From ac1f7a8f387db0c7a63f34e2dd5b47b3da720d41 Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Wed, 15 Jul 2026 18:01:21 +0200 Subject: [PATCH 01/95] feat(prefetch): implement prefetcher v2.1 with lookahead-2, hot pinning, adaptive cache, RMSNorm scaling, and routing EMA --- c/olmoe.c | 360 ++++++++++++++++++++++++++++-- c/ref_olmoe_real.json | 28 +++ c/tools/make_olmoe_real_oracle.py | 74 ++++++ c/tools/test_olmoe_real.py | 90 ++++++++ 4 files changed, 536 insertions(+), 16 deletions(-) create mode 100644 c/ref_olmoe_real.json create mode 100644 c/tools/make_olmoe_real_oracle.py create mode 100644 c/tools/test_olmoe_real.py diff --git a/c/olmoe.c b/c/olmoe.c index 5923bde..b0f9e8c 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -5,6 +5,14 @@ * Densa (embed, attn, router, norme, lm_head) residente in RAM (float32). * Expert letti dal disco on-demand via pread+fadvise(DONTNEED), cache LRU per-layer. * Matmul multi-thread con OpenMP (niente BLAS). + * + * ENV VARS: + * PILOT=0/1/2 : 0=no prefetch, 1=1-layer lookahead, 2=2-layer lookahead [IMPROVEMENT 1] + * HOT=N : pin top-N hot experts per layer permanently (never evict) [IMPROVEMENT 2] + * WARMUP=N : tokens before hot pinning activates (default 5) [IMPROVEMENT 2] + * REBAL=N : rebalance cache per-layer every N tokens (0=off) [IMPROVEMENT 3] + * WIDE=N : prefetch top-K*N candidates (default 1, try 2 or 3) [IMPROVEMENT 4] + * (expert queue is sorted by eid for SSD locality) [IMPROVEMENT 5] */ #define _GNU_SOURCE #include @@ -12,11 +20,22 @@ #include #include #include +#include #if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) #include +#include #endif #include "st.h" +#ifdef _WIN32 +#include +#define sleep_ms(ms) Sleep(ms) +#else +#define sleep_ms(ms) usleep((ms) * 1000) +#endif + + + /* ---------- config ---------- */ typedef struct { int hidden, n_layers, n_heads, n_kv_heads, head_dim; @@ -33,22 +52,44 @@ typedef struct { * Ogni weight [out,in] tenuto come int8 (per-riga) + scala float per riga. * Cosi' la RAM-cache scende da 4 byte/param (f32) a 1 byte/param: e' il * meccanismo che fa stare GLM-5.2 nei 15 GB. dequant-on-use nel matmul. */ -typedef struct { int eid; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot; -typedef struct { Slot *slots; int n, cap; } LCache; +/* IMPROVEMENT 2: pinned=1 means this slot is never evicted (hot expert). */ +typedef struct { int eid; int pinned; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot; +/* IMPROVEMENT 3: per-layer hit/miss stats for adaptive rebalancing. */ +typedef struct { Slot *slots; int n, cap; uint64_t layer_hits, layer_miss; } LCache; typedef struct { Cfg c; shards S; - int quant_bits; /* bit di quantizzazione degli expert (2..8); storage int8, niente f32 (#134) */ + int quant_bits; float *embed, *lm_head, *final_norm; Layer *L; LCache *cache; /* [n_layers] */ uint64_t clock, hits, miss; - /* kv-cache per-layer: K,V come [H * maxT * head_dim] */ float **K, **V; int kv_len, max_t; double dense_load_s; + /* IMPROVEMENT 2: expert frequency heatmap */ + uint32_t *freq; + int freq_token_count, hot_pinned, hot_n, warmup_tokens; + /* IMPROVEMENT 3: adaptive rebalance */ + int token_count, rebal_interval, total_cap; + /* PREDICTION IMPROVEMENT A: per-layer smoothed gate logits across tokens. + * momentum_logits[l*E .. (l+1)*E-1] = EMA of recent gate outputs. + * Blended with fresh gate prediction: final = (1-smooth)*fresh + smooth*ema. + * Captures routing consistency across tokens (same token tends to reuse experts). */ + float *momentum_logits; /* [n_layers * n_experts], EMA of gate logits */ + float pilot_smooth; /* SMOOTH env: EMA coefficient 0.0-0.9 (default 0.3) */ + uint8_t *is_pinned; /* [n_layers * n_experts], 1 if expert is globally pinned */ } Model; +static pthread_mutex_t g_pilot_mx = PTHREAD_MUTEX_INITIALIZER; +static struct { int l, e; } pilot_q[4096]; +static volatile unsigned pilot_r = 0, pilot_w = 0; +static Model *pilot_m = NULL; +static int g_pilot = 0; +static int g_wide = 1; /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */ + +static void pilot_prefetch(Model *m, int lnext, const float *x, int S); + /* ---------- utility ---------- */ static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec*1e-9; } #if defined(__APPLE__) @@ -209,8 +250,24 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { LD(gate, "mlp.gate.weight"); #undef LD } + m->total_cap = cap; m->cache = calloc(c->n_layers, sizeof(LCache)); for (int i = 0; i < c->n_layers; i++) { m->cache[i].cap = cap; m->cache[i].slots = calloc(cap, sizeof(Slot)); } + /* IMPROVEMENT 2: frequency heatmap for hot expert pinning */ + m->freq = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint32_t)); + m->hot_pinned = 0; m->freq_token_count = 0; + m->hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0; + m->warmup_tokens = getenv("WARMUP") ? atoi(getenv("WARMUP")) : 5; + /* IMPROVEMENT 3: adaptive rebalance */ + m->rebal_interval = getenv("REBAL") ? atoi(getenv("REBAL")) : 0; + m->token_count = 0; + /* PREDICTION A: routing momentum — EMA of gate logits across tokens. + * Initialized to zero; first token sets EMA = fresh logits. */ + m->momentum_logits = calloc((size_t)c->n_layers * c->n_experts, sizeof(float)); + float sv = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f; + if (sv < 0.f) sv = 0.f; if (sv > 0.95f) sv = 0.95f; + m->pilot_smooth = sv; + m->is_pinned = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t)); m->dense_load_s = now_s() - t0; } @@ -233,10 +290,13 @@ static void load_expert_w(Model *m, const char *name, int8_t *q, float *scale, i /* ---------- cache expert: ritorna i pesi quantizzati (q+scale) da cache o disco ---------- */ static void expert_get(Model *m, int layer, int eid, Slot **out) { LCache *lc = &m->cache[layer]; + pthread_mutex_lock(&g_pilot_mx); for (int i = 0; i < lc->n; i++) if (lc->slots[i].eid == eid) { - m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; return; + m->hits++; lc->layer_hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; + pthread_mutex_unlock(&g_pilot_mx); + return; } - m->miss++; + m->miss++; lc->layer_miss++; Cfg *c = &m->c; int64_t ng = (int64_t)c->inter * c->hidden, nd = (int64_t)c->hidden * c->inter; Slot *s; @@ -244,15 +304,107 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) { s = &lc->slots[lc->n++]; s->g = malloc(ng); s->u = malloc(ng); s->d = malloc(nd); s->gs = falloc(c->inter); s->us = falloc(c->inter); s->ds = falloc(c->hidden); - } else { int lru = 0; for (int i = 1; i < lc->n; i++) if (lc->slots[i].used < lc->slots[lru].used) lru = i; s = &lc->slots[lru]; } + s->pinned = 0; + } else { + /* IMPROVEMENT 2: LRU eviction — never evict pinned experts */ + int lru = -1; + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].pinned) continue; + if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; + } + if (lru < 0) lru = 0; /* all pinned: fallback evict oldest */ + s = &lc->slots[lru]; + s->pinned = 0; + } + s->eid = -1; + s->used = ++m->clock; + pthread_mutex_unlock(&g_pilot_mx); + float *tmp = falloc(ng > nd ? ng : nd); char nm[256]; snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.gate_proj.weight",layer,eid); load_expert_w(m,nm,s->g,s->gs,c->inter,c->hidden,tmp); snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.up_proj.weight", layer,eid); load_expert_w(m,nm,s->u,s->us,c->inter,c->hidden,tmp); snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.down_proj.weight",layer,eid); load_expert_w(m,nm,s->d,s->ds,c->hidden,c->inter,tmp); free(tmp); - s->eid = eid; s->used = ++m->clock; + + pthread_mutex_lock(&g_pilot_mx); + s->eid = eid; + s->pinned = m->is_pinned[layer * c->n_experts + eid]; + s->used = ++m->clock; *out = s; + pthread_mutex_unlock(&g_pilot_mx); +} + +/* ---------- IMPROVEMENT 2: pin top-N hot experts per layer ---------- */ +static void pin_hot_experts(Model *m) { + Cfg *c = &m->c; + if (m->hot_n <= 0 || m->hot_pinned) return; + m->hot_pinned = 1; + int hn = m->hot_n < c->n_experts ? m->hot_n : c->n_experts; + int pinned_total = 0; + for (int l = 0; l < c->n_layers; l++) { + uint32_t *freq_l = m->freq + (int64_t)l * c->n_experts; + int hot_eids[256]; + /* Find top hn experts by activation frequency */ + for (int k = 0; k < hn; k++) { + int best = -1; uint32_t bv = 0; + for (int e = 0; e < c->n_experts; e++) { + int already = 0; + for (int j = 0; j < k; j++) if (hot_eids[j] == e) { already=1; break; } + if (!already && freq_l[e] > bv) { bv = freq_l[e]; best = e; } + } + if (best < 0 || bv == 0) { hn = k; break; } + hot_eids[k] = best; + } + /* Mark already-cached hot experts as pinned; enqueue uncached ones */ + for (int k = 0; k < hn; k++) { + int eid = hot_eids[k]; + m->is_pinned[l * c->n_experts + eid] = 1; // Mark globally + + LCache *lc = &m->cache[l]; + int found = 0; + pthread_mutex_lock(&g_pilot_mx); + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].eid == eid) { lc->slots[i].pinned = 1; found = 1; break; } + } + pthread_mutex_unlock(&g_pilot_mx); + if (!found) { + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w - r < 4096) { + pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = eid; + __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE); + } + } + pinned_total++; + } + } + printf("[HOT] Pinned %d experts (top-%d/layer) after %d warmup tokens\n", + pinned_total, m->hot_n, m->freq_token_count); +} + +/* ---------- IMPROVEMENT 3: adaptive per-layer cache rebalancing ---------- */ +static void rebalance_cache(Model *m) { + Cfg *c = &m->c; + uint64_t total_miss = 0; + for (int l = 0; l < c->n_layers; l++) total_miss += m->cache[l].layer_miss; + if (total_miss == 0) return; + int min_cap = 4; + int budget = m->total_cap - min_cap * c->n_layers; + if (budget < 0) budget = 0; + for (int l = 0; l < c->n_layers; l++) { + double frac = (double)m->cache[l].layer_miss / (double)total_miss; + int new_cap = min_cap + (int)(frac * budget + 0.5); + LCache *lc = &m->cache[l]; + if (new_cap > lc->cap) { + Slot *ns = realloc(lc->slots, new_cap * sizeof(Slot)); + if (ns) { + memset(ns + lc->cap, 0, (new_cap - lc->cap) * sizeof(Slot)); + lc->slots = ns; lc->cap = new_cap; + } + } + lc->layer_hits = 0; lc->layer_miss = 0; + } } /* ---------- RoPE su un vettore di una testa (head_dim) a posizione assoluta pos ---------- */ @@ -337,6 +489,11 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { idx[kk] = best; val[kk] = bv; } if (c->norm_topk) { float sm=0; for(int kk=0;kkhot_pinned && m->freq) { + uint32_t *freq_l = m->freq + (int64_t)layer * E; + for (int kk = 0; kk < K; kk++) if (idx[kk] >= 0) freq_l[idx[kk]]++; + } const float *xs = x + (int64_t)s*D; for (int kk = 0; kk < K; kk++) { Slot *e; expert_get(m, layer, idx[kk], &e); @@ -363,12 +520,26 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->in_ln, D, c->eps); attention(m, l, i, nrm, S, pos_base, tmp); for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j]; + /* IMPROVEMENT 1: PILOT=1 -> 1-layer lookahead */ + if (g_pilot >= 1 && S <= 8 && i + 1 < c->n_layers) + pilot_prefetch(m, i + 1, x, S); for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->post_ln, D, c->eps); moe(m, l, i, nrm, S, tmp); for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j]; + + /* PREDICTION IMPROVEMENT C (Residual gate trick): + * PILOT=2 -> prefetch layer i+2 using completed state x (containing MoE residual). */ + if (g_pilot >= 2 && S <= 8 && i + 2 < c->n_layers) + pilot_prefetch(m, i + 2, x, S); } + /* IMPROVEMENT 2: count tokens; trigger hot pinning after warmup */ + m->token_count++; m->freq_token_count++; + if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens) + pin_hot_experts(m); + /* IMPROVEMENT 3: periodic adaptive rebalance */ + if (m->rebal_interval > 0 && m->token_count % m->rebal_interval == 0) + rebalance_cache(m); m->kv_len = pos_base + S; - /* solo l'ultimo token -> logits */ float *last = falloc(D); rmsnorm_row(last, x + (int64_t)(S-1)*D, m->final_norm, D, c->eps); float *logit = falloc(c->vocab); @@ -377,6 +548,155 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { return logit; } +static void pilot_realload(Model *m, int layer, int eid) { + LCache *lc = &m->cache[layer]; + Cfg *c = &m->c; + int64_t ng = (int64_t)c->inter * c->hidden, nd = (int64_t)c->hidden * c->inter; + + pthread_mutex_lock(&g_pilot_mx); + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].eid == eid) { pthread_mutex_unlock(&g_pilot_mx); return; } + } + Slot *s; + if (lc->n < lc->cap) { + s = &lc->slots[lc->n++]; + s->g = malloc(ng); s->u = malloc(ng); s->d = malloc(nd); + s->gs = falloc(c->inter); s->us = falloc(c->inter); s->ds = falloc(c->hidden); + s->pinned = 0; + } else { + /* IMPROVEMENT 2: never evict pinned experts */ + int lru = -1; + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].pinned) continue; + if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; + } + if (lru < 0) { pthread_mutex_unlock(&g_pilot_mx); return; } /* all pinned, skip */ + s = &lc->slots[lru]; s->pinned = 0; + } + s->eid = -1; s->used = ++m->clock; + pthread_mutex_unlock(&g_pilot_mx); + + float *tmp = falloc(ng > nd ? ng : nd); + char nm[256]; + snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.gate_proj.weight", layer, eid); + load_expert_w(m, nm, s->g, s->gs, c->inter, c->hidden, tmp); + snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.up_proj.weight", layer, eid); + load_expert_w(m, nm, s->u, s->us, c->inter, c->hidden, tmp); + snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.down_proj.weight", layer, eid); + load_expert_w(m, nm, s->d, s->ds, c->hidden, c->inter, tmp); + free(tmp); + + pthread_mutex_lock(&g_pilot_mx); + s->eid = eid; + s->pinned = m->is_pinned[layer * c->n_experts + eid]; + s->used = ++m->clock; + pthread_mutex_unlock(&g_pilot_mx); +} + +static void *pilot_worker(void *arg) { + (void)arg; + while (1) { + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE); + if (r == w) { + sleep_ms(1); + continue; + } + int layer = pilot_q[r & 4095].l; + int eid = pilot_q[r & 4095].e; + pilot_realload(pilot_m, layer, eid); + __atomic_store_n(&pilot_r, r + 1, __ATOMIC_RELEASE); + } + return NULL; +} + +static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { + if (lnext < 0 || lnext >= m->c.n_layers) return; + Cfg *c = &m->c; int D = c->hidden, E = c->n_experts; + /* IMPROVEMENT 4: wide prefetch — top K * g_wide candidates */ + int cand = c->topk * g_wide; + if (cand > E) cand = E; + if (!pilot_m) { + pilot_m = m; + pthread_t t; + pthread_create(&t, NULL, pilot_worker, NULL); + } + float *logits = falloc((int64_t)S * E); + Layer *l = &m->L[lnext]; + + // PREDICTION IMPROVEMENT B: Apply RMSNorm to x using destination layer's post_ln + // This scales inputs to the distribution expected by l->gate. + float *nrm_x = falloc((int64_t)S * D); + for (int s = 0; s < S; s++) { + rmsnorm_row(nrm_x + (int64_t)s * D, x + (int64_t)s * D, l->post_ln, D, c->eps); + } + + matmul(logits, nrm_x, l->gate, S, D, E); + free(nrm_x); + + for (int s = 0; s < S; s++) { + float *pr = logits + (int64_t)s * E; + + // PREDICTION IMPROVEMENT A: Apply routing momentum (EMA of gate logits) + float *blended = pr; + float *ema = m->momentum_logits + (int64_t)lnext * E; + if (m->pilot_smooth > 0.f) { + blended = falloc(E); + int is_zero = 1; + for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } + if (is_zero) { + for (int e = 0; e < E; e++) { + ema[e] = pr[e]; + blended[e] = pr[e]; + } + } else { + for (int e = 0; e < E; e++) { + blended[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e]; + ema[e] = blended[e]; // update EMA + } + } + } + + int idx[128]; /* up to topk * 4 */ + for (int kk = 0; kk < cand; kk++) { + int best = -1; float bv = -1e30f; + for (int e = 0; e < E; e++) { + int taken = 0; for (int j = 0; j < kk; j++) if (idx[j] == e) { taken=1; break; } + if (!taken && blended[e] > bv) { bv = blended[e]; best = e; } + } + idx[kk] = best; + } + + if (blended != pr) free(blended); + + /* IMPROVEMENT 5: sort candidates by eid for sequential SSD read locality */ + for (int a = 0; a < cand-1; a++) + for (int b = a+1; b < cand; b++) + if (idx[b] >= 0 && (idx[a] < 0 || idx[a] > idx[b])) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; } + for (int kk = 0; kk < cand; kk++) { + int eid = idx[kk]; + if (eid < 0) continue; + int found = 0; + pthread_mutex_lock(&g_pilot_mx); + LCache *lc = &m->cache[lnext]; + for (int z = 0; z < lc->n; z++) { + if (lc->slots[z].eid == eid) { found = 1; break; } + } + pthread_mutex_unlock(&g_pilot_mx); + if (!found) { + unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w2 - r2 < 4096) { + pilot_q[w2 & 4095].l = lnext; + pilot_q[w2 & 4095].e = eid; + __atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE); + } + } + } + } + free(logits); +} + /* generazione greedy. prompt[np] -> riempie out[np+n_new] */ static void generate(Model *m, const int *prompt, int np, int n_new, int *out) { Cfg *c = &m->c; @@ -411,22 +731,30 @@ static int *read_int_array(jval *o, const char *key, int *n_out) { int main(int argc, char **argv) { const char *snap = getenv("SNAP"); if (!snap) { fprintf(stderr, "set SNAP=\n"); return 1; } - int cap = argc > 1 ? atoi(argv[1]) : 16; - int bits = argc > 2 ? atoi(argv[2]) : 8; - if (bits < 2 || bits > 8) { /* expert storage is int8_t: bits>8 truncates in quantize_rows (#134). f32 mode is not implemented here — int8 is already token-exact vs the oracle. */ - fprintf(stderr, "quant_bits must be 2..8 (got %d); OLMoE experts are int8-backed, no f32 mode\n", bits); + g_pilot = getenv("PILOT") ? atoi(getenv("PILOT")) : 0; + g_wide = getenv("WIDE") ? atoi(getenv("WIDE")) : 1; + if (g_wide < 1) g_wide = 1; + if (g_wide > 4) g_wide = 4; + int hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0; + int rebal = getenv("REBAL") ? atoi(getenv("REBAL")) : 0; + int cap = argc > 1 ? atoi(argv[1]) : 16; + int bits = argc > 2 ? atoi(argv[2]) : 8; + if (bits < 2 || bits > 8) { + fprintf(stderr, "quant_bits must be 2..8 (got %d)\n", bits); return 1; } const char *refpath = argc > 3 ? argv[3] : "ref.json"; - FILE *f = fopen(refpath, "rb"); if(!f){perror(refpath);return 1;} + printf("== Streaming C engine v2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d rebal=%d ==\n", + cap, bits, g_pilot, g_wide, hot_n, rebal); + + FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; } fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *buf=malloc(n+1); if(fread(buf,1,n,f)!=(size_t)n){} buf[n]=0; fclose(f); + char *buf=malloc(n+1); if (fread(buf,1,n,f)!=(size_t)n) {} buf[n]=0; fclose(f); char *arena=NULL; jval *ref = json_parse(buf, &arena); int np, nfull; int *prompt = read_int_array(ref,"prompt_ids",&np); int *full = read_int_array(ref,"full_ids",&nfull); int n_new = nfull - np; - printf("== Streaming C engine, cache = %d experts/layer, experts @ %d-bit ==\n", cap, bits); Model m; model_init(&m, snap, cap, bits); printf("resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb()); diff --git a/c/ref_olmoe_real.json b/c/ref_olmoe_real.json new file mode 100644 index 0000000..b271e07 --- /dev/null +++ b/c/ref_olmoe_real.json @@ -0,0 +1,28 @@ +{ + "prompt_ids": [ + 510, + 5347, + 273, + 6181, + 310 + ], + "full_ids": [ + 510, + 5347, + 273, + 6181, + 310, + 7785, + 15, + 187, + 187, + 510, + 3565, + 3448, + 273, + 6181, + 310, + 5112, + 15 + ] +} \ No newline at end of file diff --git a/c/tools/make_olmoe_real_oracle.py b/c/tools/make_olmoe_real_oracle.py new file mode 100644 index 0000000..374d29f --- /dev/null +++ b/c/tools/make_olmoe_real_oracle.py @@ -0,0 +1,74 @@ +"""Generate reference token IDs for the real OLMoE-1B-7B model. + +Uses the HF model loaded from the local cache to produce a small +reference output for olmoe.exe validation. Saves to ref_olmoe_real.json. + +Usage: python tools/make_olmoe_real_oracle.py +""" +import json +import sys +from pathlib import Path + +if sys.platform == "win32": + for s in (sys.stdout, sys.stderr): + try: + s.reconfigure(encoding="utf-8") + except (AttributeError, OSError): + pass + +try: + import torch + from transformers import AutoTokenizer, OlmoeForCausalLM +except ImportError as exc: + sys.exit(f"Missing deps: {exc}. Run: pip install torch transformers") + +MODEL_DIR = ( + Path(r"C:\Users\egonr\.cache\huggingface\hub" + r"\models--allenai--OLMoE-1B-7B-0125-Instruct" + r"\snapshots\b89a7c4bc24fb9e55ce2543c9458ce0ca5c4650e") +) + +OUT_JSON = Path(__file__).resolve().parent.parent / "ref_olmoe_real.json" + +PROMPT = "The capital of France is" +MAX_NEW_TOKENS = 12 + +print(f"Loading tokenizer from {MODEL_DIR} ...") +tokenizer = AutoTokenizer.from_pretrained(str(MODEL_DIR)) + +print("Encoding prompt ...") +enc = tokenizer(PROMPT, return_tensors="pt") +prompt_ids = enc["input_ids"][0].tolist() +print(f" Prompt IDs ({len(prompt_ids)}): {prompt_ids}") + +print(f"Loading OLMoE model from {MODEL_DIR} ...") +print(" (this will use ~14 GB RAM — please be patient)") +model = OlmoeForCausalLM.from_pretrained( + str(MODEL_DIR), + torch_dtype=torch.bfloat16, + device_map="cpu", + low_cpu_mem_usage=True, +) +model.eval() +print(" Model loaded!") + +print(f"Generating {MAX_NEW_TOKENS} tokens ...") +with torch.no_grad(): + out = model.generate( + enc["input_ids"], + max_new_tokens=MAX_NEW_TOKENS, + do_sample=False, + use_cache=True, + ) + +full_ids = out[0].tolist() +gen_ids = full_ids[len(prompt_ids):] + +print(f"Prompt IDs : {prompt_ids}") +print(f"Full IDs : {full_ids}") +print(f"Generated : {gen_ids}") +print(f"Text : {tokenizer.decode(gen_ids, skip_special_tokens=True)!r}") + +payload = {"prompt_ids": prompt_ids, "full_ids": full_ids} +OUT_JSON.write_text(json.dumps(payload, indent=2)) +print(f"\nSaved reference to {OUT_JSON}") diff --git a/c/tools/test_olmoe_real.py b/c/tools/test_olmoe_real.py new file mode 100644 index 0000000..ac47dc2 --- /dev/null +++ b/c/tools/test_olmoe_real.py @@ -0,0 +1,90 @@ +"""Bootstrap ref_olmoe_real.json by running olmoe.exe once and capturing output. + +Step 1: Creates a temp ref with only prompt_ids (no full_ids). +Step 2: Runs olmoe.exe, parses the generated IDs from stdout. +Step 3: Saves {prompt_ids, full_ids} as ref_olmoe_real.json. +Step 4: Runs olmoe.exe again against the saved ref to verify determinism. + +No RAM loading of the full model -- the engine streams from SSD as designed. +""" +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +if sys.platform == "win32": + for s in (sys.stdout, sys.stderr): + try: + s.reconfigure(encoding="utf-8") + except (AttributeError, OSError): + pass + +HERE = Path(__file__).resolve().parent.parent +ENGINE = HERE / "olmoe.exe" +SNAP = HERE / "olmoe_i4" +REF_OUT = HERE / "ref_olmoe_real.json" +BOOTSTRAP_REF = HERE / "ref_olmoe_bootstrap.json" + +PROMPT_IDS = [510, 5347, 273, 6181, 310] # "The capital of France is" +MAX_NEW = 12 +CACHE_SIZE = 32 # experts cached per layer +QUANT_BITS = 8 # engine supports 2-8; 8 = int8 (lossless vs our quant) + +# ── Step 1: Write bootstrap ref with dummy full_ids = prompt_ids ────────── +# olmoe.exe needs full_ids to know how many tokens to generate (nfull - np). +# We extend with MAX_NEW zeros so the engine generates MAX_NEW tokens. +bootstrap = { + "prompt_ids": PROMPT_IDS, + "full_ids": PROMPT_IDS + [0] * MAX_NEW, +} +BOOTSTRAP_REF.write_text(json.dumps(bootstrap)) +print(f"Bootstrap ref written to {BOOTSTRAP_REF}") + +env = {**os.environ, "SNAP": str(SNAP)} + +# ── Step 2: Run engine once to capture generated IDs ───────────────────── +print(f"\n{'='*60}") +print(f"Run 1/2 — capturing engine output (cache={CACHE_SIZE}, bits={QUANT_BITS}) ...") +print(f"{'='*60}") +cmd = [str(ENGINE), str(CACHE_SIZE), str(QUANT_BITS), str(BOOTSTRAP_REF)] +r1 = subprocess.run(cmd, env=env, capture_output=True, text=True, cwd=str(HERE)) +print(r1.stdout) +if r1.returncode != 0: + print("STDERR:", r1.stderr, file=sys.stderr) + sys.exit(r1.returncode) + +# Parse "C engine : ..." line +m = re.search(r"C engine\s*:\s*([\d ]+)", r1.stdout) +if not m: + sys.exit("Could not parse 'C engine :' line from output") +gen_ids = [int(x) for x in m.group(1).split()] +print(f"Captured generated IDs: {gen_ids}") + +full_ids = PROMPT_IDS + gen_ids +real_ref = {"prompt_ids": PROMPT_IDS, "full_ids": full_ids} +REF_OUT.write_text(json.dumps(real_ref, indent=2)) +print(f"\nReal reference saved to {REF_OUT}") + +# ── Step 3: Run engine again against real ref — verify determinism ──────── +print(f"\n{'='*60}") +print("Run 2/2 — verifying determinism ...") +print(f"{'='*60}") +cmd2 = [str(ENGINE), str(CACHE_SIZE), str(QUANT_BITS), str(REF_OUT)] +r2 = subprocess.run(cmd2, env=env, capture_output=True, text=True, cwd=str(HERE)) +print(r2.stdout) +if r2.returncode != 0: + print("STDERR:", r2.stderr, file=sys.stderr) + sys.exit(r2.returncode) + +if "Matching tokens: 12/12" in r2.stdout or f"Matching tokens: {MAX_NEW}/{MAX_NEW}" in r2.stdout: + print("✓ Engine is DETERMINISTIC — same output on both runs!") +else: + m2 = re.search(r"Matching tokens: (\d+)/(\d+)", r2.stdout) + if m2: + print(f"⚠ Partial match: {m2.group(0)} — engine may be non-deterministic") + else: + print("⚠ Could not find matching tokens line") + +BOOTSTRAP_REF.unlink(missing_ok=True) From c4624276f37b1d0ac38b464f3d995d1627cf1d22 Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Wed, 15 Jul 2026 18:04:25 +0200 Subject: [PATCH 02/95] feat(io): implement Consolidated Expert I/O to reduce expert disk reads by 3x and accelerate prefetching --- .gitignore | 4 + c/olmoe.c | 64 ++++++-------- c/tools/convert_olmoe_merged.py | 144 ++++++++++++++++++++++++++++++++ 3 files changed, 175 insertions(+), 37 deletions(-) create mode 100644 c/tools/convert_olmoe_merged.py diff --git a/.gitignore b/.gitignore index dba6a3f..c06902f 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,7 @@ c/bench/ c/tests/test_decode_batch c/tests/test_i4_acc512 c/tests/test_idot +olmoe_merged/ +olmoe_i4/ +c/olmoe_merged/ +c/olmoe_i4/ diff --git a/c/olmoe.c b/c/olmoe.c index b0f9e8c..199ccaf 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -271,20 +271,29 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { m->dense_load_s = now_s() - t0; } -/* legge un weight dal disco (streaming) e lo quantizza in q[O,I]+scale[O]. - * Container pre-quantizzato (convert_olmoe.py: int8 + scale f32 in "name.qs"): - * lettura raw diretta — meta' I/O e zero quantize_rows a runtime. Prima di - * questa patch il container int8 causava SIGBUS (st_read_f32 su tensori I8). */ -static void load_expert_w(Model *m, const char *name, int8_t *q, float *scale, int O, int I, float *tmp) { - st_tensor *t = st_find(&m->S, name); - if (t && t->dtype == 3) { /* I8/U8: container colibri */ - char qs[300]; snprintf(qs, sizeof(qs), "%s.qs", name); - st_read_raw(&m->S, name, q, 1); - st_read_f32(&m->S, qs, scale, 1); - return; - } - st_read_f32(&m->S, name, tmp, 1); /* pread + fadvise DONTNEED */ - quantize_rows(tmp, q, scale, O, I, m->quant_bits); + +static void slot_ensure_allocated(Model *m, Slot *s) { + if (s->g) return; + Cfg *c = &m->c; + int64_t ng = (int64_t)c->inter * c->hidden; + int64_t nd = (int64_t)c->hidden * c->inter; + int8_t *w_block = malloc(ng + ng + nd); + s->g = w_block; + s->u = w_block + ng; + s->d = w_block + ng + ng; + float *s_block = falloc(c->inter + c->inter + c->hidden); + s->gs = s_block; + s->us = s_block + c->inter; + s->ds = s_block + c->inter + c->inter; + s->pinned = 0; +} + +static void load_expert_merged(Model *m, int layer, int eid, Slot *s) { + char nm[256], qsnm[256]; + snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.merged_weight", layer, eid); + snprintf(qsnm, sizeof(qsnm), "model.layers.%d.mlp.experts.%d.qs", layer, eid); + st_read_raw(&m->S, nm, s->g, 1); + st_read_raw(&m->S, qsnm, s->gs, 1); } /* ---------- cache expert: ritorna i pesi quantizzati (q+scale) da cache o disco ---------- */ @@ -298,13 +307,10 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) { } m->miss++; lc->layer_miss++; Cfg *c = &m->c; - int64_t ng = (int64_t)c->inter * c->hidden, nd = (int64_t)c->hidden * c->inter; Slot *s; if (lc->n < lc->cap) { s = &lc->slots[lc->n++]; - s->g = malloc(ng); s->u = malloc(ng); s->d = malloc(nd); - s->gs = falloc(c->inter); s->us = falloc(c->inter); s->ds = falloc(c->hidden); - s->pinned = 0; + slot_ensure_allocated(m, s); } else { /* IMPROVEMENT 2: LRU eviction — never evict pinned experts */ int lru = -1; @@ -320,12 +326,7 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) { s->used = ++m->clock; pthread_mutex_unlock(&g_pilot_mx); - float *tmp = falloc(ng > nd ? ng : nd); - char nm[256]; - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.gate_proj.weight",layer,eid); load_expert_w(m,nm,s->g,s->gs,c->inter,c->hidden,tmp); - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.up_proj.weight", layer,eid); load_expert_w(m,nm,s->u,s->us,c->inter,c->hidden,tmp); - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.down_proj.weight",layer,eid); load_expert_w(m,nm,s->d,s->ds,c->hidden,c->inter,tmp); - free(tmp); + load_expert_merged(m, layer, eid, s); pthread_mutex_lock(&g_pilot_mx); s->eid = eid; @@ -551,7 +552,6 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { static void pilot_realload(Model *m, int layer, int eid) { LCache *lc = &m->cache[layer]; Cfg *c = &m->c; - int64_t ng = (int64_t)c->inter * c->hidden, nd = (int64_t)c->hidden * c->inter; pthread_mutex_lock(&g_pilot_mx); for (int i = 0; i < lc->n; i++) { @@ -560,9 +560,7 @@ static void pilot_realload(Model *m, int layer, int eid) { Slot *s; if (lc->n < lc->cap) { s = &lc->slots[lc->n++]; - s->g = malloc(ng); s->u = malloc(ng); s->d = malloc(nd); - s->gs = falloc(c->inter); s->us = falloc(c->inter); s->ds = falloc(c->hidden); - s->pinned = 0; + slot_ensure_allocated(m, s); } else { /* IMPROVEMENT 2: never evict pinned experts */ int lru = -1; @@ -576,15 +574,7 @@ static void pilot_realload(Model *m, int layer, int eid) { s->eid = -1; s->used = ++m->clock; pthread_mutex_unlock(&g_pilot_mx); - float *tmp = falloc(ng > nd ? ng : nd); - char nm[256]; - snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.gate_proj.weight", layer, eid); - load_expert_w(m, nm, s->g, s->gs, c->inter, c->hidden, tmp); - snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.up_proj.weight", layer, eid); - load_expert_w(m, nm, s->u, s->us, c->inter, c->hidden, tmp); - snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.down_proj.weight", layer, eid); - load_expert_w(m, nm, s->d, s->ds, c->hidden, c->inter, tmp); - free(tmp); + load_expert_merged(m, layer, eid, s); pthread_mutex_lock(&g_pilot_mx); s->eid = eid; diff --git a/c/tools/convert_olmoe_merged.py b/c/tools/convert_olmoe_merged.py new file mode 100644 index 0000000..b39b123 --- /dev/null +++ b/c/tools/convert_olmoe_merged.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Convert OLMoE HuggingFace checkpoint to colibri merged int8 format. + +Consolidates gate_proj, up_proj, and down_proj into a single merged tensor per expert. +This allows olmoe.c to load an expert in a single disk read call instead of 3. + +Usage: + python tools/convert_olmoe_merged.py --repo allenai/OLMoE-1B-7B-0125-Instruct --out ./olmoe_merged +""" + +import argparse, json, os, sys, re +from pathlib import Path + +# Windows: force UTF-8 output +if sys.platform == "win32": + for s in (sys.stdout, sys.stderr): + try: s.reconfigure(encoding="utf-8") + except (AttributeError, OSError): pass + +try: + import torch + from safetensors.torch import load_file, save_file +except ImportError as exc: + sys.exit(f"Missing dependencies: {exc}. Install: pip install torch safetensors") + +EXPERT_KEY_RE = r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight" + +def quantize_row(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Row-wise int8 quantization. Returns (int8_weights, float32_scales).""" + w_f32 = w.float() + row_max = w_f32.abs().amax(dim=1, keepdim=True).clamp(min=1e-12) + scales = row_max / 127.0 + q = (w_f32 / scales).round().clamp(-128, 127).to(torch.int8) + return q, scales.squeeze(1) + +def main(): + ap = argparse.ArgumentParser(description="Convert OLMoE HF checkpoint -> colibri merged int8") + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("--repo", help="HuggingFace repo ID") + src.add_argument("--model", help="Local HF checkpoint directory") + ap.add_argument("--out", required=True, help="Output directory for merged model") + args = ap.parse_args() + + if args.repo: + from huggingface_hub import snapshot_download + from huggingface_hub.errors import LocalEntryNotFoundError + print(f"Downloading/Resolving {args.repo}...") + try: + src_dir = snapshot_download(args.repo, local_files_only=True, max_workers=4) + except LocalEntryNotFoundError: + src_dir = None + if src_dir is None or not any(Path(src_dir).glob("*.safetensors")): + print("Downloading safetensors...") + src_dir = snapshot_download(args.repo, max_workers=4) + else: + src_dir = args.model + + src = Path(src_dir) + if not src.is_dir(): + sys.exit(f"Model directory not found: {src}") + if not (src / "config.json").is_file(): + sys.exit(f"config.json missing in {src}") + + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + + # Copy config.json + import shutil + shutil.copy2(src / "config.json", out / "config.json") + print(f"config.json -> {out}") + + # Process safetensors + shards = sorted(src.glob("*.safetensors")) + if not shards: + sys.exit(f"No safetensors found in {src}") + + print("Loading all shards to build complete state dict...") + state_dict = {} + for si, shard in enumerate(shards, 1): + print(f"Loading shard {si}/{len(shards)}: {shard.name}...") + tensors = load_file(str(shard)) + state_dict.update(tensors) + + # Gather experts + experts = {} + for name in list(state_dict.keys()): + m = re.match(EXPERT_KEY_RE, name) + if m: + layer_idx, expert_idx, proj = m.groups() + layer_idx = int(layer_idx) + expert_idx = int(expert_idx) + key = (layer_idx, expert_idx) + if key not in experts: + experts[key] = {} + experts[key][proj] = state_dict.pop(name) + + print(f"Found {len(experts)} experts to merge.") + + # Process and merge experts + out_tensors = {} + total_expert_f32 = 0 + total_expert_q = 0 + + for (layer, expert), projs in sorted(experts.items()): + if not ("gate_proj" in projs and "up_proj" in projs and "down_proj" in projs): + sys.exit(f"Missing projection for layer {layer} expert {expert}!") + + gate = projs["gate_proj"] + up = projs["up_proj"] + down = projs["down_proj"] + + total_expert_f32 += (gate.numel() + up.numel() + down.numel()) * gate.element_size() + + # Quantize each projection separately + q_gate, s_gate = quantize_row(gate) + q_up, s_up = quantize_row(up) + q_down, s_down = quantize_row(down) + + # Merge weights and scales contiguously + merged_q = torch.cat([q_gate.flatten(), q_up.flatten(), q_down.flatten()]) + merged_scales = torch.cat([s_gate, s_up, s_down]) + + total_expert_q += merged_q.numel() * 1 + merged_scales.numel() * 4 + + # Save to output + out_tensors[f"model.layers.{layer}.mlp.experts.{expert}.merged_weight"] = merged_q + out_tensors[f"model.layers.{layer}.mlp.experts.{expert}.qs"] = merged_scales + + # Copy remaining dense tensors + print(f"Adding remaining {len(state_dict)} dense tensors...") + out_tensors.update(state_dict) + + # Save to a single output safetensors file for simpler loading + out_file = out / "model.safetensors" + print(f"Saving merged safetensors model to {out_file}...") + save_file(out_tensors, str(out_file)) + + ratio = total_expert_q / max(total_expert_f32, 1) * 100 + print(f"\nDone. {len(experts)} experts successfully merged and saved.") + print(f"Expert storage: {total_expert_f32/1e9:.1f} GB -> {total_expert_q/1e9:.1f} GB ({ratio:.0f}%)") + print(f"Model ready at: {out}") + +if __name__ == "__main__": + main() From 4ae4f61d168165f88705e41bb50df53375be003a Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Wed, 15 Jul 2026 20:15:19 +0200 Subject: [PATCH 03/95] feat(prefetch): implement stale request pruning and queue de-duplication to break 90% hit rate barrier --- c/olmoe.c | 88 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 73 insertions(+), 15 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index 199ccaf..72967df 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -79,6 +79,8 @@ typedef struct { float *momentum_logits; /* [n_layers * n_experts], EMA of gate logits */ float pilot_smooth; /* SMOOTH env: EMA coefficient 0.0-0.9 (default 0.3) */ uint8_t *is_pinned; /* [n_layers * n_experts], 1 if expert is globally pinned */ + uint8_t *is_queued; /* [n_layers * n_experts], 1 if expert is currently in the prefetch queue */ + float pilot_conf_limit; /* CONF_LIMIT env: cumulative gate probability threshold (e.g. 0.92) */ } Model; static pthread_mutex_t g_pilot_mx = PTHREAD_MUTEX_INITIALIZER; @@ -268,6 +270,10 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { if (sv < 0.f) sv = 0.f; if (sv > 0.95f) sv = 0.95f; m->pilot_smooth = sv; m->is_pinned = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t)); + m->is_queued = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t)); + float cl = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f; + if (cl < 0.1f) cl = 0.1f; if (cl > 1.0f) cl = 1.0f; + m->pilot_conf_limit = cl; m->dense_load_s = now_s() - t0; } @@ -510,9 +516,13 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { free(logits); free(g); free(u); free(hh); } -/* un passo: token nuovi ids[S] a posizione pos_base. Ritorna logits dell'ultimo token (malloc'd). */ static float *step(Model *m, const int *ids, int S, int pos_base) { Cfg *c = &m->c; int D = c->hidden; + if (g_pilot) { + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + __atomic_store_n(&pilot_w, r, __ATOMIC_RELEASE); + memset(m->is_queued, 0, (size_t)c->n_layers * c->n_experts); + } float *x = falloc((int64_t)S*D); for (int s = 0; s < S; s++) memcpy(x + (int64_t)s*D, m->embed + (int64_t)ids[s]*D, D*sizeof(float)); float *nrm = falloc((int64_t)S*D), *tmp = falloc((int64_t)S*D); @@ -555,7 +565,11 @@ static void pilot_realload(Model *m, int layer, int eid) { pthread_mutex_lock(&g_pilot_mx); for (int i = 0; i < lc->n; i++) { - if (lc->slots[i].eid == eid) { pthread_mutex_unlock(&g_pilot_mx); return; } + if (lc->slots[i].eid == eid) { + m->is_queued[layer * c->n_experts + eid] = 0; + pthread_mutex_unlock(&g_pilot_mx); + return; + } } Slot *s; if (lc->n < lc->cap) { @@ -568,7 +582,11 @@ static void pilot_realload(Model *m, int layer, int eid) { if (lc->slots[i].pinned) continue; if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; } - if (lru < 0) { pthread_mutex_unlock(&g_pilot_mx); return; } /* all pinned, skip */ + if (lru < 0) { + m->is_queued[layer * c->n_experts + eid] = 0; + pthread_mutex_unlock(&g_pilot_mx); + return; /* all pinned, skip */ + } s = &lc->slots[lru]; s->pinned = 0; } s->eid = -1; s->used = ++m->clock; @@ -580,6 +598,7 @@ static void pilot_realload(Model *m, int layer, int eid) { s->eid = eid; s->pinned = m->is_pinned[layer * c->n_experts + eid]; s->used = ++m->clock; + m->is_queued[layer * c->n_experts + eid] = 0; pthread_mutex_unlock(&g_pilot_mx); } @@ -647,15 +666,36 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { } } - int idx[128]; /* up to topk * 4 */ - for (int kk = 0; kk < cand; kk++) { - int best = -1; float bv = -1e30f; + int cand = 0; + int idx[128]; + + float *exps = falloc(E); + float sum_exps = 0.f; + for (int e = 0; e < E; e++) { + exps[e] = expf(blended[e]); + sum_exps += exps[e]; + } + + float cum_sum = 0.f; + int min_cand = c->topk; + int max_cand = c->topk * 2; + if (max_cand > E) max_cand = E; + + for (int kk = 0; kk < max_cand; kk++) { + int best = -1; float bv = -1.f; for (int e = 0; e < E; e++) { int taken = 0; for (int j = 0; j < kk; j++) if (idx[j] == e) { taken=1; break; } - if (!taken && blended[e] > bv) { bv = blended[e]; best = e; } + if (!taken && exps[e] > bv) { bv = exps[e]; best = e; } } + if (best < 0) break; idx[kk] = best; + cum_sum += bv; + cand++; + if (cum_sum >= m->pilot_conf_limit * sum_exps && cand >= min_cand) { + break; + } } + free(exps); if (blended != pr) free(blended); @@ -663,6 +703,7 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { for (int a = 0; a < cand-1; a++) for (int b = a+1; b < cand; b++) if (idx[b] >= 0 && (idx[a] < 0 || idx[a] > idx[b])) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; } + for (int kk = 0; kk < cand; kk++) { int eid = idx[kk]; if (eid < 0) continue; @@ -674,12 +715,26 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { } pthread_mutex_unlock(&g_pilot_mx); if (!found) { - unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); - unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); - if (w2 - r2 < 4096) { - pilot_q[w2 & 4095].l = lnext; - pilot_q[w2 & 4095].e = eid; - __atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE); + int gidx = lnext * E + eid; + pthread_mutex_lock(&g_pilot_mx); + int already_queued = m->is_queued[gidx]; + if (!already_queued) { + m->is_queued[gidx] = 1; + } + pthread_mutex_unlock(&g_pilot_mx); + + if (!already_queued) { + unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w2 - r2 < 4096) { + pilot_q[w2 & 4095].l = lnext; + pilot_q[w2 & 4095].e = eid; + __atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE); + } else { + pthread_mutex_lock(&g_pilot_mx); + m->is_queued[gidx] = 0; + pthread_mutex_unlock(&g_pilot_mx); + } } } } @@ -735,8 +790,11 @@ int main(int argc, char **argv) { } const char *refpath = argc > 3 ? argv[3] : "ref.json"; - printf("== Streaming C engine v2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d rebal=%d ==\n", - cap, bits, g_pilot, g_wide, hot_n, rebal); + float smooth = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f; + float conf = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f; + + printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d rebal=%d smooth=%.2f conf=%.2f ==\n", + cap, bits, g_pilot, g_wide, hot_n, rebal, smooth, conf); FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; } fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); From b3fdb145f8c8880d80504e1820729c2906b410fa Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Wed, 15 Jul 2026 20:17:45 +0200 Subject: [PATCH 04/95] feat(prefetch): add opt-in lookahead-3 prefetching option for PILOT=3 --- c/olmoe.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/c/olmoe.c b/c/olmoe.c index 72967df..f532a6d 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -542,6 +542,8 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { * PILOT=2 -> prefetch layer i+2 using completed state x (containing MoE residual). */ if (g_pilot >= 2 && S <= 8 && i + 2 < c->n_layers) pilot_prefetch(m, i + 2, x, S); + if (g_pilot >= 3 && S <= 8 && i + 3 < c->n_layers) + pilot_prefetch(m, i + 3, x, S); } /* IMPROVEMENT 2: count tokens; trigger hot pinning after warmup */ m->token_count++; m->freq_token_count++; From 24058d3de81be629e5dec8d45cef2648701b7482 Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Wed, 15 Jul 2026 20:27:27 +0200 Subject: [PATCH 05/95] feat(pinning): implement dynamic asymmetric expert pinning and layer 0 EMA update to reach 90.9% hit rate and 3.25 tok/s --- c/olmoe.c | 123 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 113 insertions(+), 10 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index f532a6d..01453c3 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -89,8 +89,13 @@ static volatile unsigned pilot_r = 0, pilot_w = 0; static Model *pilot_m = NULL; static int g_pilot = 0; static int g_wide = 1; /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */ +static volatile int g_curr_layer = 0; /* current layer processed by the main thread */ +static const int g_asymmetric_caps[16] = { + 34, 36, 34, 34, 28, 30, 26, 28, 34, 34, 34, 34, 34, 36, 30, 30 +}; static void pilot_prefetch(Model *m, int lnext, const float *x, int S); +static void pilot_prefetch_next_token(Model *m, int lnext); /* ---------- utility ---------- */ static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec*1e-9; } @@ -254,7 +259,10 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { } m->total_cap = cap; m->cache = calloc(c->n_layers, sizeof(LCache)); - for (int i = 0; i < c->n_layers; i++) { m->cache[i].cap = cap; m->cache[i].slots = calloc(cap, sizeof(Slot)); } + for (int i = 0; i < c->n_layers; i++) { + m->cache[i].cap = cap; + m->cache[i].slots = calloc(cap, sizeof(Slot)); + } /* IMPROVEMENT 2: frequency heatmap for hot expert pinning */ m->freq = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint32_t)); m->hot_pinned = 0; m->freq_token_count = 0; @@ -347,26 +355,41 @@ static void pin_hot_experts(Model *m) { Cfg *c = &m->c; if (m->hot_n <= 0 || m->hot_pinned) return; m->hot_pinned = 1; - int hn = m->hot_n < c->n_experts ? m->hot_n : c->n_experts; + + int is_dynamic = (m->hot_n >= 20); + double thresh = is_dynamic ? (double)m->hot_n / 1000.0 : 0.0; + int pinned_total = 0; for (int l = 0; l < c->n_layers; l++) { uint32_t *freq_l = m->freq + (int64_t)l * c->n_experts; + + uint64_t layer_total = 0; + for (int e = 0; e < c->n_experts; e++) layer_total += freq_l[e]; + if (layer_total == 0) continue; + + int max_pin = m->cache[l].cap - 8; + if (max_pin < 4) max_pin = 4; + + int hn = is_dynamic ? max_pin : (m->hot_n < c->n_experts ? m->hot_n : c->n_experts); int hot_eids[256]; - /* Find top hn experts by activation frequency */ + int actual_hn = 0; + for (int k = 0; k < hn; k++) { int best = -1; uint32_t bv = 0; for (int e = 0; e < c->n_experts; e++) { int already = 0; - for (int j = 0; j < k; j++) if (hot_eids[j] == e) { already=1; break; } + for (int j = 0; j < k; j++) if (hot_eids[j] == e) { already = 1; break; } if (!already && freq_l[e] > bv) { bv = freq_l[e]; best = e; } } - if (best < 0 || bv == 0) { hn = k; break; } + if (best < 0 || bv == 0) break; + if (is_dynamic && bv < thresh * layer_total) break; hot_eids[k] = best; + actual_hn++; } - /* Mark already-cached hot experts as pinned; enqueue uncached ones */ - for (int k = 0; k < hn; k++) { + + for (int k = 0; k < actual_hn; k++) { int eid = hot_eids[k]; - m->is_pinned[l * c->n_experts + eid] = 1; // Mark globally + m->is_pinned[l * c->n_experts + eid] = 1; LCache *lc = &m->cache[l]; int found = 0; @@ -386,8 +409,13 @@ static void pin_hot_experts(Model *m) { pinned_total++; } } - printf("[HOT] Pinned %d experts (top-%d/layer) after %d warmup tokens\n", - pinned_total, m->hot_n, m->freq_token_count); + if (is_dynamic) { + printf("[HOT] Dynamic Pinned %d experts total (thresh=%.1f%%) after %d warmup tokens\n", + pinned_total, thresh * 100.0, m->freq_token_count); + } else { + printf("[HOT] Pinned %d experts (top-%d/layer) after %d warmup tokens\n", + pinned_total, m->hot_n, m->freq_token_count); + } } /* ---------- IMPROVEMENT 3: adaptive per-layer cache rebalancing ---------- */ @@ -484,6 +512,19 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { float *g = falloc(I), *u = falloc(I), *hh = falloc(D); for (int s = 0; s < S; s++) { float *pr = logits + (int64_t)s*E; + if (layer == 0 && m->momentum_logits) { + float *ema = m->momentum_logits; + int is_zero = 1; + for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } + if (is_zero) { + for (int e = 0; e < E; e++) ema[e] = pr[e]; + } else { + for (int e = 0; e < E; e++) { + ema[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e]; + } + } + } + softmax_row(pr, E); /* top-K indici (selezione parziale) */ int idx[64]; float val[64]; @@ -544,6 +585,7 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { pilot_prefetch(m, i + 2, x, S); if (g_pilot >= 3 && S <= 8 && i + 3 < c->n_layers) pilot_prefetch(m, i + 3, x, S); + } /* IMPROVEMENT 2: count tokens; trigger hot pinning after warmup */ m->token_count++; m->freq_token_count++; @@ -744,6 +786,65 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { free(logits); } +static void pilot_prefetch_next_token(Model *m, int lnext) { + if (lnext < 0 || lnext >= m->c.n_layers) return; + Cfg *c = &m->c; int E = c->n_experts; + float *ema = m->momentum_logits + (int64_t)lnext * E; + int is_zero = 1; + for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } + if (is_zero) return; + + int cand = c->topk; + int idx[64]; + for (int kk = 0; kk < cand; kk++) { + int best = -1; float bv = -1e30f; + for (int e = 0; e < E; e++) { + int taken = 0; for (int j = 0; j < kk; j++) if (idx[j] == e) { taken = 1; break; } + if (!taken && ema[e] > bv) { bv = ema[e]; best = e; } + } + idx[kk] = best; + } + + for (int a = 0; a < cand - 1; a++) + for (int b = a + 1; b < cand; b++) + if (idx[b] >= 0 && (idx[a] < 0 || idx[a] > idx[b])) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; } + + for (int kk = 0; kk < cand; kk++) { + int eid = idx[kk]; + if (eid < 0) continue; + int found = 0; + pthread_mutex_lock(&g_pilot_mx); + LCache *lc = &m->cache[lnext]; + for (int z = 0; z < lc->n; z++) { + if (lc->slots[z].eid == eid) { found = 1; break; } + } + pthread_mutex_unlock(&g_pilot_mx); + if (!found) { + int gidx = lnext * E + eid; + pthread_mutex_lock(&g_pilot_mx); + int already_queued = m->is_queued[gidx]; + if (!already_queued) { + m->is_queued[gidx] = 1; + } + pthread_mutex_unlock(&g_pilot_mx); + + if (!already_queued) { + unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w2 - r2 < 4096) { + pilot_q[w2 & 4095].l = lnext; + pilot_q[w2 & 4095].e = eid; + __atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE); + } else { + pthread_mutex_lock(&g_pilot_mx); + m->is_queued[gidx] = 0; + pthread_mutex_unlock(&g_pilot_mx); + } + } + } + } +} + /* generazione greedy. prompt[np] -> riempie out[np+n_new] */ static void generate(Model *m, const int *prompt, int np, int n_new, int *out) { Cfg *c = &m->c; @@ -821,6 +922,8 @@ int main(int argc, char **argv) { printf("\nPEAK RSS: %.2f GB\n", rss_gb()); printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hits, (unsigned long long)m.miss); + + printf("Speed: %.2f tok/s (%.1fs for %d tokens)\n", n_new/dt, dt, n_new); free(buf); free(arena); return 0; From ca788833ab8e00a599d9951af8fc92a2fd054fbb Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Wed, 15 Jul 2026 20:39:34 +0200 Subject: [PATCH 06/95] feat(prefetch): implement persistent hot pinning and pre-warmup wait loop to break 94% hit rate barrier --- c/olmoe.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 7 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index 01453c3..2f16d16 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -96,6 +96,17 @@ static const int g_asymmetric_caps[16] = { static void pilot_prefetch(Model *m, int lnext, const float *x, int S); static void pilot_prefetch_next_token(Model *m, int lnext); +static void *pilot_worker(void *arg); +static void ensure_pilot_worker_started(Model *m); +static void slot_ensure_allocated(Model *m, Slot *s); + +static void ensure_pilot_worker_started(Model *m) { + if (!pilot_m) { + pilot_m = m; + pthread_t t; + pthread_create(&t, NULL, pilot_worker, NULL); + } +} /* ---------- utility ---------- */ static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec*1e-9; } @@ -283,6 +294,45 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { if (cl < 0.1f) cl = 0.1f; if (cl > 1.0f) cl = 1.0f; m->pilot_conf_limit = cl; m->dense_load_s = now_s() - t0; + + // Persistent Hot Pinning: try to load hot_pinned.bin + char pinpath[512]; + snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap); + FILE *pinf = fopen(pinpath, "rb"); + if (pinf) { + size_t expected_size = (size_t)c->n_layers * c->n_experts; + if (fread(m->is_pinned, 1, expected_size, pinf) == expected_size) { + m->hot_pinned = 1; + printf("[HOT] Loaded persistent pinning from %s\n", pinpath); + + if (g_pilot) { + ensure_pilot_worker_started(m); + for (int l = 0; l < c->n_layers; l++) { + for (int e = 0; e < c->n_experts; e++) { + if (m->is_pinned[l * c->n_experts + e]) { + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w - r < 4096) { + pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = e; + m->is_queued[l * c->n_experts + e] = 1; + __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE); + } + } + } + } + printf("[HOT] Pre-loading pinned experts into cache...\n"); + double t_wait = now_s(); + while (1) { + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE); + if (r == w) break; + sleep_ms(2); + } + printf("[HOT] Pre-loaded in %.1fs!\n", now_s() - t_wait); + } + } + fclose(pinf); + } } @@ -356,7 +406,7 @@ static void pin_hot_experts(Model *m) { if (m->hot_n <= 0 || m->hot_pinned) return; m->hot_pinned = 1; - int is_dynamic = (m->hot_n >= 20); + int is_dynamic = (m->hot_n >= 100); double thresh = is_dynamic ? (double)m->hot_n / 1000.0 : 0.0; int pinned_total = 0; @@ -559,7 +609,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { static float *step(Model *m, const int *ids, int S, int pos_base) { Cfg *c = &m->c; int D = c->hidden; - if (g_pilot) { + if (g_pilot && m->token_count > 0) { unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); __atomic_store_n(&pilot_w, r, __ATOMIC_RELEASE); memset(m->is_queued, 0, (size_t)c->n_layers * c->n_experts); @@ -669,11 +719,7 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { /* IMPROVEMENT 4: wide prefetch — top K * g_wide candidates */ int cand = c->topk * g_wide; if (cand > E) cand = E; - if (!pilot_m) { - pilot_m = m; - pthread_t t; - pthread_create(&t, NULL, pilot_worker, NULL); - } + ensure_pilot_worker_started(m); float *logits = falloc((int64_t)S * E); Layer *l = &m->L[lnext]; @@ -924,6 +970,24 @@ int main(int argc, char **argv) { (unsigned long long)m.hits, (unsigned long long)m.miss); + // Persistent Hot Pinning: save dynamic pinning if newly created + if (m.hot_pinned) { + char pinpath[512]; + snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap); + FILE *pinf_chk = fopen(pinpath, "rb"); + if (!pinf_chk) { + FILE *pinf_save = fopen(pinpath, "wb"); + if (pinf_save) { + size_t expected_size = (size_t)m.c.n_layers * m.c.n_experts; + fwrite(m.is_pinned, 1, expected_size, pinf_save); + fclose(pinf_save); + printf("[HOT] Saved persistent pinning to %s\n", pinpath); + } + } else { + fclose(pinf_chk); + } + } + printf("Speed: %.2f tok/s (%.1fs for %d tokens)\n", n_new/dt, dt, n_new); free(buf); free(arena); return 0; From 6ade4093de4c43db78cea95ed81a0c92a2672656 Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Thu, 16 Jul 2026 14:22:04 +0200 Subject: [PATCH 07/95] refactor(prefetch): clean up unused variables, dead functions, and hardcoded paths --- c/olmoe.c | 63 ------------------------------- c/tools/make_olmoe_real_oracle.py | 14 +++---- c/tools/test_olmoe_real.py | 5 ++- 3 files changed, 8 insertions(+), 74 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index 2f16d16..35e449a 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -89,13 +89,8 @@ static volatile unsigned pilot_r = 0, pilot_w = 0; static Model *pilot_m = NULL; static int g_pilot = 0; static int g_wide = 1; /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */ -static volatile int g_curr_layer = 0; /* current layer processed by the main thread */ -static const int g_asymmetric_caps[16] = { - 34, 36, 34, 34, 28, 30, 26, 28, 34, 34, 34, 34, 34, 36, 30, 30 -}; static void pilot_prefetch(Model *m, int lnext, const float *x, int S); -static void pilot_prefetch_next_token(Model *m, int lnext); static void *pilot_worker(void *arg); static void ensure_pilot_worker_started(Model *m); static void slot_ensure_allocated(Model *m, Slot *s); @@ -832,64 +827,6 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { free(logits); } -static void pilot_prefetch_next_token(Model *m, int lnext) { - if (lnext < 0 || lnext >= m->c.n_layers) return; - Cfg *c = &m->c; int E = c->n_experts; - float *ema = m->momentum_logits + (int64_t)lnext * E; - int is_zero = 1; - for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } - if (is_zero) return; - - int cand = c->topk; - int idx[64]; - for (int kk = 0; kk < cand; kk++) { - int best = -1; float bv = -1e30f; - for (int e = 0; e < E; e++) { - int taken = 0; for (int j = 0; j < kk; j++) if (idx[j] == e) { taken = 1; break; } - if (!taken && ema[e] > bv) { bv = ema[e]; best = e; } - } - idx[kk] = best; - } - - for (int a = 0; a < cand - 1; a++) - for (int b = a + 1; b < cand; b++) - if (idx[b] >= 0 && (idx[a] < 0 || idx[a] > idx[b])) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; } - - for (int kk = 0; kk < cand; kk++) { - int eid = idx[kk]; - if (eid < 0) continue; - int found = 0; - pthread_mutex_lock(&g_pilot_mx); - LCache *lc = &m->cache[lnext]; - for (int z = 0; z < lc->n; z++) { - if (lc->slots[z].eid == eid) { found = 1; break; } - } - pthread_mutex_unlock(&g_pilot_mx); - if (!found) { - int gidx = lnext * E + eid; - pthread_mutex_lock(&g_pilot_mx); - int already_queued = m->is_queued[gidx]; - if (!already_queued) { - m->is_queued[gidx] = 1; - } - pthread_mutex_unlock(&g_pilot_mx); - - if (!already_queued) { - unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); - unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); - if (w2 - r2 < 4096) { - pilot_q[w2 & 4095].l = lnext; - pilot_q[w2 & 4095].e = eid; - __atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE); - } else { - pthread_mutex_lock(&g_pilot_mx); - m->is_queued[gidx] = 0; - pthread_mutex_unlock(&g_pilot_mx); - } - } - } - } -} /* generazione greedy. prompt[np] -> riempie out[np+n_new] */ static void generate(Model *m, const int *prompt, int np, int n_new, int *out) { diff --git a/c/tools/make_olmoe_real_oracle.py b/c/tools/make_olmoe_real_oracle.py index 374d29f..1e638e4 100644 --- a/c/tools/make_olmoe_real_oracle.py +++ b/c/tools/make_olmoe_real_oracle.py @@ -22,29 +22,25 @@ try: except ImportError as exc: sys.exit(f"Missing deps: {exc}. Run: pip install torch transformers") -MODEL_DIR = ( - Path(r"C:\Users\egonr\.cache\huggingface\hub" - r"\models--allenai--OLMoE-1B-7B-0125-Instruct" - r"\snapshots\b89a7c4bc24fb9e55ce2543c9458ce0ca5c4650e") -) +MODEL_ID = "allenai/OLMoE-1B-7B-0125-Instruct" OUT_JSON = Path(__file__).resolve().parent.parent / "ref_olmoe_real.json" PROMPT = "The capital of France is" MAX_NEW_TOKENS = 12 -print(f"Loading tokenizer from {MODEL_DIR} ...") -tokenizer = AutoTokenizer.from_pretrained(str(MODEL_DIR)) +print(f"Loading tokenizer from {MODEL_ID} ...") +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) print("Encoding prompt ...") enc = tokenizer(PROMPT, return_tensors="pt") prompt_ids = enc["input_ids"][0].tolist() print(f" Prompt IDs ({len(prompt_ids)}): {prompt_ids}") -print(f"Loading OLMoE model from {MODEL_DIR} ...") +print(f"Loading OLMoE model from {MODEL_ID} ...") print(" (this will use ~14 GB RAM — please be patient)") model = OlmoeForCausalLM.from_pretrained( - str(MODEL_DIR), + MODEL_ID, torch_dtype=torch.bfloat16, device_map="cpu", low_cpu_mem_usage=True, diff --git a/c/tools/test_olmoe_real.py b/c/tools/test_olmoe_real.py index ac47dc2..1abf64d 100644 --- a/c/tools/test_olmoe_real.py +++ b/c/tools/test_olmoe_real.py @@ -22,8 +22,9 @@ if sys.platform == "win32": pass HERE = Path(__file__).resolve().parent.parent -ENGINE = HERE / "olmoe.exe" -SNAP = HERE / "olmoe_i4" +ext = ".exe" if sys.platform == "win32" else "" +ENGINE = HERE / f"olmoe{ext}" +SNAP = os.getenv("SNAP", str(HERE.parent / "olmoe_merged")) REF_OUT = HERE / "ref_olmoe_real.json" BOOTSTRAP_REF = HERE / "ref_olmoe_bootstrap.json" From 1ac2e7b487ad061f849b49f4905a3a11787f15eb Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Thu, 16 Jul 2026 14:36:53 +0200 Subject: [PATCH 08/95] fix(prefetch): address copilot code quality reviews on safety and concurrency --- c/olmoe.c | 70 ++++++++++++--------------------- c/tools/convert_olmoe_merged.py | 3 +- 2 files changed, 27 insertions(+), 46 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index 35e449a..36fc0b0 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -54,8 +54,7 @@ typedef struct { * meccanismo che fa stare GLM-5.2 nei 15 GB. dequant-on-use nel matmul. */ /* IMPROVEMENT 2: pinned=1 means this slot is never evicted (hot expert). */ typedef struct { int eid; int pinned; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot; -/* IMPROVEMENT 3: per-layer hit/miss stats for adaptive rebalancing. */ -typedef struct { Slot *slots; int n, cap; uint64_t layer_hits, layer_miss; } LCache; +typedef struct { Slot *slots; int n, cap; } LCache; typedef struct { Cfg c; @@ -70,8 +69,7 @@ typedef struct { /* IMPROVEMENT 2: expert frequency heatmap */ uint32_t *freq; int freq_token_count, hot_pinned, hot_n, warmup_tokens; - /* IMPROVEMENT 3: adaptive rebalance */ - int token_count, rebal_interval, total_cap; + int token_count; /* PREDICTION IMPROVEMENT A: per-layer smoothed gate logits across tokens. * momentum_logits[l*E .. (l+1)*E-1] = EMA of recent gate outputs. * Blended with fresh gate prediction: final = (1-smooth)*fresh + smooth*ema. @@ -99,7 +97,11 @@ static void ensure_pilot_worker_started(Model *m) { if (!pilot_m) { pilot_m = m; pthread_t t; - pthread_create(&t, NULL, pilot_worker, NULL); + if (pthread_create(&t, NULL, pilot_worker, NULL) != 0) { + fprintf(stderr, "Error: Failed to create pilot prefetch worker thread\n"); + exit(1); + } + pthread_detach(t); } } @@ -263,7 +265,6 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { LD(gate, "mlp.gate.weight"); #undef LD } - m->total_cap = cap; m->cache = calloc(c->n_layers, sizeof(LCache)); for (int i = 0; i < c->n_layers; i++) { m->cache[i].cap = cap; @@ -274,8 +275,6 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { m->hot_pinned = 0; m->freq_token_count = 0; m->hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0; m->warmup_tokens = getenv("WARMUP") ? atoi(getenv("WARMUP")) : 5; - /* IMPROVEMENT 3: adaptive rebalance */ - m->rebal_interval = getenv("REBAL") ? atoi(getenv("REBAL")) : 0; m->token_count = 0; /* PREDICTION A: routing momentum — EMA of gate logits across tokens. * Initialized to zero; first token sets EMA = fresh logits. */ @@ -309,7 +308,9 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); if (w - r < 4096) { pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = e; + pthread_mutex_lock(&g_pilot_mx); m->is_queued[l * c->n_experts + e] = 1; + pthread_mutex_unlock(&g_pilot_mx); __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE); } } @@ -330,13 +331,16 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { } } - static void slot_ensure_allocated(Model *m, Slot *s) { if (s->g) return; Cfg *c = &m->c; int64_t ng = (int64_t)c->inter * c->hidden; int64_t nd = (int64_t)c->hidden * c->inter; int8_t *w_block = malloc(ng + ng + nd); + if (!w_block) { + fprintf(stderr, "Error: Out of memory allocating slot weights block\n"); + exit(1); + } s->g = w_block; s->u = w_block + ng; s->d = w_block + ng + ng; @@ -360,11 +364,11 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) { LCache *lc = &m->cache[layer]; pthread_mutex_lock(&g_pilot_mx); for (int i = 0; i < lc->n; i++) if (lc->slots[i].eid == eid) { - m->hits++; lc->layer_hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; + m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; pthread_mutex_unlock(&g_pilot_mx); return; } - m->miss++; lc->layer_miss++; + m->miss++; Cfg *c = &m->c; Slot *s; if (lc->n < lc->cap) { @@ -416,6 +420,7 @@ static void pin_hot_experts(Model *m) { if (max_pin < 4) max_pin = 4; int hn = is_dynamic ? max_pin : (m->hot_n < c->n_experts ? m->hot_n : c->n_experts); + if (hn > 256) hn = 256; int hot_eids[256]; int actual_hn = 0; @@ -463,29 +468,6 @@ static void pin_hot_experts(Model *m) { } } -/* ---------- IMPROVEMENT 3: adaptive per-layer cache rebalancing ---------- */ -static void rebalance_cache(Model *m) { - Cfg *c = &m->c; - uint64_t total_miss = 0; - for (int l = 0; l < c->n_layers; l++) total_miss += m->cache[l].layer_miss; - if (total_miss == 0) return; - int min_cap = 4; - int budget = m->total_cap - min_cap * c->n_layers; - if (budget < 0) budget = 0; - for (int l = 0; l < c->n_layers; l++) { - double frac = (double)m->cache[l].layer_miss / (double)total_miss; - int new_cap = min_cap + (int)(frac * budget + 0.5); - LCache *lc = &m->cache[l]; - if (new_cap > lc->cap) { - Slot *ns = realloc(lc->slots, new_cap * sizeof(Slot)); - if (ns) { - memset(ns + lc->cap, 0, (new_cap - lc->cap) * sizeof(Slot)); - lc->slots = ns; lc->cap = new_cap; - } - } - lc->layer_hits = 0; lc->layer_miss = 0; - } -} /* ---------- RoPE su un vettore di una testa (head_dim) a posizione assoluta pos ---------- */ static void rope_head(float *x, int pos, const Cfg *c) { @@ -607,7 +589,9 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { if (g_pilot && m->token_count > 0) { unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); __atomic_store_n(&pilot_w, r, __ATOMIC_RELEASE); + pthread_mutex_lock(&g_pilot_mx); memset(m->is_queued, 0, (size_t)c->n_layers * c->n_experts); + pthread_mutex_unlock(&g_pilot_mx); } float *x = falloc((int64_t)S*D); for (int s = 0; s < S; s++) memcpy(x + (int64_t)s*D, m->embed + (int64_t)ids[s]*D, D*sizeof(float)); @@ -636,9 +620,6 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { m->token_count++; m->freq_token_count++; if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens) pin_hot_experts(m); - /* IMPROVEMENT 3: periodic adaptive rebalance */ - if (m->rebal_interval > 0 && m->token_count % m->rebal_interval == 0) - rebalance_cache(m); m->kv_len = pos_base + S; float *last = falloc(D); rmsnorm_row(last, x + (int64_t)(S-1)*D, m->final_norm, D, c->eps); @@ -711,9 +692,6 @@ static void *pilot_worker(void *arg) { static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { if (lnext < 0 || lnext >= m->c.n_layers) return; Cfg *c = &m->c; int D = c->hidden, E = c->n_experts; - /* IMPROVEMENT 4: wide prefetch — top K * g_wide candidates */ - int cand = c->topk * g_wide; - if (cand > E) cand = E; ensure_pilot_worker_started(m); float *logits = falloc((int64_t)S * E); Layer *l = &m->L[lnext]; @@ -754,16 +732,19 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { int cand = 0; int idx[128]; + float max_logit = -1e30f; + for (int e = 0; e < E; e++) { if (blended[e] > max_logit) max_logit = blended[e]; } float *exps = falloc(E); float sum_exps = 0.f; for (int e = 0; e < E; e++) { - exps[e] = expf(blended[e]); + exps[e] = expf(blended[e] - max_logit); sum_exps += exps[e]; } float cum_sum = 0.f; int min_cand = c->topk; - int max_cand = c->topk * 2; + int max_cand = c->topk * g_wide; + if (max_cand < min_cand) max_cand = min_cand; if (max_cand > E) max_cand = E; for (int kk = 0; kk < max_cand; kk++) { @@ -867,7 +848,6 @@ int main(int argc, char **argv) { if (g_wide < 1) g_wide = 1; if (g_wide > 4) g_wide = 4; int hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0; - int rebal = getenv("REBAL") ? atoi(getenv("REBAL")) : 0; int cap = argc > 1 ? atoi(argv[1]) : 16; int bits = argc > 2 ? atoi(argv[2]) : 8; if (bits < 2 || bits > 8) { @@ -879,8 +859,8 @@ int main(int argc, char **argv) { float smooth = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f; float conf = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f; - printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d rebal=%d smooth=%.2f conf=%.2f ==\n", - cap, bits, g_pilot, g_wide, hot_n, rebal, smooth, conf); + printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d smooth=%.2f conf=%.2f ==\n", + cap, bits, g_pilot, g_wide, hot_n, smooth, conf); FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; } fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); diff --git a/c/tools/convert_olmoe_merged.py b/c/tools/convert_olmoe_merged.py index b39b123..883b8df 100644 --- a/c/tools/convert_olmoe_merged.py +++ b/c/tools/convert_olmoe_merged.py @@ -20,8 +20,9 @@ if sys.platform == "win32": try: import torch from safetensors.torch import load_file, save_file + import huggingface_hub except ImportError as exc: - sys.exit(f"Missing dependencies: {exc}. Install: pip install torch safetensors") + sys.exit(f"Missing dependencies: {exc}. Install: pip install torch safetensors huggingface_hub") EXPERT_KEY_RE = r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight" From 2d8d2951ee7d93cf2adad7b89b56c0adc1dbd0c5 Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Thu, 16 Jul 2026 15:37:20 +0200 Subject: [PATCH 09/95] fix(prefetch): address second round of copilot review comments - Fix ENV VARS header: document PILOT=0-3, SMOOTH, CONF_LIMIT; remove stale REBAL entry - Fix per-layer EMA: apply routing momentum to all layers (not just layer 0) with correct offset - Fix in-flight slot race in expert_get: LRU eviction now skips slots with eid==-1 (being loaded) - Fix in-flight slot race in pilot_realload: same fix, prevents concurrent writes into active slot - Fix idx[] buffer overflow: clamp max_cand to 128 before E in pilot_prefetch --- c/olmoe.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index 36fc0b0..51e32ae 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -7,12 +7,13 @@ * Matmul multi-thread con OpenMP (niente BLAS). * * ENV VARS: - * PILOT=0/1/2 : 0=no prefetch, 1=1-layer lookahead, 2=2-layer lookahead [IMPROVEMENT 1] - * HOT=N : pin top-N hot experts per layer permanently (never evict) [IMPROVEMENT 2] - * WARMUP=N : tokens before hot pinning activates (default 5) [IMPROVEMENT 2] - * REBAL=N : rebalance cache per-layer every N tokens (0=off) [IMPROVEMENT 3] - * WIDE=N : prefetch top-K*N candidates (default 1, try 2 or 3) [IMPROVEMENT 4] - * (expert queue is sorted by eid for SSD locality) [IMPROVEMENT 5] + * PILOT=0/1/2/3 : 0=no prefetch, 1=1-layer lookahead, 2=2-layer, 3=3-layer lookahead + * HOT=N : pin top-N hot experts per layer permanently (never evict) + * WARMUP=N : tokens before hot pinning activates (default 5) + * WIDE=N : prefetch top-K*N candidates (default 1, try 2 or 3) + * SMOOTH=F : EMA coefficient for routing momentum (default 0.3, range 0.0-0.95) + * CONF_LIMIT=F : cumulative gate probability threshold for prefetch cutoff (default 0.92) + * (expert queue is sorted by eid for SSD read locality) */ #define _GNU_SOURCE #include @@ -375,13 +376,13 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) { s = &lc->slots[lc->n++]; slot_ensure_allocated(m, s); } else { - /* IMPROVEMENT 2: LRU eviction — never evict pinned experts */ + /* LRU eviction — skip pinned and in-flight (eid==-1) slots */ int lru = -1; for (int i = 0; i < lc->n; i++) { - if (lc->slots[i].pinned) continue; + if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue; if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; } - if (lru < 0) lru = 0; /* all pinned: fallback evict oldest */ + if (lru < 0) lru = 0; /* all pinned/in-flight: fallback evict oldest */ s = &lc->slots[lru]; s->pinned = 0; } @@ -539,8 +540,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { float *g = falloc(I), *u = falloc(I), *hh = falloc(D); for (int s = 0; s < S; s++) { float *pr = logits + (int64_t)s*E; - if (layer == 0 && m->momentum_logits) { - float *ema = m->momentum_logits; + if (m->momentum_logits && m->pilot_smooth > 0.f) { + float *ema = m->momentum_logits + (int64_t)layer * E; int is_zero = 1; for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } if (is_zero) { @@ -646,16 +647,16 @@ static void pilot_realload(Model *m, int layer, int eid) { s = &lc->slots[lc->n++]; slot_ensure_allocated(m, s); } else { - /* IMPROVEMENT 2: never evict pinned experts */ + /* LRU eviction — skip pinned and in-flight (eid==-1) slots */ int lru = -1; for (int i = 0; i < lc->n; i++) { - if (lc->slots[i].pinned) continue; + if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue; if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; } if (lru < 0) { m->is_queued[layer * c->n_experts + eid] = 0; pthread_mutex_unlock(&g_pilot_mx); - return; /* all pinned, skip */ + return; /* all pinned/in-flight, skip */ } s = &lc->slots[lru]; s->pinned = 0; } @@ -745,6 +746,7 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { int min_cand = c->topk; int max_cand = c->topk * g_wide; if (max_cand < min_cand) max_cand = min_cand; + if (max_cand > 128) max_cand = 128; /* idx[] buffer bound */ if (max_cand > E) max_cand = E; for (int kk = 0; kk < max_cand; kk++) { From b6bae91b66a14647659d6c7c6a6c68adbf560564 Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Thu, 16 Jul 2026 15:46:26 +0200 Subject: [PATCH 10/95] fix(prefetch): address third round of copilot review comments - Fix LRU fallback: when all evictable slots are in-flight, find oldest non-in-flight slot (pinned ok) before falling back to slot 0 - Fix pin_hot_experts: guard enqueue behind g_pilot>0, call ensure_pilot_worker_started(), and set is_queued flag to prevent duplicate in-flight loads from pilot_prefetch() - Fix token counting: increment token_count/freq_token_count by S (batch size) instead of 1 so prefill tokens are counted accurately and warmup threshold triggers at the right time --- c/olmoe.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index 51e32ae..41cf776 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -382,7 +382,15 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) { if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue; if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; } - if (lru < 0) lru = 0; /* all pinned/in-flight: fallback evict oldest */ + if (lru < 0) { + /* All slots are pinned or in-flight; find oldest non-in-flight slot + * (may be pinned, but never select one currently being loaded). */ + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].eid < 0) continue; /* never evict in-flight */ + if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; + } + } + if (lru < 0) lru = 0; /* absolute last resort: all in-flight, evict slot 0 */ s = &lc->slots[lru]; s->pinned = 0; } @@ -449,13 +457,20 @@ static void pin_hot_experts(Model *m) { if (lc->slots[i].eid == eid) { lc->slots[i].pinned = 1; found = 1; break; } } pthread_mutex_unlock(&g_pilot_mx); - if (!found) { + if (!found && g_pilot > 0) { + /* Only enqueue when the prefetch worker is active (PILOT>0). */ + ensure_pilot_worker_started(m); unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); - if (w - r < 4096) { + int gidx = l * c->n_experts + eid; + pthread_mutex_lock(&g_pilot_mx); + int already = m->is_queued[gidx]; + if (!already && w - r < 4096) { pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = eid; + m->is_queued[gidx] = 1; __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE); } + pthread_mutex_unlock(&g_pilot_mx); } pinned_total++; } @@ -617,8 +632,8 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { pilot_prefetch(m, i + 3, x, S); } - /* IMPROVEMENT 2: count tokens; trigger hot pinning after warmup */ - m->token_count++; m->freq_token_count++; + /* count actual tokens processed (S>1 during prefill) */ + m->token_count += S; m->freq_token_count += S; if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens) pin_hot_experts(m); m->kv_len = pos_base + S; From c769e04d136ad8aa48dc79745363353a263ad179 Mon Sep 17 00:00:00 2001 From: Egon Ruiter Date: Thu, 16 Jul 2026 15:58:38 +0200 Subject: [PATCH 11/95] fix(prefetch): address fourth round of copilot review comments - Fix queue flush race: clear is_queued under mutex only; never move pilot_w backwards (would break r<=w ring-buffer invariant). Worker skips stale entries via new is_queued guard at start of pilot_realload - Add early-exit in pilot_realload when is_queued==0 (entry flushed between enqueue and worker pickup), preventing unnecessary loads - Fix misleading EMA struct comment: momentum_logits is used only by the PILOT prefetcher, not blended into actual MoE routing decisions - Fix Slot pinned comment: 'never evicted' was too strong; clarify that pinned slots may be displaced under extreme all-pinned cache pressure - Use st_read_f32() for scale tensor (.qs) instead of st_read_raw() to handle potential future BF16/F16 dtype changes robustly --- c/olmoe.c | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/c/olmoe.c b/c/olmoe.c index 41cf776..21b07e8 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -53,7 +53,9 @@ typedef struct { * Ogni weight [out,in] tenuto come int8 (per-riga) + scala float per riga. * Cosi' la RAM-cache scende da 4 byte/param (f32) a 1 byte/param: e' il * meccanismo che fa stare GLM-5.2 nei 15 GB. dequant-on-use nel matmul. */ -/* IMPROVEMENT 2: pinned=1 means this slot is never evicted (hot expert). */ +/* pinned=1 means this slot is strongly preferred to keep (hot expert); it will + * not be evicted during normal LRU eviction, but may be displaced under extreme + * cache pressure when all slots are pinned or in-flight. */ typedef struct { int eid; int pinned; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot; typedef struct { Slot *slots; int n, cap; } LCache; @@ -71,10 +73,10 @@ typedef struct { uint32_t *freq; int freq_token_count, hot_pinned, hot_n, warmup_tokens; int token_count; - /* PREDICTION IMPROVEMENT A: per-layer smoothed gate logits across tokens. - * momentum_logits[l*E .. (l+1)*E-1] = EMA of recent gate outputs. - * Blended with fresh gate prediction: final = (1-smooth)*fresh + smooth*ema. - * Captures routing consistency across tokens (same token tends to reuse experts). */ + /* PREDICTION IMPROVEMENT A: per-layer EMA of gate logits across tokens. + * momentum_logits[l*E .. (l+1)*E-1] = EMA of gate outputs for layer l. + * Used exclusively by the PILOT prefetcher to stabilise routing predictions + * across tokens; does NOT affect actual MoE routing (pr is unchanged). */ float *momentum_logits; /* [n_layers * n_experts], EMA of gate logits */ float pilot_smooth; /* SMOOTH env: EMA coefficient 0.0-0.9 (default 0.3) */ uint8_t *is_pinned; /* [n_layers * n_experts], 1 if expert is globally pinned */ @@ -357,7 +359,7 @@ static void load_expert_merged(Model *m, int layer, int eid, Slot *s) { snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.merged_weight", layer, eid); snprintf(qsnm, sizeof(qsnm), "model.layers.%d.mlp.experts.%d.qs", layer, eid); st_read_raw(&m->S, nm, s->g, 1); - st_read_raw(&m->S, qsnm, s->gs, 1); + st_read_f32(&m->S, qsnm, s->gs, 0); /* scales are F32; use typed reader for dtype safety */ } /* ---------- cache expert: ritorna i pesi quantizzati (q+scale) da cache o disco ---------- */ @@ -603,8 +605,13 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { static float *step(Model *m, const int *ids, int S, int pos_base) { Cfg *c = &m->c; int D = c->hidden; if (g_pilot && m->token_count > 0) { - unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); - __atomic_store_n(&pilot_w, r, __ATOMIC_RELEASE); + /* Flush stale prefetch requests: clear is_queued so pilot_realload + * will skip any entries still sitting in pilot_q for the previous + * token. We deliberately do NOT move pilot_w backwards; that would + * break the ring-buffer invariant (pilot_r could exceed pilot_w if + * the worker consumed an entry concurrently). The worker will drain + * the stale slots harmlessly because pilot_realload already exits + * early when the expert is already cached or is_queued is clear. */ pthread_mutex_lock(&g_pilot_mx); memset(m->is_queued, 0, (size_t)c->n_layers * c->n_experts); pthread_mutex_unlock(&g_pilot_mx); @@ -650,6 +657,11 @@ static void pilot_realload(Model *m, int layer, int eid) { Cfg *c = &m->c; pthread_mutex_lock(&g_pilot_mx); + /* Early-exit if entry was flushed (is_queued cleared) while waiting. */ + if (!m->is_queued[layer * c->n_experts + eid]) { + pthread_mutex_unlock(&g_pilot_mx); + return; + } for (int i = 0; i < lc->n; i++) { if (lc->slots[i].eid == eid) { m->is_queued[layer * c->n_experts + eid] = 0; From 72d3d37231e922a6fa9afca16e08fa45842d5eb4 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Thu, 16 Jul 2026 19:39:40 +0200 Subject: [PATCH 12/95] ci: run the checks on main too (#144) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @ZacharyZcR's workflow, already green on dev (78c77bf): engine + C suite, web build + vitest, python suite, and a real CUDA syntax check. Only the workflow file is cherry-picked here — main's engine code stays at 54cfe56 and nothing else from dev comes along. main is where the checks are worth the most and where they were missing entirely. Includes the two fixes the first run exposed: the pinned action's version table stops at 12.6.2 (12.6.3 failed the install), and `nvcc ... | head -40` took its exit status from head, so the job could never fail. Co-Authored-By: ZacharyZcR <#144> Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 72 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e18f1b3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,72 @@ +name: CI + +on: + pull_request: + branches: [dev, main] + push: + branches: [dev, main] + +jobs: + engine: + name: Engine (Linux, CPU) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build glm + run: cd c && make glm + - name: C test suite + run: cd c && make test-c + + engine-cuda-syntax: + name: CUDA syntax check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install CUDA toolkit (compiler only) + uses: Jimver/cuda-toolkit@v0.2.19 + with: + # v0.2.19's version table stops at 12.6.2 — 12.6.3 fails the install + # step with "Version not available" before nvcc is ever reached. + cuda: '12.6.2' + method: network + sub-packages: '["nvcc"]' + - name: Compile backend_cuda.cu (syntax only, no GPU) + run: | + cd c + # NOT `nvcc ... | head -40`: a pipeline exits with the status of its + # LAST command, so head's 0 masked every nvcc error and the job could + # only ever be green. A check that cannot fail is worse than no check — + # it buys false confidence in the one file nothing else compiles. + nvcc -O2 -std=c++17 -arch=sm_80 -c backend_cuda.cu -o /dev/null \ + -Xcompiler=-Wall,-Wextra + echo "CUDA syntax check passed" + + web: + name: Web UI + runs-on: ubuntu-latest + defaults: + run: + working-directory: web + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + cache-dependency-path: web/package-lock.json + - run: npm ci + - name: Build + run: npm run build + - name: Test + run: npx vitest run + + python: + name: Python tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Python test suite + run: cd c && python3 -m unittest discover -s tests -p 'test_*.py' From 80f886cd22e7003ba5bcc85d7ab6dfd4b5290636 Mon Sep 17 00:00:00 2001 From: Mohamed Mastouri Date: Thu, 16 Jul 2026 21:59:16 +0300 Subject: [PATCH 13/95] convert_olmoe.py: wire --ebits through to quantization (#323) --- c/tools/convert_olmoe.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/c/tools/convert_olmoe.py b/c/tools/convert_olmoe.py index dd45806..dc25f6a 100644 --- a/c/tools/convert_olmoe.py +++ b/c/tools/convert_olmoe.py @@ -3,7 +3,10 @@ Downloads or converts a local OLMoE checkpoint (e.g., allenai/OLMoE-1B-7B-0125-Instruct). Dense weights stay as-is (engine reads BF16/F16 → F32 on load). -Expert weights get row-wise int8 quantization with float32 scales. +Expert weights get row-wise symmetric quantization to --ebits bits (default 4) +with float32 scales. Storage stays one value per int8 byte regardless of bits, +matching the engine's expert layout (olmoe.c quantize_rows) — for 4 bits the +values are simply confined to [-8, 7] with scales computed against qmax=7. Usage: python tools/convert_olmoe.py --repo allenai/OLMoE-1B-7B-0125-Instruct --out ./olmoe_i4 @@ -29,12 +32,21 @@ except ImportError as exc: EXPERT_KEY_RE = r"model\.layers\.\d+\.mlp\.experts\.\d+\.(gate_proj|up_proj|down_proj)\.weight" -def quantize_row(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: - """Row-wise int8 quantization. Returns (int8_weights, float32_scales).""" +def quantize_row(w: torch.Tensor, bits: int = 8) -> tuple[torch.Tensor, torch.Tensor]: + """Row-wise symmetric quantization to `bits` (2..8). + + Returns (int8_weights, float32_scales). Storage is one value per int8 byte + for every bit width — the engine dequantizes as q*scale and never assumes + the full int8 range — mirroring olmoe.c quantize_rows(): + qmax = 2**(bits-1) - 1 (8 -> 127, 4 -> 7, 2 -> 1) + scale = amax(|w|, row) / qmax + q = clamp(round(w / scale), -qmax-1, qmax) + """ + qmax = (1 << (bits - 1)) - 1 w_f32 = w.float() row_max = w_f32.abs().amax(dim=1, keepdim=True).clamp(min=1e-12) - scales = row_max / 127.0 - q = (w_f32 / scales).round().clamp(-128, 127).to(torch.int8) + scales = row_max / qmax + q = (w_f32 / scales).round().clamp(-qmax - 1, qmax).to(torch.int8) return q, scales.squeeze(1) @@ -50,9 +62,12 @@ def main(): src.add_argument("--model", help="Local HF checkpoint directory") ap.add_argument("--out", required=True, help="Output directory for int4 model") ap.add_argument("--ebits", type=int, default=4, - help="Expert quant bits (4 or 8, default 4)") + help="Expert quant bits (2..8, default 4)") args = ap.parse_args() + if not 2 <= args.ebits <= 8: # storage is int8_t; engine rejects the same range (olmoe.c) + sys.exit(f"--ebits must be 2..8 (got {args.ebits})") + if args.repo: from huggingface_hub import snapshot_download from huggingface_hub.errors import LocalEntryNotFoundError @@ -96,7 +111,7 @@ def main(): for name, tensor in tensors.items(): if is_expert_weight(name): expert_count += 1 - q, scales = quantize_row(tensor) + q, scales = quantize_row(tensor, args.ebits) total_expert_f32 += tensor.numel() * tensor.element_size() total_expert_q += q.numel() * 1 + scales.numel() * 4 out_tensors[name] = q @@ -109,7 +124,7 @@ def main(): ratio = total_expert_q / max(total_expert_f32, 1) * 100 print(f"ok") - print(f"\nDone. {expert_count} expert tensors quantized.") + print(f"\nDone. {expert_count} expert tensors quantized to int{args.ebits}.") print(f"Expert storage: {total_expert_f32/1e9:.1f} GB -> {total_expert_q/1e9:.1f} GB ({ratio:.0f}%)") print(f"Model ready at: {out}") print(f"\nRun: SNAP={out} ./olmoe.exe 32 4 16") From 69284facb8a04c6a2f8e633dfa48351977141cf3 Mon Sep 17 00:00:00 2001 From: Mohamed Mastouri Date: Thu, 16 Jul 2026 21:59:20 +0300 Subject: [PATCH 14/95] docs: Windows 11 native install walkthrough - toolchain, SAC, CUDA DLL, failure index (#198) --- docs/WINDOWS.md | 100 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 docs/WINDOWS.md diff --git a/docs/WINDOWS.md b/docs/WINDOWS.md new file mode 100644 index 0000000..52d83cd --- /dev/null +++ b/docs/WINDOWS.md @@ -0,0 +1,100 @@ +# Windows 11 native install — a complete walkthrough (no WSL) + +A start-to-finish, reproducible path from a fresh Windows 11 machine to GLM-5.2 generating tokens, with the GPU tier. Every step and every failure mode below was hit and verified on real hardware: Core Ultra 9 285K (AVX-VNNI) / RTX 5080 (sm_120) / 128 GB RAM / Windows 11 24H2 (issue #306). Steps are ordered so the long downloads run while you build. + +## 0. What you need + +| Piece | Why | Get it | +|---|---|---| +| git, Python 3 | clone + `coli` launcher | winget / python.org | +| MinGW-w64 gcc + make | builds the engine (MSVC can't) | `scoop install mingw-winlibs`, MSYS2, or portable **w64devkit** (no admin, unzip and go) | +| CUDA Toolkit ≥ 12.8 | GPU tier; ≥12.8 required for Blackwell/sm_120 | `winget install Nvidia.CUDA` | +| MSVC Build Tools (C++ workload) | nvcc's host compiler for the CUDA DLL | `winget install Microsoft.VisualStudio.2022.BuildTools` + "Desktop development with C++" | +| ~400 GB free on a local NVMe | the int4 model (~370–384 GB) | NTFS is fine; **never** a network mount | + +RAM: 16 GB minimum, more = bigger expert cache = faster. The build itself needs none of the CUDA/MSVC pieces — do the CPU build first, add the GPU tier later. + +## 1. Start the model download first (it's the long pole) + +```powershell +python -m pip install -U "huggingface_hub[hf_transfer]" +$env:HF_HUB_ENABLE_HF_TRANSFER = "1" +hf download --local-dir D:\glm52_i4 +``` + +Use the container recommended in the README (with **int8 MTP heads** — int4 heads silently give 0% draft acceptance). The download is resumable: if it stops, rerun the same command. Expect hours; everything below fits inside them. + +## 2. Build the engine (CPU) + +From a normal PowerShell, in the repo's `c\` directory: + +```powershell +make glm.exe ARCH=native # ARCH=native unlocks AVX-VNNI on Alder Lake+/Arrow Lake +make iobench.exe # disk benchmark, useful before committing to the download +``` + +Warnings about `#pragma comment` and unused variables are normal (MSVC-isms gcc ignores). The engine banner should print `idot: avx-vnni` on VNNI-capable CPUs — if it says avx2, you built without `ARCH=native`. + +### ⚠️ Smart App Control will block your fresh binary + +On Windows 11 machines with **Smart App Control** enforced (`VerifiedAndReputablePolicyState = 1`), running your self-compiled `glm.exe` fails with: + +``` +Program 'glm.exe' failed to run: An Application Control policy has blocked this file +``` + +This is not Defender and not Mark-of-the-Web — SAC blocks *all* unsigned, unknown binaries, which includes anything you compile yourself. **Fix:** Windows Security → App & browser control → Smart App Control settings → **Off**, then **reboot** (the policy only reloads on restart). Note SAC is one-way: re-enabling later requires resetting Windows. If the settings page is missing, the registry equivalent is setting `HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy\VerifiedAndReputablePolicyState` to `0` (admin PowerShell), then rebooting. Check your current state before touching anything: + +```powershell +(Get-ItemProperty "HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy").VerifiedAndReputablePolicyState +# 0 = off, 1 = enforced, 2 = evaluation +``` + +## 3. Build the CUDA DLL (GPU tier) + +nvcc needs MSVC as host compiler, so this one step must run from a shell with the MSVC environment: open **"x64 Native Tools Command Prompt for VS 2022"** from the Start menu (plain PowerShell will fail the `cl` check). Then: + +```cmd +make cuda-dll CUDA_ARCH=sm_120 # match your GPU: sm_120 Blackwell, sm_89 Ada, ... +make glm.exe CUDA_DLL=1 ARCH=native # relink host with the runtime loader +``` + +Two pitfalls, both fixed on current `dev` (#314) but worth knowing on older checkouts: + +- **Spaces in `CUDA_HOME`** (`C:\Program Files\...`) used to break the recipe → fixed; nvcc now comes from PATH and `"$(NVCC)"` is quoted. +- **`make glm.exe CUDA_DLL=1` after a CPU-only build** used to report `up to date` and silently keep the CPU-only binary (GPU tier never engages, no error). Current `dev` has a build-config stamp that forces the relink. On older trees: delete `glm.exe` first. + +Sanity check: first GPU run should print `[CUDA] device 0: , ... sm_XX` and `[CUDA] mode: routed experts + resident dense tensors`. + +## 4. First run + +```powershell +cd \c +$env:OMP_NUM_THREADS = "" +python coli run "Explain what a mixture-of-experts model is." --model D:\glm52_i4 --ngen 48 +``` + +The first run is cold — expect the profile to be dominated by `expert-disk` while the cache warms; hit rate climbs run over run. GPU tier on top: + +```powershell +$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_DENSE="1"; $env:CUDA_EXPERT_GB="4" +python coli run "..." --model D:\glm52_i4 --ngen 64 +``` + +Size `CUDA_EXPERT_GB` so dense (~10 GB) + experts + working set stays under your VRAM. Note MTP speculation is off by default under CUDA (#293, float-accumulation divergence between draft and verify) — `COLI_CUDA_MTP=1` opts back in. + +## 5. Reference numbers from this walkthrough's hardware + +285K / RTX 5080 / 128 GB / NVMe at 5.85 GB/s random-read (19 MB blocks, `iobench`): 0.26 tok/s cold CPU → 0.30 warm CPU (MTP 2.2–2.3 tok/forward) → 0.42 tok/s GPU tier + auto-pin, expert hit 66%, ~65% of wall time in expert-disk. Disk-bound is the expected shape at ~25% expert residency — a faster disk and more RAM move the floor, the GPU moves the compute. + +## Quick failure index + +| Symptom | Cause | Fix | +|---|---|---| +| `An Application Control policy has blocked this file` | Smart App Control | §2 — turn SAC off + **reboot** | +| `cuda-dll ... Error 1` immediately | old tree: spaced CUDA_HOME / MSVC rejects `-Wextra` | update to current `dev` (#314) | +| `glm.exe is up to date` but GPU never engages | old tree: stale CPU-only binary | update to `dev`, or delete `glm.exe` and rebuild | +| `cl.exe (MSVC) not in PATH` | built from plain PowerShell | use the x64 Native Tools prompt | +| `nvcc fatal: unsupported gpu architecture 'sm_120'` | CUDA < 12.8 | install CUDA 12.8+ | +| MTP `0% (0/0)` on CPU path | int4 MTP heads in the container | use the int8-MTP container | +| MTP `draft=0` under CUDA | intended default since #293 | `COLI_CUDA_MTP=1` to opt in | From 364b9741e26a3127c40f560ca5d9e527857445be Mon Sep 17 00:00:00 2001 From: Nicholas Beerbower Date: Wed, 15 Jul 2026 23:10:10 -0400 Subject: [PATCH 15/95] tok: o200k pre-tokenizer support, auto-detected from tokenizer.json Inkling ships an o200k-family tokenizer (case-aware Split regex, GPT-4o lineage) rather than cl100k. tok_load now detects the family from the pattern itself (\p{Lu} appears only in the o200k regex) so GLM behavior is untouched, and encode dispatches to a new pretok_chunk_o200k that replays the regex engine's backtracking order exactly: greedy optional prefix, maximally-greedy uppercase run given back until the lowercase run can match, contractions attached to letter runs, \p{N}{1,3}, and the [\r\n/]* punctuation tail. tok_unicode_o200k.h adds the two range tables the new classes need (Lu+Lt and Lm+Lo+M), generated from Python unicodedata. Validated against HF tokenizers on 357 adversarial strings (case transitions, contractions, CJK, combining marks, emoji + modifiers, zero-width chars, 300 mixed-charset fuzz cases): 357/357 identical. Co-Authored-By: Claude Fable 5 --- c/tok.h | 116 ++++++++++++++++++++- c/tok_unicode_o200k.h | 228 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 c/tok_unicode_o200k.h diff --git a/c/tok.h b/c/tok.h index 7d1140b..6b957df 100644 --- a/c/tok.h +++ b/c/tok.h @@ -19,6 +19,7 @@ #include #include "json.h" #include "tok_unicode.h" +#include "tok_unicode_o200k.h" /* ---------- hash map (chiavi binarie con lunghezza) ---------- */ typedef struct { const char *k; int klen; int v; int used; } ment; @@ -50,6 +51,7 @@ typedef struct { Special *sp; int nsp; /* added tokens, ordinati per lunghezza decrescente */ uint32_t byte2cp[256]; int byte2cp_len[256]; char byte2str[256][3]; int16_t cp2byte[1024]; + int o200k; /* pre_tokenizer regex family: 0 = cl100k (GLM), 1 = o200k (Inkling) */ } Tok; /* ---------- UTF-8 ---------- */ @@ -144,6 +146,17 @@ static void tok_load(Tok *T, const char *path){ } qsort(T->sp,T->nsp,sizeof(Special),cmp_sp_len); /* match piu' lungo per primo */ } + /* pre_tokenizer family: the o200k Split regex is recognizable by its + * case-category classes (\p{Lu}...) which cl100k does not use */ + jval *pt=json_get(root,"pre_tokenizer"); + if(pt){ + jval *ps=json_get(pt,"pretokenizers"); + if(ps&&ps->t==J_ARR) for(int i=0;ilen;i++){ + jval *pat=json_get(ps->kids[i],"pattern"); + jval *rx=pat?json_get(pat,"Regex"):NULL; + if(rx&&rx->t==J_STR&&strstr(rx->str,"\\p{Lu}")) T->o200k=1; + } + } /* arena/buf restano allocati: le stringhe (j_dup) sono malloc indipendenti e ci servono vive */ (void)arena; } @@ -241,6 +254,104 @@ static void pretok_chunk(Tok *T, const unsigned char *p, int a, int b, int *out, free(cp); free(off); } +/* ---------- pre-tokenizer o200k (Inkling / GPT-4o family) ---------- + * Split regex: + * A: [^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)? + * B: [^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)? + * C: \p{N}{1,3} D: ' ?[^\s\p{L}\p{N}]+[\r\n/]*' E: \s*[\r\n]+ F: \s+(?!\S) G: \s+ + * S1 = Lu|Lt|Lm|Lo|M, S2 = Ll|Lm|Lo|M. The letter matcher below replays the + * regex engine's backtracking order exactly: A with greedy optional prefix and + * maximally-greedy S1* given back until S2+ can take >=1 char, then B. */ +#define O2_S1(c) (is_U(c)||is_X(c)) +#define O2_S2(c) (is_X(c)||(is_L(c)&&!is_U(c))) +static uint32_t o2_low(uint32_t c){ return (c>='A'&&c<='Z')?c+32:c; } +static int o2_contraction(const uint32_t *cp, int n, int k){ + if(k=0; pfx--){ + int j0=i; + if(pfx){ + uint32_t c=cp[i]; + if(c=='\r'||c=='\n'||is_L(c)||is_N(c)||i+1>=n) continue; + j0=i+1; + } + int m1=j0; while(m1=j0; s--){ + if(s=0; pfx--){ + int j0=i; + if(pfx){ + uint32_t c=cp[i]; + if(c=='\r'||c=='\n'||is_L(c)||is_N(c)||i+1>=n) continue; + j0=i+1; + } + int m1=j0; while(m1j0){ + int k=m1; while(ki){ i=e; bpe_piece(T,p,off[start],off[i],out,no,max); continue; } + } + /* C: \p{N}{1,3} */ + if(is_N(c)){ int j=i,k=0; while(ji){ int last=-1; for(int j=i;j=0){ i=last+1; bpe_piece(T,p,off[start],off[i],out,no,max); continue; } + int end = (r id (split sugli added token, poi pretok+BPE) ---------- */ static int tok_encode(Tok *T, const char *text, int len, int *out, int max){ const unsigned char *p=(const unsigned char*)text; int no=0; int i=0; @@ -254,7 +365,10 @@ static int tok_encode(Tok *T, const char *text, int len, int *out, int max){ } } int chunk_end = (hitpos<0) ? len : hitpos; - if(chunk_end>i) pretok_chunk(T,p,i,chunk_end,out,&no,max); + if(chunk_end>i){ + if(T->o200k) pretok_chunk_o200k(T,p,i,chunk_end,out,&no,max); + else pretok_chunk(T,p,i,chunk_end,out,&no,max); + } if(hitpos<0) break; if(no + +static const uint32_t uni_U[][2] = { + {0x41,0x5A},{0xC0,0xD6},{0xD8,0xDE},{0x100,0x100},{0x102,0x102},{0x104,0x104}, + {0x106,0x106},{0x108,0x108},{0x10A,0x10A},{0x10C,0x10C},{0x10E,0x10E},{0x110,0x110}, + {0x112,0x112},{0x114,0x114},{0x116,0x116},{0x118,0x118},{0x11A,0x11A},{0x11C,0x11C}, + {0x11E,0x11E},{0x120,0x120},{0x122,0x122},{0x124,0x124},{0x126,0x126},{0x128,0x128}, + {0x12A,0x12A},{0x12C,0x12C},{0x12E,0x12E},{0x130,0x130},{0x132,0x132},{0x134,0x134}, + {0x136,0x136},{0x139,0x139},{0x13B,0x13B},{0x13D,0x13D},{0x13F,0x13F},{0x141,0x141}, + {0x143,0x143},{0x145,0x145},{0x147,0x147},{0x14A,0x14A},{0x14C,0x14C},{0x14E,0x14E}, + {0x150,0x150},{0x152,0x152},{0x154,0x154},{0x156,0x156},{0x158,0x158},{0x15A,0x15A}, + {0x15C,0x15C},{0x15E,0x15E},{0x160,0x160},{0x162,0x162},{0x164,0x164},{0x166,0x166}, + {0x168,0x168},{0x16A,0x16A},{0x16C,0x16C},{0x16E,0x16E},{0x170,0x170},{0x172,0x172}, + {0x174,0x174},{0x176,0x176},{0x178,0x179},{0x17B,0x17B},{0x17D,0x17D},{0x181,0x182}, + {0x184,0x184},{0x186,0x187},{0x189,0x18B},{0x18E,0x191},{0x193,0x194},{0x196,0x198}, + {0x19C,0x19D},{0x19F,0x1A0},{0x1A2,0x1A2},{0x1A4,0x1A4},{0x1A6,0x1A7},{0x1A9,0x1A9}, + {0x1AC,0x1AC},{0x1AE,0x1AF},{0x1B1,0x1B3},{0x1B5,0x1B5},{0x1B7,0x1B8},{0x1BC,0x1BC}, + {0x1C4,0x1C5},{0x1C7,0x1C8},{0x1CA,0x1CB},{0x1CD,0x1CD},{0x1CF,0x1CF},{0x1D1,0x1D1}, + {0x1D3,0x1D3},{0x1D5,0x1D5},{0x1D7,0x1D7},{0x1D9,0x1D9},{0x1DB,0x1DB},{0x1DE,0x1DE}, + {0x1E0,0x1E0},{0x1E2,0x1E2},{0x1E4,0x1E4},{0x1E6,0x1E6},{0x1E8,0x1E8},{0x1EA,0x1EA}, + {0x1EC,0x1EC},{0x1EE,0x1EE},{0x1F1,0x1F2},{0x1F4,0x1F4},{0x1F6,0x1F8},{0x1FA,0x1FA}, + {0x1FC,0x1FC},{0x1FE,0x1FE},{0x200,0x200},{0x202,0x202},{0x204,0x204},{0x206,0x206}, + {0x208,0x208},{0x20A,0x20A},{0x20C,0x20C},{0x20E,0x20E},{0x210,0x210},{0x212,0x212}, + {0x214,0x214},{0x216,0x216},{0x218,0x218},{0x21A,0x21A},{0x21C,0x21C},{0x21E,0x21E}, + {0x220,0x220},{0x222,0x222},{0x224,0x224},{0x226,0x226},{0x228,0x228},{0x22A,0x22A}, + {0x22C,0x22C},{0x22E,0x22E},{0x230,0x230},{0x232,0x232},{0x23A,0x23B},{0x23D,0x23E}, + {0x241,0x241},{0x243,0x246},{0x248,0x248},{0x24A,0x24A},{0x24C,0x24C},{0x24E,0x24E}, + {0x370,0x370},{0x372,0x372},{0x376,0x376},{0x37F,0x37F},{0x386,0x386},{0x388,0x38A}, + {0x38C,0x38C},{0x38E,0x38F},{0x391,0x3A1},{0x3A3,0x3AB},{0x3CF,0x3CF},{0x3D2,0x3D4}, + {0x3D8,0x3D8},{0x3DA,0x3DA},{0x3DC,0x3DC},{0x3DE,0x3DE},{0x3E0,0x3E0},{0x3E2,0x3E2}, + {0x3E4,0x3E4},{0x3E6,0x3E6},{0x3E8,0x3E8},{0x3EA,0x3EA},{0x3EC,0x3EC},{0x3EE,0x3EE}, + {0x3F4,0x3F4},{0x3F7,0x3F7},{0x3F9,0x3FA},{0x3FD,0x42F},{0x460,0x460},{0x462,0x462}, + {0x464,0x464},{0x466,0x466},{0x468,0x468},{0x46A,0x46A},{0x46C,0x46C},{0x46E,0x46E}, + {0x470,0x470},{0x472,0x472},{0x474,0x474},{0x476,0x476},{0x478,0x478},{0x47A,0x47A}, + {0x47C,0x47C},{0x47E,0x47E},{0x480,0x480},{0x48A,0x48A},{0x48C,0x48C},{0x48E,0x48E}, + {0x490,0x490},{0x492,0x492},{0x494,0x494},{0x496,0x496},{0x498,0x498},{0x49A,0x49A}, + {0x49C,0x49C},{0x49E,0x49E},{0x4A0,0x4A0},{0x4A2,0x4A2},{0x4A4,0x4A4},{0x4A6,0x4A6}, + {0x4A8,0x4A8},{0x4AA,0x4AA},{0x4AC,0x4AC},{0x4AE,0x4AE},{0x4B0,0x4B0},{0x4B2,0x4B2}, + {0x4B4,0x4B4},{0x4B6,0x4B6},{0x4B8,0x4B8},{0x4BA,0x4BA},{0x4BC,0x4BC},{0x4BE,0x4BE}, + {0x4C0,0x4C1},{0x4C3,0x4C3},{0x4C5,0x4C5},{0x4C7,0x4C7},{0x4C9,0x4C9},{0x4CB,0x4CB}, + {0x4CD,0x4CD},{0x4D0,0x4D0},{0x4D2,0x4D2},{0x4D4,0x4D4},{0x4D6,0x4D6},{0x4D8,0x4D8}, + {0x4DA,0x4DA},{0x4DC,0x4DC},{0x4DE,0x4DE},{0x4E0,0x4E0},{0x4E2,0x4E2},{0x4E4,0x4E4}, + {0x4E6,0x4E6},{0x4E8,0x4E8},{0x4EA,0x4EA},{0x4EC,0x4EC},{0x4EE,0x4EE},{0x4F0,0x4F0}, + {0x4F2,0x4F2},{0x4F4,0x4F4},{0x4F6,0x4F6},{0x4F8,0x4F8},{0x4FA,0x4FA},{0x4FC,0x4FC}, + {0x4FE,0x4FE},{0x500,0x500},{0x502,0x502},{0x504,0x504},{0x506,0x506},{0x508,0x508}, + {0x50A,0x50A},{0x50C,0x50C},{0x50E,0x50E},{0x510,0x510},{0x512,0x512},{0x514,0x514}, + {0x516,0x516},{0x518,0x518},{0x51A,0x51A},{0x51C,0x51C},{0x51E,0x51E},{0x520,0x520}, + {0x522,0x522},{0x524,0x524},{0x526,0x526},{0x528,0x528},{0x52A,0x52A},{0x52C,0x52C}, + {0x52E,0x52E},{0x531,0x556},{0x10A0,0x10C5},{0x10C7,0x10C7},{0x10CD,0x10CD},{0x13A0,0x13F5}, + {0x1C90,0x1CBA},{0x1CBD,0x1CBF},{0x1E00,0x1E00},{0x1E02,0x1E02},{0x1E04,0x1E04},{0x1E06,0x1E06}, + {0x1E08,0x1E08},{0x1E0A,0x1E0A},{0x1E0C,0x1E0C},{0x1E0E,0x1E0E},{0x1E10,0x1E10},{0x1E12,0x1E12}, + {0x1E14,0x1E14},{0x1E16,0x1E16},{0x1E18,0x1E18},{0x1E1A,0x1E1A},{0x1E1C,0x1E1C},{0x1E1E,0x1E1E}, + {0x1E20,0x1E20},{0x1E22,0x1E22},{0x1E24,0x1E24},{0x1E26,0x1E26},{0x1E28,0x1E28},{0x1E2A,0x1E2A}, + {0x1E2C,0x1E2C},{0x1E2E,0x1E2E},{0x1E30,0x1E30},{0x1E32,0x1E32},{0x1E34,0x1E34},{0x1E36,0x1E36}, + {0x1E38,0x1E38},{0x1E3A,0x1E3A},{0x1E3C,0x1E3C},{0x1E3E,0x1E3E},{0x1E40,0x1E40},{0x1E42,0x1E42}, + {0x1E44,0x1E44},{0x1E46,0x1E46},{0x1E48,0x1E48},{0x1E4A,0x1E4A},{0x1E4C,0x1E4C},{0x1E4E,0x1E4E}, + {0x1E50,0x1E50},{0x1E52,0x1E52},{0x1E54,0x1E54},{0x1E56,0x1E56},{0x1E58,0x1E58},{0x1E5A,0x1E5A}, + {0x1E5C,0x1E5C},{0x1E5E,0x1E5E},{0x1E60,0x1E60},{0x1E62,0x1E62},{0x1E64,0x1E64},{0x1E66,0x1E66}, + {0x1E68,0x1E68},{0x1E6A,0x1E6A},{0x1E6C,0x1E6C},{0x1E6E,0x1E6E},{0x1E70,0x1E70},{0x1E72,0x1E72}, + {0x1E74,0x1E74},{0x1E76,0x1E76},{0x1E78,0x1E78},{0x1E7A,0x1E7A},{0x1E7C,0x1E7C},{0x1E7E,0x1E7E}, + {0x1E80,0x1E80},{0x1E82,0x1E82},{0x1E84,0x1E84},{0x1E86,0x1E86},{0x1E88,0x1E88},{0x1E8A,0x1E8A}, + {0x1E8C,0x1E8C},{0x1E8E,0x1E8E},{0x1E90,0x1E90},{0x1E92,0x1E92},{0x1E94,0x1E94},{0x1E9E,0x1E9E}, + {0x1EA0,0x1EA0},{0x1EA2,0x1EA2},{0x1EA4,0x1EA4},{0x1EA6,0x1EA6},{0x1EA8,0x1EA8},{0x1EAA,0x1EAA}, + {0x1EAC,0x1EAC},{0x1EAE,0x1EAE},{0x1EB0,0x1EB0},{0x1EB2,0x1EB2},{0x1EB4,0x1EB4},{0x1EB6,0x1EB6}, + {0x1EB8,0x1EB8},{0x1EBA,0x1EBA},{0x1EBC,0x1EBC},{0x1EBE,0x1EBE},{0x1EC0,0x1EC0},{0x1EC2,0x1EC2}, + {0x1EC4,0x1EC4},{0x1EC6,0x1EC6},{0x1EC8,0x1EC8},{0x1ECA,0x1ECA},{0x1ECC,0x1ECC},{0x1ECE,0x1ECE}, + {0x1ED0,0x1ED0},{0x1ED2,0x1ED2},{0x1ED4,0x1ED4},{0x1ED6,0x1ED6},{0x1ED8,0x1ED8},{0x1EDA,0x1EDA}, + {0x1EDC,0x1EDC},{0x1EDE,0x1EDE},{0x1EE0,0x1EE0},{0x1EE2,0x1EE2},{0x1EE4,0x1EE4},{0x1EE6,0x1EE6}, + {0x1EE8,0x1EE8},{0x1EEA,0x1EEA},{0x1EEC,0x1EEC},{0x1EEE,0x1EEE},{0x1EF0,0x1EF0},{0x1EF2,0x1EF2}, + {0x1EF4,0x1EF4},{0x1EF6,0x1EF6},{0x1EF8,0x1EF8},{0x1EFA,0x1EFA},{0x1EFC,0x1EFC},{0x1EFE,0x1EFE}, + {0x1F08,0x1F0F},{0x1F18,0x1F1D},{0x1F28,0x1F2F},{0x1F38,0x1F3F},{0x1F48,0x1F4D},{0x1F59,0x1F59}, + {0x1F5B,0x1F5B},{0x1F5D,0x1F5D},{0x1F5F,0x1F5F},{0x1F68,0x1F6F},{0x1F88,0x1F8F},{0x1F98,0x1F9F}, + {0x1FA8,0x1FAF},{0x1FB8,0x1FBC},{0x1FC8,0x1FCC},{0x1FD8,0x1FDB},{0x1FE8,0x1FEC},{0x1FF8,0x1FFC}, + {0x2102,0x2102},{0x2107,0x2107},{0x210B,0x210D},{0x2110,0x2112},{0x2115,0x2115},{0x2119,0x211D}, + {0x2124,0x2124},{0x2126,0x2126},{0x2128,0x2128},{0x212A,0x212D},{0x2130,0x2133},{0x213E,0x213F}, + {0x2145,0x2145},{0x2183,0x2183},{0x2C00,0x2C2E},{0x2C60,0x2C60},{0x2C62,0x2C64},{0x2C67,0x2C67}, + {0x2C69,0x2C69},{0x2C6B,0x2C6B},{0x2C6D,0x2C70},{0x2C72,0x2C72},{0x2C75,0x2C75},{0x2C7E,0x2C80}, + {0x2C82,0x2C82},{0x2C84,0x2C84},{0x2C86,0x2C86},{0x2C88,0x2C88},{0x2C8A,0x2C8A},{0x2C8C,0x2C8C}, + {0x2C8E,0x2C8E},{0x2C90,0x2C90},{0x2C92,0x2C92},{0x2C94,0x2C94},{0x2C96,0x2C96},{0x2C98,0x2C98}, + {0x2C9A,0x2C9A},{0x2C9C,0x2C9C},{0x2C9E,0x2C9E},{0x2CA0,0x2CA0},{0x2CA2,0x2CA2},{0x2CA4,0x2CA4}, + {0x2CA6,0x2CA6},{0x2CA8,0x2CA8},{0x2CAA,0x2CAA},{0x2CAC,0x2CAC},{0x2CAE,0x2CAE},{0x2CB0,0x2CB0}, + {0x2CB2,0x2CB2},{0x2CB4,0x2CB4},{0x2CB6,0x2CB6},{0x2CB8,0x2CB8},{0x2CBA,0x2CBA},{0x2CBC,0x2CBC}, + {0x2CBE,0x2CBE},{0x2CC0,0x2CC0},{0x2CC2,0x2CC2},{0x2CC4,0x2CC4},{0x2CC6,0x2CC6},{0x2CC8,0x2CC8}, + {0x2CCA,0x2CCA},{0x2CCC,0x2CCC},{0x2CCE,0x2CCE},{0x2CD0,0x2CD0},{0x2CD2,0x2CD2},{0x2CD4,0x2CD4}, + {0x2CD6,0x2CD6},{0x2CD8,0x2CD8},{0x2CDA,0x2CDA},{0x2CDC,0x2CDC},{0x2CDE,0x2CDE},{0x2CE0,0x2CE0}, + {0x2CE2,0x2CE2},{0x2CEB,0x2CEB},{0x2CED,0x2CED},{0x2CF2,0x2CF2},{0xA640,0xA640},{0xA642,0xA642}, + {0xA644,0xA644},{0xA646,0xA646},{0xA648,0xA648},{0xA64A,0xA64A},{0xA64C,0xA64C},{0xA64E,0xA64E}, + {0xA650,0xA650},{0xA652,0xA652},{0xA654,0xA654},{0xA656,0xA656},{0xA658,0xA658},{0xA65A,0xA65A}, + {0xA65C,0xA65C},{0xA65E,0xA65E},{0xA660,0xA660},{0xA662,0xA662},{0xA664,0xA664},{0xA666,0xA666}, + {0xA668,0xA668},{0xA66A,0xA66A},{0xA66C,0xA66C},{0xA680,0xA680},{0xA682,0xA682},{0xA684,0xA684}, + {0xA686,0xA686},{0xA688,0xA688},{0xA68A,0xA68A},{0xA68C,0xA68C},{0xA68E,0xA68E},{0xA690,0xA690}, + {0xA692,0xA692},{0xA694,0xA694},{0xA696,0xA696},{0xA698,0xA698},{0xA69A,0xA69A},{0xA722,0xA722}, + {0xA724,0xA724},{0xA726,0xA726},{0xA728,0xA728},{0xA72A,0xA72A},{0xA72C,0xA72C},{0xA72E,0xA72E}, + {0xA732,0xA732},{0xA734,0xA734},{0xA736,0xA736},{0xA738,0xA738},{0xA73A,0xA73A},{0xA73C,0xA73C}, + {0xA73E,0xA73E},{0xA740,0xA740},{0xA742,0xA742},{0xA744,0xA744},{0xA746,0xA746},{0xA748,0xA748}, + {0xA74A,0xA74A},{0xA74C,0xA74C},{0xA74E,0xA74E},{0xA750,0xA750},{0xA752,0xA752},{0xA754,0xA754}, + {0xA756,0xA756},{0xA758,0xA758},{0xA75A,0xA75A},{0xA75C,0xA75C},{0xA75E,0xA75E},{0xA760,0xA760}, + {0xA762,0xA762},{0xA764,0xA764},{0xA766,0xA766},{0xA768,0xA768},{0xA76A,0xA76A},{0xA76C,0xA76C}, + {0xA76E,0xA76E},{0xA779,0xA779},{0xA77B,0xA77B},{0xA77D,0xA77E},{0xA780,0xA780},{0xA782,0xA782}, + {0xA784,0xA784},{0xA786,0xA786},{0xA78B,0xA78B},{0xA78D,0xA78D},{0xA790,0xA790},{0xA792,0xA792}, + {0xA796,0xA796},{0xA798,0xA798},{0xA79A,0xA79A},{0xA79C,0xA79C},{0xA79E,0xA79E},{0xA7A0,0xA7A0}, + {0xA7A2,0xA7A2},{0xA7A4,0xA7A4},{0xA7A6,0xA7A6},{0xA7A8,0xA7A8},{0xA7AA,0xA7AE},{0xA7B0,0xA7B4}, + {0xA7B6,0xA7B6},{0xA7B8,0xA7B8},{0xA7BA,0xA7BA},{0xA7BC,0xA7BC},{0xA7BE,0xA7BE},{0xA7C2,0xA7C2}, + {0xA7C4,0xA7C7},{0xA7C9,0xA7C9},{0xA7F5,0xA7F5},{0xFF21,0xFF3A},{0x10400,0x10427},{0x104B0,0x104D3}, + {0x10C80,0x10CB2},{0x118A0,0x118BF},{0x16E40,0x16E5F},{0x1D400,0x1D419},{0x1D434,0x1D44D},{0x1D468,0x1D481}, + {0x1D49C,0x1D49C},{0x1D49E,0x1D49F},{0x1D4A2,0x1D4A2},{0x1D4A5,0x1D4A6},{0x1D4A9,0x1D4AC},{0x1D4AE,0x1D4B5}, + {0x1D4D0,0x1D4E9},{0x1D504,0x1D505},{0x1D507,0x1D50A},{0x1D50D,0x1D514},{0x1D516,0x1D51C},{0x1D538,0x1D539}, + {0x1D53B,0x1D53E},{0x1D540,0x1D544},{0x1D546,0x1D546},{0x1D54A,0x1D550},{0x1D56C,0x1D585},{0x1D5A0,0x1D5B9}, + {0x1D5D4,0x1D5ED},{0x1D608,0x1D621},{0x1D63C,0x1D655},{0x1D670,0x1D689},{0x1D6A8,0x1D6C0},{0x1D6E2,0x1D6FA}, + {0x1D71C,0x1D734},{0x1D756,0x1D76E},{0x1D790,0x1D7A8},{0x1D7CA,0x1D7CA},{0x1E900,0x1E921}, +}; +static const int uni_U_n = 641; + +static const uint32_t uni_X[][2] = { + {0xAA,0xAA},{0xBA,0xBA},{0x1BB,0x1BB},{0x1C0,0x1C3},{0x294,0x294},{0x2B0,0x2C1}, + {0x2C6,0x2D1},{0x2E0,0x2E4},{0x2EC,0x2EC},{0x2EE,0x2EE},{0x300,0x36F},{0x374,0x374}, + {0x37A,0x37A},{0x483,0x489},{0x559,0x559},{0x591,0x5BD},{0x5BF,0x5BF},{0x5C1,0x5C2}, + {0x5C4,0x5C5},{0x5C7,0x5C7},{0x5D0,0x5EA},{0x5EF,0x5F2},{0x610,0x61A},{0x620,0x65F}, + {0x66E,0x6D3},{0x6D5,0x6DC},{0x6DF,0x6E8},{0x6EA,0x6EF},{0x6FA,0x6FC},{0x6FF,0x6FF}, + {0x710,0x74A},{0x74D,0x7B1},{0x7CA,0x7F5},{0x7FA,0x7FA},{0x7FD,0x7FD},{0x800,0x82D}, + {0x840,0x85B},{0x860,0x86A},{0x8A0,0x8B4},{0x8B6,0x8C7},{0x8D3,0x8E1},{0x8E3,0x963}, + {0x971,0x983},{0x985,0x98C},{0x98F,0x990},{0x993,0x9A8},{0x9AA,0x9B0},{0x9B2,0x9B2}, + {0x9B6,0x9B9},{0x9BC,0x9C4},{0x9C7,0x9C8},{0x9CB,0x9CE},{0x9D7,0x9D7},{0x9DC,0x9DD}, + {0x9DF,0x9E3},{0x9F0,0x9F1},{0x9FC,0x9FC},{0x9FE,0x9FE},{0xA01,0xA03},{0xA05,0xA0A}, + {0xA0F,0xA10},{0xA13,0xA28},{0xA2A,0xA30},{0xA32,0xA33},{0xA35,0xA36},{0xA38,0xA39}, + {0xA3C,0xA3C},{0xA3E,0xA42},{0xA47,0xA48},{0xA4B,0xA4D},{0xA51,0xA51},{0xA59,0xA5C}, + {0xA5E,0xA5E},{0xA70,0xA75},{0xA81,0xA83},{0xA85,0xA8D},{0xA8F,0xA91},{0xA93,0xAA8}, + {0xAAA,0xAB0},{0xAB2,0xAB3},{0xAB5,0xAB9},{0xABC,0xAC5},{0xAC7,0xAC9},{0xACB,0xACD}, + {0xAD0,0xAD0},{0xAE0,0xAE3},{0xAF9,0xAFF},{0xB01,0xB03},{0xB05,0xB0C},{0xB0F,0xB10}, + {0xB13,0xB28},{0xB2A,0xB30},{0xB32,0xB33},{0xB35,0xB39},{0xB3C,0xB44},{0xB47,0xB48}, + {0xB4B,0xB4D},{0xB55,0xB57},{0xB5C,0xB5D},{0xB5F,0xB63},{0xB71,0xB71},{0xB82,0xB83}, + {0xB85,0xB8A},{0xB8E,0xB90},{0xB92,0xB95},{0xB99,0xB9A},{0xB9C,0xB9C},{0xB9E,0xB9F}, + {0xBA3,0xBA4},{0xBA8,0xBAA},{0xBAE,0xBB9},{0xBBE,0xBC2},{0xBC6,0xBC8},{0xBCA,0xBCD}, + {0xBD0,0xBD0},{0xBD7,0xBD7},{0xC00,0xC0C},{0xC0E,0xC10},{0xC12,0xC28},{0xC2A,0xC39}, + {0xC3D,0xC44},{0xC46,0xC48},{0xC4A,0xC4D},{0xC55,0xC56},{0xC58,0xC5A},{0xC60,0xC63}, + {0xC80,0xC83},{0xC85,0xC8C},{0xC8E,0xC90},{0xC92,0xCA8},{0xCAA,0xCB3},{0xCB5,0xCB9}, + {0xCBC,0xCC4},{0xCC6,0xCC8},{0xCCA,0xCCD},{0xCD5,0xCD6},{0xCDE,0xCDE},{0xCE0,0xCE3}, + {0xCF1,0xCF2},{0xD00,0xD0C},{0xD0E,0xD10},{0xD12,0xD44},{0xD46,0xD48},{0xD4A,0xD4E}, + {0xD54,0xD57},{0xD5F,0xD63},{0xD7A,0xD7F},{0xD81,0xD83},{0xD85,0xD96},{0xD9A,0xDB1}, + {0xDB3,0xDBB},{0xDBD,0xDBD},{0xDC0,0xDC6},{0xDCA,0xDCA},{0xDCF,0xDD4},{0xDD6,0xDD6}, + {0xDD8,0xDDF},{0xDF2,0xDF3},{0xE01,0xE3A},{0xE40,0xE4E},{0xE81,0xE82},{0xE84,0xE84}, + {0xE86,0xE8A},{0xE8C,0xEA3},{0xEA5,0xEA5},{0xEA7,0xEBD},{0xEC0,0xEC4},{0xEC6,0xEC6}, + {0xEC8,0xECD},{0xEDC,0xEDF},{0xF00,0xF00},{0xF18,0xF19},{0xF35,0xF35},{0xF37,0xF37}, + {0xF39,0xF39},{0xF3E,0xF47},{0xF49,0xF6C},{0xF71,0xF84},{0xF86,0xF97},{0xF99,0xFBC}, + {0xFC6,0xFC6},{0x1000,0x103F},{0x1050,0x108F},{0x109A,0x109D},{0x10FC,0x10FC},{0x1100,0x1248}, + {0x124A,0x124D},{0x1250,0x1256},{0x1258,0x1258},{0x125A,0x125D},{0x1260,0x1288},{0x128A,0x128D}, + {0x1290,0x12B0},{0x12B2,0x12B5},{0x12B8,0x12BE},{0x12C0,0x12C0},{0x12C2,0x12C5},{0x12C8,0x12D6}, + {0x12D8,0x1310},{0x1312,0x1315},{0x1318,0x135A},{0x135D,0x135F},{0x1380,0x138F},{0x1401,0x166C}, + {0x166F,0x167F},{0x1681,0x169A},{0x16A0,0x16EA},{0x16F1,0x16F8},{0x1700,0x170C},{0x170E,0x1714}, + {0x1720,0x1734},{0x1740,0x1753},{0x1760,0x176C},{0x176E,0x1770},{0x1772,0x1773},{0x1780,0x17D3}, + {0x17D7,0x17D7},{0x17DC,0x17DD},{0x180B,0x180D},{0x1820,0x1878},{0x1880,0x18AA},{0x18B0,0x18F5}, + {0x1900,0x191E},{0x1920,0x192B},{0x1930,0x193B},{0x1950,0x196D},{0x1970,0x1974},{0x1980,0x19AB}, + {0x19B0,0x19C9},{0x1A00,0x1A1B},{0x1A20,0x1A5E},{0x1A60,0x1A7C},{0x1A7F,0x1A7F},{0x1AA7,0x1AA7}, + {0x1AB0,0x1AC0},{0x1B00,0x1B4B},{0x1B6B,0x1B73},{0x1B80,0x1BAF},{0x1BBA,0x1BF3},{0x1C00,0x1C37}, + {0x1C4D,0x1C4F},{0x1C5A,0x1C7D},{0x1CD0,0x1CD2},{0x1CD4,0x1CFA},{0x1D2C,0x1D6A},{0x1D78,0x1D78}, + {0x1D9B,0x1DF9},{0x1DFB,0x1DFF},{0x2071,0x2071},{0x207F,0x207F},{0x2090,0x209C},{0x20D0,0x20F0}, + {0x2135,0x2138},{0x2C7C,0x2C7D},{0x2CEF,0x2CF1},{0x2D30,0x2D67},{0x2D6F,0x2D6F},{0x2D7F,0x2D96}, + {0x2DA0,0x2DA6},{0x2DA8,0x2DAE},{0x2DB0,0x2DB6},{0x2DB8,0x2DBE},{0x2DC0,0x2DC6},{0x2DC8,0x2DCE}, + {0x2DD0,0x2DD6},{0x2DD8,0x2DDE},{0x2DE0,0x2DFF},{0x2E2F,0x2E2F},{0x3005,0x3006},{0x302A,0x302F}, + {0x3031,0x3035},{0x303B,0x303C},{0x3041,0x3096},{0x3099,0x309A},{0x309D,0x309F},{0x30A1,0x30FA}, + {0x30FC,0x30FF},{0x3105,0x312F},{0x3131,0x318E},{0x31A0,0x31BF},{0x31F0,0x31FF},{0x3400,0x4DBF}, + {0x4E00,0x9FFC},{0xA000,0xA48C},{0xA4D0,0xA4FD},{0xA500,0xA60C},{0xA610,0xA61F},{0xA62A,0xA62B}, + {0xA66E,0xA672},{0xA674,0xA67D},{0xA67F,0xA67F},{0xA69C,0xA6E5},{0xA6F0,0xA6F1},{0xA717,0xA71F}, + {0xA770,0xA770},{0xA788,0xA788},{0xA78F,0xA78F},{0xA7F7,0xA7F9},{0xA7FB,0xA827},{0xA82C,0xA82C}, + {0xA840,0xA873},{0xA880,0xA8C5},{0xA8E0,0xA8F7},{0xA8FB,0xA8FB},{0xA8FD,0xA8FF},{0xA90A,0xA92D}, + {0xA930,0xA953},{0xA960,0xA97C},{0xA980,0xA9C0},{0xA9CF,0xA9CF},{0xA9E0,0xA9EF},{0xA9FA,0xA9FE}, + {0xAA00,0xAA36},{0xAA40,0xAA4D},{0xAA60,0xAA76},{0xAA7A,0xAAC2},{0xAADB,0xAADD},{0xAAE0,0xAAEF}, + {0xAAF2,0xAAF6},{0xAB01,0xAB06},{0xAB09,0xAB0E},{0xAB11,0xAB16},{0xAB20,0xAB26},{0xAB28,0xAB2E}, + {0xAB5C,0xAB5F},{0xAB69,0xAB69},{0xABC0,0xABEA},{0xABEC,0xABED},{0xAC00,0xD7A3},{0xD7B0,0xD7C6}, + {0xD7CB,0xD7FB},{0xF900,0xFA6D},{0xFA70,0xFAD9},{0xFB1D,0xFB28},{0xFB2A,0xFB36},{0xFB38,0xFB3C}, + {0xFB3E,0xFB3E},{0xFB40,0xFB41},{0xFB43,0xFB44},{0xFB46,0xFBB1},{0xFBD3,0xFD3D},{0xFD50,0xFD8F}, + {0xFD92,0xFDC7},{0xFDF0,0xFDFB},{0xFE00,0xFE0F},{0xFE20,0xFE2F},{0xFE70,0xFE74},{0xFE76,0xFEFC}, + {0xFF66,0xFFBE},{0xFFC2,0xFFC7},{0xFFCA,0xFFCF},{0xFFD2,0xFFD7},{0xFFDA,0xFFDC},{0x10000,0x1000B}, + {0x1000D,0x10026},{0x10028,0x1003A},{0x1003C,0x1003D},{0x1003F,0x1004D},{0x10050,0x1005D},{0x10080,0x100FA}, + {0x101FD,0x101FD},{0x10280,0x1029C},{0x102A0,0x102D0},{0x102E0,0x102E0},{0x10300,0x1031F},{0x1032D,0x10340}, + {0x10342,0x10349},{0x10350,0x1037A},{0x10380,0x1039D},{0x103A0,0x103C3},{0x103C8,0x103CF},{0x10450,0x1049D}, + {0x10500,0x10527},{0x10530,0x10563},{0x10600,0x10736},{0x10740,0x10755},{0x10760,0x10767},{0x10800,0x10805}, + {0x10808,0x10808},{0x1080A,0x10835},{0x10837,0x10838},{0x1083C,0x1083C},{0x1083F,0x10855},{0x10860,0x10876}, + {0x10880,0x1089E},{0x108E0,0x108F2},{0x108F4,0x108F5},{0x10900,0x10915},{0x10920,0x10939},{0x10980,0x109B7}, + {0x109BE,0x109BF},{0x10A00,0x10A03},{0x10A05,0x10A06},{0x10A0C,0x10A13},{0x10A15,0x10A17},{0x10A19,0x10A35}, + {0x10A38,0x10A3A},{0x10A3F,0x10A3F},{0x10A60,0x10A7C},{0x10A80,0x10A9C},{0x10AC0,0x10AC7},{0x10AC9,0x10AE6}, + {0x10B00,0x10B35},{0x10B40,0x10B55},{0x10B60,0x10B72},{0x10B80,0x10B91},{0x10C00,0x10C48},{0x10D00,0x10D27}, + {0x10E80,0x10EA9},{0x10EAB,0x10EAC},{0x10EB0,0x10EB1},{0x10F00,0x10F1C},{0x10F27,0x10F27},{0x10F30,0x10F50}, + {0x10FB0,0x10FC4},{0x10FE0,0x10FF6},{0x11000,0x11046},{0x1107F,0x110BA},{0x110D0,0x110E8},{0x11100,0x11134}, + {0x11144,0x11147},{0x11150,0x11173},{0x11176,0x11176},{0x11180,0x111C4},{0x111C9,0x111CC},{0x111CE,0x111CF}, + {0x111DA,0x111DA},{0x111DC,0x111DC},{0x11200,0x11211},{0x11213,0x11237},{0x1123E,0x1123E},{0x11280,0x11286}, + {0x11288,0x11288},{0x1128A,0x1128D},{0x1128F,0x1129D},{0x1129F,0x112A8},{0x112B0,0x112EA},{0x11300,0x11303}, + {0x11305,0x1130C},{0x1130F,0x11310},{0x11313,0x11328},{0x1132A,0x11330},{0x11332,0x11333},{0x11335,0x11339}, + {0x1133B,0x11344},{0x11347,0x11348},{0x1134B,0x1134D},{0x11350,0x11350},{0x11357,0x11357},{0x1135D,0x11363}, + {0x11366,0x1136C},{0x11370,0x11374},{0x11400,0x1144A},{0x1145E,0x11461},{0x11480,0x114C5},{0x114C7,0x114C7}, + {0x11580,0x115B5},{0x115B8,0x115C0},{0x115D8,0x115DD},{0x11600,0x11640},{0x11644,0x11644},{0x11680,0x116B8}, + {0x11700,0x1171A},{0x1171D,0x1172B},{0x11800,0x1183A},{0x118FF,0x11906},{0x11909,0x11909},{0x1190C,0x11913}, + {0x11915,0x11916},{0x11918,0x11935},{0x11937,0x11938},{0x1193B,0x11943},{0x119A0,0x119A7},{0x119AA,0x119D7}, + {0x119DA,0x119E1},{0x119E3,0x119E4},{0x11A00,0x11A3E},{0x11A47,0x11A47},{0x11A50,0x11A99},{0x11A9D,0x11A9D}, + {0x11AC0,0x11AF8},{0x11C00,0x11C08},{0x11C0A,0x11C36},{0x11C38,0x11C40},{0x11C72,0x11C8F},{0x11C92,0x11CA7}, + {0x11CA9,0x11CB6},{0x11D00,0x11D06},{0x11D08,0x11D09},{0x11D0B,0x11D36},{0x11D3A,0x11D3A},{0x11D3C,0x11D3D}, + {0x11D3F,0x11D47},{0x11D60,0x11D65},{0x11D67,0x11D68},{0x11D6A,0x11D8E},{0x11D90,0x11D91},{0x11D93,0x11D98}, + {0x11EE0,0x11EF6},{0x11FB0,0x11FB0},{0x12000,0x12399},{0x12480,0x12543},{0x13000,0x1342E},{0x14400,0x14646}, + {0x16800,0x16A38},{0x16A40,0x16A5E},{0x16AD0,0x16AED},{0x16AF0,0x16AF4},{0x16B00,0x16B36},{0x16B40,0x16B43}, + {0x16B63,0x16B77},{0x16B7D,0x16B8F},{0x16F00,0x16F4A},{0x16F4F,0x16F87},{0x16F8F,0x16F9F},{0x16FE0,0x16FE1}, + {0x16FE3,0x16FE4},{0x16FF0,0x16FF1},{0x17000,0x187F7},{0x18800,0x18CD5},{0x18D00,0x18D08},{0x1B000,0x1B11E}, + {0x1B150,0x1B152},{0x1B164,0x1B167},{0x1B170,0x1B2FB},{0x1BC00,0x1BC6A},{0x1BC70,0x1BC7C},{0x1BC80,0x1BC88}, + {0x1BC90,0x1BC99},{0x1BC9D,0x1BC9E},{0x1D165,0x1D169},{0x1D16D,0x1D172},{0x1D17B,0x1D182},{0x1D185,0x1D18B}, + {0x1D1AA,0x1D1AD},{0x1D242,0x1D244},{0x1DA00,0x1DA36},{0x1DA3B,0x1DA6C},{0x1DA75,0x1DA75},{0x1DA84,0x1DA84}, + {0x1DA9B,0x1DA9F},{0x1DAA1,0x1DAAF},{0x1E000,0x1E006},{0x1E008,0x1E018},{0x1E01B,0x1E021},{0x1E023,0x1E024}, + {0x1E026,0x1E02A},{0x1E100,0x1E12C},{0x1E130,0x1E13D},{0x1E14E,0x1E14E},{0x1E2C0,0x1E2EF},{0x1E800,0x1E8C4}, + {0x1E8D0,0x1E8D6},{0x1E944,0x1E94B},{0x1EE00,0x1EE03},{0x1EE05,0x1EE1F},{0x1EE21,0x1EE22},{0x1EE24,0x1EE24}, + {0x1EE27,0x1EE27},{0x1EE29,0x1EE32},{0x1EE34,0x1EE37},{0x1EE39,0x1EE39},{0x1EE3B,0x1EE3B},{0x1EE42,0x1EE42}, + {0x1EE47,0x1EE47},{0x1EE49,0x1EE49},{0x1EE4B,0x1EE4B},{0x1EE4D,0x1EE4F},{0x1EE51,0x1EE52},{0x1EE54,0x1EE54}, + {0x1EE57,0x1EE57},{0x1EE59,0x1EE59},{0x1EE5B,0x1EE5B},{0x1EE5D,0x1EE5D},{0x1EE5F,0x1EE5F},{0x1EE61,0x1EE62}, + {0x1EE64,0x1EE64},{0x1EE67,0x1EE6A},{0x1EE6C,0x1EE72},{0x1EE74,0x1EE77},{0x1EE79,0x1EE7C},{0x1EE7E,0x1EE7E}, + {0x1EE80,0x1EE89},{0x1EE8B,0x1EE9B},{0x1EEA1,0x1EEA3},{0x1EEA5,0x1EEA9},{0x1EEAB,0x1EEBB},{0x20000,0x2A6DD}, + {0x2A700,0x2B734},{0x2B740,0x2B81D},{0x2B820,0x2CEA1},{0x2CEB0,0x2EBE0},{0x2F800,0x2FA1D},{0x30000,0x3134A}, + {0xE0100,0xE01EF}, +}; +static const int uni_X_n = 595; + +static inline int is_U(uint32_t c){ return uni_in(uni_U,uni_U_n,c); } +static inline int is_X(uint32_t c){ return uni_in(uni_X,uni_X_n,c); } +#endif From 9e2d41567c5e0902da1ef5a3daaff9a957d83997 Mon Sep 17 00:00:00 2001 From: Nicholas Beerbower Date: Thu, 16 Jul 2026 15:17:21 -0400 Subject: [PATCH 16/95] tests: o200k tokenizer coverage, no model download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/tok_o200k_tiny.json is a synthetic byte-level BPE (274 vocab, a few KB) whose Split regex is the o200k pattern; expected ids in tok_o200k_cases.txt were generated by HF tokenizers on that same file. test_tok_o200k (in TEST_BINS) scores 40/40 encode + 40/40 round-trip: case-transition splits, contractions, digit groups, the [\r\n/]* tail, whitespace branches, CJK/Greek/Cyrillic, added-token atomicity. The cl100k path is untouched by construction — dispatch requires \p{Lu} in the tokenizer's own Split pattern, which cl100k lacks — and stays covered by the GLM oracle (verified on this branch: 32/32). Co-Authored-By: Claude Fable 5 --- c/Makefile | 5 ++- c/tests/test_tok_o200k.c | 61 +++++++++++++++++++++++++++++++++++++ c/tests/tok_o200k_cases.txt | 40 ++++++++++++++++++++++++ c/tests/tok_o200k_tiny.json | 1 + 4 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 c/tests/test_tok_o200k.c create mode 100644 c/tests/tok_o200k_cases.txt create mode 100644 c/tests/tok_o200k_tiny.json diff --git a/c/Makefile b/c/Makefile index 27902cb..9887c74 100644 --- a/c/Makefile +++ b/c/Makefile @@ -166,7 +166,7 @@ else PYTHON ?= python3 endif CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) ifneq (,$(LINUX)) TEST_BINS += tests/test_uring$(EXE) endif @@ -293,6 +293,9 @@ iobench$(EXE): iobench.c compat.h tests/test_json$(EXE): tests/test_json.c json.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_tok_o200k$(EXE): tests/test_tok_o200k.c tok.h tok_unicode.h tok_unicode_o200k.h json.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + tests/test_st$(EXE): tests/test_st.c st.h json.h compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) diff --git a/c/tests/test_tok_o200k.c b/c/tests/test_tok_o200k.c new file mode 100644 index 0000000..b86bda9 --- /dev/null +++ b/c/tests/test_tok_o200k.c @@ -0,0 +1,61 @@ +/* o200k pre-tokenizer validation against HF-tokenizers-generated expectations. + * Self-contained for the test-c harness: loads tests/tok_o200k_tiny.json (a + * synthetic byte-level BPE whose Split regex is the o200k pattern — a few KB, + * no model download) and scores tests/tok_o200k_cases.txt, whose expected ids + * were produced by HF `tokenizers` on the same file. Guards the case-aware + * letter matcher, contractions, digit groups, the [\r\n/]* punctuation tail, + * whitespace branches, and added-token atomicity; round-trips every case. + * The cl100k path is untouched by construction (dispatch requires \p{Lu} in + * the tokenizer's own Split pattern) and stays covered by the GLM oracle. */ +#define _GNU_SOURCE +#include "../tok.h" + +int main(void) { + Tok T; + tok_load(&T, "tests/tok_o200k_tiny.json"); + if (!T.o200k) { fprintf(stderr, "test_tok_o200k: o200k pattern not detected\n"); return 1; } + FILE *f = fopen("tests/tok_o200k_cases.txt", "rb"); + if (!f) { perror("tests/tok_o200k_cases.txt"); return 1; } + char *line = NULL; size_t cap = 0; ssize_t nr; + int pass = 0, tot = 0, dpass = 0; + while ((nr = getline(&line, &cap, f)) >= 0) { + if (nr > 0 && line[nr-1] == '\n') line[--nr] = 0; + if (nr == 0) continue; + char *tab = strchr(line, '\t'); if (!tab) continue; + *tab = 0; + const char *text = line, *idstr = tab + 1; + char tbuf[4096]; int tn = 0; + for (const char *q = text; *q && tn < 4095; q++) { + if (q[0]=='\\' && q[1]=='n') { tbuf[tn++]='\n'; q++; } + else if (q[0]=='\\' && q[1]=='t') { tbuf[tn++]='\t'; q++; } + else if (q[0]=='\\' && q[1]=='r') { tbuf[tn++]='\r'; q++; } + else if (q[0]=='\\' && q[1]=='\\') { tbuf[tn++]='\\'; q++; } + else tbuf[tn++] = *q; + } + tbuf[tn] = 0; + int exp[512], ne = 0; + for (const char *q = idstr; *q; ) { + while (*q == ',' || *q == ' ') q++; + if (!*q) break; + exp[ne++] = atoi(q); + while (*q && *q != ',') q++; + } + int got[512]; int ng = tok_encode(&T, tbuf, tn, got, 512); + int ok = (ng == ne); + for (int i = 0; i < ng && ok; i++) ok = (got[i] == exp[i]); + tot++; if (ok) pass++; + char dec[8192]; int dn = tok_decode(&T, got, ng, dec, 8191); + int drt = (dn == tn) && !memcmp(dec, tbuf, tn); + if (drt) dpass++; + if (!ok || !drt) { + fprintf(stderr, "MISMATCH text=%s\n exp(%d):", text, ne); + for (int i = 0; i < ne; i++) fprintf(stderr, " %d", exp[i]); + fprintf(stderr, "\n got(%d):", ng); + for (int i = 0; i < ng; i++) fprintf(stderr, " %d", got[i]); + fprintf(stderr, "\n decode_ok=%d\n", drt); + } + } + fclose(f); + printf("test_tok_o200k: ENCODE %d/%d DECODE %d/%d\n", pass, tot, dpass, tot); + return (pass == tot && dpass == tot) ? 0 : 2; +} diff --git a/c/tests/tok_o200k_cases.txt b/c/tests/tok_o200k_cases.txt new file mode 100644 index 0000000..38e37fa --- /dev/null +++ b/c/tests/tok_o200k_cases.txt @@ -0,0 +1,40 @@ +hello world 259,32,119,111,114,263 +HelloWorld 72,101,257,111,262 +XMLHttpRequest 88,77,76,72,116,116,112,82,101,113,117,101,115,116 +helloWORLDhello 259,87,79,82,76,68,259 +dog's 100,111,103,270 +DON'T 68,79,78,39,84 +don't 100,111,110,39,116 +I'll've 73,39,257,39,118,101 +O'Brien 79,39,66,114,105,101,110 +the theatre 265,32,116,256,97,116,114,101 +12345 268,52,53 +a1b22c333d4444 97,49,98,50,50,99,51,51,51,100,52,52,52,52 +3.14 51,46,49,52 +http://x.com/a/b 104,116,116,112,58,47,47,120,46,99,111,109,47,97,47,98 +path/to/file 112,97,264,47,116,111,47,102,105,108,101 +a//b///c 97,47,47,98,47,47,47,99 +!!\r\n//x 33,33,13,10,47,47,120 +one\ntwo\r\nthree 111,110,101,10,116,119,111,13,10,264,114,101,101 + \n x 32,32,10,32,32,120 + 32,32,32 +a b c 97,32,32,98,32,32,32,99 +tab\there 116,269,9,256,114,101 +Café 67,97,102,101,204,129 +naiveBayes 110,97,105,118,101,66,97,121,101,115 +Éclair 195,137,99,108,97,105,114 +北京大学 229,140,151,228,186,172,229,164,167,229,173,166 +ΑΒαβ 206,145,206,146,206,177,206,178 +Иван 208,152,208,178,208,176,208,189 +ẞßscharf 225,186,158,195,159,115,99,104,97,114,102 +i̇stanbul 105,204,135,115,116,97,110,98,117,108 +hello<|endoftext|>world 259,274,119,111,114,263 +<|message_user|>hi 275,104,105 +mixedCASEand123 109,105,120,101,100,67,65,83,69,97,110,100,268 +'s 270 + 's 32,39,115 +A 65 +aB 97,66 +Ab 65,98 +AB 65,66 +ab 269 diff --git a/c/tests/tok_o200k_tiny.json b/c/tests/tok_o200k_tiny.json new file mode 100644 index 0000000..829301e --- /dev/null +++ b/c/tests/tok_o200k_tiny.json @@ -0,0 +1 @@ +{"version": "1.0", "truncation": null, "padding": null, "added_tokens": [{"id": 274, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 275, "content": "<|message_user|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}], "normalizer": null, "pre_tokenizer": {"type": "Sequence", "pretokenizers": [{"type": "Split", "pattern": {"Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, "behavior": "Isolated", "invert": false}, {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false}]}, "post_processor": null, "decoder": {"type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true}, "model": {"type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": true, "vocab": {"Ā": 0, "ā": 1, "Ă": 2, "ă": 3, "Ą": 4, "ą": 5, "Ć": 6, "ć": 7, "Ĉ": 8, "ĉ": 9, "Ċ": 10, "ċ": 11, "Č": 12, "č": 13, "Ď": 14, "ď": 15, "Đ": 16, "đ": 17, "Ē": 18, "ē": 19, "Ĕ": 20, "ĕ": 21, "Ė": 22, "ė": 23, "Ę": 24, "ę": 25, "Ě": 26, "ě": 27, "Ĝ": 28, "ĝ": 29, "Ğ": 30, "ğ": 31, "Ġ": 32, "!": 33, "\"": 34, "#": 35, "$": 36, "%": 37, "&": 38, "'": 39, "(": 40, ")": 41, "*": 42, "+": 43, ",": 44, "-": 45, ".": 46, "/": 47, "0": 48, "1": 49, "2": 50, "3": 51, "4": 52, "5": 53, "6": 54, "7": 55, "8": 56, "9": 57, ":": 58, ";": 59, "<": 60, "=": 61, ">": 62, "?": 63, "@": 64, "A": 65, "B": 66, "C": 67, "D": 68, "E": 69, "F": 70, "G": 71, "H": 72, "I": 73, "J": 74, "K": 75, "L": 76, "M": 77, "N": 78, "O": 79, "P": 80, "Q": 81, "R": 82, "S": 83, "T": 84, "U": 85, "V": 86, "W": 87, "X": 88, "Y": 89, "Z": 90, "[": 91, "\\": 92, "]": 93, "^": 94, "_": 95, "`": 96, "a": 97, "b": 98, "c": 99, "d": 100, "e": 101, "f": 102, "g": 103, "h": 104, "i": 105, "j": 106, "k": 107, "l": 108, "m": 109, "n": 110, "o": 111, "p": 112, "q": 113, "r": 114, "s": 115, "t": 116, "u": 117, "v": 118, "w": 119, "x": 120, "y": 121, "z": 122, "{": 123, "|": 124, "}": 125, "~": 126, "ġ": 127, "Ģ": 128, "ģ": 129, "Ĥ": 130, "ĥ": 131, "Ħ": 132, "ħ": 133, "Ĩ": 134, "ĩ": 135, "Ī": 136, "ī": 137, "Ĭ": 138, "ĭ": 139, "Į": 140, "į": 141, "İ": 142, "ı": 143, "IJ": 144, "ij": 145, "Ĵ": 146, "ĵ": 147, "Ķ": 148, "ķ": 149, "ĸ": 150, "Ĺ": 151, "ĺ": 152, "Ļ": 153, "ļ": 154, "Ľ": 155, "ľ": 156, "Ŀ": 157, "ŀ": 158, "Ł": 159, "ł": 160, "¡": 161, "¢": 162, "£": 163, "¤": 164, "¥": 165, "¦": 166, "§": 167, "¨": 168, "©": 169, "ª": 170, "«": 171, "¬": 172, "Ń": 173, "®": 174, "¯": 175, "°": 176, "±": 177, "²": 178, "³": 179, "´": 180, "µ": 181, "¶": 182, "·": 183, "¸": 184, "¹": 185, "º": 186, "»": 187, "¼": 188, "½": 189, "¾": 190, "¿": 191, "À": 192, "Á": 193, "Â": 194, "Ã": 195, "Ä": 196, "Å": 197, "Æ": 198, "Ç": 199, "È": 200, "É": 201, "Ê": 202, "Ë": 203, "Ì": 204, "Í": 205, "Î": 206, "Ï": 207, "Ð": 208, "Ñ": 209, "Ò": 210, "Ó": 211, "Ô": 212, "Õ": 213, "Ö": 214, "×": 215, "Ø": 216, "Ù": 217, "Ú": 218, "Û": 219, "Ü": 220, "Ý": 221, "Þ": 222, "ß": 223, "à": 224, "á": 225, "â": 226, "ã": 227, "ä": 228, "å": 229, "æ": 230, "ç": 231, "è": 232, "é": 233, "ê": 234, "ë": 235, "ì": 236, "í": 237, "î": 238, "ï": 239, "ð": 240, "ñ": 241, "ò": 242, "ó": 243, "ô": 244, "õ": 245, "ö": 246, "÷": 247, "ø": 248, "ù": 249, "ú": 250, "û": 251, "ü": 252, "ý": 253, "þ": 254, "ÿ": 255, "he": 256, "ll": 257, "hell": 258, "hello": 259, "Wo": 260, "Wor": 261, "World": 262, "ld": 263, "th": 264, "the": 265, "Ġthe": 266, "12": 267, "123": 268, "ab": 269, "'s": 270, "Ġa": 271, "./": 272, "ĊĊ": 273}, "merges": [["h", "e"], ["l", "l"], ["he", "ll"], ["hell", "o"], ["W", "o"], ["Wo", "r"], ["Wor", "ld"], ["l", "d"], ["t", "h"], ["th", "e"], ["Ġ", "the"], ["1", "2"], ["12", "3"], ["a", "b"], ["'", "s"], ["Ġ", "a"], [".", "/"], ["Ċ", "Ċ"]]}} \ No newline at end of file From 6507817d9fdf75837fb8b9795a2c4826dd480e17 Mon Sep 17 00:00:00 2001 From: Nicholas Beerbower Date: Thu, 16 Jul 2026 15:20:43 -0400 Subject: [PATCH 17/95] st: chunked pread with EINTR retry and honest short-read errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two latent bugs in every st.h reader, both hit in the field: - a single pread caps at ~2^31 bytes on Linux, so any tensor past 2.1 GB (bf16 embed/unembed tensors of large models qualify) came back silently truncated with perror printing '... : Success' (errno untouched by a short read) — the same misleading-error symptom glm.c fixed for its own reads in #236; - no EINTR retry. st_pread_full loops in ST_PREAD_CHUNK pieces (1 GB default, override for tests), retries EINTR, and reports offset + progress on failure. All five read sites converted; behavior on well-formed files is byte-identical (GLM oracle re-verified on this branch: 32/32). tests/test_st_pread builds with -DST_PREAD_CHUNK=7 so a 96-byte tensor exercises the multi-chunk loop, and forks a child against a shard truncated after st_init (init's static bounds check means the pread path only fires when a file shrinks under a live handle) asserting exit(1) with a 'short read' message and no 'Success'. Co-Authored-By: Claude Fable 5 --- c/Makefile | 5 ++- c/st.h | 43 ++++++++++++++++++++--- c/tests/test_st_pread.c | 75 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 6 deletions(-) create mode 100644 c/tests/test_st_pread.c diff --git a/c/Makefile b/c/Makefile index 27902cb..89da266 100644 --- a/c/Makefile +++ b/c/Makefile @@ -166,7 +166,7 @@ else PYTHON ?= python3 endif CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) ifneq (,$(LINUX)) TEST_BINS += tests/test_uring$(EXE) endif @@ -293,6 +293,9 @@ iobench$(EXE): iobench.c compat.h tests/test_json$(EXE): tests/test_json.c json.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_st_pread$(EXE): tests/test_st_pread.c st.h json.h compat.h + $(CC) $(CFLAGS) -DST_PREAD_CHUNK=7 $< -o $@ $(LDFLAGS) + tests/test_st$(EXE): tests/test_st.c st.h json.h compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) diff --git a/c/st.h b/c/st.h index 6b4a710..efa6e5a 100644 --- a/c/st.h +++ b/c/st.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -104,6 +105,38 @@ static int st_direct_fd(shards *S, int fd) { } /* indicizza tutti i model-*.safetensors in snap_dir */ +/* pread completo: chunk-loop (una singola pread si ferma a ~2^31 byte su Linux + * — i tensori bf16 grandi la superano), riprova su EINTR e riporta un errore + * ONESTO: perror stampava "Success" su una short-read (errno resta 0), lo + * stesso sintomo corretto in glm.c per #236. ST_PREAD_CHUNK e' sovrascrivibile + * per i test. EN: full pread — chunk loop (one pread caps at ~2^31 bytes and + * big bf16 tensors exceed it), EINTR retry, honest short-read errors. + * Exits on failure, like every st.h reader. */ +#ifndef ST_PREAD_CHUNK +#define ST_PREAD_CHUNK (1u << 30) +#endif +static void st_pread_full(int fd, void *buf, int64_t n, int64_t off, const char *tag) { + char *p = (char *)buf; + int64_t got = 0; + while (got < n) { + int64_t want = n - got; + if (want > (int64_t)ST_PREAD_CHUNK) want = ST_PREAD_CHUNK; + ssize_t r = pread(fd, p + got, (size_t)want, off + got); + if (r < 0) { + if (errno == EINTR) continue; + fprintf(stderr, "%s: %s (off %lld, %lld/%lld bytes)\n", tag, strerror(errno), + (long long)off, (long long)got, (long long)n); + exit(1); + } + if (r == 0) { + fprintf(stderr, "%s: short read at EOF (off %lld, %lld/%lld bytes) — truncated file?\n", + tag, (long long)off, (long long)got, (long long)n); + exit(1); + } + got += r; + } +} + static void st_init(shards *S, const char *snap_dir) { memset(S, 0, sizeof(*S)); S->cap = 4096; S->t = calloc(S->cap, sizeof(st_tensor)); @@ -128,7 +161,7 @@ static void st_init(shards *S, const char *snap_dir) { if (fstat(fd, &sst) != 0) { perror("fstat shard"); exit(1); } int64_t fsz = (int64_t)sst.st_size; uint64_t hlen; - if (pread(fd, &hlen, 8, 0) != 8) { perror("pread hlen"); exit(1); } + st_pread_full(fd, &hlen, 8, 0, "pread hlen"); /* file malevolo/troncato: hlen deve stare nel file dopo gli 8 byte di * prefisso e sotto il tetto. Senza questo bound hlen+1 puo' andare in * overflow (malloc(0) e poi hdr[hlen]=0 fuori limiti) o forzare una @@ -138,7 +171,7 @@ static void st_init(shards *S, const char *snap_dir) { files[fi], (unsigned long long)hlen, (long long)fsz); exit(1); } char *hdr = malloc(hlen + 1); if (!hdr) { perror("malloc safetensors header"); exit(1); } - if (pread(fd, hdr, hlen, 8) != (ssize_t)hlen) { perror("pread hdr"); exit(1); } + st_pread_full(fd, hdr, (int64_t)hlen, 8, "pread hdr"); hdr[hlen] = 0; int64_t data_start = 8 + (int64_t)hlen; char *arena = NULL; @@ -218,7 +251,7 @@ static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) { if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); } void *raw = malloc(t->nbytes); if (!raw) { fprintf(stderr, "malloc %lld bytes for tensor %s failed\n", (long long)t->nbytes, name); exit(1); } - if (pread(t->fd, raw, t->nbytes, t->off) != t->nbytes) { perror("pread data"); exit(1); } + st_pread_full(t->fd, raw, t->nbytes, t->off, "pread data"); if (t->dtype == 2) { memcpy(out, raw, t->nbytes); } else if (t->dtype == 0) { @@ -243,7 +276,7 @@ static int64_t st_nbytes(shards *S, const char *name) { static void st_read_raw(shards *S, const char *name, void *out, int drop) { st_tensor *t = st_find(S, name); if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); } - if (pread(t->fd, out, t->nbytes, t->off) != t->nbytes) { perror("pread raw"); exit(1); } + st_pread_full(t->fd, out, t->nbytes, t->off, "pread raw"); if (drop) posix_fadvise(t->fd, t->off, t->nbytes, POSIX_FADV_DONTNEED); } @@ -256,7 +289,7 @@ static void st_read_slice_f32(shards *S, const char *name, int64_t elem_off, int int esz = (t->dtype == 2) ? 4 : 2; int64_t boff = t->off + elem_off * esz, nb = n_elems * esz; void *raw = malloc(nb); - if (pread(t->fd, raw, nb, boff) != nb) { perror("pread slice"); exit(1); } + st_pread_full(t->fd, raw, nb, boff, "pread slice"); if (t->dtype == 2) memcpy(out, raw, nb); else if (t->dtype == 0) { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = bf16_to_f32(p[i]); } else { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = f16_to_f32(p[i]); } diff --git a/c/tests/test_st_pread.c b/c/tests/test_st_pread.c new file mode 100644 index 0000000..24ec789 --- /dev/null +++ b/c/tests/test_st_pread.c @@ -0,0 +1,75 @@ +/* st_pread_full: chunk loop + honest truncation errors. + * Built with -DST_PREAD_CHUNK=7 so a ~100-byte tensor takes many pread calls — + * exercising the loop that production only needs past 2^31 bytes (one pread + * caps there on Linux; big bf16 tensors exceed it). Also forks a child against + * a truncated shard and requires exit(1) with a "short read" message instead + * of the old perror("... : Success"). */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#include "../st.h" + +#define CHECK(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "%s:%d: check failed: %s\n", __FILE__, __LINE__, #condition); \ + return 1; \ + } \ +} while (0) + +static void write_snap(const char *dir, int truncate_bytes) { + char path[512]; + snprintf(path, sizeof(path), "%s/model.safetensors", dir); + unsigned char data[96]; + for (int i = 0; i < 96; i++) data[i] = (unsigned char)(i * 7 + 3); + const char *hdr = "{\"t\":{\"dtype\":\"U8\",\"shape\":[96],\"data_offsets\":[0,96]}}"; + uint64_t hlen = strlen(hdr); + FILE *f = fopen(path, "wb"); + fwrite(&hlen, 8, 1, f); + fwrite(hdr, 1, hlen, f); + fwrite(data, 1, (size_t)(96 - truncate_bytes), f); + fclose(f); +} + +int main(void) { + char dir[] = "/tmp/st_pread_XXXXXX"; + if (!mkdtemp(dir)) { char alt[] = "st_pread_XXXXXX"; if (!mkdtemp(alt)) return 1; strcpy(dir, alt); } + + /* 1) chunk loop: 96-byte tensor read 7 bytes at a time, content exact */ + write_snap(dir, 0); + shards S; st_init(&S, dir); + unsigned char out[96] = {0}; + st_read_raw(&S, "t", out, 0); + for (int i = 0; i < 96; i++) CHECK(out[i] == (unsigned char)(i * 7 + 3)); + + /* 2) shard truncated AFTER st_init (init validates static bounds, so the + * pread path only fires when the file shrinks underneath a live handle): + * child must exit(1) with an honest message, not perror's "Success" */ + char shard[512]; snprintf(shard, sizeof(shard), "%s/model.safetensors", dir); + struct stat sb; CHECK(stat(shard, &sb) == 0); + CHECK(truncate(shard, sb.st_size - 40) == 0); + int pipefd[2]; CHECK(pipe(pipefd) == 0); + pid_t pid = fork(); CHECK(pid >= 0); + if (pid == 0) { + dup2(pipefd[1], 2); close(pipefd[0]); close(pipefd[1]); + unsigned char buf[96]; + st_read_raw(&S, "t", buf, 0); /* inherited handles; must exit(1) inside */ + _exit(42); /* reaching here = bug */ + } + close(pipefd[1]); + char err[512] = {0}; + ssize_t n = read(pipefd[0], err, sizeof(err)-1); (void)n; + close(pipefd[0]); + int status = 0; waitpid(pid, &status, 0); + CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 1); + CHECK(strstr(err, "short read") != NULL); + CHECK(strstr(err, "Success") == NULL); + + char cmd[600]; snprintf(cmd, sizeof(cmd), "rm -rf %s", dir); + if (system(cmd)) {} + printf("test_st_pread: chunk loop + honest truncation error: ok\n"); + return 0; +} From ed8dab4da43356aa587d53b8a29da87d9ca50250 Mon Sep 17 00:00:00 2001 From: Nicholas Beerbower Date: Thu, 16 Jul 2026 16:32:43 -0400 Subject: [PATCH 18/95] =?UTF-8?q?test=5Fst=5Fpread:=20Windows-compatible?= =?UTF-8?q?=20=E2=80=94=20relative=20tmpdir,=20fork=20subtest=20gated=20PO?= =?UTF-8?q?SIX?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same fix pattern as test_stops: mkdtemp with a CWD-relative template (MinGW resolves Windows paths; /tmp is not one), and the fork/pipe/ truncate-based truncation subtest compiles only where those exist — Windows still runs the cross-platform chunk-loop content check. Co-Authored-By: Claude Fable 5 --- c/tests/test_st_pread.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/c/tests/test_st_pread.c b/c/tests/test_st_pread.c index 24ec789..c874c08 100644 --- a/c/tests/test_st_pread.c +++ b/c/tests/test_st_pread.c @@ -8,8 +8,10 @@ #include #include #include +#ifndef _WIN32 #include #include +#endif #include "../st.h" @@ -35,8 +37,10 @@ static void write_snap(const char *dir, int truncate_bytes) { } int main(void) { - char dir[] = "/tmp/st_pread_XXXXXX"; - if (!mkdtemp(dir)) { char alt[] = "st_pread_XXXXXX"; if (!mkdtemp(alt)) return 1; strcpy(dir, alt); } + /* relative to the CWD, per test_stops: MinGW .exe files resolve Windows + * paths and "/tmp" is not one */ + char dir[] = "test_st_pread_XXXXXX"; + if (!mkdtemp(dir)) { perror("mkdtemp"); return 1; } /* 1) chunk loop: 96-byte tensor read 7 bytes at a time, content exact */ write_snap(dir, 0); @@ -45,6 +49,7 @@ int main(void) { st_read_raw(&S, "t", out, 0); for (int i = 0; i < 96; i++) CHECK(out[i] == (unsigned char)(i * 7 + 3)); +#ifndef _WIN32 /* 2) shard truncated AFTER st_init (init validates static bounds, so the * pread path only fires when the file shrinks underneath a live handle): * child must exit(1) with an honest message, not perror's "Success" */ @@ -67,8 +72,17 @@ int main(void) { CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 1); CHECK(strstr(err, "short read") != NULL); CHECK(strstr(err, "Success") == NULL); +#else + /* fork/pipe/truncate are POSIX; Windows still runs the chunk-loop check */ + printf("test_st_pread: truncation subtest skipped on Windows\n"); +#endif - char cmd[600]; snprintf(cmd, sizeof(cmd), "rm -rf %s", dir); + char cmd[600]; +#ifdef _WIN32 + snprintf(cmd, sizeof(cmd), "rmdir /s /q %s", dir); +#else + snprintf(cmd, sizeof(cmd), "rm -rf %s", dir); +#endif if (system(cmd)) {} printf("test_st_pread: chunk loop + honest truncation error: ok\n"); return 0; From 5ad4d540ab7427e6ca134ce88fe93e3d9f869456 Mon Sep 17 00:00:00 2001 From: Nicholas Beerbower Date: Thu, 16 Jul 2026 16:33:22 -0400 Subject: [PATCH 19/95] test_tok_o200k: fgets instead of getline for the windows job MinGW's UCRT has no getline; fixed-buffer fgets with CRLF trimming reads the same case file everywhere. Co-Authored-By: Claude Fable 5 --- c/tests/test_tok_o200k.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/c/tests/test_tok_o200k.c b/c/tests/test_tok_o200k.c index b86bda9..fe62890 100644 --- a/c/tests/test_tok_o200k.c +++ b/c/tests/test_tok_o200k.c @@ -16,10 +16,13 @@ int main(void) { if (!T.o200k) { fprintf(stderr, "test_tok_o200k: o200k pattern not detected\n"); return 1; } FILE *f = fopen("tests/tok_o200k_cases.txt", "rb"); if (!f) { perror("tests/tok_o200k_cases.txt"); return 1; } - char *line = NULL; size_t cap = 0; ssize_t nr; + /* fgets, not getline: MinGW's UCRT lacks getline and this must run on + * the windows job. Case lines are short; 8 KB is generous. */ + char line[8192]; int pass = 0, tot = 0, dpass = 0; - while ((nr = getline(&line, &cap, f)) >= 0) { - if (nr > 0 && line[nr-1] == '\n') line[--nr] = 0; + while (fgets(line, sizeof(line), f)) { + size_t nr = strlen(line); + while (nr > 0 && (line[nr-1] == '\n' || line[nr-1] == '\r')) line[--nr] = 0; if (nr == 0) continue; char *tab = strchr(line, '\t'); if (!tab) continue; *tab = 0; From 81a56777df5414feb0b0afc9077f3ecaf5f2625a Mon Sep 17 00:00:00 2001 From: Tonu Samuel Date: Fri, 17 Jul 2026 08:00:26 +0300 Subject: [PATCH 20/95] glm.c: detect 9p by statfs f_type, not the "/mnt/" path prefix The slow-filesystem warning fired for any model path under /mnt/, which false-positives on native-Linux mounts (ZFS/ext4/xfs/NFS) that commonly live there. Check the actual filesystem type via statfs() against the 9p superblock magic (0x01021997) instead, so it only warns for a genuine WSL 9p mount. Linux-only (statfs); no behavior change on other platforms. --- c/glm.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/c/glm.c b/c/glm.c index 7e05ad9..511fcc5 100644 --- a/c/glm.c +++ b/c/glm.c @@ -37,6 +37,9 @@ #include /* fstat per mmap degli shard (COLI_MMAP) */ #include /* SIGINT = stop morbido del turno in serve mode */ #endif +#ifdef __linux__ +#include /* statfs: real fs-type check for the 9p warning (below) */ +#endif #if defined(_WIN32) && (defined(__x86_64__) || defined(__i386__)) #include /* hwinfo_emit: CPU brand string senza /proc */ #endif @@ -5909,9 +5912,16 @@ int main(int argc, char **argv){ m.has_mtp?"ACTIVE":"absent", g_draft); /* anche su stderr: e' il canale che le UI (coli) mostrano all'utente */ fprintf(stderr,"[MTP] %s (draft=%d)\n", m.has_mtp?"active: native speculative decoding":"absent", g_draft); - if(!strncmp(snap,"/mnt/",5)) - fprintf(stderr,"WARNING: the model is on %s (slow 9p/Windows filesystem; fadvise is ineffective).\n" - " Keep it on ext4 (for example, /home/...) for memory efficiency and speed.\n", snap); +#ifdef __linux__ + { /* Only warn for a GENUINE 9p mount (WSL Windows drives, magic 0x01021997), where + * fadvise is a no-op. The old check was `snap` starting with "/mnt/", which + * false-positives on native-Linux ZFS/ext4/xfs/NFS mounts that also live under /mnt. */ + struct statfs sfb; + if(statfs(snap,&sfb)==0 && (unsigned long)sfb.f_type==0x01021997UL) + fprintf(stderr,"WARNING: the model is on %s (9p/Windows filesystem; fadvise is ineffective).\n" + " Keep it on a native Linux fs (ext4/xfs/zfs) for memory efficiency and speed.\n", snap); + } +#endif /* HOT-STORE: PIN= [PIN_GB=g] -> top expert per frequenza fissi in RAM. * Va PRIMA di cap_for_ram: i pinnati contano nel residente. */ if(getenv("PIN")){ From 721bfcb325495216bb7eca49f1dd3a2cbd2f7896 Mon Sep 17 00:00:00 2001 From: BM Cho Date: Fri, 17 Jul 2026 17:03:49 +0800 Subject: [PATCH 21/95] docs: add Traditional Chinese README --- README.md | 4 + README.zh-TW.md | 233 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 237 insertions(+) create mode 100644 README.zh-TW.md diff --git a/README.md b/README.md index 4045e9a..797f1ca 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ colibrì — tiny engine, immense model

+

+ English · 繁體中文 +

+ **Tiny engine, immense model.** Run **GLM-5.2 (744B-parameter MoE)** on a consumer machine with ~25 GB of RAM — in pure C, with zero dependencies, by streaming experts from disk. Colibrì is a lightweight, quality-preserving MoE runtime that treats VRAM, RAM, diff --git a/README.zh-TW.md b/README.zh-TW.md new file mode 100644 index 0000000..1662617 --- /dev/null +++ b/README.zh-TW.md @@ -0,0 +1,233 @@ +

+ colibrì——小巧引擎，龐大模型 +

+ +

+ English · 繁體中文 +

+ +**小巧引擎，龐大模型。**只要約 25 GB 記憶體，就能在消費級電腦上執行 **GLM-5.2（744B 參數的 MoE）**——以零相依套件的純 C 實作，從硬碟串流載入專家。 + +Colibrì 是一套輕量、維持品質的 MoE 執行環境，將 VRAM、RAM +與儲存裝置視為統一管理的記憶體階層。高速記憶體不足可能降低速度， +但預設策略**絕不會在未告知的情況下改變模型精度或路由語意**。 + +``` +$ ./coli chat + 🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU + ✓ ready in 32s · resident 9.9 GB + › ciao! + ◆ Ciao! 😊 Come posso aiutarti oggi? +``` + +## 實際運行畫面 + +

+ colibrì 網頁儀表板——即時指標、硬體面板與專家儲存層級 +

+

網頁儀表板（./coli web）：744B 模型達到 4 tok/s、TTFT 1.6 秒、硬碟讀取 0—— +在 6× RTX 5090 上讓所有專家常駐，並即時顯示 token 指標、每輪耗時明細、 +VRAM／RAM／硬碟層級長條，以及角落的即時迷你大腦。

+ +

+ 大腦頁面——以即時皮質呈現 19,456 個專家 +

+

大腦（Brain）頁面：將全部 19,456 個專家呈現為活的皮質——顏色代表儲存層級， +亮度代表路由熱度，每輪被路由到的專家都會閃白。將游標停在專家上，即可查看其 +實測主題親和度。

+ +

+ 圖譜頁面——以 3D 星系呈現實測專家圖譜 +

+

圖譜（Atlas）頁面：將實測專家圖譜 +呈現為 3D 星系——共 13,260 個已分析專家，其中 1,041 個可重現的專門專家會按主題聚集 +（詩歌、法律、中文、SQL……）。位置取自實測路由親和度，而非學習出的嵌入向量。拖曳即可旋轉。

+ +## 核心概念 + +744B 的專家混合（Mixture-of-Experts）模型，每個 token 只會啟用約 40B 參數—— +其中每個 token 之間會變動的只有約 11 GB（被路由到的專家）： + +

+ 每個 token 只會啟用約 5.4% 的參數 +

+ +所以模型不必完整**放進**高速記憶體，而是需要正確**配置位置**： + +- **稠密部分**（注意力、共享專家、嵌入——約 17B 參數）以 int4 + **常駐 RAM**（約 9.9 GB）； +- **19,456 個路由專家**（75 個 MoE 層 × 256，加上 MTP head；每個在 int4 下約 19 MB） + **存放在硬碟**（約 370 GB），並**隨需串流載入**，搭配逐層 LRU 快取、 + 會學習的熱門專家固定儲存區，以及選用的 VRAM 層級。 + +引擎由單一 C 檔（`c/glm.c`）與少量標頭檔組成。不需要 BLAS， +執行階段不需要 Python，也不需要 GPU。 + +## 運作方式 + +### 每個 token 的處理路徑 + +

+ 路由 → 聯集 → 配置 → 重疊執行 → 學習 +

+ +每個 token 的每一層都會走過相同的五個步驟。設計目標是讓 +**配置只決定速度**——無論專家是從 VRAM 或硬碟回應，路由器的決策與權重精度都完全相同。 + +### 統一記憶體階層，取代單一記憶體門檻 + +

+ VRAM／RAM／NVMe 三層專家常駐架構 +

+ +同一套引擎涵蓋完整硬體範圍：在 25 GB 筆電上，一切都從硬碟串流載入 +（慢，但結果正確）；在大型主機上，則可讓整組專家常駐 +（`CUDA_EXPERT_GB=auto PIN_GB=all`），讓硬碟完全退出解碼路徑。 +兩端之間有一層**學習型快取**：引擎會記錄*你的*工作負載路由到哪些專家 +（`.coli_usage`，每輪更新），並自動固定最熱門的專家——colibrì 確實會越用越快。 +在多插槽主機上，`COLI_NUMA=1` 會將常駐權重交錯分配到各記憶體控制器 +（[#82](https://github.com/JustVugg/colibri/issues/82)）。 + +### 絕不為同一次硬碟讀取等待兩遍 + +快取未命中的成本很高，因此引擎大部分的巧思都用來避免或重疊處理這些讀取： +每個專家的三個矩陣相鄰儲存，並以一次 `pread` 讀取；有界非同步 I/O pool +（`PIPE=1`，預設啟用）會在常駐專家運算時載入缺少的專家；批次位置只讀取每個 +不重複專家一次（**批次聯集**）；路由前瞻執行緒（`PILOT=1`）則預先載入下一層專家—— +實測顯示，路由結果提前一層時有 **71.6% 的可預測性**。 +在 GPU 上，常駐管線（`COLI_CUDA_PIPE=2`）讓殘差流跨層保留在裝置端， +使 CPU 專家迴圈不中斷；在 Apple Silicon 上，實驗性的 +[Metal 後端](docs/metal.md)會用統一記憶體 GPU 執行批次專家運算。 + +### 忠實模型，壓縮狀態 + +前向傳遞已透過 `transformers` oracle 驗證為**逐 token 完全一致** +（teacher-forcing 32/32）。MLA 注意力儲存壓縮後的 KV 狀態——每個 token 為 576 個 +浮點數，而非 32,768 個（**縮小 57×**）——並跨重新啟動持久保存 +（`.coli_kv`）：對話可暖啟恢復，不需重新 prefill，結果與不中斷的工作階段 +逐位元組相同。DSA 稀疏注意力（GLM-5.2 的 lightning indexer）已忠實實作， +並透過強制選取所有 key，驗證可精確重現稠密注意力。 + +### 如實呈現推測式解碼 + +GLM-5.2 原生 MTP head 會起草 token，再由主模型以一次批次前向傳遞驗證—— +條件合適時每次 forward 可產生 2.2–2.8 個 token。兩條得來不易的規則已成為預設值： +MTP head 必須是 **int8**（int4 head 的接受率會崩落到 0–4%，見 +[#8](https://github.com/JustVugg/colibri/issues/8)），且草稿與驗證必須計算 +**相同函數**——`SPEC_PIN=1` 會把兩者固定在同一 kernel family +（完整鑑識過程見 [#163](https://github.com/JustVugg/colibri/issues/163)）。 +文法強制草稿（[`GRAMMAR=file.gbnf`](docs/grammar-draft.md)）可在受限 JSON 輸出中， +以近乎免費的成本提高接受率。推測式解碼是否帶來淨收益取決於快取熱度——請實測， +若不划算就使用 `DRAFT=0`。 + +## 實際成果 + +

+ 各硬體等級的實測解碼速度 +

+ +同一套引擎、同一個 int4 容器——硬體只會改變專家的存放位置。 +[完整 benchmark 表格](docs/benchmarks.md)中的重點如下： + +- **6× RTX 5090，全部常駐：**解碼 5.8–6.8 tok/s，TTFT 約 13 秒 + （[實驗紀錄](docs/experiments/glm52-6x5090-2026-07-12.md)）； +- **128 GB、僅使用 CPU 的桌上型電腦：**暖機後約 1.8 tok/s + （[#200](https://github.com/JustVugg/colibri/issues/200)）； +- **單張 RTX 5070 Ti 的筆電級電腦：**透過 GPU 常駐管線達到 1.07 tok/s + （[#273](https://github.com/JustVugg/colibri/issues/273)）； +- **25 GB 開發機：**冷啟動 0.05–0.1 tok/s——這是專案起步時已證實的下限， + 也仍是如實呈現的基準。 + +品質來自測量，而非假設：int4 容器的量化成本，以及 scale granularity／rotation +消融實驗，收錄於 [docs/benchmarks.md](docs/benchmarks.md#quality-benchmark)、 +[#108](https://github.com/JustVugg/colibri/issues/108) 與 +[#81](https://github.com/JustVugg/colibri/issues/81)。 + +## 開始使用 + +### 1. 取得模型 + +Hugging Face 上已有預先轉換的 **GLM-5.2 int4** 容器——請務必使用 +**含 int8 MTP head 的版本**： + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +> ⚠️ 原始鏡像使用 int4 MTP head → 草稿接受率為 0% +>（[#8](https://github.com/JustVugg/colibri/issues/8)）。請檢查你的版本： +> `ls -l /out-mtp-*`——正確的 int8 大小為 `3527131672 / 5366238584 / 1065950496`。 + +你也可以自行從 FP8 來源轉換——只需一條可續傳的指令，且任何時候都不需要 +在硬碟上同時存放完整的 756 GB： + +```bash +cd c && ./setup.sh # 檢查 gcc/OpenMP、建置並執行自我測試 +./coli convert --model /nvme/glm52_i4 # 逐 shard 下載並轉換（僅此一次需要 python） +``` + +### 2. 執行 + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli chat # 自動偵測 RAM 預算、快取與 MTP +COLI_MODEL=/nvme/glm52_i4 ./coli plan # 檢視規劃的 VRAM／RAM／硬碟配置 +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # 唯讀就緒檢查 +./coli web --model /nvme/glm52_i4 # 在同一個連接埠提供 API 與網頁儀表板 +./coli serve --model /nvme/glm52_i4 # 僅提供 OpenAI 相容 API +``` + +引擎執行階段是純 C——python 只供單次轉換工具與選用的 API gateway 使用。 + +### 3. 深入了解 + +| 主題 | 文件 | +|---|---| +| Benchmark、社群實測數據、品質測量 | [docs/benchmarks.md](docs/benchmarks.md) | +| 調校選項、策略、學習型快取、預先載入 | [docs/tuning.md](docs/tuning.md) | +| Windows 11 原生建置（含 CUDA DLL） | [docs/windows.md](docs/windows.md) | +| CUDA 後端、VRAM 專家層級、全部常駐 | [docs/cuda.md](docs/cuda.md) | +| Apple Silicon Metal 後端 | [docs/metal.md](docs/metal.md) | +| OpenAI 相容 API、KV slots、網頁儀表板 | [docs/api.md](docs/api.md) | +| 文法強制草稿（結構化輸出） | [docs/grammar-draft.md](docs/grammar-draft.md) | +| 環境變數完整清單 | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | + +## 支持專案 + +colibrì 最初是由一人使用 12 核心、25 GB RAM 的筆電開發； +如今它的數據來自社群中的各種真實機器。如果這個專案對你有用： + +- ⭐ 為儲存庫加星並分享； +- 🐛 以 issue 提交你的硬體 benchmark 數據——實測資料比任何其他事都更能推動專案； +- 💬 若想贊助開發或捐贈硬體，請透過 GitHub issues 聯絡。 + +## 儲存庫結構 + +``` +Makefile 根目錄建置／檢查入口 +c/ +├── glm.c 單檔 GLM 引擎 +├── st.h, tok.h, json.h 執行階段標頭檔 +├── backend_cuda.* 選用的 CUDA 層級 +├── Makefile 建置與本機檢查 +├── coli 使用者介面 CLI +├── openai_server.py OpenAI 相容 HTTP gateway +├── setup.sh 單一指令完成本機設定 +├── tools/ 離線轉換、fixtures 與 benchmarks +├── scripts/ 長時間轉換輔助工具 +└── tests/ 零相依套件的 C 與 Python 測試 +web/ 瀏覽器 UI（純 OpenAI API client） +desktop/ 包裝網頁 UI 的 Tauri v2 桌面 shell +docs/ 參考文件、實驗與媒體檔 +``` + +執行階段路徑刻意維持扁平、易讀：`glm.c` 加上少量標頭檔。 +在儲存庫根目錄執行 `make`、`make check` 與 `make clean`， +都會轉交給引擎的 Makefile。 + +## 為什麼叫做「colibrì」 + +蜂鳥只有幾公克重，能在原地懸停，並在一天內造訪上千朵花。 +這套引擎只用蜂鳥般的配給，就能讓 744B 參數的巨人運轉： +25 GB RAM、十二個 CPU 核心，以及對硬碟的大量耐心。 + +## 授權條款 + +Apache 2.0。GLM-5.2 權重由 Z.ai 以 MIT 授權發布。 From 3f239dbb922ebe4923495a7e42bce304ab2ffc4b Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Fri, 17 Jul 2026 18:39:53 +0800 Subject: [PATCH 22/95] =?UTF-8?q?tools:=20rate-scale=20the=20E8=20lattice?= =?UTF-8?q?=20ball=20by=20bit-width=20=E2=80=94=20int3-e8=20is=20now=20a?= =?UTF-8?q?=20real=20int3=20codebook,=20not=20a=20fixed=202-bit=20ball?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- c/tools/quant_ablation.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/c/tools/quant_ablation.py b/c/tools/quant_ablation.py index 689643a..dd08715 100644 --- a/c/tools/quant_ablation.py +++ b/c/tools/quant_ablation.py @@ -118,7 +118,7 @@ def quantize_param(w, bits, group, rot=False, e8=""): def _grid_or_e8(x, bits, group, e8): if e8: - return _quant_e8(x.float(), group, ball=(e8 == "-e8")) + return _quant_e8(x.float(), group, bits, ball=(e8 == "-e8")) return _quant_last_dim(x, bits, group) @@ -169,8 +169,15 @@ def _e8_ball(y, r2=10.0): return p -def _quant_e8(x, group, ball): - """Blocks of 8 along the input dim; per-group scale by MSE search over RMS multiples.""" +_E8_R2_REPORTED = set() +def _e8_radius(bits): + # E8 lattice: points within |p|^2<=r2 grow ~r2^4, so +1 bit (x256 codebook) needs r2 x4. + # Anchor: r2=10 is the ~2^16 E8P ball (2 bits over 8 dims). Scale from there. + return 10.0 * (4.0 ** (bits - 2)) + +def _quant_e8(x, group, bits, ball): + """Blocks of 8 along the input dim; per-group scale by MSE search over RMS multiples. + ball=True clamps to the rate-scaled E8 ball for `bits`; ball=False is the unbounded ideal.""" if x.shape[-1] % 8: raise SystemExit(f"-e8 needs input dim divisible by 8 (got {x.shape[-1]})") g = group or x.shape[-1] @@ -183,7 +190,11 @@ def _quant_e8(x, group, ball): for k in (0.5, 0.7, 0.9, 1.1, 1.4, 1.8, 2.4): s = rms * k yb = (xg / s).reshape(-1, g // 8, 8) - p = _e8_ball(yb) if ball else _e8_nearest(yb) + p = _e8_ball(yb, _e8_radius(bits)) if ball else _e8_nearest(yb) + if ball and bits not in _E8_R2_REPORTED: + _E8_R2_REPORTED.add(bits) + import sys as _sys + _sys.stderr.write(f"[e8] bits={bits}: ball r2={_e8_radius(bits):.1f}\n") out = (p.reshape(-1, g) * s) err = (out - xg).pow(2).sum(-1, keepdim=True) if best_err is None: From 528b3cff37353d5a4719311f89b2e2eecc2af2a6 Mon Sep 17 00:00:00 2001 From: woolcoxm <13604288+woolcoxm@users.noreply.github.com> Date: Fri, 17 Jul 2026 07:18:24 -0400 Subject: [PATCH 23/95] fix(flake): package engine, support modules, and python for Nix builds (#345) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four bugs in the Nix flake, all addressed: 1. checkPhase failed with "make: python3: No such file or directory" because `make test-c` shells out to python3 (c/Makefile: PYTHON ?= python3) but python3 wasn't in nativeBuildInputs. Add pkgs.python3. 2. `coli serve` reported "engine is not built" even though glm was packaged, because the engine landed in $out/bin while coli looks for it beside itself, in libexec/colibri, or via $COLI_ENGINE (c/coli:57- 66). The wrapper now sets COLI_ENGINE to the bundled engine. 3. `coli serve` failed with ModuleNotFoundError: openai_server because the support modules (openai_server.py, resource_plan.py, doctor.py) were not installed. Install them and put their dir on PYTHONPATH. 4. Rebuilt the install layout into a self-contained $out/lib/colibri/ that mirrors the source tree coli expects, with $out/bin/{glm,coli} as user-facing entry points (glm symlinked; coli wrapped). NOTE: flake.lock is intentionally not included — it must be generated on a Nix host via `nix flake lock` and committed separately, since it requires narHash values that only the nix tooling can compute. --- flake.nix | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/flake.nix b/flake.nix index 368b0a7..83c6de2 100644 --- a/flake.nix +++ b/flake.nix @@ -26,7 +26,9 @@ version = "1.0"; src = ./.; - nativeBuildInputs = [ pkgs.makeWrapper ]; + # python3 is needed by checkPhase: `make test-c` shells out to + # `python3 tools/run_tests.py` (see c/Makefile, PYTHON ?= python3). + nativeBuildInputs = [ pkgs.makeWrapper pkgs.python3 ]; buildInputs = [ pkgs.gcc @@ -44,18 +46,29 @@ installPhase = '' runHook preInstall - mkdir -p $out/bin - cp c/glm $out/bin/glm - # Wrap coli (the Python CLI) so it finds the right python and the engine - mkdir -p $out/share/colibri - cp c/coli $out/share/colibri/coli - chmod +x $out/share/colibri/coli - cp -r c/tools $out/share/colibri/tools + # Self-contained layout under $out/lib/colibri that mirrors the + # source tree `coli` runs in (see the path-resolution logic at the + # top of c/coli): the engine, the coli CLI script, the support + # modules it imports (openai_server.py, resource_plan.py, + # doctor.py), and tools/ all sit next to each other. + mkdir -p $out/lib/colibri/tools $out/bin + cp c/glm $out/lib/colibri/glm + cp c/coli $out/lib/colibri/coli + chmod +x $out/lib/colibri/coli + cp c/openai_server.py c/resource_plan.py c/doctor.py $out/lib/colibri/ + cp -r c/tools/* $out/lib/colibri/tools/ + # $out/bin holds the user-facing entry points. + ln -s ../lib/colibri/glm $out/bin/glm + + # Wrap coli: point it at the bundled engine (COLI_ENGINE) so it is + # found by default, and at the module dir (PYTHONPATH) so + # `import openai_server` / `resource_plan` / `doctor` resolve. makeWrapper ${pythonEnv}/bin/python $out/bin/coli \ - --add-flags "$out/share/colibri/coli" \ - --set PYTHONPATH "${pythonEnv}/${pkgs.python3.sitePackages}" + --add-flags "$out/lib/colibri/coli" \ + --set COLI_ENGINE "$out/lib/colibri/glm" \ + --set PYTHONPATH "$out/lib/colibri:${pythonEnv}/${pkgs.python3.sitePackages}" runHook postInstall ''; From 8bf4cb9a98f38f758154ccce6c2bdd77e73a9b41 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Fri, 17 Jul 2026 02:07:09 +0200 Subject: [PATCH 24/95] =?UTF-8?q?coli:=20chat=20attaches=20to=20a=20runnin?= =?UTF-8?q?g=20serve=20=E2=80=94=20the=20engine=20survives=20the=20chat=20?= =?UTF-8?q?(local)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cold-chat cost, measured on this box: every `coli chat` spawns a private engine (34-136 s of resident load) and starts with an empty expert cache (hit 4% cold vs 55% warm). Quitting throws both away. `coli chat` now probes localhost:8000 first (~1 ms when nothing listens) and, if a `coli serve` answers, runs the REPL over plain OpenAI SSE against it: stdlib urllib only, engine byte-protocol untouched. --attach [URL] forces it, --no-attach restores a private engine. reasoning_content keepalive pings are filtered; :reset starts a new conversation client-side (the server's KV slots reuse prefixes per conversation on their own). Verified against a mock SSE server (pings ignored, markdown rendered, :reset, clean exit) and against the real 744B model: two consecutive sessions, second attach instant with zero reload — the engine stayed resident at 15.7 GB across both. Honest limit: warmth carry-over BETWEEN different conversations is small here because cap=3 slots/layer is a short memory; the structural wins are the load never being repaid and same-conversation continuation. LOCAL ONLY for now — not pushed, per the current working rule. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit aa406ccab8a4501b924bc3a9f4725dd1d18a685d) --- c/coli | 136 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 134 insertions(+), 2 deletions(-) diff --git a/c/coli b/c/coli index 4c27415..74f95de 100755 --- a/c/coli +++ b/c/coli @@ -20,7 +20,7 @@ Configuration through environment variables or flags (also valid after the subco --topp P adaptive expert top-p --topk N fixed top-k --ngen N maximum response tokens --cap N cache slots/layer """ -import os, sys, subprocess, argparse, json, time, signal, shutil, threading, re, codecs, tempfile, textwrap +import os, sys, subprocess, argparse, json, time, signal, shutil, threading, re, codecs, tempfile, textwrap, struct # The engine mmaps every shard (144+ files); macOS default RLIMIT_NOFILE is 256. if sys.platform != "win32": @@ -484,9 +484,134 @@ def cmd_run(a): e=env_for(a); e["PROMPT"]=f"[gMASK]<|user|>{prompt}<|assistant|>" sys.exit(subprocess.call([GLM, str(a.cap)], env=e)) +def server_probe(base, api_key=None, timeout=1.5): + """Is a coli serve alive at `base`? Returns its model_id, or None. + Probes /health then /v1/models — both cheap, neither touches the engine.""" + import urllib.request, urllib.error + def get(path): + req=urllib.request.Request(base.rstrip("/")+path) + if api_key: req.add_header("Authorization", f"Bearer {api_key}") + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read().decode("utf-8","replace")) + try: + if get("/health").get("status")!="ok": return None + data=get("/v1/models").get("data") or [] + return data[0]["id"] if data else None + except Exception: + return None + +def chat_attached(a, base, model_id): + """The chat REPL over HTTP against a running `coli serve`. + + Why this exists (the cold-chat cost, measured): spawning a private engine + pays 34-136 s of resident load on EVERY start, and begins with an empty + expert cache — hit rate 4% cold vs 55% warm, a ~10x on early decode. A + resident server pays load once and keeps the LRU warm across sessions; + its KV slots reuse the conversation prefix, so a continued chat skips + re-prefill too. The engine byte-protocol stays untouched — this is plain + OpenAI SSE over localhost, stdlib only.""" + import urllib.request + print(f" {C.grn}✦ attached{C.r} {C.dim}to {base} · model {model_id} · the engine stays warm after you quit{C.r}") + print(f" {C.dim}type and press Enter · Ctrl-C stops the answer · :reset starts a new conversation · :q exits{C.r}\n") + msgs=[] + w=term_w()-4 + while True: + if TTY: + print(f" {C.dgray}╭{'─'*w}╮{C.r}") + try: msg=input(f" {C.dgray}│{C.r} {C.teal}{C.b}›{C.r} ") + except EOFError: print(); break + print(f" {C.dgray}╰{'─'*w}╯{C.r}") + else: + try: msg=input() + except EOFError: break + msg=msg.strip() + if msg in (":q",":quit","exit"): break + if not msg: continue + if msg==":reset": msgs=[]; print(f" {C.dim}✦ new conversation{C.r}\n"); continue + msgs.append({"role":"user","content":msg}) + body=json.dumps({"model":model_id,"messages":msgs,"stream":True, + "max_tokens":a.ngen}).encode() + req=urllib.request.Request(base.rstrip("/")+"/v1/chat/completions", data=body, + headers={"Content-Type":"application/json"}) + if a.api_key: req.add_header("Authorization", f"Bearer {a.api_key}") + print(f"\n {C.teal}◆ colibrì{C.r}") + sp=Spinner("thinking…"); sp.start() + md=MDStream(" "); reply=[]; first=True; t0=time.time(); interrupted=False + try: + with urllib.request.urlopen(req) as r: + for raw in r: + line=raw.decode("utf-8","replace").strip() + if not line.startswith("data: "): continue + data=line[6:] + if data=="[DONE]": break + try: ev=json.loads(data) + except ValueError: continue + for ch in ev.get("choices",[]): + d=ch.get("delta",{}) + txt=d.get("content") + if not txt: continue # ping/ruolo/reasoning: non è testo + if first: sp.stop(); first=False + md.feed(txt); reply.append(txt) + except KeyboardInterrupt: + interrupted=True # il server annulla la richiesta alla disconnessione + except OSError as e: + sp.stop() + print(f"\n {C.yel}[server unreachable: {e}]{C.r}"); break + sp.stop(); md.close() + if reply: msgs.append({"role":"assistant","content":"".join(reply)}) + else: msgs.pop() # turno vuoto: non sporcare la history + el=time.time()-t0 + note=" · ⏹ interrupted" if interrupted else "" + print(f"\r {C.dgray}└─ ~{len(''.join(reply))//4} tok · {el:.0f}s{note}{C.r}\n") + print(f" {C.dim}goodbye — the engine keeps running for the next chat 🐦{C.r}") + +def kv_resume_notice(model_dir): + """SERVE mode silently resumes .coli_kv from disk (glm.c kv_disk_load): a chat + started today continues a conversation from days ago, with `first=0` so the + turn is appended WITHOUT the [gMASK] prefix. The engine does announce it + on stderr — but nothing here ever shows that: the drain thread's + p.stderr.read() blocks until EOF, so on a healthy start errlog is still empty + when the status lines are printed. The warning only appeared once the engine + DIED, which is exactly when it no longer mattered. + + Measured cost of the silence: a chat inherited 670 tokens of an old Italian + session ("il mio numero preferito e 7, ricordalo!"). Every later reply came + back in Italian, and "explain fibonacci in short" was answered about the + number 7 — the model was being coherent with a context nobody could see, and + it read as a quantization bug for a day. + + So say it here, in Python, from the file itself: no pipe, no thread, no + Windows deadlock risk (see the stderr comment below).""" + p=os.path.join(model_dir, ".coli_kv") + try: + with open(p,"rb") as f: + if f.read(8)!=b"COLIKV1\0": return + h=struct.unpack("<8i", f.read(32)) + n=h[6] + if n<1: return + age=time.time()-os.path.getmtime(p) + when=f"{age/86400:.0f}d ago" if age>86400 else f"{age/3600:.0f}h ago" if age>3600 else "just now" + print(f" {C.yel}↺ resuming a saved conversation: {n} tokens, last written {when}{C.r}") + print(f" {C.dgray} it steers tone, language and topic. :reset clears it · " + f"KVSAVE=0 disables saving · delete {p} to start clean{C.r}") + except (OSError, struct.error): pass + def cmd_chat(a): + # ATTACH: a running `coli serve` beats a private engine every time — the load + # (34-136 s) and the cache warmth survive between sessions. Explicit --attach + # wins; otherwise probe localhost quietly and use it if it's there. --no-attach + # forces the old behaviour. The probe costs ~1 ms when nothing is listening. + if not getattr(a,"no_attach",False): + base=getattr(a,"attach",None) or "http://127.0.0.1:8000" + mid=server_probe(base, getattr(a,"api_key",None)) + if mid: + banner(f"chat · {mid} · attached") + chat_attached(a, base, mid); return + if getattr(a,"attach",None): + sys.exit(f"--attach: no coli serve answering at {base} (start one with: coli serve --model )") need_model(a.model) banner(f"chat · {os.path.basename(a.model)} · ram {a.ram or '-'}GB · topp {a.topp or 'off'}") + kv_resume_notice(a.model) errlog=tempfile.NamedTemporaryFile(mode="w+", suffix=".log", delete=False) e=env_for(a); e["SERVE"]="1" # stderr -> PIPE, NOT stderr=errlog (file). On Windows/MinGW, pointing the @@ -719,7 +844,14 @@ def main(): pd=sub.add_parser("doctor",parents=[common]) pd.add_argument("--json",action="store_true",help="emit a versioned JSON report") pr=sub.add_parser("run", parents=[common]); pr.add_argument("prompt", nargs="*") - sub.add_parser("chat", parents=[common]) + pc=sub.add_parser("chat", parents=[common]) + pc.add_argument("--attach", nargs="?", const="http://127.0.0.1:8000", default=None, + help="chat against a running `coli serve` instead of spawning an engine " + "(keeps the model loaded and the expert cache warm across chat sessions). " + "Bare --attach probes localhost:8000.") + pc.add_argument("--no-attach", action="store_true", + help="never auto-attach, always spawn a private engine") + pc.add_argument("--api-key", default=os.environ.get("COLI_API_KEY")) ps=sub.add_parser("serve", parents=[common]) ps.add_argument("--host",default="127.0.0.1"); ps.add_argument("--port",type=int,default=8000) ps.add_argument("--model-id",default=os.environ.get("COLI_MODEL_ID","glm-5.2-colibri")) From 7eb239328d128c70bb5e939a89cf6fd8023528fb Mon Sep 17 00:00:00 2001 From: JustVugg Date: Fri, 17 Jul 2026 13:26:48 +0200 Subject: [PATCH 25/95] =?UTF-8?q?coli:=20serve=20pidfile=20+=20coli=20stop?= =?UTF-8?q?=20=E2=80=94=20one=20command=20to=20shut=20engine=20and=20serve?= =?UTF-8?q?r=20down?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No more manual pkill: cmd_serve writes a pidfile, cmd_stop finds the server (pidfile, then /proc cmdline) and its engine (comm glm/exe/olmoe with SERVE=1 in environ — the engine re-execs for OMP tuning so its comm is 'exe', which is why every 'pkill -x glm' in history killed nothing). SIGTERM, wait, SIGKILL. --dry-run lists targets without acting. First real use took down a live serve+engine pair cleanly and released 16 GB. Co-Authored-By: Claude Opus 4.8 (cherry picked from commit 41a872c331a2a0a8655699e0171c68dd2bcda186) --- c/coli | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/c/coli b/c/coli index 74f95de..0dd9de3 100755 --- a/c/coli +++ b/c/coli @@ -744,12 +744,65 @@ def cmd_chat(a): except Exception: pass print(f" {C.teal}goodbye{C.r} {C.dim}— the hummingbird returns to its nest{C.r} 🐦\n") +def serve_pidfile(port): return os.path.join(tempfile.gettempdir(), f"coli-serve-{port}.pid") + def cmd_serve(a): need_model(a.model) + # pidfile: cosi' `coli stop` spegne tutto con un comando, senza pkill a mano. + # EN: pidfile so `coli stop` can shut everything down without manual pkill. + try: + with open(serve_pidfile(a.port),"w") as f: f.write(f"{os.getpid()} {a.model}\n") + except OSError: pass from openai_server import serve - serve(a.model, a.host, a.port, a.model_id, a.api_key, - a.cap,a.ngen,GLM,env_for(a),a.cors_origin, - a.max_queue,a.queue_timeout,a.kv_slots) + try: + serve(a.model, a.host, a.port, a.model_id, a.api_key, + a.cap,a.ngen,GLM,env_for(a),a.cors_origin, + a.max_queue,a.queue_timeout,a.kv_slots) + finally: + try: os.unlink(serve_pidfile(a.port)) + except OSError: pass + +def cmd_stop(a): + """Shut down a running `coli serve` AND its engine — one command, no pkill. + The engine re-execs itself for OMP tuning, so its process is named `exe`, + not `glm`: every `pkill -x glm` in history silently killed nothing (that is + how two 17+5 GB ghost engines OOM'd this box on 2026-07-16). This finds the + real processes: the pidfile first, then /proc by cmdline/environ — only + processes that are demonstrably ours (SERVE=1 + our SNAP, or `coli serve` + in the command line).""" + banner("stop") + targets=[] # (pid, descrizione) + pf=serve_pidfile(a.port) + try: + pid=int(open(pf).read().split()[0]) + os.kill(pid,0); targets.append((pid,f"coli serve (pidfile, port {a.port})")) + except (OSError,ValueError,IndexError): pass + for pd in os.listdir("/proc"): + if not pd.isdigit(): continue + pid=int(pd) + try: + cmd=open(f"/proc/{pd}/cmdline","rb").read().replace(b"\0",b" ").decode("utf-8","replace") + if "coli" in cmd and " serve" in cmd and pid!=os.getpid(): + if not any(p==pid for p,_ in targets): targets.append((pid,"coli serve (cmdline)")) + comm=open(f"/proc/{pd}/comm").read().strip() + if comm in ("glm","exe","olmoe"): + env=open(f"/proc/{pd}/environ","rb").read().replace(b"\0",b"\n").decode("utf-8","replace") + if "SERVE=1" in env: targets.append((pid,f"engine `{comm}` (SERVE=1)")) + except (OSError,PermissionError): continue + if not targets: + print(f" nothing running — no serve on port {a.port}, no SERVE engines"); return + for pid,desc in targets: print(f" {'would stop' if a.dry_run else 'stopping'} {pid}: {desc}") + if a.dry_run: return + for pid,_ in targets: + try: os.kill(pid, signal.SIGTERM) + except OSError: pass + time.sleep(2.0) + for pid,_ in targets: + try: os.kill(pid, signal.SIGKILL); print(f" {pid}: forced (SIGKILL)") + except OSError: pass # gia' morto: bene + try: os.unlink(pf) + except OSError: pass + print(f" {C.grn}✓ stopped{C.r} — RAM released") def cmd_web(a): """serve + open the dashboard in the browser once the API answers.""" @@ -860,6 +913,8 @@ def main(): ps.add_argument("--max-queue",type=int,default=int(os.environ.get("COLI_MAX_QUEUE","8"))) ps.add_argument("--queue-timeout",type=float,default=float(os.environ.get("COLI_QUEUE_TIMEOUT","300"))) ps.add_argument("--kv-slots",type=int,default=int(os.environ.get("COLI_KV_SLOTS","1"))) + pst=sub.add_parser("stop", parents=[common], help="shut down a running coli serve and its engine") + pst.add_argument("--port",type=int,default=8000); pst.add_argument("--dry-run",action="store_true") pw=sub.add_parser("web", parents=[common], help="serve + open the dashboard in a browser") for arg,kw in (("--host",dict(default="127.0.0.1")),("--port",dict(type=int,default=8000)), ("--model-id",dict(default=os.environ.get("COLI_MODEL_ID","glm-5.2-colibri"))), @@ -883,7 +938,7 @@ def main(): pc.add_argument("--no-mtp",action="store_true",help="skip the MTP head (no speculative drafts)") a=ap.parse_args() handler={"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"doctor":cmd_doctor, - "run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"bench":cmd_bench, + "run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"stop":cmd_stop,"bench":cmd_bench, "convert":cmd_convert,"web":cmd_web}.get(a.cmd) if handler: sys.exit(handler(a) or 0) banner(); print(__doc__) From d5327e22520017056562ddf0c5f49578f55ad2ef Mon Sep 17 00:00:00 2001 From: JustVugg Date: Fri, 17 Jul 2026 13:53:20 +0200 Subject: [PATCH 26/95] =?UTF-8?q?omp:=20cap=20LLVM=20libomp=20idle=20spin?= =?UTF-8?q?=20=E2=80=94=20a=20parked=20engine=20must=20not=20burn=203000%?= =?UTF-8?q?=20CPU=20(#341)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hot-thread tuning sets OMP_WAIT_POLICY=active and caps libgomp's spin with GOMP_SPINCOUNT=200000. LLVM libomp reads NEITHER: under active policy it sets KMP_BLOCKTIME=infinite, so once the answer ends and the engine parks on stdin, the whole OMP team spins forever — ~100% per thread, the 3000% figure reported on FreeBSD 15.1 in #341 (clang/libomp is the default toolchain neighborhood there; the same applies to macOS builds). KMP_BLOCKTIME=200 keeps the team hot for 200 ms after each parallel region — plenty to bridge back-to-back per-expert matmuls — and then sleeps it. setenv with overwrite=0, so a user-provided KMP_BLOCKTIME still wins, and libgomp builds ignore the variable entirely: on GCC toolchains this commit is a no-op by construction. Not reproduced locally (this box is gcc/libgomp and idles clean); shipped as the mechanism-matching candidate fix with a request on the issue for the reporter to verify on dev and to name their OpenMP runtime (ldd | grep omp). Co-Authored-By: Claude Opus 4.8 --- c/glm.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/c/glm.c b/c/glm.c index 645be12..30d970a 100644 --- a/c/glm.c +++ b/c/glm.c @@ -6000,6 +6000,14 @@ int main(int argc, char **argv){ !getenv("COLI_CUDA") && !getenv("COLI_METAL")){ setenv("OMP_WAIT_POLICY","active",0); /* keep the team hot across the tiny per-expert matmul regions */ setenv("GOMP_SPINCOUNT","200000",0); /* spin briefly, then yield so long disk waits don't burn a core */ + /* LLVM libomp (clang builds: FreeBSD cc, macOS, some Linux setups) does not + * read GOMP_*: with OMP_WAIT_POLICY=active it sets KMP_BLOCKTIME=infinite, + * so the idle team SPINS FOREVER once generation ends — a serve-mode engine + * parked on stdin burns ~100% x nthreads (#341, measured 3000% on FreeBSD). + * 200 ms of blocktime keeps the team hot across back-to-back expert matmuls + * and lets it sleep at the prompt. libgomp ignores KMP_*; overwrite=0 keeps + * the user's own setting authoritative. */ + setenv("KMP_BLOCKTIME","200",0); setenv("OMP_PROC_BIND","close",0); /* pack the team onto adjacent cores for cache locality */ setenv("OMP_DYNAMIC","FALSE",0); /* fixed team size: no per-region thread-count churn */ setenv("COLI_OMP_TUNED","1",1); From 03e643f006c247f3cd8cf8be212792d3b70f7d6b Mon Sep 17 00:00:00 2001 From: LordMZTE Date: Fri, 17 Jul 2026 14:02:30 +0200 Subject: [PATCH 27/95] fix(flake): create flake.lock, use --set-default for COLI_ENGINE --- flake.lock | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 2 +- 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 flake.lock diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..9b788fa --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1784160687, + "narHash": "sha256-iYL/bixrb6FlHFu/gIuBYzq6c6lM5AAXsXNSWXtIgQc=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "4382ed2b7a6839d4280a9b386db49cbc5907414d", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-26.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix index 83c6de2..584a088 100644 --- a/flake.nix +++ b/flake.nix @@ -67,7 +67,7 @@ # `import openai_server` / `resource_plan` / `doctor` resolve. makeWrapper ${pythonEnv}/bin/python $out/bin/coli \ --add-flags "$out/lib/colibri/coli" \ - --set COLI_ENGINE "$out/lib/colibri/glm" \ + --set-default COLI_ENGINE "$out/lib/colibri/glm" \ --set PYTHONPATH "$out/lib/colibri:${pythonEnv}/${pkgs.python3.sitePackages}" runHook postInstall ''; From 411f237f9403c5e79933bae211321933afdaa805 Mon Sep 17 00:00:00 2001 From: woolcoxm <13604288+woolcoxm@users.noreply.github.com> Date: Fri, 17 Jul 2026 08:26:18 -0400 Subject: [PATCH 28/95] fix(warnings): silence two -Wall/-Wextra warnings on the MinGW build A clean 'make glm' on MinGW emitted exactly two warnings, both real: 1. compat.h:240 - ignoring pragma comment [-Wunknown-pragmas] #pragma comment(lib, "psapi.lib") is an MSVC directive; MinGW/GCC warns about it. Guarded with ifdef _MSC_VER - MinGW links psapi via -lpsapi (already in the Makefile), MSVC keeps the pragma. 2. glm.c:1210 - g_numa_nodes defined but not used [-Wunused-variable] g_numa_nodes is only read/written inside ifdef __linux__ blocks, so on every non-Linux build it is a static that is never used. Moved the definition under the same __linux__ guard; nothing references it off-Linux. Verified: rm -f *.o glm.exe && make glm -> 0 warnings, 0 errors. --- c/compat.h | 4 +++- c/glm.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/c/compat.h b/c/compat.h index 82de8b6..78b52cb 100644 --- a/c/compat.h +++ b/c/compat.h @@ -237,7 +237,9 @@ static inline int compat_rename(const char *old, const char *new){ /* --- rss_gb: getrusage -> GetProcessMemoryInfo --- * ru_maxrss in KB (come Linux): rss_gb() divide per 1e6 → GB corretti. */ #include -#pragma comment(lib, "psapi.lib") +#ifdef _MSC_VER +#pragma comment(lib, "psapi.lib") /* MSVC: link psapi; MinGW/GCC uses -lpsapi */ +#endif struct rusage { long ru_maxrss; }; #define RUSAGE_SELF 0 static inline int getrusage(int who, struct rusage *r){ diff --git a/c/glm.c b/c/glm.c index 30d970a..40a1690 100644 --- a/c/glm.c +++ b/c/glm.c @@ -1207,7 +1207,9 @@ static int g_disk_split=0; /* DISK_SPLIT=1: contatori che spezzano i DISK LOAD ( * 10x (#82) — hence per-region mbind here and nothing else. Raw syscall, no libnuma * dependency; MPOL_MF_MOVE migrates pages of reused heap chunks too. Linux-only, * silent no-op elsewhere or on single-node hosts. */ -static int g_numa_nodes=0; +#ifdef __linux__ +static int g_numa_nodes=0; /* only touched under __linux__; off-Linux NUMA is a no-op */ +#endif static void numa_slab_bind(void *p, size_t n){ #ifdef __linux__ if(g_numa_nodes<2 || !p || !n) return; From 5e2be61a2c49ba9b10f6710853a85a545e1ea448 Mon Sep 17 00:00:00 2001 From: woolcoxm <13604288+woolcoxm@users.noreply.github.com> Date: Fri, 17 Jul 2026 08:27:51 -0400 Subject: [PATCH 29/95] fix(test): make test_stops build on Windows (mkdtemp compat shim) test_stops.c uses POSIX mkdtemp() to make a scratch dir in the CWD, but MinGW-w64 does not declare mkdtemp, so the test failed to compile on the Windows job - and only there: tests/test_stops.c:73:9: error: implicit declaration of function 'mkdtemp'; did you mean 'mktemp'? [-Wimplicit-function-declaration] make: *** [Makefile:318: tests/test_stops.exe] Error 1 That halted `make test` at the C-test stage on Windows (test_uring is correctly Linux-only, so test_stops was the only blocker). Added a compat_mkdtemp shim to compat.h, following the file's existing convention (every platform difference lives there; the .c stays clean): _mktemp fills the trailing X's in place (same contract as mkdtemp), then _mkdir creates the directory. Also added for _mkdir. On Linux compat.h is a complete no-op, so POSIX mkdtemp is untouched there. Verified: test_stops builds clean on MinGW and all 5 sub-cases pass: tokenizer special-flag parsing, config/generation_config eos union, no-generation_config fallback, both-configs-mutilated tokenizer sweep, and the T=NULL validation path. --- c/compat.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/c/compat.h b/c/compat.h index 82de8b6..c477170 100644 --- a/c/compat.h +++ b/c/compat.h @@ -80,6 +80,7 @@ static inline int compat_open_direct(const char *path){ #endif #include #include +#include /* _mkdir (for the mkdtemp shim below) */ #include #include #include @@ -304,6 +305,19 @@ static inline int compat_setenv(const char *name, const char *value, int overwri } #define setenv(name,value,overwrite) compat_setenv(name,value,overwrite) +/* --- mkdtemp -> _mktemp + _mkdir (POSIX mkdtemp assente su Windows) --- + * Test binaries (test_stops.c) create a scratch dir in the CWD via a + * "name_XXXXXX" template; POSIX mkdtemp fills the X's and mkdirs 0700. The + * Windows CRT has _mktemp (in-place, same XXXXXX contract) so we compose it. + * Returns the template pointer on success, NULL on failure — matching POSIX. */ +static inline char *compat_mkdtemp(char *tmpl){ + if(!tmpl) return NULL; + if(!_mktemp(tmpl)) return NULL; /* fills the trailing X's in place */ + if(_mkdir(tmpl) != 0) return NULL; /* EEXIST is impossible post-_mktemp */ + return tmpl; +} +#define mkdtemp(tmpl) compat_mkdtemp(tmpl) + #endif /* _WIN32 */ /* --- compat_aligned_free su piattaforme diverse da Windows --- From 099e0c13c4ef3e675b509bbae4504fee818c8fca Mon Sep 17 00:00:00 2001 From: woolcoxm <13604288+woolcoxm@users.noreply.github.com> Date: Fri, 17 Jul 2026 08:28:41 -0400 Subject: [PATCH 30/95] =?UTF-8?q?docs(env):=20refresh=20ENVIRONMENT.md=20?= =?UTF-8?q?=E2=80=94=20document=20all=2034=20drifted=20env=20vars?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ENVIRONMENT.md says it is "generated from source by scanning every getenv() site", but it had drifted far behind the code. Reconciling the doc against the C sources (the MAINTAINING-DOCS.md procedure) found 34 engine env vars read by the code but undocumented, and nothing stale. Added (defaults/effects taken from source, per the maintenance rules - nothing invented): Performance/tuning (11): COLI_NUMA, PILOT_TWO, COUPLE/COUPLE_K/COUPLE_D, ROUTE_TRACE, COLI_NO_FUSED_PAIR, DISK_SPLIT, I4S, SPEC_PIN, COLI_RAM_OVERCOMMIT CUDA (16): COLI_CUDA_ATTN_SHARD, COLI_CUDA_PIPE/_PIPE_SHARD/_PIPE_S_MIN, COLI_CUDA_MTP, COLI_CUDA_ASYNC, COLI_CUDA_DUAL_PROJ, COLI_CUDA_W4_PACKED, COLI_CUDA_TC_INT4/_TC_MIN_ROWS/_TC_W4A16/_TC_W4A16_MIN, COLI_CUDA_SHARED_W4A16/_SHARED_W4A16_MIN_ROWS, COLI_METAL_UNTRACKED Advanced/debug (7): SCHEMA, EXPERT_BUDGET/_EXPERIMENTAL, TOKENS, SCORE_PREFIX, REPIN_VERBOSE, PPL (olmoe-only), COLI_PROMPT (CLI section) Also bumped the "Generated from" line to dev @ d5327e2 and noted the scan now covers olmoe.c, backend_cuda.cu, backend_metal.mm (not just glm.c). Verified: the code-vs-doc diff is now empty - all 111 distinct C env vars are documented. The reverse diff (doc vars not in C code) is 14 entries, all legitimate: 11 Python-side vars correctly in the Server/CLI section, plus 3 prose constants (IOSQE_ASYNC, O_DIRECT, the VAR format word). --- docs/ENVIRONMENT.md | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 4bf2d55..535288a 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -2,7 +2,7 @@ Reference for the environment variables read by the colibrì engine. -**Generated from `upstream/dev @ 6d3ed7e`** by scanning every `getenv()` site in `c/glm.c`. Defaults and behavior are taken from the source; see [MAINTAINING-DOCS.md](MAINTAINING-DOCS.md) to regenerate this after the code changes. +**Generated from `dev @ d5327e2`** by scanning every `getenv()` site in `c/glm.c` and the other C sources (`c/olmoe.c`, `c/backend_cuda.cu`, `c/backend_metal.mm`). Defaults and behavior are taken from the source; see [MAINTAINING-DOCS.md](MAINTAINING-DOCS.md) to regenerate this after the code changes. ## Which program reads these? @@ -43,6 +43,7 @@ Format: `VAR` — default — effect. | `URING` | `0` (off) | Linux-only queued expert I/O. `URING=1` implies `PIPE=1`, forces cold reads through io-wq (`IOSQE_ASYNC`), replaces blocking loader pthreads and spin waits with batched SQEs/CQEs, and batches `PILOT_REAL` loads on a separate ring. Use `DIRECT=1` for cold NVMe to avoid page-cache copy/readahead limits. Fails clearly if the kernel denies io_uring; incompatible with `COLI_MMAP=1`. | | `DIRECT` | `0` (off) | Use `O_DIRECT`/unbuffered reads for expert slabs. Helps sustained NVMe; keeps the zero-copy GPU path. | | `COLI_NO_OMP_TUNE` | off | **Kill-switch** for the OpenMP hot-thread tuning (`OMP_WAIT_POLICY=active` spin + proc-bind). Set `=1` when the CPU is mostly waiting on the GPU (Metal) so spin doesn't steal the shared power budget. | +| `COLI_NUMA` | off (Linux only) | `COLI_NUMA=1` interleaves expert slabs across NUMA nodes via `mbind` (raw syscall, no libnuma). Helps multi-socket hosts (+7–40% expert matmul); silent no-op on single-node or non-Linux. | | `MLOCK` | `-1` (auto: on for macOS) | Wire the streamed expert cache into physical RAM (`mlock`) to dodge the memory compressor. `0` off, `1` force. | | `CAP_RAISE` | `1` (on) | Let the engine raise the expert-cache cap above `topk` when RAM allows (bigger batches). `0` fixes the cap. | | `PREFETCH` | `0` | Prefetch depth for streamed experts. | @@ -54,16 +55,26 @@ Format: `VAR` — default — effect. | `PILOT` | `0` (off) | Router-piloted cross-layer expert prefetch. | | `PILOT_REAL` | `0` (off) | Value-preserving real cross-layer prefetch loads (`PILOT_REAL=1` opts in). | | `PILOT_K` | `6` if `PILOT_REAL` else `8` | Number of experts the pilot prefetches per step. | +| `PILOT_TWO` | `0` (off) | Two-step shared-expert-corrected router prediction for the pilot. | +| `COUPLE` | unset | Path to a coupling-score file driving cross-layer expert prefetch (#176). When set, `couple_load` reads it. | +| `COUPLE_K` | `8` | Top-K coupled experts per layer when `COUPLE` is set. | +| `COUPLE_D` | `1` | Coupling lookahead depth (`1` or `2`) when `COUPLE` is set. | | `CACHE_ROUTE` | `0` (off) | Opt-in max-rank cache-aware MoE routing (pin∪LRU prefer within top-M). See [CACHE_ROUTE.md](CACHE_ROUTE.md). | | `ROUTE_J` | `2` | Sacred top ranks always taken when `CACHE_ROUTE=1`. | | `ROUTE_M` | `12` | Max-rank window for resident preference when `CACHE_ROUTE=1`. | | `ROUTE_P` | `0` | Cumulative mass window for CACHE_ROUTE (`0` = fixed M). | | `ROUTE_ALPHA` | `1` | Scale gate mass of substituted experts before renorm (`1` = off). | | `ROUTE_AGREE` | auto | Overlap% + KL vs true top-K; auto-on when `CACHE_ROUTE=1`. | +| `ROUTE_TRACE` | unset | If set to a path, logs every routing decision there (testing/analysis). | | `ABSORB` | `-1` (auto: absorbed for S≤4) | MLA attention absorption mode. | | `IDOT` | `1` | Integer dot-product kernel. `IDOT=0` uses exact f32 kernels (for A/B numerical checks). | | `COLI_POLICY` | `quality` | Resource policy: `quality`, `balanced`, or `experimental-fast`. | | `PROF` | `0` (off) | Performance profile: a startup header (machine + effective config), then per run — or per turn in serve mode, on stderr — forward-latency percentiles (p50/p90/p99/max), expert-I/O totals and cache-tier fill, phase shares of wall time, and a verdict naming the knob most likely to help on this machine. Output is additive; `PROF` unset changes nothing. | +| `COLI_NO_FUSED_PAIR` | `0` (off) | `=1` disables the fused-pair matmul kernel. | +| `DISK_SPLIT` | `0` (off) | `=1` splits the reported disk-load time across the draft/absorb/forward phases in stats. | +| `I4S` | unset | Engage the int4 `IDOT` kernel only for batch `S>=` (testing). | +| `SPEC_PIN` | `1` (on) | Speculation gate mode. `0` reverts to the legacy S-dependent speculation gates (#163). | +| `COLI_RAM_OVERCOMMIT` | off | `=1` overrides the "projected peak > MemAvailable → exit(2)" guard so a run that risks kernel OOM-kill is allowed to proceed. | --- @@ -77,7 +88,22 @@ Format: `VAR` — default — effect. | `CUDA_EXPERT_GB` | `0` | VRAM budget (GB) for caching experts on the GPU. | | `CUDA_RELEASE_HOST` | auto (`1` if >1 device) | Release host-side copies after upload. | | `COLI_CUDA_ATTN` | off | Run S≤4 attention on the GPU. | +| `COLI_CUDA_ATTN_SHARD` | off | `=1` splits KV-b heads across devices during attention load (multi-GPU). | | `COLI_CUDA_PROFILE` | off | Emit CUDA timing. | +| `COLI_CUDA_PIPE` | `0` (off) | `1` engages the multi-step attention pipeline; `2` enables the pipe2 path. | +| `COLI_CUDA_PIPE_SHARD` | off | `=1` runs the multi-device P2P head-shard attention path (opt-in for NVLink topologies; serializes ~95 MB/layer over a star PCIe topology). | +| `COLI_CUDA_PIPE_S_MIN` | `1` single-GPU, `8` multi-GPU | Minimum prefill batch S to engage the pipe2 CUDA path. | +| `COLI_CUDA_MTP` | `0` (off) | `=1` opts into MTP speculation under CUDA (off by default: cold streaming experts run on CPU where the fused-pair/IDOT kernels diverge in FP order, collapsing draft acceptance, #163/#292). | +| `COLI_CUDA_ASYNC` | on | `=0` forces synchronous `cudaMemcpy` instead of async + pinned host staging. | +| `COLI_CUDA_DUAL_PROJ` | on | `=0` issues gate+up as two separate launches instead of one fused `grouped_hidden_w4_dual`. | +| `COLI_CUDA_W4_PACKED` | on | `=0` disables the grouped packed-int4 path. | +| `COLI_CUDA_TC_INT4` | off | `=1` uses the W4A4 WMMA Tensor Core path (when all expert tensors are int4 and dims divide). | +| `COLI_CUDA_TC_MIN_ROWS` | `8` | Min rows-per-expert to engage the W4A4 Tensor Core path. | +| `COLI_CUDA_TC_W4A16` | off | `=1` uses the lossless W4A16 Tensor Core path (compute capability ≥7). | +| `COLI_CUDA_TC_W4A16_MIN` | `16` | Per-expert row threshold above which W4A16 TC tiles dispatch (smaller batches fall back to the naive kernel). | +| `COLI_CUDA_SHARED_W4A16` | off | `=1` uploads shared-expert weights and runs the shared-MLP W4A16 Tensor Core kernel. | +| `COLI_CUDA_SHARED_W4A16_MIN_ROWS` | `32` | Min row count to engage the shared-MLP W4A16 kernel. | +| `COLI_METAL_UNTRACKED` | off (Metal only) | `=1` sets `MTLResourceHazardTrackingModeUntracked` on Metal buffers (reduces hazard-tracking overhead). | --- @@ -89,8 +115,11 @@ These are for testing, benchmarking, or internal use — not part of the everyda |---|---|---| | `SPEC` | `1` | Speculative decoding on/off. | | `DRAFT` | `-1` (auto: 3 with MTP, else 0) | Number of speculative draft tokens per step. | -| `GRAMMAR` | unset | Path to a GBNF grammar file to constrain generation. | +| `GRAMMAR` | unset | Path to a GBNF grammar file to constrain generation. Takes precedence over `SCHEMA`. | +| `SCHEMA` | unset | Path to a JSON-Schema file compiled to GBNF to constrain generation (consulted only when `GRAMMAR` is empty). | | `GRAMMAR_DRAFT` | unset | Max grammar-forced draft span length. | +| `EXPERT_BUDGET` | `0` (off) | Cap experts loaded per layer (MoE-Spec). **Quarantined:** silently forced to `0` unless `EXPERT_BUDGET_EXPERIMENTAL` is set — every tested value is either no faster or incoherent (issue #303). | +| `EXPERT_BUDGET_EXPERIMENTAL` | unset | Setting it (any value) allows `EXPERT_BUDGET>0` to actually take effect (expect garbage, #294). | | `DSA` | on | Dynamic Sparse Attention indexer. `DSA=0` disables. | | `DSA_FORCE` | `0` | Force the DSA path on. | | `DSA_TOPK` | model value | Override the DSA index top-k (testing). | @@ -101,11 +130,15 @@ These are for testing, benchmarking, or internal use — not part of the everyda | `PIN_FILL` | `0` | Fill the pinned store even without usage data. | | `MTP_DEBUG` / `MTP_PRENORM` / `MTP_SWAP` | off | MTP head debugging / ablations. | | `STATS` | unset | Write an expert-usage histogram to `STATS=` at end of run. | +| `TOKENS` | unset | If set, dumps generated token ids to stderr for A/B comparison. | | `SCORE` | unset | Scoring/eval mode over `SCORE=`. | +| `SCORE_PREFIX` | on | If unset or `≠0`, prepends `[gMASK]` to scoring contexts (GLM-family only). | +| `REPIN_VERBOSE` | off | If set, prints per-swap `[REPIN]` diagnostics during VRAM repin. | | `REF` / `REF_FORCE` | `ref_glm.json` | Reference-output comparison mode. | | `REPLAY` | unset | Replay mode. | | `TF` | unset | Teacher-forcing mode. | | `CHAT_TEMPLATE` | `1` | Apply the GLM chat template (`0` = raw prompt). | +| `PPL` | off (`olmoe.c` only) | `PPL=1` enters teacher-forced NLL/perplexity meter mode in the OLMoE sister engine. | --- @@ -136,7 +169,7 @@ These are read by the Python programs (not the `glm` engine), so they don't appe - `SNAP` — model snapshot directory (required by `glm`; set from `--model`). - `SERVE`, `SERVE_BATCH` — select serve / batched-serve mode. -- `PROMPT` — one-shot text mode. +- `PROMPT` — one-shot text mode (the engine also honors `COLI_PROMPT`, preferred cross-platform; `PROMPT` is ignored on Windows if it contains cmd.exe `$`-metacharacters). - `COLI_OMP_TUNED` — internal sentinel guarding the OMP re-exec (see `COLI_NO_OMP_TUNE`); not user-facing. --- From ba00889fb9c4842efceab1ca74f1f4513737503b Mon Sep 17 00:00:00 2001 From: woolcoxm <13604288+woolcoxm@users.noreply.github.com> Date: Fri, 17 Jul 2026 09:01:23 -0400 Subject: [PATCH 31/95] =?UTF-8?q?sampling:=20partial=20top-p=20select=20in?= =?UTF-8?q?=20dist=5Fbuild=20=E2=80=94=20O(V)=20heapify=20+=20k=20pops,=20?= =?UTF-8?q?not=20O(V=20log=20V)=20qsort=20(#335)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dist_build() sorted the entire 151936-entry vocab by probability (qsort) on every sampled token whenever 0 < g_nuc < 1 — the serve default — and again per draft position under rejection sampling. Measured cost: 5.6-8.0 ms/call; the actual work is finding the few-hundred-token head whose cumulative mass reaches g_nuc. Replace the full qsort + linear scan with a max-heap partial select: - Floyd heapify g_pidx over V by descending g_pbuf prob (O(V), cache-friendly) - pop winners to the array's high end until cum >= g_nuc (k * O(log V)) - the remaining heap prefix IS the tail -> zero it, renormalize the head Winners land in g_pidx[out..V-1] in descending order, so s2 accumulates in the same order as before -> head is unchanged on tie-free shapes (ties were already unspecified under the unstable qsort). All four dist_build/dist_sample contract properties hold: g_pbuf stays id-indexed, g_pidx stays internal, the tail is fully zeroed, the head renormalizes to 1. No API change, no caller change, no new globals. c/tests/test_topp.c (new): drives the REAL dist_build via the include-glm.c pattern against an independent double-precision reimplementation of the OLD algorithm. 123 cases: 6 sizes (1..1519) x 5 shapes (uniform/peaked/geometric/ plateau/sharptail) x 4 nuc values, plus the g_nuc>=1 guard-off paths and V=1. Tie-free shapes compare head values within 1e-6 rel (float vs double renorm noise); tie shapes compare value-multisets. No scratch files -> builds clean on Windows MinGW without the unmerged mkdtemp shim (#352). --- c/Makefile | 5 +- c/glm.c | 38 ++++-- c/tests/test_topp.c | 280 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 312 insertions(+), 11 deletions(-) create mode 100644 c/tests/test_topp.c diff --git a/c/Makefile b/c/Makefile index 27902cb..395e136 100644 --- a/c/Makefile +++ b/c/Makefile @@ -166,7 +166,7 @@ else PYTHON ?= python3 endif CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) ifneq (,$(LINUX)) TEST_BINS += tests/test_uring$(EXE) endif @@ -317,6 +317,9 @@ tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c glm.c st.h uring.h json.h t tests/test_stops$(EXE): tests/test_stops.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_topp$(EXE): tests/test_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) diff --git a/c/glm.c b/c/glm.c index 30d970a..add5a71 100644 --- a/c/glm.c +++ b/c/glm.c @@ -4186,10 +4186,21 @@ static uint64_t g_rng=0x9E3779B97F4A7C15ULL; static inline double rndu(void){ g_rng^=g_rng<<13; g_rng^=g_rng>>7; g_rng^=g_rng<<17; return (double)(g_rng>>11)*(1.0/9007199254740992.0); } static float *g_pbuf=NULL; static int *g_pidx=NULL; /* buffer riusati (decode single-thread) */ -static int cmp_pdesc(const void *a,const void *b){ - float pa=g_pbuf[*(const int*)a], pb=g_pbuf[*(const int*)b]; - return papb ? -1 : 0; } -/* costruisce in g_pbuf la distribuzione target: softmax(lo/temp) troncata a top-p g_nuc */ +/* sift-down su max-heap in h[0..n), chiave = g_pbuf[h[i]] (#335: partial top-p select). + * Versione "a buco": porta il valore di radice e lo deposita solo alla fine, cosi' + * heapify e' O(V) e ogni pop e' O(log n) senza qsort sull'intero vocabolario. */ +static void topp_siftdown(int *h, int n, int i){ + int iv=h[i]; float kv=g_pbuf[iv]; + for(;;){ int l=2*i+1; + if(l>=n) break; /* foglia */ + int b=l; if(l+1g_pbuf[h[l]]) b=l+1; /* figlio maggiore */ + if(g_pbuf[h[b]]<=kv) break; /* nessun figlio supera la radice -> ferma */ + h[i]=h[b]; i=b; } + h[i]=iv; +} +/* costruisce in g_pbuf la distribuzione target: softmax(lo/temp) troncata a top-p g_nuc. + * Invariante per dist_sample: g_pbuf resta INDICIZZATO per token-id (mai riordinato); + * la coda troncata va AZZERATA in g_pbuf (dist_sample la legge direttamente per id). */ static void dist_build(const float *lo, int V){ if(!g_pbuf){ g_pbuf=falloc(V); g_pidx=malloc(V*sizeof(int)); } float mx=lo[0]; for(int i=1;imx) mx=lo[i]; @@ -4198,12 +4209,19 @@ static void dist_build(const float *lo, int V){ for(int i=0;i0 && g_nuc<1.f){ for(int i=0;i=g_nuc){ keep=i+1; break; } } - double s2=0; for(int i=keep;i=0;i--) topp_siftdown(g_pidx,V,i); /* heapify O(V) */ + /* pop verso la coda: i vincitori (testa top-p) cadono in g_pidx[out..V-1] in ordine + * DECRESCENTE, come il vecchio qsort, quindi s2 accumula nello stesso ordine -> + * head bit-identical sui casi senza pareggi (i pareggi erano gia' non specificati + * sotto il qsort instabile e restano tali). Il prefisso g_pidx[0..out-1) e' la coda. */ + double s2=0, cum=0; int out=V; + do{ int root=g_pidx[0]; /* massimo corrente */ + g_pidx[0]=g_pidx[--out]; g_pidx[out]=root; /* sposta il max in coda */ + s2+=g_pbuf[root]; cum+=g_pbuf[root]; + if(out>0) topp_siftdown(g_pidx,out,0); + } while(cum0); + for(int i=0;i=0 -> quel token e' escluso (rinormalizzando al volo) */ diff --git a/c/tests/test_topp.c b/c/tests/test_topp.c new file mode 100644 index 0000000..9276507 --- /dev/null +++ b/c/tests/test_topp.c @@ -0,0 +1,280 @@ +/* Top-p (nucleus) truncation in dist_build: the partial-select rewrite (#335) must be + * indistinguishable from the old full-vocab qsort for every shape dist_sample can see. + * + * Why this test exists (#335): dist_build() previously qsort-ed the entire 151936-entry + * vocab on every sampled token to find the few-hundred-token head whose cumulative mass + * reaches g_nuc. It now heapifies (O(V)) and pops only the head (k * O(log V)). The win + * is structural; the risk is a silent sampling-distribution change, because the contract + * is subtle: + * + * dist_sample() iterates g_pbuf[0..V-1] BY TOKEN ID and sums probabilities directly. + * So dist_build MUST leave g_pbuf indexed by id (never reordered) AND must zero every + * truncated tail entry -- merely excluding the tail from the head would leave mass on + * it and the sampled distribution would drift with no crash and no error. + * + * Strategy: drive the REAL dist_build (via the test_stops.c include-glm.c pattern) on a + * sweep of distributions and g_nuc values, and compare against an INDEPENDENT reference + * that re-implements the OLD algorithm (full qsort + zero-tail + renorm) in double on a + * private buffer. On shapes with no ties the renormalized head must be BIT-IDENTICAL to + * the reference (the issue's stated invariant: s2 accumulates in the same descending + * order). On tie shapes, where the unstable qsort already left ordering unspecified, we + * check multiset equality instead. Every shape also checks: exact-zero tails, head sums + * to 1.0, and a sane keep-count. + * + * No scratch files: the test runs entirely in memory (no mkdtemp), so it builds clean on + * the Windows MinGW CI job without the unmerged compat shim (#352). */ +#define main coli_glm_main_unused +#include "../glm.c" +#undef main + +#include + +static int g_nfails = 0; + +/* pointer set by ref_build so cmp_ref_desc can read the current reference buffer + * (the qsort comparator gets no user-data argument in C). */ +static const double *g_ref_p = NULL; + +#define FAIL(fmt, ...) do { \ + fprintf(stderr, " FAIL [%s V=%d nuc=%.3f shape=%s]: " fmt "\n", \ + label, V, nuc, shape_name, ##__VA_ARGS__); \ + g_nfails++; \ + return; \ +} while (0) + +/* ---- independent reference: the OLD algorithm, in double, on a private buffer ------- */ +/* Stable qsort by descending probability (ties broken by ascending index, which makes + * the reference deterministic regardless of the production comparator). */ +static int cmp_ref_desc(const void *a, const void *b){ + double pa = ((const double *)g_ref_p)[*(const int*)a]; + double pb = ((const double *)g_ref_p)[*(const int*)b]; + if (pa < pb) return 1; + if (pa > pb) return -1; + /* tie -> lower index first (stable, unlike the production comparator) */ + return *(const int*)a - *(const int*)b; +} + +/* Build the reference distribution into out[0..V-1] (indexed by token id), mirroring the + * old dist_build: softmax(lo/temp) truncated to top-p nuc, tail zeroed, head renormalized. + * Returns the keep-count through *keep_out. */ +static void ref_build(const float *lo, int V, double temp, double nuc, + double *out, int *pidx, int *keep_out){ + double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i]; + double s = 0, invt = 1.0 / (temp > 1e-4 ? temp : 1e-4); + for (int i = 0; i < V; i++){ out[i] = exp((lo[i]-mx)*invt); s += out[i]; } + for (int i = 0; i < V; i++) out[i] /= s; + + if (nuc > 0 && nuc < 1.0){ + for (int i = 0; i < V; i++) pidx[i] = i; + qsort(pidx, V, sizeof(int), cmp_ref_desc); + double cum = 0; int keep = V; + for (int i = 0; i < V; i++){ cum += out[pidx[i]]; if (cum >= nuc){ keep = i+1; break; } } + double s2 = 0; + for (int i = keep; i < V; i++) out[pidx[i]] = 0; + for (int i = 0; i < keep; i++) s2 += out[pidx[i]]; + for (int i = 0; i < keep; i++) out[pidx[i]] /= s2; + *keep_out = keep; + } else { + *keep_out = V; + } +} + +/* count how many production g_pbuf entries are non-zero == the head size */ +static int head_count(int V){ + int n = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) n++; return n; +} + +/* Run one case: load logits into g_pbuf via the real dist_build, compare to reference. + * shape_name is for diagnostics only. */ +static void check_case(const char *label, int V, double nuc, const char *shape_name, + const float *lo){ + /* reference on a private buffer */ + double *ref = malloc((size_t)V * sizeof(double)); + int *ridx = malloc((size_t)V * sizeof(int)); + int ref_keep = 0; + g_ref_p = ref; /* cmp_ref_desc reads this */ + ref_build(lo, V, g_temp, nuc, ref, ridx, &ref_keep); + + /* production: drive the real dist_build (writes the global g_pbuf) */ + g_nuc = (float)nuc; + dist_build(lo, V); + + int got_keep = head_count(V); + + /* 1. keep-count must match the reference exactly. The partial select and the old + * qsort keep the same NUMBER of tokens by construction (same cumulative-mass rule); + * a count divergence is a real bug, not a tie artifact. */ + if (got_keep != ref_keep) + FAIL("keep-count mismatch: got %d, ref %d", got_keep, ref_keep); + + /* 2. Detect ties across the WHOLE pre-truncation distribution, not just the kept set. + * A tie at the head/tail boundary makes which-side-a-token-lands-on interchangeable: + * both algorithms keep the right count but may keep different MEMBERS. So any input + * with a duplicated softmax value needs the relaxed multiset comparison below. We + * detect this on the reference softmax (pre-truncation) by sorting all V values. */ + int has_ties = 0; + { + double *all = malloc((size_t)V * sizeof(double)); + /* reconstruct the pre-truncation softmax the same way ref_build does */ + double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i]; + double s = 0, invt = 1.0 / (g_temp > 1e-4 ? g_temp : 1e-4); + for (int i = 0; i < V; i++){ all[i] = exp((lo[i]-mx)*invt); s += all[i]; } + for (int i = 0; i < V; i++) all[i] /= s; + for (int a = 1; a < V; a++){ double k = all[a]; int b = a-1; + while (b >= 0 && all[b] > k){ all[b+1] = all[b]; b--; } all[b+1] = k; } + for (int a = 1; a < V; a++) if (all[a] == all[a-1]){ has_ties = 1; break; } + free(all); + } + + if (has_ties){ + /* Multiset equality of the non-zero (head) values. Ties make membership + * interchangeable, so we compare sorted value-multisets, not id-aligned values. + * Tolerance is 1e-6 relative -- the engine uses float arithmetic, the reference + * double, so sub-ULP noise is expected (matches test_i4_grouped.c's convention). */ + double *got = malloc((size_t)ref_keep * sizeof(double)); + int gm = 0; + for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) got[gm++] = (double)g_pbuf[i]; + if (gm != ref_keep) + FAIL("tie-shape head size mismatch: got %d non-zero, ref %d", gm, ref_keep); + for (int a = 1; a < gm; a++){ double k = got[a]; int b = a-1; + while (b >= 0 && got[b] > k){ got[b+1] = got[b]; b--; } got[b+1] = k; } + double *rsort = malloc((size_t)ref_keep * sizeof(double)); + int rm = 0; + for (int i = 0; i < V; i++) if (ref[i] != 0.0) rsort[rm++] = ref[i]; + for (int a = 1; a < rm; a++){ double k = rsort[a]; int b = a-1; + while (b >= 0 && rsort[b] > k){ rsort[b+1] = rsort[b]; b--; } rsort[b+1] = k; } + int mm = 0; double worst = 0; + for (int i = 0; i < gm; i++){ + double d = fabs(got[i] - rsort[i]); + double rel = rsort[i] > 1e-30 ? d / rsort[i] : d; + if (rel > worst) worst = rel; + if (rel > 1e-6) mm++; + } + free(got); free(rsort); + if (mm) FAIL("tie-shape multiset mismatch: %d/%d head values differ beyond 1e-6 rel (worst %.3g)", + mm, ref_keep, worst); + } else { + /* No ties anywhere: membership is forced, so compare id-aligned head values. The + * engine computes in float (g_pbuf /= (float)s2) while the reference uses double, + * so the comparison is relative-tolerance (1e-6), not bit-exact -- the partial + * select and qsort accumulate s2 in the same descending order, so any difference + * is pure float-rounding noise, not an ordering bug. */ + int bad = 0; int first_id = -1; float gv = 0, rv = 0; double worst = 0; + for (int i = 0; i < V; i++){ + if (ref[i] == 0.0) continue; /* tail */ + float want = (float)ref[i]; + double d = fabs((double)g_pbuf[i] - (double)want); + double rel = fabs((double)want) > 1e-30 ? d / fabs((double)want) : d; + if (rel > worst) worst = rel; + if (rel > 1e-6){ + bad++; if (first_id < 0){ first_id = i; gv = g_pbuf[i]; rv = want; } + if (bad > 3) break; + } + } + if (bad) + FAIL("head not within 1e-6 rel of reference: %d entries differ (first id %d: got %.9g want %.9g, worst %.3g)", + bad, first_id, (double)gv, (double)rv, worst); + } + + /* 3. head must renormalize to 1.0 (within float epsilon) */ + double sum = 0; for (int i = 0; i < V; i++) sum += g_pbuf[i]; + if (fabs(sum - 1.0) > 1e-5) + FAIL("head does not sum to 1.0: sum=%.12g (keep=%d)", sum, got_keep); + + free(ref); free(ridx); + printf(" ok [V=%d nuc=%.3f shape=%s keep=%d%s sum=%.10f]\n", + V, nuc, shape_name, got_keep, has_ties ? " (ties)" : "", sum); +} +#undef FAIL + +/* deterministic xorshift32 RNG (matches the test_i4_grouped.c convention) */ +static uint32_t rng_state = 0x12345678u; +static uint32_t xr(void){ rng_state ^= rng_state << 13; rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; return rng_state; } +static double frand(void){ return (xr() >> 8) * (1.0 / 16777216.0); } /* [0,1) */ + +/* fill logits for a given shape. Shapes chosen to stress the comparator and the head/tail + * boundary differently. */ +static void fill_shape(float *lo, int V, int shape){ + switch (shape){ + case 0: /* uniform -> every token equal probability -> massive tie plateau */ + for (int i = 0; i < V; i++) lo[i] = 0.f; break; + case 1: /* peaked: one dominant token, rest small and distinct (no ties). + * The fixed hot-spots are clamped to V-1 so small V (incl. V=1) doesn't + * write out of bounds and corrupt heap metadata on the later free(lo). */ + for (int i = 0; i < V; i++) lo[i] = (float)(-1.0 - frand()*4.0); + lo[0] = 3.f; lo[V/3 comparator tie handling */ + for (int i = 0; i < V; i++) lo[i] = (float)(-(double)(i / 7)); /* 7-wide plateaus */ + break; + case 4: /* sharp-tail: a few hot, then a long flat floor (small tie at the floor). + * Hot count is min(12,V) so V<12 (incl. V=1) stays in bounds. */ + for (int i = 0; i < V; i++) lo[i] = -8.f; + { int hot = V<12 ? V : 12; for (int i = 0; i < hot; i++) lo[i] = (float)(2.0 - frand()); } break; + } +} + +int main(void){ + /* sizes: small for exhaustive tie detection up to near-production scale */ + int sizes[] = {1, 2, 8, 64, 257, 1519}; /* 1519 ~= V/100 of GLM-5.2 */ + double nucs[] = {0.001, 0.5, 0.9, 0.999}; /* tight -> almost-everything */ + int n_shapes = 5; + + /* temperature used by dist_build: pick a normal serving value */ + g_temp = 0.7f; + + int cases = 0; + for (size_t si = 0; si < sizeof(sizes)/sizeof(sizes[0]); si++){ + int V = sizes[si]; + /* dist_build allocates g_pbuf/g_pidx ONCE and reuses them (single-V invariant in + * real serving, where V is the constant model vocab). This sweep varies V, so free + * and force a reallocation per size -- otherwise a later, larger V would overflow + * the buffer sized for the first (smallest) V. */ + free(g_pbuf); g_pbuf = NULL; free(g_pidx); g_pidx = NULL; + float *lo = malloc((size_t)V * sizeof(float)); + for (int shape = 0; shape < n_shapes; shape++){ + fill_shape(lo, V, shape); + for (size_t ni = 0; ni < sizeof(nucs)/sizeof(nucs[0]); ni++){ + char label[32]; snprintf(label, sizeof(label), "size[%zu]/shape[%d]", si, shape); + const char *sn = (const char*[]){"uniform","peaked","geometric","plateau","sharptail"}[shape]; + check_case(label, V, nucs[ni], sn, lo); + cases++; + } + } + free(lo); + } + + /* guard-off path: g_nuc >= 1 must skip truncation entirely (full softmax kept) */ + { + int V = 256; float lo[256]; + for (int i = 0; i < V; i++) lo[i] = (float)(frand()*4 - 2); + g_nuc = 1.0f; dist_build(lo, V); + int nz = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) nz++; + if (nz != V){ fprintf(stderr, " FAIL [guard-off nuc=1.0]: %d/%d entries kept, expected all\n", nz, V); g_nfails++; } + else printf(" ok [guard-off nuc=1.0 keep=%d]\n", nz); + cases++; + + g_nuc = 0.0f; dist_build(lo, V); + nz = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) nz++; + if (nz != V){ fprintf(stderr, " FAIL [guard-off nuc=0.0]: %d/%d entries kept, expected all\n", nz, V); g_nfails++; } + else printf(" ok [guard-off nuc=0.0 keep=%d]\n", nz); + cases++; + } + + /* extreme tie edge case: V=1, single token -> keep=1 regardless of nuc */ + { + float lo[1] = {5.f}; + g_nuc = 0.5f; dist_build(lo, 1); + if (g_pbuf[0] == 0.f || !(fabs((double)g_pbuf[0] - 1.0) < 1e-6)){ + fprintf(stderr, " FAIL [V=1]: g_pbuf[0]=%.9g, expected 1.0\n", (double)g_pbuf[0]); g_nfails++; + } else printf(" ok [V=1 keep=1]\n"); + cases++; + } + + printf("\ntest_topp: %d cases run, %d failure(s)\n", cases, g_nfails); + if (g_nfails){ printf("test_topp: FAIL\n"); return 1; } + printf("test_topp: ok\n"); + return 0; +} From 37c96ee07a15669ddc94c51f61c1bc41f0c7cc51 Mon Sep 17 00:00:00 2001 From: woolcoxm <13604288+woolcoxm@users.noreply.github.com> Date: Fri, 17 Jul 2026 09:22:26 -0400 Subject: [PATCH 32/95] bench: add bench_topp -- head-to-head old-qsort vs new-heap timing on V=151936 (#335) test_topp proves correctness; bench_topp measures the latency claim. It re-implements the OLD dist_build (full-vocab qsort) inline on a private buffer and times it against the REAL new dist_build over identical inputs in one process: V=151936, temp=0.7, 3 shapes (realistic / uniform / plateau) x 4 nuc values (0.5/0.9/0.95/0.99), 2000 timed reps each, median reported. Deliberately NOT in TEST_BINS -- it's a microbench, not a gate. Build on demand: make tests/bench_topp && ./tests/bench_topp --- c/Makefile | 5 ++ c/tests/bench_topp.c | 131 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 c/tests/bench_topp.c diff --git a/c/Makefile b/c/Makefile index 395e136..bc609fa 100644 --- a/c/Makefile +++ b/c/Makefile @@ -320,6 +320,11 @@ tests/test_stops$(EXE): tests/test_stops.c glm.c st.h uring.h json.h tok.h tok_u tests/test_topp$(EXE): tests/test_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +# bench_topp is a microbenchmark (old qsort vs new heap partial-select, #335), NOT a test +# gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_topp +tests/bench_topp$(EXE): tests/bench_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) diff --git a/c/tests/bench_topp.c b/c/tests/bench_topp.c new file mode 100644 index 0000000..9c0682f --- /dev/null +++ b/c/tests/bench_topp.c @@ -0,0 +1,131 @@ +/* Microbenchmark: old (full-vocab qsort) vs new (heap partial-select) top-p truncation. + * + * This is NOT a unit test -- test_topp.c proves correctness. This measures the headline + * claim of #335: that replacing the O(V log V) qsort over all 151936 vocab entries with + * an O(V) heapify + k*log-V pops is materially faster per call, which is the win the issue + * was opened for. + * + * It re-implements the OLD dist_build inline (qsort + scan) on a private buffer so the A/B + * runs in one process, same inputs, same warm caches -- a controlled comparison. It calls + * the REAL (new) dist_build via the include-glm.c pattern on the global g_pbuf. + * + * Methodology (chosen to be honest, not to flatter the change): + * - V = 151936 (the actual GLM-5.2 vocab), g_nuc swept across the values that matter + * for serving: 0.5 / 0.9 (serve default) / 0.95 / 0.99. + * - Three logit shapes: (a) realistic peaked -- one hot token, long exponential tail, + * the shape real language-model logits take; (b) uniform -- worst case for the heap, + * maximum pop count; (c) a plateau of ties, to exercise the tie path. + * - Each (shape, nuc) is timed over N_REPEAT=2000 iterations, with the RNG/logits frozen + * so both algorithms do IDENTICAL work. We report median ns/call and the new/old ratio. + * - A warmup pass primes caches before timing. + * + * Run: make tests/bench_topp && ./tests/bench_topp (not in TEST_BINS -- not a gate) + */ +#define main coli_glm_main_unused +#include "../glm.c" +#undef main + +#include +#include + +/* ---- the OLD algorithm, verbatim from dev before #354, on a private buffer ---- */ +static float *s_pbuf; static int *s_pidx; static double *s_ref; +static int cmp_pdesc_old(const void *a, const void *b){ + double pa = s_ref[*(const int*)a], pb = s_ref[*(const int*)b]; + return pa < pb ? 1 : pa > pb ? -1 : 0; } +static void dist_build_old(const float *lo, int V, double temp, double nuc){ + double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i]; + double s = 0, invt = 1.0 / (temp > 1e-4 ? temp : 1e-4); + for (int i = 0; i < V; i++){ s_ref[i] = exp((lo[i]-mx)*invt); s += s_ref[i]; } + for (int i = 0; i < V; i++) s_ref[i] /= s; + if (nuc > 0 && nuc < 1.0){ + for (int i = 0; i < V; i++) s_pidx[i] = i; + qsort(s_pidx, V, sizeof(int), cmp_pdesc_old); + double cum = 0; int keep = V; + for (int i = 0; i < V; i++){ cum += s_ref[s_pidx[i]]; if (cum >= nuc){ keep = i+1; break; } } + double s2 = 0; + for (int i = keep; i < V; i++) s_ref[s_pidx[i]] = 0; + for (int i = 0; i < keep; i++) s2 += s_ref[s_pidx[i]]; + for (int i = 0; i < keep; i++) s_ref[s_pidx[i]] /= s2; + } + (void)s_pbuf; +} + +/* ---- timing: median of N_REPEAT runs in ns/call, sorted ascending ---- */ +#define N_REPEAT 2000 +static double bench_ns(void (*fn)(const float*,int,double,double), + const float *lo, int V, double temp, double nuc){ + static double ts[N_REPEAT]; + for (int r = 0; r < N_REPEAT; r++){ + double t0 = now_s(); + fn(lo, V, temp, nuc); + ts[r] = (now_s() - t0) * 1e9; + } + /* insertion sort the N_REPEAT samples (small), take median */ + for (int a = 1; a < N_REPEAT; a++){ double k = ts[a]; int b = a-1; + while (b >= 0 && ts[b] > k){ ts[b+1] = ts[b]; b--; } ts[b+1] = k; } + return ts[N_REPEAT/2]; +} + +/* the NEW algorithm is the real dist_build, but it writes g_pbuf (not a private buf). + * Wrap it so the bench signature matches, and set the globals it reads. */ +static void dist_build_new(const float *lo, int V, double temp, double nuc){ + g_temp = (float)temp; g_nuc = (float)nuc; + dist_build(lo, V); +} + +/* deterministic logit fill for three shapes */ +static uint32_t brng = 0xA5A5A5A5u; +static double brand(void){ brng ^= brng << 13; brng ^= brng >> 17; brng ^= brng << 5; + return (double)(brng >> 8) * (1.0 / 16777216.0); } +static void fill_realistic(float *lo, int V){ /* one hot, exponential tail -- like real logits */ + for (int i = 0; i < V; i++) lo[i] = (float)(-4.0 * brand() - (double)i * 0.0001); + lo[0] = 6.f; lo[V/50] = 4.f; lo[V/200] = 3.f; +} +static void fill_uniform(float *lo, int V){ /* worst case for the heap: max pop count */ + for (int i = 0; i < V; i++) lo[i] = 0.f; +} +static void fill_plateau(float *lo, int V){ /* ties: blocks of equal value */ + for (int i = 0; i < V; i++) lo[i] = (float)(-(double)(i / 50)); +} + +int main(void){ + int V = 151936; + float *lo = malloc((size_t)V * sizeof(float)); + s_ref = malloc((size_t)V * sizeof(double)); + s_pidx = malloc((size_t)V * sizeof(int)); + /* force the new dist_build to allocate g_pbuf/g_pidx at full V once */ + g_temp = 0.7f; g_nuc = 0.9f; dist_build(lo, V); + + double temp = 0.7; + struct { const char *name; void (*fill)(float*,int); } shapes[] = { + { "realistic", fill_realistic }, + { "uniform", fill_uniform }, + { "plateau", fill_plateau }, + }; + double nucs[] = { 0.5, 0.9, 0.95, 0.99 }; + + printf("bench_topp: top-p truncation, old (qsort) vs new (heap) V=%d temp=%.2f\n", V, temp); + printf("%-12s %6s %14s %14s %9s %9s\n", "shape", "nuc", "old ns/call", "new ns/call", "speedup", "keep"); + printf("-----------------------------------------------------------------------------\n"); + + for (size_t sh = 0; sh < sizeof(shapes)/sizeof(shapes[0]); sh++){ + shapes[sh].fill(lo, V); + for (size_t ni = 0; ni < sizeof(nucs)/sizeof(nucs[0]); ni++){ + double nuc = nucs[ni]; + /* warmup both paths so caches/branch predictors are primed */ + for (int w = 0; w < 50; w++){ dist_build_old(lo, V, temp, nuc); dist_build_new(lo, V, temp, nuc); } + double t_old = bench_ns(dist_build_old, lo, V, temp, nuc); + double t_new = bench_ns(dist_build_new, lo, V, temp, nuc); + /* keep count = non-zero entries the new path leaves (== old's keep) */ + int keep = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) keep++; + printf("%-12s %6.2f %14.0f %14.0f %8.2fx %9d\n", + shapes[sh].name, nuc, t_old, t_new, t_old / t_new, keep); + } + printf("\n"); + } + + printf("bench_topp: done (lower ns is better; speedup = old/new)\n"); + free(lo); free(s_ref); free(s_pidx); + return 0; +} From 5d163688170053352cfaa3b3bff1d5b0a8fa2b9d Mon Sep 17 00:00:00 2001 From: woolcoxm <13604288+woolcoxm@users.noreply.github.com> Date: Fri, 17 Jul 2026 09:54:40 -0400 Subject: [PATCH 33/95] =?UTF-8?q?sampling:=20partial=20top-keep=20select?= =?UTF-8?q?=20in=20attention=5Frows=20DSA=20=E2=80=94=20O(nk)=20quickselec?= =?UTF-8?q?t,=20not=20O(nk=20log=20nk)=20qsort=20(#356)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DSA lightning indexer selects the top-index_topk (2048) context keys to attend to by finding the threshold = keep-th largest attention score. It previously full-qsorted all nk scores per layer per token (O(nk log nk)) just to read one pivot value, then scanned the original array in position order to build the kept set. Replace the qsort with partial_select_desc (Hoare quickselect, median-of-three, descending): O(nk) average to partition the keep largest into a[0..k), then the threshold is min of that block. The two position-order scans (>thr then ==thr) are UNCHANGED, so the kept-position set is bit-identical -- a stronger contract than #335's sampling heap (which was multiset-only because the heap was unstable and changed accumulation order). The quickselect pivot IS by definition the keep-th largest, so the new threshold equals old tmp[keep-1] exactly. Measured (bench_dsa_select, keep=2048, median of 2000 reps): nk=2049: 119us -> 5.7us (21x) nk=8192: 626us -> 43us (15x) nk=32768: 2.8ms -> 0.28ms (10x) nk=65536: 6.6ms -> 0.47ms (14x) The gap widens with context (linear vs n-log-n). DSA only fires past index_topk, so this is precisely the long-conversation regime where decode latency matters. Adds test_dsa_select (in TEST_BINS): 129 cases asserting element-wise identical kept-set vs an independent qsort reference across shapes (random, peaked, sorted, reverse-sorted, tie-plateau, all-equal) and edges (keep==1, keep==nk). Also directly checks the partition invariant. Adds bench_dsa_select (on-demand, NOT a gate): reproduces the table above. --- c/Makefile | 10 +- c/glm.c | 47 +++++++- c/tests/bench_dsa_select.c | 132 ++++++++++++++++++++++ c/tests/test_dsa_select.c | 223 +++++++++++++++++++++++++++++++++++++ 4 files changed, 408 insertions(+), 4 deletions(-) create mode 100644 c/tests/bench_dsa_select.c create mode 100644 c/tests/test_dsa_select.c diff --git a/c/Makefile b/c/Makefile index 27902cb..41e487c 100644 --- a/c/Makefile +++ b/c/Makefile @@ -166,7 +166,7 @@ else PYTHON ?= python3 endif CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) ifneq (,$(LINUX)) TEST_BINS += tests/test_uring$(EXE) endif @@ -326,6 +326,14 @@ tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c tests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_dsa_select$(EXE): tests/test_dsa_select.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +# bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356), +# NOT a test gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_dsa_select +tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + tests/test_uring$(EXE): tests/test_uring.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) diff --git a/c/glm.c b/c/glm.c index 30d970a..1070e79 100644 --- a/c/glm.c +++ b/c/glm.c @@ -2304,6 +2304,43 @@ static int g_dsa_force=0; /* DSA_FORCE=1: selezione sempre attiva (test: top-min static int cmp_fdesc(const void *a,const void *b){ float x=*(const float*)a, y=*(const float*)b; return xy?-1:0; } +/* PARTIAL SELECT (quickselect, Hoare partition, DESCending). After this call the k + * LARGEST elements of a[0..n) are in a[0..k) in unspecified order; the (k+1)-th and + * beyond are untouched-or-smaller. O(n) average, O(n^2) pathological (mitigated by + * median-of-three below) — and unlike a full qsort it never orders more than needed. + * + * Why this exists (#356): the DSA top-keep in attention_rows previously full-qsorted + * all nk context scores (O(nk log nk)) per layer per token just to read ONE value -- + * the keep-th largest (the threshold). quickselect finds that pivot in O(nk) average, + * and the position-order scans that build dst[] are unchanged, so the kept set is + * bit-identical. Mirrors the sampling-side fix in #335 (heap partial-select there). + * + * NOT a stable partition: callers must derive the threshold and then re-scan the + * ORIGINAL array (the DSA code does exactly this) rather than reading a[0..k). */ +static void partial_select_desc(float *a, int n, int k){ + if(k<=0) return; + if(k>=n) return; /* nothing to partition: all kept */ + int lo=0, hi=n-1; + while(lo>1); + if(a[mid]>a[lo]){ float t=a[lo]; a[lo]=a[mid]; a[mid]=t; } + if(a[hi]>a[lo]){ float t=a[lo]; a[lo]=a[hi]; a[hi]=t; } + if(a[mid]>a[hi]){ float t=a[hi]; a[hi]=a[mid]; a[mid]=t; } + float piv=a[hi]; + int i=lo, j=hi; + for(;;){ + while(a[i]>piv) i++; /* desc: large values go left */ + while(j>lo && a[j]=j) break; + float t=a[i]; a[i]=a[j]; a[j]=t; i++; if(i>j) break; j--; + } + /* partition point: a[lo..i) are all >= piv, a[i..hi] are all <= piv */ + if(k<=i-1) hi=i-1; /* the k-th largest is in the left partition */ + else lo=i; /* it's in the right partition */ + } +} + /* attenzione MLA con KV-cache compressa, su token nuovi x[S,hidden], pos_base = pos del primo */ /* kvs/pos describe a ragged decode batch: each row may belong to a different * sequence. NULL keeps the original contiguous, currently-bound KV path. */ @@ -2586,10 +2623,14 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p } isc[t]=a*wsc; } - /* top-keep: soglia via qsort desc, poi scan in ordine di posizione */ + /* top-keep: threshold via PARTIAL SELECT (#356), poi scan in ordine di posizione. + * Era un qsort completo su nk (O(nk log nk)); quickselect estrae solo il + * keep-esimo valore piu' grande in O(nk) medio. La soglia (= min del blocco + * dei keep maggiori) e' identica a tmp[keep-1] del vecchio qsort, quindi i + * due scan qui sotto costruiscono dst[] bit-identical. */ float *tmp=falloc(nk); memcpy(tmp,isc,nk*sizeof(float)); - qsort(tmp,nk,sizeof(float),cmp_fdesc); - float thr=tmp[keep-1]; + partial_select_desc(tmp,nk,keep); + float thr=tmp[0]; for(int t=1;tdsa_sel+(int64_t)s*dtopk, nd=0; for(int t=0;tthr) dst[nd++]=t; for(int t=0;t +#include + +/* ---- the OLD algorithm, verbatim from dev before #356, on a private buffer ---- */ +static int cmp_pdesc_old(const void *a, const void *b){ + float x=*(const float*)a, y=*(const float*)b; return xy?-1:0; } +static void keep_old(const float *isc, int nk, int keep, int *dst, int *nd_out){ + float *tmp=malloc((size_t)nk*sizeof(float)); + memcpy(tmp,isc,(size_t)nk*sizeof(float)); + qsort(tmp,(size_t)nk,sizeof(float),cmp_pdesc_old); + float thr=tmp[keep-1]; int nd=0; + for(int t=0;tthr) dst[nd++]=t; + for(int t=0;t=0 && ts[b]>k){ ts[b+1]=ts[b]; b--; } ts[b+1]=k; } + free(dst); + return ts[N_REPEAT/2]; +} + +/* the NEW algorithm calls the real partial_select_desc + replicates the production + * threshold derivation and position scans. */ +static void keep_new(const float *isc, int nk, int keep, int *dst, int *nd_out){ + float *tmp=malloc((size_t)nk*sizeof(float)); + memcpy(tmp,isc,(size_t)nk*sizeof(float)); + partial_select_desc(tmp,nk,keep); + float thr=tmp[0]; for(int t=1;tthr) dst[nd++]=t; + for(int t=0;t> 17; brng ^= brng << 5; + return (double)(brng >> 8) * (1.0 / 16777216.0); } +static void fill_realistic(float *isc, int nk){ /* few hot, long distinct tail */ + for(int i=0;i boundary path */ + for(int i=0;i thr -> keep (strictly above threshold) + * for t: if isc[t] == thr -> keep (ties, in position order) + * -- are UNCHANGED by the rewrite. So if the threshold value is identical, + * the kept-position set is identical element-by-element (not just as a + * multiset, which is all the unstable sampling heap in #335 could promise). + * + * Strategy: drive the REAL partial_select_desc (via the include-glm.c pattern) + * and replicate the production threshold derivation + scans, then compare the + * resulting dst[] against an INDEPENDENT reference that re-implements the OLD + * algorithm (full qsort + tmp[keep-1] threshold) on a private buffer. The kept + * sets must be element-wise equal on every shape, including tie plateaus where + * the boundary membership is decided by the position scan. + * + * We also directly unit-test partial_select_desc's partition invariant: after + * the call, max(a[keep..n)) <= min(a[0..keep)) -- i.e. the keep largest really + * did land in the prefix. This catches a broken quickselect even before the + * end-to-end comparison. + * + * In-memory only (no scratch files), so it builds clean on the Windows MinGW CI + * job without the unmerged compat shim. */ +#define main coli_glm_main_unused +#include "../glm.c" +#undef main + +#include + +static int g_nfails = 0; + +#define FAIL(fmt, ...) do { \ + fprintf(stderr, " FAIL [%s nk=%d keep=%d shape=%s]: " fmt "\n", \ + label, nk, keep, shape_name, ##__VA_ARGS__); \ + g_nfails++; \ + return; \ +} while (0) + +/* ---- independent reference: the OLD algorithm (full qsort + tmp[keep-1]) ---- */ +/* qsort comparator matching the production cmp_fdesc exactly (desc, unstable). */ +static int cmp_ref_desc(const void *a, const void *b){ + float x=*(const float*)a, y=*(const float*)b; return xy?-1:0; } + +/* Reproduce the OLD glm.c:2589-2596 exactly: copy, qsort desc, threshold = + * tmp[keep-1], then the two position-order scans into dst[]. Returns nd. */ +static int keep_old(const float *isc, int nk, int keep, int *dst){ + float *tmp=malloc((size_t)nk*sizeof(float)); + memcpy(tmp,isc,(size_t)nk*sizeof(float)); + qsort(tmp,(size_t)nk,sizeof(float),cmp_ref_desc); + float thr=tmp[keep-1]; + int nd=0; + for(int t=0;tthr) dst[nd++]=t; + for(int t=0;tthr) dst[nd++]=t; + for(int t=0;t= every element of tmp[keep..n). + * (>=, not >: equal values may sit on either side of the partition boundary, + * which is fine -- the threshold is the MIN of the prefix, and the position + * scan handles ties.) */ + float top_min=INFINITY, tail_max=-INFINITY; + for(int i=0;itail_max) tail_max=tmp[i]; + if(!(top_min >= tail_max)) + FAIL("partition invariant violated: top_min=%.9g < tail_max=%.9g", top_min, tail_max); + free(tmp); +} + +/* ---- end-to-end: old vs new kept-set must be element-wise identical ---- */ +static void check_case(const char *label, int nk, int keep, const char *shape_name, + const float *isc){ + int *da=malloc((size_t)nk*sizeof(int)); + int *db=malloc((size_t)nk*sizeof(int)); + int na=keep_old(isc,nk,keep,da); + int nb=keep_new(isc,nk,keep,db); + + /* 1. both keep exactly `keep` positions (the contract: keep the top-keep by + * count). A count mismatch is a real bug, not a tie artifact. */ + if(na!=keep) FAIL("old kept %d, expected %d (old path is the reference)", na, keep); + if(nb!=keep) FAIL("new kept %d, expected %d", nb, keep); + if(na!=nb) FAIL("keep-count mismatch: old=%d new=%d", na, nb); + + /* 2. element-wise identical dst[]. This is the strong contract: because the + * threshold is derived identically and the position-order scans are byte- + * for-byte the same, the kept SET and its ORDER must match exactly. (This + * is what makes #356 cleaner than #335, which was multiset-only.) */ + int first_diff=-1; + for(int i=0;i=0) + FAIL("kept-set differs at index %d: old dst[%d]=%d new dst[%d]=%d", + first_diff, first_diff, da[first_diff], first_diff, db[first_diff]); + + /* 3. also check the partition invariant directly (catches a subtly broken + * quickselect even if the threshold happened to come out right). */ + check_partition(label,isc,nk,keep,shape_name); + + free(da); free(db); + printf(" ok [nk=%d keep=%d shape=%s]\n", nk, keep, shape_name); +} +#undef FAIL + +/* deterministic xorshift32 RNG (matches the test_i4_grouped.c / test_topp.c convention) */ +static uint32_t rng_state = 0x12345678u; +static uint32_t xr(void){ rng_state ^= rng_state << 13; rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; return rng_state; } +static double frand(void){ return (xr() >> 8) * (1.0 / 16777216.0); } /* [0,1) */ + +/* fill scores for a given shape. Shapes stress the threshold boundary and the + * quickselect's median-of-three pivot differently. */ +static void fill_shape(float *isc, int nk, int shape){ + switch(shape){ + case 0: /* uniform random distinct (no ties): the clean contract case */ + for(int i=0;i3) isc[nk/3]=1.f; if(nk>2) isc[nk/2]=0.5f; break; + case 2: /* strictly decreasing geometric (no ties): sorted input -- worst case + * for a naive quickselect; median-of-three must handle it */ + for(int i=0;i boundary membership decided + * entirely by the position scan (exercises the ==thr path) */ + for(int i=0;i degenerate threshold, all kept + * via the ==thr scan; quickselect must not infinite-loop or corrupt */ + for(int i=0;i=n / k==1 / k==n edges. */ + int nks[] = {1, 2, 8, 64, 2049, 4097, 8193}; + int keeps[] = {1, 8, 256, 1024, 2048}; + int n_shapes = 6; + + int cases = 0; + for(size_t ni=0; nin, but skip the plateau/geometric edge if nk<7) */ + fill_shape(isc,nk,shape); + for(size_t ki=0; kink) continue; /* keep<=nk invariant of the production code */ + if(keep<=0) continue; + char label[40]; snprintf(label,sizeof(label),"nk[%zu]/keep[%zu]/shape[%d]",ni,ki,shape); + const char *sn=(const char*[]){"random","peaked","decreasing","increasing","plateau","all-equal"}[shape]; + check_case(label,nk,keep,sn,isc); + cases++; + } + } + free(isc); + } + + /* edge: keep == nk (nothing to partition; both paths keep everything) */ + { + int nk=100, keep=100; float isc[100]; + for(int i=0;i every kept slot is a tie; + * the position scan must pick positions 0..keep-1 deterministically */ + { + int nk=1000, keep=500; float isc[1000]; + for(int i=0;i positions 0..%d]\n",keep,keep-1); cases++; + free(db); + } + + printf("\ntest_dsa_select: %d cases run, %d failure(s)\n", cases, g_nfails); + if(g_nfails){ printf("test_dsa_select: FAIL\n"); return 1; } + printf("test_dsa_select: ok\n"); + return 0; +} From 211d4488c36d5d54eba679fbc5775162fac52c56 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Fri, 17 Jul 2026 18:49:28 +0200 Subject: [PATCH 34/95] =?UTF-8?q?win32:=20stop=20erasing=20the=20real=20Re?= =?UTF-8?q?adFile=20error=20=E2=80=94=20pread=20failures=20name=20their=20?= =?UTF-8?q?WinErr=20(#307)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit compat_pread mapped every non-EOF ReadFile failure to EIO, so the field report was always the same three words: "Input/output error". #307 then burned three rounds of guessing across three people — storage? contention? alignment? — because the actual GetLastError code never appeared anywhere. compat_pread now stashes the code in a per-thread slot and pread_full appends it to the failure line: pread qs: Input/output error (off 780592, 0/8192 bytes, WinErr=1450) That one number is the difference between "insufficient system resources" (1450 -> memory pressure), "sharing violation" (32 -> AV interference), "device not ready" (21 -> the T: drive itself) and a dozen other distinct diagnoses. Diagnostic only: no behavior change, no retry policy — that discussion lives in #361 and should be settled AFTER the first report tells us which error we are actually retrying. Linux path untouched byte-for-byte; the Windows compile is certified by the check.yml MSYS2 job. Co-Authored-By: Claude Opus 4.8 --- c/compat.h | 7 +++++++ c/glm.c | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/c/compat.h b/c/compat.h index 82de8b6..a21bbd8 100644 --- a/c/compat.h +++ b/c/compat.h @@ -143,6 +143,12 @@ static inline int compat_fadvise(int fd, off_t off, off_t len, int advice){ * Thread-safe (no shared seek position). Gestisce offset >4 GB e chunking * per letture >2 GB (anche se i tensori individuali sono nell'ordine dei * MB-centinaia di MB, il wrapper e' robusto per ogni taglia). */ +/* Ultimo GetLastError() di una ReadFile fallita, per thread: il chiamante + * (pread_full in glm.c) lo stampa accanto a strerror. Senza questo, OGNI + * fallimento Windows collassa in "EIO -> Input/output error" e la diagnosi + * dal campo diventa un tirare a indovinare (#307: tre giri di ipotesi tra + * tre persone perche' il codice vero non compariva da nessuna parte). */ +static __thread DWORD compat_pread_lasterr __attribute__((unused)); static inline ssize_t compat_pread(int fd, void *buf, size_t n, off_t off){ intptr_t osfh = _get_osfhandle(fd); if(osfh == -1 || osfh == -2){ errno = EBADF; return -1; } @@ -158,6 +164,7 @@ static inline ssize_t compat_pread(int fd, void *buf, size_t n, off_t off){ if(!ReadFile(h, (char*)buf + total, chunk32, &rd, &ov)){ DWORD err = GetLastError(); if(err == ERROR_HANDLE_EOF) break; /* past EOF → return bytes read (0 if none, matching POSIX pread) */ + compat_pread_lasterr = err; /* preserva il codice VERO per il report (#307) */ if(err == ERROR_INVALID_HANDLE || err == ERROR_INVALID_FUNCTION) errno = EBADF; else errno = EIO; return -1; diff --git a/c/glm.c b/c/glm.c index 30d970a..003a42d 100644 --- a/c/glm.c +++ b/c/glm.c @@ -1732,8 +1732,14 @@ static int pread_full(int fd, void *buf, int64_t n, int64_t off, const char *tag while(got Date: Sat, 18 Jul 2026 01:54:31 +0800 Subject: [PATCH 35/95] cuda: batch ragged attention across independent streams --- c/backend_cuda.cu | 71 ++++++++++++++++++++++++++++++++ c/backend_cuda.h | 6 ++- c/glm.c | 26 +++++++++--- c/tests/test_ragged_attention.cu | 35 ++++++++++++++++ 4 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 c/tests/test_ragged_attention.cu diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu index 9ce142f..188330d 100644 --- a/c/backend_cuda.cu +++ b/c/backend_cuda.cu @@ -338,6 +338,40 @@ __global__ static void attention_absorb_batch_kernel(float *ctx,const float *q, ctx[((size_t)s*H+h)*V+v]=a*(fmt?wscale[row]:1.f);} } +/* Independent KV sequence per row. latent/rope are packed as [S,T,*], while + * lengths selects the valid prefix for each row. */ +__global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q, + const float *latent,const float *rope,const int *lengths, + const void *weights,const float *wscale,int fmt,int S,int H,int Q,int R, + int V,int K,int T,float scale){ + int s=blockIdx.y,h=blockIdx.x,tid=threadIdx.x,nt=lengths[s],rbase=h*(Q+V); + if(s>=S||nt<1||nt>T)return; + extern __shared__ float sm[];float *qa=sm,*cl=qa+K,*scores=cl+K,*red=scores+T; + const float *qs=q+((size_t)s*H+h)*(Q+R); + const float *ls=latent+(size_t)s*T*K,*rs=rope+(size_t)s*T*R; + for(int k=tid;k>1;n;n>>=1){if(tid>1;n;n>>=1){if(tid= bytes) return 1; if (*ptr) cudaFree(*ptr); @@ -770,6 +804,43 @@ extern "C" int coli_cuda_attention_project_batch(ColiCudaTensor *w,ColiCudaTenso return attention_absorb_batch_run(w,proj,out,q,latent,rope,S,H,Q,R,V,K,T,scale); } +extern "C" int coli_cuda_attention_project_ragged(ColiCudaTensor *w,ColiCudaTensor *proj, + float *out,const float *q,const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int T,float scale){ + if(!w||!proj||!out||!q||!latent||!rope||!lengths||S<1||S>512||T<1||T>512|| + H<1||Q<1||R<1||V<1||K<1||K>512||w->I!=K||w->O!=H*(Q+V)|| + proj->device!=w->device||proj->I!=H*V)return 0; + size_t ln=(size_t)S*T*K,rn=(size_t)S*T*R; + float *lh=(float*)std::calloc(ln,sizeof(float)),*rh=(float*)std::calloc(rn,sizeof(float)); + if(!lh||!rh){std::free(lh);std::free(rh);return 0;} + for(int s=0;sT){std::free(lh);std::free(rh);return 0;} + std::memcpy(lh+(size_t)s*T*K,latent[s],(size_t)lengths[s]*K*sizeof(float)); + std::memcpy(rh+(size_t)s*T*R,rope[s],(size_t)lengths[s]*R*sizeof(float)); + } + DeviceContext *dc=find_ctx(w->device); + if(!select_ctx(dc)){std::free(lh);std::free(rh);return 0;} + size_t qb=(size_t)S*H*(Q+R)*sizeof(float),lb=ln*sizeof(float),rb=rn*sizeof(float); + size_t cb=(size_t)S*H*V*sizeof(float),ob=(size_t)S*proj->O*sizeof(float); + int ok=reserve(&dc->aq,&dc->aq_cap,qb)&&reserve(&dc->al,&dc->al_cap,lb)&& + reserve(&dc->ar,&dc->ar_cap,rb)&&reserve(&dc->ac,&dc->ac_cap,cb)&& + reserve(&dc->y,&dc->y_cap,ob)&& + reserve_bytes(&dc->group_desc,&dc->group_desc_cap,(size_t)S*sizeof(int)); + if(ok)ok=cuda_ok(cudaMemcpyAsync(dc->aq,q,qb,cudaMemcpyHostToDevice,dc->stream),"ragged q upload")&& + cuda_ok(cudaMemcpyAsync(dc->al,lh,lb,cudaMemcpyHostToDevice,dc->stream),"ragged latent upload")&& + cuda_ok(cudaMemcpyAsync(dc->ar,rh,rb,cudaMemcpyHostToDevice,dc->stream),"ragged rope upload")&& + cuda_ok(cudaMemcpyAsync(dc->group_desc,lengths,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged lengths upload"); + std::free(lh);std::free(rh);if(!ok)return 0; + size_t shared=(size_t)(2*K+T+256)*sizeof(float); + attention_absorb_ragged_kernel<<stream>>>(dc->ac,dc->aq,dc->al,dc->ar, + (const int*)dc->group_desc,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale); + quant_matmul<<O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights, + proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I)); + return cuda_ok(cudaGetLastError(),"ragged attention launch")&& + cuda_ok(cudaMemcpyAsync(out,dc->y,ob,cudaMemcpyDeviceToHost,dc->stream),"ragged output download")&& + cuda_ok(cudaStreamSynchronize(dc->stream),"ragged attention synchronize"); +} + extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) { if (!tensor) return; DeviceContext *ctx = find_ctx(tensor->device); diff --git a/c/backend_cuda.h b/c/backend_cuda.h index acbe4bc..edddbfe 100644 --- a/c/backend_cuda.h +++ b/c/backend_cuda.h @@ -14,6 +14,7 @@ #define COLI_CUDA_DLLEXPORT #endif + #ifdef __cplusplus extern "C" { #endif @@ -92,6 +93,10 @@ COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,C const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale); +COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, + float *out,const float *q,const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale); + COLI_CUDA_DLLEXPORT void coli_cuda_tensor_free(ColiCudaTensor *tensor); COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor); COLI_CUDA_DLLEXPORT int coli_cuda_tensor_device(const ColiCudaTensor *tensor); @@ -143,4 +148,3 @@ COLI_CUDA_DLLEXPORT int coli_cuda_pipe_sync(int device); #endif #endif - diff --git a/c/glm.c b/c/glm.c index 003a42d..298065b 100644 --- a/c/glm.c +++ b/c/glm.c @@ -1644,7 +1644,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits c->index_topk, c->index_topk); } } - m->hlast=falloc(D); m->h_all=falloc((int64_t)64*D); + m->hlast=falloc(D); m->h_all=falloc((int64_t)512*D); /* byte della parte DENSA residente (embed+lm_head+attn+mlp densa+shared+norme) */ int64_t rb=qt_bytes(&m->embed)+qt_bytes(&m->lm_head); @@ -2640,7 +2640,23 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p float *sc_all = falloc((int64_t)omp_get_max_threads()*sc_cap); int cuda_core=0,cuda_projected=0; #ifdef COLI_CUDA - if(cuda_absorb&&l->n_kv_b_shard>1){ + if(kvs&&g_cuda_enabled&&getenv("COLI_CUDA_ATTN")&&atoi(getenv("COLI_CUDA_ATTN"))&& + !dnsel&&l->kv_b.cuda_eligible&&l->o.cuda_eligible&& + qt_cuda_upload(&l->kv_b)&&qt_cuda_upload(&l->o)){ + const float **rl=malloc((size_t)S*sizeof(*rl)),**rr=malloc((size_t)S*sizeof(*rr)); + int *rn=malloc((size_t)S*sizeof(*rn)); int mt=0; + if(rl&&rr&&rn){ + for(int s=0;skv_start[layer]; rn[s]=pos+1-st0; + rl[s]=coli_kv_row(kvs[s]->Lc[layer],st0,kvl); + rr[s]=coli_kv_row(kvs[s]->Rc[layer],st0,c->qk_rope); + if(rn[s]>mt)mt=rn[s]; + } + cuda_core=cuda_projected=coli_cuda_attention_project_ragged(l->kv_b.cuda,l->o.cuda, + out,Q,rl,rr,rn,S,H,c->qk_nope,c->qk_rope,vh,kvl,mt,c->attn_scale); + } + free(rl);free(rr);free(rn); + } else if(cuda_absorb&&l->n_kv_b_shard>1){ int n=l->n_kv_b_shard,st0=m->kv_start[layer],nt=pos_base+S-st0,ok=1; float *qs=falloc((int64_t)S*H*qh),*cs=falloc((int64_t)S*H*vh); for(int d=0;dh_all) memcpy(m->h_all, x, (int64_t)S*D*sizeof(float)); /* hidden di TUTTE le pos (S<=64) */ + if(m->h_all) memcpy(m->h_all, x, (int64_t)S*D*sizeof(float)); /* hidden di TUTTE le pos (S<=512) */ if(m->hlast) memcpy(m->hlast, x+(int64_t)(S-1)*D, D*sizeof(float)); float *lo=falloc((int64_t)S*c->vocab), *row=falloc(D); for(int s=0;sfinal_norm, D, c->eps); @@ -3988,8 +4004,8 @@ static float *step_all(Model *m, const int *ids, int S, int pos_base){ static float *step_decode_batch(Model *m, const DecodeRow *rows, int S){ Cfg *c=&m->c; int D=c->hidden; /* Ragged KV currently uses MLA absorption; the stack kernel is sized to 512. */ - if(!rows || S<1 || S>64 || c->kv_lora>512) return NULL; - KVState *kvs[64]; int positions[64]; + if(!rows || S<1 || S>512 || c->kv_lora>512) return NULL; + KVState *kvs[512]; int positions[512]; float *x=falloc((int64_t)S*D); for(int s=0;sLc || !rows[s].kv->Rc || !rows[s].kv->kv_start || diff --git a/c/tests/test_ragged_attention.cu b/c/tests/test_ragged_attention.cu new file mode 100644 index 0000000..c00a8eb --- /dev/null +++ b/c/tests/test_ragged_attention.cu @@ -0,0 +1,35 @@ +#include "../backend_cuda.h" + +#include +#include +#include + +int main(){ + int dev=0;if(!coli_cuda_init(&dev,1))return 77; + constexpr int S=3,H=2,Q=2,R=1,V=2,K=3,D=H*V,O=3,T=3; + std::vector w(H*(Q+V)*K),p(O*D),q(S*H*(Q+R)); + for(size_t i=0;i> l(S),r(S); + const float *lp[S],*rp[S]; + for(int s=0;s Date: Fri, 17 Jul 2026 21:49:28 +0300 Subject: [PATCH 36/95] Windows: fix non-ASCII chat prompt corruption (ANSI codepage vs UTF-8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit coli passes the chat prompt to glm.exe through the PROMPT/COLI_PROMPT environment variable. On Windows, plain getenv() is populated by the CRT from the ANSI-codepage view of the environment block, not UTF-8 — so any non-ASCII prompt text (Cyrillic, CJK, ...) is silently mangled before the byte-level tokenizer ever sees it, even though the parent process (coli's Python subprocess call) sets the value correctly via the wide env block. Add compat_getenv_utf8() in compat.h: reads the variable through GetEnvironmentVariableW and converts straight to UTF-8, bypassing the ANSI codepage entirely. No-op passthrough to getenv() on non-Windows platforms. coli_user_prompt() in glm.c now uses it for both COLI_PROMPT and PROMPT. Verified: tiny-oracle self-test still 32/32 after rebuild, and a real run against the full GLM-5.2-int4 model with a Cyrillic prompt now round-trips correctly end to end (input echoed intact, coherent Cyrillic output), where it previously produced replacement-character garbage on both input and output. --- c/compat.h | 31 +++++++++++++++++++++++++++++++ c/glm.c | 4 ++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/c/compat.h b/c/compat.h index 82de8b6..06779e0 100644 --- a/c/compat.h +++ b/c/compat.h @@ -304,8 +304,39 @@ static inline int compat_setenv(const char *name, const char *value, int overwri } #define setenv(name,value,overwrite) compat_setenv(name,value,overwrite) +/* --- getenv_utf8: read an env var as UTF-8, not through the ANSI codepage --- + * Plain getenv()/_environ are populated by the CRT from the ANSI-codepage view + * of the process environment block, not UTF-8. A parent that hands the child a + * Unicode value via CreateProcessW's wide env block (e.g. Python's subprocess + * module, which coli uses to pass the chat prompt) round-trips correctly only + * through GetEnvironmentVariableW; going through narrow getenv() re-encodes it + * via CP_ACP first, so any non-ASCII prompt text (Cyrillic, CJK, ...) comes out + * corrupted before the byte-level tokenizer ever sees it. Read the wide value + * directly and convert straight to UTF-8, bypassing the ANSI codepage entirely. + * Returned buffer is intentionally leaked: called a handful of times at + * startup, lives for the process. */ +static inline const char *compat_getenv_utf8(const char *name){ + wchar_t wname[64]; + if(MultiByteToWideChar(CP_UTF8, 0, name, -1, wname, 64) <= 0) return getenv(name); + DWORD need = GetEnvironmentVariableW(wname, NULL, 0); + if(!need) return NULL; + wchar_t *wval = (wchar_t*)malloc(need * sizeof(wchar_t)); + if(!wval) return NULL; + GetEnvironmentVariableW(wname, wval, need); + int blen = WideCharToMultiByte(CP_UTF8, 0, wval, -1, NULL, 0, NULL, NULL); + char *val = blen>0 ? (char*)malloc((size_t)blen) : NULL; + if(val) WideCharToMultiByte(CP_UTF8, 0, wval, -1, val, blen, NULL, NULL); + free(wval); + return val; +} +#define getenv_utf8(name) compat_getenv_utf8(name) + #endif /* _WIN32 */ +#ifndef getenv_utf8 +#define getenv_utf8(name) getenv(name) +#endif + /* --- compat_aligned_free su piattaforme diverse da Windows --- * Su Linux/macOS, posix_memalign usa free() normale. */ #ifndef compat_aligned_free diff --git a/c/glm.c b/c/glm.c index 00c7076..b0e419c 100644 --- a/c/glm.c +++ b/c/glm.c @@ -5701,9 +5701,9 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){ * self-test, and would "generate" from "$P$G". So on Windows a PROMPT carrying * cmd's $-metacodes is ignored; set COLI_PROMPT to pass a real prompt from cmd. */ static const char *coli_user_prompt(void){ - const char *p = getenv("COLI_PROMPT"); + const char *p = getenv_utf8("COLI_PROMPT"); if(p) return p; - p = getenv("PROMPT"); + p = getenv_utf8("PROMPT"); #ifdef _WIN32 if(p) for(const char *q=p; q[0]; q++) if(q[0]=='$' && q[1] && strchr("ABCDEFGHLNPQSTV_+|$", q[1]&~0x20)){ p=NULL; break; } From 7f70a8db5d92516c267f867d41e0a12bcd0021c8 Mon Sep 17 00:00:00 2001 From: KingIcyCreamProjects Date: Fri, 17 Jul 2026 14:08:50 -0500 Subject: [PATCH 37/95] sampling: guard against non-finite logits (was silent token-0 spew) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the default serve path (TEMP>0, 0=u, and the fallback returned token 0. The engine then emitted an unbroken run of token 0 with NO error. The greedy path was equally blind: argmax_v started bv=lo[0] and `lo[i]>NaN` is always false, so a NaN at index 0 pinned the argmax to 0. - argmax_v: skip NaN (x==x) and seed from -inf, so it returns the max finite/+Inf entry instead of being NaN-pinned to 0. Covers greedy decode and the speculative-verify argmax path. - dist_build: after the softmax sum, if s is non-finite or <=0, collapse g_pbuf to a one-hot over the finite argmax and warn once, instead of dividing every entry into NaN. Covers the nucleus and verify paths. Both are O(1)/free on the happy path (one branch after the existing loop; one extra comparison inside the existing argmax loop). Degrade + diagnose, never silently corrupt. test_logit_nan (wired into TEST_BINS): asserts argmax_v skips NaN/picks +Inf, dist_build yields a finite normalized one-hot on the max finite logit, dist_sample emits that token (not 0), and clean logits still give a valid distribution. Fails on stock dev, passes with this change. Co-Authored-By: Claude Opus 4.8 --- c/Makefile | 5 +++- c/glm.c | 15 ++++++++++- c/tests/test_logit_nan.c | 55 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 c/tests/test_logit_nan.c diff --git a/c/Makefile b/c/Makefile index feb3045..062ef75 100644 --- a/c/Makefile +++ b/c/Makefile @@ -166,7 +166,7 @@ else PYTHON ?= python3 endif CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) ifneq (,$(LINUX)) TEST_BINS += tests/test_uring$(EXE) endif @@ -331,6 +331,9 @@ tests/bench_topp$(EXE): tests/bench_topp.c glm.c st.h uring.h json.h tok.h tok_u tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_logit_nan$(EXE): tests/test_logit_nan.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) diff --git a/c/glm.c b/c/glm.c index 92a4d78..9125587 100644 --- a/c/glm.c +++ b/c/glm.c @@ -4151,7 +4151,11 @@ static void mtp_absorb(Model *m, const int *next_ids, const float *x, int S, int } static inline int argmax_v(const float *lo, int V){ - int b=0; float bv=lo[0]; for(int i=1;ibv){bv=lo[i];b=i;} return b; + /* skip NaN (x==x is false for NaN) so a poisoned logit can't pin the argmax + * to index 0 — pick the max finite/+Inf entry instead. */ + int b=-1; float bv=-INFINITY; + for(int i=0;ibv){ bv=x; b=i; } } + return b<0?0:b; } /* ---- METODO F: draft grammaticale (#48) ---- @@ -4258,6 +4262,15 @@ static void dist_build(const float *lo, int V){ float mx=lo[0]; for(int i=1;imx) mx=lo[i]; double s=0; float invt=1.f/(g_temp>1e-4f?g_temp:1e-4f); for(int i=0;i1e300){ + static int warned=0; if(!warned){ warned=1; fprintf(stderr,"[sample] non-finite logits; emitting argmax of finite entries\n"); } + int b=argmax_v(lo,V); for(int i=0;i0 && g_nuc<1.f){ for(int i=0;i=u and fell + * through to return token 0 — so the engine silently emitted an unbroken run of + * token 0 with no error, on the DEFAULT serve path (TEMP>0, 0 +#include +#define main coli_glm_main_unused +#include "../glm.c" +#undef main + +static int approx1(double x){ return x > 0.999 && x < 1.001; } + +int main(void){ + /* --- argmax_v must skip NaN (greedy decode + speculative-verify paths) --- */ + { float lo[8]={NAN,1.f,5.f,2.f,NAN,-3.f,4.f,0.f}; + assert(argmax_v(lo,8)==2 && "pick max finite (idx2=5.0), not NaN-pinned idx0"); } + { float lo[8]={3.f,INFINITY,1.f,2.f,0.f,-1.f,2.5f,1.5f}; + assert(argmax_v(lo,8)==1 && "pick the +Inf position"); } + { float lo[8]; for(int i=0;i<8;i++) lo[i]=NAN; + assert(argmax_v(lo,8)==0 && "all-NaN: no crash, defined fallback"); } + + g_temp=0.7f; g_nuc=0.9f; /* the default serve/chat sampling path */ + + /* --- dist_build: a NaN logit must yield a finite one-hot, not all-NaN --- */ + { float lo[8]={0.5f,1.f,NAN,8.f,0.2f,-1.f,0.f,0.3f}; /* max finite = idx3 (8.0) */ + dist_build(lo,8); + double sum=0; int nan=0; + for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; } + assert(!nan && "g_pbuf must be finite after a NaN logit"); + assert(approx1(sum) && "g_pbuf must normalize to 1"); + assert(approx1(g_pbuf[3]) && "mass must land on the max finite logit (idx3)"); + assert(dist_sample(8,-1)==3 && "sampler emits the finite argmax, not token 0"); } + + /* --- regression: clean logits still produce a valid distribution --- */ + { float lo[8]={0.1f,0.2f,3.0f,0.4f,0.5f,0.6f,0.7f,0.8f}; /* peak = idx2 */ + dist_build(lo,8); + double sum=0; int nan=0; + for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; } + assert(!nan && "clean softmax stays finite"); + assert(approx1(sum) && "clean softmax must sum to 1"); + assert(g_pbuf[2]>=g_pbuf[0] && "peak token keeps the most mass"); } + + printf("OK test_logit_nan: argmax_v NaN-skip + dist_build finite-collapse\n"); + return 0; +} From e2d39abd2d4951606ca5c65fe025dc5cf8ea0fec Mon Sep 17 00:00:00 2001 From: KingIcyCreamProjects Date: Fri, 17 Jul 2026 14:39:14 -0500 Subject: [PATCH 38/95] sampling: NaN-skip the mx scan too (review follow-up on #369) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review: the collapse starts one line before the sum — seeding mx=lo[0] means a NaN at index 0 makes mx NaN, every (lo[i]-mx) NaN, and the softmax is doomed at the max-finding, not the normalize. Seed mx from -INFINITY and skip NaNs (x==x), mirroring the argmax_v change; if nothing finite survives, fall back to mx=0 and let the post-sum guard decide. The isfinite(s) guard is now the second line of defense rather than the only one. Clean logits take a byte-identical path (the extra x==x compare is noise next to V expf calls). test_logit_nan gains the NaN-at-index-0 and all-NaN dist_build cases; test_topp's 123-case sweep still passes on this tree, confirming no interaction with the #354 heap select. Co-Authored-By: Claude Opus 4.8 --- c/glm.c | 8 +++++++- c/tests/test_logit_nan.c | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/c/glm.c b/c/glm.c index 9125587..54dd7fa 100644 --- a/c/glm.c +++ b/c/glm.c @@ -4259,7 +4259,13 @@ static void topp_siftdown(int *h, int n, int i){ * la coda troncata va AZZERATA in g_pbuf (dist_sample la legge direttamente per id). */ static void dist_build(const float *lo, int V){ if(!g_pbuf){ g_pbuf=falloc(V); g_pidx=malloc(V*sizeof(int)); } - float mx=lo[0]; for(int i=1;imx) mx=lo[i]; + /* NaN-skip the max scan (x==x rules out NaN): seeding mx=lo[0] let a NaN at + * index 0 poison every (lo[i]-mx) before the sum — the softmax was doomed at + * the max-finding, not the normalize. The post-sum guard below stays as the + * second line of defense. */ + float mx=-INFINITY; + for(int i=0;imx) mx=lo[i]; + if(!isfinite(mx)) mx=0.f; /* all-NaN (or +Inf) logits: let the guard below decide */ double s=0; float invt=1.f/(g_temp>1e-4f?g_temp:1e-4f); for(int i=0;i Date: Sat, 18 Jul 2026 03:50:48 +0800 Subject: [PATCH 39/95] cuda: load ragged attention entry point on Windows --- c/backend_loader.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/c/backend_loader.c b/c/backend_loader.c index b743d1b..bbcb8ca 100644 --- a/c/backend_loader.c +++ b/c/backend_loader.c @@ -61,6 +61,9 @@ typedef int (*fn_attention_absorb_batch)(ColiCudaTensor *kv_b,float *ctx,const f typedef int (*fn_attention_absorb_batch_dev)(ColiCudaTensor *kv_b_shard,float *ctx_dev, const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); typedef int (*fn_attention_absorb_kvdev)(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale); typedef int (*fn_attention_project_batch)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q,const float *latent, const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale); +typedef int (*fn_attention_project_ragged)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, + float *out,const float *q,const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale); typedef int (*fn_attention_project_batch_dev)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); typedef int (*fn_attention_project_batch_dev_out)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); typedef int (*fn_pipe_add)(int device,float *x_dev,const float *t_dev,size_t n); @@ -107,6 +110,7 @@ static struct { fn_attention_absorb_batch_dev attention_absorb_batch_dev; fn_attention_absorb_kvdev attention_absorb_kvdev; fn_attention_project_batch attention_project_batch; + fn_attention_project_ragged attention_project_ragged; fn_attention_project_batch_dev attention_project_batch_dev; fn_attention_project_batch_dev_out attention_project_batch_dev_out; fn_pipe_add pipe_add; @@ -200,6 +204,7 @@ static int coli_cuda_load(void){ RESOLVE(attention_absorb_batch_dev, fn_attention_absorb_batch_dev) RESOLVE(attention_absorb_kvdev, fn_attention_absorb_kvdev) RESOLVE(attention_project_batch, fn_attention_project_batch) + RESOLVE(attention_project_ragged, fn_attention_project_ragged) RESOLVE(attention_project_batch_dev, fn_attention_project_batch_dev) RESOLVE(attention_project_batch_dev_out, fn_attention_project_batch_dev_out) RESOLVE(pipe_add, fn_pipe_add) @@ -342,6 +347,14 @@ int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,ColiCudaTensor *o_pro return g_cuda.attention_project_batch(kv_b, o_proj, out, q, latent, rope, S, H, Q, R, V, K, T, attention_scale); } +int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, + float *out,const float *q,const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale){ + if(!coli_cuda_load()) return 0; + return g_cuda.attention_project_ragged(kv_b,o_proj,out,q,latent,rope,lengths, + S,H,Q,R,V,K,max_t,attention_scale); +} + int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){ if(!g_cuda.available){ return 0; } return g_cuda.attention_project_batch_dev(kv_b, o_proj, out, q_dev, latent_dev, rope_dev, S, H, Q, R, V, K, T, scale); From c3a90eca36fa3387e78a8f48a31e5b34a5ff1bdf Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Sat, 18 Jul 2026 04:54:59 +0800 Subject: [PATCH 40/95] profile CPU GPU tier execution costs --- c/glm.c | 49 ++++++++++++++++++++------ c/tests/test_benchmark_cuda_fixture.py | 6 +++- c/tools/benchmark_cuda_fixture.py | 28 +++++++++++++-- 3 files changed, 69 insertions(+), 14 deletions(-) diff --git a/c/glm.c b/c/glm.c index 92a4d78..8ec64a1 100644 --- a/c/glm.c +++ b/c/glm.c @@ -201,7 +201,9 @@ typedef struct { uint64_t route_slots, route_swaps; /* CACHE_ROUTE: slots chosen / substituted vs true top-K */ uint64_t route_agree_hit, route_agree_tot; /* ROUTE_AGREE: |chosen ∩ true top-K| / K */ double route_kl_sum; uint64_t route_kl_n; /* mean KL(true||chosen) on gate mass */ - double t_ewait, t_emm, t_attn, t_kvb, t_head;/* profiling: dove va il tempo (wall del + double t_ewait, t_emm, t_ecpu, t_egpu, t_route, t_p2p, t_attn, t_kvb, t_head; + uint64_t n_p2p; /* P0 execution profile: tier split + residual hops */ + /* profiling: dove va il tempo (wall del * thread di compute; il servizio disco * overlappato vive in g_edisk_ns) */ double t_aproj,t_acore,t_aout; /* attention breakdown */ @@ -307,15 +309,16 @@ static uint64_t g_prof_nlat; /* forwards recorded (monotonic static void prof_lat(double s){ g_prof_lat[g_prof_nlat++ % PROF_LAT_CAP]=s; } /* snapshot for windowed reports (serve mode: one report per turn) */ typedef struct { - double edisk,ewait,emm,attn,head; - int64_t io; uint64_t hits,miss,ereq,n_fw,n_emit,nlat; + double edisk,ewait,emm,ecpu,egpu,route,p2p,attn,head; + int64_t io; uint64_t hits,miss,ereq,n_fw,n_emit,nlat,n_p2p; } ProfBase; static void prof_base(Model *m, ProfBase *b){ b->edisk=edisk_s(); b->ewait=m->t_ewait; b->emm=m->t_emm; + b->ecpu=m->t_ecpu; b->egpu=m->t_egpu; b->route=m->t_route; b->p2p=m->t_p2p; b->attn=m->t_attn; b->head=m->t_head; b->io=atomic_load_explicit(&g_prof_io,memory_order_relaxed); b->hits=m->hits; b->miss=m->miss; b->ereq=m->ereq; - b->n_fw=m->n_fw; b->n_emit=m->n_emit; b->nlat=g_prof_nlat; + b->n_fw=m->n_fw; b->n_emit=m->n_emit; b->nlat=g_prof_nlat; b->n_p2p=m->n_p2p; } static float *falloc(int64_t n){ @@ -2848,6 +2851,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int if(!rank_buf||!rank_w){ free(rank_buf); free(rank_w); rank_buf=NULL; rank_w=NULL; do_cache_route=0; } } /* ---- FASE A: routing di tutte le S posizioni ---- */ + double route_t0=g_prof?now_s():0; int *idxs=malloc((size_t)S*K*sizeof(int)); float *ws=malloc((size_t)S*K*sizeof(float)); int *keff=malloc(S*sizeof(int)); /* router in UN matmul batch: stessa matematica, via le S chiamate S=1 */ @@ -2997,6 +3001,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int for(int d=0;dt_route+=now_s()-route_t0; if(g_route_fp) g_route_call++; if(g_couple && cp_pred && S<=8) for(int s2=0;s2g.cuda,e->u.cuda,e->d.cuda,hh,xg,nr)){ for(int r=0;rt_emm+=now_s()-t0; continue; + double dt=now_s()-t0;m->t_emm+=dt;if(g_prof)m->t_egpu+=dt;continue; } if(!e->slab) expert_host_ensure(m,layer,e); #endif @@ -3272,7 +3277,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int matmul_qt(hh, gg, &e->d, nr); for(int r=0;rt_emm += now_s()-t0; + double dt=now_s()-t0;m->t_emm+=dt;if(g_prof)m->t_ecpu+=dt; } #ifdef COLI_CUDA ColiCudaTensor *dev_g[COLI_CUDA_MAX_DEVICES][64],*dev_u[COLI_CUDA_MAX_DEVICES][64]; @@ -3280,6 +3285,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int int dev_rows[COLI_CUDA_MAX_DEVICES][64],dev_which[COLI_CUDA_MAX_DEVICES][64]; int dev_nc[COLI_CUDA_MAX_DEVICES]={0},dev_total[COLI_CUDA_MAX_DEVICES]={0}; int dev_off[COLI_CUDA_MAX_DEVICES]={0},dev_ok[COLI_CUDA_MAX_DEVICES]={0}; + double dev_time[COLI_CUDA_MAX_DEVICES]={0}; for(int di=0;dig.cuda_device==g_cuda_devices[di]) dev_total[di]+=group_n[q]; for(int di=1;di1) schedule(static) - for(int di=0;dig.cuda,e->u.cuda,e->d.cuda,hh,xg,nr)){ expert_host_ensure(m,layer,e); expert_gate_up(gg,uu,xg,&e->g,&e->u,nr); for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z]; matmul_qt(hh,gg,&e->d,nr); } + if(g_prof)m->t_ecpu+=now_s()-tc; } float *src=dev_ok[di]?group_y+(int64_t)off*D:hh; for(int r=0;rmx)mx=dev_time[di];m->t_egpu+=mx;} m->t_emm+=now_s()-tg; #endif /* No drain barrier: the per-expert pipe_wait(qof[j]) above (issued for every @@ -3925,7 +3937,11 @@ static void layers_forward_rows(Model *m, float *x, int S, int pos_base, float *dst=coli_cuda_pipe_scratch(dev,15,xb); if(dst){ if(x_dev_on<0) ok=coli_cuda_pipe_upload(dev,dst,x,xb); - else if(x_dev_on!=dev){ ok=coli_cuda_pipe_peer_copy(dev,dst,x_dev_on,x_dev,xb); } + else if(x_dev_on!=dev){ + double tp=g_prof?now_s():0; + ok=coli_cuda_pipe_peer_copy(dev,dst,x_dev_on,x_dev,xb); + if(g_prof){m->t_p2p+=now_s()-tp;m->n_p2p++;} + } else dst=x_dev; if(ok){ x_dev=dst; x_dev_on=dev; @@ -4558,6 +4574,10 @@ static void profile_print(Model *m, double elapsed){ edisk_s(),m->t_ewait,m->t_emm,m->t_attn,m->t_kvb,m->t_head,elapsed-accounted); printf("ATTENTION: projection/RoPE %.3fs | score-softmax-value %.3fs | output projection %.3fs\n", m->t_aproj,m->t_acore,m->t_aout); + if(g_prof)printf("P0-EXEC: routed CPU %.3fs | routed GPU critical %.3fs | router %.3fs | residual P2P %.3fs / %llu hop | orchestration %.3fs\n", + m->t_ecpu,m->t_egpu,m->t_route,m->t_p2p,(unsigned long long)m->n_p2p, + elapsed-m->t_ewait-m->t_emm-m->t_attn-m->t_head-m->t_route-m->t_p2p>0? + elapsed-m->t_ewait-m->t_emm-m->t_attn-m->t_head-m->t_route-m->t_p2p:0); #ifdef COLI_METAL if(g_metal_enabled){ uint64_t ok=0,fb=0,ex=0; double su=0,gp=0,sc=0; coli_metal_moe_counts(&ok,&fb,&ex); coli_metal_moe_times(&su,&gp,&sc); @@ -4571,6 +4591,7 @@ static void profile_print(Model *m, double elapsed){ static void profile_reset(Model *m){ m->t_ewait=m->t_emm=m->t_attn=m->t_kvb=m->t_head=0; + m->t_ecpu=m->t_egpu=m->t_route=m->t_p2p=0;m->n_p2p=0; m->t_aproj=m->t_acore=m->t_aout=0; atomic_store_explicit(&g_edisk_ns,0,memory_order_relaxed); } @@ -4615,11 +4636,19 @@ static void prof_report(Model *m, const ProfBase *b, double elapsed, int tokens, io_svc,io_w); fprintf(f,"[PROF] resident experts: %d pinned (%.1f GB) + %d in LRU (%.1f GB, cap %d/layer)\n", pinned,pinned*eb/1e9,lru,lru*eb/1e9,m->ecap); - double emm=m->t_emm-b->emm, attn=m->t_attn-b->attn, head=m->t_head-b->head; - double other=elapsed-io_w-emm-attn-head; if(other<0) other=0; + double emm=m->t_emm-b->emm, ecpu=m->t_ecpu-b->ecpu, egpu=m->t_egpu-b->egpu; + double route=m->t_route-b->route,p2p=m->t_p2p-b->p2p; + uint64_t np2p=m->n_p2p-b->n_p2p; + double attn=m->t_attn-b->attn, head=m->t_head-b->head; + double other=elapsed-io_w-emm-attn-head-route-p2p; if(other<0) other=0; double f_io=io_w/elapsed, f_emm=emm/elapsed, f_attn=attn/elapsed; fprintf(f,"[PROF] time shares: expert-I/O %.0f%% | expert-matmul %.0f%% | attention %.0f%% | lm_head %.0f%% | other %.0f%%\n", 100*f_io,100*f_emm,100*f_attn,100*head/elapsed,100*other/elapsed); + double slow=ecpu>egpu?ecpu:egpu,fast=ecpu1e-9?slow/fast:0.0,route,p2p,(unsigned long long)np2p, + np2p?p2p*1e3/np2p:0.0,other); if(f_io>=0.30){ fprintf(f,"[PROF] verdict: I/O-bound — %.0f%% of the time waits on expert reads (hit %.0f%%).",100*f_io,hitp); if(hitp<90) fprintf(f," More cache is the lever: raise RAM_GB (or add RAM)."); diff --git a/c/tests/test_benchmark_cuda_fixture.py b/c/tests/test_benchmark_cuda_fixture.py index a704052..ca317c9 100644 --- a/c/tests/test_benchmark_cuda_fixture.py +++ b/c/tests/test_benchmark_cuda_fixture.py @@ -1,11 +1,12 @@ import unittest -from tools.benchmark_cuda_fixture import parse_output +from tools.benchmark_cuda_fixture import parse_output, parse_p0 SAMPLE = """ REPLAY decode: 4 tokens | 12.34 tok/s PROFILE: expert-disk 1.25s | expert-matmul 2.50s | attention 0.75s | lm_head 0.10s | other -0.05s +P0-EXEC: routed CPU 1.200s | routed GPU critical 0.150s | router 0.200s | residual P2P 0.030s / 75 hop | orchestration 0.100s """ @@ -19,6 +20,9 @@ class ParseOutputTest(unittest.TestCase): with self.assertRaisesRegex(RuntimeError, "benchmark output missing"): parse_output("REPLAY decode: 4 tokens | 12.34 tok/s", "engine failed") + def test_extracts_p0_profile(self): + self.assertEqual(parse_p0(SAMPLE), [1.2, 0.15, 0.2, 0.03, 75.0, 0.1]) + if __name__ == "__main__": unittest.main() diff --git a/c/tools/benchmark_cuda_fixture.py b/c/tools/benchmark_cuda_fixture.py index bced18e..849bb55 100644 --- a/c/tools/benchmark_cuda_fixture.py +++ b/c/tools/benchmark_cuda_fixture.py @@ -17,6 +17,11 @@ PROFILE_RE = re.compile( r"\| attention ([0-9.]+)s .* lm_head ([0-9.]+)s \| other ([0-9.-]+)s" ) PROFILE_KEYS = ("disk", "expert_matmul", "attention", "lm_head", "other") +P0_RE = re.compile( + r"P0-EXEC: routed CPU ([0-9.]+)s \| routed GPU critical ([0-9.]+)s \| " + r"router ([0-9.]+)s \| residual P2P ([0-9.]+)s / ([0-9]+) hop \| orchestration ([0-9.]+)s" +) +P0_KEYS = ("routed_cpu", "routed_gpu_critical", "router", "p2p", "p2p_hops", "orchestration") def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]: @@ -30,11 +35,20 @@ def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]: return float(speed.group(1)), [disk] + [float(value) for value in rest] -def execute(engine: str, env: dict[str, str]) -> tuple[float, list[float]]: +def parse_p0(stdout: str) -> list[float]: + """Extract the optional PROF=1 execution-layer breakdown.""" + row = P0_RE.search(stdout) + if not row: + raise RuntimeError("benchmark output missing P0-EXEC profile") + return [float(value) for value in row.groups()] + + +def execute(engine: str, env: dict[str, str]) -> tuple[float, list[float], list[float]]: run = subprocess.run( [engine, "4", "4", "4"], env=env, text=True, capture_output=True, check=True ) - return parse_output(run.stdout, run.stderr) + speed, profile = parse_output(run.stdout, run.stderr) + return speed, profile, parse_p0(run.stdout) def main() -> None: @@ -63,6 +77,8 @@ def main() -> None: OMP_NUM_THREADS=str(args.threads), OMP_PROC_BIND="spread", OMP_PLACES="cores", + DRAFT="0", + PROF="1", ) execute(args.engine, base | {"STATS": str(stats)}) @@ -86,13 +102,15 @@ def main() -> None: execute(args.engine, base | extra) # warm-up speeds = {name: [] for name in modes} profiles = {name: [] for name in modes} + p0_profiles = {name: [] for name in modes} names = list(modes) for run_index in range(args.runs): order = names[run_index % len(names):] + names[:run_index % len(names)] for name in order: - speed, profile = execute(args.engine, base | modes[name]) + speed, profile, p0 = execute(args.engine, base | modes[name]) speeds[name].append(speed) profiles[name].append(profile) + p0_profiles[name].append(p0) result = {} for name in names: @@ -103,6 +121,10 @@ def main() -> None: key: statistics.median(row[index] for row in profiles[name]) for index, key in enumerate(PROFILE_KEYS) }, + "median_p0": { + key: statistics.median(row[index] for row in p0_profiles[name]) + for index, key in enumerate(P0_KEYS) + }, } print(json.dumps(result, indent=2)) From 946fcd4f9f9d7c6558f8704f2393eb97910abba7 Mon Sep 17 00:00:00 2001 From: bokiko Date: Sat, 18 Jul 2026 00:19:39 +0300 Subject: [PATCH 41/95] =?UTF-8?q?glm:=20fmt=3D4=20support=20in=20qt=5Faddr?= =?UTF-8?q?ow/qt=5Fmatvec=5Frows=20=E2=80=94=20CPU=20absorb=20path=20decod?= =?UTF-8?q?ed=20grouped=20int4=20as=20int2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An all-grouped container (kv_b_proj at fmt=4) generates one correct token and then EOS: qt_addrow and qt_matvec_rows handle fmt 0/1/2 and fall through to the int2 decoder, so grouped-int4 kv_b was unpacked as 2-bit pairs under a per-row scale that does not exist in the [O,ng] layout. Prefill (S>4, reconstruction) is unaffected, which made the failure look like an EOS bug rather than an attention bug. Same class as #298 (CUDA absorb kernels missing fmt=4), CPU side. Existing containers escape it because the recommended mixed-precision recipe keeps kv_b at int8 (#237). Adds per-group branches mirroring matmul_i4_grouped semantics. fmt 0/1/2/3 paths are untouched. Co-Authored-By: Claude --- c/glm.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/c/glm.c b/c/glm.c index 92a4d78..c869b9c 100644 --- a/c/glm.c +++ b/c/glm.c @@ -2284,6 +2284,18 @@ static void expert_prefetch(Model *m, int layer, int eid){ static void qt_addrow(const QT *t, int row, float coef, float *acc){ int I=t->I; if(t->fmt==0){ const float *w=t->qf+(int64_t)row*I; for(int i=0;ifmt==4){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); + int gs=t->gs, ng=(I+gs-1)/gs; const float *scl=t->s+(int64_t)row*ng; + for(int i=0;i+1>1]; + acc[i] +=coef*scl[i/gs] *((int)(b&0xF)-8); + acc[i+1]+=coef*scl[(i+1)/gs]*((int)(b>>4)-8); } + if(I&1){ uint8_t b=w[I>>1]; acc[I-1]+=coef*scl[(I-1)/gs]*((int)(b&0xF)-8); } return; } float c=coef*t->s[row]; if(t->fmt==1){ const int8_t *w=t->q8+(int64_t)row*I; for(int i=0;ifmt==2){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); @@ -2302,6 +2314,13 @@ static void qt_matvec_rows(const QT *t, int r0, int n, const float *x, float *y) else if(t->fmt==2){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); float s=t->s[row]; float acc=0; for(int i=0;i+1>1]; acc+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } if(I&1){ uint8_t b=w[I>>1]; acc+=((int)(b&0xF)-8)*x[I-1]; } a=acc*s; } + else if(t->fmt==4){ /* per-gruppo, come matmul_i4_grouped / per-group, as matmul_i4_grouped */ + const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); + int gs=t->gs, ng=(I+gs-1)/gs; const float *scl=t->s+(int64_t)row*ng; + for(int g=0; g*gsI?I:base+gs; float acc=0; + for(int i=base;i>1]; + acc+=(float)((i&1)?((int)(b>>4)-8):((int)(b&0xF)-8))*x[i]; } + a+=(double)acc*scl[g]; } } else { const uint8_t *w=t->q4+(int64_t)row*((I+3)/4); float s=t->s[row]; float acc=0; for(int i=0;i>2]; acc+=((int)((b>>((i&3)*2))&3)-2)*x[i]; } a=acc*s; } y[j]=(float)a; From 73e0e5ec82e98ef40bcfab5abd42c6776e6a381e Mon Sep 17 00:00:00 2001 From: KingIcyCreamProjects Date: Fri, 17 Jul 2026 16:24:33 -0500 Subject: [PATCH 42/95] docs(api): add "Connect a coding CLI or editor" recipe (#373) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documents connecting OpenAI-compatible coding CLIs to `coli serve`. Covers the common snag reported in #373 — clients like crush refuse to start without an API key even though the local endpoint needs none — by showing that any dummy key works (Colibri only enforces COLI_API_KEY if set). Concrete recipes for aider and crush, a curl smoke test, the generic base-URL/model/key pattern for other tools, and an honest tok/s-latency caveat for the streaming path. Co-Authored-By: Claude Opus 4.8 --- docs/api.md | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/docs/api.md b/docs/api.md index d3b09f1..cb29eb2 100644 --- a/docs/api.md +++ b/docs/api.md @@ -49,6 +49,67 @@ errors before streaming headers are sent. `GET /health` exposes active/queued/completed/rejected counters, and successful generation responses include `x-colibri-queue-wait-ms`. +## Connect a coding CLI or editor + +The API is OpenAI-compatible, so most coding CLIs and editor extensions work by +pointing them at Colibri as an *OpenAI-compatible* provider. Three settings: + +- **Base URL** — `http://localhost:8000/v1` +- **Model** — `glm-5.2-colibri` (or whatever you pass to `--model-id`) +- **API key** — any non-empty string, e.g. `local` + +Colibri needs **no** API key by default, but many clients refuse to start without +one — give them any dummy value. The key is only enforced if you set `COLI_API_KEY`. + +Smoke-test the endpoint first (no key needed unless you set one): + +```bash +curl http://127.0.0.1:8000/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"glm-5.2-colibri","messages":[{"role":"user","content":"hi"}]}' +``` + +**aider** + +```bash +export OPENAI_API_BASE=http://localhost:8000/v1 +export OPENAI_API_KEY=local +aider --model openai/glm-5.2-colibri # the openai/ prefix routes to OPENAI_API_BASE +``` + +**crush** — add a provider to `crush.json` (`~/.config/crush/crush.json`, or +`%USERPROFILE%\AppData\Local\crush\crush.json` on Windows): + +```json +{ + "$schema": "https://charm.land/crush.json", + "providers": { + "colibri": { + "name": "Colibri", + "type": "openai-compat", + "base_url": "http://localhost:8000/v1/", + "api_key": "local", + "models": [ + { "name": "GLM-5.2 (Colibri)", "id": "glm-5.2-colibri", + "context_window": 131072, "default_max_tokens": 1024 } + ] + } + } +} +``` + +The `"api_key": "local"` dummy is what satisfies clients that demand a key. +`context_window` is only the client's budget display — set it to whatever your +KV configuration actually allows. + +**Continue, Cline / Roo, `llm`, the OpenAI SDKs, …** — set the provider's base +URL to `http://localhost:8000/v1`, the model to `glm-5.2-colibri`, and any dummy +key (`OPENAI_API_KEY` / `OPENAI_BASE_URL` for env-based tools). + +> On the CPU-streaming path a large model decodes at roughly 1 tok/s, so +> interactive agent loops will feel slow — it connects and works, but the latency +> is very different from a hosted model. + ## Isolated KV contexts `coli serve --kv-slots N` allocates up to 16 independent sequence contexts. From 570d738ed5c7ae8a43aed5a4eb23b8948c48c41f Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Sat, 18 Jul 2026 05:34:41 +0800 Subject: [PATCH 43/95] profile effective CPU expert bandwidth --- c/glm.c | 23 +++++++++++++++++------ c/tests/test_benchmark_cuda_fixture.py | 4 ++-- c/tools/benchmark_cuda_fixture.py | 5 +++-- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/c/glm.c b/c/glm.c index 8ec64a1..f5dd33b 100644 --- a/c/glm.c +++ b/c/glm.c @@ -203,6 +203,7 @@ typedef struct { double route_kl_sum; uint64_t route_kl_n; /* mean KL(true||chosen) on gate mass */ double t_ewait, t_emm, t_ecpu, t_egpu, t_route, t_p2p, t_attn, t_kvb, t_head; uint64_t n_p2p; /* P0 execution profile: tier split + residual hops */ + uint64_t cpu_expert_rows; int64_t cpu_expert_bytes; /* profiling: dove va il tempo (wall del * thread di compute; il servizio disco * overlappato vive in g_edisk_ns) */ @@ -310,7 +311,7 @@ static void prof_lat(double s){ g_prof_lat[g_prof_nlat++ % PROF_LAT_CAP]=s; } /* snapshot for windowed reports (serve mode: one report per turn) */ typedef struct { double edisk,ewait,emm,ecpu,egpu,route,p2p,attn,head; - int64_t io; uint64_t hits,miss,ereq,n_fw,n_emit,nlat,n_p2p; + int64_t io,cpu_bytes; uint64_t hits,miss,ereq,n_fw,n_emit,nlat,n_p2p,cpu_rows; } ProfBase; static void prof_base(Model *m, ProfBase *b){ b->edisk=edisk_s(); b->ewait=m->t_ewait; b->emm=m->t_emm; @@ -319,6 +320,7 @@ static void prof_base(Model *m, ProfBase *b){ b->io=atomic_load_explicit(&g_prof_io,memory_order_relaxed); b->hits=m->hits; b->miss=m->miss; b->ereq=m->ereq; b->n_fw=m->n_fw; b->n_emit=m->n_emit; b->nlat=g_prof_nlat; b->n_p2p=m->n_p2p; + b->cpu_bytes=m->cpu_expert_bytes;b->cpu_rows=m->cpu_expert_rows; } static float *falloc(int64_t n){ @@ -3277,7 +3279,9 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int matmul_qt(hh, gg, &e->d, nr); for(int r=0;rt_emm+=dt;if(g_prof)m->t_ecpu+=dt; + double dt=now_s()-t0;m->t_emm+=dt;if(g_prof){m->t_ecpu+=dt; + m->cpu_expert_bytes+=qt_bytes(&e->g)+qt_bytes(&e->u)+qt_bytes(&e->d); + m->cpu_expert_rows+=(uint64_t)nr;} } #ifdef COLI_CUDA ColiCudaTensor *dev_g[COLI_CUDA_MAX_DEVICES][64],*dev_u[COLI_CUDA_MAX_DEVICES][64]; @@ -3320,6 +3324,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int expert_gate_up(gg,uu,xg,&e->g,&e->u,nr); for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z]; matmul_qt(hh,gg,&e->d,nr); + if(g_prof){m->cpu_expert_bytes+=qt_bytes(&e->g)+qt_bytes(&e->u)+qt_bytes(&e->d); + m->cpu_expert_rows+=(uint64_t)nr;} } if(g_prof)m->t_ecpu+=now_s()-tc; } @@ -4574,8 +4580,9 @@ static void profile_print(Model *m, double elapsed){ edisk_s(),m->t_ewait,m->t_emm,m->t_attn,m->t_kvb,m->t_head,elapsed-accounted); printf("ATTENTION: projection/RoPE %.3fs | score-softmax-value %.3fs | output projection %.3fs\n", m->t_aproj,m->t_acore,m->t_aout); - if(g_prof)printf("P0-EXEC: routed CPU %.3fs | routed GPU critical %.3fs | router %.3fs | residual P2P %.3fs / %llu hop | orchestration %.3fs\n", - m->t_ecpu,m->t_egpu,m->t_route,m->t_p2p,(unsigned long long)m->n_p2p, + if(g_prof)printf("P0-EXEC: routed CPU %.3fs / %.2f GB/s (%llu row) | routed GPU critical %.3fs | router %.3fs | residual P2P %.3fs / %llu hop | orchestration %.3fs\n", + m->t_ecpu,m->t_ecpu>0?m->cpu_expert_bytes/1e9/m->t_ecpu:0.0, + (unsigned long long)m->cpu_expert_rows,m->t_egpu,m->t_route,m->t_p2p,(unsigned long long)m->n_p2p, elapsed-m->t_ewait-m->t_emm-m->t_attn-m->t_head-m->t_route-m->t_p2p>0? elapsed-m->t_ewait-m->t_emm-m->t_attn-m->t_head-m->t_route-m->t_p2p:0); #ifdef COLI_METAL @@ -4592,6 +4599,7 @@ static void profile_print(Model *m, double elapsed){ static void profile_reset(Model *m){ m->t_ewait=m->t_emm=m->t_attn=m->t_kvb=m->t_head=0; m->t_ecpu=m->t_egpu=m->t_route=m->t_p2p=0;m->n_p2p=0; + m->cpu_expert_bytes=0;m->cpu_expert_rows=0; m->t_aproj=m->t_acore=m->t_aout=0; atomic_store_explicit(&g_edisk_ns,0,memory_order_relaxed); } @@ -4639,15 +4647,18 @@ static void prof_report(Model *m, const ProfBase *b, double elapsed, int tokens, double emm=m->t_emm-b->emm, ecpu=m->t_ecpu-b->ecpu, egpu=m->t_egpu-b->egpu; double route=m->t_route-b->route,p2p=m->t_p2p-b->p2p; uint64_t np2p=m->n_p2p-b->n_p2p; + int64_t cpu_bytes=m->cpu_expert_bytes-b->cpu_bytes; + uint64_t cpu_rows=m->cpu_expert_rows-b->cpu_rows; double attn=m->t_attn-b->attn, head=m->t_head-b->head; double other=elapsed-io_w-emm-attn-head-route-p2p; if(other<0) other=0; double f_io=io_w/elapsed, f_emm=emm/elapsed, f_attn=attn/elapsed; fprintf(f,"[PROF] time shares: expert-I/O %.0f%% | expert-matmul %.0f%% | attention %.0f%% | lm_head %.0f%% | other %.0f%%\n", 100*f_io,100*f_emm,100*f_attn,100*head/elapsed,100*other/elapsed); double slow=ecpu>egpu?ecpu:egpu,fast=ecpu1e-9?slow/fast:0.0,route,p2p,(unsigned long long)np2p, + ecpu,ecpu>0?cpu_bytes/1e9/ecpu:0.0,(unsigned long long)cpu_rows, + egpu,fast>1e-9?slow/fast:0.0,route,p2p,(unsigned long long)np2p, np2p?p2p*1e3/np2p:0.0,other); if(f_io>=0.30){ fprintf(f,"[PROF] verdict: I/O-bound — %.0f%% of the time waits on expert reads (hit %.0f%%).",100*f_io,hitp); diff --git a/c/tests/test_benchmark_cuda_fixture.py b/c/tests/test_benchmark_cuda_fixture.py index ca317c9..3feaf11 100644 --- a/c/tests/test_benchmark_cuda_fixture.py +++ b/c/tests/test_benchmark_cuda_fixture.py @@ -6,7 +6,7 @@ from tools.benchmark_cuda_fixture import parse_output, parse_p0 SAMPLE = """ REPLAY decode: 4 tokens | 12.34 tok/s PROFILE: expert-disk 1.25s | expert-matmul 2.50s | attention 0.75s | lm_head 0.10s | other -0.05s -P0-EXEC: routed CPU 1.200s | routed GPU critical 0.150s | router 0.200s | residual P2P 0.030s / 75 hop | orchestration 0.100s +P0-EXEC: routed CPU 1.200s / 123.40 GB/s (456 row) | routed GPU critical 0.150s | router 0.200s | residual P2P 0.030s / 75 hop | orchestration 0.100s """ @@ -21,7 +21,7 @@ class ParseOutputTest(unittest.TestCase): parse_output("REPLAY decode: 4 tokens | 12.34 tok/s", "engine failed") def test_extracts_p0_profile(self): - self.assertEqual(parse_p0(SAMPLE), [1.2, 0.15, 0.2, 0.03, 75.0, 0.1]) + self.assertEqual(parse_p0(SAMPLE), [1.2, 123.4, 456.0, 0.15, 0.2, 0.03, 75.0, 0.1]) if __name__ == "__main__": diff --git a/c/tools/benchmark_cuda_fixture.py b/c/tools/benchmark_cuda_fixture.py index 849bb55..26512dd 100644 --- a/c/tools/benchmark_cuda_fixture.py +++ b/c/tools/benchmark_cuda_fixture.py @@ -18,10 +18,11 @@ PROFILE_RE = re.compile( ) PROFILE_KEYS = ("disk", "expert_matmul", "attention", "lm_head", "other") P0_RE = re.compile( - r"P0-EXEC: routed CPU ([0-9.]+)s \| routed GPU critical ([0-9.]+)s \| " + r"P0-EXEC: routed CPU ([0-9.]+)s / ([0-9.]+) GB/s \(([0-9]+) row\) \| routed GPU critical ([0-9.]+)s \| " r"router ([0-9.]+)s \| residual P2P ([0-9.]+)s / ([0-9]+) hop \| orchestration ([0-9.]+)s" ) -P0_KEYS = ("routed_cpu", "routed_gpu_critical", "router", "p2p", "p2p_hops", "orchestration") +P0_KEYS = ("routed_cpu", "routed_cpu_gb_s", "routed_cpu_rows", "routed_gpu_critical", + "router", "p2p", "p2p_hops", "orchestration") def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]: From 153ee16f61b50633c6ce30008ce56c13aea49069 Mon Sep 17 00:00:00 2001 From: woolcoxm <13604288+woolcoxm@users.noreply.github.com> Date: Fri, 17 Jul 2026 18:17:29 -0400 Subject: [PATCH 44/95] =?UTF-8?q?bench:=20multi-seed=20bench=5Fdsa=5Fselec?= =?UTF-8?q?t=20=E2=80=94=20kill=20single-input=20pivot-luck=20spike=20(#35?= =?UTF-8?q?7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @KingIcyCreamProjects caught a real artifact on the 9950X3D (#357 thread): the nk=8192 cell reported ~75x, far above the real ~13-40x algorithm. Root cause was two compounding bench bugs, both verified against the code: 1. ONE frozen input per cell. partial_select_desc's median-of-three pivot is fully deterministic (no RNG), and bench_dsa_select froze the input then ran 2000 reps on that identical array -- so all 2000 reps hit the exact same pivot sequence. A single lucky input spiked one nk row (stable across re-runs because it's deterministic, not because it's real). 2. brng was a static global, initialized once and NEVER reset between cells. So each (shape,nk) cell's input depended on every prior cell's brand() draws -- reordering nks[] or adding a shape silently shifted all later inputs. Fix: - brng_seed() reseeds per (shape, nk, seed) so every cell is reproducible and independent of cell ordering. - Report the MEDIAN of N_SEEDS=11 independent inputs, each itself a median over N_REPEAT=2000 timing reps. A lucky pivot now moves one of 11 samples, not the reported number. Measured on Intel Core Ultra 9 185H (this fix, median of 11 seeds): realistic@8192 10.97x (was 13.57x single-seed on this box) uniform@8192 9.39x (was 7.48x) plateau@8192 16.11x (plateau ignores the RNG, unchanged as expected) band: 6-26x across all cells, no anomalous spikes. Correctness is unaffected (test_dsa_select, 129 cases, unchanged). This is a bench fidelity fix only -- no engine code touched. --- c/tests/bench_dsa_select.c | 61 +++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/c/tests/bench_dsa_select.c b/c/tests/bench_dsa_select.c index 1fd3e6b..a1f2014 100644 --- a/c/tests/bench_dsa_select.c +++ b/c/tests/bench_dsa_select.c @@ -59,6 +59,12 @@ static double bench_ns(void (*fn)(const float*,int,int,int*,int*), return ts[N_REPEAT/2]; } +/* Sort an array of doubles ascending (median-of-medians aggregation below). */ +static void dsort(double *a, int n){ + for(int s=1;s=0 && a[b]>k){ a[b+1]=a[b]; b--; } a[b+1]=k; } +} + /* the NEW algorithm calls the real partial_select_desc + replicates the production * threshold derivation and position scans. */ static void keep_new(const float *isc, int nk, int keep, int *dst, int *nd_out){ @@ -74,6 +80,7 @@ static void keep_new(const float *isc, int nk, int keep, int *dst, int *nd_out){ /* deterministic score fill for three shapes */ static uint32_t brng = 0xA5A5A5A5u; +static void brng_seed(uint32_t s){ brng = s; } static double brand(void){ brng ^= brng << 13; brng ^= brng >> 17; brng ^= brng << 5; return (double)(brng >> 8) * (1.0 / 16777216.0); } static void fill_realistic(float *isc, int nk){ /* few hot, long distinct tail */ @@ -87,11 +94,24 @@ static void fill_plateau(float *isc, int nk){ /* tie blocks -> boundary path for(int i=0;i Date: Sat, 18 Jul 2026 01:10:33 +0200 Subject: [PATCH 45/95] sampling: survive non-finite logits instead of emitting token 0 forever (#369) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single NaN or +Inf logit silently broke the default sampling path. +Inf became `mx`, then `expf((Inf-mx))`/`expf((NaN-mx))` is NaN, the softmax sum went NaN, every probability went NaN — and dist_sample's fallback loop `if(g_pbuf[i]>0)` is false for NaN at every index, so it returned 0. Every subsequent token: 0. No error, no warning. @KingIcyCreamProjects found it. dist_build now takes `mx` over finite logits only, gives a non-finite logit probability 0, and when the distribution is unusable (no finite logit, or a non-finite/zero sum) collapses to a delta on the finite argmax and warns ONCE on stderr — degraded, but a valid token and a visible cause, never a silent stream of zeros. The finite argmax uses the index found during the mx pass (robust even when lo[0] itself is NaN, where argmax_v would wrongly return 0). tests/test_sample_nan.c: healthy logits still sample correctly; NaN/+Inf injected at lo[0], the middle, and the last position all pick the finite argmax; an all-non-finite vocab leaves no NaN in the buffer and doesn't crash. Co-Authored-By: Claude Opus 4.8 --- c/Makefile | 5 +++- c/glm.c | 23 +++++++++++++-- c/tests/test_sample_nan.c | 62 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 c/tests/test_sample_nan.c diff --git a/c/Makefile b/c/Makefile index feb3045..7210cbb 100644 --- a/c/Makefile +++ b/c/Makefile @@ -166,7 +166,7 @@ else PYTHON ?= python3 endif CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) ifneq (,$(LINUX)) TEST_BINS += tests/test_uring$(EXE) endif @@ -328,6 +328,9 @@ tests/test_topp$(EXE): tests/test_topp.c glm.c st.h uring.h json.h tok.h tok_uni tests/bench_topp$(EXE): tests/bench_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_sample_nan$(EXE): tests/test_sample_nan.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) diff --git a/c/glm.c b/c/glm.c index bf8acb1..2ee12ed 100644 --- a/c/glm.c +++ b/c/glm.c @@ -4296,9 +4296,28 @@ static void topp_siftdown(int *h, int n, int i){ * la coda troncata va AZZERATA in g_pbuf (dist_sample la legge direttamente per id). */ static void dist_build(const float *lo, int V){ if(!g_pbuf){ g_pbuf=falloc(V); g_pidx=malloc(V*sizeof(int)); } - float mx=lo[0]; for(int i=1;imx) mx=lo[i]; + /* Un solo logit NaN/+Inf avvelenava tutto (#369): +Inf diventava mx, NaN/Inf-mx + * -> expf NaN -> s NaN -> ogni prob NaN -> dist_sample cade sul fallback + * `g_pbuf[i]>0` (NaN>0 e' falso ovunque) e ritorna 0 PER SEMPRE, in silenzio. + * Difesa: mx solo sui finiti; un logit non finito contribuisce prob 0; + * se la distribuzione degenera (tutti non finiti / somma non valida) si + * ripiega sull'argmax dei finiti e si avvisa UNA volta, mai in silenzio. */ + int mxi=-1; float mx=0; + for(int i=0;imx)){ mx=lo[i]; mxi=i; } double s=0; float invt=1.f/(g_temp>1e-4f?g_temp:1e-4f); - for(int i=0;i=0){ + for(int i=0;i=0)?mxi:0; /* mxi = argmax dei logit FINITI (robusto + * anche se lo[0] e' NaN, dove argmax_v fallirebbe) */ + for(int i=0;i0 && g_nuc<1.f){ for(int i=0;i0` e' falso su NaN ovunque). + * Ora la distribuzione degenere ripiega sull'argmax dei logit FINITI e avvisa una volta. + * + * Il test verifica: (a) con logit sani il campionamento resta corretto; (b) con NaN/+Inf + * iniettato il token scelto e' l'argmax dei FINITI (mai 0 per default), su ogni posizione + * del NaN inclusa lo[0]; (c) nessun NaN sopravvive in g_pbuf. */ +#define main coli_glm_main_unused +#include "../glm.c" +#undef main +#include + +static uint32_t rs=0x1234abcd; static uint32_t xr(){rs^=rs<<13;rs^=rs>>17;rs^=rs<<5;return rs;} + +static int pbuf_has_nan(int V){ for(int i=0;i 0 e nessun NaN nel buffer */ + for(int i=0;ipv){pv=g_pbuf[i];picked=i;} + if(b<2 && picked!=amax){ /* NaN e +Inf: delta esatto su amax */ + printf(" FAIL: %s @ %d -> picked %d, atteso argmax finito %d\n",bn[b],at,picked,amax); fail=1; + } + } + } + if(!fail) printf(" NaN/+Inf iniettato: argmax dei finiti vince, mai 0/NaN ok\n"); + + /* (c) caso estremo: TUTTI non finiti -> non deve crashare, buffer valido */ + for(int i=0;i Date: Sat, 18 Jul 2026 01:15:48 +0200 Subject: [PATCH 46/95] docs: collapse WINDOWS.md/windows.md case collision into one file (#371) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs/WINDOWS.md (the #329 install walkthrough) and docs/windows.md (the concise reference from the README restructure) differ only in case. On case-insensitive filesystems — every default Windows and macOS checkout — the two paths collide: the working tree can never be clean, and `git rebase`/`git status` refuse to run. @KingIcyCreamProjects reported it. Everything links to the lowercase docs/windows.md (README.md, README.zh-TW.md, docs/cuda.md), so that is the canonical name. This keeps it and folds in both contents: the step-by-step walkthrough (download-first, Smart App Control, CUDA DLL, first run, reference numbers, failure index) followed by the build-flags reference (AVX-VNNI, warmup). docs/WINDOWS.md is removed. No links change. Co-Authored-By: Claude Opus 4.8 --- docs/WINDOWS.md | 100 ----------------------------------------- docs/windows.md | 116 +++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 99 insertions(+), 117 deletions(-) delete mode 100644 docs/WINDOWS.md diff --git a/docs/WINDOWS.md b/docs/WINDOWS.md deleted file mode 100644 index 52d83cd..0000000 --- a/docs/WINDOWS.md +++ /dev/null @@ -1,100 +0,0 @@ -# Windows 11 native install — a complete walkthrough (no WSL) - -A start-to-finish, reproducible path from a fresh Windows 11 machine to GLM-5.2 generating tokens, with the GPU tier. Every step and every failure mode below was hit and verified on real hardware: Core Ultra 9 285K (AVX-VNNI) / RTX 5080 (sm_120) / 128 GB RAM / Windows 11 24H2 (issue #306). Steps are ordered so the long downloads run while you build. - -## 0. What you need - -| Piece | Why | Get it | -|---|---|---| -| git, Python 3 | clone + `coli` launcher | winget / python.org | -| MinGW-w64 gcc + make | builds the engine (MSVC can't) | `scoop install mingw-winlibs`, MSYS2, or portable **w64devkit** (no admin, unzip and go) | -| CUDA Toolkit ≥ 12.8 | GPU tier; ≥12.8 required for Blackwell/sm_120 | `winget install Nvidia.CUDA` | -| MSVC Build Tools (C++ workload) | nvcc's host compiler for the CUDA DLL | `winget install Microsoft.VisualStudio.2022.BuildTools` + "Desktop development with C++" | -| ~400 GB free on a local NVMe | the int4 model (~370–384 GB) | NTFS is fine; **never** a network mount | - -RAM: 16 GB minimum, more = bigger expert cache = faster. The build itself needs none of the CUDA/MSVC pieces — do the CPU build first, add the GPU tier later. - -## 1. Start the model download first (it's the long pole) - -```powershell -python -m pip install -U "huggingface_hub[hf_transfer]" -$env:HF_HUB_ENABLE_HF_TRANSFER = "1" -hf download --local-dir D:\glm52_i4 -``` - -Use the container recommended in the README (with **int8 MTP heads** — int4 heads silently give 0% draft acceptance). The download is resumable: if it stops, rerun the same command. Expect hours; everything below fits inside them. - -## 2. Build the engine (CPU) - -From a normal PowerShell, in the repo's `c\` directory: - -```powershell -make glm.exe ARCH=native # ARCH=native unlocks AVX-VNNI on Alder Lake+/Arrow Lake -make iobench.exe # disk benchmark, useful before committing to the download -``` - -Warnings about `#pragma comment` and unused variables are normal (MSVC-isms gcc ignores). The engine banner should print `idot: avx-vnni` on VNNI-capable CPUs — if it says avx2, you built without `ARCH=native`. - -### ⚠️ Smart App Control will block your fresh binary - -On Windows 11 machines with **Smart App Control** enforced (`VerifiedAndReputablePolicyState = 1`), running your self-compiled `glm.exe` fails with: - -``` -Program 'glm.exe' failed to run: An Application Control policy has blocked this file -``` - -This is not Defender and not Mark-of-the-Web — SAC blocks *all* unsigned, unknown binaries, which includes anything you compile yourself. **Fix:** Windows Security → App & browser control → Smart App Control settings → **Off**, then **reboot** (the policy only reloads on restart). Note SAC is one-way: re-enabling later requires resetting Windows. If the settings page is missing, the registry equivalent is setting `HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy\VerifiedAndReputablePolicyState` to `0` (admin PowerShell), then rebooting. Check your current state before touching anything: - -```powershell -(Get-ItemProperty "HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy").VerifiedAndReputablePolicyState -# 0 = off, 1 = enforced, 2 = evaluation -``` - -## 3. Build the CUDA DLL (GPU tier) - -nvcc needs MSVC as host compiler, so this one step must run from a shell with the MSVC environment: open **"x64 Native Tools Command Prompt for VS 2022"** from the Start menu (plain PowerShell will fail the `cl` check). Then: - -```cmd -make cuda-dll CUDA_ARCH=sm_120 # match your GPU: sm_120 Blackwell, sm_89 Ada, ... -make glm.exe CUDA_DLL=1 ARCH=native # relink host with the runtime loader -``` - -Two pitfalls, both fixed on current `dev` (#314) but worth knowing on older checkouts: - -- **Spaces in `CUDA_HOME`** (`C:\Program Files\...`) used to break the recipe → fixed; nvcc now comes from PATH and `"$(NVCC)"` is quoted. -- **`make glm.exe CUDA_DLL=1` after a CPU-only build** used to report `up to date` and silently keep the CPU-only binary (GPU tier never engages, no error). Current `dev` has a build-config stamp that forces the relink. On older trees: delete `glm.exe` first. - -Sanity check: first GPU run should print `[CUDA] device 0: , ... sm_XX` and `[CUDA] mode: routed experts + resident dense tensors`. - -## 4. First run - -```powershell -cd \c -$env:OMP_NUM_THREADS = "" -python coli run "Explain what a mixture-of-experts model is." --model D:\glm52_i4 --ngen 48 -``` - -The first run is cold — expect the profile to be dominated by `expert-disk` while the cache warms; hit rate climbs run over run. GPU tier on top: - -```powershell -$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_DENSE="1"; $env:CUDA_EXPERT_GB="4" -python coli run "..." --model D:\glm52_i4 --ngen 64 -``` - -Size `CUDA_EXPERT_GB` so dense (~10 GB) + experts + working set stays under your VRAM. Note MTP speculation is off by default under CUDA (#293, float-accumulation divergence between draft and verify) — `COLI_CUDA_MTP=1` opts back in. - -## 5. Reference numbers from this walkthrough's hardware - -285K / RTX 5080 / 128 GB / NVMe at 5.85 GB/s random-read (19 MB blocks, `iobench`): 0.26 tok/s cold CPU → 0.30 warm CPU (MTP 2.2–2.3 tok/forward) → 0.42 tok/s GPU tier + auto-pin, expert hit 66%, ~65% of wall time in expert-disk. Disk-bound is the expected shape at ~25% expert residency — a faster disk and more RAM move the floor, the GPU moves the compute. - -## Quick failure index - -| Symptom | Cause | Fix | -|---|---|---| -| `An Application Control policy has blocked this file` | Smart App Control | §2 — turn SAC off + **reboot** | -| `cuda-dll ... Error 1` immediately | old tree: spaced CUDA_HOME / MSVC rejects `-Wextra` | update to current `dev` (#314) | -| `glm.exe is up to date` but GPU never engages | old tree: stale CPU-only binary | update to `dev`, or delete `glm.exe` and rebuild | -| `cl.exe (MSVC) not in PATH` | built from plain PowerShell | use the x64 Native Tools prompt | -| `nvcc fatal: unsupported gpu architecture 'sm_120'` | CUDA < 12.8 | install CUDA 12.8+ | -| MTP `0% (0/0)` on CPU path | int4 MTP heads in the container | use the int8-MTP container | -| MTP `draft=0` under CUDA | intended default since #293 | `COLI_CUDA_MTP=1` to opt in | diff --git a/docs/windows.md b/docs/windows.md index 94a4f69..ecb81e7 100644 --- a/docs/windows.md +++ b/docs/windows.md @@ -1,25 +1,107 @@ -# Windows 11 (native, no WSL) +# Windows 11 native install — a complete walkthrough (no WSL) -colibrì builds and runs natively on Windows 11 x86-64 with MinGW-w64. The port -adds a `_WIN32` compatibility layer in `c/compat.h` that maps POSIX I/O to the -Windows API (pread → ReadFile+OVERLAPPED, posix_fadvise no-op, aligned -allocation, MoveFileEx rename, GlobalMemoryStatusEx RAM detection). All platform -differences stay in `compat.h`; the engine source is unchanged. +A start-to-finish, reproducible path from a fresh Windows 11 machine to GLM-5.2 generating tokens, with the GPU tier. Every step and every failure mode below was hit and verified on real hardware: Core Ultra 9 285K (AVX-VNNI) / RTX 5080 (sm_120) / 128 GB RAM / Windows 11 24H2 (issue #306). Steps are ordered so the long downloads run while you build. -**Toolchain:** GCC via [winlibs](https://winlibs.com/) or MSYS2 MinGW-w64. -Tested with GCC 16.1.0 (x86_64-ucrt-posix-seh). +## 0. What you need + +| Piece | Why | Get it | +|---|---|---| +| git, Python 3 | clone + `coli` launcher | winget / python.org | +| MinGW-w64 gcc + make | builds the engine (MSVC can't) | `scoop install mingw-winlibs`, MSYS2, or portable **w64devkit** (no admin, unzip and go) | +| CUDA Toolkit ≥ 12.8 | GPU tier; ≥12.8 required for Blackwell/sm_120 | `winget install Nvidia.CUDA` | +| MSVC Build Tools (C++ workload) | nvcc's host compiler for the CUDA DLL | `winget install Microsoft.VisualStudio.2022.BuildTools` + "Desktop development with C++" | +| ~400 GB free on a local NVMe | the int4 model (~370–384 GB) | NTFS is fine; **never** a network mount | + +RAM: 16 GB minimum, more = bigger expert cache = faster. The build itself needs none of the CUDA/MSVC pieces — do the CPU build first, add the GPU tier later. + +## 1. Start the model download first (it's the long pole) ```powershell -# One-time toolchain install (pick one): -scoop install mingw-winlibs # portable, no shell needed -# or: pacman -S mingw-w64-x86_64-gcc make # via MSYS2 +python -m pip install -U "huggingface_hub[hf_transfer]" +$env:HF_HUB_ENABLE_HF_TRANSFER = "1" +hf download --local-dir D:\glm52_i4 +``` -# Build (from c/ directory): -make glm.exe # GLM-5.2 engine (static, no DLL dependencies) -make olmoe.exe # OLMoE engine (same shims) -make iobench.exe # disk I/O benchmark -make test-c # run C tests -make test-python # run Python tests (requires python) +Use the container recommended in the README (with **int8 MTP heads** — int4 heads silently give 0% draft acceptance). The download is resumable: if it stops, rerun the same command. Expect hours; everything below fits inside them. + +## 2. Build the engine (CPU) + +From a normal PowerShell, in the repo's `c\` directory: + +```powershell +make glm.exe ARCH=native # ARCH=native unlocks AVX-VNNI on Alder Lake+/Arrow Lake +make iobench.exe # disk benchmark, useful before committing to the download +``` + +Warnings about `#pragma comment` and unused variables are normal (MSVC-isms gcc ignores). The engine banner should print `idot: avx-vnni` on VNNI-capable CPUs — if it says avx2, you built without `ARCH=native`. + +### ⚠️ Smart App Control will block your fresh binary + +On Windows 11 machines with **Smart App Control** enforced (`VerifiedAndReputablePolicyState = 1`), running your self-compiled `glm.exe` fails with: + +``` +Program 'glm.exe' failed to run: An Application Control policy has blocked this file +``` + +This is not Defender and not Mark-of-the-Web — SAC blocks *all* unsigned, unknown binaries, which includes anything you compile yourself. **Fix:** Windows Security → App & browser control → Smart App Control settings → **Off**, then **reboot** (the policy only reloads on restart). Note SAC is one-way: re-enabling later requires resetting Windows. If the settings page is missing, the registry equivalent is setting `HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy\VerifiedAndReputablePolicyState` to `0` (admin PowerShell), then rebooting. Check your current state before touching anything: + +```powershell +(Get-ItemProperty "HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy").VerifiedAndReputablePolicyState +# 0 = off, 1 = enforced, 2 = evaluation +``` + +## 3. Build the CUDA DLL (GPU tier) + +nvcc needs MSVC as host compiler, so this one step must run from a shell with the MSVC environment: open **"x64 Native Tools Command Prompt for VS 2022"** from the Start menu (plain PowerShell will fail the `cl` check). Then: + +```cmd +make cuda-dll CUDA_ARCH=sm_120 # match your GPU: sm_120 Blackwell, sm_89 Ada, ... +make glm.exe CUDA_DLL=1 ARCH=native # relink host with the runtime loader +``` + +Two pitfalls, both fixed on current `dev` (#314) but worth knowing on older checkouts: + +- **Spaces in `CUDA_HOME`** (`C:\Program Files\...`) used to break the recipe → fixed; nvcc now comes from PATH and `"$(NVCC)"` is quoted. +- **`make glm.exe CUDA_DLL=1` after a CPU-only build** used to report `up to date` and silently keep the CPU-only binary (GPU tier never engages, no error). Current `dev` has a build-config stamp that forces the relink. On older trees: delete `glm.exe` first. + +Sanity check: first GPU run should print `[CUDA] device 0: , ... sm_XX` and `[CUDA] mode: routed experts + resident dense tensors`. + +## 4. First run + +```powershell +cd \c +$env:OMP_NUM_THREADS = "" +python coli run "Explain what a mixture-of-experts model is." --model D:\glm52_i4 --ngen 48 +``` + +The first run is cold — expect the profile to be dominated by `expert-disk` while the cache warms; hit rate climbs run over run. GPU tier on top: + +```powershell +$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_DENSE="1"; $env:CUDA_EXPERT_GB="4" +python coli run "..." --model D:\glm52_i4 --ngen 64 +``` + +Size `CUDA_EXPERT_GB` so dense (~10 GB) + experts + working set stays under your VRAM. Note MTP speculation is off by default under CUDA (#293, float-accumulation divergence between draft and verify) — `COLI_CUDA_MTP=1` opts back in. + +## 5. Reference numbers from this walkthrough's hardware + +285K / RTX 5080 / 128 GB / NVMe at 5.85 GB/s random-read (19 MB blocks, `iobench`): 0.26 tok/s cold CPU → 0.30 warm CPU (MTP 2.2–2.3 tok/forward) → 0.42 tok/s GPU tier + auto-pin, expert hit 66%, ~65% of wall time in expert-disk. Disk-bound is the expected shape at ~25% expert residency — a faster disk and more RAM move the floor, the GPU moves the compute. + +## Quick failure index + +| Symptom | Cause | Fix | +|---|---|---| +| `An Application Control policy has blocked this file` | Smart App Control | §2 — turn SAC off + **reboot** | +| `cuda-dll ... Error 1` immediately | old tree: spaced CUDA_HOME / MSVC rejects `-Wextra` | update to current `dev` (#314) | +| `glm.exe is up to date` but GPU never engages | old tree: stale CPU-only binary | update to `dev`, or delete `glm.exe` and rebuild | +| `cl.exe (MSVC) not in PATH` | built from plain PowerShell | use the x64 Native Tools prompt | +| `nvcc fatal: unsupported gpu architecture 'sm_120'` | CUDA < 12.8 | install CUDA 12.8+ | +| MTP `0% (0/0)` on CPU path | int4 MTP heads in the container | use the int8-MTP container | +| MTP `draft=0` under CUDA | intended default since #293 | `COLI_CUDA_MTP=1` to opt in | + +--- + +## Reference: build flags & warmup # AVX-VNNI: Intel Alder Lake+ (and Meteor Lake+) CPUs have a 128-bit int8 # dot-product instruction (VPDPBUSD) the engine can use for ~1.3x faster From db09466d3dc95015c77c8953d083d8d568f0657d Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sat, 18 Jul 2026 01:24:43 +0200 Subject: [PATCH 47/95] convert(fp8->int4): --mtp/--indexer respected on the --indir path, no more container overwrite (#355) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The local (`--indir`) branch of main() ignored --mtp and --indexer entirely: it always called convert_shard() without keep_mtp/keep_idx and always wrote `out-NNNNN.safetensors`. So the documented two-pass workflow — convert the main model into an outdir, then run `--mtp` into the SAME outdir for the head — did the opposite on the local path: with --mtp defaulting ebits to 8 and keep_mtp staying False, it silently re-converted the whole model to per-row int8 and overwrote the finished fmt=4 container shard by shard, printing nothing wrong. @mohamedmastouri2000-boop lost 137 of 141 freshly-converted g64 shards to this while building the public container for #298/#326. The --indir branch now mirrors the download path: keep_mtp=a.mtp / keep_idx=a.indexer passed through, output named out-mtp-/out-idx-/out- by mode, empty shards skipped (an MTP pass emits only shards containing layer n_layers), and config/tokenizer copied only on the main pass (the head/idx passes land in an already-complete outdir). Verified with a synthetic 2-layer + MTP-shard model: after the main pass, a second --mtp pass into the same outdir leaves out-00000's md5 BYTE-IDENTICAL and writes out-mtp-00000 alongside it. The bug would have changed that md5. Reported-by: mohamedmastouri2000-boop <#355> Co-Authored-By: Claude Opus 4.8 --- c/tools/convert_fp8_to_int4.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py index 382125e..fb2fb73 100644 --- a/c/tools/convert_fp8_to_int4.py +++ b/c/tools/convert_fp8_to_int4.py @@ -379,14 +379,32 @@ def main(): if a.indir: # conversione locale (test) shards = sorted(glob.glob(os.path.join(a.indir, "*.safetensors"))) from safetensors.numpy import save_file + # BUG #355: questo ramo ignorava --mtp/--indexer. Con --mtp scriveva + # out-NNNNN (gli STESSI nomi di una conversione normale) in ebits=8 e + # keep_mtp=False -> il "secondo passaggio MTP" nella stessa outdir + # SOVRASCRIVEVA il container gia' finito con una riconversione int8 + # completa, in silenzio (137/141 shard distrutti prima di accorgersene). + # Ora il ramo locale rispecchia il download path: prefisso corretto, + # flag passate, shard vuoti saltati. + prefix = "out-mtp-" if a.mtp else "out-idx-" if a.indexer else "out-" + n = 0 for i, sp in enumerate(shards): - out = {}; convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits, group_size=a.group_size, bits_map=bits_map) - save_file(out, os.path.join(a.outdir, f"out-{i:05d}.safetensors")) - # copia config + tokenizer - for fn in ["config.json"]: - src = os.path.join(a.indir, fn) - if os.path.exists(src): shutil.copy(src, a.outdir) - print(f"converted {len(shards)} shards -> {a.outdir}") + out = {} + convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits, + keep_mtp=a.mtp, keep_idx=a.indexer, + group_size=a.group_size, bits_map=bits_map) + if not out: # shard senza MTP/idx: niente file (come il download path) + continue + save_file(out, os.path.join(a.outdir, f"{prefix}{n:05d}.safetensors")) + n += 1 + # config/tokenizer solo per la conversione principale — i passaggi mtp/idx + # vanno nella stessa outdir di un container gia' completo di metadati. + if not a.mtp and not a.indexer: + for fn in ["config.json"]: + src = os.path.join(a.indir, fn) + if os.path.exists(src): shutil.copy(src, a.outdir) + tag = "MTP" if a.mtp else "indexer" if a.indexer else "main" + print(f"converted {n} {tag} shard(s) -> {a.outdir} ({prefix}NNNNN)") return # reale: scarica shard per shard, converte, cancella From 679c0742b2ec09a85d9987f85230125658e86d80 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sat, 18 Jul 2026 01:43:50 +0200 Subject: [PATCH 48/95] telemetry: split expert hits into pin-tier vs LRU ecache (#336) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The expert lookup counted a hit identically whether the pinned hot-store or the LRU ecache served it — both Phase C branches bumped the single `m->hits`. With a warm pin profile the pin tier absorbs most hot experts, so the ecache could be serving anywhere from ~0% to most hits and the logs couldn't tell which. That made every cache-policy question unanswerable, including #223's "at what cap does the eviction policy start to win?" (a flat A/B can mean "pin absorbed everything" or "genuine floor" and the lumped counter can't distinguish them). Two counters `hit_pin`/`hit_ecache` bumped at the two lookup branches (the existing `m->hits++` stays, so all existing math is unchanged), snapshotted in ProfBase and reset alongside `hits`. Surfaced in the human-readable summaries: decode: expert hit rate 31.3% (pin 22.1% + lru 9.2%) [PROF]: hit 31.3% (X pin + Y lru / Z load) tiny: hit rate 88.1% (0 pin + 74 lru / 10 miss) The serve-mux STAT protocol line is untouched (openai_server.py parses it positionally). Invariant hit_pin + hit_ecache == hits holds by construction — exactly two sites bump hits and each also bumps one split counter, nothing else mutates hits. Verified numerically on the tiny oracle: 0 pin + 74 lru = 74 hits, 88.1%. Zero cost: two increments on paths that already increment. Reported-by: KingIcyCreamProjects <#336> Co-Authored-By: Claude Opus 4.8 --- c/glm.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/c/glm.c b/c/glm.c index 2ee12ed..1446606 100644 --- a/c/glm.c +++ b/c/glm.c @@ -196,6 +196,7 @@ typedef struct { uint64_t mtp_prop, mtp_acc; /* statistica acceptance */ int **eroute; int *enr; /* metodo C: routing dell'ULTIMO token per layer */ uint64_t eclock, hits, miss, ereq; + uint64_t hit_pin, hit_ecache; /* split di hits per tier (#336): pin vs LRU ecache */ uint64_t gpu_expert_calls; int gpu_expert_count; int64_t gpu_expert_bytes; uint64_t n_fw, n_emit; /* metodo E: forward di decode / token emessi */ uint64_t route_slots, route_swaps; /* CACHE_ROUTE: slots chosen / substituted vs true top-K */ @@ -312,6 +313,7 @@ static void prof_lat(double s){ g_prof_lat[g_prof_nlat++ % PROF_LAT_CAP]=s; } typedef struct { double edisk,ewait,emm,ecpu,egpu,route,p2p,attn,head; int64_t io,cpu_bytes; uint64_t hits,miss,ereq,n_fw,n_emit,nlat,n_p2p,cpu_rows; + uint64_t hit_pin,hit_ecache; } ProfBase; static void prof_base(Model *m, ProfBase *b){ b->edisk=edisk_s(); b->ewait=m->t_ewait; b->emm=m->t_emm; @@ -319,6 +321,7 @@ static void prof_base(Model *m, ProfBase *b){ b->attn=m->t_attn; b->head=m->t_head; b->io=atomic_load_explicit(&g_prof_io,memory_order_relaxed); b->hits=m->hits; b->miss=m->miss; b->ereq=m->ereq; + b->hit_pin=m->hit_pin; b->hit_ecache=m->hit_ecache; b->n_fw=m->n_fw; b->n_emit=m->n_emit; b->nlat=g_prof_nlat; b->n_p2p=m->n_p2p; b->cpu_bytes=m->cpu_expert_bytes;b->cpu_rows=m->cpu_expert_rows; } @@ -3132,9 +3135,9 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int ESlot *use[64]; int missk[64]; int qof[64]; int nmiss=0; for(int j=0;jpin[layer]; - for(int z=0;znpin[layer];z++) if(P[z].eid==eid){ m->hits++; use[j]=&P[z]; break; } + for(int z=0;znpin[layer];z++) if(P[z].eid==eid){ m->hits++; m->hit_pin++; use[j]=&P[z]; break; } if(!use[j]){ ESlot *Sl=m->ecache[layer]; int nn=m->ecn[layer]; - for(int z=0;zhits++; Sl[z].used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); use[j]=&Sl[z]; break; } } + for(int z=0;zhits++; m->hit_ecache++; Sl[z].used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); use[j]=&Sl[z]; break; } } if(!use[j]){ qof[j]=nmiss; use[j]=&m->ws[nmiss]; missk[nmiss++]=j; m->miss++; if(g_disk_split){ if(m->ld_ctx==1) m->miss_draft++; else if(m->ld_ctx==2) m->miss_absorb++; } } } @@ -4674,11 +4677,12 @@ static void prof_report(Model *m, const ProfBase *b, double elapsed, int tokens, for(int i=0;i<=c->n_layers;i++){ if(m->npin)pinned+=m->npin[i]; if(m->ecn)lru+=m->ecn[i]; } double io_w=m->t_ewait-b->ewait; /* stall the compute thread felt */ double io_svc=edisk_s()-b->edisk; /* read service on the loading threads (overlaps compute) */ + uint64_t dhp=m->hit_pin-b->hit_pin, dhe=m->hit_ecache-b->hit_ecache; /* split #336 */ fprintf(f,"[PROF] expert I/O: %.3f GB fetched (%.1f MB/token, %.2f GB/s over the run%s) | " - "hit %.1f%% (%llu hit / %llu load) | %.1f loads/token | %.1fs read service / %.1fs felt wait\n", + "hit %.1f%% (%llu pin + %llu lru / %llu load) | %.1f loads/token | %.1fs read service / %.1fs felt wait\n", io/1e9, tokens>0?io/1e6/tokens:0.0, io/1e9/elapsed, g_mmap?"; COLI_MMAP=1: page cache may serve part":"", - hitp,(unsigned long long)dh,(unsigned long long)dm, tokens>0?(double)dq/tokens:0.0, + hitp,(unsigned long long)dhp,(unsigned long long)dhe,(unsigned long long)dm, tokens>0?(double)dq/tokens:0.0, io_svc,io_w); fprintf(f,"[PROF] resident experts: %d pinned (%.1f GB) + %d in LRU (%.1f GB, cap %d/layer)\n", pinned,pinned*eb/1e9,lru,lru*eb/1e9,m->ecap); @@ -4724,7 +4728,7 @@ static void run_replay(Model *m, const int *full, int nfull, int np){ if(np<2||nfull<=np){ fprintf(stderr,"REPLAY requires a non-empty prompt and continuation\n"); return; } kv_alloc(m,nfull+2); float *logit=step(m,full,np-1,0); free(logit); - m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; + m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; m->hit_pin=m->hit_ecache=0; profile_reset(m); ProfBase pb; prof_base(m,&pb); double t0=now_s(); int steps=0; @@ -4774,7 +4778,7 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ } prefill_t=now_s()-prefill_t; printf("PROFILO PREFILL (%.2fs):\n",prefill_t); profile_print(m,prefill_t); - m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; + m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; m->hit_pin=m->hit_ecache=0; m->n_emit=m->n_fw=0; g_last_repin=0; profile_reset(m); @@ -4787,8 +4791,9 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ double tot=m->hits+m->miss; int nsp=0; for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; printf("\n---\nprefill %d tokens in %.2fs | decode %d tokens in %.2fs (%.2f tok/s) | " - "expert hit rate %.1f%% | RSS %.2f GB", - np,prefill_t,produced,dt,produced/dt,tot?100.0*m->hits/tot:0.0,rss_gb()); + "expert hit rate %.1f%% (pin %.1f%% + lru %.1f%%) | RSS %.2f GB", /* split #336: quale tier serve gli hit */ + np,prefill_t,produced,dt,produced/dt,tot?100.0*m->hits/tot:0.0, + tot?100.0*m->hit_pin/tot:0.0, tot?100.0*m->hit_ecache/tot:0.0, rss_gb()); if(g_cache_route && m->route_slots) printf(" | swap %.1f%% (%llu/%llu)", 100.0*m->route_swaps/m->route_slots, @@ -6535,8 +6540,9 @@ int main(int argc, char **argv){ double tot=m.hits+m.miss; printf("N-gram speculation (DRAFT=%d): %.2f tokens/forward (%llu forwards per %llu tokens)\n", g_draft, m.n_fw?(double)m.n_emit/m.n_fw:1.0, (unsigned long long)m.n_fw, (unsigned long long)m.n_emit); - printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu) | RSS: %.2f GB | %.1f tok/s\n", - tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hits, (unsigned long long)m.miss, rss_gb(), n_new/dt); + printf("Expert cache hit rate: %.1f%% (%llu pin + %llu lru / %llu miss) | RSS: %.2f GB | %.1f tok/s\n", + tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hit_pin, (unsigned long long)m.hit_ecache, + (unsigned long long)m.miss, rss_gb(), n_new/dt); profile_print(&m,dt); if(g_prof) prof_report(&m,&pb,dt,n_new,stdout); #ifdef COLI_CUDA From b09273f73d33bc2b18ac1c7a8328a2ad69451ea7 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sat, 18 Jul 2026 01:57:17 +0200 Subject: [PATCH 49/95] serve: MTP status message tells the truth in multiplexed mode (#358) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `coli serve` runs the engine with SERVE_BATCH=1 (openai_server.py), which selects run_serve_mux, which sets g_draft=0 — speculation isn't ragged-safe across the multi-slot batch. But the "[MTP] active: native speculative decoding (draft=N)" line is printed in main() BEFORE the serve path is chosen, so `coli serve DRAFT=8` announced draft=8 and then silently disabled it. @LordMZTE reported the misleading message. The load line and the [MTP] stderr line now detect the mux case (SERVE + SERVE_BATCH) and report it truthfully: "MTP DISABLED (multiplexed serve)" with draft=0, plus a one-line explanation that single-client interactive use (`coli chat`, which spawns run_serve without SERVE_BATCH) keeps MTP. No more draft=8 claim on a path that runs draft=0. This is the honesty half of #358. The feature half — MTP inside the HTTP server for a single client — is a real enhancement but needs engine work: the mux decode kernel would have to run the speculative path when exactly one KV slot is active (S=1 is not actually ragged), and it needs a server round-trip test. Tracked separately; the message no longer lies in the meantime. Reported-by: LordMZTE <#358> Co-Authored-By: Claude Opus 4.8 --- c/glm.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/c/glm.c b/c/glm.c index 1446606..4c4feef 100644 --- a/c/glm.c +++ b/c/glm.c @@ -6386,11 +6386,22 @@ int main(int argc, char **argv){ #endif } if(getenv("DSA_TOPK")) m.c.index_topk=atoi(getenv("DSA_TOPK")); /* override per test */ + /* Il path MUX (SERVE_BATCH=1, cioe' `coli serve`) forza g_draft=0 sotto — + * la speculazione non e' ragged-safe nel batch multi-slot. Segnalarlo QUI, + * altrimenti "MTP active (draft=8)" mentirebbe: il messaggio e' stampato + * prima della scelta del path (run_serve_mux, sotto), e con DRAFT=8 diceva + * "active" per poi disabilitarlo in silenzio (#358, LordMZTE). */ + int mux_will_disable_mtp = getenv("SERVE") && getenv("SERVE_BATCH") && atoi(getenv("SERVE_BATCH")); + int eff_draft = mux_will_disable_mtp ? 0 : g_draft; printf("loaded in %.2fs | resident dense: %.2f MB | layers=%d experts=%d | MTP %s (draft=%d)\n", now_s()-t0, m.resident_bytes/(1024.0*1024.0), m.c.n_layers, m.c.n_experts, - m.has_mtp?"ACTIVE":"absent", g_draft); + m.has_mtp?(mux_will_disable_mtp?"DISABLED (multiplexed serve)":"ACTIVE"):"absent", eff_draft); /* anche su stderr: e' il canale che le UI (coli) mostrano all'utente */ - fprintf(stderr,"[MTP] %s (draft=%d)\n", m.has_mtp?"active: native speculative decoding":"absent", g_draft); + if(mux_will_disable_mtp && m.has_mtp) + fprintf(stderr,"[MTP] disabled in multiplexed serve (SERVE_BATCH=1): speculation is not " + "ragged-safe across KV slots. Single-client interactive use (`coli chat`) keeps MTP.\n"); + else + fprintf(stderr,"[MTP] %s (draft=%d)\n", m.has_mtp?"active: native speculative decoding":"absent", eff_draft); #ifdef __linux__ { /* Only warn for a GENUINE 9p mount (WSL Windows drives, magic 0x01021997), where * fadvise is a no-op. The old check was `snap` starting with "/mnt/", which From 8a1dccae3b1a4c82ea9173e3d1ea8c002f14e785 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sat, 18 Jul 2026 02:33:23 +0200 Subject: [PATCH 50/95] docs(api): tell coding-CLI users about the prefill cost before they hit it (#373) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit yurivict connected crush per the walkthrough and got 'thought for 1h8m and did nothing' — which is not a hang: agent CLIs send a 10-20k-token system preamble, and prefill on the CPU-streaming path runs at a few tok/s (attention-bound, #153). An hour of silent prefill looks exactly like a dead server. The note now spells out the arithmetic, the curl smoke-test that separates slow-but-working from broken, and honest guidance on what agent workloads are (not) viable. Co-Authored-By: Claude Opus 4.8 --- docs/api.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/api.md b/docs/api.md index cb29eb2..62010bd 100644 --- a/docs/api.md +++ b/docs/api.md @@ -106,9 +106,24 @@ KV configuration actually allows. URL to `http://localhost:8000/v1`, the model to `glm-5.2-colibri`, and any dummy key (`OPENAI_API_KEY` / `OPENAI_BASE_URL` for env-based tools). -> On the CPU-streaming path a large model decodes at roughly 1 tok/s, so -> interactive agent loops will feel slow — it connects and works, but the latency -> is very different from a hosted model. +> **Set your expectations before connecting an agentic CLI.** Two costs dominate, +> and the first one is invisible until you know it's there: +> +> 1. **Prefill.** Coding agents (crush, aider in repo-map mode, Cline, …) send a +> large system prompt plus tool definitions — often 10–20k tokens — *before +> your first word*. Prefill on the CPU-streaming path runs at a few tokens per +> second (it is attention-bound, see #153), so a 15k-token agent preamble is +> **an hour of silent "thinking" before the first output token**. The client +> looks hung; it isn't. Smoke-test with the tiny `curl` above first — if that +> answers in about a minute, the pipeline works and what you're paying for is +> prompt size. +> 2. **Decode.** Roughly 1 tok/s for a large model, so multi-turn agent loops +> (which re-pay the growing context every turn) compound the cost. +> +> Practical guidance: single surgical asks with a short context work; iterative +> agent sessions against a disk-streaming 744B model do not resemble a hosted +> API and mostly won't be worth the wait. If your client lets you trim or disable +> its system preamble and tool catalog, do it. ## Isolated KV contexts From f8e4dc95b5add6cd55105b25731293d2c17fae44 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sat, 18 Jul 2026 12:28:27 +0200 Subject: [PATCH 51/95] serve: honor --ngen when the client omits max_tokens; convert: print the resolved plan (#382, #383) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two operator-surprise fixes: openai_server.py (#382, LordMZTE): a request without max_tokens defaulted to min(256, limit) — so `coli serve --ngen 32768` still cut every answer at 256 tokens for clients that omit the field. The operator's configured budget IS the default now; generation still ends at EOS, so it's a cap, not a target. convert_fp8_to_int4.py (#383, bokiko): the resolved conversion plan (mode, source, ebits/io/x bits, grouped-vs-per-row) prints as a [PLAN] line before any work. The two traps in #383 — --mtp defaulting ebits to 8 with the grouped branch silently gated off, and --mtp appearing to ignore --indir — are already defused by the #355 fix (the --mtp pass emits ONLY head tensors on the local path), but a 3.5-hour job must show its plan at second 1, not in a post-hoc size sanity check. bokiko's exact invocation now prints: [PLAN] mode: MTP head only | source: local ./fp8 | experts 8-bit, ... | PER-ROW (grouped branch needs bits<=4; ebits=8 disables it) Co-Authored-By: Claude Opus 4.8 --- c/openai_server.py | 5 ++++- c/tools/convert_fp8_to_int4.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/c/openai_server.py b/c/openai_server.py index d55fbc8..f56e20a 100644 --- a/c/openai_server.py +++ b/c/openai_server.py @@ -406,7 +406,10 @@ def generation_options(body, limit): maximum = body.get("max_tokens") maximum_param = "max_tokens" if maximum is None: - maximum = min(256, limit) + # Client omitted max_tokens: honor the operator's configured budget (--max-tokens / + # --ngen), not an arbitrary 256 — `coli serve --ngen 32768` must mean 32768 (#382). + # Generation still ends at EOS, so this is a cap, not a target. + maximum = limit temperature = body.get("temperature") top_p = body.get("top_p") temperature = 0.7 if temperature is None else temperature diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py index fb2fb73..9cd6937 100644 --- a/c/tools/convert_fp8_to_int4.py +++ b/c/tools/convert_fp8_to_int4.py @@ -299,6 +299,16 @@ def main(): if bits_map: print(f"[MIXED] precision map: " + ", ".join(f"{k}={v}bit" for k,v in sorted(bits_map.items()))) + # Il PIANO risolto, PRIMA di toccare qualunque cosa (#383): --mtp/--indexer cambiano il + # default di ebits a 8 (testa int4 = acceptance ~0%, issue #8) e il ramo grouped e' + # gated su bits<=4 — combinazioni sorprendenti devono mostrarsi al secondo 1 di un job + # da ore, non nel size-check dopo. EN: print the RESOLVED plan before doing anything. + mode = "MTP head only" if a.mtp else "DSA indexer only" if a.indexer else "main model" + grp = f"grouped gs={a.group_size} (fmt=4)" if (a.group_size and a.ebits <= 4) else \ + (f"PER-ROW (grouped branch needs bits<=4; ebits={a.ebits} disables it)" if a.group_size else "per-row") + print(f"[PLAN] mode: {mode} | source: {'local ' + a.indir if a.indir else 'download ' + a.repo} | " + f"experts {a.ebits}-bit, embed/lm_head {a.io_bits}-bit, x {a.xbits}-bit | {grp}") + if a.selftest_nvfp4: import torch # 1) LUT e2m1: i 16 codici devono decodificare esattamente ai valori attesi. From 6d3a6168e40ef2d490f708e20baacd0908b67d92 Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Sat, 18 Jul 2026 04:14:46 +0800 Subject: [PATCH 52/95] cuda: keep ragged KV resident across decode steps --- c/backend_cuda.cu | 127 ++++++++++++++++++++++++------- c/backend_cuda.h | 3 +- c/backend_loader.c | 8 +- c/glm.c | 17 +++-- c/tests/test_ragged_attention.cu | 10 ++- 5 files changed, 122 insertions(+), 43 deletions(-) diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu index 188330d..413e53b 100644 --- a/c/backend_cuda.cu +++ b/c/backend_cuda.cu @@ -8,12 +8,21 @@ #include #include +struct RaggedKVEntry { + const void *key; + const float *host_l,*host_r; + float *latent,*rope; + int length,capacity,K,R; +}; + struct ColiCudaTensor { void *weights; float *scales; size_t weight_bytes; int fmt, I, O, device; int tracked; + RaggedKVEntry ragged[512]; + int ragged_count; }; typedef struct { @@ -23,7 +32,7 @@ typedef struct { size_t x_cap, y_cap, gate_cap, up_cap; uint8_t *qx; float *qscale; size_t qx_cap, qscale_cap; - float *host_x,*host_y; size_t host_x_cap,host_y_cap; + float *host_x,*host_y,*host_kv; size_t host_x_cap,host_y_cap,host_kv_cap; float *aq,*al,*ar,*ac; size_t aq_cap,al_cap,ar_cap,ac_cap; float *pipe_buf[24]; size_t pipe_cap[24]; /* scratch persistenti del resident pipeline */ cudaStream_t stream; @@ -338,17 +347,17 @@ __global__ static void attention_absorb_batch_kernel(float *ctx,const float *q, ctx[((size_t)s*H+h)*V+v]=a*(fmt?wscale[row]:1.f);} } -/* Independent KV sequence per row. latent/rope are packed as [S,T,*], while - * lengths selects the valid prefix for each row. */ +/* Independent device-resident KV sequence per row. lengths selects the valid + * prefix; latent/rope point at paged caches updated by the host wrapper. */ __global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q, - const float *latent,const float *rope,const int *lengths, + const float *const *latent,const float *const *rope,const int *lengths, const void *weights,const float *wscale,int fmt,int S,int H,int Q,int R, int V,int K,int T,float scale){ int s=blockIdx.y,h=blockIdx.x,tid=threadIdx.x,nt=lengths[s],rbase=h*(Q+V); if(s>=S||nt<1||nt>T)return; extern __shared__ float sm[];float *qa=sm,*cl=qa+K,*scores=cl+K,*red=scores+T; const float *qs=q+((size_t)s*H+h)*(Q+R); - const float *ls=latent+(size_t)s*T*K,*rs=rope+(size_t)s*T*R; + const float *ls=latent[s],*rs=rope[s]; for(int k=tid;k= bytes) return 1; if (*ptr) cudaFree(*ptr); @@ -440,16 +458,17 @@ extern "C" void coli_cuda_shutdown(void) { for(int b=0;b<24;b++) if(ctx->pipe_buf[b]) cudaFree(ctx->pipe_buf[b]); if (ctx->host_x) cudaFreeHost(ctx->host_x); if (ctx->host_y) cudaFreeHost(ctx->host_y); + if (ctx->host_kv) cudaFreeHost(ctx->host_kv); if (ctx->stream) cudaStreamDestroy(ctx->stream); if (ctx->group_desc) cudaFree(ctx->group_desc); ctx->x = ctx->y = ctx->gate = ctx->up = nullptr; ctx->qx=nullptr; ctx->qscale=nullptr; ctx->aq=ctx->al=ctx->ar=ctx->ac=nullptr; - ctx->host_x=ctx->host_y=nullptr;ctx->stream=nullptr; + ctx->host_x=ctx->host_y=ctx->host_kv=nullptr;ctx->stream=nullptr; ctx->x_cap = ctx->y_cap = ctx->gate_cap = ctx->up_cap = 0; ctx->qx_cap=ctx->qscale_cap=0; ctx->aq_cap=ctx->al_cap=ctx->ar_cap=ctx->ac_cap=0; - ctx->host_x_cap=ctx->host_y_cap=0; + ctx->host_x_cap=ctx->host_y_cap=ctx->host_kv_cap=0; ctx->group_desc=nullptr; ctx->group_desc_cap=0; } g_nctx = 0; @@ -805,35 +824,81 @@ extern "C" int coli_cuda_attention_project_batch(ColiCudaTensor *w,ColiCudaTenso } extern "C" int coli_cuda_attention_project_ragged(ColiCudaTensor *w,ColiCudaTensor *proj, - float *out,const float *q,const float *const *latent,const float *const *rope, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, const int *lengths,int S,int H,int Q,int R,int V,int K,int T,float scale){ - if(!w||!proj||!out||!q||!latent||!rope||!lengths||S<1||S>512||T<1||T>512|| + if(!w||!proj||!out||!q||!keys||!latent||!rope||!lengths||S<1||S>512||T<1||T>8192|| H<1||Q<1||R<1||V<1||K<1||K>512||w->I!=K||w->O!=H*(Q+V)|| proj->device!=w->device||proj->I!=H*V)return 0; - size_t ln=(size_t)S*T*K,rn=(size_t)S*T*R; - float *lh=(float*)std::calloc(ln,sizeof(float)),*rh=(float*)std::calloc(rn,sizeof(float)); - if(!lh||!rh){std::free(lh);std::free(rh);return 0;} - for(int s=0;sT){std::free(lh);std::free(rh);return 0;} - std::memcpy(lh+(size_t)s*T*K,latent[s],(size_t)lengths[s]*K*sizeof(float)); - std::memcpy(rh+(size_t)s*T*R,rope[s],(size_t)lengths[s]*R*sizeof(float)); - } DeviceContext *dc=find_ctx(w->device); - if(!select_ctx(dc)){std::free(lh);std::free(rh);return 0;} - size_t qb=(size_t)S*H*(Q+R)*sizeof(float),lb=ln*sizeof(float),rb=rn*sizeof(float); + if(!select_ctx(dc))return 0; + float **dl=(float**)std::malloc((size_t)S*sizeof(*dl)); + float **dr=(float**)std::malloc((size_t)S*sizeof(*dr)); + int *old=(int*)std::malloc((size_t)S*sizeof(*old)); + int *add=(int*)std::malloc((size_t)S*sizeof(*add)); + int *off=(int*)std::malloc((size_t)S*sizeof(*off));int packed_n=0; + if(!dl||!dr||!old||!add||!off){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;} + for(int s=0;sT){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;} + RaggedKVEntry *e=nullptr; + for(int i=0;iragged_count;i++)if(w->ragged[i].key==keys[s]){e=&w->ragged[i];break;} + if(!e){ + if(w->ragged_count>=512){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;} + e=&w->ragged[w->ragged_count++];std::memset(e,0,sizeof(*e));e->key=keys[s]; + } + if(e->K!=K||e->R!=R||e->host_l!=latent[s]||e->host_r!=rope[s]||lengths[s]length){ + if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope); + e->latent=e->rope=nullptr;e->length=e->capacity=0; + e->K=K;e->R=R;e->host_l=latent[s];e->host_r=rope[s]; + } + if(lengths[s]>e->capacity){ + int cap=(lengths[s]+63)&~63;float *nl=nullptr,*nr=nullptr; + if(!cuda_ok(cudaMalloc(&nl,(size_t)cap*K*sizeof(float)),"ragged KV latent page")|| + !cuda_ok(cudaMalloc(&nr,(size_t)cap*R*sizeof(float)),"ragged KV rope page")){ + if(nl)cudaFree(nl);if(nr)cudaFree(nr);std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0; + } + if(e->length){ + cudaMemcpyAsync(nl,e->latent,(size_t)e->length*K*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream); + cudaMemcpyAsync(nr,e->rope,(size_t)e->length*R*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream); + } + if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope); + e->latent=nl;e->rope=nr;e->capacity=cap; + } + dl[s]=e->latent;dr[s]=e->rope;old[s]=e->length;add[s]=lengths[s]-e->length; + off[s]=packed_n;packed_n+=add[s]*(K+R); + } + size_t qb=(size_t)S*H*(Q+R)*sizeof(float); size_t cb=(size_t)S*H*V*sizeof(float),ob=(size_t)S*proj->O*sizeof(float); - int ok=reserve(&dc->aq,&dc->aq_cap,qb)&&reserve(&dc->al,&dc->al_cap,lb)&& - reserve(&dc->ar,&dc->ar_cap,rb)&&reserve(&dc->ac,&dc->ac_cap,cb)&& - reserve(&dc->y,&dc->y_cap,ob)&& - reserve_bytes(&dc->group_desc,&dc->group_desc_cap,(size_t)S*sizeof(int)); + size_t pb=(size_t)packed_n*sizeof(float); + size_t desc=(size_t)S*(2*sizeof(float*)+4*sizeof(int)); + int ok=reserve(&dc->aq,&dc->aq_cap,qb)&&reserve(&dc->ac,&dc->ac_cap,cb)&& + reserve(&dc->y,&dc->y_cap,ob)&&reserve_bytes(&dc->group_desc,&dc->group_desc_cap,desc)&& + (!pb||(reserve(&dc->al,&dc->al_cap,pb)&&reserve_pinned(&dc->host_kv,&dc->host_kv_cap,pb))); + char *db=(char*)dc->group_desc;float **ddl=(float**)db,**ddr=ddl+S; + int *dn=(int*)(ddr+S),*dold=dn+S,*dadd=dold+S,*doff=dadd+S; + if(ok&&pb){ + for(int s=0;shost_kv+off[s]; + std::memcpy(p,latent[s]+(size_t)old[s]*K,(size_t)add[s]*K*sizeof(float)); + std::memcpy(p+(size_t)add[s]*K,rope[s]+(size_t)old[s]*R,(size_t)add[s]*R*sizeof(float)); + } + ok=cuda_ok(cudaMemcpyAsync(dc->al,dc->host_kv,pb,cudaMemcpyHostToDevice,dc->stream),"ragged KV append upload"); + } if(ok)ok=cuda_ok(cudaMemcpyAsync(dc->aq,q,qb,cudaMemcpyHostToDevice,dc->stream),"ragged q upload")&& - cuda_ok(cudaMemcpyAsync(dc->al,lh,lb,cudaMemcpyHostToDevice,dc->stream),"ragged latent upload")&& - cuda_ok(cudaMemcpyAsync(dc->ar,rh,rb,cudaMemcpyHostToDevice,dc->stream),"ragged rope upload")&& - cuda_ok(cudaMemcpyAsync(dc->group_desc,lengths,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged lengths upload"); - std::free(lh);std::free(rh);if(!ok)return 0; + cuda_ok(cudaMemcpyAsync(ddl,dl,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged latent pointers")&& + cuda_ok(cudaMemcpyAsync(ddr,dr,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged rope pointers")&& + cuda_ok(cudaMemcpyAsync(dn,lengths,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged lengths upload")&& + cuda_ok(cudaMemcpyAsync(dold,old,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged old lengths")&& + cuda_ok(cudaMemcpyAsync(dadd,add,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append lengths")&& + cuda_ok(cudaMemcpyAsync(doff,off,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append offsets"); + if(ok&&pb)ragged_kv_append<<stream>>>(ddl,ddr,dc->al,dold,dadd,doff,K,R); + if(ok)for(int s=0;sragged_count;i++)if(w->ragged[i].key==keys[s]){w->ragged[i].length=lengths[s];break;} + } + std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);if(!ok)return 0; size_t shared=(size_t)(2*K+T+256)*sizeof(float); - attention_absorb_ragged_kernel<<stream>>>(dc->ac,dc->aq,dc->al,dc->ar, - (const int*)dc->group_desc,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale); + attention_absorb_ragged_kernel<<stream>>>(dc->ac,dc->aq,ddl,ddr, + dn,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale); quant_matmul<<O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights, proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I)); return cuda_ok(cudaGetLastError(),"ragged attention launch")&& @@ -852,6 +917,10 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) { } if (tensor->weights) cudaFree(tensor->weights); if (tensor->scales) cudaFree(tensor->scales); + for(int i=0;iragged_count;i++){ + if(tensor->ragged[i].latent)cudaFree(tensor->ragged[i].latent); + if(tensor->ragged[i].rope)cudaFree(tensor->ragged[i].rope); + } std::free(tensor); } diff --git a/c/backend_cuda.h b/c/backend_cuda.h index edddbfe..63c1f11 100644 --- a/c/backend_cuda.h +++ b/c/backend_cuda.h @@ -94,7 +94,8 @@ COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,C int V,int K,int T,float attention_scale); COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, - float *out,const float *q,const float *const *latent,const float *const *rope, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale); COLI_CUDA_DLLEXPORT void coli_cuda_tensor_free(ColiCudaTensor *tensor); diff --git a/c/backend_loader.c b/c/backend_loader.c index bbcb8ca..eedbd50 100644 --- a/c/backend_loader.c +++ b/c/backend_loader.c @@ -62,7 +62,8 @@ typedef int (*fn_attention_absorb_batch_dev)(ColiCudaTensor *kv_b_shard,float *c typedef int (*fn_attention_absorb_kvdev)(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale); typedef int (*fn_attention_project_batch)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q,const float *latent, const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale); typedef int (*fn_attention_project_ragged)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, - float *out,const float *q,const float *const *latent,const float *const *rope, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale); typedef int (*fn_attention_project_batch_dev)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); typedef int (*fn_attention_project_batch_dev_out)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); @@ -348,10 +349,11 @@ int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,ColiCudaTensor *o_pro } int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, - float *out,const float *q,const float *const *latent,const float *const *rope, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale){ if(!coli_cuda_load()) return 0; - return g_cuda.attention_project_ragged(kv_b,o_proj,out,q,latent,rope,lengths, + return g_cuda.attention_project_ragged(kv_b,o_proj,out,q,keys,latent,rope,lengths, S,H,Q,R,V,K,max_t,attention_scale); } diff --git a/c/glm.c b/c/glm.c index 70da320..a45fbc9 100644 --- a/c/glm.c +++ b/c/glm.c @@ -2717,18 +2717,20 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p !dnsel&&l->kv_b.cuda_eligible&&l->o.cuda_eligible&& qt_cuda_upload(&l->kv_b)&&qt_cuda_upload(&l->o)){ const float **rl=malloc((size_t)S*sizeof(*rl)),**rr=malloc((size_t)S*sizeof(*rr)); + const void **rk=malloc((size_t)S*sizeof(*rk)); int *rn=malloc((size_t)S*sizeof(*rn)); int mt=0; - if(rl&&rr&&rn){ + if(rk&&rl&&rr&&rn){ for(int s=0;skv_start[layer]; rn[s]=pos+1-st0; + rk[s]=kvs[s]; rl[s]=coli_kv_row(kvs[s]->Lc[layer],st0,kvl); rr[s]=coli_kv_row(kvs[s]->Rc[layer],st0,c->qk_rope); if(rn[s]>mt)mt=rn[s]; } cuda_core=cuda_projected=coli_cuda_attention_project_ragged(l->kv_b.cuda,l->o.cuda, - out,Q,rl,rr,rn,S,H,c->qk_nope,c->qk_rope,vh,kvl,mt,c->attn_scale); + out,Q,rk,rl,rr,rn,S,H,c->qk_nope,c->qk_rope,vh,kvl,mt,c->attn_scale); } - free(rl);free(rr);free(rn); + free(rk);free(rl);free(rr);free(rn); } else if(cuda_absorb&&l->n_kv_b_shard>1){ int n=l->n_kv_b_shard,st0=m->kv_start[layer],nt=pos_base+S-st0,ok=1; float *qs=falloc((int64_t)S*H*qh),*cs=falloc((int64_t)S*H*vh); @@ -5267,7 +5269,7 @@ static void run_serve_mux(Model *m, const char *snap){ g_draft=0; /* one scheduler owns every forward; MTP/speculation is not ragged-safe */ int maxctx=getenv("CTX")?atoi(getenv("CTX")):4096; int nctx=getenv("KV_SLOTS")?atoi(getenv("KV_SLOTS")):1; - if(nctx<1||nctx>16){fprintf(stderr,"KV_SLOTS deve essere tra 1 e 16\n");exit(2);} + if(nctx<1||nctx>512){fprintf(stderr,"KV_SLOTS must be between 1 and 512\n");exit(2);} g_kvsave=getenv("KVSAVE")?atoi(getenv("KVSAVE")):1; KVState *initial=m->kv; free(initial->kv_start); free(initial); ServeCtx *ctx=calloc(nctx,sizeof(*ctx)); ServeReq *req=calloc(nctx,sizeof(*req)); @@ -5323,7 +5325,7 @@ static void run_serve_mux(Model *m, const char *snap){ } active=0; for(int i=0;i1?atoi(argv[1]):64; int ebits= argc>2?atoi(argv[2]):8; int dbits= argc>3?atoi(argv[3]):ebits; - if(getenv("SERVE") && (kv_slot_count()<1 || kv_slot_count()>16)){ - fprintf(stderr,"KV_SLOTS must be between 1 and 16\n"); return 2; + int kv_limit=(getenv("SERVE_BATCH")&&atoi(getenv("SERVE_BATCH")))?512:16; + if(getenv("SERVE") && (kv_slot_count()<1 || kv_slot_count()>kv_limit)){ + fprintf(stderr,"KV_SLOTS must be between 1 and %d\n",kv_limit); return 2; } #ifdef COLI_CUDA if(getenv("COLI_CUDA") && atoi(getenv("COLI_CUDA"))){ diff --git a/c/tests/test_ragged_attention.cu b/c/tests/test_ragged_attention.cu index c00a8eb..dd1e2f8 100644 --- a/c/tests/test_ragged_attention.cu +++ b/c/tests/test_ragged_attention.cu @@ -16,14 +16,18 @@ int main(){ !coli_cuda_tensor_upload(&tp,p.data(),nullptr,0,D,O,dev))return 1; int n[S]={1,2,3};std::vector> l(S),r(S); const float *lp[S],*rp[S]; + const void *keys[S]; for(int s=0;s Date: Sat, 18 Jul 2026 18:38:40 +0800 Subject: [PATCH 53/95] plan NUMA interleave on multi-socket Linux --- c/resource_plan.py | 24 +++++++++++++++++++++++- c/tests/test_resource_plan.py | 19 +++++++++++++++++-- docs/ENVIRONMENT.md | 2 +- docs/benchmarks.md | 8 ++++++-- 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/c/resource_plan.py b/c/resource_plan.py index 6b3f2fe..5f17a6a 100644 --- a/c/resource_plan.py +++ b/c/resource_plan.py @@ -203,6 +203,22 @@ def physical_cpu_count(): return os.cpu_count() or 1 +def cpu_socket_count(): + """Return the number of physical CPU sockets visible to this process.""" + if not sys.platform.startswith("linux"): + return 1 + try: + result = subprocess.run(["lscpu", "-p=socket"], text=True, + capture_output=True, check=True, timeout=5) + sockets = {int(line) for line in result.stdout.splitlines() + if line and not line.startswith("#")} + if sockets: + return len(sockets) + except (OSError, ValueError, subprocess.SubprocessError): + pass + return 1 + + POLICIES = { "quality": {"preserve_quantization": True, "preserve_router": True}, "balanced": {"preserve_quantization": True, "preserve_router": True}, @@ -212,11 +228,12 @@ POLICIES = { def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, available_memory=None, available_disk=None, gpus=None, - policy="quality", physical_cpus=None): + policy="quality", physical_cpus=None, cpu_sockets=None): if policy not in POLICIES: raise ValueError(f"unknown policy: {policy}") info = analyze_model(model) physical_cpus = physical_cpu_count() if physical_cpus is None else physical_cpus + cpu_sockets = cpu_socket_count() if cpu_sockets is None else cpu_sockets cfg = info["config"] available_memory = memory_available() if available_memory is None else available_memory if available_disk is None: @@ -286,6 +303,7 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, "quality_preserving": policy != "experimental-fast"}, "model": {key: value for key, value in info.items() if key != "config"}, "cpu": {"physical_cores": max(1, int(physical_cpus)), + "sockets": max(1, int(cpu_sockets)), "thread_policy": "physical-cores"}, "tiers": { "disk": {"role": "cold-backing", "model_bytes": info["model_bytes"], @@ -318,6 +336,10 @@ def environment_for_plan(plan, env=None, cuda_enabled=True): # ("Affinity not supported on this configuration"): non impostarle li'. result.setdefault("OMP_PROC_BIND", "spread") result.setdefault("OMP_PLACES", "cores") + if sys.platform.startswith("linux") and plan["cpu"].get("sockets", 1) > 1: + # Selectively interleave large expert/dense slabs across memory controllers. + # Unlike blanket numactl interleave, this leaves CUDA staging buffers local. + result.setdefault("COLI_NUMA", "1") if plan["policy"]["name"] == "balanced": result.setdefault("REPIN", "64") ram = plan["tiers"]["ram"] diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.py index 023cb93..b184c80 100644 --- a/c/tests/test_resource_plan.py +++ b/c/tests/test_resource_plan.py @@ -10,6 +10,7 @@ from resource_plan import ( GB, analyze_model, build_plan, + cpu_socket_count, environment_for_plan, format_plan, memory_available, @@ -66,15 +67,19 @@ class ResourcePlanTest(unittest.TestCase): # 0 slots/layer. The value must be a sane positive number of bytes. self.assertGreater(memory_available(), 0) + def test_cpu_socket_count_is_positive(self): + self.assertGreaterEqual(cpu_socket_count(), 1) + def test_builds_bounded_three_tier_plan(self): gpus = [{"index": 0, "name": "test-gpu", "total_bytes": 12 * GB, "free_bytes": 10 * GB}] plan = build_plan(self.model, ram_gb=16, context=32, vram_gb=20, available_memory=32 * GB, available_disk=100 * GB, gpus=gpus, - physical_cpus=24) + physical_cpus=24, cpu_sockets=2) self.assertEqual(plan["version"], 2) self.assertEqual(plan["policy"]["name"], "quality") self.assertEqual(plan["cpu"]["physical_cores"], 24) + self.assertEqual(plan["cpu"]["sockets"], 2) self.assertTrue(plan["policy"]["preserve_quantization"]) self.assertFalse(plan["tiers"]["vram"]["requires_host_backing"]) self.assertEqual(plan["tiers"]["ram"]["budget_bytes"], 16 * GB) @@ -105,7 +110,7 @@ class ResourcePlanTest(unittest.TestCase): {"index": 1, "name": "b", "total_bytes": 12 * GB, "free_bytes": 10 * GB}, ] plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB, - available_disk=1, gpus=gpus) + available_disk=1, gpus=gpus, cpu_sockets=2) env = environment_for_plan(plan, {"RAM_GB": "12", "PIN": "stats.txt", "COLI_GPUS": "1"}) self.assertEqual(env["RAM_GB"], "12") @@ -125,6 +130,16 @@ class ResourcePlanTest(unittest.TestCase): "OMP_PROC_BIND": "close"}) self.assertEqual(explicit_threads["OMP_NUM_THREADS"], "7") self.assertEqual(explicit_threads["OMP_PROC_BIND"], "close") + + if sys.platform.startswith("linux"): + self.assertEqual(env["COLI_NUMA"], "1") + explicit_numa = environment_for_plan(plan, {"COLI_NUMA": "0"}) + self.assertEqual(explicit_numa["COLI_NUMA"], "0") + + def test_single_socket_plan_does_not_enable_numa(self): + plan = build_plan(self.model, available_memory=16 * GB, available_disk=1, + gpus=[], physical_cpus=8, cpu_sockets=1) + self.assertNotIn("COLI_NUMA", environment_for_plan(plan)) def test_cpu_binary_does_not_apply_gpu_tier(self): plan = build_plan(self.model, available_memory=16 * GB, available_disk=1, gpus=[{"index": 0, "name": "a", "total_bytes": 8 * GB, diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 535288a..9f0d749 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -43,7 +43,7 @@ Format: `VAR` — default — effect. | `URING` | `0` (off) | Linux-only queued expert I/O. `URING=1` implies `PIPE=1`, forces cold reads through io-wq (`IOSQE_ASYNC`), replaces blocking loader pthreads and spin waits with batched SQEs/CQEs, and batches `PILOT_REAL` loads on a separate ring. Use `DIRECT=1` for cold NVMe to avoid page-cache copy/readahead limits. Fails clearly if the kernel denies io_uring; incompatible with `COLI_MMAP=1`. | | `DIRECT` | `0` (off) | Use `O_DIRECT`/unbuffered reads for expert slabs. Helps sustained NVMe; keeps the zero-copy GPU path. | | `COLI_NO_OMP_TUNE` | off | **Kill-switch** for the OpenMP hot-thread tuning (`OMP_WAIT_POLICY=active` spin + proc-bind). Set `=1` when the CPU is mostly waiting on the GPU (Metal) so spin doesn't steal the shared power budget. | -| `COLI_NUMA` | off (Linux only) | `COLI_NUMA=1` interleaves expert slabs across NUMA nodes via `mbind` (raw syscall, no libnuma). Helps multi-socket hosts (+7–40% expert matmul); silent no-op on single-node or non-Linux. | +| `COLI_NUMA` | auto in generated plans on multi-socket Linux; otherwise off | `COLI_NUMA=1` selectively interleaves large expert and dense slabs across NUMA nodes via `mbind` (raw syscall, no libnuma). Helps multi-socket hosts (+7–40% expert matmul); silent no-op on single-node or non-Linux. Explicit `COLI_NUMA=0` overrides the generated plan. | | `MLOCK` | `-1` (auto: on for macOS) | Wire the streamed expert cache into physical RAM (`mlock`) to dodge the memory compressor. `0` off, `1` force. | | `CAP_RAISE` | `1` (on) | Let the engine raise the expert-cache cap above `topk` when RAM allows (bigger batches). `0` fixes the cap. | | `PREFETCH` | `0` | Prefetch depth for streamed experts. | diff --git a/docs/benchmarks.md b/docs/benchmarks.md index f9c83f2..09d3752 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -116,8 +116,12 @@ can match an RTX 5090 on expert matmul ([#101](https://github.com/JustVugg/colib so **the GPU tier earns its VRAM only when the CPU is the weak link**. On multi-socket hosts, NUMA placement is a further lever: interleaving the resident weights across nodes measured **+13% (2-socket) and +40% (4-socket CPU-only)** -([#82](https://github.com/JustVugg/colibri/issues/82)) — but never blanket-interleave -a GPU host (measured 10× regression via the DMA staging pages). +([#82](https://github.com/JustVugg/colibri/issues/82)). On a 2-socket Xeon Silver +4510 host with 6× RTX 5090, selective `COLI_NUMA=1` raised effective CPU-expert +bandwidth from **42.42 to 58.26/65.89 GB/s** and greedy decode from **7.66 to +9.02/9.17 tok/s** (64 tokens, `TEMP=0 DRAFT=0`, byte-identical output). Do not +blanket-interleave a GPU host: it also spreads DMA staging pages and has measured +up to a 10× regression; generated plans enable only the selective slab policy. ## Quality benchmark From 6a2ab47f3b644a07e7a0841741bf08509dd57075 Mon Sep 17 00:00:00 2001 From: bokiko Date: Sat, 18 Jul 2026 15:15:26 +0300 Subject: [PATCH 54/95] convert: resolve --mtp/--indexer shards from the local index instead of scanning all of them (#383) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When --indir contains model.safetensors.index.json, the head/indexer passes convert only the shards that actually hold the requested tensors (3 instead of scanning 141 for --mtp — every empty scan still opens a 5 GB shard). Prints the selection in the [PLAN] block. Without the index the full scan is unchanged. Output is byte-identical to the unfiltered path (sha256-verified on the GLM-5.2 FP8 head shards, with a decoy shard proving the filter excludes rather than reorders). Co-Authored-By: Claude --- c/tools/convert_fp8_to_int4.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py index 9cd6937..f851c37 100644 --- a/c/tools/convert_fp8_to_int4.py +++ b/c/tools/convert_fp8_to_int4.py @@ -389,6 +389,25 @@ def main(): if a.indir: # conversione locale (test) shards = sorted(glob.glob(os.path.join(a.indir, "*.safetensors"))) from safetensors.numpy import save_file + # #383: se l'indice c'e', i passaggi --mtp/--indexer convertono SOLO gli shard + # che contengono i tensori richiesti (3 invece di scandire tutti i 141 — ogni + # scansione a vuoto apre comunque uno shard da 5 GB). Senza indice: scansione + # completa come prima. + # EN: #383: when the index is present, the --mtp/--indexer passes convert ONLY + # the shards that hold the requested tensors (3 instead of scanning all 141 — + # every empty scan still opens a 5 GB shard). Without the index: full scan as + # before. + if a.mtp or a.indexer: + idxp = os.path.join(a.indir, "model.safetensors.index.json") + if os.path.exists(idxp): + wmap = json.load(open(idxp))["weight_map"] + if a.mtp: + want = {v for k, v in wmap.items() if k.startswith(f"model.layers.{a.n_layers}.")} + else: + want = {v for k, v in wmap.items() if "indexer" in k and 0 <= layer_idx(k) < a.n_layers} + keep = [sp for sp in shards if os.path.basename(sp) in want] + print(f"[PLAN] index: {len(keep)}/{len(shards)} local shard(s) hold the requested tensors") + shards = keep # BUG #355: questo ramo ignorava --mtp/--indexer. Con --mtp scriveva # out-NNNNN (gli STESSI nomi di una conversione normale) in ebits=8 e # keep_mtp=False -> il "secondo passaggio MTP" nella stessa outdir From 799fe41f88a963d039af3dd9c56f85c3b26f3e82 Mon Sep 17 00:00:00 2001 From: andres-gb10 Date: Sat, 18 Jul 2026 16:45:19 +0100 Subject: [PATCH 55/95] coli: honor THINK=1 in run mode coli run hardcoded the nothink template (<|assistant|>), so THINK=1 had no effect in one-shot runs while serve mode honors it (glm.c serve template picks vs from the same env var). Pick the template the same way here. Co-Authored-By: Claude Fable 5 --- c/coli | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/c/coli b/c/coli index 0dd9de3..c7fdaa2 100755 --- a/c/coli +++ b/c/coli @@ -480,8 +480,11 @@ def cmd_run(a): need_model(a.model) prompt=" ".join(a.prompt) if a.prompt else sys.exit('usage: coli run "your prompt"') banner("run") - # template ufficiale GLM-5.2: niente \n dopo i ruoli; = risposta diretta (nothink) - e=env_for(a); e["PROMPT"]=f"[gMASK]<|user|>{prompt}<|assistant|>" + # template ufficiale GLM-5.2: niente \n dopo i ruoli; = risposta diretta (nothink). + # THINK=1 lascia aperto, stessa convenzione del serve mode (glm.c). EN: THINK=1 leaves + # open so the engine emits its reasoning block; the default stays nothink. + tk="" if os.environ.get("THINK","0")=="1" else "" + e=env_for(a); e["PROMPT"]=f"[gMASK]<|user|>{prompt}<|assistant|>{tk}" sys.exit(subprocess.call([GLM, str(a.cap)], env=e)) def server_probe(base, api_key=None, timeout=1.5): From 308e269f468797211a32884f2a942083ac773aac Mon Sep 17 00:00:00 2001 From: andres-gb10 Date: Sat, 18 Jul 2026 21:11:40 +0100 Subject: [PATCH 56/95] convert(fp8->int4): --indir copies the full metadata set and resumes interrupted passes (#383) Two gaps in the local path, both raised in #383: - The main --indir pass copied only config.json while the download path copies four files; without tokenizer.json the converted container can't run chat/serve. Copy the same four, print what was copied, and warn when the source is missing any (tokenizer.json called out explicitly). - Local passes restarted from shard 0 on every rerun. out-NNNNN names count EMITTED shards, not input indexes (shards with no relevant tensors emit nothing), so the download path's exists-check can't be mirrored directly: a sidecar manifest per prefix records input -> output (or empty) plus the conversion parameters, written atomically after each shard. Rerun skips what matches and refuses a parameter mismatch on the same outdir instead of mixing containers (the #355 failure mode). Co-Authored-By: Claude Fable 5 --- c/tools/convert_fp8_to_int4.py | 66 +++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py index 9cd6937..bef3f74 100644 --- a/c/tools/convert_fp8_to_int4.py +++ b/c/tools/convert_fp8_to_int4.py @@ -397,24 +397,72 @@ def main(): # Ora il ramo locale rispecchia il download path: prefisso corretto, # flag passate, shard vuoti saltati. prefix = "out-mtp-" if a.mtp else "out-idx-" if a.indexer else "out-" - n = 0 + # RIPRESA (#383): i nomi out-NNNNN contano gli shard EMESSI, non l'indice di + # input (gli shard senza tensori rilevanti non producono file), quindi "il + # file esiste" non basta per saltare il lavoro gia' fatto. Un manifest + # sidecar ricorda input -> output (o "vuoto") e con quali parametri: la + # ripresa salta solo cio' che combacia, e parametri diversi sulla stessa + # outdir vengono rifiutati invece di mescolare container (il modo #355). + # EN: RESUME (#383): out-NNNNN names count EMITTED shards, not the input + # EN: index (shards with no relevant tensors emit no file), so "the file + # EN: exists" is not enough to skip completed work. A sidecar manifest + # EN: records input -> output (or "empty") plus the conversion parameters: + # EN: resume skips only what matches, and different parameters on the same + # EN: outdir are refused instead of mixing containers (the #355 failure mode). + params = {"ebits": a.ebits, "io_bits": a.io_bits, "xbits": a.xbits, + "group_size": a.group_size, "n_layers": a.n_layers, "bits_map": bits_map} + prog_path = os.path.join(a.outdir, f".{prefix}progress.json") + prog = {} + if os.path.exists(prog_path): + try: prog = json.loads(open(prog_path).read()) + except (OSError, ValueError): prog = {} + if prog and prog.get("params") != params: + print(f"ERROR: {prog_path} records a conversion with {prog.get('params')};\n" + f" this run uses {params}. Refusing to mix conversions in the same " + f"outdir — use a fresh --outdir (or delete the manifest and the " + f"{prefix}*.safetensors shards to redo).") + return + done = prog.setdefault("shards", {}); prog["params"] = params + n = 0; fresh = 0; skipped = 0 for i, sp in enumerate(shards): + key = os.path.basename(sp) + prev = done.get(key) # None = mai visto; "" = visto, vuoto; nome = emesso + if prev is not None and (prev == "" or os.path.exists(os.path.join(a.outdir, prev))): + if prev: n += 1 + skipped += 1 + continue out = {} convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits, keep_mtp=a.mtp, keep_idx=a.indexer, group_size=a.group_size, bits_map=bits_map) if not out: # shard senza MTP/idx: niente file (come il download path) - continue - save_file(out, os.path.join(a.outdir, f"{prefix}{n:05d}.safetensors")) - n += 1 - # config/tokenizer solo per la conversione principale — i passaggi mtp/idx - # vanno nella stessa outdir di un container gia' completo di metadati. + done[key] = "" + else: + name = f"{prefix}{n:05d}.safetensors" + save_file(out, os.path.join(a.outdir, name)) + done[key] = name; n += 1; fresh += 1 + tmp_prog = prog_path + ".tmp" # scrittura atomica: una ripresa non vede mai un manifest mezzo scritto + with open(tmp_prog, "w") as f: json.dump(prog, f, indent=1) # EN: atomic write: a resume never sees a half-written manifest + os.replace(tmp_prog, prog_path) + if skipped: print(f"[RESUME] {skipped} shard(s) already done in {a.outdir}, skipped") + # metadati per la conversione principale: gli stessi quattro file del download + # path — senza tokenizer.json chat/serve non partono. I passaggi mtp/idx vanno + # nella stessa outdir di un container gia' completo di metadati. + # EN: metadata for the main pass: the same four files as the download path — + # EN: chat/serve won't start without tokenizer.json. The mtp/idx passes target + # EN: an outdir whose container already has its metadata. if not a.mtp and not a.indexer: - for fn in ["config.json"]: + copied, missing = [], [] + for fn in ["config.json", "tokenizer.json", "tokenizer_config.json", "generation_config.json"]: src = os.path.join(a.indir, fn) - if os.path.exists(src): shutil.copy(src, a.outdir) + if os.path.exists(src): shutil.copy(src, a.outdir); copied.append(fn) + else: missing.append(fn) + print(f"[META] copied from {a.indir}: {', '.join(copied) if copied else 'nothing'}") + if missing: + print(f"[META] WARNING: not found in {a.indir}: {', '.join(missing)}" + + (" — chat/serve need tokenizer.json" if "tokenizer.json" in missing else "")) tag = "MTP" if a.mtp else "indexer" if a.indexer else "main" - print(f"converted {n} {tag} shard(s) -> {a.outdir} ({prefix}NNNNN)") + print(f"converted {fresh} {tag} shard(s), {n} in container -> {a.outdir} ({prefix}NNNNN)") return # reale: scarica shard per shard, converte, cancella From d6676c10e9d78b0f59ced6e51a4709bc2a22b5c5 Mon Sep 17 00:00:00 2001 From: recviking Date: Sat, 18 Jul 2026 16:27:29 -0400 Subject: [PATCH 57/95] =?UTF-8?q?tools:=20repair=5Fmtp=5Fint8.py=20?= =?UTF-8?q?=E2=80=94=20fix=20int4-converted=20MTP=20heads=20in=20place;=20?= =?UTF-8?q?warn=20on=20--mtp=20--ebits=20<8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-row int4 quantization of the MTP head's eh_proj [D,2D] zeroes its ENTIRE embedding half: the tensor's two column halves differ in scale by ~20-30x per row (embedding-half absmax ~0.05, hidden-half ~1.5 on GLM-5.2), so a single per-row scale (absmax/7) puts every embedding-half weight below half a quantization step and np.rint rounds it to exact zero (packed bytes 0x88). The draft head then cannot see the input token and MTP acceptance collapses to ~0% — the mechanism behind issue #8's measurement (int4: 0-4%; int8: 39-59%), and the reason --mtp already defaults to --ebits 8. This showed up in the wild: a published pre-converted container had its MTP shards made with an explicit --ebits 4 and drafts were pure garbage on every backend (deterministic, data-side; verified byte-for-byte by reproducing the published bytes with quant_int4 on the official BF16 rows). - tools/repair_mtp_int8.py: repairs such a container IN PLACE — finds the MTP layer's per-row-int4 dense tensors (eh_proj, q/kv/o projections, shared experts; routed experts untouched), re-downloads only those from the FP8 source repo (~355 MB of HTTP range reads, no torch, no token), requantizes at int8 with quant_int8's exact math, and rewrites the shards atomically with *.bak-int4 backups. --dry-run to inspect. The engine picks up int8 automatically (qt_from_disk detects format by blob size). Validated on GLM-5.2 744B: MTP acceptance 0% -> 100% (greedy), 1.11 -> 3.20 tokens/forward, decode 0.18 -> 0.30 tok/s (Metal, 4-bit KV). - convert_fp8_to_int4.py: print a warning when --mtp is combined with --ebits <8 and per-row scales (the exact footgun above); --group-size 128 remains a valid int4 alternative since group scales give the embedding half its own scale. Co-Authored-By: Claude Fable 5 --- c/tools/convert_fp8_to_int4.py | 11 ++ c/tools/repair_mtp_int8.py | 229 +++++++++++++++++++++++++++++++++ 2 files changed, 240 insertions(+) create mode 100644 c/tools/repair_mtp_int8.py diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py index 382125e..e398d6f 100644 --- a/c/tools/convert_fp8_to_int4.py +++ b/c/tools/convert_fp8_to_int4.py @@ -286,6 +286,17 @@ def main(): # testa MTP a int4 = acceptance ~0-4% (misurato, issue #8): il draft sbaglia sempre # e la speculazione non parte mai. A int8: 39-59%, 2.2-2.8 token/forward. a.ebits = 8 if (a.mtp or a.indexer) else 4 + if a.mtp and a.ebits < 8 and a.group_size <= 0: + # Non solo lossy: eh_proj ha ~20-30x di asimmetria di scala fra le due meta' di + # colonna, quindi l'int4 per-riga (UNA scala per riga) arrotonda a ZERO l'intera + # meta' embedding -> il draft non vede il token -> acceptance ~0% (issue #8). + # EN: not merely lossy: eh_proj has ~20-30x column-scale asymmetry, so per-row + # EN: int4 rounds its ENTIRE embedding half to exact zeros -> the draft cannot + # EN: see the input token -> ~0% acceptance (issue #8). A container converted + # EN: this way is repairable in place with tools/repair_mtp_int8.py. + print(f"WARNING: --mtp with --ebits {a.ebits} and per-row scales ZEROES eh_proj's " + "embedding half -> MTP acceptance ~0% (issue #8). Use the default --ebits 8, " + "or add --group-size 128 for group-scaled int4.") if a.xbits is None: a.xbits = a.ebits # Build per-type bits map. If a type-specific arg is set, use it; otherwise the diff --git a/c/tools/repair_mtp_int8.py b/c/tools/repair_mtp_int8.py new file mode 100644 index 0000000..695e99f --- /dev/null +++ b/c/tools/repair_mtp_int8.py @@ -0,0 +1,229 @@ +"""Repair an existing colibri int4 container whose MTP head was quantized at int4. + +WHY THIS EXISTS + `model.layers..eh_proj.weight` [D, 2D] multiplies the MTP concat + [embedding_norm ; hidden_norm], and its two column halves differ in scale by + ~20-30x per row (embedding-half absmax ~0.05, hidden-half ~1.5 on GLM-5.2). + Per-row int4 uses ONE scale (= absmax/7) per row, so every embedding-half + weight lands below half a quantization step and np.rint rounds the ENTIRE + embedding half to exact zeros (packed bytes 0x88). The MTP head then drafts + garbage: acceptance ~0% (issue #8 measured 0-4% at int4; 39-59% at int8 — + which is why `convert_fp8_to_int4.py --mtp` defaults to --ebits 8). + + A container converted (or downloaded) with an int4 MTP head does not need a + full re-conversion: this script re-downloads ONLY the affected dense tensors + (~355 MB of HTTP range reads against the FP8 source repo), requantizes them + at int8 with the converter's exact math, and patches the local shards in + place. Originals are kept beside as *.bak-int4. + +WHAT IT TOUCHES + The MTP layer's dense tensors only (eh_proj, q_a/q_b/kv_a/kv_b/o_proj, + shared_experts.*): the ones that stream into RAM once and stay resident. + Routed experts (model.layers..mlp.experts.*) are NOT touched — they are + statistically like the main layers' experts and int4 is acceptable there. + The engine auto-detects int8 vs int4 by blob size (qt_from_disk), so no + engine or config change is needed. Cost: ~+133 MB on disk / resident RAM. + +USAGE + python3 tools/repair_mtp_int8.py --snap /path/to/glm52_i4 # repair + python3 tools/repair_mtp_int8.py --snap /path/to/glm52_i4 --dry-run # inspect only + + --source-repo defaults to zai-org/GLM-5.2-FP8 (the checkpoint the public + int4 containers were converted from). Requires numpy and network access; + no torch, no HF token (public repo, anonymous range reads). +""" +import argparse, glob, json, os, ssl, struct, sys, urllib.request +import numpy as np + +# macOS python.org builds ship no CA bundle: use certifi when available (Linux +# system Pythons generally have working system certs and skip this). +try: + import certifi + _SSL_CTX = ssl.create_default_context(cafile=certifi.where()) +except ImportError: + _SSL_CTX = ssl.create_default_context() + + +# ---------- HTTP range reads against the source repo ---------- +def http_range(url, start, length, tries=5): + req = urllib.request.Request(url, headers={"User-Agent": "colibri-mtp-repair", + "Range": f"bytes={start}-{start+length-1}"}) + for attempt in range(tries): + try: + with urllib.request.urlopen(req, timeout=30, context=_SSL_CTX) as r: + data = r.read() + if len(data) == length: + return data + except KeyboardInterrupt: + raise + except Exception as ex: + if attempt == tries - 1: + raise RuntimeError(f"range read failed for {url}: {ex}") + raise RuntimeError(f"short range read for {url}") + + +class SourceRepo: + def __init__(self, repo, revision="main"): + self.base = f"https://huggingface.co/{repo}/resolve/{revision}/" + with urllib.request.urlopen(self.base + "model.safetensors.index.json", timeout=30, context=_SSL_CTX) as r: + self.wmap = json.loads(r.read())["weight_map"] + self._hdr = {} + + def _shard_header(self, shard): + if shard not in self._hdr: + n = struct.unpack("> 3) & 0xF + mant = (b & 7).astype(np.float64) + v = (sign * np.where(e > 0, (1 + mant / 8) * np.exp2(e.astype(np.float64) - 7), + mant / 8 * np.exp2(-6.0))).reshape(m["shape"]) + sn = name + "_scale_inv" + sshard = self.wmap[sn] + shdr, sbase = self._shard_header(sshard) + sm = shdr[sn] + so0, so1 = sm["data_offsets"] + sc = np.frombuffer(http_range(self.base + sshard, sbase + so0, so1 - so0), + dtype=np.float32).reshape(sm["shape"]) + O, I = m["shape"] + scf = np.repeat(np.repeat(sc, 128, axis=0)[:O], 128, axis=1)[:, :I] + return (v * scf).astype(np.float32) + raise ValueError(f"{name}: unsupported source dtype {m['dtype']}") + + +# ---------- quantization: identical to convert_fp8_to_int4.quant_int8 ---------- +def quant_int8(w): + amax = np.abs(w).max(axis=1, keepdims=True) + s = np.maximum(amax / 127, 1e-8) + q = np.clip(np.rint(w / s), -128, 127).astype(np.int8) + return q.reshape(-1).view(np.uint8).copy(), s[:, 0].astype(np.float32) + + +# ---------- local safetensors IO (no deps; preserves byte-identity of untouched tensors) ---------- +def read_shard(path): + with open(path, "rb") as fh: + n = struct.unpack(" [tensor names to repair] + already_ok, skipped_experts = [], 0 + for f in sorted(glob.glob(os.path.join(a.snap, "*.safetensors"))): + with open(f, "rb") as fh: + n = struct.unpack(" Date: Sun, 19 Jul 2026 05:01:41 +0800 Subject: [PATCH 58/95] release: version infrastructure + GitHub Release workflow - c/version.py: single source of truth (__version__ = "1.0.0") - coli: reads version.py, banner shows dynamic version, --version flag - .github/workflows/release.yml: tag push triggers cross-platform build (Linux x86_64, macOS ARM64, Windows x86_64) and creates a GitHub Release with packaged binaries + changelog notes - CHANGELOG.md: v1.0.0 baseline documenting all shipped features To cut a release: 1. bump c/version.py 2. add a CHANGELOG section 3. git tag v1.0.0 && git push --tags --- .github/workflows/release.yml | 99 +++++++++++++++++++++++++++++++++++ CHANGELOG.md | 53 +++++++++++++++++++ c/coli | 5 +- c/version.py | 3 ++ 4 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release.yml create mode 100644 CHANGELOG.md create mode 100644 c/version.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..245d24f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,99 @@ +name: Release + +on: + push: + tags: ['v*'] + +permissions: + contents: write + +jobs: + build: + strategy: + matrix: + include: + - os: ubuntu-latest + name: linux-x86_64 + ext: "" + make_args: "ARCH=x86-64-v3" + - os: macos-latest + name: macos-arm64 + ext: "" + make_args: "" + - os: windows-latest + name: windows-x86_64 + ext: ".exe" + make_args: "" + shell: msys2 + runs-on: ${{ matrix.os }} + defaults: + run: + shell: ${{ matrix.shell || 'bash' }} + steps: + - uses: actions/checkout@v4 + + - if: matrix.shell == 'msys2' + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: false + install: >- + make + mingw-w64-ucrt-x86_64-gcc + + - if: matrix.os == 'macos-latest' + run: brew install libomp + + - name: Build engine + run: | + cd c + make glm ${{ matrix.make_args }} + ls -lh glm${{ matrix.ext }} + + - name: Package + run: | + TAG=${GITHUB_REF#refs/tags/} + mkdir -p dist + cp c/glm${{ matrix.ext }} dist/colibri-${TAG}-${{ matrix.name }}${{ matrix.ext }} + cp c/coli dist/ + cp c/version.py dist/ + cp c/openai_server.py dist/ + cp c/resource_plan.py dist/ + cp c/doctor.py dist/ + cp LICENSE dist/ + cd dist + if [ "${{ matrix.ext }}" = ".exe" ]; then + 7z a colibri-${TAG}-${{ matrix.name }}.zip * + else + tar czf colibri-${TAG}-${{ matrix.name }}.tar.gz * + fi + + - uses: actions/upload-artifact@v4 + with: + name: colibri-${{ matrix.name }} + path: dist/colibri-*.* + + release: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + path: artifacts + merge-multiple: true + + - name: Create GitHub Release + env: + GH_TOKEN: ${{ github.token }} + run: | + TAG=${GITHUB_REF#refs/tags/} + # Extract the latest version section from CHANGELOG + NOTES=$(awk "/^## \\[${TAG#v}\\]/{found=1;next} /^## \\[/{if(found)exit} found{print}" CHANGELOG.md) + if [ -z "$NOTES" ]; then + NOTES="Release ${TAG}" + fi + gh release create "$TAG" artifacts/* \ + --title "colibrì ${TAG}" \ + --notes "$NOTES" diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6177966 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,53 @@ +# Changelog + +All notable changes to colibrì are documented here. +Format follows [Keep a Changelog](https://keepachangelog.com/). + +## [1.0.0] — 2026-07-19 + +First tagged release. The engine has been running in production since late June +2026; this tag marks the baseline for semantic versioning going forward. + +### Highlights + +- **GLM-5.2 (744B MoE)** runs on ~25 GB RAM in pure C, streaming experts from disk +- **Three-tier placement**: VRAM (hot) / RAM (warm) / NVMe (cold), with a learning + cache that pins your workload's hottest experts automatically +- **CUDA backend**: multi-GPU expert tier, dense tensor distribution, batched + ragged attention, resident pipeline (`COLI_CUDA_PIPE=2`) +- **Metal backend** (Apple Silicon): batched expert SwiGLU + fused decode attention + on unified memory GPU +- **MTP speculation**: native GLM-5.2 draft heads, grammar-forced drafts, kernel- + pinned verification (`SPEC_PIN=1`) +- **OpenAI-compatible API**: `coli serve` with SSE streaming, KV slots, bounded + queue, web dashboard (`coli web`) +- **Web UI**: chat with live metrics, expert cortex brain page, profiling breakdown, + expert atlas 3-D galaxy +- **Cross-platform**: Linux, macOS, Windows 11 (native MinGW), PowerPC; CI on all three +- **Auto-tune**: `coli plan --auto-tier` classifies the bottleneck and derives + MTP/PIPE/NUMA/PIN settings with explanations + +### Engine + +- Token-exact validation against `transformers` oracle (teacher-forcing 32/32) +- Compressed MLA KV cache (576 floats/token, 57× smaller), persisted across + restarts (`.coli_kv`, zero re-prefill) +- DSA sparse attention (lightning indexer), faithfully implemented +- Router-lookahead prefetch (`PILOT=1`, 71.6% predictive) +- Async expert I/O pool (`PIPE=1`), io_uring batching (`URING=1`) +- NUMA-aware expert placement (`COLI_NUMA=1`, +13–40% on multi-socket) +- AVX2 / AVX-512 / AVX-VNNI / ARM NEON / NEON-i8mm / POWER VSX kernels +- int4 / int8 / int2 / grouped-int4 (fmt=4) quantization formats + +### Tools + +- `coli convert` — FP8→int4 one-shard-at-a-time converter +- `coli doctor` — read-only setup diagnostics +- `coli plan` — resource planner with auto-tune prescription +- `coli bench` — MMLU / HellaSwag / ARC quality benchmarks +- Expert atlas (`tools/analyze.py --web`) — measured topic affinity for 19,456 experts + +### Community + +- 30+ hardware datapoints in the benchmark tracker +- Contributions from 20+ authors across engine, docs, tooling, and ports diff --git a/c/coli b/c/coli index 4c27415..81bc787 100755 --- a/c/coli +++ b/c/coli @@ -40,6 +40,8 @@ if sys.platform == "win32": except (AttributeError, OSError): pass HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +from version import __version__ as _version # Run-in-place (source checkout, "cd c && ./coli ..."): the engine, the # support modules (resource_plan.py, doctor.py, openai_server.py) and @@ -115,7 +117,7 @@ def sprite_lines(): def banner(sub=""): sp=sprite_lines() txt=[ - f"{C.teal}{C.b}colibrì{C.r} {C.dim}v1.0{C.r}", + f"{C.teal}{C.b}colibrì{C.r} {C.dim}v{_version}{C.r}", f"{C.dim}tiny engine, immense model{C.r}", f"{C.gray}GLM-5.2 · 744B MoE · int4 · streaming CPU{C.r}", f"{C.dgray}{sub}{C.r}" if sub else "", @@ -712,6 +714,7 @@ def main(): common.add_argument("--topp", type=float, default=0); common.add_argument("--topk", type=int, default=0) common.add_argument("--temp", type=float, default=None) # temperatura token (0=greedy, default 1.0+nucleus .95) ap=argparse.ArgumentParser(prog="coli", parents=[common], description="colibrì — run GLM-5.2 locally") + ap.add_argument("--version", action="version", version=f"colibrì {_version}") sub=ap.add_subparsers(dest="cmd") sub.add_parser("build", parents=[common]); sub.add_parser("info", parents=[common]) pp=sub.add_parser("plan",parents=[common]) diff --git a/c/version.py b/c/version.py new file mode 100644 index 0000000..9951ca1 --- /dev/null +++ b/c/version.py @@ -0,0 +1,3 @@ +"""Single source of truth for the colibrì version number.""" + +__version__ = "1.0.0" From d2d3a7b5598f0feb49ccc8e4665c2b7375d4afb2 Mon Sep 17 00:00:00 2001 From: Stonki13 <204933532+Stonki13@users.noreply.github.com> Date: Sat, 18 Jul 2026 23:18:16 +0200 Subject: [PATCH 59/95] Fix CUDA detection on Windows: cuda_binary()/cuda_linkage() always returned False MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both coli's cuda_binary() and doctor.py's cuda_linkage() detect CUDA support by running `ldd` on the engine binary and looking for a linked libcudart — but that's Linux-only (cuda_binary() checks sys.platform != "linux", and cuda_linkage() checks os.name != "posix", both short-circuiting to False on win32). Windows CUDA_DLL=1 builds never link libcudart at all: glm.exe loads coli_cuda.dll dynamically via LoadLibrary at startup (backend_loader.c), so there's no import-table entry for ldd/dumpbin to find in the first place. The practical effect: `coli doctor` always reported "NVIDIA GPU detected but the engine is CPU-only" on Windows, and `coli run/chat/serve --gpu ...` always hard-exited with "--gpu needs the CUDA build" — even on a correctly built CUDA_DLL=1 binary with coli_cuda.dll sitting right next to glm.exe. Fix: on win32, detect a COLI_CUDA build by scanning glm.exe for the marker string "[CUDA] mode: routed experts", which only exists in code compiled under #ifdef COLI_CUDA (see glm.c's cuda init block), then confirm coli_cuda.dll actually sits next to the binary — mirroring the Linux "linked but missing" distinction. Linux/macOS detection is unchanged. Verified on Windows 11 with mingw-w64 GCC 16.1.0 + CUDA 13.2 + RTX 5080: `coli doctor` now reports "CUDA engine and devices are available", and `coli run --gpu 0 --vram 8` populates VRAM (confirmed via the engine's own "[CUDA] resident set: N tensors, X GB VRAM" runtime log) instead of exiting. --- c/coli | 24 ++++++++++++++++++------ c/doctor.py | 35 ++++++++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/c/coli b/c/coli index 4c27415..4566ea6 100755 --- a/c/coli +++ b/c/coli @@ -141,12 +141,24 @@ def need_model(model): sys.exit(f"{C.yel}engine is not built.{C.r} Run: coli build") def cuda_binary(): - if not os.path.exists(GLM) or sys.platform != "linux": return False - try: - linked=subprocess.run(["ldd",GLM],capture_output=True,text=True,timeout=3) - return any("libcudart" in line and "not found" not in line - for line in linked.stdout.splitlines()) - except (OSError,subprocess.SubprocessError): return False + if not os.path.exists(GLM): return False + if sys.platform == "linux": + try: + linked=subprocess.run(["ldd",GLM],capture_output=True,text=True,timeout=3) + return any("libcudart" in line and "not found" not in line + for line in linked.stdout.splitlines()) + except (OSError,subprocess.SubprocessError): return False + if sys.platform == "win32": + # Windows CUDA_DLL=1 builds never link libcudart directly: glm.exe loads + # coli_cuda.dll at runtime via LoadLibrary (backend_loader.c), so there's no + # import-table entry for ldd/dumpbin to see. Detect the COLI_CUDA build via a + # marker string baked into glm.c's #ifdef COLI_CUDA block instead, and require + # coli_cuda.dll to actually sit next to glm.exe (else CUDA init fails at startup). + try: + with open(GLM,"rb") as f: built=b"[CUDA] mode: routed experts" in f.read() + except OSError: return False + return built and os.path.exists(os.path.join(os.path.dirname(GLM),"coli_cuda.dll")) + return False def resource_request(a, env): ctx=a.ctx or int(env.get("CTX",4096)) diff --git a/c/doctor.py b/c/doctor.py index 4cb5a42..cbe9c87 100644 --- a/c/doctor.py +++ b/c/doctor.py @@ -19,16 +19,33 @@ def _check(identifier, status, summary, **details): def cuda_linkage(engine_path): """Return CUDA linkage state without loading the executable or CUDA runtime.""" - if not Path(engine_path).is_file() or os.name != "posix": + engine = Path(engine_path) + if not engine.is_file(): return {"linked": False, "missing": False} - try: - result = subprocess.run(["ldd", str(engine_path)], capture_output=True, text=True, - timeout=3, check=False) - except (OSError, subprocess.SubprocessError): - return {"linked": False, "missing": False} - lines = [line for line in result.stdout.splitlines() if "libcudart" in line] - return {"linked": any("not found" not in line for line in lines), - "missing": any("not found" in line for line in lines)} + if os.name == "posix": + try: + result = subprocess.run(["ldd", str(engine)], capture_output=True, text=True, + timeout=3, check=False) + except (OSError, subprocess.SubprocessError): + return {"linked": False, "missing": False} + lines = [line for line in result.stdout.splitlines() if "libcudart" in line] + return {"linked": any("not found" not in line for line in lines), + "missing": any("not found" in line for line in lines)} + if sys.platform == "win32": + # Windows CUDA_DLL=1 builds never link libcudart directly: glm.exe loads + # coli_cuda.dll at runtime via LoadLibrary (backend_loader.c), so there's no + # import-table entry for ldd/dumpbin to see. Detect the COLI_CUDA build via a + # marker string baked into glm.c's #ifdef COLI_CUDA block instead, and require + # coli_cuda.dll to actually sit next to glm.exe (else CUDA init fails at startup). + try: + built = b"[CUDA] mode: routed experts" in engine.read_bytes() + except OSError: + return {"linked": False, "missing": False} + if not built: + return {"linked": False, "missing": False} + dll_present = (engine.parent / "coli_cuda.dll").is_file() + return {"linked": dll_present, "missing": not dll_present} + return {"linked": False, "missing": False} def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *, From d86a6b93ad4f70045138a13907682b5f936c3355 Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Sun, 19 Jul 2026 06:10:14 +0800 Subject: [PATCH 60/95] packaging: pyproject.toml + editorconfig + clang-format Python packaging: - pyproject.toml: pip install colibri-engine (editable dev install works) - colibri/ package with __version__, CLI entry point delegating to c/coli - Optional dependency groups: [convert] (numpy, huggingface_hub), [oracle] (torch, transformers, safetensors), [bench] (tokenizers, datasets) Code style: - .editorconfig: consistent indent/charset across all file types - .clang-format: LLVM-based, 120 col, matches existing engine style Usage: pip install -e . # dev install (CLI + serve, no heavy deps) pip install -e .[convert] # adds converter dependencies pip install -e .[oracle] # adds torch/transformers for oracle validation --- .clang-format | 10 +++++++++ .editorconfig | 24 ++++++++++++++++++++ colibri/__init__.py | 5 +++++ colibri/_version.py | 3 +++ colibri/cli.py | 30 +++++++++++++++++++++++++ pyproject.toml | 53 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 125 insertions(+) create mode 100644 .clang-format create mode 100644 .editorconfig create mode 100644 colibri/__init__.py create mode 100644 colibri/_version.py create mode 100644 colibri/cli.py create mode 100644 pyproject.toml diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..f911652 --- /dev/null +++ b/.clang-format @@ -0,0 +1,10 @@ +BasedOnStyle: LLVM +IndentWidth: 4 +ColumnLimit: 120 +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: AllIfsAndElse +AllowShortLoopsOnASingleLine: true +BreakBeforeBraces: Attach +PointerAlignment: Right +SpaceAfterCStyleCast: false +SortIncludes: false diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..e0acabb --- /dev/null +++ b/.editorconfig @@ -0,0 +1,24 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 + +[*.{c,h,cu}] +indent_size = 4 + +[*.{ts,tsx,js,json,css}] +indent_size = 2 + +[Makefile] +indent_style = tab + +[*.yml] +indent_size = 2 + +[*.md] +trim_trailing_whitespace = false diff --git a/colibri/__init__.py b/colibri/__init__.py new file mode 100644 index 0000000..77f7061 --- /dev/null +++ b/colibri/__init__.py @@ -0,0 +1,5 @@ +"""colibrì — tiny engine, immense model.""" + +from colibri._version import __version__ + +__all__ = ["__version__"] diff --git a/colibri/_version.py b/colibri/_version.py new file mode 100644 index 0000000..9951ca1 --- /dev/null +++ b/colibri/_version.py @@ -0,0 +1,3 @@ +"""Single source of truth for the colibrì version number.""" + +__version__ = "1.0.0" diff --git a/colibri/cli.py b/colibri/cli.py new file mode 100644 index 0000000..3f19234 --- /dev/null +++ b/colibri/cli.py @@ -0,0 +1,30 @@ +"""Entry point for `coli` when installed via pip. + +Delegates to the original c/coli script which handles all subcommands. +This wrapper exists so `pip install colibri-engine` creates a `coli` console +script that works without the user having to add c/ to PATH manually. +""" + +import os +import sys +import runpy + + +def main(): + here = os.path.dirname(os.path.abspath(__file__)) + engine_dir = os.path.join(os.path.dirname(here), "c") + coli_script = os.path.join(engine_dir, "coli") + + if not os.path.exists(coli_script): + sys.exit( + "colibri engine directory not found.\n" + "Install from source: git clone + pip install -e ." + ) + + sys.path.insert(0, engine_dir) + sys.argv[0] = coli_script + runpy.run_path(coli_script, run_name="__main__") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..18f8f85 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,53 @@ +[build-system] +requires = ["setuptools>=68.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "colibri-engine" +dynamic = ["version"] +description = "Tiny engine, immense model — run GLM-5.2 (744B MoE) locally" +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.10" +authors = [ + {name = "JustVugg"}, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Science/Research", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] + +[project.optional-dependencies] +convert = [ + "numpy", + "huggingface_hub", +] +oracle = [ + "torch>=2.0", + "transformers>=4.40", + "safetensors", +] +bench = [ + "tokenizers", + "datasets", +] + +[project.scripts] +coli = "colibri.cli:main" + +[project.urls] +Homepage = "https://github.com/JustVugg/colibri" +Issues = "https://github.com/JustVugg/colibri/issues" + +[tool.setuptools.dynamic] +version = {attr = "colibri._version.__version__"} + +[tool.setuptools.packages.find] +where = ["."] +include = ["colibri*"] From 2122c004d9bdbaa387c64ae4d6c00f24ace128b1 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sun, 19 Jul 2026 10:56:00 +0200 Subject: [PATCH 61/95] serve: end-to-end tool-calling regression test + unparsed-marker diagnosis (#401) The gateway's tool-calling path had unit coverage (parse_tool_calls, render_chat) but nothing exercised the real subprocess wire protocol or the HTTP surface a coding client actually hits. #401 reports plain-text replies where tool_calls were expected; every documented path checks out, so pin the whole path down with a mock engine speaking SUBMIT/DATA/DONE and assert: - non-stream: tool_calls populated, finish_reason tool_calls, no raw markers - stream: markers suppressed across 20-way chunk splits, tool_calls delta - tool-result round trip: <|observation|> rendering, text reply - no tools: plain text untouched Also emit a stderr diagnosis when tools are declared and tool-call markers are present in the reply but the strict parse matches nothing (typically quantization-mangled output) pointing at COLI_TOOL_SALVAGE=1 -- the likely field condition behind #401. Co-Authored-By: Claude Fable 5 --- c/openai_server.py | 8 ++ c/tests/test_openai_tools_e2e.py | 178 +++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 c/tests/test_openai_tools_e2e.py diff --git a/c/openai_server.py b/c/openai_server.py index f56e20a..3900ba6 100644 --- a/c/openai_server.py +++ b/c/openai_server.py @@ -271,6 +271,14 @@ def parse_tool_calls(reply, tools=None): salvaged.append(name) calls.append({"id": "call_" + uuid.uuid4().hex[:24], "type": "function", "function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)}}) + if tools and not calls and re.search(r"||", reply): + # Diagnosi per la #401: il client ha dichiarato i tools e il modello ha PROVATO la + # sintassi, ma il parse rigoroso non ha agganciato nulla (tipico output int4 storpiato). + # EN: #401 field diagnosis: tools were declared and the model attempted the syntax, + # EN: but the strict parse matched nothing (typically quantization-mangled output). + sys.stderr.write("[api] tools declared and tool-call markers present, but no call " + "parsed -- output may be quantization-mangled; try COLI_TOOL_SALVAGE=1\n") + sys.stderr.flush() text = _BOX_RE.sub("", reply) if THINK_CLOSE in text: text = text.split(THINK_CLOSE, 1)[1] diff --git a/c/tests/test_openai_tools_e2e.py b/c/tests/test_openai_tools_e2e.py new file mode 100644 index 0000000..3eabee1 --- /dev/null +++ b/c/tests/test_openai_tools_e2e.py @@ -0,0 +1,178 @@ +"""End-to-end tool-calling test for the OpenAI gateway (#401). + +Unlike the unit tests in test_openai_server.py (which call parse_tool_calls / +render_chat directly), this suite runs openai_server.py as a real subprocess +against a mock engine that speaks the actual SERVE wire protocol +(READY / SUBMIT / DATA / DONE), then talks to it over real HTTP. It pins down +the full path a coding client exercises: tool declaration rendering, marker +suppression in streamed deltas (across chunk boundaries), tool_calls in both +response shapes, and the <|observation|> round trip. +""" +import json +import os +import socket +import subprocess +import sys +import tempfile +import unittest +import urllib.request +from pathlib import Path + +SERVER = Path(__file__).resolve().parent.parent / "openai_server.py" +MODEL_ID = "glm-5.2-colibri" + +# Mock engine: replies are keyed on the prompt so one process covers every case. +# Prompts received are appended to MOCK_LOG for assertions on the rendering. +MOCK_ENGINE = r'''#!/usr/bin/env python3 +import sys, os +out, inp = sys.stdout.buffer, sys.stdin.buffer +out.write(b"\x01\x01READY\x01\x01\n" + b"STAT 0 0 0 0 0\n"); out.flush() + +def reply(rid, text, chunks=1): + data = text.encode("utf-8") + n = max(1, len(data) // chunks) + for i in range(0, len(data), n): + part = data[i:i+n] + out.write(("DATA %s %d\n" % (rid, len(part))).encode() + part + b"\n"); out.flush() + out.write(("DONE %s STAT %d 1.0 50.0 10.0 42 0\n" % (rid, len(text.split()))).encode()) + out.flush() + +while True: + line = inp.readline() + if not line: break + f = line.decode().strip().split() + if not f or f[0] != "SUBMIT": continue + rid, plen = f[1], int(f[3]) + prompt = inp.read(plen).decode("utf-8", "replace"); inp.read(1) + with open(os.environ["MOCK_LOG"], "a") as log: + log.write(prompt + "\n\x00\n") + if "" in prompt: + reply(rid, "25 degrees and sunny in Rome.") + elif "weather in Rome" in prompt: + reply(rid, "get_weathercity" + "Rome") + elif "weather in Milan" in prompt: + # split across many tiny DATA chunks: streamed marker suppression must + # hold even when a marker straddles a chunk boundary + reply(rid, "Checking. get_weathercity" + "Milan", chunks=20) + else: + reply(rid, "Hello from the mock engine.") +''' + +TOOLS = [{"type": "function", "function": { + "name": "get_weather", + "description": "Current weather for a city", + "parameters": {"type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"]}}}] + + +class ToolCallingE2E(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + mock = Path(cls.tmp.name) / "mock_engine.py" + mock.write_text(MOCK_ENGINE) + mock.chmod(0o755) + cls.mock_log = Path(cls.tmp.name) / "prompts.log" + cls.mock_log.touch() + with socket.socket() as probe: # free port, then hand it to the server + probe.bind(("127.0.0.1", 0)) + cls.port = probe.getsockname()[1] + env = dict(os.environ, MOCK_LOG=str(cls.mock_log)) + cls.server = subprocess.Popen( + [sys.executable, str(SERVER), "--model", cls.tmp.name, + "--engine", str(mock), "--port", str(cls.port)], + env=env, stderr=subprocess.DEVNULL) + cls.base = f"http://127.0.0.1:{cls.port}/v1" + for _ in range(100): + try: + urllib.request.urlopen(cls.base + "/models", timeout=2) + return + except OSError: + if cls.server.poll() is not None: + raise RuntimeError("gateway exited during startup") + import time + time.sleep(0.1) + raise RuntimeError("gateway did not come up") + + @classmethod + def tearDownClass(cls): + cls.server.terminate() + cls.server.wait(timeout=5) + cls.tmp.cleanup() + + def post(self, body, stream=False): + req = urllib.request.Request( + self.base + "/chat/completions", json.dumps(body).encode(), + {"Content-Type": "application/json"}) + resp = urllib.request.urlopen(req, timeout=30) + if not stream: + return json.loads(resp.read()) + events = [] + for raw in resp: + line = raw.decode().strip() + if line.startswith("data: ") and line != "data: [DONE]": + events.append(json.loads(line[6:])) + return events + + def test_tool_call_non_stream(self): + r = self.post({"model": MODEL_ID, "tools": TOOLS, + "messages": [{"role": "user", + "content": "What is the weather in Rome?"}]}) + choice = r["choices"][0] + self.assertEqual(choice["finish_reason"], "tool_calls") + calls = choice["message"]["tool_calls"] + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0]["function"]["name"], "get_weather") + self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"city": "Rome"}) + self.assertNotIn("", choice["message"].get("content") or "") + + def test_tool_call_streamed_markers_suppressed(self): + events = self.post({"model": MODEL_ID, "tools": TOOLS, "stream": True, + "messages": [{"role": "user", + "content": "What is the weather in Milan?"}]}, + stream=True) + deltas = [e["choices"][0]["delta"] for e in events if e["choices"]] + text = "".join(d.get("content") or "" for d in deltas) + self.assertNotIn("", text) + self.assertNotIn("", text) + calls = [d["tool_calls"] for d in deltas if d.get("tool_calls")] + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0][0]["function"]["name"], "get_weather") + self.assertEqual(json.loads(calls[0][0]["function"]["arguments"]), + {"city": "Milan"}) + finish = [e["choices"][0]["finish_reason"] for e in events + if e["choices"] and e["choices"][0].get("finish_reason")] + self.assertEqual(finish, ["tool_calls"]) + + def test_tool_result_round_trip(self): + r = self.post({"model": MODEL_ID, "tools": TOOLS, "messages": [ + {"role": "user", "content": "What is the weather in Rome?"}, + {"role": "assistant", "content": None, "tool_calls": [ + {"id": "call_x", "type": "function", + "function": {"name": "get_weather", + "arguments": "{\"city\": \"Rome\"}"}}]}, + {"role": "tool", "tool_call_id": "call_x", + "content": "25 degrees, sunny"}]}) + choice = r["choices"][0] + self.assertEqual(choice["finish_reason"], "stop") + self.assertFalse(choice["message"].get("tool_calls")) + self.assertIn("25 degrees", choice["message"]["content"]) + rendered = self.mock_log.read_text().split("\x00")[-2] + self.assertIn("<|observation|>25 degrees, sunny", + rendered) + self.assertIn("# Tools", rendered) + self.assertIn('"get_weather"', rendered) + + def test_no_tools_plain_text(self): + r = self.post({"model": MODEL_ID, + "messages": [{"role": "user", "content": "Hi!"}]}) + choice = r["choices"][0] + self.assertEqual(choice["finish_reason"], "stop") + self.assertIn("mock engine", choice["message"]["content"]) + + +if __name__ == "__main__": + unittest.main() From 0f606bc4465df0473f39e809f9510b318708631e Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sun, 19 Jul 2026 11:01:26 +0200 Subject: [PATCH 62/95] test: skip the tools e2e suite on Windows (shebang mock engine) CreateProcess cannot exec a shebang script, so the gateway exits during setUpClass on the windows CI job. The gateway logic under test is platform-independent and stays covered by the POSIX jobs. Co-Authored-By: Claude Fable 5 --- c/tests/test_openai_tools_e2e.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/c/tests/test_openai_tools_e2e.py b/c/tests/test_openai_tools_e2e.py index 3eabee1..d921843 100644 --- a/c/tests/test_openai_tools_e2e.py +++ b/c/tests/test_openai_tools_e2e.py @@ -68,6 +68,10 @@ TOOLS = [{"type": "function", "function": { "required": ["city"]}}}] +@unittest.skipUnless(os.name == "posix", + "the mock engine is a shebang script the gateway execs directly; " + "Windows CreateProcess cannot run it. The gateway logic under test " + "is platform-independent and covered by the POSIX CI jobs.") class ToolCallingE2E(unittest.TestCase): @classmethod def setUpClass(cls): From c90e2cc4382e4a7f7f48914a03a9b0c181c38694 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sun, 19 Jul 2026 11:02:31 +0200 Subject: [PATCH 63/95] =?UTF-8?q?glm:=20measured-RSS=20guard=20=E2=80=94?= =?UTF-8?q?=20the=20RAM=20budget=20enforces=20itself=20at=20the=20safe=20p?= =?UTF-8?q?oint=20(#403)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cap_for_ram's projection is an estimate: on the GB10 (#403) long generations overshot it by ~40 GB (projected 74.4, real 115.6) and the kernel killed the engine three times. Run D of the issue proves a low cap CONTAINS the growth; this guard does that automatically, keyed on MEASURED RSS instead of the projection. At the repin safe point (no moe in flight), every ~16 emitted tokens: if RSS exceeds the resolved budget (RAM_GB/auto, or an explicit RSS_GUARD_GB ceiling), free the least-used LRU expert slabs in place and lower ecap so the cache cannot regrow. Slabs are >128 KB so glibc returns the pages to the kernel immediately -- RSS actually drops. Eviction never compacts the array: with PILOT_REAL the pilot worker holds pointers into ecache[] across its preads, so the slot stays in place with eid=-1/used=0 (first candidate for reuse); reserved slots (eid<0) are never touched and victim selection happens under g_pilot_mx. resident_bytes is left alone: LRU slots are never accounted there (only pin + dense). Co-Authored-By: Claude Fable 5 --- c/glm.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/c/glm.c b/c/glm.c index 11c09a6..a3e7ab7 100644 --- a/c/glm.c +++ b/c/glm.c @@ -4924,7 +4924,68 @@ static int repin_pick(Model *m, RepinCand *out, int maxc){ } return nb; } +/* ---- RSS GUARD (#403) ----------------------------------------------------- + * La proiezione di cap_for_ram e' una STIMA: sul GB10 (#403) le generazioni + * lunghe l'hanno sforata di ~40 GB (proiettato 74.4, reale 115.6 -> 3 kill del + * kernel). La run D dell'issue prova che un cap piu' basso CONTIENE la crescita: + * questa guardia lo fa da sola, sull'RSS MISURATO invece che sul proiettato. + * Al safe point (stessa sede di repin: nessun moe in volo), ogni ~16 token + * emessi: se l'RSS supera il budget, svuota gli slot LRU meno usati e abbassa + * ecap perche' non ricrescano. Gli slab sono >128KB (mmap'd da glibc): la free + * restituisce le pagine al kernel subito, quindi l'RSS scende davvero. + * Lo slot NON viene compattato via: resta al suo posto con eid=-1/used=0 (primo + * candidato al riuso), perche' con PILOT_REAL il worker tiene puntatori dentro + * ecache[] durante i suoi pread e uno spostamento li invaliderebbe; per lo + * stesso motivo gli slot eid<0 (riservati/in caricamento) non si toccano e la + * selezione avviene sotto g_pilot_mx. resident_bytes resta invariato: gli slot + * LRU non sono mai contati li' (solo pin e densa). + * EN: evict = free the slab in place (eid=-1, used=0, never compact: PILOT_REAL + * EN: holds pointers into ecache[] across its preads), skip eid<0 reservations, + * EN: select under g_pilot_mx. RSS_GUARD_GB= forces an explicit ceiling. */ +static double g_ram_budget_gb=0; /* budget risolto, scritto da cap_for_ram */ +static uint64_t g_rssg_last=0; +static void rss_guard(Model *m){ + double lim = getenv("RSS_GUARD_GB") ? atof(getenv("RSS_GUARD_GB")) : g_ram_budget_gb; + if(lim<=0) return; + if(m->n_emit - g_rssg_last < 16) return; + g_rssg_last = m->n_emit; + double rss=rss_gb(); + if(rss <= lim*1.02+0.3) return; /* tolleranza: 2% + 300MB */ + Cfg *c=&m->c; + int64_t need=(int64_t)((rss-lim)*1e9), freed=0; int dropped=0; + for(int pass=0; pass<8 && freedn_layers && freedecache || !m->ecache[l]) continue; + pthread_mutex_lock(&g_pilot_mx); + int nn=m->ecn[l], lru=-1; + for(int z=0;zecache[l][z]; + if(cand->eid<0 || !cand->slab) continue; + if(lru<0 || cand->usedecache[l][lru].used) lru=z; + } + if(lru<0){ pthread_mutex_unlock(&g_pilot_mx); continue; } + ESlot *s=&m->ecache[l][lru]; + s->eid=-1; /* nascosto: nessun hit/evict altrui */ + pthread_mutex_unlock(&g_pilot_mx); + int64_t sb=s->slab_cap + s->fslab_cap*4; +#ifdef COLI_METAL + if(s->slab && g_metal_enabled) coli_metal_unregister(s->slab); +#endif + compat_aligned_free(s->slab); free(s->fslab); + s->slab=NULL; s->fslab=NULL; s->slab_cap=s->fslab_cap=0; + QT *q[3]={&s->g,&s->u,&s->d}; + for(int k=0;k<3;k++){ q[k]->qf=NULL; q[k]->q8=NULL; q[k]->q4=NULL; q[k]->s=NULL; } + s->used=0; /* primo candidato al riuso */ + freed += sb; dropped++; + } + if(m->ecap>2) m->ecap--; /* il tetto scende: niente ricrescita */ + } + if(dropped) + fprintf(stderr,"[RAM-GUARD] RSS %.1f GB over the %.1f GB budget (#403): " + "dropped %d cached experts, cap -> %d\n", rss, lim, dropped, m->ecap); +} static void repin_pass_limit(Model *m,int limit){ + rss_guard(m); /* #403: il budget si fa rispettare sull'RSS MISURATO */ if(g_repin<=0) return; if(m->n_emit - g_last_repin < (uint64_t)g_repin) return; g_last_repin = m->n_emit; @@ -6024,6 +6085,7 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){ if(auto_b){ ram_gb = g_mem_avail_boot*0.88; /* misurata PRIMA del load: il residente gia' * allocato viene sottratto sotto, non due volte */ if(ram_gb<4){ fprintf(stderr,"[RAM] MemAvailable is unreadable or too low; assuming 8 GB\n"); ram_gb=8; } } + g_ram_budget_gb = ram_gb; /* #403: la RSS-guard usa il budget RISOLTO */ /* slack ONESTO, non forfettario (l'OOM del 2026-07-04 veniva da qui): * ws[64] slab del working-set (si materializzano TUTTI nel prefill batch-union), * KV cache a max_ctx, kvb_all della ricostruzione k/v in attention, From 3cd2674f68e2015f9b4168841c796a9fcbf32ed6 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sun, 19 Jul 2026 12:29:33 +0200 Subject: [PATCH 64/95] =?UTF-8?q?release:=20fix=20the=20Windows=20job=20sh?= =?UTF-8?q?ell=20=E2=80=94=20'msys2=20{0}',=20not=20'msys2'=20(+=20inherit?= =?UTF-8?q?=20PATH=20for=207z)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub Actions shells are format strings; the bare 'msys2' string made the v1.0.0 tag build fail before its first step ('Invalid shell option'). Same invocation ci.yml already uses. path-type: inherit so the Package step can reach the runner's 7z.exe. Co-Authored-By: Claude Fable 5 --- .github/workflows/release.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 245d24f..e471fb9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,9 @@ jobs: name: windows-x86_64 ext: ".exe" make_args: "" - shell: msys2 + # GitHub shells are format strings: 'msys2 {0}', never bare 'msys2' + # (the v1.0.0 tag build failed on exactly this). Same as ci.yml. + shell: "msys2 {0}" runs-on: ${{ matrix.os }} defaults: run: @@ -32,11 +34,14 @@ jobs: steps: - uses: actions/checkout@v4 - - if: matrix.shell == 'msys2' + - if: matrix.os == 'windows-latest' uses: msys2/setup-msys2@v2 with: msystem: UCRT64 update: false + # inherit: the Package step calls the runner's 7z.exe, which lives on + # the Windows PATH; the default minimal path would not see it. + path-type: inherit install: >- make mingw-w64-ucrt-x86_64-gcc From ca39e5333f1addb67df62b836ee2d9e661fd91e4 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sun, 19 Jul 2026 12:38:11 +0200 Subject: [PATCH 65/95] packaging: single version source + honest editable-install semantics (on top of #396) colibri/_version.py now reads c/version.py (#394's single source of truth -- coli --version, the release workflow, and pip metadata can no longer drift), with an importlib.metadata fallback for the installed-wheel case where c/ is not on disk. README documents that pip install -e . is the supported form: the engine lives in c/ and is not packaged into a standalone wheel. Verified in a clean venv: pip install -e . -> colibri.__version__ == 1.0.0 read from c/version.py, coli entrypoint on PATH and functional. Co-Authored-By: Claude Fable 5 --- README.md | 4 ++++ colibri/_version.py | 24 ++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 797f1ca..e5d1651 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,10 @@ COLI_MODEL=/nvme/glm52_i4 ./coli doctor # read-only readiness check The engine at runtime is pure C — python is only used by the one-time converter and the optional API gateway. +Prefer a `coli` command on your PATH? From a checkout, `pip install -e .` +registers it (the engine itself still lives in `c/` — this is an editable +install from the clone, not a standalone wheel). + ### 3. Go deeper | topic | doc | diff --git a/colibri/_version.py b/colibri/_version.py index 9951ca1..174442a 100644 --- a/colibri/_version.py +++ b/colibri/_version.py @@ -1,3 +1,23 @@ -"""Single source of truth for the colibrì version number.""" +"""Version accessor for the pip package. -__version__ = "1.0.0" +The single source of truth is c/version.py (#394): coli --version and the +GitHub Release workflow read it, so the pip metadata must read the SAME file +instead of carrying a second literal that would drift on the first bump. + +From a checkout (the supported install: `pip install -e .`) the file is read +directly. From an installed wheel c/ is not on disk, so fall back to the +package metadata that setuptools baked at build time from that same file. +""" +from pathlib import Path + +try: + _ns = {} + exec((Path(__file__).resolve().parent.parent / "c" / "version.py").read_text(), _ns) + __version__ = _ns["__version__"] +except OSError: + from importlib.metadata import PackageNotFoundError, version + + try: + __version__ = version("colibri-engine") + except PackageNotFoundError: + __version__ = "0.0.0+unknown" From 72e36772f51605fa947efe380614c85a395c556c Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sun, 19 Jul 2026 13:05:56 +0200 Subject: [PATCH 66/95] security: reject malformed model tensors at the untrusted-mirror boundary Colibri loads model directories and safetensors from mirrors it does not control, so the file's declared shapes and byte spans are attacker-influenced input. Three memory-safety holes on that boundary, independently confirmed (incl. a from-scratch adversarial audit that re-derived the same two) and present in the shipped v1.0.0: - st.h st_read_f32: numel came from the shape, nbytes from the offsets, with no cross-check. A crafted tensor whose shape inflates numel past nbytes made the BF16/F16 loop read past the malloc'd raw buffer and the F32 memcpy write past the caller's config-sized destination (heap OOB read + write). Now enforce numel*esz == nbytes before any copy. - st.h header parse: the shape product could overflow int64 to a small/negative numel that would then pass the cross-check. Guard each multiply. - glm.c qt_resolve_fmt (new, replaces the three duplicated "?1:?2:3" fmt sites in qt_from_disk and both expert_load arms): the old inference SILENTLY fell to int2 for any unrecognized weight byte count, so a too-short weight became a valid int2 whose matmul read O*I nibbles past the buffer; and an oversized scale array overflowed the per-row t->s. Now the weight bytes must match a known int8/int4/int2 layout and the scale array must match the expected per-row (O) or grouped (O*ng) cardinality, else refuse. - glm.c config/generation_config slurp: unbounded ftell -> malloc(n+1) gave a hostile file a load-time OOM, and on malloc failure b[got]=0 was a NULL deref. Cap at 256 MB and NULL-check. Verified: TF token-exactness unchanged on every quant format (full-precision 32/32, int4 11/32, int2 1/32, mix 5/32 -- byte-identical to the pre-change binary); fmt=4 grouped path preserved (the scale check is by construction the same condition detect_group_size already imposed); a hand-crafted hostile safetensors is refused cleanly; ASan+UBSan clean on legit and hostile loads (only the pre-existing intentional startup leaks remain). These are the C trust-boundary items of #368, landed as a minimal standalone fix; the server-side and build items of that PR follow via its rebase. Co-Authored-By: Claude Fable 5 --- c/glm.c | 51 ++++++++++++++++++++++++++++++++++++--------------- c/st.h | 23 ++++++++++++++++++++++- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/c/glm.c b/c/glm.c index a3e7ab7..26e7fc4 100644 --- a/c/glm.c +++ b/c/glm.c @@ -1336,7 +1336,12 @@ static jval* cfg_root(const char *snap, char **arena){ char p[2048]; snprintf(p,sizeof(p),"%s/config.json",snap); FILE *f=fopen(p,"rb"); if(!f){perror(p);exit(1);} fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *b=malloc(n+1); size_t got=fread(b,1,n,f); b[got]=0; fclose(f); + /* SEC: config.json arriva dalla dir modello non fidata. Limita la dimensione + * (un file ostile enorme = OOM al load) e controlla la malloc: senza il NULL + * check, b[got]=0 su malloc fallita era un NULL-deref. */ + if(n<0 || n>(256L<<20)){ fprintf(stderr,"%s: size %ld out of range (0..256 MB)\n",p,n); exit(1); } + char *b=malloc((size_t)n+1); if(!b){ fprintf(stderr,"OOM reading %s (%ld bytes)\n",p,n); exit(1); } + size_t got=fread(b,1,(size_t)n,f); b[got]=0; fclose(f); if((long)got!=n) fprintf(stderr,"warning: short read on %s (%ld of %ld)\n",p,(long)got,n); return json_parse(b,arena); } @@ -1375,8 +1380,9 @@ static void load_cfg(Cfg *c, const char *snap){ FILE *gf=fopen(gp,"rb"); /* assente = nessun problema: e' opzionale */ if(gf){ fseek(gf,0,SEEK_END); long gn=ftell(gf); fseek(gf,0,SEEK_SET); - if(gn>0){ - char *gb=malloc(gn+1); size_t gg=fread(gb,1,gn,gf); gb[gg]=0; + char *gb = (gn>0 && gn<=(256L<<20)) ? malloc((size_t)gn+1) : NULL; /* SEC: cap + NULL check */ + if(gb){ + size_t gg=fread(gb,1,(size_t)gn,gf); gb[gg]=0; char *ga=NULL; jval *gr=json_parse(gb,&ga); jval *ge=gr?json_get(gr,"eos_token_id"):NULL; if(ge){ @@ -1445,6 +1451,27 @@ static int detect_group_size(int O, int I, int64_t ns){ return 0; } +/* SEC: risolve e VALIDA il formato quantizzato di un tensore [O,I] letto da un + * container non fidato (mirror). L'inferenza precedente (`?1:?2:3`) cadeva su + * int2 per QUALSIASI conteggio byte non riconosciuto: un peso troppo corto + * diventava un int2 valido e il matmul leggeva oltre il buffer (O*I nibble a + * 4/byte). Qui i byte del peso devono corrispondere a un layout noto e i byte + * della scala alla cardinalita' attesa (O per-row, O*ng per-gruppo) — altrimenti + * si termina invece di sforare. Ritorna fmt (1/2/3/4) e scrive *gs. */ +static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns, int *gs){ + int64_t exp_i8=(int64_t)O*I, exp_i4=(int64_t)O*((I+1)/2), exp_i2=(int64_t)O*((I+3)/4); + int fmt = (nb==exp_i8)?1 : (nb==exp_i4)?2 : (nb==exp_i2)?3 : 0; + if(!fmt){ + fprintf(stderr,"%s: quantized weight is %lld bytes — no int8/int4/int2 layout for [%d,%d], refusing (untrusted container)\n", + name,(long long)nb,O,I); exit(1); } + *gs=0; + if(fmt==2){ int g=detect_group_size(O,I,ns); if(g>0){ fmt=4; *gs=g; } } + int64_t exp_scale = (fmt==4)? (int64_t)O*((I+*gs-1)/(*gs)) : (int64_t)O; /* in FLOAT */ + if(ns != exp_scale*4){ + fprintf(stderr,"%s: scale array is %lld bytes — expected %lld for [%d,%d] fmt=%d, refusing (untrusted container)\n", + name,(long long)ns,(long long)(exp_scale*4),O,I,fmt); exit(1); } + return fmt; +} /* costruisce un QT [O,I] dal disco in `t` (buffer riusabili tra chiamate). * - se esiste `name.qs`: pesi GIA' quantizzati nel container (U8 qdata + F32 scala) -> letti diretti * - altrimenti: tensore pieno (f32/bf16) -> quantizzato a runtime a `bits` (oracolo tiny / pesi pieni) @@ -1454,12 +1481,11 @@ static void qt_from_disk(Model *m, const char *name, int O, int I, int bits, int if(st_has(&m->S,sn)){ int64_t nb=st_nbytes(&m->S,name); int64_t ns=st_nbytes(&m->S,sn); /* scale bytes (F32) */ - /* Detect int4-grouped (fmt=4): packed int4 weight bytes BUT scale array is - * larger than O*4 — the group size is derived from the scale-array size. */ - int fmt = (nb==(int64_t)O*I)?1 : (nb==(int64_t)O*((I+1)/2))?2 : 3; + /* fmt=4 int4-grouped: byte int4 ma scala > O*4 — gs deriva dalla scala. + * qt_resolve_fmt valida entrambi i conteggi contro [O,I] e termina se + * non fidati (SEC). */ int gs=0; - if(fmt==2) gs=detect_group_size(O,I,ns); - if(gs>0) fmt=4; + int fmt = qt_resolve_fmt(name,O,I,nb,ns,&gs); if(fmt==1){ if(t->fmt!=1||!t->q8){ t->fmt=1; t->O=O; t->I=I; t->gs=0; t->q8=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q8,drop); } else if(fmt==4){ int ng=(I+gs-1)/gs; if(t->fmt!=4||!t->q4){ t->fmt=4; t->O=O; t->I=I; t->gs=gs; t->q4=qalloc(nb); t->s=falloc((int64_t)O*ng); } @@ -1802,11 +1828,8 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ QT *qt[3]={&s->g,&s->u,&s->d}; int OO[3]={I,I,D}, II[3]={D,D,I}; for(int k=0;k<3;k++){ int64_t nb=tw[k]->nbytes; - int fmt=(nb==(int64_t)OO[k]*II[k])?1:(nb==(int64_t)OO[k]*((II[k]+1)/2))?2:3; - /* detect grouped int4 (fmt=4): int4 weight bytes + larger scale array */ int gs=0; - if(fmt==2) gs=detect_group_size(OO[k],II[k],tq[k]->nbytes); - if(gs>0) fmt=4; + int fmt=qt_resolve_fmt(tw[k]->name,OO[k],II[k],nb,tq[k]->nbytes,&gs); qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL; qt[k]->q8=(int8_t*)((char*)bw[k]+tw[k]->off); qt[k]->q4=(uint8_t*)((char*)bw[k]+tw[k]->off); qt[k]->s=(float*)((char*)bq[k]+tq[k]->off); @@ -1931,10 +1954,8 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ QT *qt[3]={&s->g,&s->u,&s->d}; int OO[3]={I,I,D}, II[3]={D,D,I}; for(int k=0;k<3;k++){ int64_t nb=tw[k]->nbytes; - int fmt = (nb==(int64_t)OO[k]*II[k])?1 : (nb==(int64_t)OO[k]*((II[k]+1)/2))?2 : 3; int gs=0; - if(fmt==2) gs=detect_group_size(OO[k],II[k],tq[k]->nbytes); - if(gs>0) fmt=4; + int fmt=qt_resolve_fmt(tw[k]->name,OO[k],II[k],nb,tq[k]->nbytes,&gs); qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL; qt[k]->q8=(int8_t*)(s->slab+pos[k]); qt[k]->q4=s->slab+pos[k]; qt[k]->s=fp[k]; } diff --git a/c/st.h b/c/st.h index efa6e5a..a2c2c3f 100644 --- a/c/st.h +++ b/c/st.h @@ -200,7 +200,19 @@ static void st_init(shards *S, const char *snap_dir) { if (a0 < 0 || b0 < a0 || data_start + b0 > fsz) { fprintf(stderr, "%s: tensor '%s' data_offsets [%lld,%lld] out of file bounds (%lld)\n", files[fi], name, (long long)a0, (long long)b0, (long long)fsz); exit(1); } - int64_t numel = 1; for (int k = 0; k < shp->len; k++) numel *= (int64_t)shp->kids[k]->num; + /* SEC: lo shape viene da un file non fidato (mirror). Senza il guard + * di overflow, uno shape tipo [65535,65535,65535,...] fa avvolgere + * numel a un valore piccolo/negativo che poi passerebbe il cross-check + * numel*esz==nbytes in st_read_f32, riaprendo l'OOB. */ + int64_t numel = 1; int bad_shape = 0; + for (int k = 0; k < shp->len; k++) { + int64_t d = (int64_t)shp->kids[k]->num; + if (d < 0 || (d != 0 && numel > INT64_MAX / d)) { bad_shape = 1; break; } + numel *= d; + } + if (bad_shape) { + fprintf(stderr, "%s: tensor '%s' shape overflows int64 — refusing (hostile or corrupt file)\n", + files[fi], name); exit(1); } if (S->n == S->cap) { S->cap *= 2; S->t = realloc(S->t, S->cap*sizeof(st_tensor)); } st_tensor *t = &S->t[S->n++]; t->name = strdup(name); t->fd = fd; t->off = data_start + a0; @@ -249,6 +261,15 @@ static void st_prefetch(shards *S, const char *name) { static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) { st_tensor *t = st_find(S, name); if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); } + /* SEC: numel viene dallo shape, nbytes dagli offset — due campi indipendenti + * del file. Se non concordano, la memcpy F32 (nbytes) o i loop BF16/F16 + * (numel elementi da un raw di soli nbytes) sforano il buffer del chiamante, + * che e' dimensionato sul config, non sul file. Il chiamante che alloca su + * st_numel resta coerente; questo blocca l'ingresso ostile a monte. */ + int esz = (t->dtype == 2) ? 4 : 2; + if (t->numel < 0 || t->numel > t->nbytes / esz || t->numel * (int64_t)esz != t->nbytes) { + fprintf(stderr, "%s: tensor '%s' shape/bytes mismatch (numel %lld, %lld bytes, dtype %d) — refusing (hostile or corrupt file)\n", + name, name, (long long)t->numel, (long long)t->nbytes, t->dtype); exit(1); } void *raw = malloc(t->nbytes); if (!raw) { fprintf(stderr, "malloc %lld bytes for tensor %s failed\n", (long long)t->nbytes, name); exit(1); } st_pread_full(t->fd, raw, t->nbytes, t->off, "pread data"); From 845af6378d212f07906ba635b2ae4f7e4f2b0c64 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sun, 19 Jul 2026 14:08:19 +0200 Subject: [PATCH 67/95] docs: beginner-friendly Quick Start guide for Linux/Windows/macOS (#414) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds docs/quickstart.md — a step-by-step, no-experience-assumed walkthrough from installing the build tools to the first coli chat, with per-OS copy-paste commands (Ubuntu apt, Windows MSYS2 or prebuilt binary, macOS brew), the ready-made HF int4 container plus the self-convert path, and an honest 'what to expect' on disk-bound speed. Commands verified against setup.sh and the coli subcommands; every cross-linked doc exists. Linked from the README's Get started section. Closes #414 Co-Authored-By: Claude Fable 5 --- README.md | 4 ++ docs/quickstart.md | 164 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 docs/quickstart.md diff --git a/README.md b/README.md index e5d1651..f9cf3f0 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,10 @@ scale-granularity/rotation ablations live in ## Get started +> **New here?** The [Quick Start guide](docs/quickstart.md) walks through +> install → build → model → first chat step by step for Linux, Windows, and +> macOS, with copy-paste commands and no assumed background. + ### 1. Get the model A pre-converted **GLM-5.2 int4** container is on Hugging Face — **use the diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 0000000..7d6a9bf --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,164 @@ +# Quick Start — from zero to a running model + +A step-by-step guide for first-time users on **Linux**, **Windows**, and **macOS**. +No prior experience with C, CUDA, or model conversion is assumed. If you get +stuck, `./coli doctor` (below) tells you exactly what's missing. + +> **What you're setting up:** colibrì runs a very large Mixture-of-Experts model +> (e.g. GLM-5.2, 744B parameters) on a normal machine by streaming the model's +> experts from disk instead of needing them all in RAM. The engine is a single +> C program; Python is only used once, to prepare the model files. + +--- + +## 0. What you need first (prerequisites) + +| | Minimum | Recommended | +|---|---|---| +| **RAM** | ~16 GB | 24 GB+ | +| **Free disk** | ~380 GB for the int4 model | a fast NVMe SSD (streaming speed = your token speed) | +| **OS** | Linux, Windows 10/11, or macOS | any | +| **Tools** | a C compiler + `make` + `git` + `python3` | — | + +You do **not** need a GPU. A GPU only helps if you have one; the engine runs +CPU-only by default. + +--- + +## 1. Install the build tools + +### Linux (Ubuntu / Debian) + +```bash +sudo apt update +sudo apt install -y build-essential git python3 +``` + +`build-essential` gives you `gcc`, `make`, and OpenMP (libgomp) — everything the +engine needs. + +### Windows + +You have two options. + +**Option A — download a prebuilt binary (no compiler needed).** +Grab the latest `colibri--windows-x86_64.zip` from the +[Releases page](https://github.com/JustVugg/colibri/releases), unzip it, and +skip to [step 3](#3-get-the-model). Python 3 (from +[python.org](https://www.python.org/downloads/)) is still needed if you want to +convert a model yourself. + +**Option B — build from source with MSYS2.** +Install [MSYS2](https://www.msys2.org/), open the **UCRT64** shell, and run: + +```bash +pacman -S --needed mingw-w64-ucrt-x86_64-gcc make git python +``` + +### macOS + +```bash +xcode-select --install # C compiler (clang) +brew install libomp git python # OpenMP for multithreading +``` + +--- + +## 2. Get the code and build the engine + +```bash +git clone https://github.com/JustVugg/colibri.git +cd colibri/c +./setup.sh +``` + +`setup.sh` checks your compiler and OpenMP, builds the engine, and runs a tiny +self-test. When it prints: + +``` +engine self-test: 32/32 (expected 32/32) +``` + +the engine is working correctly. (On Windows Option A you already have the +binary — you can skip this step.) + +--- + +## 3. Get the model + +You have two paths. + +### Easiest — download a ready-made int4 container + +A pre-converted **GLM-5.2 int4** model is on Hugging Face. **Use the version +with the int8 MTP heads** (the plain int4 heads disable speculative decoding — +see [#8](https://github.com/JustVugg/colibri/issues/8)): + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +Download it into a folder on a fast disk, e.g. `/nvme/glm52_i4` (Linux/macOS) or +`D:\glm52_i4` (Windows). It is about **372 GB**, so make sure you have the space. + +### Or convert it yourself from the FP8 source + +One resumable command downloads and converts the model shard by shard, so it +never needs the full ~756 GB on disk at once: + +```bash +./coli convert --model /nvme/glm52_i4 +``` + +This step uses Python and runs only once. Safe to interrupt and re-run — it +resumes where it left off. + +--- + +## 4. Run it + +Point `COLI_MODEL` at the folder from step 3 and start chatting: + +```bash +# Linux / macOS +COLI_MODEL=/nvme/glm52_i4 ./coli chat + +# Windows (UCRT64 shell) +COLI_MODEL=/d/glm52_i4 ./coli chat +``` + +Useful first commands: + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # read-only check: is everything ready? +COLI_MODEL=/nvme/glm52_i4 ./coli plan # shows where the model will live (RAM/disk/GPU) +COLI_MODEL=/nvme/glm52_i4 ./coli chat --topp 0.85 # faster: reads less from disk, same quality +``` + +> **Tip:** `--topp 0.85` is worth adding on a disk-bound machine — it reads +> fewer expert bytes per token with no quality loss, which directly means more +> tokens per second. + +--- + +## 5. What to expect + +- **First launch loads the resident weights** (~10 GB) — this takes a moment. +- **Speed depends on your disk.** The experts stream from storage, so a fast + NVMe SSD is the single biggest factor in tokens/second. On a slow or shared + disk, generation can be well under 1 token/second — that's expected, and it's + the honest cost of running a 744B model on a small machine. +- **It's still the full model.** Placement only changes speed, never the model's + answers or precision. + +If something doesn't work, run `./coli doctor` — it reports exactly what's +missing (compiler, model files, permissions) and how to fix it. + +--- + +## Where to go next + +| Topic | Doc | +|---|---| +| Windows native build (and CUDA DLL) | [docs/windows.md](windows.md) | +| Tuning: cache, prefetch, speculation | [docs/tuning.md](tuning.md) | +| OpenAI-compatible API + web dashboard | [docs/api.md](api.md) | +| Every environment variable | [docs/ENVIRONMENT.md](ENVIRONMENT.md) | From cfcc742591a03a79754f68a7f1a6f0a7ad1738a5 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Sun, 19 Jul 2026 14:58:32 +0200 Subject: [PATCH 68/95] metal: advance the LFRU recency clock on the GPU-prerouted decode path (#417) On Metal, when routing is precomputed on the GPU (g_pre_idx), the moe fast path bumps eusage/ehit/eheat for the selected experts but skips the one thing the full CPU router does at the equivalent site: elast[layer][e] = ++eaccess_clock. So the session-local recency clock advances during prefill (full router) but freezes the moment GPU-prerouted decode starts, and REPIN's tier_pick_lfru() tie-breaker then runs on stale recency for the rest of the run. Mirror the exact update the non-Metal path already does. Inside #ifdef COLI_METAL, so CPU/CUDA are untouched; elast only feeds the LFRU eviction heuristic, so this cannot affect output, only which experts REPIN keeps warm. Found and reported by @monotophic with a source-level trace repro (ELAST_TRACE). Fix is inspection-verified against line ~3055; needs a Metal build to exercise end-to-end. Closes #417 Co-Authored-By: Claude Fable 5 --- c/glm.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/c/glm.c b/c/glm.c index 26e7fc4..0f97ad2 100644 --- a/c/glm.c +++ b/c/glm.c @@ -2931,6 +2931,11 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int m->eusage[layer][idxs[(int64_t)s*K+kk]]++; ehit_mark(m,layer,idxs[(int64_t)s*K+kk]); if(m->eheat[layer][idxs[(int64_t)s*K+kk]]eheat[layer][idxs[(int64_t)s*K+kk]]++; + /* #417: la scorciatoia GPU-prerouted deve far avanzare l'orologio di recency + * come il percorso router completo (riga ~3055), altrimenti elast/eaccess_clock + * si congelano a fine prefill e il tie-breaker LFRU di REPIN gira su punteggi + * stantii durante il decode su Metal. */ + m->elast[layer][idxs[(int64_t)s*K+kk]]=++m->eaccess_clock; } for(int d=0;d Date: Sun, 19 Jul 2026 03:56:56 +0800 Subject: [PATCH 69/95] refactor: split glm.c into colibri.c + 4 header modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename glm.c → colibri.c and extract four self-contained modules into header-only files (same pattern as st.h/tier.h/grammar.h): quant.h (672 lines) — SIMD matmul kernels, quantization sample.h (143 lines) — RNG, top-p sampling, stop-set kv_persist.h (121 lines) — .coli_kv disk persistence telemetry.h (189 lines) — dashboard protocol, stats, usage Main engine file shrinks from 6588 to 5396 lines (−18%). Build system: primary target is now colibri$(EXE); `make glm` remains as a phony alias for backward compat. CI, setup.sh, coli CLI, and all 10 test files that include the engine are updated. make check passes (C + Python, 73 tests, zero warnings). --- .github/workflows/ci.yml | 8 +- .gitignore | 2 + c/Makefile | 53 +- c/coli | 16 +- c/{glm.c => colibri.c} | 1239 +---------------------------- c/kv_persist.h | 121 +++ c/quant.h | 672 ++++++++++++++++ c/sample.h | 143 ++++ c/setup.sh | 4 +- c/telemetry.h | 189 +++++ c/tests/bench_dsa_select.c | 2 +- c/tests/bench_topp.c | 2 +- c/tests/test_dsa_select.c | 2 +- c/tests/test_i4_grouped.c | 2 +- c/tests/test_idot.c | 2 +- c/tests/test_kv_alloc.c | 2 +- c/tests/test_makefile_platform.py | 4 +- c/tests/test_sample_nan.c | 2 +- c/tests/test_stops.c | 2 +- c/tests/test_topp.c | 2 +- c/tests/test_uring.c | 2 +- 21 files changed, 1206 insertions(+), 1265 deletions(-) rename c/{glm.c => colibri.c} (82%) create mode 100644 c/kv_persist.h create mode 100644 c/quant.h create mode 100644 c/sample.h create mode 100644 c/telemetry.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72b7149..eb7e872 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Build glm - run: cd c && make glm + - name: Build colibri + run: cd c && make colibri - name: C test suite run: cd c && make test-c @@ -105,11 +105,11 @@ jobs: make cuda-dll CUDA_ARCH=sm_80 test -f coli_cuda.dll || { echo "cuda-dll reported success but produced no DLL" >&2; exit 1; } echo "coli_cuda.dll built (MSVC host)" - - name: make glm CUDA_DLL=1 (host links backend_loader, not cudart) + - name: make colibri CUDA_DLL=1 (host links backend_loader, not cudart) shell: msys2 {0} run: | cd c - make glm CUDA_DLL=1 + make colibri CUDA_DLL=1 test -f glm.exe || { echo "glm CUDA_DLL=1 reported success but produced no exe" >&2; exit 1; } echo "glm.exe built against the DLL loader" diff --git a/.gitignore b/.gitignore index a65fd3d..b3e1193 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ desktop/src-tauri/target/ desktop/src-tauri/gen/ # binari compilati (si rigenerano con make / coli build) +c/colibri +c/colibri.exe c/glm c/glm.exe c/olmoe diff --git a/c/Makefile b/c/Makefile index 1f8583b..e33ad37 100644 --- a/c/Makefile +++ b/c/Makefile @@ -56,7 +56,7 @@ OMPL = endif CFLAGS = -O3 $(OMPC) -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function # Opt-in: ARCH=native appends -mcpu=native (arm64 clang uses -mcpu, not -march), -# which unlocks the i8mm SMMLA int8/int4 dot kernels in glm.c. ARCH unset -> +# which unlocks the i8mm SMMLA int8/int4 dot kernels in colibri.c. ARCH unset -> # no -mcpu, default build byte-identical. Apple clang knows apple-m4 / native. ifneq ($(ARCH),) CFLAGS += -mcpu=$(ARCH) @@ -70,7 +70,7 @@ else ifneq ($(IS_WIN),) # ARCH default = x86-64-v3 (portable binary with AVX2). For max speed on THIS # machine use ARCH=native: on AVX-VNNI CPUs (Intel Alder Lake+, Meteor Lake+) # it also unlocks the 128-bit VPDPBUSD int8/int4 dot kernel (dot_i8i8/dot_i4i8), -# which the x86-64-v3 baseline does not define. The #ifdef guards in glm.c mean +# which the x86-64-v3 baseline does not define. The #ifdef guards in colibri.c mean # a v3 build simply compiles out the VNNI path - safe on any x86-64. CC = gcc ARCH ?= x86-64-v3 @@ -207,17 +207,18 @@ LDFLAGS += -framework Metal -framework Foundation -lc++ METAL_OBJ = backend_metal.o endif -all: glm$(EXE) +all: colibri$(EXE) -# phony 'glm' → 'glm.exe' on Windows (so 'make glm' and 'coli build' work on every platform) -glm: glm$(EXE) +# phony targets — 'glm' kept for backward compatibility +colibri: colibri$(EXE) +glm: colibri$(EXE) # Config stamp: make only tracks file timestamps, not flag changes. Without this, -# `make glm.exe CUDA_DLL=1` after a prior CPU-only build reports "up to date" and -# silently keeps the CPU-only binary (no CUDA loader) — a build that looks like it -# worked but isn't. We record the build-affecting flags in .build-config and rewrite -# it ONLY when they change (evaluated here at parse time, so the file's timestamp -# moves exactly when the config moves). glm.exe and the CUDA/loader objects depend +# `make colibri.exe CUDA_DLL=1` after a prior CPU-only build reports "up to date" +# and silently keeps the CPU-only binary (no CUDA loader) — a build that looks like +# it worked but isn't. We record the build-affecting flags in .build-config and +# rewrite it ONLY when they change (evaluated here at parse time, so the file's +# timestamp moves exactly when the config moves). The binary and CUDA/loader objects depend # on it, so they relink on a config change and stay put otherwise. (#306) BUILD_CONFIG := $(CC)|$(CFLAGS)|$(LDFLAGS)|CUDA=$(CUDA)|CUDA_DLL=$(CUDA_DLL)|ARCH=$(ARCH)|CUDA_ARCH=$(CUDA_ARCH)|METAL=$(METAL) BUILD_CONFIG_OLD := $(shell cat .build-config 2>/dev/null) @@ -226,8 +227,8 @@ $(shell printf '%s' '$(BUILD_CONFIG)' > .build-config) endif .build-config: ; -glm$(EXE): glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h $(CUDA_OBJ) $(METAL_OBJ) .build-config - $(CC) $(CFLAGS) glm.c $(CUDA_OBJ) $(METAL_OBJ) -o glm$(EXE) $(LDFLAGS) +colibri$(EXE): colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h quant.h sample.h kv_persist.h telemetry.h $(CUDA_OBJ) $(METAL_OBJ) .build-config + $(CC) $(CFLAGS) colibri.c $(CUDA_OBJ) $(METAL_OBJ) -o colibri$(EXE) $(LDFLAGS) # Windows runtime loader object: resolves coli_cuda_* from coli_cuda.dll. backend_loader.o: backend_loader.c backend_cuda.h compat.h .build-config @@ -285,7 +286,7 @@ PORTABLE_ARCH = native endif portable: - $(MAKE) glm$(EXE) ARCH=$(PORTABLE_ARCH) + $(MAKE) colibri$(EXE) ARCH=$(PORTABLE_ARCH) iobench$(EXE): iobench.c compat.h $(CC) $(CFLAGS) iobench.c -o iobench$(EXE) $(LDFLAGS) @@ -311,27 +312,27 @@ tests/test_schema_gbnf$(EXE): tests/test_schema_gbnf.c schema_gbnf.h grammar.h j tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_idot$(EXE): tests/test_idot.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_idot$(EXE): tests/test_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_stops$(EXE): tests/test_stops.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_stops$(EXE): tests/test_stops.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_topp$(EXE): tests/test_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_topp$(EXE): tests/test_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) # bench_topp is a microbenchmark (old qsort vs new heap partial-select, #335), NOT a test # gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_topp -tests/bench_topp$(EXE): tests/bench_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/bench_topp$(EXE): tests/bench_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_sample_nan$(EXE): tests/test_sample_nan.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_sample_nan$(EXE): tests/test_sample_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c colibri.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_logit_nan$(EXE): tests/test_logit_nan.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h @@ -343,15 +344,15 @@ tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c tests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_dsa_select$(EXE): tests/test_dsa_select.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_dsa_select$(EXE): tests/test_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) # bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356), # NOT a test gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_dsa_select -tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_uring$(EXE): tests/test_uring.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_uring$(EXE): tests/test_uring.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) test-c: $(TEST_BINS) @@ -368,12 +369,12 @@ check: $(MAKE) portable $(MAKE) test -install: glm$(EXE) olmoe$(EXE) +install: colibri$(EXE) olmoe$(EXE) $(INSTALL) -d $(DESTDIR)$(BINDIR) $(INSTALL) -d $(DESTDIR)$(LIBEXECDIR) $(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)/tools $(INSTALL) -m 755 coli $(DESTDIR)$(BINDIR)/coli - $(INSTALL) -m 755 glm$(EXE) $(DESTDIR)$(LIBEXECDIR)/glm$(EXE) + $(INSTALL) -m 755 colibri$(EXE) $(DESTDIR)$(LIBEXECDIR)/colibri$(EXE) $(INSTALL) -m 755 olmoe$(EXE) $(DESTDIR)$(LIBEXECDIR)/olmoe$(EXE) $(INSTALL) -m 644 resource_plan.py doctor.py openai_server.py $(DESTDIR)$(LIBEXECDIR)/ $(INSTALL) -m 644 tools/*.py $(DESTDIR)$(LIBEXECDIR)/tools/ @@ -387,4 +388,4 @@ clean: bench: iobench$(EXE) @if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) — run: ./iobench$(EXE) "; fi -.PHONY: all glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench +.PHONY: all colibri glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench diff --git a/c/coli b/c/coli index 803c22b..d6728fe 100755 --- a/c/coli +++ b/c/coli @@ -54,16 +54,20 @@ from version import __version__ as _version # guess is right (e.g. a custom packaging layout). _EXE = ".exe" if sys.platform == "win32" else "" _LIBEXEC = os.path.join(os.path.dirname(HERE), "libexec", "colibri") +_here_colibri = os.path.join(HERE, "colibri" + _EXE) _here_glm = os.path.join(HERE, "glm" + _EXE) if os.environ.get("COLI_ENGINE"): GLM = os.environ["COLI_ENGINE"] TOOLS = os.path.join(os.path.dirname(GLM), "tools") +elif os.path.exists(_here_colibri): + GLM = _here_colibri + TOOLS = os.path.join(HERE, "tools") elif os.path.exists(_here_glm): GLM = _here_glm TOOLS = os.path.join(HERE, "tools") else: - GLM = os.path.join(_LIBEXEC, "glm" + _EXE) + GLM = os.path.join(_LIBEXEC, "colibri" + _EXE) TOOLS = os.path.join(_LIBEXEC, "tools") sys.path.insert(0, _LIBEXEC) # so `import resource_plan`, `doctor`, `openai_server` still resolve @@ -241,13 +245,13 @@ def env_for(a): e["COLI_CUDA"]="0"; e.pop("CUDA_EXPERT_GB",None); e.pop("CUDA_DENSE",None) else: if not cuda_binary(): - sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} make glm CUDA=1 (this binary is CPU-only)") + sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} make colibri CUDA=1 (this binary is CPU-only)") e["COLI_CUDA"]="1" if a.gpu!="auto": e["COLI_GPUS"]=a.gpu e.setdefault("CUDA_DENSE","1") if a.vram and a.gpu!="none": if not cuda_binary(): - sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} make glm CUDA=1 (this binary is CPU-only)") + sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} make colibri CUDA=1 (this binary is CPU-only)") e["COLI_CUDA"]="1"; e["CUDA_EXPERT_GB"]=str(a.vram) return e @@ -421,8 +425,8 @@ def cmd_build(a): banner("build") if not os.path.exists(os.path.join(HERE, "Makefile")): sys.exit(f"{C.yel}coli build{C.r} only works from a source checkout (this is an installed copy).\n" - f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c glm.") - sys.exit(subprocess.call(["make","-C",HERE,"glm"])) + f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c colibri.") + sys.exit(subprocess.call(["make","-C",HERE,"colibri"])) def cmd_info(a): banner("info") @@ -802,7 +806,7 @@ def cmd_stop(a): if "coli" in cmd and " serve" in cmd and pid!=os.getpid(): if not any(p==pid for p,_ in targets): targets.append((pid,"coli serve (cmdline)")) comm=open(f"/proc/{pd}/comm").read().strip() - if comm in ("glm","exe","olmoe"): + if comm in ("colibri","glm","exe","olmoe"): env=open(f"/proc/{pd}/environ","rb").read().replace(b"\0",b"\n").decode("utf-8","replace") if "SERVE=1" in env: targets.append((pid,f"engine `{comm}` (SERVE=1)")) except (OSError,PermissionError): continue diff --git a/c/glm.c b/c/colibri.c similarity index 82% rename from c/glm.c rename to c/colibri.c index 0f97ad2..554a7fa 100644 --- a/c/glm.c +++ b/c/colibri.c @@ -73,21 +73,6 @@ static int g_metal_gemm_min=16; /* COLI_METAL_GEMM_MIN: min rows to send a mat static const int *g_pre_idx; static const float *g_pre_w; static const int *g_pre_keff; static const float *g_pre_sh; /* output dello shared expert gia' calcolato su GPU */ #endif -#ifdef __AVX2__ -#include -static inline float hsum256(__m256 v){ /* somma orizzontale di 8 float */ - __m128 lo=_mm256_castps256_ps128(v), hi=_mm256_extractf128_ps(v,1); - lo=_mm_add_ps(lo,hi); __m128 sh=_mm_movehl_ps(lo,lo); lo=_mm_add_ps(lo,sh); - sh=_mm_shuffle_ps(lo,lo,1); lo=_mm_add_ss(lo,sh); return _mm_cvtss_f32(lo); -} -#elif defined(__ARM_NEON) -#include /* Apple Silicon / aarch64: kernel NEON */ -#elif defined(__VSX__) -#include /* POWER8+ (ppc64le): kernel VSX */ -#undef vector /* igiene: si usano __vector/__bool espliciti */ -#undef pixel -#undef bool -#endif #ifdef __APPLE__ #include /* host_statistics64: MemAvailable di macOS */ #endif @@ -218,13 +203,21 @@ typedef struct { uint64_t bytes_mtp, bytes_main; /* byte letti da disco per tipo layer */ } Model; -static void usage_save(Model *m); /* cache che impara: definita accanto a stats_dump */ -static void tiers_emit(Model *m); -static void ehit_mark(Model *m, int layer, int eid); -static void emap_emit(Model *m); -static void hits_emit(Model *m); -static void hwinfo_emit(Model *m); -static int64_t expert_bytes_probe(Model *m, int ebits); /* PROF: tier sizes in the report */ +#include "quant.h" +static int g_no_fused_pair=0; +static int g_spec_pin=1; +static int g_spec_live=0; +static inline int spec_pinned(void){ return g_spec_pin && g_spec_live; } + +static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot); +static void matmul_qt(float *y, const float *x, QT *w, int S){ matmul_qt_ex(y,x,w,S,1); } + +static void expert_gate_up(float *g,float *u,const float *x,QT *wg,QT *wu,int S){ + if(!g_no_fused_pair&&!spec_pinned()&&S==1&&wg->fmt==2&&wu->fmt==2&&wg->I==wu->I&&wg->O==wu->O) + matmul_i4_pair(g,u,x,wg->q4,wg->s,wu->q4,wu->s,wg->I,wg->O); + else { matmul_qt(g,x,wg,S); matmul_qt(u,x,wu,S); } +} + static int g_repin; static uint64_t g_last_repin; #ifdef COLI_CUDA @@ -332,713 +325,15 @@ static float *falloc(int64_t n){ if(n<0 || (uint64_t)n > SIZE_MAX/sizeof(float)){ fprintf(stderr,"falloc: n=%lld is out of range\n",(long long)n); exit(1); } float *p=malloc((size_t)n*sizeof(float)); if(!p){fprintf(stderr,"OOM\n");exit(1);} return p; } -/* ---- Accumulatore int4->float a 512 bit / 512-bit int4->float accumulator ---- - * Stessa matematica lossless di matmul_i4 (nibble->f32, FMA), ma 32 pesi/iter su - * due catene FMA indipendenti. NON bit-identico al vecchio ordine: la riduzione - * ad albero accumula MENO errore della somma sequenziale (misurato 2-4x più - * vicino all'oracolo double sulle forme reali; perplexity invariata, +4-7% sul - * decode con routing CPU-heavy — vedi docs/experiments/glm52-6x5090-2026-07-12.md). - * EN: same lossless math as matmul_i4, 32 weights/iter on two independent FMA - * chains. Not bit-identical to the old order: tree reduction accumulates LESS - * rounding than sequential summation. I4_ACC512=0 restores the old order (A/B). */ -#if defined(__AVX512F__) && defined(__AVX512BW__) -static int g_i4_acc512=1; -static inline float dot_i4f_avx512(const uint8_t *w,const float *x,int I){ - const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8); - __m512 acc0=_mm512_setzero_ps(),acc1=_mm512_setzero_ps(); int i=0; - for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1))); - __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi); - __m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8)); - __m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8)); - acc0=_mm512_fmadd_ps(_mm512_loadu_ps(x+i),w0,acc0); - acc1=_mm512_fmadd_ps(_mm512_loadu_ps(x+i+16),w1,acc1); - } - return _mm512_reduce_add_ps(_mm512_add_ps(acc0,acc1)); -} -/* selftest contro il riferimento scalare (I4_ACC512_TEST=1): copre l'ordine dei - * nibble e ogni multiplo di 32. / selftest vs the scalar reference. */ -static int i4_acc512_selftest(void){ - enum { N=224 }; uint8_t w[(N+1)/2]; float x[N]; - for(int i=0;i>1]=(uint8_t)(q+8); - else w[i>>1]|=(uint8_t)((q+8)<<4); - x[i]=(float)(((i*29+7)%101)-50)/37.f; - } - for(int n=32;n<=N;n+=32){ - float ref=0; for(int i=0;i>1]>>((i&1)*4))&15)-8); - float got=dot_i4f_avx512(w,x,n),tol=2e-5f*(1.f+fabsf(ref)); - if(fabsf(got-ref)>tol){ fprintf(stderr,"AVX512 i4 selftest n=%d: %.9g != %.9g\n",n,got,ref); return 0; } - } - return 1; -} -#endif -/* y[S,O] = x[S,I] @ W^T, W[O,I] f32 */ -static void matmul(float *y, const float *x, const float *W, int S, int I, int O){ - #pragma omp parallel for schedule(static) - for (int o=0;o>1))); /* 8 byte=16 nibble */ - __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i nib=_mm_unpacklo_epi8(lo,hi); /* nibble in ordine */ - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } - a=hsum256(acc); -#elif defined(__ARM_NEON) - const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); - float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); - for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); /* 8 byte=16 nibble */ - uint8x8x2_t z=vzip_u8(vand_u8(by,m4), vshr_n_u8(by,4)); /* nibble in ordine */ - int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[0]),b8)); - int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[1]),b8)); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } - a=vaddvq_f32(vaddq_f32(ac0,ac1)); -#endif -#if defined(__AVX512F__) && defined(__AVX512BW__) - } -#endif - for(;i+1>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8; - a += xs[i]*(float)lo + xs[i+1]*(float)hi; } - if(i>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; } - y[(int64_t)s*O+o]=a*sc; } } -} -/* y[S,O] = x[S,I] @ W^T with W int4 packed (2/byte) + per-GROUP scales (fmt=4). - * Same nibble math as matmul_i4, but the scale changes every `gs` elements along I. - * The accumulator resets at each group boundary: dot(x[grp], w[grp]) * scale[grp]. - * gs MUST be a multiple of 16 (the AVX2 vector width). */ -static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const float *scale, - int S, int I, int O, int gs){ - int rb=(I+1)/2; int ng=(I+gs-1)/gs; - #pragma omp parallel for schedule(static) - for(int o=0;oI) glen=I-base; - float sc=scl[g]; - int i=base; -#ifdef __AVX2__ - const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8); - __m256 acc=_mm256_setzero_ps(); - for(; i+16<=base+glen; i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1))); - __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i nib=_mm_unpacklo_epi8(lo,hi); - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } - a+=hsum256(acc)*sc; -#endif - /* scalar tail for the group remainder */ - for(; i>1]; - a+=(xs[i]*(float)((int)(byte&0xF)-8)+xs[i+1]*(float)((int)(byte>>4)-8))*sc; } - else { uint8_t byte=w[i>>1]; a+=xs[i]*(float)((int)(byte&0xF)-8)*sc; } - } - } - y[(int64_t)s*O+o]=a; - } - } -} -/* Decode hot path for gate+up: same exact q4 dot products as matmul_i4, but one - * OpenMP dispatch covers both matrices. KTransformers uses persistent pools; - * this keeps colibri dependency-free while removing one team launch/expert. */ -static void matmul_i4_pair(float *yg, float *yu, const float *x, - const uint8_t *qg, const float *sg, - const uint8_t *qu, const float *su, int I, int O){ - int rb=(I+1)/2; - #pragma omp parallel for schedule(static) - for(int z=0;z<2*O;z++){ - int o=z>1))); - __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i nib=_mm_unpacklo_epi8(lo,hi); - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); } - a=hsum256(acc); -#elif defined(__ARM_NEON) - const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); - float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0); - for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); - uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4)); - int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8)); - int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8)); - ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); - ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); - ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); - ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } - a=vaddvq_f32(vaddq_f32(ac0,ac1)); -#endif -#if defined(__AVX512F__) && defined(__AVX512BW__) - } -#endif - for(;i+1>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); } - if(i>1]&15)-8); - (z=2) non calcolano la STESSA funzione. Tre interruttori dipendono da S: il gate - * int4-IDOT (S>=g_i4s — asimmetrico proprio dove g_i4s>1), la fusione gate+up solo-S==1, - * e la soglia righe del GEMM Metal. Con SPEC_PIN=1 (default) ogni forward emesso mentre - * i draft del modello sono attivi resta sulla famiglia di kernel di S=1: draft e verifica - * coincidono per costruzione. Prefill e decode non speculativo sono intoccati. - * EN: MTP acceptance collapses when the draft (S=1) and verify (S>=2) forwards do not - * compute the SAME function. Three switches are S-dependent: the int4 IDOT gate - * (S>=g_i4s — asymmetric exactly on ISAs where g_i4s>1), the S==1-only gate+up fusion, - * and the Metal GEMM row threshold. SPEC_PIN=1 (default) pins every forward issued - * while model drafts are live to the platform's S=1 kernel family, so draft and verify - * agree by construction; prefill and non-speculative decode are untouched. - * SPEC_PIN=0 restores the S-dependent gates (A/B). */ -static int g_spec_pin=1; -static int g_spec_live=0; /* set by spec_decode while drafts are live */ -static inline int spec_pinned(void){ return g_spec_pin && g_spec_live; } -static void expert_gate_up(float *g,float *u,const float *x,QT *wg,QT *wu,int S){ - if(!g_no_fused_pair&&!spec_pinned()&&S==1&&wg->fmt==2&&wu->fmt==2&&wg->I==wu->I&&wg->O==wu->O) - matmul_i4_pair(g,u,x,wg->q4,wg->s,wu->q4,wu->s,wg->I,wg->O); - else { matmul_qt(g,x,wg,S); matmul_qt(u,x,wu,S); } -} -/* y[S,O] = x[S,I] @ W^T con W int2 impacchettato (4 valori/byte) + scala[O]. nibble 2-bit -> [-2,1]. */ -static void matmul_i2(float *y, const float *x, const uint8_t *q2, const float *scale, int S, int I, int O){ - int rb=(I+3)/4; - #pragma omp parallel for schedule(static) - for (int o=0;o>2))); /* 4 byte=16 valori */ - __m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2); - __m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2); - __m128i lo=_mm_unpacklo_epi8(p0,p1), hi=_mm_unpacklo_epi8(p2,p3); - __m128i nib=_mm_unpacklo_epi16(lo,hi); /* 16 valori in ordine */ - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b2)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b2)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } - a=hsum256(acc); -#elif defined(__ARM_NEON) - const uint8x8_t m2v=vdup_n_u8(3); const int8x8_t b2v=vdup_n_s8(2); - float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); - for(;i+16<=I;i+=16){ uint32_t wd; memcpy(&wd, w+(i>>2), 4); /* 4 byte=16 valori */ - uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd)); - uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v)); - uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6)); - uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0])); - int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[0]),b2v)); /* 16 valori in ordine */ - int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[1]),b2v)); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } - a=vaddvq_f32(vaddq_f32(ac0,ac1)); -#endif - for(;i>2]; int sh=(i&3)*2; a += xs[i]*(float)((int)((byte>>sh)&3)-2); } - y[(int64_t)s*O+o]=a*sc; } } -} -/* ---- KERNEL INTERI (IDOT): attivazioni quantizzate a int8 per riga (absmax/127, - * stile Q8_0), prodotto scalare INTERO via maddubs/madd AVX2 — niente conversione - * f32 dei pesi nel ciclo caldo. ~2-3x sui matmul quantizzati; errore aggiunto ~0.3% - * RMS per matmul (attivazione int8), IDOT=0 torna al percorso f32 esatto. */ -#if defined(__AVX512VNNI__) && defined(__AVX512BW__) -#define IDOT_KERNEL "avx512-vnni" -#elif defined(__AVXVNNI__) && defined(__AVX2__) -#define IDOT_KERNEL "avx-vnni" -#elif defined(__AVX2__) -#define IDOT_KERNEL "avx2" -#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) -#define IDOT_KERNEL "neon-i8mm" -#elif defined(__ARM_NEON) -#define IDOT_KERNEL "neon" -#elif defined(__VSX__) -#define IDOT_KERNEL "vsx" -#else -#define IDOT_KERNEL "scalar" -#endif -static int g_idot=1; -#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) -static int g_i4s=1; /* SDOT presente: int4 IDOT conviene anche a S=1 (decode). Misurato - * su Apple M-series: +14%%, expert-matmul -16%%. EN: with SDOT, int4 - * IDOT pays even at S=1 (decode); measured on Apple M-series. */ -#elif defined(__VSX__) -static int g_i4s=1; /* POWER8 vec_msum: qui il fallback f32 e' SCALARE, quindi l'IDOT - * int4 conviene anche a S=1. Misurato su POWER8 S824 (vedi PR). - * EN: on VSX the f32 fallback is plain scalar C, so int4 IDOT - * pays even at S=1. Measured on a POWER8 S824 (see PR). */ -#else -static int g_i4s=2; /* senza SDOT / altrove: soglia originale (misura AVX2 dell'autore). - * EN: without SDOT / elsewhere: original threshold (author's AVX2). */ -#endif -static inline float qrow_i8(const float *x, int8_t *q, int I){ - float amax=0; for(int i=0;iamax)amax=a; } - float s=amax/127.f; if(s<1e-12f) s=1e-12f; float inv=1.f/s; - for(int i=0;i s32 directly, 64 bytes/iter, no 16-bit intermediate. - * AVX-512 has no vpsignb: |w| via abs, sign folded into x with a mask-negate - * (w==0 -> product 0 either way). |x|<=127 (qrow_i8), |w|<=128 as u8: each - * s32 lane adds <= 4*128*127, safe up to I=16384 like the AVX2 bound. */ - __m512i acc=_mm512_setzero_si512(); - for(;i+64<=I;i+=64){ - __m512i wv=_mm512_loadu_si512((const void*)(w+i)); - __m512i xv=_mm512_loadu_si512((const void*)(x+i)); - __mmask64 neg=_mm512_movepi8_mask(wv); - __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv); - acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs); - } - sum=_mm512_reduce_add_epi32(acc); -#elif defined(__AVXVNNI__) && defined(__AVX2__) - /* AVX-VNNI 128-bit: vpdpbusd u8*s8 -> s32, 16 byte/iter. Stesso trucco del - * segno della variante 512-bit: |w| via abs, segno piegato in x con maschera - * (w==0 -> product 0). __AVX2__ serve per _mm_sign_epi8 / abs. */ - __m128i acc=_mm_setzero_si128(); - for(;i+16<=I;i+=16){ - __m128i wv=_mm_loadu_si128((const __m128i*)(w+i)); - __m128i xv=_mm_loadu_si128((const __m128i*)(x+i)); - __m128i xs=_mm_sign_epi8(xv,wv); /* x * sign(w); _mm_sign zona __AVX2__ */ - acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(wv),xs); - } - sum=hsum128_i32(acc); -#elif defined(__AVX2__) - __m256i acc=_mm256_setzero_si256(); const __m256i ones=_mm256_set1_epi16(1); - for(;i+32<=I;i+=32){ - __m256i wv=_mm256_loadu_si256((const __m256i*)(w+i)); - __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); - __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); - acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); - } - sum=hsum256_i32(acc); -#elif defined(__ARM_NEON) - /* ARM: SDOT nativo se disponibile (Apple Silicon: sempre); altrimenti vmull/vpadal. - * Stesso bound anti-overflow del trucco AVX2: coppie <= 128*127*2 = 32512 < 32767. */ -#if defined(__ARM_FEATURE_DOTPROD) - /* 4 accumulatori indipendenti: SDOT ha latenza ~3-4 cicli, con un solo acc la - * catena seriale strozza il core a ~26 GB/s di pesi; con 4 lane indipendenti il - * dot diventa memory-bound (misurato su M4: 26 -> 63 GB/s per core, 2.4x). */ - int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); - for(;i+64<=I;i+=64){ - a0=vdotq_s32(a0,vld1q_s8(w+i), vld1q_s8(x+i)); - a1=vdotq_s32(a1,vld1q_s8(w+i+16),vld1q_s8(x+i+16)); - a2=vdotq_s32(a2,vld1q_s8(w+i+32),vld1q_s8(x+i+32)); - a3=vdotq_s32(a3,vld1q_s8(w+i+48),vld1q_s8(x+i+48)); - } - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - for(;i+16<=I;i+=16) acc=vdotq_s32(acc,vld1q_s8(w+i),vld1q_s8(x+i)); - sum=vaddvq_s32(acc); -#else - int32x4_t acc=vdupq_n_s32(0); - for(;i+16<=I;i+=16){ - int8x16_t wv=vld1q_s8(w+i), xv=vld1q_s8(x+i); - int16x8_t p=vmull_s8(vget_low_s8(wv),vget_low_s8(xv)); - p=vmlal_s8(p,vget_high_s8(wv),vget_high_s8(xv)); - acc=vpadalq_s16(acc,p); - } - sum=vaddvq_s32(acc); -#endif -#elif defined(__VSX__) - /* POWER8: vec_msum (s8 x u8 -> s32) somma i prodotti byte DIRETTAMENTE in lane - * s32, 16 byte/iter: il bound anti-saturazione a 16 bit di maddubs qui non serve. - * Stesso trucco del segno (|w| u8 per x*sign(w) s8), ma |w| via select+sub MODULO - * e non vec_abs: -128 deve diventare 128 u8, non saturare a 127. - * EN: vec_msum accumulates byte products straight into s32 lanes; |w| is built - * with a modulo subtract select instead of vec_abs so w=-128 wraps to 128 (u8) - * rather than saturating to 127. |x|<=127 from qrow_i8, so x negation is safe. */ - __vector signed int acc=vec_splats(0); - const __vector signed char vz=vec_splats((signed char)0); - for(;i+16<=I;i+=16){ - __vector signed char wv=vec_xl(0,(const signed char*)(w+i)); - __vector signed char xv=vec_xl(0,(const signed char*)(x+i)); - __vector __bool char neg=vec_cmplt(wv,vz); - __vector signed char xs=vec_sel(xv,vec_sub(vz,xv),neg); - __vector unsigned char wa=(__vector unsigned char)vec_sel(wv,vec_sub(vz,wv),neg); - acc=vec_msum(xs,wa,acc); - } - sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); -#endif - for(;i int8 [-8,7] al volo, poi stesso trucco */ -static inline int32_t dot_i4i8(const uint8_t *w4, const int8_t *x, int I){ - int32_t sum=0; int i=0; -#if defined(__AVX512VNNI__) && defined(__AVX512BW__) - /* 32 bytes = 64 nibbles -> int8 in [-8,7], one vpdpbusd per 64 values. - * 256-bit unpack leaves values in per-128-lane order [0-15][32-47]/[16-31][48-63]; - * dot pairing is order-invariant, so permute x's 128-bit blocks to match - * instead of re-ordering w (one vpermq per iter, off the critical unpack path). */ - const __m256i m4v=_mm256_set1_epi8(0x0F); - const __m512i b8v=_mm512_set1_epi8(8); - const __m512i xidx=_mm512_setr_epi64(0,1,4,5,2,3,6,7); - __m512i acc=_mm512_setzero_si512(); - for(;i+64<=I;i+=64){ - __m256i by=_mm256_loadu_si256((const __m256i*)(w4+(i>>1))); - __m256i lo=_mm256_and_si256(by,m4v), hi=_mm256_and_si256(_mm256_srli_epi16(by,4),m4v); - __m256i z0=_mm256_unpacklo_epi8(lo,hi), z1=_mm256_unpackhi_epi8(lo,hi); - __m512i wv=_mm512_sub_epi8(_mm512_inserti64x4(_mm512_castsi256_si512(z0),z1,1),b8v); - __m512i xv=_mm512_permutexvar_epi64(xidx,_mm512_loadu_si512((const void*)(x+i))); - __mmask64 neg=_mm512_movepi8_mask(wv); - __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv); - acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs); - } - sum=_mm512_reduce_add_epi32(acc); -#elif defined(__AVXVNNI__) && defined(__AVX2__) - /* AVX-VNNI 128-bit, int4: 16 byte = 32 nibble -> int8 [-8,7] in due half - * (n0/n1), ciascuno alimentato a un vpdpbusd da 16 byte. Stesso unpack - * 128-bit del ramo AVX2 sotto; 32 elementi/iter come li. */ - const __m128i m4=_mm_set1_epi8(0x0F); const __m128i b8=_mm_set1_epi8(8); - __m128i acc=_mm_setzero_si128(); - for(;i+32<=I;i+=32){ - __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); /* 16 byte = 32 nibble */ - __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); /* nibble in ordine */ - __m128i w0=_mm_sub_epi8(n0,b8), w1=_mm_sub_epi8(n1,b8); - __m128i x0=_mm_loadu_si128((const __m128i*)(x+i)); - __m128i x1=_mm_loadu_si128((const __m128i*)(x+i+16)); - acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0)); - acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1)); - } - sum=hsum128_i32(acc); -#elif defined(__AVX2__) - const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi8(8); - const __m256i ones=_mm256_set1_epi16(1); - __m256i acc=_mm256_setzero_si256(); - for(;i+32<=I;i+=32){ - __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); /* 16 byte = 32 nibble */ - __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); /* in ordine */ - __m256i wv=_mm256_sub_epi8(_mm256_set_m128i(n1,n0),b8); - __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); - __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); - acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); - } - sum=hsum256_i32(acc); -#elif defined(__ARM_NEON) - const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8); -#if defined(__ARM_FEATURE_DOTPROD) - /* 4 accumulatori indipendenti (vedi dot_i8i8): spezza la catena seriale su acc. - * Misurato su M4: 12.4 -> 29.9 GB/s di pesi per core (2.4x). */ - int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); - for(;i+64<=I;i+=64){ - uint8x16_t byA=vld1q_u8(w4+(i>>1)), byB=vld1q_u8(w4+(i>>1)+16); - uint8x16x2_t zA=vzipq_u8(vandq_u8(byA,m4q), vshrq_n_u8(byA,4)); /* nibble in ordine */ - uint8x16x2_t zB=vzipq_u8(vandq_u8(byB,m4q), vshrq_n_u8(byB,4)); - a0=vdotq_s32(a0,vsubq_s8(vreinterpretq_s8_u8(zA.val[0]),b8q),vld1q_s8(x+i)); - a1=vdotq_s32(a1,vsubq_s8(vreinterpretq_s8_u8(zA.val[1]),b8q),vld1q_s8(x+i+16)); - a2=vdotq_s32(a2,vsubq_s8(vreinterpretq_s8_u8(zB.val[0]),b8q),vld1q_s8(x+i+32)); - a3=vdotq_s32(a3,vsubq_s8(vreinterpretq_s8_u8(zB.val[1]),b8q),vld1q_s8(x+i+48)); - } - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - for(;i+32<=I;i+=32){ - uint8x16_t by=vld1q_u8(w4+(i>>1)); /* 16 byte = 32 nibble */ - uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); /* nibble in ordine */ - acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q),vld1q_s8(x+i)); - acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q),vld1q_s8(x+i+16)); - } - sum=vaddvq_s32(acc); -#else - int32x4_t acc=vdupq_n_s32(0); - for(;i+32<=I;i+=32){ - uint8x16_t by=vld1q_u8(w4+(i>>1)); /* 16 byte = 32 nibble */ - uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); /* nibble in ordine */ - int8x16_t w0=vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q); - int8x16_t w1=vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q); - int8x16_t x0=vld1q_s8(x+i), x1=vld1q_s8(x+i+16); - int16x8_t p=vmull_s8(vget_low_s8(w0),vget_low_s8(x0)); /* |w|<=8: nessun overflow */ - p=vmlal_s8(p,vget_high_s8(w0),vget_high_s8(x0)); - acc=vpadalq_s16(acc,p); - p=vmull_s8(vget_low_s8(w1),vget_low_s8(x1)); - p=vmlal_s8(p,vget_high_s8(w1),vget_high_s8(x1)); - acc=vpadalq_s16(acc,p); - } - sum=vaddvq_s32(acc); -#endif -#elif defined(__VSX__) - /* 16 byte = 32 nibble. vec_mergeh/vec_mergel su ppc64le (GCC) interallacciano come - * unpacklo/unpackhi x86 (verificato empiricamente su POWER8): i nibble escono in - * ordine di memoria. |w|<=8 dopo il -8, quindi stesso trucco del segno di dot_i8i8. - * EN: vec_mergeh/l on ppc64le interleave like x86 unpacklo/hi (verified on POWER8), - * so nibbles come out in memory order; then the same sign trick as dot_i8i8. */ - const __vector unsigned char m4v=vec_splats((unsigned char)0x0F); - const __vector unsigned char sh4=vec_splats((unsigned char)4); - const __vector signed char b8v=vec_splats((signed char)8); - const __vector signed char vz=vec_splats((signed char)0); - __vector signed int acc=vec_splats(0); - for(;i+32<=I;i+=32){ - __vector unsigned char by=vec_xl(0,w4+(i>>1)); /* 16 byte = 32 nibble */ - __vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4); - __vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v); - __vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v); - __vector signed char x0=vec_xl(0,(const signed char*)(x+i)); - __vector signed char x1=vec_xl(0,(const signed char*)(x+i+16)); - __vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz); - acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0), - (__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc); - acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1), - (__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc); - } - sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); -#endif - for(;i+1>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } - if(i>1]; sum+=((int)(b&0xF)-8)*x[i]; } - return sum; -} -#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) -/* SMMLA (i8mm): vmmlaq_s32 vede ogni int8x16_t come matrice 2x8 row-major (byte 0-7 = - * riga 0, byte 8-15 = riga 1) e accumula C += A*B^T nel 2x2 int32: lane0=a0.b0, - * lane1=a0.b1, lane2=a1.b0, lane3=a1.b1. vcombine di due mezze-righe costruisce la - * matrice: A = due righe di peso (o,o+1), B = due righe di attivazione (s,s+1), quindi - * meta' traffico pesi e doppio lavoro per istruzione a S>=2. EN: vmmlaq_s32 treats each - * int8x16_t as a 2x8 row-major matrix and does C += A*B^T on a 2x2 int32 tile; vcombine - * of vget_low/high halves builds the 2-row register from two weight/activation rows. */ -static inline int32x4_t mm_tile16(int32x4_t acc, int8x16_t wo, int8x16_t wo1, - int8x16_t xs, int8x16_t xs1){ - acc=vmmlaq_s32(acc, vcombine_s8(vget_low_s8(wo), vget_low_s8(wo1)), - vcombine_s8(vget_low_s8(xs), vget_low_s8(xs1))); - return vmmlaq_s32(acc, vcombine_s8(vget_high_s8(wo), vget_high_s8(wo1)), - vcombine_s8(vget_high_s8(xs), vget_high_s8(xs1))); -} -static void matmul_q_idot_mm(float *y, const int8_t *xq, const float *sx, const int8_t *q, - const float *scale, int S, int I, int O){ - #pragma omp parallel for schedule(static) - for(int o=0;o<(O&~1);o+=2){ - const int8_t *wo=q+(int64_t)o*I, *wo1=q+(int64_t)(o+1)*I; - float sc0=scale[o], sc1=scale[o+1]; - for(int s=0;s<(S&~1);s+=2){ - const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I; - /* 4 accumulatori indipendenti: una sola catena vmmla e' latency-bound. - * EN: 4 independent accumulators; a single vmmla chain is latency-bound. */ - int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0; - for(;i+64<=I;i+=64){ - a0=mm_tile16(a0,vld1q_s8(wo+i), vld1q_s8(wo1+i), vld1q_s8(xs+i), vld1q_s8(xs1+i)); - a1=mm_tile16(a1,vld1q_s8(wo+i+16),vld1q_s8(wo1+i+16),vld1q_s8(xs+i+16),vld1q_s8(xs1+i+16)); - a2=mm_tile16(a2,vld1q_s8(wo+i+32),vld1q_s8(wo1+i+32),vld1q_s8(xs+i+32),vld1q_s8(xs1+i+32)); - a3=mm_tile16(a3,vld1q_s8(wo+i+48),vld1q_s8(wo1+i+48),vld1q_s8(xs+i+48),vld1q_s8(xs1+i+48)); - } - for(;i+16<=I;i+=16) - a0=mm_tile16(a0,vld1q_s8(wo+i),vld1q_s8(wo1+i),vld1q_s8(xs+i),vld1q_s8(xs1+i)); - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); - int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); - for(;i>1)), byo1=vld1q_u8(wo1+(i>>1)); - uint8x16_t cyo=vld1q_u8(wo+(i>>1)+16), cyo1=vld1q_u8(wo1+(i>>1)+16); - uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); - uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); - uint8x16x2_t ko =vzipq_u8(vandq_u8(cyo, m4q), vshrq_n_u8(cyo, 4)); - uint8x16x2_t ko1=vzipq_u8(vandq_u8(cyo1,m4q), vshrq_n_u8(cyo1,4)); - a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), - vld1q_s8(xs+i), vld1q_s8(xs1+i)); - a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), - vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); - a2=mm_tile16(a2, vsubq_s8(vreinterpretq_s8_u8(ko.val[0]),b8q), - vsubq_s8(vreinterpretq_s8_u8(ko1.val[0]),b8q), - vld1q_s8(xs+i+32), vld1q_s8(xs1+i+32)); - a3=mm_tile16(a3, vsubq_s8(vreinterpretq_s8_u8(ko.val[1]),b8q), - vsubq_s8(vreinterpretq_s8_u8(ko1.val[1]),b8q), - vld1q_s8(xs+i+48), vld1q_s8(xs1+i+48)); - } - for(;i+32<=I;i+=32){ - uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1)); - uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); - uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); - a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), - vld1q_s8(xs+i), vld1q_s8(xs1+i)); - a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), - vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); - } - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); - int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); - for(;i+1>1], bo1=wo1[i>>1]; - int a0=(int)(bo&0xF)-8, a1=(int)(bo>>4)-8, b0=(int)(bo1&0xF)-8, b1=(int)(bo1>>4)-8; - int u0=xs[i],u1=xs[i+1],v0=xs1[i],v1=xs1[i+1]; - d00+=a0*u0+a1*u1; d01+=a0*v0+a1*v1; d10+=b0*u0+b1*u1; d11+=b0*v0+b1*v1; } - if(i>1], bo1=wo1[i>>1]; - int a0=(int)(bo&0xF)-8, b0=(int)(bo1&0xF)-8; - d00+=a0*xs[i]; d01+=a0*xs1[i]; d10+=b0*xs[i]; d11+=b0*xs1[i]; } - y[(int64_t)s*O+o] =(float)d00*sc0*sx[s]; - y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s]; - y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1]; - y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1]; - } - if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I; - y[(int64_t)s*O+o] =(float)dot_i4i8(wo, xs,I)*sc0*sx[s]; - y[(int64_t)s*O+(o+1)]=(float)dot_i4i8(wo1,xs,I)*sc1*sx[s]; } - } - if(O&1){ int o=O-1; const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o]; - #pragma omp parallel for schedule(static) - for(int s=0;s=2){ matmul_q_idot_mm(y,xq,sx,q,scale,S,I,O); return; } -#endif - #pragma omp parallel for schedule(static) - for(int o=0;o=2){ matmul_i4_idot_mm(y,xq,sx,q4,scale,S,I,O); return; } -#endif - #pragma omp parallel for schedule(static) - for(int o=0;og_qscratch.xq_cap){ - int8_t *p=realloc(g_qscratch.xq,xn); - if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); } - g_qscratch.xq=p; g_qscratch.xq_cap=xn; - } - if(sn>g_qscratch.sx_cap){ - float *p=realloc(g_qscratch.sx,sn*sizeof(float)); - if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); } - g_qscratch.sx=p; g_qscratch.sx_cap=sn; - } - *xq=g_qscratch.xq; *sx=g_qscratch.sx; -} - -/* allow_idot=0: forza il kernel int4/int8 ESATTO (attivazioni f32). Serve alle proiezioni di - * attenzione: sono sensibili alla quantizzazione int8 delle attivazioni dell'IDOT. Misurato su - * GLM-5.2 int4, 1023 token, log-lik -5040.33 (esatto) -> -5160.47 (IDOT) = +0.117 nat/token, - * ~+12% perplexity. Gli altri matmul del prefill (o_proj, kv_b, expert) tengono l'IDOT. - * EN: allow_idot=0 forces the EXACT int4/int8 kernel (f32 activations). The attention - * projections need it: IDOT's int8 activation quantization costs +0.117 nats/token there - * (~+12% perplexity), measured. Every other prefill matmul keeps IDOT as before. */ -static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot); -static void matmul_qt(float *y, const float *x, QT *w, int S){ matmul_qt_ex(y,x,w,S,1); } static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot){ #ifdef COLI_METAL - /* Large row-batches (prefill: kv_b reconstruction, o_proj, dense MLP, step_all logits) - * amortize Metal's ~5ms submit latency; small-S decode matmuls stay on CPU (NEON wins). - * Weights must be registered (all dense QT allocs are, via qalloc). */ if(g_metal_enabled && S>=g_metal_gemm_min && !spec_pinned() && (w->fmt==1||w->fmt==2) && !omp_in_parallel()){ const void *wp = w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4; if(coli_metal_gemm(y,x,wp,w->s,w->fmt,S,w->I,w->O)) return; } #endif #ifdef COLI_CUDA - /* The CUDA backend owns persistent copies only for model-resident tensors. - * Streaming expert slots are reused for different IDs and must never enter - * this cache. Nested OpenMP calls stay on CPU because each device context - * intentionally owns one synchronous scratch stream in this stage. */ if(g_cuda_enabled && w->cuda_eligible && !w->cuda_failed && !omp_in_parallel()){ const void *weights = w->fmt==0 ? (const void*)w->qf : w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4; @@ -1049,21 +344,7 @@ static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot) } #endif if(w->fmt==0){ matmul(y,x,w->qf,S,w->I,w->O); return; } - /* fmt=4: grouped int4 — always use the exact grouped kernel (no IDOT approximation, - * since the whole point of grouped scales is better quality). */ if(w->fmt==4){ matmul_i4_grouped(y,x,w->q4,w->s,S,w->I,w->O,w->gs); return; } - /* int8 IDOT vince sempre (1.4-2.5x). int4 IDOT: l'autore su AVX2 trovo' che a S=1 - * non ripaga (soglia S>=2); ma su ARM/SDOT il singolo token CONVIENE (vedi g_i4s / - * PR #9 per il gemello VNNI). Soglia configurabile con I4S. - * EN: int8 IDOT always wins (1.4-2.5x). int4 IDOT: on AVX2 the author found S=1 didn't - * pay (S>=2 gate); on ARM/SDOT single-token DOES pay (see g_i4s / PR #9 for the VNNI - * twin). Threshold configurable via I4S. */ - /* #163: sotto SPEC_PIN il gate int4-IDOT usa la decisione di S=1 per OGNI S, cosi' - * draft e verifica restano sulla stessa famiglia. (CUDA non e' toccato: la sua - * condizione non dipende da S, quindi e' gia' coerente tra draft e verifica.) - * EN: under SPEC_PIN the int4 IDOT gate uses the S=1 decision for EVERY S, so draft - * and verify stay in one family. (CUDA untouched: its condition is S-independent, - * hence already draft/verify-consistent.) */ if(allow_idot && g_idot && (w->fmt==1 || (w->fmt==2 && (spec_pinned() ? g_i4s<=1 : S>=g_i4s)))){ int I=w->I; int8_t *xq; float *sx; if(S<0 || I<0 || (size_t)S>SIZE_MAX/(size_t)(I?I:1)){ fprintf(stderr,"matmul_qt: shape overflow\n"); exit(1); } @@ -1078,49 +359,6 @@ static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot) else matmul_i4(y,x,w->q4,w->s,S,w->I,w->O); } -/* quantizza w[O,I] f32 -> int8 q[O,I] + scala[O] simmetrica per riga */ -static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits){ - int qmax=(1<<(bits-1))-1; - #pragma omp parallel for schedule(static) - for(int o=0;oamax)amax=a; } - float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; - int8_t *qr=q+(int64_t)o*I; - for(int i=0;iqmax)v=qmax; if(v<-qmax-1)v=-qmax-1; qr[i]=(int8_t)v; } - } -} -/* quantizza w[O,I] f32 -> int4 impacchettato (2/byte) + scala[O]. - * bits<=4: valori in [-qmax-1,qmax] stanno in un nibble [-8,7]; memorizzati come v+8 (0..15). */ -static void pack_int4(const float *w, uint8_t *q4, float *scale, int O, int I, int bits){ - int qmax=(1<<(bits-1))-1, rb=(I+1)/2; - #pragma omp parallel for schedule(static) - for(int o=0;oamax)amax=a; } - float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; - uint8_t *qr=q4+(int64_t)o*rb; - for(int i=0;iqmax)v0=qmax; if(v0<-8)v0=-8; - int v1=0; if(i+1qmax)v1=qmax; if(v1<-8)v1=-8; } - qr[i>>1] = (uint8_t)((v0+8) | ((v1+8)<<4)); - } - } -} - -/* quantizza w[O,I] f32 -> int2 impacchettato (4/byte) + scala[O]. valori nibble 2-bit in [-2,1]. */ -static void pack_int2(const float *w, uint8_t *q2, float *scale, int O, int I, int bits){ - int qmax=(1<<(bits-1))-1, rb=(I+3)/4; - #pragma omp parallel for schedule(static) - for(int o=0;oamax)amax=a; } - float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; - uint8_t *qr=q2+(int64_t)o*rb; - for(int i=0;iqmax)v=qmax; if(v<-2)v=-2; byte|=(uint8_t)((v+2)<<(k*2)); } - qr[i>>2]=byte; - } - } -} - static int g_nopack=0; /* NOPACK=1 -> tiene i valori <=4bit in contenitore int8 (per validare il packing) */ static int g_drop=0; /* DROP=1 -> scarta le pagine expart dopo l'uso. Default 0: le lascia in * page-cache (buff/cache, NON RSS) come L2 gratuito -> sfrutta lo @@ -1206,6 +444,11 @@ static int g_disk_split=0; /* DISK_SPLIT=1: contatori che spezzano i DISK LOAD ( * (int4), con i byte letti. Default OFF: a flag spento gli atomic * non vengono MAI toccati (zero overhead), le righe extra di stats * non vengono stampate. Solo misura: nessun effetto sull'output. */ + +#include "sample.h" +#include "kv_persist.h" +#include "telemetry.h" + /* Aligned allocator for dense QT weights/scales: under METAL, page-align + register so the * GPU reads them zero-copy (no upload duplicate). Plain malloc otherwise. */ /* ---- COLI_NUMA=1 (#82): interleave the expert slabs across NUMA nodes ---- @@ -4239,13 +3482,14 @@ static void mtp_absorb(Model *m, const int *next_ids, const float *x, int S, int } static inline int argmax_v(const float *lo, int V){ - /* skip NaN (x==x is false for NaN) so a poisoned logit can't pin the argmax - * to index 0 — pick the max finite/+Inf entry instead. */ int b=-1; float bv=-INFINITY; for(int i=0;ibv){ bv=x; b=i; } } return b<0?0:b; } +static void repin_pass_limit(Model *m,int limit); +static void repin_pass(Model *m){ repin_pass_limit(m,16); } + /* ---- METODO F: draft grammaticale (#48) ---- * gr_feed consuma i byte di ogni token EMESSO e tiene il walker in sync con l'output; * grammar_draft propone lo span FORZATO successivo (un solo byte legale per posizione) @@ -4326,113 +3570,6 @@ static int grammar_draft(int *draft, int cap){ * Rejection sampling di Leviathan: accetta il draft x_d con prob p(x_d); al rifiuto * ricampiona da p con x_d azzerato e rinormalizzato. La distribuzione risultante e' * ESATTAMENTE p: la speculazione resta invisibile all'output anche col sampling. */ -static uint64_t g_rng=0x9E3779B97F4A7C15ULL; -static inline double rndu(void){ g_rng^=g_rng<<13; g_rng^=g_rng>>7; g_rng^=g_rng<<17; - return (double)(g_rng>>11)*(1.0/9007199254740992.0); } -static float *g_pbuf=NULL; static int *g_pidx=NULL; /* buffer riusati (decode single-thread) */ -/* sift-down su max-heap in h[0..n), chiave = g_pbuf[h[i]] (#335: partial top-p select). - * Versione "a buco": porta il valore di radice e lo deposita solo alla fine, cosi' - * heapify e' O(V) e ogni pop e' O(log n) senza qsort sull'intero vocabolario. */ -static void topp_siftdown(int *h, int n, int i){ - int iv=h[i]; float kv=g_pbuf[iv]; - for(;;){ int l=2*i+1; - if(l>=n) break; /* foglia */ - int b=l; if(l+1g_pbuf[h[l]]) b=l+1; /* figlio maggiore */ - if(g_pbuf[h[b]]<=kv) break; /* nessun figlio supera la radice -> ferma */ - h[i]=h[b]; i=b; } - h[i]=iv; -} -/* costruisce in g_pbuf la distribuzione target: softmax(lo/temp) troncata a top-p g_nuc. - * Invariante per dist_sample: g_pbuf resta INDICIZZATO per token-id (mai riordinato); - * la coda troncata va AZZERATA in g_pbuf (dist_sample la legge direttamente per id). */ -static void dist_build(const float *lo, int V){ - if(!g_pbuf){ g_pbuf=falloc(V); g_pidx=malloc(V*sizeof(int)); } - /* Un solo logit NaN/+Inf avvelenava tutto (#369): +Inf diventava mx, NaN/Inf-mx - * -> expf NaN -> s NaN -> ogni prob NaN -> dist_sample cade sul fallback - * `g_pbuf[i]>0` (NaN>0 e' falso ovunque) e ritorna 0 PER SEMPRE, in silenzio. - * Difesa: mx solo sui finiti; un logit non finito contribuisce prob 0; - * se la distribuzione degenera (tutti non finiti / somma non valida) si - * ripiega sull'argmax dei finiti e si avvisa UNA volta, mai in silenzio. */ - int mxi=-1; float mx=0; - for(int i=0;imx)){ mx=lo[i]; mxi=i; } - double s=0; float invt=1.f/(g_temp>1e-4f?g_temp:1e-4f); - if(mxi>=0){ - for(int i=0;i=0)?mxi:0; /* mxi = argmax dei logit FINITI (robusto - * anche se lo[0] e' NaN, dove argmax_v fallirebbe) */ - for(int i=0;i0 && g_nuc<1.f){ - for(int i=0;i=0;i--) topp_siftdown(g_pidx,V,i); /* heapify O(V) */ - /* pop verso la coda: i vincitori (testa top-p) cadono in g_pidx[out..V-1] in ordine - * DECRESCENTE, come il vecchio qsort, quindi s2 accumula nello stesso ordine -> - * head bit-identical sui casi senza pareggi (i pareggi erano gia' non specificati - * sotto il qsort instabile e restano tali). Il prefisso g_pidx[0..out-1) e' la coda. */ - double s2=0, cum=0; int out=V; - do{ int root=g_pidx[0]; /* massimo corrente */ - g_pidx[0]=g_pidx[--out]; g_pidx[out]=root; /* sposta il max in coda */ - s2+=g_pbuf[root]; cum+=g_pbuf[root]; - if(out>0) topp_siftdown(g_pidx,out,0); - } while(cum0); - for(int i=0;i=0 -> quel token e' escluso (rinormalizzando al volo) */ -static int dist_sample(int V, int ban){ - double z = 1.0 - (ban>=0 ? g_pbuf[ban] : 0.0); if(z<=1e-12) z=1e-12; - double u = rndu()*z, cum=0; - for(int i=0;i=u) return i; } - for(int i=V-1;i>=0;i--) if(i!=ban && g_pbuf[i]>0) return i; - return 0; -} -/* prossimo token dai logits: greedy se g_temp<=0, altrimenti sampling. - * ban = token escluso perche' rifiutato dalla verifica speculativa precedente. */ -static int pick_tok(const float *lo, int V, int ban){ - if(g_temp<=0) return argmax_v(lo,V); - dist_build(lo,V); - return dist_sample(V,ban); -} - -/* stop-set attivo (popolato da run_text/run_serve dal config; vuoto in validazione, - * dove si genera un numero fisso di token da confrontare con l'oracolo) */ -static int g_stop[64], g_nstop=0; /* config eos + ogni added-token "special" del tokenizer */ -static void repin_pass_limit(Model *m,int limit); -static void repin_pass(Model *m){ repin_pass_limit(m,16); } -static inline int is_stop(int t){ for(int i=0;i solo gli stop del config (validazione/oracolo, dove il tokenizer non serve). */ -static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){ - g_nstop=0; - for(int i=0;in_stop && g_nstop<64;i++) g_stop[g_nstop++]=c->stop_ids[i]; - if(tok_eos>=0 && !is_stop(tok_eos) && g_nstop<64) g_stop[g_nstop++]=tok_eos; - int nsp=0; - /* DIFESA IN PROFONDITA' (woolcoxm, #298): il tokenizer marca "special":true i token di - * CONTROLLO -- <|user|>, <|assistant|>, <|observation|>, , [gMASK], i marker - * image/video/audio. Nessuno di questi e' contenuto legittimo di una risposta: se il - * modello ne emette uno, il turno e' finito (infatti GLM ne elenca tre fra gli eos - * ufficiali). Senza questo, uno di quei token non elencato nel config veniva - * DETOKENIZZATO E STAMPATO IN CHAT come testo, e la generazione proseguiva oltre la - * fine reale -- l'"added stuff on the end" riportato su un checkpoint convertito. - * Fidarsi del config di pesi convertiti da terzi e' precisamente cio' che non - * possiamo controllare; il flag del tokenizer lo possiamo leggere. - * NB: // hanno "special":false e restano contenuto vero. */ - if(T) for(int id=0; idn_ids && g_nstop<64; id++) - if(T->id_special[id] && !is_stop(id)){ g_stop[g_nstop++]=id; nsp++; } - fprintf(stderr,"[stop] %d stop tokens:",g_nstop); - for(int i=0;imx){mx=lo[i];best=i;} } - double se=0; for(int i=0;i .. " (T=ctxlen+contlen) * output: riga " " per richiesta. @@ -5101,117 +4226,6 @@ static void repin_pass_limit(Model *m,int limit){ * si appendono SOLO le posizioni nuove e si riscrive nrec per ultimo: un crash a meta' * append lascia nrec vecchio = file coerente. La riga KV del layer MTP non si salva: * al resume kv_start=-1 e la finestra di draft riparte da sola. */ -static int g_kvsave=1; -#define KV_MAGIC "COLIKV1\0" -static void kv_hdr(Model *m, int32_t *h, int nrec){ - Cfg *c=&m->c; int nic=0; - for(int i=0;in_layers;i++) if(m->Ic && m->Ic[i]) nic++; - h[0]=c->n_layers; h[1]=c->kv_lora; h[2]=c->qk_rope; - h[3]=m->has_dsa?c->index_hd:0; h[4]=nic; h[5]=c->vocab; h[6]=nrec; h[7]=0; -} -/* Bytes of one on-disk record: [tok i32][Lc+Rc per layer][Ic per DSA layer]. - * Layout matches what kv_disk_append writes and kv_disk_load reads. */ -static int64_t kv_rec_bytes(Model *m){ - Cfg *c=&m->c; - int64_t rec = 4 + (int64_t)c->n_layers*(c->kv_lora+c->qk_rope)*4; - if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) rec+=(int64_t)c->index_hd*4; - return rec; -} -/* Open the persistent handle lazily; write the header if the file is new. After - * this returns successfully, k->disk_fp is valid for the engine's lifetime and - * positioned at end-of-header (nrec==0 case) or wherever the caller seeks. */ -static int kv_disk_open(Model *m){ - KVState *k=m->kv; - if(k->disk_fp) return 1; - k->disk_fp=fopen(k->disk_path,"r+b"); - if(!k->disk_fp){ /* not there yet -> create + header */ - k->disk_fp=fopen(k->disk_path,"wb"); - if(!k->disk_fp) return 0; - int32_t h[8]; kv_hdr(m,h,0); - fwrite(KV_MAGIC,1,8,k->disk_fp); fwrite(h,4,8,k->disk_fp); - fflush(k->disk_fp); - fclose(k->disk_fp); - k->disk_fp=fopen(k->disk_path,"r+b"); /* reopen r+b for append */ - if(!k->disk_fp) return 0; - } - return 1; -} -static void kv_disk_truncate(Model *m, int nrec){ - if(!g_kvsave) return; - KVState *k=m->kv; - if(k->disk_fp){ fclose(k->disk_fp); k->disk_fp=NULL; } /* drop to shrink on disc */ - FILE *f=fopen(k->disk_path,"r+b"); - if(!f){ k->disk_nrec=0; return; } - k->disk_nrec=nrec; - int32_t nr=nrec; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); - fflush(f); fclose(f); -} -static void kv_disk_reset(Model *m){ kv_disk_truncate(m,0); } -static void kv_disk_append(Model *m, const int *hist, int len){ - KVState *k=m->kv; - if(!g_kvsave || len<=k->disk_nrec) return; - Cfg *c=&m->c; - if(!kv_disk_open(m)) return; - FILE *f=k->disk_fp; - int64_t rec = kv_rec_bytes(m); - /* grow the contiguous staging buffer if the record is larger (#1 batching) */ - if(rec > k->disk_buf_cap){ - uint8_t *nb=realloc(k->disk_buf, rec); - if(!nb) return; /* OOM: skip this turn, retry next */ - k->disk_buf=nb; k->disk_buf_cap=rec; - } - fseek(f, 8+8*4 + (int64_t)k->disk_nrec*rec, SEEK_SET); - for(int p=k->disk_nrec;pdisk_buf; /* pack token + every layer into one record */ - *(int32_t*)b = hist[p]; b+=4; - for(int i=0;in_layers;i++){ - memcpy(b, m->Lc[i]+(int64_t)p*c->kv_lora, (size_t)c->kv_lora*4); b+=c->kv_lora*4; - memcpy(b, m->Rc[i]+(int64_t)p*c->qk_rope,(size_t)c->qk_rope*4); b+=c->qk_rope*4; - } - if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]){ - memcpy(b, m->Ic[i]+(int64_t)p*c->index_hd, (size_t)c->index_hd*4); b+=c->index_hd*4; - } - fwrite(k->disk_buf, 1, (size_t)rec, f); /* one fwrite per position (was ~157) */ - } - fflush(f); /* dati prima, contatore poi */ - int32_t nr=len; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); - fflush(f); /* persist the counter too */ - k->disk_nrec=len; -} -static int kv_disk_load(Model *m, int *hist, int maxctx){ - if(!g_kvsave) return 0; - KVState *k=m->kv; - Cfg *c=&m->c; - FILE *f=fopen(k->disk_path,"rb"); if(!f) return 0; - char mg[8]; int32_t h[8], w[8]; kv_hdr(m,w,0); - if(fread(mg,1,8,f)!=8 || memcmp(mg,KV_MAGIC,8) || fread(h,4,8,f)!=8 || - h[0]!=w[0]||h[1]!=w[1]||h[2]!=w[2]||h[3]!=w[3]||h[4]!=w[4]||h[5]!=w[5]){ - fprintf(stderr,"[KV] ignoring .coli_kv from a different model or version\n"); fclose(f); return 0; } - int nrec=h[6]; - if(nrec<1){ fclose(f); return 0; } - if(nrec>=maxctx-8-g_draft){ - fprintf(stderr,"[KV] saved conversation (%d tokens) exceeds the context: starting over\n",nrec); - fclose(f); return 0; } - double t0=now_s(); - for(int p=0;pn_layers;i++){ - if(fread(m->Lc[i]+(int64_t)p*c->kv_lora, 4, c->kv_lora, f)!=(size_t)c->kv_lora || - fread(m->Rc[i]+(int64_t)p*c->qk_rope, 4, c->qk_rope, f)!=(size_t)c->qk_rope){ nrec=p; goto out; } - } - if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) - if(fread(m->Ic[i]+(int64_t)p*c->index_hd, 4, c->index_hd, f)!=(size_t)c->index_hd){ nrec=p; goto out; } - } -out: - fclose(f); - if(nrec>0){ - if(m->has_mtp) m->kv_start[c->n_layers]=-1; /* la finestra MTP riparte da sola */ - fprintf(stderr,"[KV] resumed conversation from disk: %d tokens in %.1fs (no re-prefill)\n", - nrec, now_s()-t0); - } - k->disk_nrec=nrec; - return nrec; -} typedef struct { KVState kv; int *hist, len, first; } ServeCtx; static double kv_pool_bytes(Model *m, int max_ctx); @@ -5622,212 +4636,7 @@ static int *read_arr(jval*o,const char*k,int*n){ if(!r){ fprintf(stderr,"OOM read_arr\n"); exit(1); } for(int i=0;ilen;i++) r[i]=(int)a->kids[i]->num; *n=a->len; return r; } -/* byte residenti di un tensore [O,I] al numero di bit dato (specchio di qt_bytes) */ -static int64_t tbytes(int O,int I,int bits){ - if(bits>=16) return (int64_t)O*I*4; - if(bits>=5) return (int64_t)O*I + (int64_t)O*4; - return (int64_t)O*((I+1)/2) + (int64_t)O*4; -} -/* byte VERI di un expert: dal container se pre-quantizzato, altrimenti stima da ebits */ -static int64_t expert_bytes_probe(Model *m, int ebits){ - Cfg *c=&m->c; int64_t eb=0; char nm[256]; - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.gate_proj.weight",c->first_dense); - if(st_nbytes(&m->S,nm)>0){ - const char *suf[3]={"gate_proj","up_proj","down_proj"}; - for(int k=0;k<3;k++){ - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight",c->first_dense,suf[k]); - eb+=st_nbytes(&m->S,nm); - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight.qs",c->first_dense,suf[k]); - int64_t q=st_nbytes(&m->S,nm); if(q>0) eb+=q; - } - } - if(eb<=0) eb = tbytes(c->moe_inter,c->hidden,ebits)*2 + tbytes(c->hidden,c->moe_inter,ebits); - return eb; -} - -/* TIERS: fotografia della piramide expert per la dashboard web — - * "TIERS " sul canale di protocollo. - * ram = pinnati non-VRAM + LRU corrente; disk = tutto il resto. */ -/* BRAIN MAP (dashboard): per-turn expert hit bitmap + full residency/heat map. - * g_ehit[layer][eid]=1 quando l'expert viene instradato in questo turno; - * hits_emit lo serializza e lo azzera. emap_emit fotografa tier+heat di TUTTI. */ -static uint8_t **g_ehit; -static void ehit_mark(Model *m, int layer, int eid){ - if(!g_ehit){ Cfg *c=&m->c; - g_ehit=calloc(c->n_layers+1,sizeof(uint8_t*)); - for(int i=0;i<=c->n_layers;i++) g_ehit[i]=calloc(c->n_experts,1); - } - g_ehit[layer][eid]=1; -} - -/* HWINFO: hardware snapshot for the web dashboard — emitted once at READY. */ -/* CPU model + cores + RAM (GB); empty/zero where unavailable. - * Shared by the dashboard HWINFO line and the PROF=1 header. */ -static void hw_probe(char *cpu, size_t cn, int *cores, double *ram_total, double *ram_avail){ - cpu[0]=0; -#ifdef _WIN32 - /* niente /proc su Windows: brand string via CPUID (0x80000002..4), zero - * dipendenze extra. La dashboard mostrava "0 GB RAM / 0 cores" perche' - * tutto questo blocco era solo-Linux mentre il ramo CUDA funzionava. */ -#if defined(__x86_64__) || defined(__i386__) - { unsigned int r[12]={0}; unsigned int *w=r; - for(unsigned int f=0x80000002u; f<=0x80000004u; f++,w+=4) - __get_cpuid(f,&w[0],&w[1],&w[2],&w[3]); - char *b=(char*)r; b[47]=0; while(*b==' ')b++; - snprintf(cpu,cn,"%s",b); } -#endif -#else - FILE *ci=fopen("/proc/cpuinfo","r"); - if(ci){ char ln[256]; - while(fgets(ln,sizeof(ln),ci)) if(!strncmp(ln,"model name",10)){ - char *p=strchr(ln,':'); if(p){ p++; while(*p==' ')p++; - int n=(int)strlen(p); if(n>0&&p[n-1]=='\n')p[--n]=0; - snprintf(cpu,cn,"%s",p); } break; } - fclose(ci); } -#endif - *cores=0; -#ifdef _WIN32 - { SYSTEM_INFO si; GetSystemInfo(&si); *cores=(int)si.dwNumberOfProcessors; } -#elif defined(_SC_NPROCESSORS_ONLN) - *cores=(int)sysconf(_SC_NPROCESSORS_ONLN); -#endif - *ram_total=*ram_avail=0; -#ifdef _WIN32 - compat_meminfo(ram_total,ram_avail); /* GlobalMemoryStatusEx, gia' in compat.h */ -#else - FILE *mi=fopen("/proc/meminfo","r"); - if(mi){ char ln[256]; double mt=0,ma=0; - while(fgets(ln,sizeof(ln),mi)){ - if(sscanf(ln,"MemTotal: %lf",&mt)==1) *ram_total=mt/1e6; - if(sscanf(ln,"MemAvailable: %lf",&ma)==1) *ram_avail=ma/1e6; - } fclose(mi); } -#endif -} - -static void hwinfo_emit(Model *m){ - Cfg *c=&m->c; (void)c; /* silence -Wunused on builds without /proc (#148 report) */ - char cpu[256]; int cores; double ram_total,ram_avail; - hw_probe(cpu,sizeof(cpu),&cores,&ram_total,&ram_avail); - /* GPU */ - int ngpu=0; double vram_total=0; - char gpu_name[128]=""; -#ifdef COLI_CUDA - ngpu=g_cuda_ndev; vram_total=m->gpu_expert_bytes/1e9; - for(int i=0;i0){ - /* We don't have a device-name API; parse from the init log line stored in stderr. - * Simpler: just read the nvidia driver sysfs or use a fixed label. */ - snprintf(gpu_name,sizeof(gpu_name),"CUDA device x%d",g_cuda_ndev); - } -#endif - printf("HWINFO %d %.1f %.1f %d %.1f %s|%s\n", - cores,ram_total,ram_avail,ngpu,vram_total,cpu,gpu_name); - fflush(stdout); -} - -static void tiers_emit(Model *m){ - Cfg *c=&m->c; int nsp=0; - for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; - int total=(nsp+(m->has_mtp?1:0))*c->n_experts; - int pinned=0,lru=0; - for(int i=0;i<=c->n_layers;i++){ pinned+=m->npin?m->npin[i]:0; lru+=m->ecn?m->ecn[i]:0; } - int vram=0; double vram_gb=0; -#ifdef COLI_CUDA - vram=m->gpu_expert_count; vram_gb=m->gpu_expert_bytes/1e9; -#endif - int ram=pinned-vram+lru; if(ram<0) ram=0; - int disk=total-vram-ram; if(disk<0) disk=0; - double eb=(double)expert_bytes_probe(m,m->ebits); - printf("TIERS %d %d %d %.2f %.2f\n",vram,ram,disk,vram_gb,ram*eb/1e9); - fflush(stdout); -} - -/* EMAP: 1 byte/expert (2bit tier: 0=disk 1=RAM 2=VRAM | 6bit heat log2-bucket), - * righe = layer sparsi in ordine (+MTP se presente), colonne = n_experts. Hex. */ -static void emap_emit(Model *m){ - Cfg *c=&m->c; - int rows=0; - for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; - int has_mtp = m->has_mtp && m->eusage[c->n_layers]; - if(has_mtp) rows++; - int cols=c->n_experts; - char *hex=malloc((size_t)rows*cols*2+1); int w=0; - for(int i=0;i<=c->n_layers;i++){ - int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); - if(!is_row) continue; - for(int e=0;epin[i]; - for(int z=0;znpin[i];z++) if(P[z].eid==e){ -#ifdef COLI_CUDA - tier = P[z].g.cuda?2:1; -#else - tier = 1; -#endif - break; } - if(!tier && m->ecache && m->ecache[i]) - for(int z=0;zecn[i];z++) if(m->ecache[i][z].eid==e){ tier=1; break; } - uint32_t u = m->eusage[i]?m->eusage[i][e]:0; - int heat=0; while(u){ heat++; u>>=1; } if(heat>63) heat=63; - int b=(tier<<6)|heat; - hex[w++]="0123456789abcdef"[b>>4]; hex[w++]="0123456789abcdef"[b&15]; - } - } - hex[w]=0; - printf("EMAP %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); -} - -/* HITS: bitmap 1bit/expert (stesso ordine di EMAP), poi azzera per il turno dopo. */ -static void hits_emit(Model *m){ - Cfg *c=&m->c; if(!g_ehit) return; - int rows=0; - for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; - int has_mtp = m->has_mtp && m->eusage[c->n_layers]; - if(has_mtp) rows++; - int cols=c->n_experts, nb=(rows*cols+7)/8; - uint8_t *bm=calloc(nb,1); int bit=0; - for(int i=0;i<=c->n_layers;i++){ - int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); - if(!is_row) continue; - for(int e=0;e>3]|=1<<(bit&7); g_ehit[i][e]=0; } - } - char *hex=malloc((size_t)nb*2+1); int w=0; - for(int b=0;b>4]; hex[w++]="0123456789abcdef"[bm[b]&15]; } - hex[w]=0; - printf("HITS %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); free(bm); -} - -/* scarica su file l'istogramma d'uso degli expert: righe "layer eid count" (per PIN). - * Include la riga MTP (layer n_layers). Scrittura atomica (tmp+rename): viene chiamata - * anche a ogni turno di serve e il processo puo' morire in qualsiasi momento. */ -static void stats_dump_q(Model *m, const char *path, int quiet){ - char tmp[2100]; snprintf(tmp,sizeof(tmp),"%s.tmp",path); - FILE *f=fopen(tmp,"w"); if(!f){ if(!quiet) perror(tmp); return; } - Cfg *c=&m->c; int64_t tot=0, nz=0; - for(int i=0;i<=c->n_layers;i++){ if(!m->eusage[i]) continue; - for(int e=0;en_experts;e++) if(m->eusage[i][e]){ fprintf(f,"%d %d %u\n",i,e,m->eusage[i][e]); tot+=m->eusage[i][e]; nz++; } } - fclose(f); rename(tmp,path); - if(!quiet) fprintf(stderr,"[STATS] %lld selections across %lld distinct experts -> %s\n",(long long)tot,(long long)nz,path); -} -static void stats_dump(Model *m, const char *path){ stats_dump_q(m,path,0); } - -/* CACHE CHE IMPARA: istogramma d'uso PERSISTENTE in /.coli_usage. - * Caricato all'avvio (i contatori ripartono dalla storia), salvato a ogni turno: - * piu' usi colibri', meglio l'auto-pin conosce i TUOI expert caldi. */ -static char g_usage_path[2100]=""; -static int64_t usage_load(Model *m, const char *path){ - FILE *f=fopen(path,"r"); if(!f) return 0; - Cfg *c=&m->c; int l,e; uint32_t cnt; int64_t tot=0; - while(fscanf(f,"%d %d %u",&l,&e,&cnt)==3) - if(l>=0&&l<=c->n_layers&&e>=0&&en_experts&&m->eusage[l]){ m->eusage[l][e]+=cnt; tot+=cnt; } - fclose(f); return tot; -} -static void usage_save(Model *m){ if(g_usage_path[0]) stats_dump_q(m,g_usage_path,1); } +/* telemetry, stats, usage persistence — moved to telemetry.h */ /* HOT-STORE ("il redis del colibri'"): carica in RAM, UNA VOLTA e per sempre, i top expert * per frequenza d'uso misurata (file STATS di un run precedente), entro un budget in GB. diff --git a/c/kv_persist.h b/c/kv_persist.h new file mode 100644 index 0000000..51289a8 --- /dev/null +++ b/c/kv_persist.h @@ -0,0 +1,121 @@ +/* kv_persist.h — .coli_kv on-disk KV cache persistence. + * Conversations reopen warm across engine restarts: the compressed MLA KV-cache + * is appended incrementally after every turn, crash-safe (nrec written last). + * Include after Model/KVState/Cfg are defined; requires now_s() and g_draft. */ +#ifndef KV_PERSIST_H +#define KV_PERSIST_H + +static int g_kvsave=1; +#define KV_MAGIC "COLIKV1\0" + +static void kv_hdr(Model *m, int32_t *h, int nrec){ + Cfg *c=&m->c; int nic=0; + for(int i=0;in_layers;i++) if(m->Ic && m->Ic[i]) nic++; + h[0]=c->n_layers; h[1]=c->kv_lora; h[2]=c->qk_rope; + h[3]=m->has_dsa?c->index_hd:0; h[4]=nic; h[5]=c->vocab; h[6]=nrec; h[7]=0; +} + +static int64_t kv_rec_bytes(Model *m){ + Cfg *c=&m->c; + int64_t rec = 4 + (int64_t)c->n_layers*(c->kv_lora+c->qk_rope)*4; + if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) rec+=(int64_t)c->index_hd*4; + return rec; +} + +static int kv_disk_open(Model *m){ + KVState *k=m->kv; + if(k->disk_fp) return 1; + k->disk_fp=fopen(k->disk_path,"r+b"); + if(!k->disk_fp){ + k->disk_fp=fopen(k->disk_path,"wb"); + if(!k->disk_fp) return 0; + int32_t h[8]; kv_hdr(m,h,0); + fwrite(KV_MAGIC,1,8,k->disk_fp); fwrite(h,4,8,k->disk_fp); + fflush(k->disk_fp); + fclose(k->disk_fp); + k->disk_fp=fopen(k->disk_path,"r+b"); + if(!k->disk_fp) return 0; + } + return 1; +} + +static void kv_disk_truncate(Model *m, int nrec){ + if(!g_kvsave) return; + KVState *k=m->kv; + if(k->disk_fp){ fclose(k->disk_fp); k->disk_fp=NULL; } + FILE *f=fopen(k->disk_path,"r+b"); + if(!f){ k->disk_nrec=0; return; } + k->disk_nrec=nrec; + int32_t nr=nrec; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); + fflush(f); fclose(f); +} + +static void kv_disk_reset(Model *m){ kv_disk_truncate(m,0); } + +static void kv_disk_append(Model *m, const int *hist, int len){ + KVState *k=m->kv; + if(!g_kvsave || len<=k->disk_nrec) return; + Cfg *c=&m->c; + if(!kv_disk_open(m)) return; + FILE *f=k->disk_fp; + int64_t rec = kv_rec_bytes(m); + if(rec > k->disk_buf_cap){ + uint8_t *nb=realloc(k->disk_buf, rec); + if(!nb) return; + k->disk_buf=nb; k->disk_buf_cap=rec; + } + fseek(f, 8+8*4 + (int64_t)k->disk_nrec*rec, SEEK_SET); + for(int p=k->disk_nrec;pdisk_buf; + *(int32_t*)b = hist[p]; b+=4; + for(int i=0;in_layers;i++){ + memcpy(b, m->Lc[i]+(int64_t)p*c->kv_lora, (size_t)c->kv_lora*4); b+=c->kv_lora*4; + memcpy(b, m->Rc[i]+(int64_t)p*c->qk_rope,(size_t)c->qk_rope*4); b+=c->qk_rope*4; + } + if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]){ + memcpy(b, m->Ic[i]+(int64_t)p*c->index_hd, (size_t)c->index_hd*4); b+=c->index_hd*4; + } + fwrite(k->disk_buf, 1, (size_t)rec, f); + } + fflush(f); + int32_t nr=len; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); + fflush(f); + k->disk_nrec=len; +} + +static int kv_disk_load(Model *m, int *hist, int maxctx){ + if(!g_kvsave) return 0; + KVState *k=m->kv; + Cfg *c=&m->c; + FILE *f=fopen(k->disk_path,"rb"); if(!f) return 0; + char mg[8]; int32_t h[8], w[8]; kv_hdr(m,w,0); + if(fread(mg,1,8,f)!=8 || memcmp(mg,KV_MAGIC,8) || fread(h,4,8,f)!=8 || + h[0]!=w[0]||h[1]!=w[1]||h[2]!=w[2]||h[3]!=w[3]||h[4]!=w[4]||h[5]!=w[5]){ + fprintf(stderr,"[KV] ignoring .coli_kv from a different model or version\n"); fclose(f); return 0; } + int nrec=h[6]; + if(nrec<1){ fclose(f); return 0; } + if(nrec>=maxctx-8-g_draft){ + fprintf(stderr,"[KV] saved conversation (%d tokens) exceeds the context: starting over\n",nrec); + fclose(f); return 0; } + double t0=now_s(); + for(int p=0;pn_layers;i++){ + if(fread(m->Lc[i]+(int64_t)p*c->kv_lora, 4, c->kv_lora, f)!=(size_t)c->kv_lora || + fread(m->Rc[i]+(int64_t)p*c->qk_rope, 4, c->qk_rope, f)!=(size_t)c->qk_rope){ nrec=p; goto out; } + } + if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) + if(fread(m->Ic[i]+(int64_t)p*c->index_hd, 4, c->index_hd, f)!=(size_t)c->index_hd){ nrec=p; goto out; } + } +out: + fclose(f); + if(nrec>0){ + if(m->has_mtp) m->kv_start[c->n_layers]=-1; + fprintf(stderr,"[KV] resumed conversation from disk: %d tokens in %.1fs (no re-prefill)\n", + nrec, now_s()-t0); + } + k->disk_nrec=nrec; + return nrec; +} + +#endif /* KV_PERSIST_H */ diff --git a/c/quant.h b/c/quant.h new file mode 100644 index 0000000..928de96 --- /dev/null +++ b/c/quant.h @@ -0,0 +1,672 @@ +/* quant.h — quantized matmul kernels (header-only, all functions static). + * Multi-architecture SIMD: AVX2 / AVX-512 / AVX-VNNI / ARM NEON / NEON-SDOT / + * NEON-i8mm / POWER VSX. Pure compute — no Model or QT dependency. */ +#ifndef COLI_QUANT_H +#define COLI_QUANT_H + +#include +#include +#include +#include +#include + +#ifdef _OPENMP +#include +#endif + +/* ---- SIMD includes -------------------------------------------------------- */ +#ifdef __AVX2__ +#include +static inline float hsum256(__m256 v){ + __m128 lo=_mm256_castps256_ps128(v), hi=_mm256_extractf128_ps(v,1); + lo=_mm_add_ps(lo,hi); __m128 sh=_mm_movehl_ps(lo,lo); lo=_mm_add_ps(lo,sh); + sh=_mm_shuffle_ps(lo,lo,1); lo=_mm_add_ss(lo,sh); return _mm_cvtss_f32(lo); +} +static inline int hsum256_i32(__m256i v){ + __m128i lo=_mm256_castsi256_si128(v), hi=_mm256_extracti128_si256(v,1); + lo=_mm_add_epi32(lo,hi); lo=_mm_hadd_epi32(lo,lo); lo=_mm_hadd_epi32(lo,lo); + return _mm_cvtsi128_si32(lo); +} +#endif +#if defined(__AVXVNNI__) && defined(__AVX2__) +static inline int hsum128_i32(__m128i v){ + v=_mm_hadd_epi32(v,v); v=_mm_hadd_epi32(v,v); return _mm_cvtsi128_si32(v); +} +#endif +#ifdef __ARM_NEON +#include +#endif +#ifdef __VSX__ +#include +#undef vector +#undef pixel +#undef bool +#endif + +/* ---- AVX-512 int4->float accumulator -------------------------------------- */ +#if defined(__AVX512F__) && defined(__AVX512BW__) +static int g_i4_acc512=1; +static inline float dot_i4f_avx512(const uint8_t *w,const float *x,int I){ + const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8); + __m512 acc0=_mm512_setzero_ps(),acc1=_mm512_setzero_ps(); int i=0; + for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi); + __m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8)); + __m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8)); + acc0=_mm512_fmadd_ps(_mm512_loadu_ps(x+i),w0,acc0); + acc1=_mm512_fmadd_ps(_mm512_loadu_ps(x+i+16),w1,acc1); + } + return _mm512_reduce_add_ps(_mm512_add_ps(acc0,acc1)); +} +static int i4_acc512_selftest(void){ + enum { N=224 }; uint8_t w[(N+1)/2]; float x[N]; + for(int i=0;i>1]=(uint8_t)(q+8); + else w[i>>1]|=(uint8_t)((q+8)<<4); + x[i]=(float)(((i*29+7)%101)-50)/37.f; + } + for(int n=32;n<=N;n+=32){ + float ref=0; for(int i=0;i>1]>>((i&1)*4))&15)-8); + float got=dot_i4f_avx512(w,x,n),tol=2e-5f*(1.f+fabsf(ref)); + if(fabsf(got-ref)>tol){ fprintf(stderr,"AVX512 i4 selftest n=%d: %.9g != %.9g\n",n,got,ref); return 0; } + } + return 1; +} +#endif + +/* ---- y[S,O] = x[S,I] @ W^T, W[O,I] f32 ---------------------------------- */ +static void matmul(float *y, const float *x, const float *W, int S, int I, int O){ + #pragma omp parallel for schedule(static) + for (int o=0;o>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i nib=_mm_unpacklo_epi8(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } + a=hsum256(acc); +#elif defined(__ARM_NEON) + const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); + float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); + for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); + uint8x8x2_t z=vzip_u8(vand_u8(by,m4), vshr_n_u8(by,4)); + int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[0]),b8)); + int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[1]),b8)); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); +#endif +#if defined(__AVX512F__) && defined(__AVX512BW__) + } +#endif + for(;i+1>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8; + a += xs[i]*(float)lo + xs[i+1]*(float)hi; } + if(i>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; } + y[(int64_t)s*O+o]=a*sc; } } +} + +/* ---- y[S,O] = x[S,I] @ W^T, W int4 packed + per-GROUP scales (fmt=4) ----- */ +static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const float *scale, + int S, int I, int O, int gs){ + int rb=(I+1)/2; int ng=(I+gs-1)/gs; + #pragma omp parallel for schedule(static) + for(int o=0;oI) glen=I-base; + float sc=scl[g]; + int i=base; +#ifdef __AVX2__ + const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8); + __m256 acc=_mm256_setzero_ps(); + for(; i+16<=base+glen; i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i nib=_mm_unpacklo_epi8(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } + a+=hsum256(acc)*sc; +#endif + for(; i>1]; + a+=(xs[i]*(float)((int)(byte&0xF)-8)+xs[i+1]*(float)((int)(byte>>4)-8))*sc; } + else { uint8_t byte=w[i>>1]; a+=xs[i]*(float)((int)(byte&0xF)-8)*sc; } + } + } + y[(int64_t)s*O+o]=a; + } + } +} + +/* ---- fused gate+up: one OMP dispatch for both matrices -------------------- */ +static void matmul_i4_pair(float *yg, float *yu, const float *x, + const uint8_t *qg, const float *sg, + const uint8_t *qu, const float *su, int I, int O){ + int rb=(I+1)/2; + #pragma omp parallel for schedule(static) + for(int z=0;z<2*O;z++){ + int o=z>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i nib=_mm_unpacklo_epi8(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); } + a=hsum256(acc); +#elif defined(__ARM_NEON) + const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); + float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0); + for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); + uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4)); + int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8)); + int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8)); + ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); +#endif +#if defined(__AVX512F__) && defined(__AVX512BW__) + } +#endif + for(;i+1>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); } + if(i>1]&15)-8); + (z>2))); + __m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2); + __m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2); + __m128i lo=_mm_unpacklo_epi8(p0,p1), hi=_mm_unpacklo_epi8(p2,p3); + __m128i nib=_mm_unpacklo_epi16(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b2)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b2)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } + a=hsum256(acc); +#elif defined(__ARM_NEON) + const uint8x8_t m2v=vdup_n_u8(3); const int8x8_t b2v=vdup_n_s8(2); + float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); + for(;i+16<=I;i+=16){ uint32_t wd; memcpy(&wd, w+(i>>2), 4); + uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd)); + uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v)); + uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6)); + uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0])); + int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[0]),b2v)); + int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[1]),b2v)); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); +#endif + for(;i>2]; int sh=(i&3)*2; a += xs[i]*(float)((int)((byte>>sh)&3)-2); } + y[(int64_t)s*O+o]=a*sc; } } +} + +/* ---- IDOT: integer dot kernels (int8-quantized activations) --------------- */ +#if defined(__AVX512VNNI__) && defined(__AVX512BW__) +#define IDOT_KERNEL "avx512-vnni" +#elif defined(__AVXVNNI__) && defined(__AVX2__) +#define IDOT_KERNEL "avx-vnni" +#elif defined(__AVX2__) +#define IDOT_KERNEL "avx2" +#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) +#define IDOT_KERNEL "neon-i8mm" +#elif defined(__ARM_NEON) +#define IDOT_KERNEL "neon" +#elif defined(__VSX__) +#define IDOT_KERNEL "vsx" +#else +#define IDOT_KERNEL "scalar" +#endif +static int g_idot=1; +#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) +static int g_i4s=1; +#elif defined(__VSX__) +static int g_i4s=1; +#else +static int g_i4s=2; +#endif + +static inline float qrow_i8(const float *x, int8_t *q, int I){ + float amax=0; for(int i=0;iamax)amax=a; } + float s=amax/127.f; if(s<1e-12f) s=1e-12f; float inv=1.f/s; + for(int i=0;i>1))); + __m256i lo=_mm256_and_si256(by,m4v), hi=_mm256_and_si256(_mm256_srli_epi16(by,4),m4v); + __m256i z0=_mm256_unpacklo_epi8(lo,hi), z1=_mm256_unpackhi_epi8(lo,hi); + __m512i wv=_mm512_sub_epi8(_mm512_inserti64x4(_mm512_castsi256_si512(z0),z1,1),b8v); + __m512i xv=_mm512_permutexvar_epi64(xidx,_mm512_loadu_si512((const void*)(x+i))); + __mmask64 neg=_mm512_movepi8_mask(wv); + __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv); + acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs); + } + sum=_mm512_reduce_add_epi32(acc); +#elif defined(__AVXVNNI__) && defined(__AVX2__) + const __m128i m4=_mm_set1_epi8(0x0F); const __m128i b8=_mm_set1_epi8(8); + __m128i acc=_mm_setzero_si128(); + for(;i+32<=I;i+=32){ + __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); + __m128i w0=_mm_sub_epi8(n0,b8), w1=_mm_sub_epi8(n1,b8); + __m128i x0=_mm_loadu_si128((const __m128i*)(x+i)); + __m128i x1=_mm_loadu_si128((const __m128i*)(x+i+16)); + acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0)); + acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1)); + } + sum=hsum128_i32(acc); +#elif defined(__AVX2__) + const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi8(8); + const __m256i ones=_mm256_set1_epi16(1); + __m256i acc=_mm256_setzero_si256(); + for(;i+32<=I;i+=32){ + __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); + __m256i wv=_mm256_sub_epi8(_mm256_set_m128i(n1,n0),b8); + __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); + __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); + acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); + } + sum=hsum256_i32(acc); +#elif defined(__ARM_NEON) + const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8); +#if defined(__ARM_FEATURE_DOTPROD) + int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); + for(;i+64<=I;i+=64){ + uint8x16_t byA=vld1q_u8(w4+(i>>1)), byB=vld1q_u8(w4+(i>>1)+16); + uint8x16x2_t zA=vzipq_u8(vandq_u8(byA,m4q), vshrq_n_u8(byA,4)); + uint8x16x2_t zB=vzipq_u8(vandq_u8(byB,m4q), vshrq_n_u8(byB,4)); + a0=vdotq_s32(a0,vsubq_s8(vreinterpretq_s8_u8(zA.val[0]),b8q),vld1q_s8(x+i)); + a1=vdotq_s32(a1,vsubq_s8(vreinterpretq_s8_u8(zA.val[1]),b8q),vld1q_s8(x+i+16)); + a2=vdotq_s32(a2,vsubq_s8(vreinterpretq_s8_u8(zB.val[0]),b8q),vld1q_s8(x+i+32)); + a3=vdotq_s32(a3,vsubq_s8(vreinterpretq_s8_u8(zB.val[1]),b8q),vld1q_s8(x+i+48)); + } + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + for(;i+32<=I;i+=32){ + uint8x16_t by=vld1q_u8(w4+(i>>1)); + uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); + acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q),vld1q_s8(x+i)); + acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q),vld1q_s8(x+i+16)); + } + sum=vaddvq_s32(acc); +#else + int32x4_t acc=vdupq_n_s32(0); + for(;i+32<=I;i+=32){ + uint8x16_t by=vld1q_u8(w4+(i>>1)); + uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); + int8x16_t w0=vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q); + int8x16_t w1=vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q); + int8x16_t x0=vld1q_s8(x+i), x1=vld1q_s8(x+i+16); + int16x8_t p=vmull_s8(vget_low_s8(w0),vget_low_s8(x0)); + p=vmlal_s8(p,vget_high_s8(w0),vget_high_s8(x0)); + acc=vpadalq_s16(acc,p); + p=vmull_s8(vget_low_s8(w1),vget_low_s8(x1)); + p=vmlal_s8(p,vget_high_s8(w1),vget_high_s8(x1)); + acc=vpadalq_s16(acc,p); + } + sum=vaddvq_s32(acc); +#endif +#elif defined(__VSX__) + const __vector unsigned char m4v=vec_splats((unsigned char)0x0F); + const __vector unsigned char sh4=vec_splats((unsigned char)4); + const __vector signed char b8v=vec_splats((signed char)8); + const __vector signed char vz=vec_splats((signed char)0); + __vector signed int acc=vec_splats(0); + for(;i+32<=I;i+=32){ + __vector unsigned char by=vec_xl(0,w4+(i>>1)); + __vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4); + __vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v); + __vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v); + __vector signed char x0=vec_xl(0,(const signed char*)(x+i)); + __vector signed char x1=vec_xl(0,(const signed char*)(x+i+16)); + __vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz); + acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0), + (__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc); + acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1), + (__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc); + } + sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); +#endif + for(;i+1>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } + if(i>1]; sum+=((int)(b&0xF)-8)*x[i]; } + return sum; +} + +/* ---- ARM i8mm SMMLA tiled kernels ---------------------------------------- */ +#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) +static inline int32x4_t mm_tile16(int32x4_t acc, int8x16_t wo, int8x16_t wo1, + int8x16_t xs, int8x16_t xs1){ + acc=vmmlaq_s32(acc, vcombine_s8(vget_low_s8(wo), vget_low_s8(wo1)), + vcombine_s8(vget_low_s8(xs), vget_low_s8(xs1))); + return vmmlaq_s32(acc, vcombine_s8(vget_high_s8(wo), vget_high_s8(wo1)), + vcombine_s8(vget_high_s8(xs), vget_high_s8(xs1))); +} +static void matmul_q_idot_mm(float *y, const int8_t *xq, const float *sx, const int8_t *q, + const float *scale, int S, int I, int O){ + #pragma omp parallel for schedule(static) + for(int o=0;o<(O&~1);o+=2){ + const int8_t *wo=q+(int64_t)o*I, *wo1=q+(int64_t)(o+1)*I; + float sc0=scale[o], sc1=scale[o+1]; + for(int s=0;s<(S&~1);s+=2){ + const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I; + int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0; + for(;i+64<=I;i+=64){ + a0=mm_tile16(a0,vld1q_s8(wo+i), vld1q_s8(wo1+i), vld1q_s8(xs+i), vld1q_s8(xs1+i)); + a1=mm_tile16(a1,vld1q_s8(wo+i+16),vld1q_s8(wo1+i+16),vld1q_s8(xs+i+16),vld1q_s8(xs1+i+16)); + a2=mm_tile16(a2,vld1q_s8(wo+i+32),vld1q_s8(wo1+i+32),vld1q_s8(xs+i+32),vld1q_s8(xs1+i+32)); + a3=mm_tile16(a3,vld1q_s8(wo+i+48),vld1q_s8(wo1+i+48),vld1q_s8(xs+i+48),vld1q_s8(xs1+i+48)); + } + for(;i+16<=I;i+=16) + a0=mm_tile16(a0,vld1q_s8(wo+i),vld1q_s8(wo1+i),vld1q_s8(xs+i),vld1q_s8(xs1+i)); + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); + int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); + for(;i>1)), byo1=vld1q_u8(wo1+(i>>1)); + uint8x16_t cyo=vld1q_u8(wo+(i>>1)+16), cyo1=vld1q_u8(wo1+(i>>1)+16); + uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); + uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); + uint8x16x2_t ko =vzipq_u8(vandq_u8(cyo, m4q), vshrq_n_u8(cyo, 4)); + uint8x16x2_t ko1=vzipq_u8(vandq_u8(cyo1,m4q), vshrq_n_u8(cyo1,4)); + a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), + vld1q_s8(xs+i), vld1q_s8(xs1+i)); + a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), + vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); + a2=mm_tile16(a2, vsubq_s8(vreinterpretq_s8_u8(ko.val[0]),b8q), + vsubq_s8(vreinterpretq_s8_u8(ko1.val[0]),b8q), + vld1q_s8(xs+i+32), vld1q_s8(xs1+i+32)); + a3=mm_tile16(a3, vsubq_s8(vreinterpretq_s8_u8(ko.val[1]),b8q), + vsubq_s8(vreinterpretq_s8_u8(ko1.val[1]),b8q), + vld1q_s8(xs+i+48), vld1q_s8(xs1+i+48)); + } + for(;i+32<=I;i+=32){ + uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1)); + uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); + uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); + a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), + vld1q_s8(xs+i), vld1q_s8(xs1+i)); + a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), + vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); + } + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); + int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); + for(;i+1>1], bo1=wo1[i>>1]; + int a0=(int)(bo&0xF)-8, a1=(int)(bo>>4)-8, b0=(int)(bo1&0xF)-8, b1=(int)(bo1>>4)-8; + int u0=xs[i],u1=xs[i+1],v0=xs1[i],v1=xs1[i+1]; + d00+=a0*u0+a1*u1; d01+=a0*v0+a1*v1; d10+=b0*u0+b1*u1; d11+=b0*v0+b1*v1; } + if(i>1], bo1=wo1[i>>1]; + int a0=(int)(bo&0xF)-8, b0=(int)(bo1&0xF)-8; + d00+=a0*xs[i]; d01+=a0*xs1[i]; d10+=b0*xs[i]; d11+=b0*xs1[i]; } + y[(int64_t)s*O+o] =(float)d00*sc0*sx[s]; + y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s]; + y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1]; + y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1]; + } + if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I; + y[(int64_t)s*O+o] =(float)dot_i4i8(wo, xs,I)*sc0*sx[s]; + y[(int64_t)s*O+(o+1)]=(float)dot_i4i8(wo1,xs,I)*sc1*sx[s]; } + } + if(O&1){ int o=O-1; const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o]; + #pragma omp parallel for schedule(static) + for(int s=0;s=2){ matmul_q_idot_mm(y,xq,sx,q,scale,S,I,O); return; } +#endif + #pragma omp parallel for schedule(static) + for(int o=0;o=2){ matmul_i4_idot_mm(y,xq,sx,q4,scale,S,I,O); return; } +#endif + #pragma omp parallel for schedule(static) + for(int o=0;og_qscratch.xq_cap){ + int8_t *p=realloc(g_qscratch.xq,xn); + if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); } + g_qscratch.xq=p; g_qscratch.xq_cap=xn; + } + if(sn>g_qscratch.sx_cap){ + float *p=realloc(g_qscratch.sx,sn*sizeof(float)); + if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); } + g_qscratch.sx=p; g_qscratch.sx_cap=sn; + } + *xq=g_qscratch.xq; *sx=g_qscratch.sx; +} + +/* ---- f32 -> quantized packing --------------------------------------------- */ +static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits){ + int qmax=(1<<(bits-1))-1; + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; + int8_t *qr=q+(int64_t)o*I; + for(int i=0;iqmax)v=qmax; if(v<-qmax-1)v=-qmax-1; qr[i]=(int8_t)v; } + } +} +static void pack_int4(const float *w, uint8_t *q4, float *scale, int O, int I, int bits){ + int qmax=(1<<(bits-1))-1, rb=(I+1)/2; + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; + uint8_t *qr=q4+(int64_t)o*rb; + for(int i=0;iqmax)v0=qmax; if(v0<-8)v0=-8; + int v1=0; if(i+1qmax)v1=qmax; if(v1<-8)v1=-8; } + qr[i>>1] = (uint8_t)((v0+8) | ((v1+8)<<4)); + } + } +} +static void pack_int2(const float *w, uint8_t *q2, float *scale, int O, int I, int bits){ + int qmax=(1<<(bits-1))-1, rb=(I+3)/4; + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; + uint8_t *qr=q2+(int64_t)o*rb; + for(int i=0;iqmax)v=qmax; if(v<-2)v=-2; byte|=(uint8_t)((v+2)<<(k*2)); } + qr[i>>2]=byte; + } + } +} + +#endif /* COLI_QUANT_H */ diff --git a/c/sample.h b/c/sample.h new file mode 100644 index 0000000..abce88c --- /dev/null +++ b/c/sample.h @@ -0,0 +1,143 @@ +/* sample.h — sampling (temperature + nucleus) and stop-set management. + * Header-only: all functions are static — include from the main engine file. */ +#ifndef SAMPLE_H +#define SAMPLE_H + +#include +#include +#include +#include "tok.h" + +/* ---- RNG (xorshift64*) -------------------------------------------------- */ +static uint64_t g_rng = 0x9E3779B97F4A7C15ULL; +static inline double rndu(void){ + g_rng ^= g_rng << 13; g_rng ^= g_rng >> 7; g_rng ^= g_rng << 17; + return (double)(g_rng >> 11) * (1.0 / 9007199254740992.0); +} + +/* ---- argmax over a float vector ----------------------------------------- */ +static inline int argmax_v(const float *lo, int V){ + int b = 0; float bv = lo[0]; + for (int i = 1; i < V; i++) if (lo[i] > bv) { bv = lo[i]; b = i; } + return b; +} + +/* ---- distribution buffers (reused, single-threaded decode) --------------- */ +static float *g_pbuf = NULL; +static int *g_pidx = NULL; + +/* sift-down on max-heap in h[0..n), key = g_pbuf[h[i]] (#335: partial top-p). + * "hole" variant: carries the root value and deposits only at the end, so + * heapify is O(V) and each pop is O(log n) without qsort on the full vocab. */ +static void topp_siftdown(int *h, int n, int i){ + int iv = h[i]; float kv = g_pbuf[iv]; + for (;;) { + int l = 2*i + 1; + if (l >= n) break; + int b = l; if (l+1 < n && g_pbuf[h[l+1]] > g_pbuf[h[l]]) b = l+1; + if (g_pbuf[h[b]] <= kv) break; + h[i] = h[b]; i = b; + } + h[i] = iv; +} + +/* build the target distribution in g_pbuf: softmax(lo/temp) truncated to + * top-p g_nuc. Invariant: g_pbuf stays indexed by token-id (never reordered); + * the truncated tail is zeroed (dist_sample reads by id directly). + * Requires: g_temp, g_nuc, falloc() — declared in the main engine file. */ +static void dist_build(const float *lo, int V){ + if (!g_pbuf) { g_pbuf = falloc(V); g_pidx = malloc(V * sizeof(int)); } + int mxi = -1; float mx = 0; + for (int i = 0; i < V; i++) + if (isfinite(lo[i]) && (mxi < 0 || lo[i] > mx)) { mx = lo[i]; mxi = i; } + double s = 0; float invt = 1.f / (g_temp > 1e-4f ? g_temp : 1e-4f); + if (mxi >= 0) { + for (int i = 0; i < V; i++) { + g_pbuf[i] = isfinite(lo[i]) ? expf((lo[i] - mx) * invt) : 0.f; + s += g_pbuf[i]; + } + } + if (mxi < 0 || !isfinite(s) || s <= 0.0) { + static int warned = 0; + if (!warned) { warned = 1; fprintf(stderr, + "[SAMPLE] warning: non-finite logits (NaN/Inf) — falling back to argmax; " + "output may be degraded. This usually means a numerical blow-up upstream.\n"); } + int a = (mxi >= 0) ? mxi : 0; + for (int i = 0; i < V; i++) g_pbuf[i] = 0.f; + g_pbuf[a] = 1.f; + return; + } + for (int i = 0; i < V; i++) g_pbuf[i] /= (float)s; + if (g_nuc > 0 && g_nuc < 1.f) { + for (int i = 0; i < V; i++) g_pidx[i] = i; + for (int i = V/2-1; i >= 0; i--) topp_siftdown(g_pidx, V, i); + double s2 = 0, cum = 0; int out = V; + do { + int root = g_pidx[0]; + g_pidx[0] = g_pidx[--out]; g_pidx[out] = root; + s2 += g_pbuf[root]; cum += g_pbuf[root]; + if (out > 0) topp_siftdown(g_pidx, out, 0); + } while (cum < g_nuc && out > 0); + for (int i = 0; i < out; i++) g_pbuf[g_pidx[i]] = 0; + float s2f = (float)s2; + for (int i = out; i < V; i++) g_pbuf[g_pidx[i]] /= s2f; + } +} + +/* sample from g_pbuf; ban>=0 excludes that token (renormalizing on the fly) */ +static int dist_sample(int V, int ban){ + double z = 1.0 - (ban >= 0 ? g_pbuf[ban] : 0.0); + if (z <= 1e-12) z = 1e-12; + double u = rndu() * z, cum = 0; + for (int i = 0; i < V; i++) { if (i == ban) continue; cum += g_pbuf[i]; if (cum >= u) return i; } + for (int i = V-1; i >= 0; i--) if (i != ban && g_pbuf[i] > 0) return i; + return 0; +} + +/* next token from logits: greedy if g_temp<=0, sampling otherwise. + * ban = token excluded because it was rejected by speculative verification. */ +static int pick_tok(const float *lo, int V, int ban){ + if (g_temp <= 0) return argmax_v(lo, V); + dist_build(lo, V); + return dist_sample(V, ban); +} + +/* ---- stop set ----------------------------------------------------------- */ +static int g_stop[64], g_nstop = 0; +static inline int is_stop(int t){ + for (int i = 0; i < g_nstop; i++) if (t == g_stop[i]) return 1; + return 0; +} +/* T=NULL -> config stops only (validation/oracle, where the tokenizer is not needed). */ +static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){ + g_nstop = 0; + for (int i = 0; i < c->n_stop && g_nstop < 64; i++) g_stop[g_nstop++] = c->stop_ids[i]; + if (tok_eos >= 0 && !is_stop(tok_eos) && g_nstop < 64) g_stop[g_nstop++] = tok_eos; + int nsp = 0; + if (T) for (int id = 0; id < T->n_ids && g_nstop < 64; id++) + if (T->id_special[id] && !is_stop(id)) { g_stop[g_nstop++] = id; nsp++; } + fprintf(stderr, "[stop] %d stop tokens:", g_nstop); + for (int i = 0; i < g_nstop; i++) fprintf(stderr, " %d", g_stop[i]); + if (nsp) fprintf(stderr, " (%d from the tokenizer's special set)", nsp); + fprintf(stderr, "\n"); +} +static void stops_arm(const Cfg *c, int tok_eos){ stops_arm_tok(c, tok_eos, NULL); } + +/* ---- log-prob of a target token given the logit vector ------------------- */ +static double logprob_target(const float *lo, int V, int target, int *am){ + float mx = lo[0]; int best = 0; + for (int i = 1; i < V; i++) if (lo[i] > mx) { mx = lo[i]; best = i; } + double se = 0; + for (int i = 0; i < V; i++) se += exp((double)lo[i] - mx); + if (am) *am = (best == target); + return (double)(lo[target] - mx) - log(se); +} + +/* "glm" in model_type, case-insensitive */ +static int mt_is_glm(const char *s){ + if (s) for (; *s; s++) + if ((s[0]|32) == 'g' && (s[1]|32) == 'l' && (s[2]|32) == 'm') return 1; + return 0; +} + +#endif /* SAMPLE_H */ diff --git a/c/setup.sh b/c/setup.sh index e31e60a..dcb851d 100755 --- a/c/setup.sh +++ b/c/setup.sh @@ -32,11 +32,11 @@ esac # 2) build: nativa (veloce, per QUESTA macchina). Per un binario da distribuire: make portable echo " building (ARCH=${ARCH:-native})…" -make -s glm ARCH="${ARCH:-native}" +make -s colibri ARCH="${ARCH:-native}" # 3) self-test sull'oracolo tiny, se presente if [ -d glm_tiny ] && [ -f ref_glm.json ]; then - r=$(SNAP=./glm_tiny TF=1 ./glm 64 16 16 2>/dev/null | grep -oE "[0-9]+/[0-9]+ positions" || true) + r=$(SNAP=./glm_tiny TF=1 ./colibri 64 16 16 2>/dev/null | grep -oE "[0-9]+/[0-9]+ positions" || true) echo " engine self-test: ${r:-?} (expected 32/32)" fi diff --git a/c/telemetry.h b/c/telemetry.h new file mode 100644 index 0000000..3c3fe85 --- /dev/null +++ b/c/telemetry.h @@ -0,0 +1,189 @@ +/* telemetry.h — dashboard protocol lines, stats/usage persistence, hardware probe. + * Include after Model/Cfg/QT/ESlot/shards and st.h are defined; requires + * qt_bytes(), now_s(), rss_gb(), edisk_s(), and the g_cuda_* globals (ifdef). */ +#ifndef TELEMETRY_H +#define TELEMETRY_H + +static int64_t tbytes(int O,int I,int bits){ + if(bits>=16) return (int64_t)O*I*4; + if(bits>=5) return (int64_t)O*I + (int64_t)O*4; + return (int64_t)O*((I+1)/2) + (int64_t)O*4; +} + +static int64_t expert_bytes_probe(Model *m, int ebits){ + Cfg *c=&m->c; int64_t eb=0; char nm[256]; + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.gate_proj.weight",c->first_dense); + if(st_nbytes(&m->S,nm)>0){ + const char *suf[3]={"gate_proj","up_proj","down_proj"}; + for(int k=0;k<3;k++){ + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight",c->first_dense,suf[k]); + eb+=st_nbytes(&m->S,nm); + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight.qs",c->first_dense,suf[k]); + int64_t q=st_nbytes(&m->S,nm); if(q>0) eb+=q; + } + } + if(eb<=0) eb = tbytes(c->moe_inter,c->hidden,ebits)*2 + tbytes(c->hidden,c->moe_inter,ebits); + return eb; +} + +/* BRAIN MAP: per-turn expert hit bitmap for the dashboard. */ +static uint8_t **g_ehit; +static void ehit_mark(Model *m, int layer, int eid){ + if(!g_ehit){ Cfg *c=&m->c; + g_ehit=calloc(c->n_layers+1,sizeof(uint8_t*)); + for(int i=0;i<=c->n_layers;i++) g_ehit[i]=calloc(c->n_experts,1); + } + g_ehit[layer][eid]=1; +} + +/* CPU model + cores + RAM (GB); empty/zero where unavailable. */ +static void hw_probe(char *cpu, size_t cn, int *cores, double *ram_total, double *ram_avail){ + cpu[0]=0; +#ifdef _WIN32 +#if defined(__x86_64__) || defined(__i386__) + { unsigned int r[12]={0}; unsigned int *w=r; + for(unsigned int f=0x80000002u; f<=0x80000004u; f++,w+=4) + __get_cpuid(f,&w[0],&w[1],&w[2],&w[3]); + char *b=(char*)r; b[47]=0; while(*b==' ')b++; + snprintf(cpu,cn,"%s",b); } +#endif +#else + FILE *ci=fopen("/proc/cpuinfo","r"); + if(ci){ char ln[256]; + while(fgets(ln,sizeof(ln),ci)) if(!strncmp(ln,"model name",10)){ + char *p=strchr(ln,':'); if(p){ p++; while(*p==' ')p++; + int n=(int)strlen(p); if(n>0&&p[n-1]=='\n')p[--n]=0; + snprintf(cpu,cn,"%s",p); } break; } + fclose(ci); } +#endif + *cores=0; +#ifdef _WIN32 + { SYSTEM_INFO si; GetSystemInfo(&si); *cores=(int)si.dwNumberOfProcessors; } +#elif defined(_SC_NPROCESSORS_ONLN) + *cores=(int)sysconf(_SC_NPROCESSORS_ONLN); +#endif + *ram_total=*ram_avail=0; +#ifdef _WIN32 + compat_meminfo(ram_total,ram_avail); +#else + FILE *mi=fopen("/proc/meminfo","r"); + if(mi){ char ln[256]; double mt=0,ma=0; + while(fgets(ln,sizeof(ln),mi)){ + if(sscanf(ln,"MemTotal: %lf",&mt)==1) *ram_total=mt/1e6; + if(sscanf(ln,"MemAvailable: %lf",&ma)==1) *ram_avail=ma/1e6; + } fclose(mi); } +#endif +} + +static void hwinfo_emit(Model *m){ + Cfg *c=&m->c; (void)c; + char cpu[256]; int cores; double ram_total,ram_avail; + hw_probe(cpu,sizeof(cpu),&cores,&ram_total,&ram_avail); + int ngpu=0; double vram_total=0; + char gpu_name[128]=""; +#ifdef COLI_CUDA + ngpu=g_cuda_ndev; vram_total=m->gpu_expert_bytes/1e9; + for(int i=0;i0) + snprintf(gpu_name,sizeof(gpu_name),"CUDA device x%d",g_cuda_ndev); +#endif + printf("HWINFO %d %.1f %.1f %d %.1f %s|%s\n", + cores,ram_total,ram_avail,ngpu,vram_total,cpu,gpu_name); + fflush(stdout); +} + +static void tiers_emit(Model *m){ + Cfg *c=&m->c; int nsp=0; + for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; + int total=(nsp+(m->has_mtp?1:0))*c->n_experts; + int pinned=0,lru=0; + for(int i=0;i<=c->n_layers;i++){ pinned+=m->npin?m->npin[i]:0; lru+=m->ecn?m->ecn[i]:0; } + int vram=0; double vram_gb=0; +#ifdef COLI_CUDA + vram=m->gpu_expert_count; vram_gb=m->gpu_expert_bytes/1e9; +#endif + int ram=pinned-vram+lru; if(ram<0) ram=0; + int disk=total-vram-ram; if(disk<0) disk=0; + double eb=(double)expert_bytes_probe(m,m->ebits); + printf("TIERS %d %d %d %.2f %.2f\n",vram,ram,disk,vram_gb,ram*eb/1e9); + fflush(stdout); +} + +static void emap_emit(Model *m){ + Cfg *c=&m->c; + int rows=0; + for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; + int has_mtp = m->has_mtp && m->eusage[c->n_layers]; + if(has_mtp) rows++; + int cols=c->n_experts; + char *hex=malloc((size_t)rows*cols*2+1); int w=0; + for(int i=0;i<=c->n_layers;i++){ + int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); + if(!is_row) continue; + for(int e=0;epin[i]; + for(int z=0;znpin[i];z++) if(P[z].eid==e){ +#ifdef COLI_CUDA + tier = P[z].g.cuda?2:1; +#else + tier = 1; +#endif + break; } + if(!tier && m->ecache && m->ecache[i]) + for(int z=0;zecn[i];z++) if(m->ecache[i][z].eid==e){ tier=1; break; } + uint32_t u = m->eusage[i]?m->eusage[i][e]:0; + int heat=0; while(u){ heat++; u>>=1; } if(heat>63) heat=63; + int b=(tier<<6)|heat; + hex[w++]="0123456789abcdef"[b>>4]; hex[w++]="0123456789abcdef"[b&15]; + } + } + hex[w]=0; + printf("EMAP %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); +} + +static void hits_emit(Model *m){ + Cfg *c=&m->c; if(!g_ehit) return; + int rows=0; + for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; + int has_mtp = m->has_mtp && m->eusage[c->n_layers]; + if(has_mtp) rows++; + int cols=c->n_experts, nb=(rows*cols+7)/8; + uint8_t *bm=calloc(nb,1); int bit=0; + for(int i=0;i<=c->n_layers;i++){ + int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); + if(!is_row) continue; + for(int e=0;e>3]|=1<<(bit&7); g_ehit[i][e]=0; } + } + char *hex=malloc((size_t)nb*2+1); int w=0; + for(int b=0;b>4]; hex[w++]="0123456789abcdef"[bm[b]&15]; } + hex[w]=0; + printf("HITS %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); free(bm); +} + +static void stats_dump_q(Model *m, const char *path, int quiet){ + char tmp[2100]; snprintf(tmp,sizeof(tmp),"%s.tmp",path); + FILE *f=fopen(tmp,"w"); if(!f){ if(!quiet) perror(tmp); return; } + Cfg *c=&m->c; int64_t tot=0, nz=0; + for(int i=0;i<=c->n_layers;i++){ if(!m->eusage[i]) continue; + for(int e=0;en_experts;e++) if(m->eusage[i][e]){ fprintf(f,"%d %d %u\n",i,e,m->eusage[i][e]); tot+=m->eusage[i][e]; nz++; } } + fclose(f); rename(tmp,path); + if(!quiet) fprintf(stderr,"[STATS] %lld selections across %lld distinct experts -> %s\n",(long long)tot,(long long)nz,path); +} +static void stats_dump(Model *m, const char *path){ stats_dump_q(m,path,0); } + +static char g_usage_path[2100]=""; +static int64_t usage_load(Model *m, const char *path){ + FILE *f=fopen(path,"r"); if(!f) return 0; + Cfg *c=&m->c; int l,e; uint32_t cnt; int64_t tot=0; + while(fscanf(f,"%d %d %u",&l,&e,&cnt)==3) + if(l>=0&&l<=c->n_layers&&e>=0&&en_experts&&m->eusage[l]){ m->eusage[l][e]+=cnt; tot+=cnt; } + fclose(f); return tot; +} +static void usage_save(Model *m){ if(g_usage_path[0]) stats_dump_q(m,g_usage_path,1); } + +#endif /* TELEMETRY_H */ diff --git a/c/tests/bench_dsa_select.c b/c/tests/bench_dsa_select.c index a1f2014..676e4c2 100644 --- a/c/tests/bench_dsa_select.c +++ b/c/tests/bench_dsa_select.c @@ -24,7 +24,7 @@ * Run: make tests/bench_dsa_select && ./tests/bench_dsa_select (not in TEST_BINS) */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main #include diff --git a/c/tests/bench_topp.c b/c/tests/bench_topp.c index 9c0682f..b284308 100644 --- a/c/tests/bench_topp.c +++ b/c/tests/bench_topp.c @@ -22,7 +22,7 @@ * Run: make tests/bench_topp && ./tests/bench_topp (not in TEST_BINS -- not a gate) */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main #include diff --git a/c/tests/test_dsa_select.c b/c/tests/test_dsa_select.c index 99fecb0..ef3fd7b 100644 --- a/c/tests/test_dsa_select.c +++ b/c/tests/test_dsa_select.c @@ -31,7 +31,7 @@ * In-memory only (no scratch files), so it builds clean on the Windows MinGW CI * job without the unmerged compat shim. */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main #include diff --git a/c/tests/test_i4_grouped.c b/c/tests/test_i4_grouped.c index 2a9521e..04e8d88 100644 --- a/c/tests/test_i4_grouped.c +++ b/c/tests/test_i4_grouped.c @@ -18,7 +18,7 @@ * index, a wrong group boundary or a swapped nibble cannot hide under it — * those are O(1) relative errors, not O(1e-6). */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static uint32_t rng_state=0xC0FFEEu; diff --git a/c/tests/test_idot.c b/c/tests/test_idot.c index b7109d4..343535c 100644 --- a/c/tests/test_idot.c +++ b/c/tests/test_idot.c @@ -7,7 +7,7 @@ * (sign-trick kernels must treat |−128| as 128 unsigned, not saturate to 127), * and random data at qrow_i8's contract (|x| <= 127, w full int8 range). */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static uint32_t rng_state=0x12345678u; diff --git a/c/tests/test_kv_alloc.c b/c/tests/test_kv_alloc.c index 40ec00d..d080c08 100644 --- a/c/tests/test_kv_alloc.c +++ b/c/tests/test_kv_alloc.c @@ -4,7 +4,7 @@ * arrays twice on the second call -> allocator abort. No model file needed: * the CPU path of kv_alloc only reads c->n_layers/kv_lora/qk_rope. */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main int main(void){ diff --git a/c/tests/test_makefile_platform.py b/c/tests/test_makefile_platform.py index 3c72880..50b4c7a 100644 --- a/c/tests/test_makefile_platform.py +++ b/c/tests/test_makefile_platform.py @@ -35,7 +35,7 @@ class MakefilePlatformTests(unittest.TestCase): env["PATH"] = "" result = subprocess.run( - [MAKE, "--no-print-directory", "-B", "-n", "glm"], + [MAKE, "--no-print-directory", "-B", "-n", "colibri"], cwd=C_DIR, env=env, text=True, @@ -43,7 +43,7 @@ class MakefilePlatformTests(unittest.TestCase): check=True, ) - self.assertIn("-o glm.exe", result.stdout) + self.assertIn("-o colibri.exe", result.stdout) self.assertIn("-fopenmp", result.stdout) self.assertIn("-static", result.stdout) diff --git a/c/tests/test_sample_nan.c b/c/tests/test_sample_nan.c index fc33268..4f31def 100644 --- a/c/tests/test_sample_nan.c +++ b/c/tests/test_sample_nan.c @@ -6,7 +6,7 @@ * iniettato il token scelto e' l'argmax dei FINITI (mai 0 per default), su ogni posizione * del NaN inclusa lo[0]; (c) nessun NaN sopravvive in g_pbuf. */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main #include diff --git a/c/tests/test_stops.c b/c/tests/test_stops.c index 00f922b..5e7e9d2 100644 --- a/c/tests/test_stops.c +++ b/c/tests/test_stops.c @@ -20,7 +20,7 @@ * Defense 2 is what makes this robust against checkpoints we don't control: * even with BOTH configs mutilated, a control token cannot leak into a reply. */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static const char *TOKJSON = diff --git a/c/tests/test_topp.c b/c/tests/test_topp.c index 9276507..49f45d6 100644 --- a/c/tests/test_topp.c +++ b/c/tests/test_topp.c @@ -24,7 +24,7 @@ * No scratch files: the test runs entirely in memory (no mkdtemp), so it builds clean on * the Windows MinGW CI job without the unmerged compat shim (#352). */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main #include diff --git a/c/tests/test_uring.c b/c/tests/test_uring.c index e64b9f2..625c79b 100644 --- a/c/tests/test_uring.c +++ b/c/tests/test_uring.c @@ -6,7 +6,7 @@ #include #include #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static int fail(const char *s){ fprintf(stderr,"FAIL: %s\n",s); return 1; } From 420a0720c31808753d03784de56a5fcd1a51a61d Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Sun, 19 Jul 2026 04:02:00 +0800 Subject: [PATCH 70/95] docs: add simplified Chinese and Italian README, update language nav MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New files: README.zh-CN.md — simplified Chinese (大陆用词) README.it.md — Italian (the project's "mother tongue") All four READMEs now link to each other in a consistent nav bar. Updated zh-TW to reflect glm.c → colibri.c rename and new headers. --- README.it.md | 260 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 +- README.zh-CN.md | 237 +++++++++++++++++++++++++++++++++++++++++++ README.zh-TW.md | 12 ++- 4 files changed, 506 insertions(+), 5 deletions(-) create mode 100644 README.it.md create mode 100644 README.zh-CN.md diff --git a/README.it.md b/README.it.md new file mode 100644 index 0000000..76e3c21 --- /dev/null +++ b/README.it.md @@ -0,0 +1,260 @@ +

+ colibrì — motore piccolo, modello immenso +

+ +

+ English · 简体中文 · 繁體中文 · Italiano +

+ +**Motore piccolo, modello immenso.** Esegui **GLM-5.2 (744 miliardi di parametri, MoE)** su un computer consumer con ~25 GB di RAM — in C puro, zero dipendenze, caricando gli expert dal disco in streaming. + +Colibrì è un runtime MoE leggero e che preserva la qualità: tratta VRAM, RAM e +disco come un'unica gerarchia di memoria gestita. Se la memoria veloce non basta +il modello rallenta, ma la policy predefinita **non cambia mai silenziosamente la +precisione del modello né la semantica del router**. + +``` +$ ./coli chat + 🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU + ✓ ready in 32s · resident 9.9 GB + › ciao! + ◆ Ciao! 😊 Come posso aiutarti oggi? +``` + +## Guardalo in azione + +

+ dashboard web di colibrì — metriche live, pannello hardware, livelli degli expert +

+

La dashboard web (./coli web): un modello da 744B a 4 tok/s, TTFT 1.6 s, disco 0 — +residenza completa degli expert su 6× RTX 5090, con metriche token in tempo reale, breakdown dei tempi per turno, +la barra dei livelli VRAM/RAM/disco e il mini-cervello live nell'angolo.

+ +

+ la pagina Brain — 19.456 expert come una corteccia vivente +

+

La pagina Brain: tutti i 19.456 expert come una corteccia vivente — il colore indica +il livello di archiviazione, la luminosità il calore di routing, e ogni expert instradato in un turno +lampeggia bianco. Passando il cursore si vede l'affinità +tematica misurata dell'expert.

+ +

+ la pagina Atlas — l'atlante misurato degli expert come una galassia 3D +

+

La pagina Atlas: l'atlante +misurato degli expert come una galassia 3D — 13.260 expert caratterizzati, 1.041 specialisti +replicabili che si raggruppano per argomento (poesia, legge, cinese, SQL…). La posizione deriva +dall'affinità di routing misurata, non da un embedding appreso. Trascinare per ruotare.

+ +## L'idea + +Un modello Mixture-of-Experts da 744B attiva solo ~40B parametri per token — e +solo ~11 GB di quelli cambiano da un token all'altro (gli expert instradati): + +

+ solo ~5.4% dei parametri è attivo per token +

+ +Il modello non ha bisogno di *stare* in memoria veloce — ha bisogno di essere +**piazzato**: + +- la **parte densa** (attenzione, expert condivisi, embedding — ~17B parametri) + resta **residente in RAM a int4** (~9.9 GB); +- i **19.456 expert instradati** (75 layer MoE × 256 + la testa MTP, ~19 MB + ciascuno a int4) stanno **su disco** (~370 GB) e vengono **caricati on demand + in streaming**, con una cache LRU per layer, un hot-store pinnato che impara, + e un livello VRAM opzionale. + +Il motore è un singolo file C (`c/colibri.c`) più header piccoli. Niente BLAS, +niente Python a runtime, niente GPU obbligatoria. + +## Come funziona + +### Il percorso di ogni token + +

+ instrada → unione → piazza → sovrapponi → impara +

+ +Ogni layer di ogni token percorre gli stessi cinque passi. L'obiettivo +progettuale è che **il piazzamento decide solo la velocità** — le decisioni +del router e la precisione dei pesi sono identiche sia che un expert risponda +dalla VRAM sia dal disco. + +### Una gerarchia di memoria, non un requisito di memoria + +

+ residenza expert a tre livelli: VRAM / RAM / NVMe +

+ +Lo stesso motore copre l'intero spettro: su un portatile da 25 GB tutto viene +caricato dal disco in streaming (lento, ma corretto); su un host grande l'intero +set di expert diventa residente (`CUDA_EXPERT_GB=auto PIN_GB=all`) e il disco +esce completamente dal percorso di decode. Tra i livelli c'è una **cache che +impara**: il motore registra quali expert il *tuo* carico di lavoro instrada +(`.coli_usage`, aggiornato a ogni turno) e fissa automaticamente i più caldi — +colibrì diventa letteralmente più veloce man mano che lo usi. Sugli host +multi-socket, `COLI_NUMA=1` interlaccia i pesi residenti tra i controller di +memoria ([#82](https://github.com/JustVugg/colibri/issues/82)). + +### Mai aspettare il disco due volte + +I miss nella cache costano caro, quindi il motore investe la maggior parte +della sua astuzia per evitarli e sovrapporli: le tre matrici di ogni expert sono +memorizzate contigue e lette con un unico `pread`; un pool I/O asincrono +limitato (`PIPE=1`, attivo per default) carica gli expert mancanti mentre quelli +residenti calcolano; le posizioni in batch leggono ogni expert unico una sola +volta (**batch-union**); un thread di lookahead del router (`PILOT=1`) fa il +prefetch degli expert del layer successivo — il routing è misurabilmente +**prevedibile al 71.6% un layer in anticipo**. Sulle GPU, la pipeline residente +(`COLI_CUDA_PIPE=2`) mantiene il flusso residuo on-device tra i layer, così il +loop CPU degli expert procede senza interruzioni; su Apple Silicon un backend +[Metal](docs/metal.md) sperimentale esegue la matmul batch degli expert sulla +GPU a memoria unificata. + +### Modello fedele, stato compresso + +Il forward pass è validato **token-esatto contro un oracle `transformers`** +(teacher-forcing 32/32). L'attenzione MLA memorizza uno stato KV compresso — 576 +float/token invece di 32.768 (**57× più piccolo**) — e lo persiste tra i +riavvii (`.coli_kv`): le conversazioni riaprono "calde", senza alcun re-prefill, +byte-identiche a una sessione ininterrotta. L'attenzione sparsa DSA (il +lightning indexer di GLM-5.2) è implementata fedelmente e validata forzando la +selezione di tutte le chiavi per riprodurre esattamente l'attenzione densa. + +### Decodifica speculativa, onestamente + +La testa MTP nativa di GLM-5.2 propone token che il modello principale verifica +in un unico forward batch — 2.2–2.8 token/forward quando conviene. Due regole +conquistate a caro prezzo sono i default: la testa MTP deve essere **int8** (le +teste int4 crollano al 0–4% di accettazione, +[#8](https://github.com/JustVugg/colibri/issues/8)), e draft e verifica devono +calcolare **la stessa funzione** — `SPEC_PIN=1` fissa entrambi sulla stessa +famiglia di kernel ([#163](https://github.com/JustVugg/colibri/issues/163) +contiene l'intera indagine forense). I draft forzati da grammatica +([`GRAMMAR=file.gbnf`](docs/grammar-draft.md)) aggiungono accettazione quasi +gratuita sull'output JSON vincolato. Se la speculazione conviene dipende dalla +temperatura della cache — misura, e usa `DRAFT=0` quando non paga. + +## Cosa ottiene + +

+ velocità di decode misurata per classe hardware +

+ +Stesso motore, stesso container int4 — cambia solo dove risiedono gli expert. +Punti salienti dalle [tabelle benchmark complete](docs/benchmarks.md): + +- **6× RTX 5090, residenza completa:** 5.8–6.8 tok/s in decode, TTFT ~13 s + ([log dell'esperimento](docs/experiments/glm52-6x5090-2026-07-12.md)); +- **desktop solo-CPU da 128 GB:** ~1.8 tok/s a cache calda + ([#200](https://github.com/JustVugg/colibri/issues/200)); +- **singola RTX 5070 Ti, classe laptop:** 1.07 tok/s tramite la pipeline + GPU-residente ([#273](https://github.com/JustVugg/colibri/issues/273)); +- **macchina di sviluppo da 25 GB:** 0.05–0.1 tok/s a freddo — il punto di + partenza dimostrato da cui è nato il progetto, e ancora oggi la baseline onesta. + +La qualità è misurata, non presunta: il costo di quantizzazione del container +int4 e le ablazioni su granularità delle scale e rotazione sono in +[docs/benchmarks.md](docs/benchmarks.md#quality-benchmark) e +[#108](https://github.com/JustVugg/colibri/issues/108)/[#81](https://github.com/JustVugg/colibri/issues/81). + +## Per iniziare + +### 1. Scarica il modello + +Un container **GLM-5.2 int4** pre-convertito è su Hugging Face — **usa la +versione con le teste MTP int8**: + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +> ⚠️ Il mirror originale contiene teste MTP int4 → accettazione dei draft allo 0% +> ([#8](https://github.com/JustVugg/colibri/issues/8)). Verifica la tua versione: +> `ls -l /out-mtp-*` — int8 (corretto) è `3527131672 / 5366238584 / 1065950496`. + +Oppure converti tu stesso dalla sorgente FP8 — un unico comando riprendibile che +non richiede mai i 756 GB completi su disco contemporaneamente: + +```bash +cd c && ./setup.sh # verifica gcc/OpenMP, compila, autotest +./coli convert --model /nvme/glm52_i4 # scarica e converti shard per shard (python, una tantum) +``` + +### 2. Esegui + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli chat # budget RAM, cache e MTP rilevati automaticamente +COLI_MODEL=/nvme/glm52_i4 ./coli plan # mostra il piazzamento pianificato VRAM/RAM/disco +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # controllo di idoneità (sola lettura) +./coli web --model /nvme/glm52_i4 # API + dashboard web sulla stessa porta +./coli serve --model /nvme/glm52_i4 # solo API compatibile OpenAI +``` + +Il motore a runtime è puro C — python si usa solo per il convertitore (una tantum) +e per il gateway API opzionale. + +### 3. Approfondisci + +| argomento | documento | +|---|---| +| Benchmark, dati dalla comunità, misurazioni di qualità | [docs/benchmarks.md](docs/benchmarks.md) | +| Parametri di tuning, policy, cache che impara, prefetch | [docs/tuning.md](docs/tuning.md) | +| Build nativa su Windows 11 (con CUDA DLL) | [docs/windows.md](docs/windows.md) | +| Backend CUDA, livello expert in VRAM, residenza completa | [docs/cuda.md](docs/cuda.md) | +| Backend Metal per Apple Silicon | [docs/metal.md](docs/metal.md) | +| API compatibile OpenAI, KV slot, dashboard web | [docs/api.md](docs/api.md) | +| Draft forzati da grammatica (output strutturato) | [docs/grammar-draft.md](docs/grammar-draft.md) | +| Inventario delle variabili d'ambiente | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | + +## Sostenere il progetto + +colibrì è nato come progetto di una sola persona su un portatile con 12 core +e 25 GB di RAM; oggi i suoi numeri arrivano da una comunità di macchine reali. +Se ti è utile: + +- ⭐ metti una stella al repository e condividilo; +- 🐛 apri issue con i numeri di benchmark del tuo hardware — i datapoint + fanno avanzare questo progetto più di qualsiasi altra cosa; +- 💬 contattaci via GitHub issues per sponsorizzare lo sviluppo o donare hardware. + +## Struttura del repository + +``` +Makefile punto d'ingresso root per build/check +c/ +├── colibri.c motore principale +├── quant.h kernel matmul quantizzati (SIMD multi-architettura) +├── sample.h campionamento, RNG, set di stop +├── kv_persist.h persistenza KV su disco (.coli_kv) +├── telemetry.h protocollo dashboard, statistiche, usage +├── st.h, tok.h, json.h header di runtime +├── backend_cuda.* livello CUDA opzionale +├── Makefile build e check locali +├── coli CLI utente +├── openai_server.py gateway HTTP compatibile OpenAI +├── setup.sh setup locale in un solo comando +├── tools/ conversione offline, fixture e benchmark +├── scripts/ helper per conversioni lunghe +└── tests/ test C e Python senza dipendenze +web/ UI browser (puro client API OpenAI) +desktop/ shell desktop Tauri v2 che racchiude la web UI +docs/ documentazione di riferimento, esperimenti, media +``` + +Il percorso a runtime resta intenzionalmente piatto e leggibile: `colibri.c` +più i suoi header. Dalla radice del repository, `make`, `make check` e +`make clean` delegano al Makefile del motore. + +## Perché "colibrì" + +Il colibrì pesa pochi grammi, sta sospeso nel vuoto e visita un migliaio di +fiori al giorno. Questo motore tiene in vita un gigante da 744 miliardi di +parametri con le razioni di un colibrì: 25 GB di RAM, dodici core CPU e +tanta pazienza col disco. + +Il nome è rimasto in italiano perché questa è la lingua in cui è stato scritto +il primo prototipo — i commenti nel codice lo testimoniano ancora. + +## Licenza + +Apache 2.0. I pesi di GLM-5.2 sono rilasciati da Z.ai sotto licenza MIT. diff --git a/README.md b/README.md index f9cf3f0..d6f9bcb 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

- English · 繁體中文 + English · 简体中文 · 繁體中文 · Italiano

**Tiny engine, immense model.** Run **GLM-5.2 (744B-parameter MoE)** on a consumer machine with ~25 GB of RAM — in pure C, with zero dependencies, by streaming experts from disk. diff --git a/README.zh-CN.md b/README.zh-CN.md new file mode 100644 index 0000000..352de33 --- /dev/null +++ b/README.zh-CN.md @@ -0,0 +1,237 @@ +

+ colibrì——小巧引擎，庞大模型 +

+ +

+ English · 简体中文 · 繁體中文 · Italiano +

+ +**小巧引擎，庞大模型。**只需约 25 GB 内存，就能在消费级电脑上运行 **GLM-5.2（744B 参数的 MoE）**——以零依赖的纯 C 实现，从磁盘流式加载专家。 + +Colibrì 是一套轻量、保持模型质量的 MoE 运行时，将 VRAM、RAM +与存储设备视为统一管理的内存层级。高速内存不足可能降低速度， +但默认策略**绝不会在未告知的情况下改变模型精度或路由语义**。 + +``` +$ ./coli chat + 🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU + ✓ ready in 32s · resident 9.9 GB + › ciao! + ◆ Ciao! 😊 Come posso aiutarti oggi? +``` + +## 实际运行效果 + +

+ colibrì 网页仪表盘——实时指标、硬件面板与专家存储层级 +

+

网页仪表盘（./coli web）：744B 模型达到 4 tok/s、TTFT 1.6 秒、磁盘读取 0—— +在 6× RTX 5090 上让所有专家常驻，并实时显示 token 指标、每轮耗时明细、 +VRAM／RAM／磁盘层级条，以及角落的实时迷你大脑。

+ +

+ 大脑页面——以实时皮层呈现 19,456 个专家 +

+

大脑（Brain）页面：将全部 19,456 个专家呈现为活的皮层——颜色代表存储层级， +亮度代表路由热度，每轮被路由到的专家都会闪白。将光标停在专家上，即可查看其 +实测主题亲和度。

+ +

+ 图谱页面——以 3D 星系呈现实测专家图谱 +

+

图谱（Atlas）页面：将实测专家图谱 +呈现为 3D 星系——共 13,260 个已分析专家，其中 1,041 个可复现的专门专家会按主题聚集 +（诗歌、法律、中文、SQL……）。位置取自实测路由亲和度，而非学习出的嵌入向量。拖拽即可旋转。

+ +## 核心概念 + +744B 的专家混合（Mixture-of-Experts）模型，每个 token 只会激活约 40B 参数—— +其中每个 token 之间会变动的只有约 11 GB（被路由到的专家）： + +

+ 每个 token 只会激活约 5.4% 的参数 +

+ +所以模型不必完整**装进**高速内存，而是需要正确**放置**： + +- **稠密部分**（注意力、共享专家、嵌入——约 17B 参数）以 int4 + **常驻 RAM**（约 9.9 GB）； +- **19,456 个路由专家**（75 个 MoE 层 × 256，加上 MTP head；每个在 int4 下约 19 MB） + **存放在磁盘**（约 370 GB），并**按需流式加载**，配合逐层 LRU 缓存、 + 会学习的热门专家固定存储区，以及可选的 VRAM 层级。 + +引擎是一个 C 主文件（`c/colibri.c`）加上若干头文件。不需要 BLAS， +运行时不需要 Python，也不需要 GPU。 + +## 工作原理 + +### 每个 token 的处理路径 + +

+ 路由 → 并集 → 放置 → 重叠执行 → 学习 +

+ +每个 token 的每一层都会经过相同的五个步骤。设计目标是让 +**放置只决定速度**——无论专家是从 VRAM 还是磁盘响应，路由器的决策与权重精度都完全相同。 + +### 统一内存层级，取代单一内存门槛 + +

+ VRAM／RAM／NVMe 三层专家常驻架构 +

+ +同一套引擎覆盖完整硬件范围：在 25 GB 笔记本上，一切都从磁盘流式加载 +（慢，但结果正确）；在大内存主机上，则可让整组专家常驻 +（`CUDA_EXPERT_GB=auto PIN_GB=all`），让磁盘完全退出解码路径。 +两端之间有一层**学习型缓存**：引擎会记录*你的*工作负载路由到哪些专家 +（`.coli_usage`，每轮更新），并自动固定最热门的专家——colibrì 确实会越用越快。 +在多路主机上，`COLI_NUMA=1` 会将常驻权重交错分配到各内存控制器 +（[#82](https://github.com/JustVugg/colibri/issues/82)）。 + +### 绝不为同一次磁盘读取等待两遍 + +缓存未命中的代价很高，因此引擎大部分的巧思都用来避免或重叠这些读取： +每个专家的三个矩阵相邻存储，并以一次 `pread` 读取；有界异步 I/O 池 +（`PIPE=1`，默认启用）会在常驻专家计算时加载缺失的专家；批量位置只读取每个 +不重复专家一次（**批量并集**）；路由前瞻线程（`PILOT=1`）则预取下一层专家—— +实测显示，路由结果提前一层时有 **71.6% 的可预测性**。 +在 GPU 上，常驻管线（`COLI_CUDA_PIPE=2`）让残差流跨层保留在设备端， +使 CPU 专家循环不中断；在 Apple Silicon 上，实验性的 +[Metal 后端](docs/metal.md)会用统一内存 GPU 执行批量专家运算。 + +### 忠实模型，压缩状态 + +前向传播已通过 `transformers` oracle 验证为**逐 token 完全一致** +（teacher-forcing 32/32）。MLA 注意力存储压缩后的 KV 状态——每个 token 为 576 个 +浮点数，而非 32,768 个（**缩小 57×**）——并跨重启持久保存 +（`.coli_kv`）：对话可暖启恢复，不需重新 prefill，结果与不中断的会话 +逐字节相同。DSA 稀疏注意力（GLM-5.2 的 lightning indexer）已忠实实现， +并通过强制选取所有 key，验证可精确复现稠密注意力。 + +### 诚实的推测解码 + +GLM-5.2 原生 MTP head 会起草 token，再由主模型以一次批量前向传播验证—— +条件合适时每次 forward 可产生 2.2–2.8 个 token。两条来之不易的规则已成为默认值： +MTP head 必须是 **int8**（int4 head 的接受率会崩塌到 0–4%，见 +[#8](https://github.com/JustVugg/colibri/issues/8)），且草稿与验证必须计算 +**相同函数**——`SPEC_PIN=1` 会把两者固定在同一 kernel family +（完整取证过程见 [#163](https://github.com/JustVugg/colibri/issues/163)）。 +语法强制草稿（[`GRAMMAR=file.gbnf`](docs/grammar-draft.md)）可在受限 JSON 输出中， +以近乎免费的代价提高接受率。推测解码是否带来净收益取决于缓存热度——请实测， +若不划算就使用 `DRAFT=0`。 + +## 实际成果 + +

+ 各硬件级别的实测解码速度 +

+ +同一套引擎、同一个 int4 容器——硬件只会改变专家的存放位置。 +[完整 benchmark 表格](docs/benchmarks.md)中的重点如下： + +- **6× RTX 5090，全部常驻：**解码 5.8–6.8 tok/s，TTFT 约 13 秒 + （[实验记录](docs/experiments/glm52-6x5090-2026-07-12.md)）； +- **128 GB、仅使用 CPU 的台式机：**热缓存后约 1.8 tok/s + （[#200](https://github.com/JustVugg/colibri/issues/200)）； +- **单张 RTX 5070 Ti 的笔记本级主机：**通过 GPU 常驻管线达到 1.07 tok/s + （[#273](https://github.com/JustVugg/colibri/issues/273)）； +- **25 GB 开发机：**冷启动 0.05–0.1 tok/s——这是项目起步时已证实的下限， + 也仍是诚实的基准。 + +质量来自测量，而非假设：int4 容器的量化损失，以及 scale granularity／rotation +消融实验，收录于 [docs/benchmarks.md](docs/benchmarks.md#quality-benchmark)、 +[#108](https://github.com/JustVugg/colibri/issues/108) 与 +[#81](https://github.com/JustVugg/colibri/issues/81)。 + +## 开始使用 + +### 1. 获取模型 + +Hugging Face 上已有预转换的 **GLM-5.2 int4** 容器——请务必使用 +**含 int8 MTP head 的版本**： + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +> ⚠️ 原始镜像使用 int4 MTP head → 草稿接受率为 0% +>（[#8](https://github.com/JustVugg/colibri/issues/8)）。请检查你的版本： +> `ls -l /out-mtp-*`——正确的 int8 大小为 `3527131672 / 5366238584 / 1065950496`。 + +你也可以自行从 FP8 源转换——只需一条可断点续传的命令，且任何时候都不需要 +在磁盘上同时存放完整的 756 GB： + +```bash +cd c && ./setup.sh # 检查 gcc/OpenMP、构建并运行自测 +./coli convert --model /nvme/glm52_i4 # 逐 shard 下载并转换（仅此一次需要 python） +``` + +### 2. 运行 + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli chat # 自动检测 RAM 预算、缓存与 MTP +COLI_MODEL=/nvme/glm52_i4 ./coli plan # 查看规划的 VRAM／RAM／磁盘配置 +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # 只读就绪检查 +./coli web --model /nvme/glm52_i4 # 在同一端口提供 API 与网页仪表盘 +./coli serve --model /nvme/glm52_i4 # 仅提供 OpenAI 兼容 API +``` + +引擎运行时是纯 C——python 只供一次性转换工具与可选的 API gateway 使用。 + +### 3. 深入了解 + +| 主题 | 文档 | +|---|---| +| Benchmark、社区实测数据、质量测量 | [docs/benchmarks.md](docs/benchmarks.md) | +| 调优选项、策略、学习型缓存、预取 | [docs/tuning.md](docs/tuning.md) | +| Windows 11 原生构建（含 CUDA DLL） | [docs/windows.md](docs/windows.md) | +| CUDA 后端、VRAM 专家层级、全部常驻 | [docs/cuda.md](docs/cuda.md) | +| Apple Silicon Metal 后端 | [docs/metal.md](docs/metal.md) | +| OpenAI 兼容 API、KV slots、网页仪表盘 | [docs/api.md](docs/api.md) | +| 语法强制草稿（结构化输出） | [docs/grammar-draft.md](docs/grammar-draft.md) | +| 环境变量完整清单 | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | + +## 支持项目 + +colibrì 最初由一人使用 12 核心、25 GB RAM 的笔记本开发； +如今它的数据来自社区中各种真实机器。如果这个项目对你有用： + +- ⭐ 为仓库加星并分享； +- 🐛 以 issue 提交你的硬件 benchmark 数据——实测数据比任何其他事都更能推动项目； +- 💬 若想赞助开发或捐赠硬件，请通过 GitHub issues 联系。 + +## 仓库结构 + +``` +Makefile 根目录构建／检查入口 +c/ +├── colibri.c 引擎主文件 +├── quant.h 量化 matmul 内核（SIMD 多架构） +├── sample.h 采样与 stop-set 管理 +├── kv_persist.h .coli_kv 磁盘持久化 +├── telemetry.h 仪表盘协议、统计与用量持久化 +├── st.h, tok.h, json.h 运行时头文件 +├── backend_cuda.* 可选的 CUDA 层级 +├── Makefile 构建与本地检查 +├── coli 用户界面 CLI +├── openai_server.py OpenAI 兼容 HTTP gateway +├── setup.sh 一条命令完成本地设置 +├── tools/ 离线转换、fixtures 与 benchmarks +├── scripts/ 长时间转换辅助工具 +└── tests/ 零依赖的 C 与 Python 测试 +web/ 浏览器 UI（纯 OpenAI API client） +desktop/ 封装网页 UI 的 Tauri v2 桌面 shell +docs/ 参考文档、实验与媒体文件 +``` + +运行时路径刻意保持扁平、易读：`colibri.c` 加上若干头文件。 +在仓库根目录执行 `make`、`make check` 与 `make clean`， +都会转发给引擎的 Makefile。 + +## 为什么叫"colibrì" + +蜂鸟只有几克重，能在原地悬停，并在一天内造访上千朵花。 +这套引擎只用蜂鸟般的配给，就能让 744B 参数的巨人运转： +25 GB RAM、十二个 CPU 核心，以及对磁盘的大量耐心。 + +## 许可证 + +Apache 2.0。GLM-5.2 权重由 Z.ai 以 MIT 许可发布。 diff --git a/README.zh-TW.md b/README.zh-TW.md index 1662617..2576eba 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -3,7 +3,7 @@

- English · 繁體中文 + English · 简体中文 · 繁體中文 · Italiano

**小巧引擎，龐大模型。**只要約 25 GB 記憶體，就能在消費級電腦上執行 **GLM-5.2（744B 參數的 MoE）**——以零相依套件的純 C 實作，從硬碟串流載入專家。 @@ -60,7 +60,7 @@ VRAM／RAM／硬碟層級長條，以及角落的即時迷你大腦。

**存放在硬碟**（約 370 GB），並**隨需串流載入**，搭配逐層 LRU 快取、 會學習的熱門專家固定儲存區，以及選用的 VRAM 層級。 -引擎由單一 C 檔（`c/glm.c`）與少量標頭檔組成。不需要 BLAS， +引擎由主 C 檔（`c/colibri.c`）與多個標頭檔模組組成。不需要 BLAS， 執行階段不需要 Python，也不需要 GPU。 ## 運作方式 @@ -203,7 +203,11 @@ colibrì 最初是由一人使用 12 核心、25 GB RAM 的筆電開發； ``` Makefile 根目錄建置／檢查入口 c/ -├── glm.c 單檔 GLM 引擎 +├── colibri.c GLM 引擎主檔 +├── quant.h 量化 matmul kernel +├── sample.h 取樣與 stop-set +├── kv_persist.h .coli_kv 磁碟持久化 +├── telemetry.h 儀表板協定、統計 ├── st.h, tok.h, json.h 執行階段標頭檔 ├── backend_cuda.* 選用的 CUDA 層級 ├── Makefile 建置與本機檢查 @@ -218,7 +222,7 @@ desktop/ 包裝網頁 UI 的 Tauri v2 桌面 shell docs/ 參考文件、實驗與媒體檔 ``` -執行階段路徑刻意維持扁平、易讀：`glm.c` 加上少量標頭檔。 +執行階段路徑刻意維持扁平、易讀：`colibri.c` 加上模組化標頭檔。 在儲存庫根目錄執行 `make`、`make check` 與 `make clean`， 都會轉交給引擎的 Makefile。 From e486574442724ff4ba79808a1ede254368b0638a Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Sun, 19 Jul 2026 04:22:46 +0800 Subject: [PATCH 71/95] =?UTF-8?q?web:=20i18n=20support=20=E2=80=94=20Engli?= =?UTF-8?q?sh,=20=E7=AE=80=E4=BD=93=E4=B8=AD=E6=96=87,=20=E7=B9=81?= =?UTF-8?q?=E9=AB=94=E4=B8=AD=E6=96=87,=20Italiano?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lightweight i18n without react-i18next: a LocaleProvider context + useLocale() hook with {{var}} interpolation, auto-detect from navigator.language, persisted to localStorage. 98 translation keys across 4 locale files (en/zh-CN/zh-TW/it). All user-visible strings in App/Brain/Profiling/ErrorBoundary are now t() calls. Language switcher added to the sidebar footer. Build clean (tsc + vite), 18 tests pass. --- web/src/App.tsx | 121 +++++++++++++++++++----------------- web/src/Brain.tsx | 43 +++++++------ web/src/ErrorBoundary.tsx | 21 +++++-- web/src/Profiling.tsx | 58 ++++++++---------- web/src/i18n/en.ts | 125 ++++++++++++++++++++++++++++++++++++++ web/src/i18n/index.ts | 78 ++++++++++++++++++++++++ web/src/i18n/it.ts | 113 ++++++++++++++++++++++++++++++++++ web/src/i18n/zh-CN.ts | 113 ++++++++++++++++++++++++++++++++++ web/src/i18n/zh-TW.ts | 113 ++++++++++++++++++++++++++++++++++ web/src/index.css | 4 +- web/src/main.tsx | 5 +- 11 files changed, 675 insertions(+), 119 deletions(-) create mode 100644 web/src/i18n/en.ts create mode 100644 web/src/i18n/index.ts create mode 100644 web/src/i18n/it.ts create mode 100644 web/src/i18n/zh-CN.ts create mode 100644 web/src/i18n/zh-TW.ts diff --git a/web/src/App.tsx b/web/src/App.tsx index a33cc88..39c9f3b 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -9,6 +9,7 @@ import { Database, Feather, Gauge, + Globe, HardDrive, KeyRound, Layers, @@ -34,6 +35,7 @@ import { Brain } from "./Brain" import { Profiling } from "./Profiling" import { persistPublicSettings, stored } from "@/lib/storage" import { cn } from "@/lib/utils" +import { useLocale } from "./i18n" const message = (role: ChatMessage["role"], content: string): ChatMessage => { let id: string @@ -42,15 +44,12 @@ const message = (role: ChatMessage["role"], content: string): ChatMessage => { } export default function App() { - // When the page is served by the engine itself (coli web), same-origin is the - // right default: no CORS, no manual endpoint editing. The Vite dev server - // (port 5173) keeps the classic default. + const { t, locale, setLocale, locales } = useLocale() + const servedByEngine = typeof window !== "undefined" && window.location.port !== "5173" && window.location.protocol.startsWith("http") const defaultBase = servedByEngine ? `${window.location.origin}/v1` : "http://127.0.0.1:8000/v1" const [baseUrl, setBaseUrl] = useState(() => { const saved = stored(localStorage, "colibri.baseUrl", defaultBase) - // migrate: a stored FACTORY default pointing at another origin would trip CORS - // when the page is engine-served — upgrade it to same-origin once. if (servedByEngine && saved === "http://127.0.0.1:8000/v1" && defaultBase !== saved) return defaultBase return saved }) @@ -120,7 +119,7 @@ export default function App() { const result = await getHealth(baseUrl, apiKey) if (!disposed) { setHealth(result); setHealthError("") } } catch (cause) { - if (!disposed) setHealthError(cause instanceof Error ? cause.message : "Runtime metrics unavailable") + if (!disposed) setHealthError(cause instanceof Error ? cause.message : "status.runtimeUnavailable") } } const timer = window.setInterval(() => void poll(), 5000) @@ -155,13 +154,13 @@ export default function App() { } catch (cause) { if (!controller.signal.aborted) { setHealth(null) - setHealthError(cause instanceof Error ? cause.message : "Runtime metrics unavailable") + setHealthError(cause instanceof Error ? cause.message : "status.runtimeUnavailable") } } } catch (cause) { if (controller.signal.aborted) return setConnected(false) - setError(cause instanceof Error ? cause.message : "Could not reach the server.") + setError(cause instanceof Error ? cause.message : "status.serverError") } finally { if (probeRef.current === controller) { probeRef.current = null; setConnecting(false) } } @@ -227,7 +226,7 @@ export default function App() { if (controller.signal.aborted) { updateMessages((current) => current.filter((item) => item.id !== assistant.id || item.content)) } else { - setError(cause instanceof Error ? cause.message : "Generation failed.") + setError(cause instanceof Error ? cause.message : "status.generationFailed") updateMessages((current) => current.filter((item) => item.id !== assistant.id || item.content)) } } finally { @@ -241,22 +240,22 @@ export default function App() {
-
ACTIVE MODEL{model}
+
{t("topbar.activeModel")}{model}
- - - + + +
- {loading && tokenCount > 0 ? {tokenCount} tokens : null} - {!loading && tokPerSec != null ? {tokPerSec.toFixed(1)} tok/s : null} + {loading && tokenCount > 0 ? {t("topbar.tokens", { n: tokenCount })} : null} + {!loading && tokPerSec != null ? {t("topbar.tokPerSec", { n: tokPerSec.toFixed(1) })} : null} {!loading && ttft != null ? TTFT {(ttft/1000).toFixed(1)}s : null} {!loading && lastRun?.usage ? {lastRun.usage.prompt_tokens}→{lastRun.usage.completion_tokens} : null} {lastRun?.queueWaitMs != null ? queue {Math.round(lastRun.queueWaitMs)}ms : null} - slot {cacheSlot + 1} - + {t("topbar.slot", { n: cacheSlot + 1 })} +
@@ -335,11 +342,11 @@ export default function App() { {!messages.length ? (
- COLIBRÌ ENGINE -

Ask the giant.
Keep the machine yours.

-

Connect to a local colibrì server and stream responses directly from your hardware. Nothing leaves the endpoint you choose.

+ {t("hero.title")} +

{t("hero.subtitle")}
{t("hero.tagline")}

+

{t("hero.description")}

- {["Explain how expert routing works", "Write a small C benchmark", "Compare RAM and VRAM caching"].map((item) => )} + {[t("prompts.routing"), t("prompts.benchmark"), t("prompts.caching")].map((item) => )}
) : ( @@ -347,7 +354,7 @@ export default function App() { {messages.map((item) => (
{item.role === "user" ? "Y" : }
-
{item.role === "user" ? "You" : "colibrì"}
{item.content || }
+
{item.role === "user" ? t("chat.you") : t("chat.colibri")}
{item.content || }
))}
@@ -356,10 +363,10 @@ export default function App() {
- {error &&
{error}
} + {error &&
{t(error)}
}
-