url/colibri
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

quant: generalize fmt=4 group-size detection (g64/g128/g256, not just 128)

Browse Source 
The grouped-int4 (fmt=4) loader hard-coded gs=128 at all three detection
sites (resident weights + the two expert-load paths: mmap and pread-slab).
A checkpoint converted with --group-size 64 (or any non-128 size) wrote a
valid file that the engine then misdetected as plain per-row int4 (fmt=2)
and read the wrong number of scales -> garbage output, silently.

The conversion path (convert_fp8_to_int4.py --group-size) already emits
arbitrary group sizes, and the compute kernel (matmul_i4_grouped) is
fully gs-generic (only constraint: gs multiple of 16, the AVX2 width).
The loader was the sole gap.

Replace the three hardcoded checks with one shared helper, detect_group_size(),
that derives gs from the scale-array byte count by probing candidate sizes
{16,32,48,64,96,128,192,256} finest-first. Data-driven: any listed size
just works; per-row int4 (ns == O*4) correctly returns gs=0 -> stays fmt=2.

Verified against real GLM-5.2 expert dims (gate/up O=2048,I=6144 and down
O=6144,I=2048): g64/g128/g256 all detect correctly, and per-row is never
misdetected as grouped. g128 (the only previously-supported size) is
unchanged -> no regression for existing checkpoints.

This unblocks the g64 lane that ZacharyZcR's #225 ablation identified as
beating shipped per-row int4 (-7.5pp vs -9.3pp) at ~19% fewer bits: the
converter can produce it, and now the engine can load it.

Refs #225
This commit is contained in:
woolcoxm
2026-07-15 15:54:37 -04:00
parent 704b4d7eeb
commit 55576941cc
1 changed files with 30 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c/glm.c
+30 -15
View File
@@ -1282,6 +1282,29 @@ static void load_cfg(Cfg *c, const char *snap){
free(ar);
}
/* Derive the fmt=4 group size from the scale-array byte count. A grouped-int4
* tensor stores ceil(I/gs) f32 scales per output row, so:
* ns_bytes == O * ceil(I/gs) * 4 => gs == I * 4 / (ns_bytes/O - ... )
* We probe candidate group sizes (must be a multiple of 16, the AVX2 vector
* width the grouped kernel requires) from finest to coarsest and return the
* first whose predicted scale-array size matches ns_bytes. Returns 0 if no
* candidate fits (then it's plain per-row int4, fmt=2, not grouped).
* Data-driven: g64/g128/g256 all just work; adding a size means listing it. */
static int detect_group_size(int O, int I, int64_t ns){
if(O<=0 || ns<=(int64_t)O*4 || I<=0) return 0; /* not grouped */
/* ns/O is the per-row scale bytes; groups = (ns/O)/4; gs = ceil(I/groups).
* Probe from small gs (finest granularity) upward so the most granular
* match wins — that's what we want, since finer groups are unambiguous. */
static const int cands[]={16,32,48,64,96,128,192,256};
for(int ci=0; ci<(int)(sizeof(cands)/sizeof(cands[0])); ci++){
int gs=cands[ci];
if(gs>I) break;
int ng=(I+gs-1)/gs;
if(ns==(int64_t)O*ng*4) return gs;
}
return 0;
}
/* costruisce un QT [O,I] dal disco in `t` (buffer riusabili tra chiamate).
* - se esiste `name.qs`: pesi GIA' quantizzati nel container (U8 qdata + F32 scala) -> letti diretti
* - altrimenti: tensore pieno (f32/bf16) -> quantizzato a runtime a `bits` (oracolo tiny / pesi pieni)
@@ -1292,15 +1315,11 @@ static void qt_from_disk(Model *m, const char *name, int O, int I, int bits, int
int64_t nb=st_nbytes(&m->S,name);
int64_t ns=st_nbytes(&m->S,sn); /* scale bytes (F32) */
/* Detect int4-grouped (fmt=4): packed int4 weight bytes BUT scale array is
* larger than O*4 — check if it matches O*ceil(I/gs)*4 for gs=128. */
* larger than O*4 — the group size is derived from the scale-array size. */
int fmt = (nb==(int64_t)O*I)?1 : (nb==(int64_t)O*((I+1)/2))?2 : 3;
int gs=0;
if(fmt==2 && ns > (int64_t)O*4){
/* could be grouped; try gs=128 */
int ng128=(I+127)/128;
if(ns==(int64_t)O*ng128*4){ fmt=4; gs=128; }
/* future: try other group sizes here */
}
if(fmt==2) gs=detect_group_size(O,I,ns);
if(gs>0) fmt=4;
if(fmt==1){ if(t->fmt!=1||!t->q8){ t->fmt=1; t->O=O; t->I=I; t->gs=0; t->q8=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q8,drop); }
else if(fmt==4){ int ng=(I+gs-1)/gs;
if(t->fmt!=4||!t->q4){ t->fmt=4; t->O=O; t->I=I; t->gs=gs; t->q4=qalloc(nb); t->s=falloc((int64_t)O*ng); }
@@ -1630,10 +1649,8 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){
int fmt=(nb==(int64_t)OO[k]*II[k])?1:(nb==(int64_t)OO[k]*((II[k]+1)/2))?2:3;
/* detect grouped int4 (fmt=4): int4 weight bytes + larger scale array */
int gs=0;
if(fmt==2 && tq[k]->nbytes > (int64_t)OO[k]*4){
int ng128=(II[k]+127)/128;
if(tq[k]->nbytes==(int64_t)OO[k]*ng128*4){ fmt=4; gs=128; }
}
if(fmt==2) gs=detect_group_size(OO[k],II[k],tq[k]->nbytes);
if(gs>0) fmt=4;
qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL;
qt[k]->q8=(int8_t*)((char*)bw[k]+tw[k]->off); qt[k]->q4=(uint8_t*)((char*)bw[k]+tw[k]->off);
qt[k]->s=(float*)((char*)bq[k]+tq[k]->off);
@@ -1749,10 +1766,8 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){
int64_t nb=tw[k]->nbytes;
int fmt = (nb==(int64_t)OO[k]*II[k])?1 : (nb==(int64_t)OO[k]*((II[k]+1)/2))?2 : 3;
int gs=0;
if(fmt==2 && tq[k]->nbytes > (int64_t)OO[k]*4){
int ng128=(II[k]+127)/128;
if(tq[k]->nbytes==(int64_t)OO[k]*ng128*4){ fmt=4; gs=128; }
}
if(fmt==2) gs=detect_group_size(OO[k],II[k],tq[k]->nbytes);
if(gs>0) fmt=4;
qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL;
qt[k]->q8=(int8_t*)(s->slab+pos[k]); qt[k]->q4=s->slab+pos[k]; qt[k]->s=fp[k];
}