url/colibri
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

int3-g64 (fmt=5): per-group-scale 3-bit weight format — engine, converter, tests

Browse Source 
New weight format: int3 with ONE f32 scale per 64-input group (3.5 bits/weight
effective). Per group: 16B low plane (2 bits/val, int2 layout) + 8B high plane
(1 bit/val), values [-4,3] stored v+4. Same quantization math as
tools/quant_ablation.py _quant_last_dim(bits=3, group=64) from #132, whose
OLMoE ablation measured int3-g64 BEATING the shipped per-row int4 on quality
(-7.5 vs -9.3pp) at ~14% fewer bits.

Engine (placed per the #391 split): matmul_i3 + pack_int3_g64 + I3_* layout
helpers in quant.h next to their kernel family; fmt=5 branches in colibri.c's
qt_bytes/qt_alloc/qt_fill/matmul_qt/embed_row/qt_addrow/qt_matvec_rows.

Format detection now goes through #413's qt_resolve_fmt: fmt=5 registers its
distinct weight-byte layout O*ceil(I/64)*24 and its scale cardinality
O*ceil(I/64) there, validated against [O,I] like every other format. int3-g64
and grouped-int4-at-gs=64 carry the SAME scale count, so the weight bytes are
the int3 tag; row formats keep precedence for the small-I shapes where byte
counts coincide. The io_uring expert path still used the raw ?1:?2:3 byte
inference (it missed fmt=4 grouping entirely and never set gs) — converted to
qt_resolve_fmt like the other two expert paths.

Backends: qt_cuda_upload returns 0 for fmt=5 (tensor stays CPU-side, the
documented fallback), the dense CUDA matmul gate excludes fmt=5, and Metal's
existing fmt gates (gemm fmt<=3, moe fmt 1/2) already reject it.

Converter: quant_int3_g64 in convert_fp8_to_int4.py; --ebits 3/--xbits 3 now
emit it (previously bits=3 silently produced int4).

Tests: tests/test_int3.c (bit-exact pack/unpack vs reference, matmul_i3 vs
dequant-matmul incl. short tail groups and the real GLM I=7168, QT plumbing,
qt_resolve_fmt disambiguation incl. the same-scale-count fmt=4/fmt=5 pair,
outlier-rows RMS: int3-g64 3.3x lower error than per-row int4),
tests/test_int3_load.c (hand-rolled .safetensors fixture through st_init +
qt_from_disk: fmt=5 detected and loaded next to an int4 control tensor),
tests/test_int3_convert.py (NumPy pack round-trip vs independent decoder).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
FABIOTESS
2026-07-20 17:31:05 +01:00
parent ebd85a781b
commit 5e42e70ec3
7 changed files with 503 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c/quant.h
+90
View File
@@ -268,6 +268,68 @@ static void matmul_i2(float *y, const float *x, const uint8_t *q2, const float *
y[(int64_t)s*O+o]=a*sc; } }
}
/* ---- int3-g64 (fmt=5): 3-bit weights with ONE f32 scale per 64-input group -
* Per group: 16B low plane (2 bits/val, int2 layout) + 8B high plane (1 bit/val),
* values in [-4,3] stored v+4. 3.5 bits/weight effective — the quality/size point
* the #132 OLMoE ablation measured BEATING per-row int4. */
#define I3_GROUP 64
#define I3_GBYTES 24 /* 16B low plane + 8B high plane per group */
static inline int64_t i3_groups(int I){ return ((int64_t)I + I3_GROUP - 1) / I3_GROUP; }
static inline int64_t i3_rowbytes(int I){ return i3_groups(I) * I3_GBYTES; }
/* Dequant-on-use with PER-GROUP scale. Exact f32 path only (no IDOT in v1: int8
* activations don't compose with per-group accumulation without a kernel
* restructure — follow-up). NEON: low plane = matmul_i2's unpack, high plane
* expanded via vtst on bit masks; x86 stays scalar for now (follow-up). */
static void matmul_i3(float *y, const float *x, const uint8_t *q3, const float *scale, int S, int I, int O){
int64_t ng=i3_groups(I), rb=i3_rowbytes(I);
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){
const uint8_t *wrow=q3+(int64_t)o*rb;
const float *srow=scale+(int64_t)o*ng;
for(int s=0;s<S;s++){
const float *xs=x+(int64_t)s*I;
float acc=0;
for(int64_t g=0; g<ng; g++){
const uint8_t *lo=wrow+g*I3_GBYTES, *hi=lo+16;
int base=(int)(g*I3_GROUP), n = I-base < I3_GROUP ? I-base : I3_GROUP;
float a=0; int k=0;
#if defined(__ARM_NEON)
if(n==I3_GROUP){
const uint8x8_t m2v=vdup_n_u8(3); const int8x16_t b4q=vdupq_n_s8(4);
const uint8x16_t bitm={1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128};
const uint8x16_t fourq=vdupq_n_u8(4);
float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
for(;k+16<=I3_GROUP;k+=16){
uint32_t wd; memcpy(&wd, lo+(k>>2), 4); /* 4 bytes = 16 low-plane values */
uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd));
uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v));
uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6));
uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0]));
uint8x16_t lov=vcombine_u8(vreinterpret_u8_u16(zz.val[0]), vreinterpret_u8_u16(zz.val[1]));
uint8x16_t hv=vcombine_u8(vdup_n_u8(hi[k>>3]), vdup_n_u8(hi[(k>>3)+1]));
uint8x16_t hb=vandq_u8(vtstq_u8(hv,bitm), fourq); /* 4 where high bit set */
int8x16_t wq=vsubq_s8(vreinterpretq_s8_u8(vaddq_u8(lov,hb)), b4q); /* [-4,3] in order */
int16x8_t w0=vmovl_s8(vget_low_s8(wq)), w1=vmovl_s8(vget_high_s8(wq));
ac0=vfmaq_f32(ac0, vld1q_f32(xs+base+k), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
ac1=vfmaq_f32(ac1, vld1q_f32(xs+base+k+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
ac0=vfmaq_f32(ac0, vld1q_f32(xs+base+k+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
ac1=vfmaq_f32(ac1, vld1q_f32(xs+base+k+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1))));
}
a=vaddvq_f32(vaddq_f32(ac0,ac1));
}
#endif
for(;k<n;k++){
unsigned u=((lo[k>>2]>>((k&3)*2))&3) | (((hi[k>>3]>>(k&7))&1)<<2);
a += xs[base+k]*(float)((int)u-4);
}
acc += a*srow[g];
}
y[(int64_t)s*O+o]=acc;
}
}
}
/* ---- IDOT: integer dot kernels (int8-quantized activations) --------------- */
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
#define IDOT_KERNEL "avx512-vnni"
@@ -689,6 +751,34 @@ static void pack_int4(const float *w, uint8_t *q4, float *scale, int O, int I, i
}
}
}
/* quantize w[O,I] f32 -> int3-g64 (fmt=5): per 64-input group, symmetric absmax
* (qmax=3, clamp [-4,3], stored v+4), 16B low plane + 8B high plane, ONE f32 scale
* per group. Same math as tools/quant_ablation.py `_quant_last_dim(bits=3, group=64)`
* (#132), here with real bit packing. */
static void pack_int3_g64(const float *w, uint8_t *q3, float *scale, int O, int I){
int64_t ng=i3_groups(I), rb=i3_rowbytes(I);
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){
const float *wr=w+(int64_t)o*I;
uint8_t *qr=q3+(int64_t)o*rb;
float *sr=scale+(int64_t)o*ng;
for(int64_t g=0; g<ng; g++){
int base=(int)(g*I3_GROUP), n = I-base < I3_GROUP ? I-base : I3_GROUP;
float amax=0;
for(int k=0;k<n;k++){ float a=fabsf(wr[base+k]); if(a>amax)amax=a; }
float s=amax/3.f; if(s<1e-8f)s=1e-8f; sr[g]=s;
uint8_t *lo=qr+g*I3_GBYTES, *hi=lo+16;
memset(lo,0,I3_GBYTES);
for(int k=0;k<n;k++){
int v=(int)lrintf(wr[base+k]/s); if(v>3)v=3; if(v<-4)v=-4;
unsigned u=(unsigned)(v+4); /* 0..7 */
lo[k>>2] |= (uint8_t)((u&3)<<((k&3)*2));
hi[k>>3] |= (uint8_t)(((u>>2)&1)<<(k&7));
}
}
}
}
static void pack_int2(const float *w, uint8_t *q2, float *scale, int O, int I, int bits){
int qmax=(1<<(bits-1))-1, rb=(I+3)/4;
#pragma omp parallel for schedule(static)