url/colibri
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cuda+engine: full fmt=4 (grouped int4 gs=64) support + diagnostic harness

Browse Source 
The new g64 model quantizes experts with group-size 64 (fmt=4): one f32
scale per 64 elements instead of per-row. This required changes across
the entire compute stack — CUDA kernel, engine dispatch, and dequant
helpers — plus a new fused gate+up kernel to recover the ~40% throughput
that the generic grouped path costs.

CUDA backend (backend_cuda.cu):
- ColiCudaTensor: add gs/ng fields for group-size metadata
- row_bytes/weight_at: fmt=4 uses same packed int4 layout as fmt=2
- quant_matmul: apply per-group scales (scales[o*ng+g]) for fmt=4,
  matching the CPU matmul_i4_grouped accumulation exactly
- coli_cuda_tensor_upload: allocate O*ng scales (not O) for fmt=4,
  store gs/ng on the tensor
- coli_cuda_tensor_update: handle grouped scales on refresh
- coli_cuda_tensor_free/tensor_bytes: VRAM accounting uses O*ng
- coli_cuda_expert_group: return 0 for fmt=4 (fall back to correct
  per-expert path, since grouped_hidden/grouped_down kernels only
  handle per-row scales)
- All 11 quant_matmul call sites updated to pass gs,ng

Engine (glm.c):
- qt_cuda_upload: pass t->gs to tensor_upload
- matmul dispatch (line 816): pass w->gs to coli_cuda_matmul
- kv_b shard upload: pass l->kv_b.gs
- kv_b shard rb computation: add fmt=4 case ((I+1)/2)
- embed_row: add fmt=4 branch with per-group scale dequant
- qt_addrow/qt_matvec_rows: add fmt=4 branches (CPU MLA absorption path)
- expert_gate_up: add matmul_i4_grouped_pair — fused gate+up for fmt=4
  that reads x once instead of twice (+40% tok/s, 0.77 -> 1.08)
- run_text: prepend [gMASK]<sop> prefix for GLM models (#108 fix —
  without it, PROMPT mode generates garbage on all GLM snapshots)

API (backend_cuda.h, backend_loader.c):
- coli_cuda_tensor_upload and coli_cuda_matmul signatures: add gs param
- Loader typedefs and wrappers thread gs through the DLL boundary

Tests:
- bench_tensor_core.cu, test_backend_cuda.cu, test_pipe_cuda.cu:
  update all calls to new API signature (append gs=0 for non-grouped)

New tool: c/tools/diag_harness.py
- Comprehensive model diagnostic harness (system probe, correctness
  smoke, deep PROFILE diagnostic, quality benchmarks via eval_glm.py,
  throughput with/without MTP). Outputs JSON + Markdown reports.

Performance (GLM-5.2 744B g64 / RTX 5070 Ti / 32GB RAM):
  broken CUDA:           0.05 tok/s (every tensor fell back to CPU)
  fixed CUDA:            0.30 tok/s (6x — CUDA working)
  + full opt stack:      0.77 tok/s (CACHE_ROUTE + EXPERT_BUDGET)
  + fused grouped pair:  1.08 tok/s (+40% from gate+up fusion)
This commit is contained in:
woolcoxm
2026-07-16 01:09:47 -04:00
parent 86987154f5
commit 6e29260635
8 changed files with 1524 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c/tests/bench_tensor_core.cu
+3 -3
View File
@@ -30,9 +30,9 @@ int main(){
for(int i=0;i<D;i++)ds[i]=0.006f+(i%7)*0.0002f;
for(size_t i=0;i<x.size();i++)x[i]=std::sin((float)(i+1)*0.013f)*2.f;
ColiCudaTensor *g=nullptr,*u=nullptr,*d=nullptr;
if(!coli_cuda_tensor_upload(&g,hidden.data(),hs.data(),2,D,I,device)||
!coli_cuda_tensor_upload(&u,hidden.data(),hs.data(),2,D,I,device)||
!coli_cuda_tensor_upload(&d,down.data(),ds.data(),2,I,D,device))return 2;
if(!coli_cuda_tensor_upload(&g,hidden.data(),hs.data(),2,D,I,device,0)||
!coli_cuda_tensor_upload(&u,hidden.data(),hs.data(),2,D,I,device,0)||
!coli_cuda_tensor_upload(&d,down.data(),ds.data(),2,I,D,device,0))return 2;
for(int rows: {1,2,4,8}){
double scalar=run(g,u,d,x.data(),a.data(),rows,3,0);
double packed=run(g,u,d,x.data(),b.data(),rows,3,1);
c/tests/test_backend_cuda.cu
+11 -11
View File
@@ -99,7 +99,7 @@ int main(int argc, char **argv) {
const int8_t q8b[8]={-1,-2,-3,-4, 1,-2,3,-4};
const float s8b[2]={1.f,.5f},want8b[4]={10.f,15.f,-3.f,-2.5f};
if(!coli_cuda_tensor_update(t8,q8b,s8b)||
!coli_cuda_matmul(&t8,got,x,q8b,s8b,1,2,4,2,d0)||
!coli_cuda_matmul(&t8,got,x,q8b,s8b,1,2,4,2,d0,0)||
!close_enough(got,want8b,4))return 1;
/* Rows [-8,-1,0,7] and [1,2,3,4], packed low nibble first. */
@@ -107,26 +107,26 @@ int main(int argc, char **argv) {
const float s4[2] = {1.0f, 0.25f};
const float want4[2] = {-34.0f, -2.5f};
ColiCudaTensor *t4 = nullptr;
if (!coli_cuda_matmul(&t4, got, x, q4, s4, 2, 1, 4, 2, d1) || !close_enough(got, want4, 2)) return 1;
if (!coli_cuda_matmul(&t4, got, x, q4, s4, 2, 1, 4, 2, d1, 0) || !close_enough(got, want4, 2)) return 1;
const uint8_t q2[2] = {0xe4, 0x1b};
const float s2[2] = {0.5f, 2.0f};
const float want2[2] = {-2.0f, 12.0f};
ColiCudaTensor *t2 = nullptr;
if (!coli_cuda_matmul(&t2, got, x, q2, s2, 3, 1, 4, 2, d1) || !close_enough(got, want2, 2)) return 1;
if (!coli_cuda_matmul(&t2, got, x, q2, s2, 3, 1, 4, 2, d1, 0) || !close_enough(got, want2, 2)) return 1;
const float wf[8] = {1, 0, -1, 2, 0.5f, 0.5f, 0.5f, 0.5f};
const float wantf[2] = {-10.0f, -1.0f};
ColiCudaTensor *tf = nullptr;
if (!coli_cuda_matmul(&tf, got, x, wf, nullptr, 0, 1, 4, 2, d0) || !close_enough(got, wantf, 2)) return 1;
if (!coli_cuda_matmul(&tf, got, x, wf, nullptr, 0, 1, 4, 2, d0, 0) || !close_enough(got, wantf, 2)) return 1;
const float eg[8] = {1,0,0,0, 0,1,0,0};
const float eu[8] = {1,0,0,0, 0,1,0,0};
const float ed[8] = {1,0, 0,1, 1,1, 1,-1};
ColiCudaTensor *tg=nullptr,*tu=nullptr,*td=nullptr;
if (!coli_cuda_tensor_upload(&tg,eg,nullptr,0,4,2,d0) ||
!coli_cuda_tensor_upload(&tu,eu,nullptr,0,4,2,d0) ||
!coli_cuda_tensor_upload(&td,ed,nullptr,0,2,4,d0)) return 1;
if (!coli_cuda_tensor_upload(&tg,eg,nullptr,0,4,2,d0,0) ||
!coli_cuda_tensor_upload(&tu,eu,nullptr,0,4,2,d0,0) ||
!coli_cuda_tensor_upload(&td,ed,nullptr,0,2,4,d0,0)) return 1;
float expert[8], want_expert[8];
for(int s=0;s<2;s++){
float a=x[s*4], b=x[s*4+1];
@@ -144,7 +144,7 @@ int main(int argc, char **argv) {
const float aw[16]={1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1};
const float aq[4]={1,2,.5f,-.5f},al[12]={1,0,0,0, 0,1,0,0, 0,0,1,0};
const float ar[6]={1,0, 0,1, 1,1};float actx[2],aref[2];
ColiCudaTensor *at=nullptr;if(!coli_cuda_tensor_upload(&at,aw,nullptr,0,4,4,d0))return 1;
ColiCudaTensor *at=nullptr;if(!coli_cuda_tensor_upload(&at,aw,nullptr,0,4,4,d0,0))return 1;
float score[3];for(int t=0;t<3;t++)score[t]=aq[0]*al[t*4]+aq[1]*al[t*4+1]+aq[2]*ar[t*2]+aq[3]*ar[t*2+1];
float mx=score[0],z=0;for(int t=1;t<3;t++)mx=score[t]>mx?score[t]:mx;
for(int t=0;t<3;t++){score[t]=std::exp(score[t]-mx);z+=score[t];}for(int t=0;t<3;t++)score[t]/=z;
@@ -163,9 +163,9 @@ int main(int argc, char **argv) {
for(int i=0;i<32;i++)ws4[i]=0.01f+(i%5)*0.002f;
for(int i=0;i<64;i++)gx4[i]=std::sin((float)(i+1)*0.17f)*2.f;
ColiCudaTensor *g4=nullptr,*u4=nullptr,*d4=nullptr;
if(!coli_cuda_tensor_upload(&g4,w4,ws4,2,32,32,d0)||
!coli_cuda_tensor_upload(&u4,w4,ws4,2,32,32,d0)||
!coli_cuda_tensor_upload(&d4,w4,ws4,2,32,32,d0))return 1;
if(!coli_cuda_tensor_upload(&g4,w4,ws4,2,32,32,d0,0)||
!coli_cuda_tensor_upload(&u4,w4,ws4,2,32,32,d0,0)||
!coli_cuda_tensor_upload(&d4,w4,ws4,2,32,32,d0,0))return 1;
ColiCudaTensor *gg4[2]={g4,g4},*ug4[2]={u4,u4},*dg4[2]={d4,d4};
if(!coli_cuda_expert_group(gg4,ug4,dg4,group_rows,2,scalar4,gx4))return 1;
setenv("COLI_CUDA_TC_INT4","1",1);
c/tests/test_pipe_cuda.cu
+1 -1
View File
@@ -119,7 +119,7 @@ int main(void){
for(int i=0;i<O;i++) sc[i]=0.01f+0.001f*(i%7);
for(size_t i=0;i<(size_t)S*K;i++) x[i]=rndf();
ColiCudaTensor *t=NULL;
ok&=coli_cuda_matmul(&t,ref,x,w4,sc,2,S,K,O,0); /* host path = reference */
ok&=coli_cuda_matmul(&t,ref,x,w4,sc,2,S,K,O,0,0); /* host path = reference */
float *xd=(float*)coli_cuda_pipe_alloc(0,(size_t)S*K*4);
float *yd=(float*)coli_cuda_pipe_alloc(0,(size_t)S*O*4);
ok&=coli_cuda_pipe_upload(0,xd,x,(size_t)S*K*4);