woolcoxm
6e29260635
The new g64 model quantizes experts with group-size 64 (fmt=4): one f32 scale per 64 elements instead of per-row. This required changes across the entire compute stack — CUDA kernel, engine dispatch, and dequant helpers — plus a new fused gate+up kernel to recover the ~40% throughput that the generic grouped path costs. CUDA backend (backend_cuda.cu): - ColiCudaTensor: add gs/ng fields for group-size metadata - row_bytes/weight_at: fmt=4 uses same packed int4 layout as fmt=2 - quant_matmul: apply per-group scales (scales[o*ng+g]) for fmt=4, matching the CPU matmul_i4_grouped accumulation exactly - coli_cuda_tensor_upload: allocate O*ng scales (not O) for fmt=4, store gs/ng on the tensor - coli_cuda_tensor_update: handle grouped scales on refresh - coli_cuda_tensor_free/tensor_bytes: VRAM accounting uses O*ng - coli_cuda_expert_group: return 0 for fmt=4 (fall back to correct per-expert path, since grouped_hidden/grouped_down kernels only handle per-row scales) - All 11 quant_matmul call sites updated to pass gs,ng Engine (glm.c): - qt_cuda_upload: pass t->gs to tensor_upload - matmul dispatch (line 816): pass w->gs to coli_cuda_matmul - kv_b shard upload: pass l->kv_b.gs - kv_b shard rb computation: add fmt=4 case ((I+1)/2) - embed_row: add fmt=4 branch with per-group scale dequant - qt_addrow/qt_matvec_rows: add fmt=4 branches (CPU MLA absorption path) - expert_gate_up: add matmul_i4_grouped_pair — fused gate+up for fmt=4 that reads x once instead of twice (+40% tok/s, 0.77 -> 1.08) - run_text: prepend [gMASK]<sop> prefix for GLM models (#108 fix — without it, PROMPT mode generates garbage on all GLM snapshots) API (backend_cuda.h, backend_loader.c): - coli_cuda_tensor_upload and coli_cuda_matmul signatures: add gs param - Loader typedefs and wrappers thread gs through the DLL boundary Tests: - bench_tensor_core.cu, test_backend_cuda.cu, test_pipe_cuda.cu: update all calls to new API signature (append gs=0 for non-grouped) New tool: c/tools/diag_harness.py - Comprehensive model diagnostic harness (system probe, correctness smoke, deep PROFILE diagnostic, quality benchmarks via eval_glm.py, throughput with/without MTP). Outputs JSON + Markdown reports. Performance (GLM-5.2 744B g64 / RTX 5070 Ti / 32GB RAM): broken CUDA: 0.05 tok/s (every tensor fell back to CPU) fixed CUDA: 0.30 tok/s (6x — CUDA working) + full opt stack: 0.77 tok/s (CACHE_ROUTE + EXPERT_BUDGET) + fused grouped pair: 1.08 tok/s (+40% from gate+up fusion)
46 lines
2.4 KiB
Plaintext
46 lines
2.4 KiB
Plaintext
#include "../backend_cuda.h"
|
|
|
|
#include <chrono>
|
|
#include <cmath>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <vector>
|
|
|
|
static double run(ColiCudaTensor *g,ColiCudaTensor *u,ColiCudaTensor *d,
|
|
const float *x,float *y,int rows,int iterations,int mode){
|
|
ColiCudaTensor *gs[1]={g},*us[1]={u},*ds[1]={d}; int rs[1]={rows};
|
|
if(mode==2){setenv("COLI_CUDA_TC_INT4","1",1);setenv("COLI_CUDA_TC_MIN_ROWS","1",1);}
|
|
else unsetenv("COLI_CUDA_TC_INT4");
|
|
setenv("COLI_CUDA_W4_PACKED",mode==0?"0":"1",1);
|
|
if(!coli_cuda_expert_group(gs,us,ds,rs,1,y,x))std::exit(2);
|
|
auto begin=std::chrono::steady_clock::now();
|
|
for(int i=0;i<iterations;i++)if(!coli_cuda_expert_group(gs,us,ds,rs,1,y,x))std::exit(2);
|
|
auto end=std::chrono::steady_clock::now();
|
|
return std::chrono::duration<double,std::milli>(end-begin).count()/iterations;
|
|
}
|
|
|
|
int main(){
|
|
constexpr int D=6144,I=2048,O=8;
|
|
int device=0;if(!coli_cuda_init(&device,1))return 77;
|
|
std::vector<unsigned char> hidden((size_t)I*D/2),down((size_t)D*I/2);
|
|
std::vector<float> hs(I),ds(D),x((size_t)O*D),a((size_t)O*D),b((size_t)O*D),c((size_t)O*D);
|
|
for(size_t i=0;i<hidden.size();i++)hidden[i]=(unsigned char)((i*17+29)&255);
|
|
for(size_t i=0;i<down.size();i++)down[i]=(unsigned char)((i*13+41)&255);
|
|
for(int i=0;i<I;i++)hs[i]=0.006f+(i%11)*0.0002f;
|
|
for(int i=0;i<D;i++)ds[i]=0.006f+(i%7)*0.0002f;
|
|
for(size_t i=0;i<x.size();i++)x[i]=std::sin((float)(i+1)*0.013f)*2.f;
|
|
ColiCudaTensor *g=nullptr,*u=nullptr,*d=nullptr;
|
|
if(!coli_cuda_tensor_upload(&g,hidden.data(),hs.data(),2,D,I,device,0)||
|
|
!coli_cuda_tensor_upload(&u,hidden.data(),hs.data(),2,D,I,device,0)||
|
|
!coli_cuda_tensor_upload(&d,down.data(),ds.data(),2,I,D,device,0))return 2;
|
|
for(int rows: {1,2,4,8}){
|
|
double scalar=run(g,u,d,x.data(),a.data(),rows,3,0);
|
|
double packed=run(g,u,d,x.data(),b.data(),rows,3,1);
|
|
double tc=run(g,u,d,x.data(),c.data(),rows,3,2);
|
|
double pe=0,te=0,ref=0;for(int i=0;i<rows*D;i++){double p=b[i]-a[i],t=c[i]-a[i];pe+=p*p;te+=t*t;ref+=(double)a[i]*a[i];}
|
|
std::printf("rows=%d scalar_ms=%.3f packed_ms=%.3f packed_speedup=%.3fx packed_rms=%.7f tensor_ms=%.3f tensor_speedup=%.3fx tensor_rms=%.5f\n",
|
|
rows,scalar,packed,scalar/packed,std::sqrt(pe/(ref+1e-20)),tc,scalar/tc,std::sqrt(te/(ref+1e-20)));
|
|
}
|
|
coli_cuda_tensor_free(g);coli_cuda_tensor_free(u);coli_cuda_tensor_free(d);coli_cuda_shutdown();
|
|
}
|