cuda: grouped-int4 (fmt=4) support in the expert-group kernels (#334)
The grouped MoE kernels were per-row-only: GroupDesc had no group-size fields, row_bytes() returned 0 for fmt=4, the scale buffer was hardcoded to O floats, and a fmt=4 group that reached the generic path would have been silently decoded as int2. This closed the GPU expert tier to every grouped container — including the g64 quality line (#225) and the E8 lattice route (#347) whose whole point is fitting more experts in VRAM. - ColiCudaTensor gains gs/scale_count; upload allocates O*ceil(I/gs) scales for fmt=4 and applies the same offset->signed nibble conversion as fmt=2 (identical packing). New ABI entry coli_cuda_tensor_upload_g carries gs without touching the existing symbol — an old Windows DLL missing it returns 0 and the tensor simply stays CPU-side. - GroupDesc gains per-tensor group sizes; new grouped_hidden_g4_dual / grouped_down_g4 apply the per-group scale inside the accumulation (gs is required even, so a packed byte never straddles groups; gs=0 degrades to per-row, letting fmt=2 members ride the same launch). - coli_cuda_expert_group routes any group containing fmt=4 through the g4 kernels; pure-fmt=2 groups keep the existing paths byte-identical. The generic fallback now explicitly rejects fmt=4 instead of decoding garbage (#334's prevention note, made real). tests/test_grouped_g4_cuda.cu: kernel-vs-CPU oracle over 50 trials x 3 experts — gs=64, a non-divisible tail group (200 % 64), and a per-row member in the same launch: zero mismatches on a 5090. make check 77/77; CPU, CUDA and MinGW builds clean.
This commit is contained in:
@@ -46,6 +46,7 @@ typedef int (*fn_attention_absorb)(ColiCudaTensor *kv_b, float *ctx,
|
||||
int R, int V, int K, int T, float attention_scale);
|
||||
typedef int (*fn_tensor_upload)(ColiCudaTensor **tensor, const void *weights,
|
||||
const float *scales, int fmt, int I, int O, int device);
|
||||
typedef int (*fn_tensor_upload_g)(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device, int gs);
|
||||
typedef int (*fn_matmul)(ColiCudaTensor **tensor, float *y, const float *x,
|
||||
const void *weights, const float *scales,
|
||||
int fmt, int S, int I, int O, int device);
|
||||
@@ -102,6 +103,7 @@ static struct {
|
||||
fn_expert_group expert_group;
|
||||
fn_attention_absorb attention_absorb;
|
||||
fn_tensor_upload tensor_upload;
|
||||
fn_tensor_upload_g tensor_upload_g;
|
||||
fn_matmul matmul;
|
||||
fn_tensor_free tensor_free;
|
||||
fn_tensor_bytes tensor_bytes;
|
||||
@@ -196,6 +198,7 @@ static int coli_cuda_load(void){
|
||||
RESOLVE(expert_group, fn_expert_group)
|
||||
RESOLVE(attention_absorb, fn_attention_absorb)
|
||||
RESOLVE(tensor_upload, fn_tensor_upload)
|
||||
RESOLVE(tensor_upload_g, fn_tensor_upload_g)
|
||||
RESOLVE(matmul, fn_matmul)
|
||||
RESOLVE(tensor_free, fn_tensor_free)
|
||||
RESOLVE(tensor_bytes, fn_tensor_bytes)
|
||||
@@ -302,6 +305,11 @@ int coli_cuda_tensor_upload(ColiCudaTensor **tensor, const void *weights,
|
||||
return g_cuda.tensor_upload(tensor, weights, scales, fmt, I, O, device);
|
||||
}
|
||||
|
||||
int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device, int gs){
|
||||
if(!g_cuda.available || !g_cuda.tensor_upload_g){ return 0; }
|
||||
return g_cuda.tensor_upload_g(tensor, weights, scales, fmt, I, O, device, gs);
|
||||
}
|
||||
|
||||
int coli_cuda_matmul(ColiCudaTensor **tensor, float *y, const float *x,
|
||||
const void *weights, const float *scales,
|
||||
int fmt, int S, int I, int O, int device){
|
||||
|
||||
Reference in New Issue
Block a user