url/colibri
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #451 from ZacharyZcR/feat/cuda-grouped-g64

Browse Source 
cuda: grouped-int4 (fmt=4) support in the expert-group kernels — opens the GPU tier to g64 and E8 containers (#334)
This commit is contained in:
Vincenzo Fornaro
2026-07-20 18:06:45 +02:00
committed by GitHub
parent 78a773e539 a74e3e0c3a
commit e78fcfcbca
5 changed files with 206 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c/backend_loader.c
+8
View File
@@ -46,6 +46,7 @@ typedef int (*fn_attention_absorb)(ColiCudaTensor *kv_b, float *ctx,
int R, int V, int K, int T, float attention_scale);
typedef int (*fn_tensor_upload)(ColiCudaTensor **tensor, const void *weights,
const float *scales, int fmt, int I, int O, int device);
typedef int (*fn_tensor_upload_g)(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device, int gs);
typedef int (*fn_matmul)(ColiCudaTensor **tensor, float *y, const float *x,
const void *weights, const float *scales,
int fmt, int S, int I, int O, int device);
@@ -103,6 +104,7 @@ static struct {
fn_expert_group expert_group;
fn_attention_absorb attention_absorb;
fn_tensor_upload tensor_upload;
fn_tensor_upload_g tensor_upload_g;
fn_matmul matmul;
fn_tensor_free tensor_free;
fn_tensor_bytes tensor_bytes;
@@ -198,6 +200,7 @@ static int coli_cuda_load(void){
RESOLVE(expert_group, fn_expert_group)
RESOLVE(attention_absorb, fn_attention_absorb)
RESOLVE(tensor_upload, fn_tensor_upload)
RESOLVE(tensor_upload_g, fn_tensor_upload_g)
RESOLVE(matmul, fn_matmul)
RESOLVE(tensor_free, fn_tensor_free)
RESOLVE(tensor_bytes, fn_tensor_bytes)
@@ -305,6 +308,11 @@ int coli_cuda_tensor_upload(ColiCudaTensor **tensor, const void *weights,
return g_cuda.tensor_upload(tensor, weights, scales, fmt, I, O, device);
}
int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor, const void *weights, const float *scales, int fmt, int I, int O, int device, int gs){
if(!g_cuda.available || !g_cuda.tensor_upload_g){ return 0; }
return g_cuda.tensor_upload_g(tensor, weights, scales, fmt, I, O, device, gs);
}
int coli_cuda_matmul(ColiCudaTensor **tensor, float *y, const float *x,
const void *weights, const float *scales,
int fmt, int S, int I, int O, int device){