woolcoxm
6e29260635
The new g64 model quantizes experts with group-size 64 (fmt=4): one f32 scale per 64 elements instead of per-row. This required changes across the entire compute stack — CUDA kernel, engine dispatch, and dequant helpers — plus a new fused gate+up kernel to recover the ~40% throughput that the generic grouped path costs. CUDA backend (backend_cuda.cu): - ColiCudaTensor: add gs/ng fields for group-size metadata - row_bytes/weight_at: fmt=4 uses same packed int4 layout as fmt=2 - quant_matmul: apply per-group scales (scales[o*ng+g]) for fmt=4, matching the CPU matmul_i4_grouped accumulation exactly - coli_cuda_tensor_upload: allocate O*ng scales (not O) for fmt=4, store gs/ng on the tensor - coli_cuda_tensor_update: handle grouped scales on refresh - coli_cuda_tensor_free/tensor_bytes: VRAM accounting uses O*ng - coli_cuda_expert_group: return 0 for fmt=4 (fall back to correct per-expert path, since grouped_hidden/grouped_down kernels only handle per-row scales) - All 11 quant_matmul call sites updated to pass gs,ng Engine (glm.c): - qt_cuda_upload: pass t->gs to tensor_upload - matmul dispatch (line 816): pass w->gs to coli_cuda_matmul - kv_b shard upload: pass l->kv_b.gs - kv_b shard rb computation: add fmt=4 case ((I+1)/2) - embed_row: add fmt=4 branch with per-group scale dequant - qt_addrow/qt_matvec_rows: add fmt=4 branches (CPU MLA absorption path) - expert_gate_up: add matmul_i4_grouped_pair — fused gate+up for fmt=4 that reads x once instead of twice (+40% tok/s, 0.77 -> 1.08) - run_text: prepend [gMASK]<sop> prefix for GLM models (#108 fix — without it, PROMPT mode generates garbage on all GLM snapshots) API (backend_cuda.h, backend_loader.c): - coli_cuda_tensor_upload and coli_cuda_matmul signatures: add gs param - Loader typedefs and wrappers thread gs through the DLL boundary Tests: - bench_tensor_core.cu, test_backend_cuda.cu, test_pipe_cuda.cu: update all calls to new API signature (append gs=0 for non-grouped) New tool: c/tools/diag_harness.py - Comprehensive model diagnostic harness (system probe, correctness smoke, deep PROFILE diagnostic, quality benchmarks via eval_glm.py, throughput with/without MTP). Outputs JSON + Markdown reports. Performance (GLM-5.2 744B g64 / RTX 5070 Ti / 32GB RAM): broken CUDA: 0.05 tok/s (every tensor fell back to CPU) fixed CUDA: 0.30 tok/s (6x — CUDA working) + full opt stack: 0.77 tok/s (CACHE_ROUTE + EXPERT_BUDGET) + fused grouped pair: 1.08 tok/s (+40% from gate+up fusion)
170 lines
9.7 KiB
C
170 lines
9.7 KiB
C
#ifndef COLIBRI_BACKEND_CUDA_H
|
|
#define COLIBRI_BACKEND_CUDA_H
|
|
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
/* COLI_CUDA_DLLEXPORT marks functions exported from coli_cuda.dll on Windows.
|
|
* Define COLI_CUDA_BUILDING_DLL when compiling the .cu into the DLL (so the
|
|
* functions are __declspec(dllexport)); the host loader does NOT include this
|
|
* header's declarations — it resolves symbols at runtime via GetProcAddress. */
|
|
#if defined(_WIN32) && defined(COLI_CUDA_BUILDING_DLL)
|
|
#define COLI_CUDA_DLLEXPORT __declspec(dllexport)
|
|
#else
|
|
#define COLI_CUDA_DLLEXPORT
|
|
#endif
|
|
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
#define COLI_CUDA_MAX_DEVICES 16
|
|
|
|
/* Opaque, persistent device copy of one resident quantized tensor. */
|
|
typedef struct ColiCudaTensor ColiCudaTensor;
|
|
|
|
/* Devices are CUDA ordinals, not positions in the input list. */
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_init(const int *devices, int count);
|
|
COLI_CUDA_DLLEXPORT void coli_cuda_shutdown(void);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_device_count(void);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_device_at(int index);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_mem_info(int device, size_t *free_bytes, size_t *total_bytes);
|
|
/* device < 0 returns aggregate statistics for all configured devices. */
|
|
COLI_CUDA_DLLEXPORT void coli_cuda_stats(int device, size_t *tensor_count, size_t *tensor_bytes);
|
|
COLI_CUDA_DLLEXPORT void coli_cuda_group_stats(uint64_t *calls, uint64_t *experts, uint64_t *rows,
|
|
double *h2d_ms, double *kernel_ms, double *d2h_ms);
|
|
|
|
/* Upload without executing, so capacity failures happen during model startup. */
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload_g(ColiCudaTensor **tensor,
|
|
const void *weights, const float *scales,
|
|
int fmt, int I, int O, int device, int gs);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_upload(ColiCudaTensor **tensor,
|
|
const void *weights, const float *scales,
|
|
int fmt, int I, int O, int device, int gs);
|
|
|
|
/*
|
|
* y[S,O] = x[S,I] @ W[O,I]^T.
|
|
* fmt matches QT in glm.c: 0=f32, 1=int8, 2=int4, 3=int2, 4=grouped int4.
|
|
* gs is the group size for fmt=4 (0 for all other formats).
|
|
* The first successful call uploads W and its scales; later calls reuse it.
|
|
* Returns 1 on success and 0 when CUDA is not initialized or the format is invalid.
|
|
*/
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_matmul(ColiCudaTensor **tensor,
|
|
float *y, const float *x,
|
|
const void *weights, const float *scales,
|
|
int fmt, int S, int I, int O, int device, int gs);
|
|
|
|
/* Fused expert pipeline: y = down(silu(gate(x)) * up(x)). All three tensors
|
|
* must already be resident on one device. Activations cross PCIe once in
|
|
* each direction instead of once per matrix. */
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up,
|
|
ColiCudaTensor *down, float *y, const float *x, int S);
|
|
|
|
/* Prefill-oriented shared expert path. INT4 weights stay packed in global
|
|
* memory, activations are converted to FP16 per tile, and Tensor Cores
|
|
* accumulate into FP32. Unlike COLI_CUDA_TC_INT4 this does not quantize the
|
|
* activation to INT4. */
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate, ColiCudaTensor *up,
|
|
ColiCudaTensor *down, float *y,
|
|
const float *x, int S);
|
|
|
|
/* Packed group of same-shaped experts. Inputs and outputs contain sum(rows)
|
|
* consecutive [D] rows in call order. */
|
|
/* Async issue/take split of the group call below (Inc.4): issue launches on the
|
|
* device stream and returns; take syncs and returns the pinned result rows (valid
|
|
* until the next issue on that device). Small totals only (<=8 rows); one
|
|
* outstanding issue per device. */
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_expert_group_issue(ColiCudaTensor *const *gates,
|
|
ColiCudaTensor *const *ups,
|
|
ColiCudaTensor *const *downs,
|
|
const int *rows, int count, const float *x);
|
|
COLI_CUDA_DLLEXPORT const float *coli_cuda_expert_group_take(int device);
|
|
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_expert_group(ColiCudaTensor *const *gates,
|
|
ColiCudaTensor *const *ups,
|
|
ColiCudaTensor *const *downs,
|
|
const int *rows, int count,
|
|
float *y, const float *x);
|
|
|
|
/* Decode-only MLA weight-absorption core for one token. kv_b is [H*(Q+V),K]. */
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb(ColiCudaTensor *kv_b,float *ctx,const float *q,
|
|
const float *latent,const float *rope,int H,int Q,
|
|
int R,int V,int K,int T,float attention_scale);
|
|
|
|
/* Causal MLA absorption for S contiguous rows from one sequence. The KV
|
|
* arrays contain T rows ending at the final query; query s attends T-S+s+1
|
|
* rows. One transfer and one launch replace S host round-trips. */
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb_batch(ColiCudaTensor *kv_b,float *ctx,const float *q,
|
|
const float *latent,const float *rope,int S,
|
|
int H,int Q,int R,int V,int K,int T,
|
|
float attention_scale);
|
|
|
|
/* Same attention batch followed immediately by resident o_proj on the same
|
|
* device. Only the final [S,D] tensor crosses back to the host. */
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
|
|
float *out,const float *q,const float *latent,
|
|
const float *rope,int S,int H,int Q,int R,
|
|
int V,int K,int T,float attention_scale);
|
|
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
|
|
float *out,const float *q,const void *const *keys,
|
|
const float *const *latent,const float *const *rope,
|
|
const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale);
|
|
|
|
COLI_CUDA_DLLEXPORT void coli_cuda_tensor_free(ColiCudaTensor *tensor);
|
|
COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_device(const ColiCudaTensor *tensor);
|
|
|
|
/* Replace a resident tensor's contents without reallocating its device slot. */
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_update(ColiCudaTensor *tensor,
|
|
const void *weights, const float *scales);
|
|
|
|
/* ---- resident-pipeline primitives (Inc.0): device-pointer entry points ---- */
|
|
COLI_CUDA_DLLEXPORT float *coli_cuda_pipe_scratch(int device,int slot,size_t bytes);
|
|
COLI_CUDA_DLLEXPORT void *coli_cuda_pipe_alloc(int device,size_t bytes);
|
|
COLI_CUDA_DLLEXPORT void coli_cuda_pipe_free(int device,void *p);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_upload(int device,void *dst,const void *src,size_t bytes);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_download(int device,const void *src,void *dst,size_t bytes);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev,
|
|
const float *w_dev,int S,int D,float eps);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev,int rows,
|
|
int stride,int offset,int R,int heads,float theta);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_silu_mul(int device,float *gate_dev,const float *up_dev,size_t n);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_add(int device,float *x_dev,const float *t_dev,size_t n);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *partial_dev,
|
|
const int *rows_dev,int nrows,int D);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_gemm(ColiCudaTensor *t,float *y_dev,const float *x_dev,int S);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_dev,
|
|
const float *w_dev,int S,int D,float eps,
|
|
int xstride,int ystride);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows,
|
|
int stride,int offset,int R,int heads,float theta);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_router(int device,const float *x_dev,
|
|
const void *rw_dev,const void *rb_dev,int D,int E,int Ksel,
|
|
float topp,int norm_topk,float routed_scale,
|
|
int *idx_host,float *w_host,int *keff_host);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src,
|
|
int spitch,int width,int height);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
|
|
float *out,const float *q_dev,const float *latent_dev,const float *rope_dev,
|
|
int S,int H,int Q,int R,int V,int K,int T,float scale);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *kv_b_shard,float *ctx_dev,
|
|
const float *q_dev,const float *latent_dev,const float *rope_dev,
|
|
int S,int H,int Q,int R,int V,int K,int T,float scale);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb_kvdev(ColiCudaTensor *kv_b,float *ctx,const float *q,
|
|
const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T,
|
|
float scale);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_peer_copy(int dst_dev,float *dst,int src_dev,
|
|
const float *src,size_t bytes);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
|
|
float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev,
|
|
int S,int H,int Q,int R,int V,int K,int T,float scale);
|
|
COLI_CUDA_DLLEXPORT int coli_cuda_pipe_sync(int device);
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|
|
|
|
#endif
|