diff --git a/c/backend_cuda.h b/c/backend_cuda.h index c84a265..acbe4bc 100644 --- a/c/backend_cuda.h +++ b/c/backend_cuda.h @@ -60,7 +60,7 @@ COLI_CUDA_DLLEXPORT int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTenso * memory, activations are converted to FP16 per tile, and Tensor Cores * accumulate into FP32. Unlike COLI_CUDA_TC_INT4 this does not quantize the * activation to INT4. */ -int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate, ColiCudaTensor *up, +COLI_CUDA_DLLEXPORT int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate, ColiCudaTensor *up, ColiCudaTensor *down, float *y, const float *x, int S); @@ -97,46 +97,46 @@ COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor); COLI_CUDA_DLLEXPORT int coli_cuda_tensor_device(const ColiCudaTensor *tensor); /* Replace a resident tensor's contents without reallocating its device slot. */ -int coli_cuda_tensor_update(ColiCudaTensor *tensor, +COLI_CUDA_DLLEXPORT int coli_cuda_tensor_update(ColiCudaTensor *tensor, const void *weights, const float *scales); /* ---- resident-pipeline primitives (Inc.0): device-pointer entry points ---- */ -float *coli_cuda_pipe_scratch(int device,int slot,size_t bytes); -void *coli_cuda_pipe_alloc(int device,size_t bytes); -void coli_cuda_pipe_free(int device,void *p); -int coli_cuda_pipe_upload(int device,void *dst,const void *src,size_t bytes); -int coli_cuda_pipe_download(int device,const void *src,void *dst,size_t bytes); -int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev, +COLI_CUDA_DLLEXPORT float *coli_cuda_pipe_scratch(int device,int slot,size_t bytes); +COLI_CUDA_DLLEXPORT void *coli_cuda_pipe_alloc(int device,size_t bytes); +COLI_CUDA_DLLEXPORT void coli_cuda_pipe_free(int device,void *p); +COLI_CUDA_DLLEXPORT int coli_cuda_pipe_upload(int device,void *dst,const void *src,size_t bytes); +COLI_CUDA_DLLEXPORT int coli_cuda_pipe_download(int device,const void *src,void *dst,size_t bytes); +COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps); -int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev,int rows, +COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev,int rows, int stride,int offset,int R,int heads,float theta); -int coli_cuda_pipe_silu_mul(int device,float *gate_dev,const float *up_dev,size_t n); -int coli_cuda_pipe_add(int device,float *x_dev,const float *t_dev,size_t n); -int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *partial_dev, +COLI_CUDA_DLLEXPORT int coli_cuda_pipe_silu_mul(int device,float *gate_dev,const float *up_dev,size_t n); +COLI_CUDA_DLLEXPORT int coli_cuda_pipe_add(int device,float *x_dev,const float *t_dev,size_t n); +COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *partial_dev, const int *rows_dev,int nrows,int D); -int coli_cuda_pipe_gemm(ColiCudaTensor *t,float *y_dev,const float *x_dev,int S); -int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_dev, +COLI_CUDA_DLLEXPORT int coli_cuda_pipe_gemm(ColiCudaTensor *t,float *y_dev,const float *x_dev,int S); +COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps, int xstride,int ystride); -int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows, +COLI_CUDA_DLLEXPORT int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows, int stride,int offset,int R,int heads,float theta); -int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src, +COLI_CUDA_DLLEXPORT int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src, int spitch,int width,int height); -int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, +COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); -int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *kv_b_shard,float *ctx_dev, +COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *kv_b_shard,float *ctx_dev, const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); -int coli_cuda_attention_absorb_kvdev(ColiCudaTensor *kv_b,float *ctx,const float *q, +COLI_CUDA_DLLEXPORT int coli_cuda_attention_absorb_kvdev(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale); -int coli_cuda_pipe_peer_copy(int dst_dev,float *dst,int src_dev, +COLI_CUDA_DLLEXPORT int coli_cuda_pipe_peer_copy(int dst_dev,float *dst,int src_dev, const float *src,size_t bytes); -int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, +COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); -int coli_cuda_pipe_sync(int device); +COLI_CUDA_DLLEXPORT int coli_cuda_pipe_sync(int device); #ifdef __cplusplus } diff --git a/c/backend_loader.c b/c/backend_loader.c index 6fea70c..0635e08 100644 --- a/c/backend_loader.c +++ b/c/backend_loader.c @@ -52,6 +52,35 @@ typedef void (*fn_tensor_free)(ColiCudaTensor *tensor); typedef size_t (*fn_tensor_bytes)(const ColiCudaTensor *tensor); typedef int (*fn_tensor_device)(const ColiCudaTensor *tensor); +/* --- #111 GPU resident pipeline additions (matched to backend_cuda.h) --- */ + + +/* --- #111 GPU resident pipeline additions (matched to backend_cuda.h) --- */ +typedef int (*fn_attention_absorb_batch)(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent,const float *rope,int S, int H,int Q,int R,int V,int K,int T, float attention_scale); +typedef int (*fn_attention_absorb_batch_dev)(ColiCudaTensor *kv_b_shard,float *ctx_dev, const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); +typedef int (*fn_attention_absorb_kvdev)(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale); +typedef int (*fn_attention_project_batch)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q,const float *latent, const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale); +typedef int (*fn_attention_project_batch_dev)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); +typedef int (*fn_attention_project_batch_dev_out)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); +typedef int (*fn_pipe_add)(int device,float *x_dev,const float *t_dev,size_t n); +typedef void * (*fn_pipe_alloc)(int device,size_t bytes); +typedef int (*fn_pipe_copy2d)(int device,float *dst,int dpitch,const float *src, int spitch,int width,int height); +typedef int (*fn_pipe_download)(int device,const void *src,void *dst,size_t bytes); +typedef void (*fn_pipe_free)(int device,void *p); +typedef int (*fn_pipe_gemm)(ColiCudaTensor *t,float *y_dev,const float *x_dev,int S); +typedef int (*fn_pipe_peer_copy)(int dst_dev,float *dst,int src_dev, const float *src,size_t bytes); +typedef int (*fn_pipe_rmsnorm)(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps); +typedef int (*fn_pipe_rmsnorm_s)(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps, int xstride,int ystride); +typedef int (*fn_pipe_rope)(int device,float *v_dev,const int *pos_dev,int rows, int stride,int offset,int R,int heads,float theta); +typedef int (*fn_pipe_rope_base)(int device,float *v_dev,int pos_base,int rows, int stride,int offset,int R,int heads,float theta); +typedef int (*fn_pipe_rows_add)(int device,float *x_dev,const float *partial_dev, const int *rows_dev,int nrows,int D); +typedef float * (*fn_pipe_scratch)(int device,int slot,size_t bytes); +typedef int (*fn_pipe_silu_mul)(int device,float *gate_dev,const float *up_dev,size_t n); +typedef int (*fn_pipe_sync)(int device); +typedef int (*fn_pipe_upload)(int device,void *dst,const void *src,size_t bytes); +typedef int (*fn_shared_mlp_w4a16)(ColiCudaTensor *gate, ColiCudaTensor *up, ColiCudaTensor *down, float *y, const float *x, int S); +typedef int (*fn_tensor_update)(ColiCudaTensor *tensor, const void *weights, const float *scales); + /* Resolved pointers, plus a flag so we attempt the load at most once. */ static struct { int loaded; /* 1 = load attempted (success or fail), 0 = not yet */ @@ -72,6 +101,31 @@ static struct { fn_tensor_free tensor_free; fn_tensor_bytes tensor_bytes; fn_tensor_device tensor_device; + + fn_attention_absorb_batch attention_absorb_batch; + fn_attention_absorb_batch_dev attention_absorb_batch_dev; + fn_attention_absorb_kvdev attention_absorb_kvdev; + fn_attention_project_batch attention_project_batch; + fn_attention_project_batch_dev attention_project_batch_dev; + fn_attention_project_batch_dev_out attention_project_batch_dev_out; + fn_pipe_add pipe_add; + fn_pipe_alloc pipe_alloc; + fn_pipe_copy2d pipe_copy2d; + fn_pipe_download pipe_download; + fn_pipe_free pipe_free; + fn_pipe_gemm pipe_gemm; + fn_pipe_peer_copy pipe_peer_copy; + fn_pipe_rmsnorm pipe_rmsnorm; + fn_pipe_rmsnorm_s pipe_rmsnorm_s; + fn_pipe_rope pipe_rope; + fn_pipe_rope_base pipe_rope_base; + fn_pipe_rows_add pipe_rows_add; + fn_pipe_scratch pipe_scratch; + fn_pipe_silu_mul pipe_silu_mul; + fn_pipe_sync pipe_sync; + fn_pipe_upload pipe_upload; + fn_shared_mlp_w4a16 shared_mlp_w4a16; + fn_tensor_update tensor_update; } g_cuda; /* Resolve the DLL and all 11 symbols. Returns 1 on success, 0 otherwise. @@ -119,6 +173,31 @@ static int coli_cuda_load(void){ RESOLVE(tensor_free, fn_tensor_free) RESOLVE(tensor_bytes, fn_tensor_bytes) RESOLVE(tensor_device, fn_tensor_device) + + RESOLVE(attention_absorb_batch, fn_attention_absorb_batch) + RESOLVE(attention_absorb_batch_dev, fn_attention_absorb_batch_dev) + RESOLVE(attention_absorb_kvdev, fn_attention_absorb_kvdev) + RESOLVE(attention_project_batch, fn_attention_project_batch) + RESOLVE(attention_project_batch_dev, fn_attention_project_batch_dev) + RESOLVE(attention_project_batch_dev_out, fn_attention_project_batch_dev_out) + RESOLVE(pipe_add, fn_pipe_add) + RESOLVE(pipe_alloc, fn_pipe_alloc) + RESOLVE(pipe_copy2d, fn_pipe_copy2d) + RESOLVE(pipe_download, fn_pipe_download) + RESOLVE(pipe_free, fn_pipe_free) + RESOLVE(pipe_gemm, fn_pipe_gemm) + RESOLVE(pipe_peer_copy, fn_pipe_peer_copy) + RESOLVE(pipe_rmsnorm, fn_pipe_rmsnorm) + RESOLVE(pipe_rmsnorm_s, fn_pipe_rmsnorm_s) + RESOLVE(pipe_rope, fn_pipe_rope) + RESOLVE(pipe_rope_base, fn_pipe_rope_base) + RESOLVE(pipe_rows_add, fn_pipe_rows_add) + RESOLVE(pipe_scratch, fn_pipe_scratch) + RESOLVE(pipe_silu_mul, fn_pipe_silu_mul) + RESOLVE(pipe_sync, fn_pipe_sync) + RESOLVE(pipe_upload, fn_pipe_upload) + RESOLVE(shared_mlp_w4a16, fn_shared_mlp_w4a16) + RESOLVE(tensor_update, fn_tensor_update) #undef RESOLVE g_cuda.available = 1; @@ -216,4 +295,129 @@ int coli_cuda_tensor_device(const ColiCudaTensor *tensor){ return g_cuda.tensor_device(tensor); } +/* ---- #111 pipeline wrappers ---- */ + + +/* ---- #111 pipeline wrappers (see header for semantics) ---- */ + +int coli_cuda_attention_absorb_batch(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent,const float *rope,int S, int H,int Q,int R,int V,int K,int T, float attention_scale){ + if(!g_cuda.available){ return 0; } + return g_cuda.attention_absorb_batch(kv_b, ctx, q, latent, rope, S, H, Q, R, V, K, T, attention_scale); +} + +int coli_cuda_attention_absorb_batch_dev(ColiCudaTensor *kv_b_shard,float *ctx_dev, const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){ + if(!g_cuda.available){ return 0; } + return g_cuda.attention_absorb_batch_dev(kv_b_shard, ctx_dev, q_dev, latent_dev, rope_dev, S, H, Q, R, V, K, T, scale); +} + +int coli_cuda_attention_absorb_kvdev(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale){ + if(!g_cuda.available){ return 0; } + return g_cuda.attention_absorb_kvdev(kv_b, ctx, q, latent_dev, rope_dev, H, Q, R, V, K, T, scale); +} + +int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q,const float *latent, const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale){ + if(!g_cuda.available){ return 0; } + return g_cuda.attention_project_batch(kv_b, o_proj, out, q, latent, rope, S, H, Q, R, V, K, T, attention_scale); +} + +int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){ + if(!g_cuda.available){ return 0; } + return g_cuda.attention_project_batch_dev(kv_b, o_proj, out, q_dev, latent_dev, rope_dev, S, H, Q, R, V, K, T, scale); +} + +int coli_cuda_attention_project_batch_dev_out(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){ + if(!g_cuda.available){ return 0; } + return g_cuda.attention_project_batch_dev_out(kv_b, o_proj, out_dev, q_dev, latent_dev, rope_dev, S, H, Q, R, V, K, T, scale); +} + +int coli_cuda_pipe_add(int device,float *x_dev,const float *t_dev,size_t n){ + if(!g_cuda.available){ return 0; } + return g_cuda.pipe_add(device, x_dev, t_dev, n); +} + +void * coli_cuda_pipe_alloc(int device,size_t bytes){ + if(!g_cuda.available){ return NULL; } + return g_cuda.pipe_alloc(device, bytes); +} + +int coli_cuda_pipe_copy2d(int device,float *dst,int dpitch,const float *src, int spitch,int width,int height){ + if(!g_cuda.available){ return 0; } + return g_cuda.pipe_copy2d(device, dst, dpitch, src, spitch, width, height); +} + +int coli_cuda_pipe_download(int device,const void *src,void *dst,size_t bytes){ + if(!g_cuda.available){ return 0; } + return g_cuda.pipe_download(device, src, dst, bytes); +} + +void coli_cuda_pipe_free(int device,void *p){ + if(!g_cuda.available){ return; } + g_cuda.pipe_free(device, p); +} + +int coli_cuda_pipe_gemm(ColiCudaTensor *t,float *y_dev,const float *x_dev,int S){ + if(!g_cuda.available){ return 0; } + return g_cuda.pipe_gemm(t, y_dev, x_dev, S); +} + +int coli_cuda_pipe_peer_copy(int dst_dev,float *dst,int src_dev, const float *src,size_t bytes){ + if(!g_cuda.available){ return 0; } + return g_cuda.pipe_peer_copy(dst_dev, dst, src_dev, src, bytes); +} + +int coli_cuda_pipe_rmsnorm(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps){ + if(!g_cuda.available){ return 0; } + return g_cuda.pipe_rmsnorm(device, y_dev, x_dev, w_dev, S, D, eps); +} + +int coli_cuda_pipe_rmsnorm_s(int device,float *y_dev,const float *x_dev, const float *w_dev,int S,int D,float eps, int xstride,int ystride){ + if(!g_cuda.available){ return 0; } + return g_cuda.pipe_rmsnorm_s(device, y_dev, x_dev, w_dev, S, D, eps, xstride, ystride); +} + +int coli_cuda_pipe_rope(int device,float *v_dev,const int *pos_dev,int rows, int stride,int offset,int R,int heads,float theta){ + if(!g_cuda.available){ return 0; } + return g_cuda.pipe_rope(device, v_dev, pos_dev, rows, stride, offset, R, heads, theta); +} + +int coli_cuda_pipe_rope_base(int device,float *v_dev,int pos_base,int rows, int stride,int offset,int R,int heads,float theta){ + if(!g_cuda.available){ return 0; } + return g_cuda.pipe_rope_base(device, v_dev, pos_base, rows, stride, offset, R, heads, theta); +} + +int coli_cuda_pipe_rows_add(int device,float *x_dev,const float *partial_dev, const int *rows_dev,int nrows,int D){ + if(!g_cuda.available){ return 0; } + return g_cuda.pipe_rows_add(device, x_dev, partial_dev, rows_dev, nrows, D); +} + +float * coli_cuda_pipe_scratch(int device,int slot,size_t bytes){ + if(!g_cuda.available){ return NULL; } + return g_cuda.pipe_scratch(device, slot, bytes); +} + +int coli_cuda_pipe_silu_mul(int device,float *gate_dev,const float *up_dev,size_t n){ + if(!g_cuda.available){ return 0; } + return g_cuda.pipe_silu_mul(device, gate_dev, up_dev, n); +} + +int coli_cuda_pipe_sync(int device){ + if(!g_cuda.available){ return 0; } + return g_cuda.pipe_sync(device); +} + +int coli_cuda_pipe_upload(int device,void *dst,const void *src,size_t bytes){ + if(!g_cuda.available){ return 0; } + return g_cuda.pipe_upload(device, dst, src, bytes); +} + +int coli_cuda_shared_mlp_w4a16(ColiCudaTensor *gate, ColiCudaTensor *up, ColiCudaTensor *down, float *y, const float *x, int S){ + if(!g_cuda.available){ return 0; } + return g_cuda.shared_mlp_w4a16(gate, up, down, y, x, S); +} + +int coli_cuda_tensor_update(ColiCudaTensor *tensor, const void *weights, const float *scales){ + if(!g_cuda.available){ return 0; } + return g_cuda.tensor_update(tensor, weights, scales); +} + #endif /* _WIN32 */