fix: layer_cuda_shard_kvb scale offset for fmt=4 (multi-GPU, per JustVugg review)
The kv_b shard scale pointer used h0*(Q+V) which is correct for per-row scales (fmt=2: one scale per row). For fmt=4 (grouped), there are ng scales per row, so the offset must be h0*(Q+V)*ng. Without this, the shard reads from the wrong scale position on multi-GPU, producing silent corruption. Single-GPU is unaffected (no sharding). Fix: const float *scale=l->kv_b.s+(int64_t)h0*(Q+V)*(gs>0?ng:1); Refs #298
This commit is contained in:
+1
-1
@@ -1647,7 +1647,7 @@ static void layer_cuda_shard_kvb(Layer *l,int H,int Q,int V){
|
||||
for(int d=0,h0=0;d<g_cuda_ndev;d++){
|
||||
int hn=H/g_cuda_ndev+(d<H%g_cuda_ndev),rows=hn*(Q+V);
|
||||
const void *part=weights+(int64_t)h0*(Q+V)*rb;
|
||||
const float *scale=l->kv_b.s+(int64_t)h0*(Q+V);
|
||||
const float *scale=l->kv_b.s+(int64_t)h0*(Q+V)*(l->kv_b.gs>0?(l->kv_b.I+l->kv_b.gs-1)/l->kv_b.gs:1);
|
||||
if(!coli_cuda_tensor_upload(&l->kv_b_shard[d],part,scale,l->kv_b.fmt,l->kv_b.I,rows,g_cuda_devices[d],l->kv_b.gs))return;
|
||||
l->shard_h0[d]=h0;l->shard_hn[d]=hn;l->n_kv_b_shard++;h0+=hn;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user