url/colibri
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #447 from monotophic/metal/rtop8-parallel

Browse Source 
metal: parallelize the single-threaded r_top8 selection kernel (bit-exact, +6.7% decode)
This commit is contained in:
Vincenzo Fornaro
2026-07-20 18:07:44 +02:00
committed by GitHub
parent 06961728d8 9f30916003
commit aa825d361a
3 changed files with 240 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c/backend_metal.h
+17
View File
@@ -84,6 +84,23 @@ int coli_metal_layer_decode(float *x,
int coli_metal_gemm(float *y, const float *x, const void *weights, const float *scales,
int fmt, int S, int I, int O); /* large-batch sync GEMM; 0 -> CPU */
/* Parallel top-8 expert selection (r_top8_par): run ONE top-8 selection kernel standalone
* on host arrays — par=0 the serial r_top8, par=1 the parallel exact-match replica gated
* in the engine by COLI_RTOP8 (default ON; COLI_RTOP8=0 opts out to the serial kernel).
* Exists so the metal-test suite (and any battery probe) can prove serial/parallel
* equivalence on the ENGINE build's own compiled shaders, not just in the bench tool.
* sig[S*E], bias[E], idx[S*K], w[S*K], keff[S].
* Expert-count generality: the parallel kernel's blocked-lane design (ch[8]/32-lane
* threadgroup) is validated correct for arbitrary E<=256, including non-multiples of the
* 32-lane width and small E (see metal-test's E=24/E=168/E=256 cases — 168 is the REAP
* expert-pruned package width from #428/#426). For E>256 (out of contract) this function
* transparently falls back to the serial kernel even when par=1 is requested, and the
* same automatic fallback is wired into the engine dispatch site — "par" is a request,
* never a guarantee, so no caller can reach the unguarded parallel path out of contract.
* Returns 1 on success, 0 if Metal is unavailable. */
int coli_metal_rtop8(int par, const float *sig, const float *bias, int S, int E, int K,
int Ksel, float topp, int normk, float rscale,
int *idx, float *w, int *keff);
void coli_metal_attn_counts(uint64_t *ok, double *wall, double *kernel);
void coli_metal_attn_lat(double *ksched, double *gsched);
int coli_metal_attn_decode(const float *x,
c/backend_metal.mm
+135 -5
View File
@@ -219,6 +219,63 @@ kernel void r_top8(device const float* sig [[buffer(0)]], device const float* bi
if(normk){ float sm=0; for(int kk=0;kk<Ke;kk++) sm+=ww[kk]; sm+=1e-20f; for(int kk=0;kk<Ke;kk++) ww[kk]/=sm; }
for(int kk=0;kk<Ke;kk++) ww[kk]*=rscale;
}
// parallel replica of r_top8's selection on ONE SIMDGROUP per row instead of one serial
// thread (bench/kernels @ 27bfe83: serial r_top8 measured 0.465 ms/layer, ~55% of the
// layer CB; this replica ~93x faster with exactly matching output). EXACT-MATCH is the
// contract: each lane owns ceil(E/32) contiguous experts (blocked) and keeps a taken
// bitmask; per selection step: lane-local strict-'>' ascending max (lowest index wins
// within a lane, matching the serial ascending scan), then a shuffle-down argmax
// reduction where ties prefer the LOWER index — together exactly the serial kernel's
// first-max-wins order. The topp/normk/rscale tail is the serial code verbatim on lane 0
// (same ops, same order => bitwise-identical results; metal-test enforces this with
// memcmp). Contract: E<=256 (ch[8]/taken mask sizing: ceil(E/32)<=8) — the defensive
// return below makes an out-of-contract dispatch a visible no-op (idx/w/keff untouched),
// never an OOB write; both call sites (coli_metal_layer_decode's dispatch and the
// standalone coli_metal_rtop8 runner) additionally gate on E<=256 in host code before
// selecting this pipeline at all, so the return here is defense-in-depth, not the only
// guard. Sentinel-per-lane design (ch[j]=-1e30f for e>=E) makes non-multiple-of-32 E
// and small E correct without special-casing — validated for E=24, E=168 (REAP
// expert-pruned packages, see the upstream feature-request thread) and E=256 by metal-test.
// ASSUMES SIMD width 32 (shuffle offsets 16..1, 32-thread threadgroup per row): enforced
// at init — coli_metal_init clears g_rtop8_width_ok (and therefore both call sites' use
// of this pipeline) if threadExecutionWidth != 32.
kernel void r_top8_par(device const float* sig [[buffer(0)]], device const float* bias [[buffer(1)]],
device int* idx [[buffer(2)]], device float* w [[buffer(3)]],
device int* keff [[buffer(4)]], constant int& E [[buffer(5)]],
constant int& K [[buffer(6)]], constant int& Ksel [[buffer(7)]],
constant float& topp [[buffer(8)]], constant int& normk [[buffer(9)]],
constant float& rscale [[buffer(10)]],
uint s [[threadgroup_position_in_grid]],
uint slane [[thread_index_in_simdgroup]]) {
if(E>256) return;
device const float* sg=sig+(long)s*E;
device int* id_=idx+(long)s*K; device float* ww=w+(long)s*K;
int per=(E+31)/32, base=(int)slane*per;
float ch[8]; uint taken=0u;
for(int j=0;j<per;j++){ int e=base+j; ch[j]=(e<E)?sg[e]+bias[e]:-1e30f; }
for(int kk=0;kk<Ksel;kk++){
float bv=-1e30f; int bi=0x7FFFFFFF;
for(int j=0;j<per;j++) if(!(taken&(1u<<j)) && ch[j]>bv){ bv=ch[j]; bi=base+j; }
for(uint off=16;off>0;off>>=1){
float ov=simd_shuffle_down(bv,off); int oi=simd_shuffle_down(bi,off);
if(ov>bv || (ov==bv && oi<bi)){ bv=ov; bi=oi; }
}
bv=simd_broadcast(bv,0); bi=simd_broadcast(bi,0);
if(bi>=base && bi<base+per) taken|=1u<<(bi-base);
if(slane==0){ id_[kk]=bi; ww[kk]=sg[bi]; }
}
if(slane!=0) return;
int Ke=Ksel;
if(topp>0.0f && topp<1.0f){
for(int a=1;a<Ksel;a++){ int ii=id_[a]; float wv=ww[a]; int b=a-1;
while(b>=0 && ww[b]<wv){ ww[b+1]=ww[b]; id_[b+1]=id_[b]; b--; } ww[b+1]=wv; id_[b+1]=ii; }
float tot=1e-20f; for(int kk=0;kk<Ksel;kk++) tot+=ww[kk];
float cum=0; for(int kk=0;kk<Ksel;kk++){ cum+=ww[kk]; if(cum>=topp*tot){ Ke=kk+1; break; } }
}
keff[s]=Ke;
if(normk){ float sm=0; for(int kk=0;kk<Ke;kk++) sm+=ww[kk]; sm+=1e-20f; for(int kk=0;kk<Ke;kk++) ww[kk]/=sm; }
for(int kk=0;kk<Ke;kk++) ww[kk]*=rscale;
}
)METAL";
struct ColiMetalTensor {
@@ -231,7 +288,15 @@ static id<MTLDevice> g_dev;
static id<MTLCommandQueue> g_queue;
static id<MTLComputePipelineState> g_gemv, g_moe_gemv, g_moe_silu;
static id<MTLComputePipelineState> g_a_rms, g_a_rope, g_a_copy, g_a_qabs, g_a_score, g_a_smax, g_a_clat, g_a_ctx;
static id<MTLComputePipelineState> g_a_add, g_r_router, g_r_top8;
static id<MTLComputePipelineState> g_a_add, g_r_router, g_r_top8, g_r_top8p;
static int g_rtop8_par = 1; // COLI_RTOP8 (default ON); COLI_RTOP8=0 opts out to the
// serial kernel — see coli_metal_init.
static int g_rtop8_width_ok = 1; // hardware fact, independent of the policy gate above:
// false if this device's threadExecutionWidth != 32.
// Consulted by BOTH the engine dispatch site and the
// standalone coli_metal_rtop8 runner, so no caller can
// reach r_top8_par's 32-lane reduction on an unsafe
// device even by explicitly requesting par=1.
static size_t g_tensor_count, g_tensor_bytes;
static uint64_t g_moe_ok, g_moe_fb, g_moe_experts; // GPU blocks / CPU-fallback blocks / experts on GPU
static double g_t_setup, g_t_gpu, g_t_scatter, g_t_kernel; // per-block time breakdown (seconds)
@@ -284,6 +349,8 @@ extern "C" int coli_metal_init(void) {
if (g_dev) return 1;
if (getenv("COLI_METAL_UNTRACKED") && atoi(getenv("COLI_METAL_UNTRACKED")))
g_res_opts = MTLResourceStorageModeShared | MTLResourceHazardTrackingModeUntracked;
{ const char *e = getenv("COLI_RTOP8"); // default ON; COLI_RTOP8=0 opts out
if (e && atoi(e) == 0) g_rtop8_par = 0; }
@autoreleasepool {
g_dev = MTLCreateSystemDefaultDevice();
if (!g_dev) return 0;
@@ -299,8 +366,23 @@ extern "C" int coli_metal_init(void) {
auto P=[&](const char*n){ return [g_dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@(n)] error:&err]; };
g_a_rms=P("a_rmsnorm"); g_a_rope=P("a_rope"); g_a_copy=P("a_copy");
g_a_qabs=P("a_qabs"); g_a_score=P("a_score"); g_a_smax=P("a_smax"); g_a_clat=P("a_clat"); g_a_ctx=P("a_ctx");
g_a_add=P("a_add"); g_r_router=P("r_router"); g_r_top8=P("r_top8");
if(!g_a_add||!g_r_router||!g_r_top8){ fprintf(stderr,"[metal] tail pipelines failed\n"); g_dev=nil; return 0; }
g_a_add=P("a_add"); g_r_router=P("r_router"); g_r_top8=P("r_top8"); g_r_top8p=P("r_top8_par");
if(!g_a_add||!g_r_router||!g_r_top8||!g_r_top8p){ fprintf(stderr,"[metal] tail pipelines failed\n"); g_dev=nil; return 0; }
// r_top8_par's reduction hardcodes SIMD width 32 (shuffle-down offsets 16..1, one
// 32-thread threadgroup per row). True on all Apple Silicon shipped to date, but a
// non-32-width device would reduce wrongly AND race multiple lane-0 writers, so this
// is a hard safety fact (g_rtop8_width_ok), not just a policy default: it gates BOTH
// the engine dispatch site and the standalone coli_metal_rtop8 runner (degrade-to-safe,
// same pattern as the pool/ring fallbacks elsewhere) — no caller can opt back into an
// unsafe reduction on such a device, even by explicitly requesting par=1.
if ([g_r_top8p threadExecutionWidth] != 32) {
g_rtop8_width_ok = 0;
if (g_rtop8_par)
fprintf(stderr, "[metal] COLI_RTOP8 parallel top-8 disabled: threadExecutionWidth=%lu "
"!= 32 (r_top8_par's reduction assumes 32-lane simdgroups) — serial "
"r_top8 in use\n", (unsigned long)[g_r_top8p threadExecutionWidth]);
g_rtop8_par = 0;
}
if (!g_gemv || !g_moe_gemv || !g_moe_silu || !g_a_rms || !g_a_rope || !g_a_copy ||
!g_a_qabs || !g_a_score || !g_a_smax || !g_a_clat || !g_a_ctx) {
fprintf(stderr, "[metal] pipeline failed\n"); g_dev = nil; return 0; }
@@ -597,12 +679,22 @@ extern "C" int coli_metal_layer_decode(float *x,
// 5) silu(gate)*up + exact top-K select
[e setComputePipelineState:g_moe_silu]; [e setBuffer:ash1_ offset:0 atIndex:0]; [e setBuffer:ash2_ offset:0 atIndex:1];
[e dispatchThreads:MTLSizeMake((size_t)S*SI,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)];
{ [e setComputePipelineState:g_r_top8];
{ // COLI_RTOP8 (default ON) swaps the serial 1-thread-per-row select for the exact-
// match 1-simdgroup-per-row replica (same buffers/args; only pipeline+grid change).
// E<=256 is required by r_top8_par's ch[8]/32-lane blocking contract; this call
// site's E is always 256 today (layer_forward_rows' own architecture-shape gate in
// colibri.c requires c->n_experts==256 to reach coli_metal_layer_decode at all —
// see PR body "Scope statement") but the check is kept here too, defense-in-depth,
// so a future relaxation of that gate (e.g. to admit REAP-pruned E=168 models into
// the fused path) degrades safely to the serial kernel instead of mis-dispatching.
int use_par = g_rtop8_par && g_rtop8_width_ok && E<=256;
[e setComputePipelineState:use_par?g_r_top8p:g_r_top8];
[e setBuffer:asig_ offset:0 atIndex:0]; [e setBuffer:rbB offset:rboff atIndex:1];
[e setBuffer:aidx_ offset:0 atIndex:2]; [e setBuffer:aw_ offset:0 atIndex:3]; [e setBuffer:akeff_ offset:0 atIndex:4];
[e setBytes:&E length:4 atIndex:5]; [e setBytes:&K length:4 atIndex:6]; [e setBytes:&Ksel length:4 atIndex:7];
[e setBytes:&topp length:4 atIndex:8]; [e setBytes:&normk length:4 atIndex:9]; [e setBytes:&rscale length:4 atIndex:10];
[e dispatchThreads:MTLSizeMake(S,1,1) threadsPerThreadgroup:MTLSizeMake(S,1,1)]; }
if(use_par) [e dispatchThreadgroups:MTLSizeMake(S,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)];
else [e dispatchThreads:MTLSizeMake(S,1,1) threadsPerThreadgroup:MTLSizeMake(S,1,1)]; }
BAR();
// 6) shared down
bind_gemv(e,shd_w,shd_s,shd_fmt,SI,AH,ash1_,ashout_,S);
@@ -651,6 +743,44 @@ extern "C" int coli_metal_gemm(float *y, const float *x, const void *wp, const f
return 1;
}
// Standalone single-kernel runner for the top-8 select (see backend_metal.h). Fresh
// shared buffers per call (a test/probe path, not a hot path); grids exactly as the
// engine dispatch site: serial = S threads of one S-wide threadgroup, parallel = S
// threadgroups x 32 (one simdgroup per row). "par" is a REQUEST, not a guarantee: same
// E<=256 and SIMD-width-32 host-side checks as the engine dispatch site gate the actual
// pipeline choice, so a caller (including metal-test itself) can never reach the parallel
// kernel out of contract by asking for it — par=1 with E>256, or on a non-32-wide device,
// transparently runs the serial kernel instead and still returns 1 (success).
extern "C" int coli_metal_rtop8(int par, const float *sig, const float *bias, int S, int E, int K,
int Ksel, float topp, int normk, float rscale,
int *idx, float *w, int *keff) {
if (!g_dev || S < 1 || E < 1 || K < 1 || Ksel < 1 || Ksel > K) return 0;
int use_par = par && g_r_top8p && g_rtop8_width_ok && E<=256;
@autoreleasepool {
id<MTLBuffer> bs=[g_dev newBufferWithBytes:sig length:(size_t)S*E*4 options:MTLResourceStorageModeShared];
id<MTLBuffer> bb=[g_dev newBufferWithBytes:bias length:(size_t)E*4 options:MTLResourceStorageModeShared];
id<MTLBuffer> bi=[g_dev newBufferWithLength:(size_t)S*K*4 options:MTLResourceStorageModeShared];
id<MTLBuffer> bw=[g_dev newBufferWithLength:(size_t)S*K*4 options:MTLResourceStorageModeShared];
id<MTLBuffer> bk=[g_dev newBufferWithLength:(size_t)S*4 options:MTLResourceStorageModeShared];
if(!bs||!bb||!bi||!bw||!bk) return 0;
memset(bi.contents,0xFF,(size_t)S*K*4); // poison: untouched slots stay visible
id<MTLCommandBuffer> cb=[g_queue commandBuffer]; id<MTLComputeCommandEncoder> e=[cb computeCommandEncoder];
[e setComputePipelineState:use_par?g_r_top8p:g_r_top8];
[e setBuffer:bs offset:0 atIndex:0]; [e setBuffer:bb offset:0 atIndex:1];
[e setBuffer:bi offset:0 atIndex:2]; [e setBuffer:bw offset:0 atIndex:3]; [e setBuffer:bk offset:0 atIndex:4];
[e setBytes:&E length:4 atIndex:5]; [e setBytes:&K length:4 atIndex:6]; [e setBytes:&Ksel length:4 atIndex:7];
[e setBytes:&topp length:4 atIndex:8]; [e setBytes:&normk length:4 atIndex:9]; [e setBytes:&rscale length:4 atIndex:10];
if(use_par) [e dispatchThreadgroups:MTLSizeMake((NSUInteger)S,1,1) threadsPerThreadgroup:MTLSizeMake(32,1,1)];
else [e dispatchThreads:MTLSizeMake((NSUInteger)S,1,1) threadsPerThreadgroup:MTLSizeMake((NSUInteger)S,1,1)];
[e endEncoding]; [cb commit]; [cb waitUntilCompleted];
if(cb.status==MTLCommandBufferStatusError){ fprintf(stderr,"[metal] rtop8 cmdbuf error\n"); return 0; }
memcpy(idx,bi.contents,(size_t)S*K*4);
memcpy(w,bw.contents,(size_t)S*K*4);
memcpy(keff,bk.contents,(size_t)S*4);
}
return 1;
}
extern "C" void coli_metal_tensor_free(ColiMetalTensor *t) {
if (!t) return;
g_tensor_count--; g_tensor_bytes -= t->wbytes;
c/tests/test_backend_metal.mm
+88
View File
@@ -177,6 +177,68 @@ static int run_attn(int S, int pos_base, const char* name){
return pass?0:1;
}
// serial r_top8 vs parallel r_top8_par on the ENGINE build's own compiled shaders — the
// exact-match contract (same indices, same order, same weights bitwise, same keff)
// enforced with memcmp, per adversarial input family. `mode` selects the input
// construction; see the inventory at the call sites in main(). E is a parameter (not
// hardcoded 256) so the same helper drives both the original E=256 fuzz and the
// expert-count-generality cases (E=24 <32-lane-width, E=168 REAP-pruned, E=200
// lane-straddling boundary, E=257 out-of-contract auto-serial-fallback proof).
static int run_rtop8(int mode, int S, int E, float topp, int normk, float rscale, const char *name) {
const int K=8, Ksel=8;
std::vector<float> sig((size_t)S*E), bias(E);
srand(4242+mode*17+S+E);
for (int e=0;e<E;e++) bias[e]=((rand()%2001)-1000)/1000.f;
for (int s=0;s<S;s++) for (int e=0;e<E;e++) {
float *v=&sig[(size_t)s*E+e];
switch (mode) {
case 0: *v=(float)(rand()%10000)/10000.f; break; // generic sigmoid-like
case 1: *v=0.5f; break; // ALL EQUAL: pure tie-break test
case 2: *v=(float)((e/2)%8)/8.f; break; // massed duplicates (paired+cyclic ties)
case 3: *v=(e%2)?1e-40f:2e-40f; break; // denormal logits (flush behavior must match)
case 4: *v=(float)(rand()%3)/2.f; break; // 3-level ties across the whole row
// boundary-forcing: elevate the LAST 4 valid experts (E-4..E-1) to near-max choice
// so they are guaranteed in the top-8. For an E whose per-lane block size doesn't
// divide E evenly, E-1's lane straddles the E boundary (real indices below E,
// sentinel -1e30f at/above E in the SAME ch[] block) -- e.g. E=200: per=ceil(200/
// 32)=7, lane 28 owns indices 196..202, of which 196-199 are real and 200-202 are
// sentinel. Forcing selection onto 196-199 exercises exactly that lane's per-index
// e<E boundary check, rather than hoping random data happens to land there.
case 5: *v=(e>=E-4)?1.0f:(float)(rand()%10000)/10000.f; break;
default: *v=(float)(rand()%10000)/10000.f; break;
}
}
if (mode==1) for (int e=0;e<E;e++) bias[e]=0.25f; // choice fully tied too
if (mode==3) for (int e=0;e<E;e++) bias[e]=(e%3)?3e-40f:-3e-40f; // denormal bias as well
if (mode==5) for (int e=E-4;e<E;e++) bias[e]=1.0f; // combined choice = 2.0, max possible
std::vector<int> is((size_t)S*K), ip((size_t)S*K); std::vector<float> ws((size_t)S*K), wp((size_t)S*K);
std::vector<int> ks(S), kp(S);
if (!coli_metal_rtop8(0,sig.data(),bias.data(),S,E,K,Ksel,topp,normk,rscale,is.data(),ws.data(),ks.data()) ||
!coli_metal_rtop8(1,sig.data(),bias.data(),S,E,K,Ksel,topp,normk,rscale,ip.data(),wp.data(),kp.data())) {
printf(" %-34s FAIL (rtop8 runner returned 0)\n", name); return 1; }
int ok = memcmp(is.data(),ip.data(),(size_t)S*K*4)==0 &&
memcmp(ws.data(),wp.data(),(size_t)S*K*4)==0 && // bitwise: same ops, same order
memcmp(ks.data(),kp.data(),(size_t)S*4)==0;
if (mode==5 && ok) {
// Don't just trust the input design -- confirm the straddling lane's valid segment
// (E-4..E-1) was actually selected, in EVERY row, so this case can't silently
// degrade into an unrelated pass if the input construction above ever changes.
for (int s=0;s<S;s++) { int seen=0;
for (int k=0;k<K;k++) if (ip[(size_t)s*K+k]>=E-4 && ip[(size_t)s*K+k]<E) seen++;
if (seen<4) { printf(" %-34s *** boundary segment not exercised (row %d saw %d/4) -- test setup bug\n", name, s, seen); return 1; }
}
}
if (!ok) {
printf(" %-34s *** MISMATCH\n", name);
for (int s=0;s<S;s++){ printf(" row %d keff %d/%d:",s,ks[s],kp[s]);
for(int k=0;k<K;k++) printf(" [%d]%d/%d %.6g/%.6g",k,is[s*K+k],ip[s*K+k],ws[s*K+k],wp[s*K+k]);
printf("\n"); }
return 1;
}
printf(" %-34s ok (serial==parallel bitwise, S=%d E=%d)\n", name, S, E);
return 0;
}
int main(void) {
if (!coli_metal_init()) { printf("Metal unavailable (skipping)\n"); return 0; }
printf("Metal backend kernel tests:\n");
@@ -215,6 +277,32 @@ int main(void) {
fail |= run_attn(1, 37, "attn S=1 pos=37");
fail |= run_attn(4, 12, "attn S=4 pos=12 (MTP)");
fail |= run_attn(3, 0, "attn S=3 pos=0");
printf("Metal top-8 select serial-vs-parallel tests (exact-match contract, E=256):\n");
fail |= run_rtop8(0, 1, 256, 0.0f, 1, 1.0f, "top8 generic S=1");
fail |= run_rtop8(0, 4, 256, 0.0f, 1, 1.0f, "top8 generic S=4");
fail |= run_rtop8(1, 1, 256, 0.0f, 1, 1.0f, "top8 ALL-EQUAL ties");
fail |= run_rtop8(2, 4, 256, 0.0f, 1, 1.0f, "top8 massed dup ties S=4");
fail |= run_rtop8(4, 2, 256, 0.0f, 0, 2.5f, "top8 3-level ties rscale");
fail |= run_rtop8(3, 1, 256, 0.0f, 1, 1.0f, "top8 denormal logits");
fail |= run_rtop8(0, 1, 256, 0.01f, 1, 1.0f, "top8 topp=0.01 (Ke=1 edge)");
fail |= run_rtop8(2, 1, 256, 0.6f, 1, 1.0f, "top8 topp=0.6 tied weights");
fail |= run_rtop8(0, 4, 256, 0.999f,1, 1.75f, "top8 topp=0.999 S=4");
fail |= run_rtop8(1, 2, 256, 0.5f, 0, 1.0f, "top8 topp on ALL-EQUAL");
printf("Metal top-8 select expert-count-generality tests (E!=256, REAP/#428 motivated):\n");
fail |= run_rtop8(0, 1, 168, 0.0f, 1, 1.0f, "top8 E=168 (REAP) generic S=1");
fail |= run_rtop8(2, 4, 168, 0.0f, 1, 1.0f, "top8 E=168 (REAP) massed dup ties S=4");
fail |= run_rtop8(0, 1, 24, 0.0f, 1, 1.0f, "top8 E=24 (<32 lane width) generic");
fail |= run_rtop8(1, 1, 24, 0.0f, 1, 1.0f, "top8 E=24 (<32 lane width) ALL-EQUAL ties");
// E=200: per-lane block size ceil(200/32)=7, and 200 is NOT a multiple of 7, so lane 28
// (indices 196..202) straddles the boundary -- 196-199 real, 200-202 sentinel -1e30f in
// the SAME ch[] block. E=24 and E=168 above both happen to divide evenly by their own
// per (24/1, 168/6), so no case before this one exercised a lane whose ch[] mixes real
// and sentinel indices. mode 5 deterministically forces indices 196-199 into the top-8
// (see run_rtop8) and asserts they were actually selected, rather than hoping random
// data lands there -- proving by TEST what the per-index `e<E` check was proven by
// reading (both kernels agree bitwise on a selection that requires that check to fire).
fail |= run_rtop8(5, 4, 200, 0.0f, 1, 1.0f, "top8 E=200 (lane straddles E boundary)");
fail |= run_rtop8(0, 1, 257, 0.0f, 1, 1.0f, "top8 E=257 (>256, auto-serial-fallback)");
printf(fail? "metal backend tests: FAILED\n" : "metal backend tests: ok\n");
coli_metal_shutdown();
return fail;