From 153ee16f61b50633c6ce30008ce56c13aea49069 Mon Sep 17 00:00:00 2001
From: woolcoxm <13604288+woolcoxm@users.noreply.github.com>
Date: Fri, 17 Jul 2026 18:17:29 -0400
Subject: [PATCH] =?UTF-8?q?bench:=20multi-seed=20bench=5Fdsa=5Fselect=20?=
 =?UTF-8?q?=E2=80=94=20kill=20single-input=20pivot-luck=20spike=20(#357)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

@KingIcyCreamProjects caught a real artifact on the 9950X3D (#357 thread): the
nk=8192 cell reported ~75x, far above the real ~13-40x algorithm. Root cause was
two compounding bench bugs, both verified against the code:

  1. ONE frozen input per cell. partial_select_desc's median-of-three pivot is
     fully deterministic (no RNG), and bench_dsa_select froze the input then ran
     2000 reps on that identical array -- so all 2000 reps hit the exact same
     pivot sequence. A single lucky input spiked one nk row (stable across re-runs
     because it's deterministic, not because it's real).
  2. brng was a static global, initialized once and NEVER reset between cells.
     So each (shape,nk) cell's input depended on every prior cell's brand() draws
     -- reordering nks[] or adding a shape silently shifted all later inputs.

Fix:
  - brng_seed() reseeds per (shape, nk, seed) so every cell is reproducible and
    independent of cell ordering.
  - Report the MEDIAN of N_SEEDS=11 independent inputs, each itself a median over
    N_REPEAT=2000 timing reps. A lucky pivot now moves one of 11 samples, not the
    reported number.

Measured on Intel Core Ultra 9 185H (this fix, median of 11 seeds):
  realistic@8192  10.97x   (was 13.57x single-seed on this box)
  uniform@8192     9.39x   (was 7.48x)
  plateau@8192    16.11x   (plateau ignores the RNG, unchanged as expected)
  band: 6-26x across all cells, no anomalous spikes.

Correctness is unaffected (test_dsa_select, 129 cases, unchanged). This is a bench
fidelity fix only -- no engine code touched.
---
 c/tests/bench_dsa_select.c | 61 +++++++++++++++++++++++++++++---------
 1 file changed, 47 insertions(+), 14 deletions(-)

diff --git a/c/tests/bench_dsa_select.c b/c/tests/bench_dsa_select.c
index 1fd3e6b..a1f2014 100644
--- a/c/tests/bench_dsa_select.c
+++ b/c/tests/bench_dsa_select.c
@@ -59,6 +59,12 @@ static double bench_ns(void (*fn)(const float*,int,int,int*,int*),
     return ts[N_REPEAT/2];
 }
 
+/* Sort an array of doubles ascending (median-of-medians aggregation below). */
+static void dsort(double *a, int n){
+    for(int s=1;s<n;s++){ double k=a[s]; int b=s-1;
+        while(b>=0 && a[b]>k){ a[b+1]=a[b]; b--; } a[b+1]=k; }
+}
+
 /* the NEW algorithm calls the real partial_select_desc + replicates the production
  * threshold derivation and position scans. */
 static void keep_new(const float *isc, int nk, int keep, int *dst, int *nd_out){
@@ -74,6 +80,7 @@ static void keep_new(const float *isc, int nk, int keep, int *dst, int *nd_out){
 
 /* deterministic score fill for three shapes */
 static uint32_t brng = 0xA5A5A5A5u;
+static void brng_seed(uint32_t s){ brng = s; }
 static double brand(void){ brng ^= brng << 13; brng ^= brng >> 17; brng ^= brng << 5;
     return (double)(brng >> 8) * (1.0 / 16777216.0); }
 static void fill_realistic(float *isc, int nk){   /* few hot, long distinct tail */
@@ -87,11 +94,24 @@ static void fill_plateau(float *isc, int nk){     /* tie blocks -> boundary path
     for(int i=0;i<nk;i++) isc[i]=(float)(-(double)(i/7));
 }
 
+/* Multi-seed aggregation. Per @KingIcyCreamProjects (#357 thread): with a single frozen
+ * input per cell, quickselect's deterministic median-of-three pivot means one lucky input
+ * can spike a single nk row (an observed ~75x at nk=8192 on a 9950X3D that was really a
+ * ~13-40x algorithm). Two bugs compounded it: (1) the old bench drew ONE input per cell;
+ * (2) brng was never reset, so each cell's input depended on every prior cell's draws --
+ * reordering nks[] silently shifted all later inputs. Both fixed here: brng is reseeded
+ * per (shape, nk, seed), and we take the MEDIAN of N_SEEDS independent inputs, each itself
+ * a median over N_REPEAT timing reps. A lucky pivot now moves one of the N_SEEDS samples,
+ * not the reported number. */
+
 int main(void){
     int keep = 2048;   /* GLM-5.2 index_topk */
     int nks[] = {2049, 4096, 8192, 16384, 32768, 65536};
+    const int N_SEEDS = 11;   /* odd so the median is a real sample, not interpolated */
     float *isc = malloc((size_t)65536*sizeof(float));
     int *dst   = malloc((size_t)65536*sizeof(int)); int nd;
+    double *old_seeds = malloc((size_t)N_SEEDS*sizeof(double));
+    double *new_seeds = malloc((size_t)N_SEEDS*sizeof(double));
 
     struct { const char *name; void (*fill)(float*,int); } shapes[] = {
         { "realistic", fill_realistic },
@@ -99,27 +119,40 @@ int main(void){
         { "plateau",   fill_plateau   },
     };
 
-    printf("bench_dsa_select: DSA top-keep, old (qsort) vs new (partial-select)  keep=%d\n", keep);
+    printf("bench_dsa_select: DSA top-keep, old (qsort) vs new (partial-select)  keep=%d  (median of %d seeds x %d reps)\n",
+           keep, N_SEEDS, N_REPEAT);
     printf("%-12s %7s %14s %14s %9s\n", "shape", "nk", "old ns/call", "new ns/call", "speedup");
     printf("------------------------------------------------------------------------\n");
 
     for(size_t sh=0; sh<sizeof(shapes)/sizeof(shapes[0]); sh++){
         for(size_t ni=0; ni<sizeof(nks)/sizeof(nks[0]); ni++){
             int nk=nks[ni];
-            shapes[sh].fill(isc,nk);
-            /* warmup both paths so caches/branch predictors are primed */
-            for(int w=0; w<50; w++){ keep_old(isc,nk,keep,dst,&nd); keep_new(isc,nk,keep,dst,&nd); }
-            /* sanity: both must keep exactly `keep` (correctness is test_dsa_select's
-             * job, but a count divergence here would make the timing meaningless) */
-            keep_old(isc,nk,keep,dst,&nd); int na=nd;
-            keep_new(isc,nk,keep,dst,&nd); int nb=nd;
-            if(na!=keep || nb!=keep){
-                printf("%-12s %7d   (BAD COUNTS: old=%d new=%d, skipped)\n",
-                       shapes[sh].name, nk, na, nb);
+            int bad = 0;
+            for(int sd=0; sd<N_SEEDS; sd++){
+                /* reseed per (shape,nk,seed) so each cell's input is reproducible and
+                 * independent of cell ordering, and so lucky pivots are sampled, not fixed. */
+                brng_seed(0xA5A5A5A5u + (uint32_t)(sd*0x9E3779B9u));
+                shapes[sh].fill(isc,nk);
+                /* warmup both paths so caches/branch predictors are primed */
+                for(int w=0; w<50; w++){ keep_old(isc,nk,keep,dst,&nd); keep_new(isc,nk,keep,dst,&nd); }
+                /* sanity: both must keep exactly `keep` (correctness is test_dsa_select's
+                 * job, but a count divergence here would make the timing meaningless) */
+                keep_old(isc,nk,keep,dst,&nd); int na=nd;
+                keep_new(isc,nk,keep,dst,&nd); int nb=nd;
+                if(na!=keep || nb!=keep){ bad++; continue; }
+                old_seeds[sd] = bench_ns(keep_old,isc,nk,keep);
+                new_seeds[sd] = bench_ns(keep_new,isc,nk,keep);
+            }
+            if(bad == N_SEEDS){
+                printf("%-12s %7d   (BAD COUNTS on all %d seeds, skipped)\n",
+                       shapes[sh].name, nk, bad);
                 continue;
             }
-            double t_old=bench_ns(keep_old,isc,nk,keep);
-            double t_new=bench_ns(keep_new,isc,nk,keep);
+            /* report median-of-seed-medians: robust to a single lucky/unlucky pivot */
+            dsort(old_seeds, N_SEEDS);
+            dsort(new_seeds, N_SEEDS);
+            double t_old = old_seeds[N_SEEDS/2];
+            double t_new = new_seeds[N_SEEDS/2];
             printf("%-12s %7d %14.0f %14.0f %8.2fx\n",
                    shapes[sh].name, nk, t_old, t_new, t_old/t_new);
         }
@@ -127,6 +160,6 @@ int main(void){
     }
 
     printf("bench_dsa_select: done (lower ns is better; speedup = old/new)\n");
-    free(isc); free(dst);
+    free(isc); free(dst); free(old_seeds); free(new_seeds);
     return 0;
 }