url/colibri
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync: dev

Browse Source
This commit is contained in:
noobdev-ph
2026-07-19 21:47:34 +08:00
parent 468b190db9 61004dcb84
commit 26d6b2d662
50 changed files with 3638 additions and 1452 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c/Makefile
+28 -27
View File
@@ -56,7 +56,7 @@ OMPL =
endif
CFLAGS = -O3 $(OMPC) -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function
# Opt-in: ARCH=native appends -mcpu=native (arm64 clang uses -mcpu, not -march),
# which unlocks the i8mm SMMLA int8/int4 dot kernels in glm.c. ARCH unset ->
# which unlocks the i8mm SMMLA int8/int4 dot kernels in colibri.c. ARCH unset ->
# no -mcpu, default build byte-identical. Apple clang knows apple-m4 / native.
ifneq ($(ARCH),)
CFLAGS += -mcpu=$(ARCH)
@@ -70,7 +70,7 @@ else ifneq ($(IS_WIN),)
# ARCH default = x86-64-v3 (portable binary with AVX2). For max speed on THIS
# machine use ARCH=native: on AVX-VNNI CPUs (Intel Alder Lake+, Meteor Lake+)
# it also unlocks the 128-bit VPDPBUSD int8/int4 dot kernel (dot_i8i8/dot_i4i8),
# which the x86-64-v3 baseline does not define. The #ifdef guards in glm.c mean
# which the x86-64-v3 baseline does not define. The #ifdef guards in colibri.c mean
# a v3 build simply compiles out the VNNI path - safe on any x86-64.
CC = gcc
ARCH ?= x86-64-v3
@@ -207,17 +207,18 @@ LDFLAGS += -framework Metal -framework Foundation -lc++
METAL_OBJ = backend_metal.o
endif
all: glm$(EXE)
all: colibri$(EXE)
# phony 'glm' → 'glm.exe' on Windows (so 'make glm' and 'coli build' work on every platform)
glm: glm$(EXE)
# phony targets — 'glm' kept for backward compatibility
colibri: colibri$(EXE)
glm: colibri$(EXE)
# Config stamp: make only tracks file timestamps, not flag changes. Without this,
# `make glm.exe CUDA_DLL=1` after a prior CPU-only build reports "up to date" and
# silently keeps the CPU-only binary (no CUDA loader) — a build that looks like it
# worked but isn't. We record the build-affecting flags in .build-config and rewrite
# it ONLY when they change (evaluated here at parse time, so the file's timestamp
# moves exactly when the config moves). glm.exe and the CUDA/loader objects depend
# `make colibri.exe CUDA_DLL=1` after a prior CPU-only build reports "up to date"
# and silently keeps the CPU-only binary (no CUDA loader) — a build that looks like
# it worked but isn't. We record the build-affecting flags in .build-config and
# rewrite it ONLY when they change (evaluated here at parse time, so the file's
# timestamp moves exactly when the config moves). The binary and CUDA/loader objects depend
# on it, so they relink on a config change and stay put otherwise. (#306)
BUILD_CONFIG := $(CC)|$(CFLAGS)|$(LDFLAGS)|CUDA=$(CUDA)|CUDA_DLL=$(CUDA_DLL)|ARCH=$(ARCH)|CUDA_ARCH=$(CUDA_ARCH)|METAL=$(METAL)
BUILD_CONFIG_OLD := $(shell cat .build-config 2>/dev/null)
@@ -226,8 +227,8 @@ $(shell printf '%s' '$(BUILD_CONFIG)' > .build-config)
endif
.build-config: ;
glm$(EXE): glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h $(CUDA_OBJ) $(METAL_OBJ) .build-config
$(CC) $(CFLAGS) glm.c $(CUDA_OBJ) $(METAL_OBJ) -o glm$(EXE) $(LDFLAGS)
colibri$(EXE): colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h quant.h sample.h kv_persist.h telemetry.h $(CUDA_OBJ) $(METAL_OBJ) .build-config
$(CC) $(CFLAGS) colibri.c $(CUDA_OBJ) $(METAL_OBJ) -o colibri$(EXE) $(LDFLAGS)
# Windows runtime loader object: resolves coli_cuda_* from coli_cuda.dll.
backend_loader.o: backend_loader.c backend_cuda.h compat.h .build-config
@@ -285,7 +286,7 @@ PORTABLE_ARCH = native
endif
portable:
$(MAKE) glm$(EXE) ARCH=$(PORTABLE_ARCH)
$(MAKE) colibri$(EXE) ARCH=$(PORTABLE_ARCH)
iobench$(EXE): iobench.c compat.h
$(CC) $(CFLAGS) iobench.c -o iobench$(EXE) $(LDFLAGS)
@@ -311,30 +312,30 @@ tests/test_schema_gbnf$(EXE): tests/test_schema_gbnf.c schema_gbnf.h grammar.h j
tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_idot$(EXE): tests/test_idot.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
tests/test_idot$(EXE): tests/test_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_stops$(EXE): tests/test_stops.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
tests/test_stops$(EXE): tests/test_stops.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_topp$(EXE): tests/test_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
tests/test_topp$(EXE): tests/test_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
# bench_topp is a microbenchmark (old qsort vs new heap partial-select, #335), NOT a test
# gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_topp
tests/bench_topp$(EXE): tests/bench_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
tests/bench_topp$(EXE): tests/bench_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_sample_nan$(EXE): tests/test_sample_nan.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
tests/test_sample_nan$(EXE): tests/test_sample_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c colibri.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_logit_nan$(EXE): tests/test_logit_nan.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
tests/test_logit_nan$(EXE): tests/test_logit_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c
@@ -343,15 +344,15 @@ tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c
tests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_dsa_select$(EXE): tests/test_dsa_select.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
tests/test_dsa_select$(EXE): tests/test_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
# bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356),
# NOT a test gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_dsa_select
tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_uring$(EXE): tests/test_uring.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
tests/test_uring$(EXE): tests/test_uring.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
test-c: $(TEST_BINS)
@@ -368,12 +369,12 @@ check:
$(MAKE) portable
$(MAKE) test
install: glm$(EXE) olmoe$(EXE)
install: colibri$(EXE) olmoe$(EXE)
$(INSTALL) -d $(DESTDIR)$(BINDIR)
$(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)
$(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)/tools
$(INSTALL) -m 755 coli $(DESTDIR)$(BINDIR)/coli
$(INSTALL) -m 755 glm$(EXE) $(DESTDIR)$(LIBEXECDIR)/glm$(EXE)
$(INSTALL) -m 755 colibri$(EXE) $(DESTDIR)$(LIBEXECDIR)/colibri$(EXE)
$(INSTALL) -m 755 olmoe$(EXE) $(DESTDIR)$(LIBEXECDIR)/olmoe$(EXE)
$(INSTALL) -m 644 resource_plan.py doctor.py openai_server.py $(DESTDIR)$(LIBEXECDIR)/
$(INSTALL) -m 644 tools/*.py $(DESTDIR)$(LIBEXECDIR)/tools/
@@ -387,4 +388,4 @@ clean:
bench: iobench$(EXE)
@if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) — run: ./iobench$(EXE) <file> <MB> <iters> <threads> <direct 0|1>"; fi
.PHONY: all glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench
.PHONY: all colibri glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench
c/coli
+10 -6
View File
@@ -54,16 +54,20 @@ from version import __version__ as _version
# guess is right (e.g. a custom packaging layout).
_EXE = ".exe" if sys.platform == "win32" else ""
_LIBEXEC = os.path.join(os.path.dirname(HERE), "libexec", "colibri")
_here_colibri = os.path.join(HERE, "colibri" + _EXE)
_here_glm = os.path.join(HERE, "glm" + _EXE)
if os.environ.get("COLI_ENGINE"):
GLM = os.environ["COLI_ENGINE"]
TOOLS = os.path.join(os.path.dirname(GLM), "tools")
elif os.path.exists(_here_colibri):
GLM = _here_colibri
TOOLS = os.path.join(HERE, "tools")
elif os.path.exists(_here_glm):
GLM = _here_glm
TOOLS = os.path.join(HERE, "tools")
else:
GLM = os.path.join(_LIBEXEC, "glm" + _EXE)
GLM = os.path.join(_LIBEXEC, "colibri" + _EXE)
TOOLS = os.path.join(_LIBEXEC, "tools")
sys.path.insert(0, _LIBEXEC) # so `import resource_plan`, `doctor`, `openai_server` still resolve
@@ -241,13 +245,13 @@ def env_for(a):
e["COLI_CUDA"]="0"; e.pop("CUDA_EXPERT_GB",None); e.pop("CUDA_DENSE",None)
else:
if not cuda_binary():
sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} make glm CUDA=1 (this binary is CPU-only)")
sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} make colibri CUDA=1 (this binary is CPU-only)")
e["COLI_CUDA"]="1"
if a.gpu!="auto": e["COLI_GPUS"]=a.gpu
e.setdefault("CUDA_DENSE","1")
if a.vram and a.gpu!="none":
if not cuda_binary():
sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} make glm CUDA=1 (this binary is CPU-only)")
sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} make colibri CUDA=1 (this binary is CPU-only)")
e["COLI_CUDA"]="1"; e["CUDA_EXPERT_GB"]=str(a.vram)
return e
@@ -421,8 +425,8 @@ def cmd_build(a):
banner("build")
if not os.path.exists(os.path.join(HERE, "Makefile")):
sys.exit(f"{C.yel}coli build{C.r} only works from a source checkout (this is an installed copy).\n"
f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c glm.")
sys.exit(subprocess.call(["make","-C",HERE,"glm"]))
f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c colibri.")
sys.exit(subprocess.call(["make","-C",HERE,"colibri"]))
def cmd_info(a):
banner("info")
@@ -802,7 +806,7 @@ def cmd_stop(a):
if "coli" in cmd and " serve" in cmd and pid!=os.getpid():
if not any(p==pid for p,_ in targets): targets.append((pid,"coli serve (cmdline)"))
comm=open(f"/proc/{pd}/comm").read().strip()
if comm in ("glm","exe","olmoe"):
if comm in ("colibri","glm","exe","olmoe"):
env=open(f"/proc/{pd}/environ","rb").read().replace(b"\0",b"\n").decode("utf-8","replace")
if "SERVE=1" in env: targets.append((pid,f"engine `{comm}` (SERVE=1)"))
except (OSError,PermissionError): continue
c/glm.c → c/colibri.c
+65 -1235
View File
File diff suppressed because it is too large Load Diff
c/kv_persist.h
+121
View File
@@ -0,0 +1,121 @@
/* kv_persist.h — .coli_kv on-disk KV cache persistence.
* Conversations reopen warm across engine restarts: the compressed MLA KV-cache
* is appended incrementally after every turn, crash-safe (nrec written last).
* Include after Model/KVState/Cfg are defined; requires now_s() and g_draft. */
#ifndef KV_PERSIST_H
#define KV_PERSIST_H
static int g_kvsave=1;
#define KV_MAGIC "COLIKV1\0"
static void kv_hdr(Model *m, int32_t *h, int nrec){
Cfg *c=&m->c; int nic=0;
for(int i=0;i<c->n_layers;i++) if(m->Ic && m->Ic[i]) nic++;
h[0]=c->n_layers; h[1]=c->kv_lora; h[2]=c->qk_rope;
h[3]=m->has_dsa?c->index_hd:0; h[4]=nic; h[5]=c->vocab; h[6]=nrec; h[7]=0;
}
static int64_t kv_rec_bytes(Model *m){
Cfg *c=&m->c;
int64_t rec = 4 + (int64_t)c->n_layers*(c->kv_lora+c->qk_rope)*4;
if(m->has_dsa) for(int i=0;i<c->n_layers;i++) if(m->Ic[i]) rec+=(int64_t)c->index_hd*4;
return rec;
}
static int kv_disk_open(Model *m){
KVState *k=m->kv;
if(k->disk_fp) return 1;
k->disk_fp=fopen(k->disk_path,"r+b");
if(!k->disk_fp){
k->disk_fp=fopen(k->disk_path,"wb");
if(!k->disk_fp) return 0;
int32_t h[8]; kv_hdr(m,h,0);
fwrite(KV_MAGIC,1,8,k->disk_fp); fwrite(h,4,8,k->disk_fp);
fflush(k->disk_fp);
fclose(k->disk_fp);
k->disk_fp=fopen(k->disk_path,"r+b");
if(!k->disk_fp) return 0;
}
return 1;
}
static void kv_disk_truncate(Model *m, int nrec){
if(!g_kvsave) return;
KVState *k=m->kv;
if(k->disk_fp){ fclose(k->disk_fp); k->disk_fp=NULL; }
FILE *f=fopen(k->disk_path,"r+b");
if(!f){ k->disk_nrec=0; return; }
k->disk_nrec=nrec;
int32_t nr=nrec; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f);
fflush(f); fclose(f);
}
static void kv_disk_reset(Model *m){ kv_disk_truncate(m,0); }
static void kv_disk_append(Model *m, const int *hist, int len){
KVState *k=m->kv;
if(!g_kvsave || len<=k->disk_nrec) return;
Cfg *c=&m->c;
if(!kv_disk_open(m)) return;
FILE *f=k->disk_fp;
int64_t rec = kv_rec_bytes(m);
if(rec > k->disk_buf_cap){
uint8_t *nb=realloc(k->disk_buf, rec);
if(!nb) return;
k->disk_buf=nb; k->disk_buf_cap=rec;
}
fseek(f, 8+8*4 + (int64_t)k->disk_nrec*rec, SEEK_SET);
for(int p=k->disk_nrec;p<len;p++){
uint8_t *b=k->disk_buf;
*(int32_t*)b = hist[p]; b+=4;
for(int i=0;i<c->n_layers;i++){
memcpy(b, m->Lc[i]+(int64_t)p*c->kv_lora, (size_t)c->kv_lora*4); b+=c->kv_lora*4;
memcpy(b, m->Rc[i]+(int64_t)p*c->qk_rope,(size_t)c->qk_rope*4); b+=c->qk_rope*4;
}
if(m->has_dsa) for(int i=0;i<c->n_layers;i++) if(m->Ic[i]){
memcpy(b, m->Ic[i]+(int64_t)p*c->index_hd, (size_t)c->index_hd*4); b+=c->index_hd*4;
}
fwrite(k->disk_buf, 1, (size_t)rec, f);
}
fflush(f);
int32_t nr=len; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f);
fflush(f);
k->disk_nrec=len;
}
static int kv_disk_load(Model *m, int *hist, int maxctx){
if(!g_kvsave) return 0;
KVState *k=m->kv;
Cfg *c=&m->c;
FILE *f=fopen(k->disk_path,"rb"); if(!f) return 0;
char mg[8]; int32_t h[8], w[8]; kv_hdr(m,w,0);
if(fread(mg,1,8,f)!=8 || memcmp(mg,KV_MAGIC,8) || fread(h,4,8,f)!=8 ||
h[0]!=w[0]||h[1]!=w[1]||h[2]!=w[2]||h[3]!=w[3]||h[4]!=w[4]||h[5]!=w[5]){
fprintf(stderr,"[KV] ignoring .coli_kv from a different model or version\n"); fclose(f); return 0; }
int nrec=h[6];
if(nrec<1){ fclose(f); return 0; }
if(nrec>=maxctx-8-g_draft){
fprintf(stderr,"[KV] saved conversation (%d tokens) exceeds the context: starting over\n",nrec);
fclose(f); return 0; }
double t0=now_s();
for(int p=0;p<nrec;p++){
int32_t tk; if(fread(&tk,4,1,f)!=1){ nrec=p; break; } hist[p]=tk;
for(int i=0;i<c->n_layers;i++){
if(fread(m->Lc[i]+(int64_t)p*c->kv_lora, 4, c->kv_lora, f)!=(size_t)c->kv_lora ||
fread(m->Rc[i]+(int64_t)p*c->qk_rope, 4, c->qk_rope, f)!=(size_t)c->qk_rope){ nrec=p; goto out; }
}
if(m->has_dsa) for(int i=0;i<c->n_layers;i++) if(m->Ic[i])
if(fread(m->Ic[i]+(int64_t)p*c->index_hd, 4, c->index_hd, f)!=(size_t)c->index_hd){ nrec=p; goto out; }
}
out:
fclose(f);
if(nrec>0){
if(m->has_mtp) m->kv_start[c->n_layers]=-1;
fprintf(stderr,"[KV] resumed conversation from disk: %d tokens in %.1fs (no re-prefill)\n",
nrec, now_s()-t0);
}
k->disk_nrec=nrec;
return nrec;
}
#endif /* KV_PERSIST_H */
c/olmoe.c
+529 -38
View File
@@ -5,6 +5,15 @@
* Densa (embed, attn, router, norme, lm_head) residente in RAM (float32).
* Expert letti dal disco on-demand via pread+fadvise(DONTNEED), cache LRU per-layer.
* Matmul multi-thread con OpenMP (niente BLAS).
*
* ENV VARS:
* PILOT=0/1/2/3 : 0=no prefetch, 1=1-layer lookahead, 2=2-layer, 3=3-layer lookahead
* HOT=N : pin top-N hot experts per layer permanently (never evict)
* WARMUP=N : tokens before hot pinning activates (default 5)
* WIDE=N : prefetch top-K*N candidates (default 1, try 2 or 3)
* SMOOTH=F : EMA coefficient for routing momentum (default 0.3, range 0.0-0.95)
* CONF_LIMIT=F : cumulative gate probability threshold for prefetch cutoff (default 0.92)
* (expert queue is sorted by eid for SSD read locality)
*/
#define _GNU_SOURCE
#include <stdio.h>
@@ -12,11 +21,22 @@
#include <string.h>
#include <math.h>
#include <time.h>
#include <pthread.h>
#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__)
#include <sys/resource.h>
#include <unistd.h>
#endif
#include "st.h"
#ifdef _WIN32
#include <windows.h>
#define sleep_ms(ms) Sleep(ms)
#else
#define sleep_ms(ms) usleep((ms) * 1000)
#endif
/* ---------- config ---------- */
typedef struct {
int hidden, n_layers, n_heads, n_kv_heads, head_dim;
@@ -33,22 +53,61 @@ typedef struct {
* Ogni weight [out,in] tenuto come int8 (per-riga) + scala float per riga.
* Cosi' la RAM-cache scende da 4 byte/param (f32) a 1 byte/param: e' il
* meccanismo che fa stare GLM-5.2 nei 15 GB. dequant-on-use nel matmul. */
typedef struct { int eid; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot;
/* pinned=1 means this slot is strongly preferred to keep (hot expert); it will
* not be evicted during normal LRU eviction, but may be displaced under extreme
* cache pressure when all slots are pinned or in-flight. */
typedef struct { int eid; int pinned; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot;
typedef struct { Slot *slots; int n, cap; } LCache;
typedef struct {
Cfg c;
shards S;
int quant_bits; /* bit di quantizzazione degli expert (2..8); storage int8, niente f32 (#134) */
int quant_bits;
float *embed, *lm_head, *final_norm;
Layer *L;
LCache *cache; /* [n_layers] */
uint64_t clock, hits, miss;
/* kv-cache per-layer: K,V come [H * maxT * head_dim] */
float **K, **V; int kv_len, max_t;
double dense_load_s;
/* IMPROVEMENT 2: expert frequency heatmap */
uint32_t *freq;
int freq_token_count, hot_pinned, hot_n, warmup_tokens;
int token_count;
/* PREDICTION IMPROVEMENT A: per-layer EMA of gate logits across tokens.
* momentum_logits[l*E .. (l+1)*E-1] = EMA of gate outputs for layer l.
* Used exclusively by the PILOT prefetcher to stabilise routing predictions
* across tokens; does NOT affect actual MoE routing (pr is unchanged). */
float *momentum_logits; /* [n_layers * n_experts], EMA of gate logits */
float pilot_smooth; /* SMOOTH env: EMA coefficient 0.0-0.9 (default 0.3) */
uint8_t *is_pinned; /* [n_layers * n_experts], 1 if expert is globally pinned */
uint8_t *is_queued; /* [n_layers * n_experts], 1 if expert is currently in the prefetch queue */
float pilot_conf_limit; /* CONF_LIMIT env: cumulative gate probability threshold (e.g. 0.92) */
} Model;
static pthread_mutex_t g_pilot_mx = PTHREAD_MUTEX_INITIALIZER;
static struct { int l, e; } pilot_q[4096];
static volatile unsigned pilot_r = 0, pilot_w = 0;
static Model *pilot_m = NULL;
static int g_pilot = 0;
static int g_wide = 1; /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */
static void pilot_prefetch(Model *m, int lnext, const float *x, int S);
static void *pilot_worker(void *arg);
static void ensure_pilot_worker_started(Model *m);
static void slot_ensure_allocated(Model *m, Slot *s);
static void ensure_pilot_worker_started(Model *m) {
if (!pilot_m) {
pilot_m = m;
pthread_t t;
if (pthread_create(&t, NULL, pilot_worker, NULL) != 0) {
fprintf(stderr, "Error: Failed to create pilot prefetch worker thread\n");
exit(1);
}
pthread_detach(t);
}
}
/* ---------- utility ---------- */
static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec*1e-9; }
#if defined(__APPLE__)
@@ -210,51 +269,224 @@ static void model_init(Model *m, const char *snap, int cap, int bits) {
#undef LD
}
m->cache = calloc(c->n_layers, sizeof(LCache));
for (int i = 0; i < c->n_layers; i++) { m->cache[i].cap = cap; m->cache[i].slots = calloc(cap, sizeof(Slot)); }
for (int i = 0; i < c->n_layers; i++) {
m->cache[i].cap = cap;
m->cache[i].slots = calloc(cap, sizeof(Slot));
}
/* IMPROVEMENT 2: frequency heatmap for hot expert pinning */
m->freq = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint32_t));
m->hot_pinned = 0; m->freq_token_count = 0;
m->hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0;
m->warmup_tokens = getenv("WARMUP") ? atoi(getenv("WARMUP")) : 5;
m->token_count = 0;
/* PREDICTION A: routing momentum — EMA of gate logits across tokens.
* Initialized to zero; first token sets EMA = fresh logits. */
m->momentum_logits = calloc((size_t)c->n_layers * c->n_experts, sizeof(float));
float sv = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f;
if (sv < 0.f) sv = 0.f; if (sv > 0.95f) sv = 0.95f;
m->pilot_smooth = sv;
m->is_pinned = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t));
m->is_queued = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t));
float cl = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f;
if (cl < 0.1f) cl = 0.1f; if (cl > 1.0f) cl = 1.0f;
m->pilot_conf_limit = cl;
m->dense_load_s = now_s() - t0;
// Persistent Hot Pinning: try to load hot_pinned.bin
char pinpath[512];
snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap);
FILE *pinf = fopen(pinpath, "rb");
if (pinf) {
size_t expected_size = (size_t)c->n_layers * c->n_experts;
if (fread(m->is_pinned, 1, expected_size, pinf) == expected_size) {
m->hot_pinned = 1;
printf("[HOT] Loaded persistent pinning from %s\n", pinpath);
if (g_pilot) {
ensure_pilot_worker_started(m);
for (int l = 0; l < c->n_layers; l++) {
for (int e = 0; e < c->n_experts; e++) {
if (m->is_pinned[l * c->n_experts + e]) {
unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED);
unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE);
if (w - r < 4096) {
pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = e;
pthread_mutex_lock(&g_pilot_mx);
m->is_queued[l * c->n_experts + e] = 1;
pthread_mutex_unlock(&g_pilot_mx);
__atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE);
}
}
}
}
printf("[HOT] Pre-loading pinned experts into cache...\n");
double t_wait = now_s();
while (1) {
unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE);
unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE);
if (r == w) break;
sleep_ms(2);
}
printf("[HOT] Pre-loaded in %.1fs!\n", now_s() - t_wait);
}
}
fclose(pinf);
}
}
/* legge un weight dal disco (streaming) e lo quantizza in q[O,I]+scale[O].
* Container pre-quantizzato (convert_olmoe.py: int8 + scale f32 in "name.qs"):
* lettura raw diretta — meta' I/O e zero quantize_rows a runtime. Prima di
* questa patch il container int8 causava SIGBUS (st_read_f32 su tensori I8). */
static void load_expert_w(Model *m, const char *name, int8_t *q, float *scale, int O, int I, float *tmp) {
st_tensor *t = st_find(&m->S, name);
if (t && t->dtype == 3) { /* I8/U8: container colibri */
char qs[300]; snprintf(qs, sizeof(qs), "%s.qs", name);
st_read_raw(&m->S, name, q, 1);
st_read_f32(&m->S, qs, scale, 1);
return;
static void slot_ensure_allocated(Model *m, Slot *s) {
if (s->g) return;
Cfg *c = &m->c;
int64_t ng = (int64_t)c->inter * c->hidden;
int64_t nd = (int64_t)c->hidden * c->inter;
int8_t *w_block = malloc(ng + ng + nd);
if (!w_block) {
fprintf(stderr, "Error: Out of memory allocating slot weights block\n");
exit(1);
}
st_read_f32(&m->S, name, tmp, 1); /* pread + fadvise DONTNEED */
quantize_rows(tmp, q, scale, O, I, m->quant_bits);
s->g = w_block;
s->u = w_block + ng;
s->d = w_block + ng + ng;
float *s_block = falloc(c->inter + c->inter + c->hidden);
s->gs = s_block;
s->us = s_block + c->inter;
s->ds = s_block + c->inter + c->inter;
s->pinned = 0;
}
static void load_expert_merged(Model *m, int layer, int eid, Slot *s) {
char nm[256], qsnm[256];
snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.merged_weight", layer, eid);
snprintf(qsnm, sizeof(qsnm), "model.layers.%d.mlp.experts.%d.qs", layer, eid);
st_read_raw(&m->S, nm, s->g, 1);
st_read_f32(&m->S, qsnm, s->gs, 0); /* scales are F32; use typed reader for dtype safety */
}
/* ---------- cache expert: ritorna i pesi quantizzati (q+scale) da cache o disco ---------- */
static void expert_get(Model *m, int layer, int eid, Slot **out) {
LCache *lc = &m->cache[layer];
pthread_mutex_lock(&g_pilot_mx);
for (int i = 0; i < lc->n; i++) if (lc->slots[i].eid == eid) {
m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; return;
m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i];
pthread_mutex_unlock(&g_pilot_mx);
return;
}
m->miss++;
Cfg *c = &m->c;
int64_t ng = (int64_t)c->inter * c->hidden, nd = (int64_t)c->hidden * c->inter;
Slot *s;
if (lc->n < lc->cap) {
s = &lc->slots[lc->n++];
s->g = malloc(ng); s->u = malloc(ng); s->d = malloc(nd);
s->gs = falloc(c->inter); s->us = falloc(c->inter); s->ds = falloc(c->hidden);
} else { int lru = 0; for (int i = 1; i < lc->n; i++) if (lc->slots[i].used < lc->slots[lru].used) lru = i; s = &lc->slots[lru]; }
float *tmp = falloc(ng > nd ? ng : nd);
char nm[256];
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.gate_proj.weight",layer,eid); load_expert_w(m,nm,s->g,s->gs,c->inter,c->hidden,tmp);
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.up_proj.weight", layer,eid); load_expert_w(m,nm,s->u,s->us,c->inter,c->hidden,tmp);
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.down_proj.weight",layer,eid); load_expert_w(m,nm,s->d,s->ds,c->hidden,c->inter,tmp);
free(tmp);
s->eid = eid; s->used = ++m->clock;
slot_ensure_allocated(m, s);
} else {
/* LRU eviction — skip pinned and in-flight (eid==-1) slots */
int lru = -1;
for (int i = 0; i < lc->n; i++) {
if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue;
if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i;
}
if (lru < 0) {
/* All slots are pinned or in-flight; find oldest non-in-flight slot
* (may be pinned, but never select one currently being loaded). */
for (int i = 0; i < lc->n; i++) {
if (lc->slots[i].eid < 0) continue; /* never evict in-flight */
if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i;
}
}
if (lru < 0) lru = 0; /* absolute last resort: all in-flight, evict slot 0 */
s = &lc->slots[lru];
s->pinned = 0;
}
s->eid = -1;
s->used = ++m->clock;
pthread_mutex_unlock(&g_pilot_mx);
load_expert_merged(m, layer, eid, s);
pthread_mutex_lock(&g_pilot_mx);
s->eid = eid;
s->pinned = m->is_pinned[layer * c->n_experts + eid];
s->used = ++m->clock;
*out = s;
pthread_mutex_unlock(&g_pilot_mx);
}
/* ---------- IMPROVEMENT 2: pin top-N hot experts per layer ---------- */
static void pin_hot_experts(Model *m) {
Cfg *c = &m->c;
if (m->hot_n <= 0 || m->hot_pinned) return;
m->hot_pinned = 1;
int is_dynamic = (m->hot_n >= 100);
double thresh = is_dynamic ? (double)m->hot_n / 1000.0 : 0.0;
int pinned_total = 0;
for (int l = 0; l < c->n_layers; l++) {
uint32_t *freq_l = m->freq + (int64_t)l * c->n_experts;
uint64_t layer_total = 0;
for (int e = 0; e < c->n_experts; e++) layer_total += freq_l[e];
if (layer_total == 0) continue;
int max_pin = m->cache[l].cap - 8;
if (max_pin < 4) max_pin = 4;
int hn = is_dynamic ? max_pin : (m->hot_n < c->n_experts ? m->hot_n : c->n_experts);
if (hn > 256) hn = 256;
int hot_eids[256];
int actual_hn = 0;
for (int k = 0; k < hn; k++) {
int best = -1; uint32_t bv = 0;
for (int e = 0; e < c->n_experts; e++) {
int already = 0;
for (int j = 0; j < k; j++) if (hot_eids[j] == e) { already = 1; break; }
if (!already && freq_l[e] > bv) { bv = freq_l[e]; best = e; }
}
if (best < 0 || bv == 0) break;
if (is_dynamic && bv < thresh * layer_total) break;
hot_eids[k] = best;
actual_hn++;
}
for (int k = 0; k < actual_hn; k++) {
int eid = hot_eids[k];
m->is_pinned[l * c->n_experts + eid] = 1;
LCache *lc = &m->cache[l];
int found = 0;
pthread_mutex_lock(&g_pilot_mx);
for (int i = 0; i < lc->n; i++) {
if (lc->slots[i].eid == eid) { lc->slots[i].pinned = 1; found = 1; break; }
}
pthread_mutex_unlock(&g_pilot_mx);
if (!found && g_pilot > 0) {
/* Only enqueue when the prefetch worker is active (PILOT>0). */
ensure_pilot_worker_started(m);
unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED);
unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE);
int gidx = l * c->n_experts + eid;
pthread_mutex_lock(&g_pilot_mx);
int already = m->is_queued[gidx];
if (!already && w - r < 4096) {
pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = eid;
m->is_queued[gidx] = 1;
__atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE);
}
pthread_mutex_unlock(&g_pilot_mx);
}
pinned_total++;
}
}
if (is_dynamic) {
printf("[HOT] Dynamic Pinned %d experts total (thresh=%.1f%%) after %d warmup tokens\n",
pinned_total, thresh * 100.0, m->freq_token_count);
} else {
printf("[HOT] Pinned %d experts (top-%d/layer) after %d warmup tokens\n",
pinned_total, m->hot_n, m->freq_token_count);
}
}
/* ---------- RoPE su un vettore di una testa (head_dim) a posizione assoluta pos ---------- */
static void rope_head(float *x, int pos, const Cfg *c) {
int h = c->head_dim / 2;
@@ -325,6 +557,19 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
float *g = falloc(I), *u = falloc(I), *hh = falloc(D);
for (int s = 0; s < S; s++) {
float *pr = logits + (int64_t)s*E;
if (m->momentum_logits && m->pilot_smooth > 0.f) {
float *ema = m->momentum_logits + (int64_t)layer * E;
int is_zero = 1;
for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } }
if (is_zero) {
for (int e = 0; e < E; e++) ema[e] = pr[e];
} else {
for (int e = 0; e < E; e++) {
ema[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e];
}
}
}
softmax_row(pr, E);
/* top-K indici (selezione parziale) */
int idx[64]; float val[64];
@@ -337,6 +582,11 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
idx[kk] = best; val[kk] = bv;
}
if (c->norm_topk) { float sm=0; for(int kk=0;kk<K;kk++) sm+=val[kk]; for(int kk=0;kk<K;kk++) val[kk]/=sm; }
/* IMPROVEMENT 2: update activation heatmap (before pinning activates) */
if (!m->hot_pinned && m->freq) {
uint32_t *freq_l = m->freq + (int64_t)layer * E;
for (int kk = 0; kk < K; kk++) if (idx[kk] >= 0) freq_l[idx[kk]]++;
}
const float *xs = x + (int64_t)s*D;
for (int kk = 0; kk < K; kk++) {
Slot *e; expert_get(m, layer, idx[kk], &e);
@@ -352,9 +602,20 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
free(logits); free(g); free(u); free(hh);
}
/* un passo: token nuovi ids[S] a posizione pos_base. Ritorna logits dell'ultimo token (malloc'd). */
static float *step(Model *m, const int *ids, int S, int pos_base) {
Cfg *c = &m->c; int D = c->hidden;
if (g_pilot && m->token_count > 0) {
/* Flush stale prefetch requests: clear is_queued so pilot_realload
* will skip any entries still sitting in pilot_q for the previous
* token. We deliberately do NOT move pilot_w backwards; that would
* break the ring-buffer invariant (pilot_r could exceed pilot_w if
* the worker consumed an entry concurrently). The worker will drain
* the stale slots harmlessly because pilot_realload already exits
* early when the expert is already cached or is_queued is clear. */
pthread_mutex_lock(&g_pilot_mx);
memset(m->is_queued, 0, (size_t)c->n_layers * c->n_experts);
pthread_mutex_unlock(&g_pilot_mx);
}
float *x = falloc((int64_t)S*D);
for (int s = 0; s < S; s++) memcpy(x + (int64_t)s*D, m->embed + (int64_t)ids[s]*D, D*sizeof(float));
float *nrm = falloc((int64_t)S*D), *tmp = falloc((int64_t)S*D);
@@ -363,12 +624,26 @@ static float *step(Model *m, const int *ids, int S, int pos_base) {
for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->in_ln, D, c->eps);
attention(m, l, i, nrm, S, pos_base, tmp);
for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j];
/* IMPROVEMENT 1: PILOT=1 -> 1-layer lookahead */
if (g_pilot >= 1 && S <= 8 && i + 1 < c->n_layers)
pilot_prefetch(m, i + 1, x, S);
for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->post_ln, D, c->eps);
moe(m, l, i, nrm, S, tmp);
for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j];
/* PREDICTION IMPROVEMENT C (Residual gate trick):
* PILOT=2 -> prefetch layer i+2 using completed state x (containing MoE residual). */
if (g_pilot >= 2 && S <= 8 && i + 2 < c->n_layers)
pilot_prefetch(m, i + 2, x, S);
if (g_pilot >= 3 && S <= 8 && i + 3 < c->n_layers)
pilot_prefetch(m, i + 3, x, S);
}
/* count actual tokens processed (S>1 during prefill) */
m->token_count += S; m->freq_token_count += S;
if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens)
pin_hot_experts(m);
m->kv_len = pos_base + S;
/* solo l'ultimo token -> logits */
float *last = falloc(D);
rmsnorm_row(last, x + (int64_t)(S-1)*D, m->final_norm, D, c->eps);
float *logit = falloc(c->vocab);
@@ -377,6 +652,192 @@ static float *step(Model *m, const int *ids, int S, int pos_base) {
return logit;
}
static void pilot_realload(Model *m, int layer, int eid) {
LCache *lc = &m->cache[layer];
Cfg *c = &m->c;
pthread_mutex_lock(&g_pilot_mx);
/* Early-exit if entry was flushed (is_queued cleared) while waiting. */
if (!m->is_queued[layer * c->n_experts + eid]) {
pthread_mutex_unlock(&g_pilot_mx);
return;
}
for (int i = 0; i < lc->n; i++) {
if (lc->slots[i].eid == eid) {
m->is_queued[layer * c->n_experts + eid] = 0;
pthread_mutex_unlock(&g_pilot_mx);
return;
}
}
Slot *s;
if (lc->n < lc->cap) {
s = &lc->slots[lc->n++];
slot_ensure_allocated(m, s);
} else {
/* LRU eviction — skip pinned and in-flight (eid==-1) slots */
int lru = -1;
for (int i = 0; i < lc->n; i++) {
if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue;
if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i;
}
if (lru < 0) {
m->is_queued[layer * c->n_experts + eid] = 0;
pthread_mutex_unlock(&g_pilot_mx);
return; /* all pinned/in-flight, skip */
}
s = &lc->slots[lru]; s->pinned = 0;
}
s->eid = -1; s->used = ++m->clock;
pthread_mutex_unlock(&g_pilot_mx);
load_expert_merged(m, layer, eid, s);
pthread_mutex_lock(&g_pilot_mx);
s->eid = eid;
s->pinned = m->is_pinned[layer * c->n_experts + eid];
s->used = ++m->clock;
m->is_queued[layer * c->n_experts + eid] = 0;
pthread_mutex_unlock(&g_pilot_mx);
}
static void *pilot_worker(void *arg) {
(void)arg;
while (1) {
unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE);
unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE);
if (r == w) {
sleep_ms(1);
continue;
}
int layer = pilot_q[r & 4095].l;
int eid = pilot_q[r & 4095].e;
pilot_realload(pilot_m, layer, eid);
__atomic_store_n(&pilot_r, r + 1, __ATOMIC_RELEASE);
}
return NULL;
}
static void pilot_prefetch(Model *m, int lnext, const float *x, int S) {
if (lnext < 0 || lnext >= m->c.n_layers) return;
Cfg *c = &m->c; int D = c->hidden, E = c->n_experts;
ensure_pilot_worker_started(m);
float *logits = falloc((int64_t)S * E);
Layer *l = &m->L[lnext];
// PREDICTION IMPROVEMENT B: Apply RMSNorm to x using destination layer's post_ln
// This scales inputs to the distribution expected by l->gate.
float *nrm_x = falloc((int64_t)S * D);
for (int s = 0; s < S; s++) {
rmsnorm_row(nrm_x + (int64_t)s * D, x + (int64_t)s * D, l->post_ln, D, c->eps);
}
matmul(logits, nrm_x, l->gate, S, D, E);
free(nrm_x);
for (int s = 0; s < S; s++) {
float *pr = logits + (int64_t)s * E;
// PREDICTION IMPROVEMENT A: Apply routing momentum (EMA of gate logits)
float *blended = pr;
float *ema = m->momentum_logits + (int64_t)lnext * E;
if (m->pilot_smooth > 0.f) {
blended = falloc(E);
int is_zero = 1;
for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } }
if (is_zero) {
for (int e = 0; e < E; e++) {
ema[e] = pr[e];
blended[e] = pr[e];
}
} else {
for (int e = 0; e < E; e++) {
blended[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e];
ema[e] = blended[e]; // update EMA
}
}
}
int cand = 0;
int idx[128];
float max_logit = -1e30f;
for (int e = 0; e < E; e++) { if (blended[e] > max_logit) max_logit = blended[e]; }
float *exps = falloc(E);
float sum_exps = 0.f;
for (int e = 0; e < E; e++) {
exps[e] = expf(blended[e] - max_logit);
sum_exps += exps[e];
}
float cum_sum = 0.f;
int min_cand = c->topk;
int max_cand = c->topk * g_wide;
if (max_cand < min_cand) max_cand = min_cand;
if (max_cand > 128) max_cand = 128; /* idx[] buffer bound */
if (max_cand > E) max_cand = E;
for (int kk = 0; kk < max_cand; kk++) {
int best = -1; float bv = -1.f;
for (int e = 0; e < E; e++) {
int taken = 0; for (int j = 0; j < kk; j++) if (idx[j] == e) { taken=1; break; }
if (!taken && exps[e] > bv) { bv = exps[e]; best = e; }
}
if (best < 0) break;
idx[kk] = best;
cum_sum += bv;
cand++;
if (cum_sum >= m->pilot_conf_limit * sum_exps && cand >= min_cand) {
break;
}
}
free(exps);
if (blended != pr) free(blended);
/* IMPROVEMENT 5: sort candidates by eid for sequential SSD read locality */
for (int a = 0; a < cand-1; a++)
for (int b = a+1; b < cand; b++)
if (idx[b] >= 0 && (idx[a] < 0 || idx[a] > idx[b])) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; }
for (int kk = 0; kk < cand; kk++) {
int eid = idx[kk];
if (eid < 0) continue;
int found = 0;
pthread_mutex_lock(&g_pilot_mx);
LCache *lc = &m->cache[lnext];
for (int z = 0; z < lc->n; z++) {
if (lc->slots[z].eid == eid) { found = 1; break; }
}
pthread_mutex_unlock(&g_pilot_mx);
if (!found) {
int gidx = lnext * E + eid;
pthread_mutex_lock(&g_pilot_mx);
int already_queued = m->is_queued[gidx];
if (!already_queued) {
m->is_queued[gidx] = 1;
}
pthread_mutex_unlock(&g_pilot_mx);
if (!already_queued) {
unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED);
unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE);
if (w2 - r2 < 4096) {
pilot_q[w2 & 4095].l = lnext;
pilot_q[w2 & 4095].e = eid;
__atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE);
} else {
pthread_mutex_lock(&g_pilot_mx);
m->is_queued[gidx] = 0;
pthread_mutex_unlock(&g_pilot_mx);
}
}
}
}
}
free(logits);
}
/* generazione greedy. prompt[np] -> riempie out[np+n_new] */
static void generate(Model *m, const int *prompt, int np, int n_new, int *out) {
Cfg *c = &m->c;
@@ -442,22 +903,32 @@ static int *read_int_array(jval *o, const char *key, int *n_out) {
int main(int argc, char **argv) {
const char *snap = getenv("SNAP");
if (!snap) { fprintf(stderr, "set SNAP=<snapshot directory>\n"); return 1; }
int cap = argc > 1 ? atoi(argv[1]) : 16;
int bits = argc > 2 ? atoi(argv[2]) : 8;
if (bits < 2 || bits > 8) { /* expert storage is int8_t: bits>8 truncates in quantize_rows (#134). f32 mode is not implemented here — int8 is already token-exact vs the oracle. */
fprintf(stderr, "quant_bits must be 2..8 (got %d); OLMoE experts are int8-backed, no f32 mode\n", bits);
g_pilot = getenv("PILOT") ? atoi(getenv("PILOT")) : 0;
g_wide = getenv("WIDE") ? atoi(getenv("WIDE")) : 1;
if (g_wide < 1) g_wide = 1;
if (g_wide > 4) g_wide = 4;
int hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0;
int cap = argc > 1 ? atoi(argv[1]) : 16;
int bits = argc > 2 ? atoi(argv[2]) : 8;
if (bits < 2 || bits > 8) {
fprintf(stderr, "quant_bits must be 2..8 (got %d)\n", bits);
return 1;
}
const char *refpath = argc > 3 ? argv[3] : "ref.json";
FILE *f = fopen(refpath, "rb"); if(!f){perror(refpath);return 1;}
float smooth = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f;
float conf = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f;
printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d smooth=%.2f conf=%.2f ==\n",
cap, bits, g_pilot, g_wide, hot_n, smooth, conf);
FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; }
fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);
char *buf=malloc(n+1); if(fread(buf,1,n,f)!=(size_t)n){} buf[n]=0; fclose(f);
char *buf=malloc(n+1); if (fread(buf,1,n,f)!=(size_t)n) {} buf[n]=0; fclose(f);
char *arena=NULL; jval *ref = json_parse(buf, &arena);
int np, nfull; int *prompt = read_int_array(ref,"prompt_ids",&np); int *full = read_int_array(ref,"full_ids",&nfull);
int n_new = nfull - np;
printf("== Streaming C engine, cache = %d experts/layer, experts @ %d-bit ==\n", cap, bits);
Model m; model_init(&m, snap, cap, bits);
printf("resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb());
@@ -487,6 +958,26 @@ int main(int argc, char **argv) {
printf("\nPEAK RSS: %.2f GB\n", rss_gb());
printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0,
(unsigned long long)m.hits, (unsigned long long)m.miss);
// Persistent Hot Pinning: save dynamic pinning if newly created
if (m.hot_pinned) {
char pinpath[512];
snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap);
FILE *pinf_chk = fopen(pinpath, "rb");
if (!pinf_chk) {
FILE *pinf_save = fopen(pinpath, "wb");
if (pinf_save) {
size_t expected_size = (size_t)m.c.n_layers * m.c.n_experts;
fwrite(m.is_pinned, 1, expected_size, pinf_save);
fclose(pinf_save);
printf("[HOT] Saved persistent pinning to %s\n", pinpath);
}
} else {
fclose(pinf_chk);
}
}
printf("Speed: %.2f tok/s (%.1fs for %d tokens)\n", n_new/dt, dt, n_new);
free(buf); free(arena);
return 0;
c/quant.h
+672
View File
@@ -0,0 +1,672 @@
/* quant.h — quantized matmul kernels (header-only, all functions static).
* Multi-architecture SIMD: AVX2 / AVX-512 / AVX-VNNI / ARM NEON / NEON-SDOT /
* NEON-i8mm / POWER VSX. Pure compute — no Model or QT dependency. */
#ifndef COLI_QUANT_H
#define COLI_QUANT_H
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <stdint.h>
#ifdef _OPENMP
#include <omp.h>
#endif
/* ---- SIMD includes -------------------------------------------------------- */
#ifdef __AVX2__
#include <immintrin.h>
static inline float hsum256(__m256 v){
__m128 lo=_mm256_castps256_ps128(v), hi=_mm256_extractf128_ps(v,1);
lo=_mm_add_ps(lo,hi); __m128 sh=_mm_movehl_ps(lo,lo); lo=_mm_add_ps(lo,sh);
sh=_mm_shuffle_ps(lo,lo,1); lo=_mm_add_ss(lo,sh); return _mm_cvtss_f32(lo);
}
static inline int hsum256_i32(__m256i v){
__m128i lo=_mm256_castsi256_si128(v), hi=_mm256_extracti128_si256(v,1);
lo=_mm_add_epi32(lo,hi); lo=_mm_hadd_epi32(lo,lo); lo=_mm_hadd_epi32(lo,lo);
return _mm_cvtsi128_si32(lo);
}
#endif
#if defined(__AVXVNNI__) && defined(__AVX2__)
static inline int hsum128_i32(__m128i v){
v=_mm_hadd_epi32(v,v); v=_mm_hadd_epi32(v,v); return _mm_cvtsi128_si32(v);
}
#endif
#ifdef __ARM_NEON
#include <arm_neon.h>
#endif
#ifdef __VSX__
#include <altivec.h>
#undef vector
#undef pixel
#undef bool
#endif
/* ---- AVX-512 int4->float accumulator -------------------------------------- */
#if defined(__AVX512F__) && defined(__AVX512BW__)
static int g_i4_acc512=1;
static inline float dot_i4f_avx512(const uint8_t *w,const float *x,int I){
const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8);
__m512 acc0=_mm512_setzero_ps(),acc1=_mm512_setzero_ps(); int i=0;
for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1)));
__m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
__m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi);
__m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8));
__m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8));
acc0=_mm512_fmadd_ps(_mm512_loadu_ps(x+i),w0,acc0);
acc1=_mm512_fmadd_ps(_mm512_loadu_ps(x+i+16),w1,acc1);
}
return _mm512_reduce_add_ps(_mm512_add_ps(acc0,acc1));
}
static int i4_acc512_selftest(void){
enum { N=224 }; uint8_t w[(N+1)/2]; float x[N];
for(int i=0;i<N;i++){
int q=((i*13+5)&15)-8;
if(!(i&1)) w[i>>1]=(uint8_t)(q+8);
else w[i>>1]|=(uint8_t)((q+8)<<4);
x[i]=(float)(((i*29+7)%101)-50)/37.f;
}
for(int n=32;n<=N;n+=32){
float ref=0; for(int i=0;i<n;i++) ref+=x[i]*(float)(((w[i>>1]>>((i&1)*4))&15)-8);
float got=dot_i4f_avx512(w,x,n),tol=2e-5f*(1.f+fabsf(ref));
if(fabsf(got-ref)>tol){ fprintf(stderr,"AVX512 i4 selftest n=%d: %.9g != %.9g\n",n,got,ref); return 0; }
}
return 1;
}
#endif
/* ---- y[S,O] = x[S,I] @ W^T, W[O,I] f32 ---------------------------------- */
static void matmul(float *y, const float *x, const float *W, int S, int I, int O){
#pragma omp parallel for schedule(static)
for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;
for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a+=xs[i]*w[i]; y[(int64_t)s*O+o]=a; } }
}
/* ---- y[S,O] = x[S,I] @ W^T, W int8 per-row + scale[O] ------------------- */
static void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int S, int I, int O){
#pragma omp parallel for schedule(static)
for (int o=0;o<O;o++){ const int8_t *w=q+(int64_t)o*I; float sc=scale[o];
for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
#ifdef __AVX2__
__m256 acc=_mm256_setzero_ps();
for(;i+8<=I;i+=8){ __m256i wi=_mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*)(w+i)));
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), _mm256_cvtepi32_ps(wi), acc); }
a=hsum256(acc);
#elif defined(__ARM_NEON)
float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
for(;i+8<=I;i+=8){ int16x8_t w16=vmovl_s8(vld1_s8(w+i));
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w16))));
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w16)))); }
a=vaddvq_f32(vaddq_f32(ac0,ac1));
#endif
for(;i<I;i++) a+=xs[i]*(float)w[i]; y[(int64_t)s*O+o]=a*sc; } }
}
/* ---- y[S,O] = x[S,I] @ W^T, W int4 packed (2/byte) + scale[O] ------------ */
static void matmul_i4(float *y, const float *x, const uint8_t *q4, const float *scale, int S, int I, int O){
int rb=(I+1)/2;
#pragma omp parallel for schedule(static)
for (int o=0;o<O;o++){ const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
#if defined(__AVX512F__) && defined(__AVX512BW__)
if(g_i4_acc512){ a=dot_i4f_avx512(w,xs,I); i=I&~31; }
else {
#endif
#ifdef __AVX2__
const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
__m256 acc=_mm256_setzero_ps();
for(;i+16<=I;i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));
__m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
__m128i nib=_mm_unpacklo_epi8(lo,hi);
__m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8));
__m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8));
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc);
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); }
a=hsum256(acc);
#elif defined(__ARM_NEON)
const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8);
float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1));
uint8x8x2_t z=vzip_u8(vand_u8(by,m4), vshr_n_u8(by,4));
int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[0]),b8));
int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[1]),b8));
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
a=vaddvq_f32(vaddq_f32(ac0,ac1));
#endif
#if defined(__AVX512F__) && defined(__AVX512BW__)
}
#endif
for(;i+1<I;i+=2){ uint8_t byte=w[i>>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8;
a += xs[i]*(float)lo + xs[i+1]*(float)hi; }
if(i<I){ uint8_t byte=w[i>>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; }
y[(int64_t)s*O+o]=a*sc; } }
}
/* ---- y[S,O] = x[S,I] @ W^T, W int4 packed + per-GROUP scales (fmt=4) ----- */
static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const float *scale,
int S, int I, int O, int gs){
int rb=(I+1)/2; int ng=(I+gs-1)/gs;
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){
const uint8_t *w=q4+(int64_t)o*rb;
const float *scl=scale+(int64_t)o*ng;
for(int s=0;s<S;s++){
const float *xs=x+(int64_t)s*I; float a=0;
for(int g=0; g*gs<I; g++){
int base=g*gs; int glen=gs; if(base+glen>I) glen=I-base;
float sc=scl[g];
int i=base;
#ifdef __AVX2__
const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
__m256 acc=_mm256_setzero_ps();
for(; i+16<=base+glen; i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));
__m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
__m128i nib=_mm_unpacklo_epi8(lo,hi);
__m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8));
__m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8));
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc);
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); }
a+=hsum256(acc)*sc;
#endif
for(; i<base+glen; i+=2){
if(i+1<base+glen){ uint8_t byte=w[i>>1];
a+=(xs[i]*(float)((int)(byte&0xF)-8)+xs[i+1]*(float)((int)(byte>>4)-8))*sc; }
else { uint8_t byte=w[i>>1]; a+=xs[i]*(float)((int)(byte&0xF)-8)*sc; }
}
}
y[(int64_t)s*O+o]=a;
}
}
}
/* ---- fused gate+up: one OMP dispatch for both matrices -------------------- */
static void matmul_i4_pair(float *yg, float *yu, const float *x,
const uint8_t *qg, const float *sg,
const uint8_t *qu, const float *su, int I, int O){
int rb=(I+1)/2;
#pragma omp parallel for schedule(static)
for(int z=0;z<2*O;z++){
int o=z<O?z:z-O; const uint8_t *w=(z<O?qg:qu)+(int64_t)o*rb;
float a=0; int i=0;
#if defined(__AVX512F__) && defined(__AVX512BW__)
if(g_i4_acc512){ a=dot_i4f_avx512(w,x,I); i=I&~31; }
else {
#endif
#ifdef __AVX2__
const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
__m256 acc=_mm256_setzero_ps();
for(;i+16<=I;i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));
__m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
__m128i nib=_mm_unpacklo_epi8(lo,hi);
__m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8));
__m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8));
acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc);
acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); }
a=hsum256(acc);
#elif defined(__ARM_NEON)
const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8);
float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0);
for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1));
uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4));
int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8));
int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8));
ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
a=vaddvq_f32(vaddq_f32(ac0,ac1));
#endif
#if defined(__AVX512F__) && defined(__AVX512BW__)
}
#endif
for(;i+1<I;i+=2){ uint8_t b=w[i>>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); }
if(i<I) a+=x[i]*(float)((w[i>>1]&15)-8);
(z<O?yg:yu)[o]=a*(z<O?sg:su)[o];
}
}
/* ---- y[S,O] = x[S,I] @ W^T, W int2 packed (4/byte) + scale[O] ------------ */
static void matmul_i2(float *y, const float *x, const uint8_t *q2, const float *scale, int S, int I, int O){
int rb=(I+3)/4;
#pragma omp parallel for schedule(static)
for (int o=0;o<O;o++){ const uint8_t *w=q2+(int64_t)o*rb; float sc=scale[o];
for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
#ifdef __AVX2__
const __m128i m2=_mm_set1_epi8(0x03); const __m256i b2=_mm256_set1_epi32(2);
__m256 acc=_mm256_setzero_ps();
for(;i+16<=I;i+=16){ __m128i by=_mm_cvtsi32_si128(*(const int*)(w+(i>>2)));
__m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2);
__m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2);
__m128i lo=_mm_unpacklo_epi8(p0,p1), hi=_mm_unpacklo_epi8(p2,p3);
__m128i nib=_mm_unpacklo_epi16(lo,hi);
__m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b2));
__m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b2));
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc);
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); }
a=hsum256(acc);
#elif defined(__ARM_NEON)
const uint8x8_t m2v=vdup_n_u8(3); const int8x8_t b2v=vdup_n_s8(2);
float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
for(;i+16<=I;i+=16){ uint32_t wd; memcpy(&wd, w+(i>>2), 4);
uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd));
uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v));
uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6));
uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0]));
int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[0]),b2v));
int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[1]),b2v));
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
a=vaddvq_f32(vaddq_f32(ac0,ac1));
#endif
for(;i<I;i++){ uint8_t byte=w[i>>2]; int sh=(i&3)*2; a += xs[i]*(float)((int)((byte>>sh)&3)-2); }
y[(int64_t)s*O+o]=a*sc; } }
}
/* ---- IDOT: integer dot kernels (int8-quantized activations) --------------- */
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
#define IDOT_KERNEL "avx512-vnni"
#elif defined(__AVXVNNI__) && defined(__AVX2__)
#define IDOT_KERNEL "avx-vnni"
#elif defined(__AVX2__)
#define IDOT_KERNEL "avx2"
#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
#define IDOT_KERNEL "neon-i8mm"
#elif defined(__ARM_NEON)
#define IDOT_KERNEL "neon"
#elif defined(__VSX__)
#define IDOT_KERNEL "vsx"
#else
#define IDOT_KERNEL "scalar"
#endif
static int g_idot=1;
#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
static int g_i4s=1;
#elif defined(__VSX__)
static int g_i4s=1;
#else
static int g_i4s=2;
#endif
static inline float qrow_i8(const float *x, int8_t *q, int I){
float amax=0; for(int i=0;i<I;i++){ float a=fabsf(x[i]); if(a>amax)amax=a; }
float s=amax/127.f; if(s<1e-12f) s=1e-12f; float inv=1.f/s;
for(int i=0;i<I;i++) q[i]=(int8_t)lrintf(x[i]*inv);
return s;
}
/* dot int8*int8 */
static inline int32_t dot_i8i8(const int8_t *w, const int8_t *x, int I){
int32_t sum=0; int i=0;
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
__m512i acc=_mm512_setzero_si512();
for(;i+64<=I;i+=64){
__m512i wv=_mm512_loadu_si512((const void*)(w+i));
__m512i xv=_mm512_loadu_si512((const void*)(x+i));
__mmask64 neg=_mm512_movepi8_mask(wv);
__m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv);
acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs);
}
sum=_mm512_reduce_add_epi32(acc);
#elif defined(__AVXVNNI__) && defined(__AVX2__)
__m128i acc=_mm_setzero_si128();
for(;i+16<=I;i+=16){
__m128i wv=_mm_loadu_si128((const __m128i*)(w+i));
__m128i xv=_mm_loadu_si128((const __m128i*)(x+i));
__m128i xs=_mm_sign_epi8(xv,wv);
acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(wv),xs);
}
sum=hsum128_i32(acc);
#elif defined(__AVX2__)
__m256i acc=_mm256_setzero_si256(); const __m256i ones=_mm256_set1_epi16(1);
for(;i+32<=I;i+=32){
__m256i wv=_mm256_loadu_si256((const __m256i*)(w+i));
__m256i xv=_mm256_loadu_si256((const __m256i*)(x+i));
__m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv));
acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones));
}
sum=hsum256_i32(acc);
#elif defined(__ARM_NEON)
#if defined(__ARM_FEATURE_DOTPROD)
int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0);
for(;i+64<=I;i+=64){
a0=vdotq_s32(a0,vld1q_s8(w+i), vld1q_s8(x+i));
a1=vdotq_s32(a1,vld1q_s8(w+i+16),vld1q_s8(x+i+16));
a2=vdotq_s32(a2,vld1q_s8(w+i+32),vld1q_s8(x+i+32));
a3=vdotq_s32(a3,vld1q_s8(w+i+48),vld1q_s8(x+i+48));
}
int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
for(;i+16<=I;i+=16) acc=vdotq_s32(acc,vld1q_s8(w+i),vld1q_s8(x+i));
sum=vaddvq_s32(acc);
#else
int32x4_t acc=vdupq_n_s32(0);
for(;i+16<=I;i+=16){
int8x16_t wv=vld1q_s8(w+i), xv=vld1q_s8(x+i);
int16x8_t p=vmull_s8(vget_low_s8(wv),vget_low_s8(xv));
p=vmlal_s8(p,vget_high_s8(wv),vget_high_s8(xv));
acc=vpadalq_s16(acc,p);
}
sum=vaddvq_s32(acc);
#endif
#elif defined(__VSX__)
__vector signed int acc=vec_splats(0);
const __vector signed char vz=vec_splats((signed char)0);
for(;i+16<=I;i+=16){
__vector signed char wv=vec_xl(0,(const signed char*)(w+i));
__vector signed char xv=vec_xl(0,(const signed char*)(x+i));
__vector __bool char neg=vec_cmplt(wv,vz);
__vector signed char xs=vec_sel(xv,vec_sub(vz,xv),neg);
__vector unsigned char wa=(__vector unsigned char)vec_sel(wv,vec_sub(vz,wv),neg);
acc=vec_msum(xs,wa,acc);
}
sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3);
#endif
for(;i<I;i++) sum+=(int32_t)w[i]*x[i];
return sum;
}
/* dot int4(packed)*int8 */
static inline int32_t dot_i4i8(const uint8_t *w4, const int8_t *x, int I){
int32_t sum=0; int i=0;
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
const __m256i m4v=_mm256_set1_epi8(0x0F);
const __m512i b8v=_mm512_set1_epi8(8);
const __m512i xidx=_mm512_setr_epi64(0,1,4,5,2,3,6,7);
__m512i acc=_mm512_setzero_si512();
for(;i+64<=I;i+=64){
__m256i by=_mm256_loadu_si256((const __m256i*)(w4+(i>>1)));
__m256i lo=_mm256_and_si256(by,m4v), hi=_mm256_and_si256(_mm256_srli_epi16(by,4),m4v);
__m256i z0=_mm256_unpacklo_epi8(lo,hi), z1=_mm256_unpackhi_epi8(lo,hi);
__m512i wv=_mm512_sub_epi8(_mm512_inserti64x4(_mm512_castsi256_si512(z0),z1,1),b8v);
__m512i xv=_mm512_permutexvar_epi64(xidx,_mm512_loadu_si512((const void*)(x+i)));
__mmask64 neg=_mm512_movepi8_mask(wv);
__m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv);
acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs);
}
sum=_mm512_reduce_add_epi32(acc);
#elif defined(__AVXVNNI__) && defined(__AVX2__)
const __m128i m4=_mm_set1_epi8(0x0F); const __m128i b8=_mm_set1_epi8(8);
__m128i acc=_mm_setzero_si128();
for(;i+32<=I;i+=32){
__m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1)));
__m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
__m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi);
__m128i w0=_mm_sub_epi8(n0,b8), w1=_mm_sub_epi8(n1,b8);
__m128i x0=_mm_loadu_si128((const __m128i*)(x+i));
__m128i x1=_mm_loadu_si128((const __m128i*)(x+i+16));
acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0));
acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1));
}
sum=hsum128_i32(acc);
#elif defined(__AVX2__)
const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi8(8);
const __m256i ones=_mm256_set1_epi16(1);
__m256i acc=_mm256_setzero_si256();
for(;i+32<=I;i+=32){
__m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1)));
__m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
__m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi);
__m256i wv=_mm256_sub_epi8(_mm256_set_m128i(n1,n0),b8);
__m256i xv=_mm256_loadu_si256((const __m256i*)(x+i));
__m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv));
acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones));
}
sum=hsum256_i32(acc);
#elif defined(__ARM_NEON)
const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8);
#if defined(__ARM_FEATURE_DOTPROD)
int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0);
for(;i+64<=I;i+=64){
uint8x16_t byA=vld1q_u8(w4+(i>>1)), byB=vld1q_u8(w4+(i>>1)+16);
uint8x16x2_t zA=vzipq_u8(vandq_u8(byA,m4q), vshrq_n_u8(byA,4));
uint8x16x2_t zB=vzipq_u8(vandq_u8(byB,m4q), vshrq_n_u8(byB,4));
a0=vdotq_s32(a0,vsubq_s8(vreinterpretq_s8_u8(zA.val[0]),b8q),vld1q_s8(x+i));
a1=vdotq_s32(a1,vsubq_s8(vreinterpretq_s8_u8(zA.val[1]),b8q),vld1q_s8(x+i+16));
a2=vdotq_s32(a2,vsubq_s8(vreinterpretq_s8_u8(zB.val[0]),b8q),vld1q_s8(x+i+32));
a3=vdotq_s32(a3,vsubq_s8(vreinterpretq_s8_u8(zB.val[1]),b8q),vld1q_s8(x+i+48));
}
int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
for(;i+32<=I;i+=32){
uint8x16_t by=vld1q_u8(w4+(i>>1));
uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4));
acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q),vld1q_s8(x+i));
acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q),vld1q_s8(x+i+16));
}
sum=vaddvq_s32(acc);
#else
int32x4_t acc=vdupq_n_s32(0);
for(;i+32<=I;i+=32){
uint8x16_t by=vld1q_u8(w4+(i>>1));
uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4));
int8x16_t w0=vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q);
int8x16_t w1=vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q);
int8x16_t x0=vld1q_s8(x+i), x1=vld1q_s8(x+i+16);
int16x8_t p=vmull_s8(vget_low_s8(w0),vget_low_s8(x0));
p=vmlal_s8(p,vget_high_s8(w0),vget_high_s8(x0));
acc=vpadalq_s16(acc,p);
p=vmull_s8(vget_low_s8(w1),vget_low_s8(x1));
p=vmlal_s8(p,vget_high_s8(w1),vget_high_s8(x1));
acc=vpadalq_s16(acc,p);
}
sum=vaddvq_s32(acc);
#endif
#elif defined(__VSX__)
const __vector unsigned char m4v=vec_splats((unsigned char)0x0F);
const __vector unsigned char sh4=vec_splats((unsigned char)4);
const __vector signed char b8v=vec_splats((signed char)8);
const __vector signed char vz=vec_splats((signed char)0);
__vector signed int acc=vec_splats(0);
for(;i+32<=I;i+=32){
__vector unsigned char by=vec_xl(0,w4+(i>>1));
__vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4);
__vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v);
__vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v);
__vector signed char x0=vec_xl(0,(const signed char*)(x+i));
__vector signed char x1=vec_xl(0,(const signed char*)(x+i+16));
__vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz);
acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0),
(__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc);
acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1),
(__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc);
}
sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3);
#endif
for(;i+1<I;i+=2){ uint8_t b=w4[i>>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; }
if(i<I){ uint8_t b=w4[i>>1]; sum+=((int)(b&0xF)-8)*x[i]; }
return sum;
}
/* ---- ARM i8mm SMMLA tiled kernels ---------------------------------------- */
#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
static inline int32x4_t mm_tile16(int32x4_t acc, int8x16_t wo, int8x16_t wo1,
int8x16_t xs, int8x16_t xs1){
acc=vmmlaq_s32(acc, vcombine_s8(vget_low_s8(wo), vget_low_s8(wo1)),
vcombine_s8(vget_low_s8(xs), vget_low_s8(xs1)));
return vmmlaq_s32(acc, vcombine_s8(vget_high_s8(wo), vget_high_s8(wo1)),
vcombine_s8(vget_high_s8(xs), vget_high_s8(xs1)));
}
static void matmul_q_idot_mm(float *y, const int8_t *xq, const float *sx, const int8_t *q,
const float *scale, int S, int I, int O){
#pragma omp parallel for schedule(static)
for(int o=0;o<(O&~1);o+=2){
const int8_t *wo=q+(int64_t)o*I, *wo1=q+(int64_t)(o+1)*I;
float sc0=scale[o], sc1=scale[o+1];
for(int s=0;s<(S&~1);s+=2){
const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I;
int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0;
for(;i+64<=I;i+=64){
a0=mm_tile16(a0,vld1q_s8(wo+i), vld1q_s8(wo1+i), vld1q_s8(xs+i), vld1q_s8(xs1+i));
a1=mm_tile16(a1,vld1q_s8(wo+i+16),vld1q_s8(wo1+i+16),vld1q_s8(xs+i+16),vld1q_s8(xs1+i+16));
a2=mm_tile16(a2,vld1q_s8(wo+i+32),vld1q_s8(wo1+i+32),vld1q_s8(xs+i+32),vld1q_s8(xs1+i+32));
a3=mm_tile16(a3,vld1q_s8(wo+i+48),vld1q_s8(wo1+i+48),vld1q_s8(xs+i+48),vld1q_s8(xs1+i+48));
}
for(;i+16<=I;i+=16)
a0=mm_tile16(a0,vld1q_s8(wo+i),vld1q_s8(wo1+i),vld1q_s8(xs+i),vld1q_s8(xs1+i));
int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1);
int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3);
for(;i<I;i++){ int a=wo[i],b=wo1[i],u=xs[i],v=xs1[i];
d00+=a*u; d01+=a*v; d10+=b*u; d11+=b*v; }
y[(int64_t)s*O+o] =(float)d00*sc0*sx[s];
y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s];
y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1];
y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1];
}
if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I;
y[(int64_t)s*O+o] =(float)dot_i8i8(wo, xs,I)*sc0*sx[s];
y[(int64_t)s*O+(o+1)]=(float)dot_i8i8(wo1,xs,I)*sc1*sx[s]; }
}
if(O&1){ int o=O-1; const int8_t *w=q+(int64_t)o*I; float sc=scale[o];
#pragma omp parallel for schedule(static)
for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i8i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
}
static void matmul_i4_idot_mm(float *y, const int8_t *xq, const float *sx, const uint8_t *q4,
const float *scale, int S, int I, int O){
int rb=(I+1)/2;
#pragma omp parallel for schedule(static)
for(int o=0;o<(O&~1);o+=2){
const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8);
const uint8_t *wo=q4+(int64_t)o*rb, *wo1=q4+(int64_t)(o+1)*rb;
float sc0=scale[o], sc1=scale[o+1];
for(int s=0;s<(S&~1);s+=2){
const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I;
int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0;
for(;i+64<=I;i+=64){
uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1));
uint8x16_t cyo=vld1q_u8(wo+(i>>1)+16), cyo1=vld1q_u8(wo1+(i>>1)+16);
uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4));
uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4));
uint8x16x2_t ko =vzipq_u8(vandq_u8(cyo, m4q), vshrq_n_u8(cyo, 4));
uint8x16x2_t ko1=vzipq_u8(vandq_u8(cyo1,m4q), vshrq_n_u8(cyo1,4));
a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q),
vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q),
vld1q_s8(xs+i), vld1q_s8(xs1+i));
a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q),
vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q),
vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16));
a2=mm_tile16(a2, vsubq_s8(vreinterpretq_s8_u8(ko.val[0]),b8q),
vsubq_s8(vreinterpretq_s8_u8(ko1.val[0]),b8q),
vld1q_s8(xs+i+32), vld1q_s8(xs1+i+32));
a3=mm_tile16(a3, vsubq_s8(vreinterpretq_s8_u8(ko.val[1]),b8q),
vsubq_s8(vreinterpretq_s8_u8(ko1.val[1]),b8q),
vld1q_s8(xs+i+48), vld1q_s8(xs1+i+48));
}
for(;i+32<=I;i+=32){
uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1));
uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4));
uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4));
a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q),
vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q),
vld1q_s8(xs+i), vld1q_s8(xs1+i));
a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q),
vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q),
vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16));
}
int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1);
int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3);
for(;i+1<I;i+=2){ uint8_t bo=wo[i>>1], bo1=wo1[i>>1];
int a0=(int)(bo&0xF)-8, a1=(int)(bo>>4)-8, b0=(int)(bo1&0xF)-8, b1=(int)(bo1>>4)-8;
int u0=xs[i],u1=xs[i+1],v0=xs1[i],v1=xs1[i+1];
d00+=a0*u0+a1*u1; d01+=a0*v0+a1*v1; d10+=b0*u0+b1*u1; d11+=b0*v0+b1*v1; }
if(i<I){ uint8_t bo=wo[i>>1], bo1=wo1[i>>1];
int a0=(int)(bo&0xF)-8, b0=(int)(bo1&0xF)-8;
d00+=a0*xs[i]; d01+=a0*xs1[i]; d10+=b0*xs[i]; d11+=b0*xs1[i]; }
y[(int64_t)s*O+o] =(float)d00*sc0*sx[s];
y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s];
y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1];
y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1];
}
if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I;
y[(int64_t)s*O+o] =(float)dot_i4i8(wo, xs,I)*sc0*sx[s];
y[(int64_t)s*O+(o+1)]=(float)dot_i4i8(wo1,xs,I)*sc1*sx[s]; }
}
if(O&1){ int o=O-1; const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
#pragma omp parallel for schedule(static)
for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i4i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
}
#endif
/* ---- IDOT dispatch (int8-quantized activations) --------------------------- */
static void matmul_q_idot(float *y, const int8_t *xq, const float *sx, const int8_t *q,
const float *scale, int S, int I, int O){
#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
if(S>=2){ matmul_q_idot_mm(y,xq,sx,q,scale,S,I,O); return; }
#endif
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){ const int8_t *w=q+(int64_t)o*I; float sc=scale[o];
for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i8i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
}
static void matmul_i4_idot(float *y, const int8_t *xq, const float *sx, const uint8_t *q4,
const float *scale, int S, int I, int O){
int rb=(I+1)/2;
#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
if(S>=2){ matmul_i4_idot_mm(y,xq,sx,q4,scale,S,I,O); return; }
#endif
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){ const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i4i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
}
/* ---- per-thread quantization scratch -------------------------------------- */
typedef struct { int8_t *xq; size_t xq_cap; float *sx; size_t sx_cap; } QScratch;
static _Thread_local QScratch g_qscratch;
static void quant_scratch(size_t xn, size_t sn, int8_t **xq, float **sx){
if(xn>g_qscratch.xq_cap){
int8_t *p=realloc(g_qscratch.xq,xn);
if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); }
g_qscratch.xq=p; g_qscratch.xq_cap=xn;
}
if(sn>g_qscratch.sx_cap){
float *p=realloc(g_qscratch.sx,sn*sizeof(float));
if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); }
g_qscratch.sx=p; g_qscratch.sx_cap=sn;
}
*xq=g_qscratch.xq; *sx=g_qscratch.sx;
}
/* ---- f32 -> quantized packing --------------------------------------------- */
static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits){
int qmax=(1<<(bits-1))-1;
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){ const float *wr=w+(int64_t)o*I; float amax=0;
for(int i=0;i<I;i++){ float a=fabsf(wr[i]); if(a>amax)amax=a; }
float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s;
int8_t *qr=q+(int64_t)o*I;
for(int i=0;i<I;i++){ int v=(int)lrintf(wr[i]/s); if(v>qmax)v=qmax; if(v<-qmax-1)v=-qmax-1; qr[i]=(int8_t)v; }
}
}
static void pack_int4(const float *w, uint8_t *q4, float *scale, int O, int I, int bits){
int qmax=(1<<(bits-1))-1, rb=(I+1)/2;
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){ const float *wr=w+(int64_t)o*I; float amax=0;
for(int i=0;i<I;i++){ float a=fabsf(wr[i]); if(a>amax)amax=a; }
float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s;
uint8_t *qr=q4+(int64_t)o*rb;
for(int i=0;i<I;i+=2){
int v0=(int)lrintf(wr[i]/s); if(v0>qmax)v0=qmax; if(v0<-8)v0=-8;
int v1=0; if(i+1<I){ v1=(int)lrintf(wr[i+1]/s); if(v1>qmax)v1=qmax; if(v1<-8)v1=-8; }
qr[i>>1] = (uint8_t)((v0+8) | ((v1+8)<<4));
}
}
}
static void pack_int2(const float *w, uint8_t *q2, float *scale, int O, int I, int bits){
int qmax=(1<<(bits-1))-1, rb=(I+3)/4;
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){ const float *wr=w+(int64_t)o*I; float amax=0;
for(int i=0;i<I;i++){ float a=fabsf(wr[i]); if(a>amax)amax=a; }
float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s;
uint8_t *qr=q2+(int64_t)o*rb;
for(int i=0;i<I;i+=4){ uint8_t byte=0;
for(int k=0;k<4 && i+k<I;k++){ int v=(int)lrintf(wr[i+k]/s); if(v>qmax)v=qmax; if(v<-2)v=-2; byte|=(uint8_t)((v+2)<<(k*2)); }
qr[i>>2]=byte;
}
}
}
#endif /* COLI_QUANT_H */
c/ref_olmoe_real.json
+28
View File
@@ -0,0 +1,28 @@
{
"prompt_ids": [
510,
5347,
273,
6181,
310
],
"full_ids": [
510,
5347,
273,
6181,
310,
7785,
15,
187,
187,
510,
3565,
3448,
273,
6181,
310,
5112,
15
]
}
c/sample.h
+143
View File
@@ -0,0 +1,143 @@
/* sample.h — sampling (temperature + nucleus) and stop-set management.
* Header-only: all functions are static — include from the main engine file. */
#ifndef SAMPLE_H
#define SAMPLE_H
#include <math.h>
#include <stdio.h>
#include <string.h>
#include "tok.h"
/* ---- RNG (xorshift64*) -------------------------------------------------- */
static uint64_t g_rng = 0x9E3779B97F4A7C15ULL;
static inline double rndu(void){
g_rng ^= g_rng << 13; g_rng ^= g_rng >> 7; g_rng ^= g_rng << 17;
return (double)(g_rng >> 11) * (1.0 / 9007199254740992.0);
}
/* ---- argmax over a float vector ----------------------------------------- */
static inline int argmax_v(const float *lo, int V){
int b=-1; float bv=-INFINITY;
for(int i=0;i<V;i++){ float x=lo[i]; if(x==x && x>bv){ bv=x; b=i; } }
return b<0?0:b;
}
/* ---- distribution buffers (reused, single-threaded decode) --------------- */
static float *g_pbuf = NULL;
static int *g_pidx = NULL;
/* sift-down on max-heap in h[0..n), key = g_pbuf[h[i]] (#335: partial top-p).
* "hole" variant: carries the root value and deposits only at the end, so
* heapify is O(V) and each pop is O(log n) without qsort on the full vocab. */
static void topp_siftdown(int *h, int n, int i){
int iv = h[i]; float kv = g_pbuf[iv];
for (;;) {
int l = 2*i + 1;
if (l >= n) break;
int b = l; if (l+1 < n && g_pbuf[h[l+1]] > g_pbuf[h[l]]) b = l+1;
if (g_pbuf[h[b]] <= kv) break;
h[i] = h[b]; i = b;
}
h[i] = iv;
}
/* build the target distribution in g_pbuf: softmax(lo/temp) truncated to
* top-p g_nuc. Invariant: g_pbuf stays indexed by token-id (never reordered);
* the truncated tail is zeroed (dist_sample reads by id directly).
* Requires: g_temp, g_nuc, falloc() — declared in the main engine file. */
static void dist_build(const float *lo, int V){
if (!g_pbuf) { g_pbuf = falloc(V); g_pidx = malloc(V * sizeof(int)); }
int mxi = -1; float mx = 0;
for (int i = 0; i < V; i++)
if (isfinite(lo[i]) && (mxi < 0 || lo[i] > mx)) { mx = lo[i]; mxi = i; }
double s = 0; float invt = 1.f / (g_temp > 1e-4f ? g_temp : 1e-4f);
if (mxi >= 0) {
for (int i = 0; i < V; i++) {
g_pbuf[i] = isfinite(lo[i]) ? expf((lo[i] - mx) * invt) : 0.f;
s += g_pbuf[i];
}
}
if (mxi < 0 || !isfinite(s) || s <= 0.0) {
static int warned = 0;
if (!warned) { warned = 1; fprintf(stderr,
"[SAMPLE] warning: non-finite logits (NaN/Inf) — falling back to argmax; "
"output may be degraded. This usually means a numerical blow-up upstream.\n"); }
int a = (mxi >= 0) ? mxi : 0;
for (int i = 0; i < V; i++) g_pbuf[i] = 0.f;
g_pbuf[a] = 1.f;
return;
}
for (int i = 0; i < V; i++) g_pbuf[i] /= (float)s;
if (g_nuc > 0 && g_nuc < 1.f) {
for (int i = 0; i < V; i++) g_pidx[i] = i;
for (int i = V/2-1; i >= 0; i--) topp_siftdown(g_pidx, V, i);
double s2 = 0, cum = 0; int out = V;
do {
int root = g_pidx[0];
g_pidx[0] = g_pidx[--out]; g_pidx[out] = root;
s2 += g_pbuf[root]; cum += g_pbuf[root];
if (out > 0) topp_siftdown(g_pidx, out, 0);
} while (cum < g_nuc && out > 0);
for (int i = 0; i < out; i++) g_pbuf[g_pidx[i]] = 0;
float s2f = (float)s2;
for (int i = out; i < V; i++) g_pbuf[g_pidx[i]] /= s2f;
}
}
/* sample from g_pbuf; ban>=0 excludes that token (renormalizing on the fly) */
static int dist_sample(int V, int ban){
double z = 1.0 - (ban >= 0 ? g_pbuf[ban] : 0.0);
if (z <= 1e-12) z = 1e-12;
double u = rndu() * z, cum = 0;
for (int i = 0; i < V; i++) { if (i == ban) continue; cum += g_pbuf[i]; if (cum >= u) return i; }
for (int i = V-1; i >= 0; i--) if (i != ban && g_pbuf[i] > 0) return i;
return 0;
}
/* next token from logits: greedy if g_temp<=0, sampling otherwise.
* ban = token excluded because it was rejected by speculative verification. */
static int pick_tok(const float *lo, int V, int ban){
if (g_temp <= 0) return argmax_v(lo, V);
dist_build(lo, V);
return dist_sample(V, ban);
}
/* ---- stop set ----------------------------------------------------------- */
static int g_stop[64], g_nstop = 0;
static inline int is_stop(int t){
for (int i = 0; i < g_nstop; i++) if (t == g_stop[i]) return 1;
return 0;
}
/* T=NULL -> config stops only (validation/oracle, where the tokenizer is not needed). */
static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){
g_nstop = 0;
for (int i = 0; i < c->n_stop && g_nstop < 64; i++) g_stop[g_nstop++] = c->stop_ids[i];
if (tok_eos >= 0 && !is_stop(tok_eos) && g_nstop < 64) g_stop[g_nstop++] = tok_eos;
int nsp = 0;
if (T) for (int id = 0; id < T->n_ids && g_nstop < 64; id++)
if (T->id_special[id] && !is_stop(id)) { g_stop[g_nstop++] = id; nsp++; }
fprintf(stderr, "[stop] %d stop tokens:", g_nstop);
for (int i = 0; i < g_nstop; i++) fprintf(stderr, " %d", g_stop[i]);
if (nsp) fprintf(stderr, " (%d from the tokenizer's special set)", nsp);
fprintf(stderr, "\n");
}
static void stops_arm(const Cfg *c, int tok_eos){ stops_arm_tok(c, tok_eos, NULL); }
/* ---- log-prob of a target token given the logit vector ------------------- */
static double logprob_target(const float *lo, int V, int target, int *am){
float mx = lo[0]; int best = 0;
for (int i = 1; i < V; i++) if (lo[i] > mx) { mx = lo[i]; best = i; }
double se = 0;
for (int i = 0; i < V; i++) se += exp((double)lo[i] - mx);
if (am) *am = (best == target);
return (double)(lo[target] - mx) - log(se);
}
/* "glm" in model_type, case-insensitive */
static int mt_is_glm(const char *s){
if (s) for (; *s; s++)
if ((s[0]|32) == 'g' && (s[1]|32) == 'l' && (s[2]|32) == 'm') return 1;
return 0;
}
#endif /* SAMPLE_H */
c/setup.sh
+2 -2
View File
@@ -32,11 +32,11 @@ esac
# 2) build: nativa (veloce, per QUESTA macchina). Per un binario da distribuire: make portable
echo " building (ARCH=${ARCH:-native})…"
make -s glm ARCH="${ARCH:-native}"
make -s colibri ARCH="${ARCH:-native}"
# 3) self-test sull'oracolo tiny, se presente
if [ -d glm_tiny ] && [ -f ref_glm.json ]; then
r=$(SNAP=./glm_tiny TF=1 ./glm 64 16 16 2>/dev/null | grep -oE "[0-9]+/[0-9]+ positions" || true)
r=$(SNAP=./glm_tiny TF=1 ./colibri 64 16 16 2>/dev/null | grep -oE "[0-9]+/[0-9]+ positions" || true)
echo " engine self-test: ${r:-?} (expected 32/32)"
fi
c/st.h
+22 -1
View File
@@ -200,7 +200,19 @@ static void st_init(shards *S, const char *snap_dir) {
if (a0 < 0 || b0 < a0 || data_start + b0 > fsz) {
fprintf(stderr, "%s: tensor '%s' data_offsets [%lld,%lld] out of file bounds (%lld)\n",
files[fi], name, (long long)a0, (long long)b0, (long long)fsz); exit(1); }
int64_t numel = 1; for (int k = 0; k < shp->len; k++) numel *= (int64_t)shp->kids[k]->num;
/* SEC: lo shape viene da un file non fidato (mirror). Senza il guard
* di overflow, uno shape tipo [65535,65535,65535,...] fa avvolgere
* numel a un valore piccolo/negativo che poi passerebbe il cross-check
* numel*esz==nbytes in st_read_f32, riaprendo l'OOB. */
int64_t numel = 1; int bad_shape = 0;
for (int k = 0; k < shp->len; k++) {
int64_t d = (int64_t)shp->kids[k]->num;
if (d < 0 || (d != 0 && numel > INT64_MAX / d)) { bad_shape = 1; break; }
numel *= d;
}
if (bad_shape) {
fprintf(stderr, "%s: tensor '%s' shape overflows int64 — refusing (hostile or corrupt file)\n",
files[fi], name); exit(1); }
if (S->n == S->cap) { S->cap *= 2; S->t = realloc(S->t, S->cap*sizeof(st_tensor)); }
st_tensor *t = &S->t[S->n++];
t->name = strdup(name); t->fd = fd; t->off = data_start + a0;
@@ -249,6 +261,15 @@ static void st_prefetch(shards *S, const char *name) {
static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) {
st_tensor *t = st_find(S, name);
if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); }
/* SEC: numel viene dallo shape, nbytes dagli offset — due campi indipendenti
* del file. Se non concordano, la memcpy F32 (nbytes) o i loop BF16/F16
* (numel elementi da un raw di soli nbytes) sforano il buffer del chiamante,
* che e' dimensionato sul config, non sul file. Il chiamante che alloca su
* st_numel resta coerente; questo blocca l'ingresso ostile a monte. */
int esz = (t->dtype == 2) ? 4 : 2;
if (t->numel < 0 || t->numel > t->nbytes / esz || t->numel * (int64_t)esz != t->nbytes) {
fprintf(stderr, "%s: tensor '%s' shape/bytes mismatch (numel %lld, %lld bytes, dtype %d) — refusing (hostile or corrupt file)\n",
name, name, (long long)t->numel, (long long)t->nbytes, t->dtype); exit(1); }
void *raw = malloc(t->nbytes);
if (!raw) { fprintf(stderr, "malloc %lld bytes for tensor %s failed\n", (long long)t->nbytes, name); exit(1); }
st_pread_full(t->fd, raw, t->nbytes, t->off, "pread data");
c/telemetry.h
+189
View File
@@ -0,0 +1,189 @@
/* telemetry.h — dashboard protocol lines, stats/usage persistence, hardware probe.
* Include after Model/Cfg/QT/ESlot/shards and st.h are defined; requires
* qt_bytes(), now_s(), rss_gb(), edisk_s(), and the g_cuda_* globals (ifdef). */
#ifndef TELEMETRY_H
#define TELEMETRY_H
static int64_t tbytes(int O,int I,int bits){
if(bits>=16) return (int64_t)O*I*4;
if(bits>=5) return (int64_t)O*I + (int64_t)O*4;
return (int64_t)O*((I+1)/2) + (int64_t)O*4;
}
static int64_t expert_bytes_probe(Model *m, int ebits){
Cfg *c=&m->c; int64_t eb=0; char nm[256];
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.gate_proj.weight",c->first_dense);
if(st_nbytes(&m->S,nm)>0){
const char *suf[3]={"gate_proj","up_proj","down_proj"};
for(int k=0;k<3;k++){
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight",c->first_dense,suf[k]);
eb+=st_nbytes(&m->S,nm);
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight.qs",c->first_dense,suf[k]);
int64_t q=st_nbytes(&m->S,nm); if(q>0) eb+=q;
}
}
if(eb<=0) eb = tbytes(c->moe_inter,c->hidden,ebits)*2 + tbytes(c->hidden,c->moe_inter,ebits);
return eb;
}
/* BRAIN MAP: per-turn expert hit bitmap for the dashboard. */
static uint8_t **g_ehit;
static void ehit_mark(Model *m, int layer, int eid){
if(!g_ehit){ Cfg *c=&m->c;
g_ehit=calloc(c->n_layers+1,sizeof(uint8_t*));
for(int i=0;i<=c->n_layers;i++) g_ehit[i]=calloc(c->n_experts,1);
}
g_ehit[layer][eid]=1;
}
/* CPU model + cores + RAM (GB); empty/zero where unavailable. */
static void hw_probe(char *cpu, size_t cn, int *cores, double *ram_total, double *ram_avail){
cpu[0]=0;
#ifdef _WIN32
#if defined(__x86_64__) || defined(__i386__)
{ unsigned int r[12]={0}; unsigned int *w=r;
for(unsigned int f=0x80000002u; f<=0x80000004u; f++,w+=4)
__get_cpuid(f,&w[0],&w[1],&w[2],&w[3]);
char *b=(char*)r; b[47]=0; while(*b==' ')b++;
snprintf(cpu,cn,"%s",b); }
#endif
#else
FILE *ci=fopen("/proc/cpuinfo","r");
if(ci){ char ln[256];
while(fgets(ln,sizeof(ln),ci)) if(!strncmp(ln,"model name",10)){
char *p=strchr(ln,':'); if(p){ p++; while(*p==' ')p++;
int n=(int)strlen(p); if(n>0&&p[n-1]=='\n')p[--n]=0;
snprintf(cpu,cn,"%s",p); } break; }
fclose(ci); }
#endif
*cores=0;
#ifdef _WIN32
{ SYSTEM_INFO si; GetSystemInfo(&si); *cores=(int)si.dwNumberOfProcessors; }
#elif defined(_SC_NPROCESSORS_ONLN)
*cores=(int)sysconf(_SC_NPROCESSORS_ONLN);
#endif
*ram_total=*ram_avail=0;
#ifdef _WIN32
compat_meminfo(ram_total,ram_avail);
#else
FILE *mi=fopen("/proc/meminfo","r");
if(mi){ char ln[256]; double mt=0,ma=0;
while(fgets(ln,sizeof(ln),mi)){
if(sscanf(ln,"MemTotal: %lf",&mt)==1) *ram_total=mt/1e6;
if(sscanf(ln,"MemAvailable: %lf",&ma)==1) *ram_avail=ma/1e6;
} fclose(mi); }
#endif
}
static void hwinfo_emit(Model *m){
Cfg *c=&m->c; (void)c;
char cpu[256]; int cores; double ram_total,ram_avail;
hw_probe(cpu,sizeof(cpu),&cores,&ram_total,&ram_avail);
int ngpu=0; double vram_total=0;
char gpu_name[128]="";
#ifdef COLI_CUDA
ngpu=g_cuda_ndev; vram_total=m->gpu_expert_bytes/1e9;
for(int i=0;i<g_cuda_ndev;i++){
size_t fr=0,to=0; coli_cuda_mem_info(g_cuda_devices[i],&fr,&to);
if(!i) vram_total=(double)to*g_cuda_ndev/1e9;
}
if(g_cuda_ndev>0)
snprintf(gpu_name,sizeof(gpu_name),"CUDA device x%d",g_cuda_ndev);
#endif
printf("HWINFO %d %.1f %.1f %d %.1f %s|%s\n",
cores,ram_total,ram_avail,ngpu,vram_total,cpu,gpu_name);
fflush(stdout);
}
static void tiers_emit(Model *m){
Cfg *c=&m->c; int nsp=0;
for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse) nsp++;
int total=(nsp+(m->has_mtp?1:0))*c->n_experts;
int pinned=0,lru=0;
for(int i=0;i<=c->n_layers;i++){ pinned+=m->npin?m->npin[i]:0; lru+=m->ecn?m->ecn[i]:0; }
int vram=0; double vram_gb=0;
#ifdef COLI_CUDA
vram=m->gpu_expert_count; vram_gb=m->gpu_expert_bytes/1e9;
#endif
int ram=pinned-vram+lru; if(ram<0) ram=0;
int disk=total-vram-ram; if(disk<0) disk=0;
double eb=(double)expert_bytes_probe(m,m->ebits);
printf("TIERS %d %d %d %.2f %.2f\n",vram,ram,disk,vram_gb,ram*eb/1e9);
fflush(stdout);
}
static void emap_emit(Model *m){
Cfg *c=&m->c;
int rows=0;
for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse) rows++;
int has_mtp = m->has_mtp && m->eusage[c->n_layers];
if(has_mtp) rows++;
int cols=c->n_experts;
char *hex=malloc((size_t)rows*cols*2+1); int w=0;
for(int i=0;i<=c->n_layers;i++){
int is_row = (i<c->n_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp);
if(!is_row) continue;
for(int e=0;e<cols;e++){
int tier=0;
ESlot *P=m->pin[i];
for(int z=0;z<m->npin[i];z++) if(P[z].eid==e){
#ifdef COLI_CUDA
tier = P[z].g.cuda?2:1;
#else
tier = 1;
#endif
break; }
if(!tier && m->ecache && m->ecache[i])
for(int z=0;z<m->ecn[i];z++) if(m->ecache[i][z].eid==e){ tier=1; break; }
uint32_t u = m->eusage[i]?m->eusage[i][e]:0;
int heat=0; while(u){ heat++; u>>=1; } if(heat>63) heat=63;
int b=(tier<<6)|heat;
hex[w++]="0123456789abcdef"[b>>4]; hex[w++]="0123456789abcdef"[b&15];
}
}
hex[w]=0;
printf("EMAP %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex);
}
static void hits_emit(Model *m){
Cfg *c=&m->c; if(!g_ehit) return;
int rows=0;
for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse) rows++;
int has_mtp = m->has_mtp && m->eusage[c->n_layers];
if(has_mtp) rows++;
int cols=c->n_experts, nb=(rows*cols+7)/8;
uint8_t *bm=calloc(nb,1); int bit=0;
for(int i=0;i<=c->n_layers;i++){
int is_row = (i<c->n_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp);
if(!is_row) continue;
for(int e=0;e<cols;e++,bit++)
if(g_ehit[i][e]){ bm[bit>>3]|=1<<(bit&7); g_ehit[i][e]=0; }
}
char *hex=malloc((size_t)nb*2+1); int w=0;
for(int b=0;b<nb;b++){ hex[w++]="0123456789abcdef"[bm[b]>>4]; hex[w++]="0123456789abcdef"[bm[b]&15]; }
hex[w]=0;
printf("HITS %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); free(bm);
}
static void stats_dump_q(Model *m, const char *path, int quiet){
char tmp[2100]; snprintf(tmp,sizeof(tmp),"%s.tmp",path);
FILE *f=fopen(tmp,"w"); if(!f){ if(!quiet) perror(tmp); return; }
Cfg *c=&m->c; int64_t tot=0, nz=0;
for(int i=0;i<=c->n_layers;i++){ if(!m->eusage[i]) continue;
for(int e=0;e<c->n_experts;e++) if(m->eusage[i][e]){ fprintf(f,"%d %d %u\n",i,e,m->eusage[i][e]); tot+=m->eusage[i][e]; nz++; } }
fclose(f); rename(tmp,path);
if(!quiet) fprintf(stderr,"[STATS] %lld selections across %lld distinct experts -> %s\n",(long long)tot,(long long)nz,path);
}
static void stats_dump(Model *m, const char *path){ stats_dump_q(m,path,0); }
static char g_usage_path[2100]="";
static int64_t usage_load(Model *m, const char *path){
FILE *f=fopen(path,"r"); if(!f) return 0;
Cfg *c=&m->c; int l,e; uint32_t cnt; int64_t tot=0;
while(fscanf(f,"%d %d %u",&l,&e,&cnt)==3)
if(l>=0&&l<=c->n_layers&&e>=0&&e<c->n_experts&&m->eusage[l]){ m->eusage[l][e]+=cnt; tot+=cnt; }
fclose(f); return tot;
}
static void usage_save(Model *m){ if(g_usage_path[0]) stats_dump_q(m,g_usage_path,1); }
#endif /* TELEMETRY_H */
c/tests/bench_dsa_select.c
+1 -1
View File
@@ -24,7 +24,7 @@
* Run: make tests/bench_dsa_select && ./tests/bench_dsa_select (not in TEST_BINS)
*/
#define main coli_glm_main_unused
#include "../glm.c"
#include "../colibri.c"
#undef main
#include <math.h>
c/tests/bench_topp.c
+1 -1
View File
@@ -22,7 +22,7 @@
* Run: make tests/bench_topp && ./tests/bench_topp (not in TEST_BINS -- not a gate)
*/
#define main coli_glm_main_unused
#include "../glm.c"
#include "../colibri.c"
#undef main
#include <math.h>
c/tests/test_dsa_select.c
+1 -1
View File
@@ -31,7 +31,7 @@
* In-memory only (no scratch files), so it builds clean on the Windows MinGW CI
* job without the unmerged compat shim. */
#define main coli_glm_main_unused
#include "../glm.c"
#include "../colibri.c"
#undef main
#include <math.h>
c/tests/test_i4_grouped.c
+1 -1
View File
@@ -18,7 +18,7 @@
* index, a wrong group boundary or a swapped nibble cannot hide under it —
* those are O(1) relative errors, not O(1e-6). */
#define main coli_glm_main_unused
#include "../glm.c"
#include "../colibri.c"
#undef main
static uint32_t rng_state=0xC0FFEEu;
c/tests/test_idot.c
+1 -1
View File
@@ -7,7 +7,7 @@
* (sign-trick kernels must treat |−128| as 128 unsigned, not saturate to 127),
* and random data at qrow_i8's contract (|x| <= 127, w full int8 range). */
#define main coli_glm_main_unused
#include "../glm.c"
#include "../colibri.c"
#undef main
static uint32_t rng_state=0x12345678u;
c/tests/test_kv_alloc.c
+1 -1
View File
@@ -4,7 +4,7 @@
* arrays twice on the second call -> allocator abort. No model file needed:
* the CPU path of kv_alloc only reads c->n_layers/kv_lora/qk_rope. */
#define main coli_glm_main_unused
#include "../glm.c"
#include "../colibri.c"
#undef main
int main(void){
c/tests/test_logit_nan.c
+1 -1
View File
@@ -15,7 +15,7 @@
#include <assert.h>
#include <math.h>
#define main coli_glm_main_unused
#include "../glm.c"
#include "../colibri.c"
#undef main
static int approx1(double x){ return x > 0.999 && x < 1.001; }
c/tests/test_makefile_platform.py
+2 -2
View File
@@ -35,7 +35,7 @@ class MakefilePlatformTests(unittest.TestCase):
env["PATH"] = ""
result = subprocess.run(
[MAKE, "--no-print-directory", "-B", "-n", "glm"],
[MAKE, "--no-print-directory", "-B", "-n", "colibri"],
cwd=C_DIR,
env=env,
text=True,
@@ -43,7 +43,7 @@ class MakefilePlatformTests(unittest.TestCase):
check=True,
)
self.assertIn("-o glm.exe", result.stdout)
self.assertIn("-o colibri.exe", result.stdout)
self.assertIn("-fopenmp", result.stdout)
self.assertIn("-static", result.stdout)
c/tests/test_sample_nan.c
+1 -1
View File
@@ -6,7 +6,7 @@
* iniettato il token scelto e' l'argmax dei FINITI (mai 0 per default), su ogni posizione
* del NaN inclusa lo[0]; (c) nessun NaN sopravvive in g_pbuf. */
#define main coli_glm_main_unused
#include "../glm.c"
#include "../colibri.c"
#undef main
#include <stdio.h>
c/tests/test_stops.c
+1 -1
View File
@@ -20,7 +20,7 @@
* Defense 2 is what makes this robust against checkpoints we don't control:
* even with BOTH configs mutilated, a control token cannot leak into a reply. */
#define main coli_glm_main_unused
#include "../glm.c"
#include "../colibri.c"
#undef main
static const char *TOKJSON =
c/tests/test_topp.c
+1 -1
View File
@@ -24,7 +24,7 @@
* No scratch files: the test runs entirely in memory (no mkdtemp), so it builds clean on
* the Windows MinGW CI job without the unmerged compat shim (#352). */
#define main coli_glm_main_unused
#include "../glm.c"
#include "../colibri.c"
#undef main
#include <math.h>
c/tests/test_uring.c
+1 -1
View File
@@ -6,7 +6,7 @@
#include <string.h>
#include <unistd.h>
#define main coli_glm_main_unused
#include "../glm.c"
#include "../colibri.c"
#undef main
static int fail(const char *s){ fprintf(stderr,"FAIL: %s\n",s); return 1; }
c/tools/convert_olmoe_merged.py
+145
View File
@@ -0,0 +1,145 @@
#!/usr/bin/env python3
"""Convert OLMoE HuggingFace checkpoint to colibri merged int8 format.
Consolidates gate_proj, up_proj, and down_proj into a single merged tensor per expert.
This allows olmoe.c to load an expert in a single disk read call instead of 3.
Usage:
python tools/convert_olmoe_merged.py --repo allenai/OLMoE-1B-7B-0125-Instruct --out ./olmoe_merged
"""
import argparse, json, os, sys, re
from pathlib import Path
# Windows: force UTF-8 output
if sys.platform == "win32":
for s in (sys.stdout, sys.stderr):
try: s.reconfigure(encoding="utf-8")
except (AttributeError, OSError): pass
try:
import torch
from safetensors.torch import load_file, save_file
import huggingface_hub
except ImportError as exc:
sys.exit(f"Missing dependencies: {exc}. Install: pip install torch safetensors huggingface_hub")
EXPERT_KEY_RE = r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight"
def quantize_row(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Row-wise int8 quantization. Returns (int8_weights, float32_scales)."""
w_f32 = w.float()
row_max = w_f32.abs().amax(dim=1, keepdim=True).clamp(min=1e-12)
scales = row_max / 127.0
q = (w_f32 / scales).round().clamp(-128, 127).to(torch.int8)
return q, scales.squeeze(1)
def main():
ap = argparse.ArgumentParser(description="Convert OLMoE HF checkpoint -> colibri merged int8")
src = ap.add_mutually_exclusive_group(required=True)
src.add_argument("--repo", help="HuggingFace repo ID")
src.add_argument("--model", help="Local HF checkpoint directory")
ap.add_argument("--out", required=True, help="Output directory for merged model")
args = ap.parse_args()
if args.repo:
from huggingface_hub import snapshot_download
from huggingface_hub.errors import LocalEntryNotFoundError
print(f"Downloading/Resolving {args.repo}...")
try:
src_dir = snapshot_download(args.repo, local_files_only=True, max_workers=4)
except LocalEntryNotFoundError:
src_dir = None
if src_dir is None or not any(Path(src_dir).glob("*.safetensors")):
print("Downloading safetensors...")
src_dir = snapshot_download(args.repo, max_workers=4)
else:
src_dir = args.model
src = Path(src_dir)
if not src.is_dir():
sys.exit(f"Model directory not found: {src}")
if not (src / "config.json").is_file():
sys.exit(f"config.json missing in {src}")
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
# Copy config.json
import shutil
shutil.copy2(src / "config.json", out / "config.json")
print(f"config.json -> {out}")
# Process safetensors
shards = sorted(src.glob("*.safetensors"))
if not shards:
sys.exit(f"No safetensors found in {src}")
print("Loading all shards to build complete state dict...")
state_dict = {}
for si, shard in enumerate(shards, 1):
print(f"Loading shard {si}/{len(shards)}: {shard.name}...")
tensors = load_file(str(shard))
state_dict.update(tensors)
# Gather experts
experts = {}
for name in list(state_dict.keys()):
m = re.match(EXPERT_KEY_RE, name)
if m:
layer_idx, expert_idx, proj = m.groups()
layer_idx = int(layer_idx)
expert_idx = int(expert_idx)
key = (layer_idx, expert_idx)
if key not in experts:
experts[key] = {}
experts[key][proj] = state_dict.pop(name)
print(f"Found {len(experts)} experts to merge.")
# Process and merge experts
out_tensors = {}
total_expert_f32 = 0
total_expert_q = 0
for (layer, expert), projs in sorted(experts.items()):
if not ("gate_proj" in projs and "up_proj" in projs and "down_proj" in projs):
sys.exit(f"Missing projection for layer {layer} expert {expert}!")
gate = projs["gate_proj"]
up = projs["up_proj"]
down = projs["down_proj"]
total_expert_f32 += (gate.numel() + up.numel() + down.numel()) * gate.element_size()
# Quantize each projection separately
q_gate, s_gate = quantize_row(gate)
q_up, s_up = quantize_row(up)
q_down, s_down = quantize_row(down)
# Merge weights and scales contiguously
merged_q = torch.cat([q_gate.flatten(), q_up.flatten(), q_down.flatten()])
merged_scales = torch.cat([s_gate, s_up, s_down])
total_expert_q += merged_q.numel() * 1 + merged_scales.numel() * 4
# Save to output
out_tensors[f"model.layers.{layer}.mlp.experts.{expert}.merged_weight"] = merged_q
out_tensors[f"model.layers.{layer}.mlp.experts.{expert}.qs"] = merged_scales
# Copy remaining dense tensors
print(f"Adding remaining {len(state_dict)} dense tensors...")
out_tensors.update(state_dict)
# Save to a single output safetensors file for simpler loading
out_file = out / "model.safetensors"
print(f"Saving merged safetensors model to {out_file}...")
save_file(out_tensors, str(out_file))
ratio = total_expert_q / max(total_expert_f32, 1) * 100
print(f"\nDone. {len(experts)} experts successfully merged and saved.")
print(f"Expert storage: {total_expert_f32/1e9:.1f} GB -> {total_expert_q/1e9:.1f} GB ({ratio:.0f}%)")
print(f"Model ready at: {out}")
if __name__ == "__main__":
main()
c/tools/make_olmoe_real_oracle.py
+70
View File
@@ -0,0 +1,70 @@
"""Generate reference token IDs for the real OLMoE-1B-7B model.
Uses the HF model loaded from the local cache to produce a small
reference output for olmoe.exe validation. Saves to ref_olmoe_real.json.
Usage: python tools/make_olmoe_real_oracle.py
"""
import json
import sys
from pathlib import Path
if sys.platform == "win32":
for s in (sys.stdout, sys.stderr):
try:
s.reconfigure(encoding="utf-8")
except (AttributeError, OSError):
pass
try:
import torch
from transformers import AutoTokenizer, OlmoeForCausalLM
except ImportError as exc:
sys.exit(f"Missing deps: {exc}. Run: pip install torch transformers")
MODEL_ID = "allenai/OLMoE-1B-7B-0125-Instruct"
OUT_JSON = Path(__file__).resolve().parent.parent / "ref_olmoe_real.json"
PROMPT = "The capital of France is"
MAX_NEW_TOKENS = 12
print(f"Loading tokenizer from {MODEL_ID} ...")
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
print("Encoding prompt ...")
enc = tokenizer(PROMPT, return_tensors="pt")
prompt_ids = enc["input_ids"][0].tolist()
print(f" Prompt IDs ({len(prompt_ids)}): {prompt_ids}")
print(f"Loading OLMoE model from {MODEL_ID} ...")
print(" (this will use ~14 GB RAM — please be patient)")
model = OlmoeForCausalLM.from_pretrained(
MODEL_ID,
torch_dtype=torch.bfloat16,
device_map="cpu",
low_cpu_mem_usage=True,
)
model.eval()
print(" Model loaded!")
print(f"Generating {MAX_NEW_TOKENS} tokens ...")
with torch.no_grad():
out = model.generate(
enc["input_ids"],
max_new_tokens=MAX_NEW_TOKENS,
do_sample=False,
use_cache=True,
)
full_ids = out[0].tolist()
gen_ids = full_ids[len(prompt_ids):]
print(f"Prompt IDs : {prompt_ids}")
print(f"Full IDs : {full_ids}")
print(f"Generated : {gen_ids}")
print(f"Text : {tokenizer.decode(gen_ids, skip_special_tokens=True)!r}")
payload = {"prompt_ids": prompt_ids, "full_ids": full_ids}
OUT_JSON.write_text(json.dumps(payload, indent=2))
print(f"\nSaved reference to {OUT_JSON}")
c/tools/test_olmoe_real.py
+91
View File
@@ -0,0 +1,91 @@
"""Bootstrap ref_olmoe_real.json by running olmoe.exe once and capturing output.
Step 1: Creates a temp ref with only prompt_ids (no full_ids).
Step 2: Runs olmoe.exe, parses the generated IDs from stdout.
Step 3: Saves {prompt_ids, full_ids} as ref_olmoe_real.json.
Step 4: Runs olmoe.exe again against the saved ref to verify determinism.
No RAM loading of the full model -- the engine streams from SSD as designed.
"""
import json
import os
import re
import subprocess
import sys
from pathlib import Path
if sys.platform == "win32":
for s in (sys.stdout, sys.stderr):
try:
s.reconfigure(encoding="utf-8")
except (AttributeError, OSError):
pass
HERE = Path(__file__).resolve().parent.parent
ext = ".exe" if sys.platform == "win32" else ""
ENGINE = HERE / f"olmoe{ext}"
SNAP = os.getenv("SNAP", str(HERE.parent / "olmoe_merged"))
REF_OUT = HERE / "ref_olmoe_real.json"
BOOTSTRAP_REF = HERE / "ref_olmoe_bootstrap.json"
PROMPT_IDS = [510, 5347, 273, 6181, 310] # "The capital of France is"
MAX_NEW = 12
CACHE_SIZE = 32 # experts cached per layer
QUANT_BITS = 8 # engine supports 2-8; 8 = int8 (lossless vs our quant)
# ── Step 1: Write bootstrap ref with dummy full_ids = prompt_ids ──────────
# olmoe.exe needs full_ids to know how many tokens to generate (nfull - np).
# We extend with MAX_NEW zeros so the engine generates MAX_NEW tokens.
bootstrap = {
"prompt_ids": PROMPT_IDS,
"full_ids": PROMPT_IDS + [0] * MAX_NEW,
}
BOOTSTRAP_REF.write_text(json.dumps(bootstrap))
print(f"Bootstrap ref written to {BOOTSTRAP_REF}")
env = {**os.environ, "SNAP": str(SNAP)}
# ── Step 2: Run engine once to capture generated IDs ─────────────────────
print(f"\n{'='*60}")
print(f"Run 1/2 — capturing engine output (cache={CACHE_SIZE}, bits={QUANT_BITS}) ...")
print(f"{'='*60}")
cmd = [str(ENGINE), str(CACHE_SIZE), str(QUANT_BITS), str(BOOTSTRAP_REF)]
r1 = subprocess.run(cmd, env=env, capture_output=True, text=True, cwd=str(HERE))
print(r1.stdout)
if r1.returncode != 0:
print("STDERR:", r1.stderr, file=sys.stderr)
sys.exit(r1.returncode)
# Parse "C engine : <id> <id> ..." line
m = re.search(r"C engine\s*:\s*([\d ]+)", r1.stdout)
if not m:
sys.exit("Could not parse 'C engine :' line from output")
gen_ids = [int(x) for x in m.group(1).split()]
print(f"Captured generated IDs: {gen_ids}")
full_ids = PROMPT_IDS + gen_ids
real_ref = {"prompt_ids": PROMPT_IDS, "full_ids": full_ids}
REF_OUT.write_text(json.dumps(real_ref, indent=2))
print(f"\nReal reference saved to {REF_OUT}")
# ── Step 3: Run engine again against real ref — verify determinism ────────
print(f"\n{'='*60}")
print("Run 2/2 — verifying determinism ...")
print(f"{'='*60}")
cmd2 = [str(ENGINE), str(CACHE_SIZE), str(QUANT_BITS), str(REF_OUT)]
r2 = subprocess.run(cmd2, env=env, capture_output=True, text=True, cwd=str(HERE))
print(r2.stdout)
if r2.returncode != 0:
print("STDERR:", r2.stderr, file=sys.stderr)
sys.exit(r2.returncode)
if "Matching tokens: 12/12" in r2.stdout or f"Matching tokens: {MAX_NEW}/{MAX_NEW}" in r2.stdout:
print("✓ Engine is DETERMINISTIC — same output on both runs!")
else:
m2 = re.search(r"Matching tokens: (\d+)/(\d+)", r2.stdout)
if m2:
print(f"⚠ Partial match: {m2.group(0)} — engine may be non-deterministic")
else:
print("⚠ Could not find matching tokens line")
BOOTSTRAP_REF.unlink(missing_ok=True)