diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..f911652 --- /dev/null +++ b/.clang-format @@ -0,0 +1,10 @@ +BasedOnStyle: LLVM +IndentWidth: 4 +ColumnLimit: 120 +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: AllIfsAndElse +AllowShortLoopsOnASingleLine: true +BreakBeforeBraces: Attach +PointerAlignment: Right +SpaceAfterCStyleCast: false +SortIncludes: false diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..e0acabb --- /dev/null +++ b/.editorconfig @@ -0,0 +1,24 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 + +[*.{c,h,cu}] +indent_size = 4 + +[*.{ts,tsx,js,json,css}] +indent_size = 2 + +[Makefile] +indent_style = tab + +[*.yml] +indent_size = 2 + +[*.md] +trim_trailing_whitespace = false diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000..bdbc3a3 --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,49 @@ +# CI: run the repo's own dependency-free gate (`make check` = clean + portable +# CPU build + C unit suites + Python stdlib tests) on the three claimed +# platforms. No model downloads, no CUDA, no external deps — by design (#140). +name: check + +on: + push: + branches: [main, dev] + pull_request: + branches: [main, dev] + +jobs: + linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: make check + run: make -C c check + + windows: + # The job that would have caught #68/#137 pre-merge: native MinGW-w64 + # (MSYS2/UCRT64), the exact toolchain the README's Windows port targets. + runs-on: windows-latest + defaults: + run: + shell: msys2 {0} + steps: + - uses: actions/checkout@v4 + - uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: false + install: >- + make + mingw-w64-ucrt-x86_64-gcc + mingw-w64-ucrt-x86_64-python + - name: make check + run: make -C c check + + macos: + # clang; libomp for the threaded path (Makefile falls back to + # single-threaded automatically if it's ever missing). + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - name: install libomp + run: brew install libomp + - name: make check + run: make -C c check diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e18f1b3..f213363 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Build glm - run: cd c && make glm + - name: Build colibri + run: cd c && make colibri - name: C test suite run: cd c && make test-c @@ -41,6 +41,78 @@ jobs: -Xcompiler=-Wall,-Wextra echo "CUDA syntax check passed" + windows-cuda-build: + # The Windows CUDA path has no coverage anywhere: `engine-cuda-syntax` above + # compiles with nvcc's *GCC* host on Linux, and check.yml's windows job is + # MinGW/UCRT64 CPU-only by design (#140). But nvcc on Windows requires MSVC + # as its host compiler — it does not accept MinGW — so `make cuda-dll` runs a + # toolchain nothing else in CI touches. That gap is not theoretical: every + # bug in #158 was a *build* failure on this path (MSVC rejects the GCC-style + # -Xcompiler=-Wall,-Wextra with "D8021 invalid numeric argument '/Wextra'", + # unresolvable CUDA_HOME/NVCC defaults, POSIX setenv in the kernel test), and + # #314 was CUDA_HOME with spaces — the layout the CUDA installer ships by + # default. Both classes are compile-time and need no GPU to catch. + # + # Build-only ON PURPOSE: GitHub's hosted runners have no NVIDIA device, so + # this job proves the Windows+MSVC CUDA build stays buildable, NOT that the + # kernels or the DLL loader behave on real silicon. That still needs hardware + # (see #157). Claiming otherwise would be the false confidence the + # engine-cuda-syntax comment above already warns about. + name: CUDA build (Windows, MSVC host) + # windows-2022, NOT windows-latest: the latest image now ships Visual Studio + # 18 (MSVC 14.5x), and CUDA's crt/host_config.h hard-errors on any host newer + # than VS 2022 ("Only the versions between 2017 and 2022 (inclusive) are + # supported"). That is a real constraint for every CUDA user on Windows, not + # a CI quirk — pinning tracks what the toolkit actually supports. Revisit when + # a CUDA release accepts VS 18; -allow-unsupported-compiler would only mask it. + runs-on: windows-2022 + steps: + - uses: actions/checkout@v4 + - name: MSVC environment (puts cl.exe on PATH for nvcc -ccbin) + uses: ilammy/msvc-dev-cmd@v1 + - name: Install CUDA toolkit (compiler only) + uses: Jimver/cuda-toolkit@v0.2.19 + with: + # Same pin as engine-cuda-syntax: v0.2.19's version table stops at + # 12.6.2. This installs to the default "C:\Program Files\NVIDIA GPU + # Computing Toolkit\..." path, so CUDA_HOME is space-bearing here — + # which is exactly the #314 regression this job would have caught. + cuda: '12.6.2' + method: network + # cudart as well as nvcc: unlike the Linux job (which only needs to + # *compile* backend_cuda.cu), cuda-dll links it, and on Windows the + # runtime headers/import lib ship as a separate installer component — + # with '["nvcc"]' alone this fails at `#include `. + sub-packages: '["nvcc", "cudart"]' + - uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: false + # inherit: cl.exe (msvc-dev-cmd) and nvcc (cuda-toolkit) are added to + # the *Windows* PATH by the steps above; without inheriting it the + # recipe's `command -v` guards fail inside the MSYS2 shell. + path-type: inherit + install: >- + make + mingw-w64-ucrt-x86_64-gcc + - name: make cuda-dll (nvcc + MSVC host) + shell: msys2 {0} + run: | + cd c + # CUDA_ARCH is pinned: the default is `native`, which asks the driver + # what card is present — there is none here, so it must be explicit. + # sm_80 matches engine-cuda-syntax and is supported by the 12.6 pin. + make cuda-dll CUDA_ARCH=sm_80 + test -f coli_cuda.dll || { echo "cuda-dll reported success but produced no DLL" >&2; exit 1; } + echo "coli_cuda.dll built (MSVC host)" + - name: make colibri CUDA_DLL=1 (host links backend_loader, not cudart) + shell: msys2 {0} + run: | + cd c + make colibri CUDA_DLL=1 + test -f colibri.exe || { echo "colibri CUDA_DLL=1 reported success but produced no exe" >&2; exit 1; } + echo "colibri.exe built against the DLL loader" + web: name: Web UI runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..e471fb9 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,104 @@ +name: Release + +on: + push: + tags: ['v*'] + +permissions: + contents: write + +jobs: + build: + strategy: + matrix: + include: + - os: ubuntu-latest + name: linux-x86_64 + ext: "" + make_args: "ARCH=x86-64-v3" + - os: macos-latest + name: macos-arm64 + ext: "" + make_args: "" + - os: windows-latest + name: windows-x86_64 + ext: ".exe" + make_args: "" + # GitHub shells are format strings: 'msys2 {0}', never bare 'msys2' + # (the v1.0.0 tag build failed on exactly this). Same as ci.yml. + shell: "msys2 {0}" + runs-on: ${{ matrix.os }} + defaults: + run: + shell: ${{ matrix.shell || 'bash' }} + steps: + - uses: actions/checkout@v4 + + - if: matrix.os == 'windows-latest' + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: false + # inherit: the Package step calls the runner's 7z.exe, which lives on + # the Windows PATH; the default minimal path would not see it. + path-type: inherit + install: >- + make + mingw-w64-ucrt-x86_64-gcc + + - if: matrix.os == 'macos-latest' + run: brew install libomp + + - name: Build engine + run: | + cd c + make glm ${{ matrix.make_args }} + ls -lh glm${{ matrix.ext }} + + - name: Package + run: | + TAG=${GITHUB_REF#refs/tags/} + mkdir -p dist + cp c/glm${{ matrix.ext }} dist/colibri-${TAG}-${{ matrix.name }}${{ matrix.ext }} + cp c/coli dist/ + cp c/version.py dist/ + cp c/openai_server.py dist/ + cp c/resource_plan.py dist/ + cp c/doctor.py dist/ + cp LICENSE dist/ + cd dist + if [ "${{ matrix.ext }}" = ".exe" ]; then + 7z a colibri-${TAG}-${{ matrix.name }}.zip * + else + tar czf colibri-${TAG}-${{ matrix.name }}.tar.gz * + fi + + - uses: actions/upload-artifact@v4 + with: + name: colibri-${{ matrix.name }} + path: dist/colibri-*.* + + release: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + path: artifacts + merge-multiple: true + + - name: Create GitHub Release + env: + GH_TOKEN: ${{ github.token }} + run: | + TAG=${GITHUB_REF#refs/tags/} + # Extract the latest version section from CHANGELOG + NOTES=$(awk "/^## \\[${TAG#v}\\]/{found=1;next} /^## \\[/{if(found)exit} found{print}" CHANGELOG.md) + if [ -z "$NOTES" ]; then + NOTES="Release ${TAG}" + fi + gh release create "$TAG" artifacts/* \ + --title "colibrì ${TAG}" \ + --notes "$NOTES" diff --git a/.gitignore b/.gitignore index 981e740..b3e1193 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ desktop/src-tauri/target/ desktop/src-tauri/gen/ # binari compilati (si rigenerano con make / coli build) +c/colibri +c/colibri.exe c/glm c/glm.exe c/olmoe @@ -68,3 +70,7 @@ c/tests/test_decode_batch c/tests/test_i4_acc512 c/tests/test_idot c/tests/test_uring +olmoe_merged/ +olmoe_i4/ +c/olmoe_merged/ +c/olmoe_i4/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6177966 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,53 @@ +# Changelog + +All notable changes to colibrì are documented here. +Format follows [Keep a Changelog](https://keepachangelog.com/). + +## [1.0.0] — 2026-07-19 + +First tagged release. The engine has been running in production since late June +2026; this tag marks the baseline for semantic versioning going forward. + +### Highlights + +- **GLM-5.2 (744B MoE)** runs on ~25 GB RAM in pure C, streaming experts from disk +- **Three-tier placement**: VRAM (hot) / RAM (warm) / NVMe (cold), with a learning + cache that pins your workload's hottest experts automatically +- **CUDA backend**: multi-GPU expert tier, dense tensor distribution, batched + ragged attention, resident pipeline (`COLI_CUDA_PIPE=2`) +- **Metal backend** (Apple Silicon): batched expert SwiGLU + fused decode attention + on unified memory GPU +- **MTP speculation**: native GLM-5.2 draft heads, grammar-forced drafts, kernel- + pinned verification (`SPEC_PIN=1`) +- **OpenAI-compatible API**: `coli serve` with SSE streaming, KV slots, bounded + queue, web dashboard (`coli web`) +- **Web UI**: chat with live metrics, expert cortex brain page, profiling breakdown, + expert atlas 3-D galaxy +- **Cross-platform**: Linux, macOS, Windows 11 (native MinGW), PowerPC; CI on all three +- **Auto-tune**: `coli plan --auto-tier` classifies the bottleneck and derives + MTP/PIPE/NUMA/PIN settings with explanations + +### Engine + +- Token-exact validation against `transformers` oracle (teacher-forcing 32/32) +- Compressed MLA KV cache (576 floats/token, 57× smaller), persisted across + restarts (`.coli_kv`, zero re-prefill) +- DSA sparse attention (lightning indexer), faithfully implemented +- Router-lookahead prefetch (`PILOT=1`, 71.6% predictive) +- Async expert I/O pool (`PIPE=1`), io_uring batching (`URING=1`) +- NUMA-aware expert placement (`COLI_NUMA=1`, +13–40% on multi-socket) +- AVX2 / AVX-512 / AVX-VNNI / ARM NEON / NEON-i8mm / POWER VSX kernels +- int4 / int8 / int2 / grouped-int4 (fmt=4) quantization formats + +### Tools + +- `coli convert` — FP8→int4 one-shard-at-a-time converter +- `coli doctor` — read-only setup diagnostics +- `coli plan` — resource planner with auto-tune prescription +- `coli bench` — MMLU / HellaSwag / ARC quality benchmarks +- Expert atlas (`tools/analyze.py --web`) — measured topic affinity for 19,456 experts + +### Community + +- 30+ hardware datapoints in the benchmark tracker +- Contributions from 20+ authors across engine, docs, tooling, and ports diff --git a/README.it.md b/README.it.md new file mode 100644 index 0000000..76e3c21 --- /dev/null +++ b/README.it.md @@ -0,0 +1,260 @@ +

+ colibrì — motore piccolo, modello immenso +

+ +

+ English · 简体中文 · 繁體中文 · Italiano +

+ +**Motore piccolo, modello immenso.** Esegui **GLM-5.2 (744 miliardi di parametri, MoE)** su un computer consumer con ~25 GB di RAM — in C puro, zero dipendenze, caricando gli expert dal disco in streaming. + +Colibrì è un runtime MoE leggero e che preserva la qualità: tratta VRAM, RAM e +disco come un'unica gerarchia di memoria gestita. Se la memoria veloce non basta +il modello rallenta, ma la policy predefinita **non cambia mai silenziosamente la +precisione del modello né la semantica del router**. + +``` +$ ./coli chat + 🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU + ✓ ready in 32s · resident 9.9 GB + › ciao! + ◆ Ciao! 😊 Come posso aiutarti oggi? +``` + +## Guardalo in azione + +

+ dashboard web di colibrì — metriche live, pannello hardware, livelli degli expert +

+

La dashboard web (./coli web): un modello da 744B a 4 tok/s, TTFT 1.6 s, disco 0 — +residenza completa degli expert su 6× RTX 5090, con metriche token in tempo reale, breakdown dei tempi per turno, +la barra dei livelli VRAM/RAM/disco e il mini-cervello live nell'angolo.

+ +

+ la pagina Brain — 19.456 expert come una corteccia vivente +

+

La pagina Brain: tutti i 19.456 expert come una corteccia vivente — il colore indica +il livello di archiviazione, la luminosità il calore di routing, e ogni expert instradato in un turno +lampeggia bianco. Passando il cursore si vede l'affinità +tematica misurata dell'expert.

+ +

+ la pagina Atlas — l'atlante misurato degli expert come una galassia 3D +

+

La pagina Atlas: l'atlante +misurato degli expert come una galassia 3D — 13.260 expert caratterizzati, 1.041 specialisti +replicabili che si raggruppano per argomento (poesia, legge, cinese, SQL…). La posizione deriva +dall'affinità di routing misurata, non da un embedding appreso. Trascinare per ruotare.

+ +## L'idea + +Un modello Mixture-of-Experts da 744B attiva solo ~40B parametri per token — e +solo ~11 GB di quelli cambiano da un token all'altro (gli expert instradati): + +

+ solo ~5.4% dei parametri è attivo per token +

+ +Il modello non ha bisogno di *stare* in memoria veloce — ha bisogno di essere +**piazzato**: + +- la **parte densa** (attenzione, expert condivisi, embedding — ~17B parametri) + resta **residente in RAM a int4** (~9.9 GB); +- i **19.456 expert instradati** (75 layer MoE × 256 + la testa MTP, ~19 MB + ciascuno a int4) stanno **su disco** (~370 GB) e vengono **caricati on demand + in streaming**, con una cache LRU per layer, un hot-store pinnato che impara, + e un livello VRAM opzionale. + +Il motore è un singolo file C (`c/colibri.c`) più header piccoli. Niente BLAS, +niente Python a runtime, niente GPU obbligatoria. + +## Come funziona + +### Il percorso di ogni token + +

+ instrada → unione → piazza → sovrapponi → impara +

+ +Ogni layer di ogni token percorre gli stessi cinque passi. L'obiettivo +progettuale è che **il piazzamento decide solo la velocità** — le decisioni +del router e la precisione dei pesi sono identiche sia che un expert risponda +dalla VRAM sia dal disco. + +### Una gerarchia di memoria, non un requisito di memoria + +

+ residenza expert a tre livelli: VRAM / RAM / NVMe +

+ +Lo stesso motore copre l'intero spettro: su un portatile da 25 GB tutto viene +caricato dal disco in streaming (lento, ma corretto); su un host grande l'intero +set di expert diventa residente (`CUDA_EXPERT_GB=auto PIN_GB=all`) e il disco +esce completamente dal percorso di decode. Tra i livelli c'è una **cache che +impara**: il motore registra quali expert il *tuo* carico di lavoro instrada +(`.coli_usage`, aggiornato a ogni turno) e fissa automaticamente i più caldi — +colibrì diventa letteralmente più veloce man mano che lo usi. Sugli host +multi-socket, `COLI_NUMA=1` interlaccia i pesi residenti tra i controller di +memoria ([#82](https://github.com/JustVugg/colibri/issues/82)). + +### Mai aspettare il disco due volte + +I miss nella cache costano caro, quindi il motore investe la maggior parte +della sua astuzia per evitarli e sovrapporli: le tre matrici di ogni expert sono +memorizzate contigue e lette con un unico `pread`; un pool I/O asincrono +limitato (`PIPE=1`, attivo per default) carica gli expert mancanti mentre quelli +residenti calcolano; le posizioni in batch leggono ogni expert unico una sola +volta (**batch-union**); un thread di lookahead del router (`PILOT=1`) fa il +prefetch degli expert del layer successivo — il routing è misurabilmente +**prevedibile al 71.6% un layer in anticipo**. Sulle GPU, la pipeline residente +(`COLI_CUDA_PIPE=2`) mantiene il flusso residuo on-device tra i layer, così il +loop CPU degli expert procede senza interruzioni; su Apple Silicon un backend +[Metal](docs/metal.md) sperimentale esegue la matmul batch degli expert sulla +GPU a memoria unificata. + +### Modello fedele, stato compresso + +Il forward pass è validato **token-esatto contro un oracle `transformers`** +(teacher-forcing 32/32). L'attenzione MLA memorizza uno stato KV compresso — 576 +float/token invece di 32.768 (**57× più piccolo**) — e lo persiste tra i +riavvii (`.coli_kv`): le conversazioni riaprono "calde", senza alcun re-prefill, +byte-identiche a una sessione ininterrotta. L'attenzione sparsa DSA (il +lightning indexer di GLM-5.2) è implementata fedelmente e validata forzando la +selezione di tutte le chiavi per riprodurre esattamente l'attenzione densa. + +### Decodifica speculativa, onestamente + +La testa MTP nativa di GLM-5.2 propone token che il modello principale verifica +in un unico forward batch — 2.2–2.8 token/forward quando conviene. Due regole +conquistate a caro prezzo sono i default: la testa MTP deve essere **int8** (le +teste int4 crollano al 0–4% di accettazione, +[#8](https://github.com/JustVugg/colibri/issues/8)), e draft e verifica devono +calcolare **la stessa funzione** — `SPEC_PIN=1` fissa entrambi sulla stessa +famiglia di kernel ([#163](https://github.com/JustVugg/colibri/issues/163) +contiene l'intera indagine forense). I draft forzati da grammatica +([`GRAMMAR=file.gbnf`](docs/grammar-draft.md)) aggiungono accettazione quasi +gratuita sull'output JSON vincolato. Se la speculazione conviene dipende dalla +temperatura della cache — misura, e usa `DRAFT=0` quando non paga. + +## Cosa ottiene + +

+ velocità di decode misurata per classe hardware +

+ +Stesso motore, stesso container int4 — cambia solo dove risiedono gli expert. +Punti salienti dalle [tabelle benchmark complete](docs/benchmarks.md): + +- **6× RTX 5090, residenza completa:** 5.8–6.8 tok/s in decode, TTFT ~13 s + ([log dell'esperimento](docs/experiments/glm52-6x5090-2026-07-12.md)); +- **desktop solo-CPU da 128 GB:** ~1.8 tok/s a cache calda + ([#200](https://github.com/JustVugg/colibri/issues/200)); +- **singola RTX 5070 Ti, classe laptop:** 1.07 tok/s tramite la pipeline + GPU-residente ([#273](https://github.com/JustVugg/colibri/issues/273)); +- **macchina di sviluppo da 25 GB:** 0.05–0.1 tok/s a freddo — il punto di + partenza dimostrato da cui è nato il progetto, e ancora oggi la baseline onesta. + +La qualità è misurata, non presunta: il costo di quantizzazione del container +int4 e le ablazioni su granularità delle scale e rotazione sono in +[docs/benchmarks.md](docs/benchmarks.md#quality-benchmark) e +[#108](https://github.com/JustVugg/colibri/issues/108)/[#81](https://github.com/JustVugg/colibri/issues/81). + +## Per iniziare + +### 1. Scarica il modello + +Un container **GLM-5.2 int4** pre-convertito è su Hugging Face — **usa la +versione con le teste MTP int8**: + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +> ⚠️ Il mirror originale contiene teste MTP int4 → accettazione dei draft allo 0% +> ([#8](https://github.com/JustVugg/colibri/issues/8)). Verifica la tua versione: +> `ls -l /out-mtp-*` — int8 (corretto) è `3527131672 / 5366238584 / 1065950496`. + +Oppure converti tu stesso dalla sorgente FP8 — un unico comando riprendibile che +non richiede mai i 756 GB completi su disco contemporaneamente: + +```bash +cd c && ./setup.sh # verifica gcc/OpenMP, compila, autotest +./coli convert --model /nvme/glm52_i4 # scarica e converti shard per shard (python, una tantum) +``` + +### 2. Esegui + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli chat # budget RAM, cache e MTP rilevati automaticamente +COLI_MODEL=/nvme/glm52_i4 ./coli plan # mostra il piazzamento pianificato VRAM/RAM/disco +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # controllo di idoneità (sola lettura) +./coli web --model /nvme/glm52_i4 # API + dashboard web sulla stessa porta +./coli serve --model /nvme/glm52_i4 # solo API compatibile OpenAI +``` + +Il motore a runtime è puro C — python si usa solo per il convertitore (una tantum) +e per il gateway API opzionale. + +### 3. Approfondisci + +| argomento | documento | +|---|---| +| Benchmark, dati dalla comunità, misurazioni di qualità | [docs/benchmarks.md](docs/benchmarks.md) | +| Parametri di tuning, policy, cache che impara, prefetch | [docs/tuning.md](docs/tuning.md) | +| Build nativa su Windows 11 (con CUDA DLL) | [docs/windows.md](docs/windows.md) | +| Backend CUDA, livello expert in VRAM, residenza completa | [docs/cuda.md](docs/cuda.md) | +| Backend Metal per Apple Silicon | [docs/metal.md](docs/metal.md) | +| API compatibile OpenAI, KV slot, dashboard web | [docs/api.md](docs/api.md) | +| Draft forzati da grammatica (output strutturato) | [docs/grammar-draft.md](docs/grammar-draft.md) | +| Inventario delle variabili d'ambiente | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | + +## Sostenere il progetto + +colibrì è nato come progetto di una sola persona su un portatile con 12 core +e 25 GB di RAM; oggi i suoi numeri arrivano da una comunità di macchine reali. +Se ti è utile: + +- ⭐ metti una stella al repository e condividilo; +- 🐛 apri issue con i numeri di benchmark del tuo hardware — i datapoint + fanno avanzare questo progetto più di qualsiasi altra cosa; +- 💬 contattaci via GitHub issues per sponsorizzare lo sviluppo o donare hardware. + +## Struttura del repository + +``` +Makefile punto d'ingresso root per build/check +c/ +├── colibri.c motore principale +├── quant.h kernel matmul quantizzati (SIMD multi-architettura) +├── sample.h campionamento, RNG, set di stop +├── kv_persist.h persistenza KV su disco (.coli_kv) +├── telemetry.h protocollo dashboard, statistiche, usage +├── st.h, tok.h, json.h header di runtime +├── backend_cuda.* livello CUDA opzionale +├── Makefile build e check locali +├── coli CLI utente +├── openai_server.py gateway HTTP compatibile OpenAI +├── setup.sh setup locale in un solo comando +├── tools/ conversione offline, fixture e benchmark +├── scripts/ helper per conversioni lunghe +└── tests/ test C e Python senza dipendenze +web/ UI browser (puro client API OpenAI) +desktop/ shell desktop Tauri v2 che racchiude la web UI +docs/ documentazione di riferimento, esperimenti, media +``` + +Il percorso a runtime resta intenzionalmente piatto e leggibile: `colibri.c` +più i suoi header. Dalla radice del repository, `make`, `make check` e +`make clean` delegano al Makefile del motore. + +## Perché "colibrì" + +Il colibrì pesa pochi grammi, sta sospeso nel vuoto e visita un migliaio di +fiori al giorno. Questo motore tiene in vita un gigante da 744 miliardi di +parametri con le razioni di un colibrì: 25 GB di RAM, dodici core CPU e +tanta pazienza col disco. + +Il nome è rimasto in italiano perché questa è la lingua in cui è stato scritto +il primo prototipo — i commenti nel codice lo testimoniano ancora. + +## Licenza + +Apache 2.0. I pesi di GLM-5.2 sono rilasciati da Z.ai sotto licenza MIT. diff --git a/README.md b/README.md index 2e22048..d6f9bcb 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,16 @@ colibrì — tiny engine, immense model

+

+ English · 简体中文 · 繁體中文 · Italiano +

+ **Tiny engine, immense model.** Run **GLM-5.2 (744B-parameter MoE)** on a consumer machine with ~25 GB of RAM — in pure C, with zero dependencies, by streaming experts from disk. -Colibrì is a lightweight, quality-preserving MoE runtime that treats VRAM, -RAM, and storage as one managed memory hierarchy. Insufficient fast memory may -reduce speed, but the default policy never silently changes model precision or -router semantics. +Colibrì is a lightweight, quality-preserving MoE runtime that treats VRAM, RAM, +and storage as one managed memory hierarchy. Insufficient fast memory may reduce +speed, but the default policy **never silently changes model precision or router +semantics**. ``` $ ./coli chat @@ -17,14 +21,14 @@ $ ./coli chat ◆ Ciao! 😊 Come posso aiutarti oggi? ``` - ## See it running

colibrì web dashboard — live metrics, hardware panel, expert tiers

-

The web dashboard (./coli web): a 744B model answering at 4+ tok/s end-to-end on 6× RTX 5090 — -with live token metrics, the hardware panel, and the VRAM/RAM/disk expert tiers.

+

The web dashboard (./coli web): a 744B model at 4 tok/s, TTFT 1.6 s, disk 0 — +full expert residency on 6× RTX 5090, with live token metrics, the per-turn time breakdown, +the VRAM/RAM/disk tier bar and the live mini-brain in the corner.

the Brain page — 19,456 experts as a live cortex @@ -33,613 +37,182 @@ with live token metrics, the hardware panel, and the VRAM/RAM/disk expert tiers. brightness is routing heat, and every expert routed in a turn flashes white. Hovering shows the expert's measured topic affinity.

-## Contents - -- [The idea](#the-idea) -- [See it running](#see-it-running) -- [What's implemented](#whats-implemented) -- [Honest numbers](#honest-numbers-wsl2-12-cores-25-gb-ram-nvme-via-vhdx) -- [Download the model](#download-the-model) -- [Web dashboard](#web-dashboard) -- [Got a better machine?](#got-a-better-machine-try-it--heres-what-to-expect) +

+ the Atlas page — the measured expert atlas as a 3-D galaxy +

+

The Atlas page: the measured expert atlas +as a 3-D galaxy — 13,260 characterised experts, 1,041 replicated specialists clustering by topic +(poetry, law, Chinese, SQL…). Position is measured routing affinity, not a learned embedding. Drag to spin.

## The idea -A 744B Mixture-of-Experts model activates only ~40B parameters per token — and only ~11 GB of those change from token to token (the routed experts). So: +A 744B Mixture-of-Experts model activates only ~40B parameters per token — and +only ~11 GB of those change from token to token (the routed experts): -- the **dense part** (attention, shared experts, embeddings — ~17B params) stays **resident in RAM at int4** (~9.9 GB); -- the **19,456 routed experts** (75 MoE layers × 256 experts + the MTP head, ~19 MB each at int4) live **on disk** (~370 GB) and are **streamed on demand**, with a per-layer LRU cache, an optional pinned hot-store, and the OS page cache as a free L2. +

+ only ~5.4% of parameters are active per token +

-The engine is a single C file (`c/glm.c`) plus small headers. No BLAS, no Python at runtime, no GPU required (an opt-in CUDA tier for pinned experts exists — see below). +So the model doesn't need to *fit* in fast memory — it needs to be **placed**: -## What's implemented +- the **dense part** (attention, shared experts, embeddings — ~17B params) stays + **resident in RAM at int4** (~9.9 GB); +- the **19,456 routed experts** (75 MoE layers × 256 + the MTP head, ~19 MB each + at int4) live **on disk** (~370 GB) and are **streamed on demand**, with a + per-layer LRU cache, a learned pinned hot-store, and an optional VRAM tier. -- **Faithful GLM-5.2 (`glm_moe_dsa`) forward** — validated token-exact against a `transformers` oracle (teacher-forcing 32/32, greedy 20/20 on a tiny-random model with the real architecture). -- **MLA attention** (q/kv-LoRA, interleaved partial RoPE) with **compressed KV-cache**: 576 floats/token instead of 32,768 (57× smaller — GLM-5.2 has 64 heads and no GQA). -- **DeepSeek-V3-style sigmoid router** (noaux_tc, routed_scaling_factor), shared expert, first-3-dense layers. -- **Native MTP speculative decoding** — GLM-5.2's own multi-token-prediction head (layer 78) drafts tokens that the main model verifies in one batched forward. **The head must be int8** (the converter does this by default): at int4 draft acceptance collapses to 0–4% and speculation never engages; at int8 it's 39–59% acceptance, **2.2–2.8 tokens/forward** (community-measured, [#8](https://github.com/JustVugg/colibri/issues/8)). Lossless *in exact arithmetic* — but **not byte-identical to non-speculative greedy in practice** ([#100](https://github.com/JustVugg/colibri/issues/100)). This isn't MTP-specific: colibrì's quantized integer kernels are shape-dependent, so any batched (S>1) or GPU forward rounds slightly differently from the single-token path, and int4 GLM-5.2 sits close enough to argmax ties that such a rounding change can flip a token. MTP, the CUDA expert tier, and batched prefill are three different ways to trip the same sensitivity (community-confirmed in #100: swapping only the kernel family forks greedy output on 3/5 prompts, with **zero speculation**). Every emitted token is still the argmax of a *valid* forward — the continuation stays correct — it just isn't the same stream. For byte-exact reproducibility: `DRAFT=0` (no speculation), plus `IDOT=0 COLI_CUDA=0` if you also want kernel-family/GPU independence. Under sampling, rejection sampling keeps the distribution correct. Honest caveat from the same measurement: on a **cold** cache each verified draft routes to extra experts (~660 → ~1100 expert-loads/token), so speculation can be a net *time* loss until the cache/pin warms up. -- **Grammar-forced speculative drafts** (`GRAMMAR=file.gbnf`, [#48](https://github.com/JustVugg/colibri/issues/48)) — on constrained-output workloads (JSON/NDJSON, function calling, structured extraction) the grammar itself is a third draft source: wherever it admits exactly **one** legal byte (braces, quotes, key names, enum bodies), that forced span is tokenized and injected as pre-accepted drafts with ~1.0 acceptance — no draft head, no lookup table, and it engages even with the int4 MTP head from [#8](https://github.com/JustVugg/colibri/issues/8). It never constrains sampling: forced spans are verified in the same batch-union forward as any draft, so a wrong or out-of-sync grammar cannot change the output — worst case is rejected drafts, and an adaptive guard turns the source off below 50% acceptance. Byte-level GBNF subset (literals, char classes, `| ( ) ? * +`, comments); `GRAMMAR_DRAFT=n` caps the forced span per forward (default 24). Composes with `DRAFT`/MTP, which fill the free-text gaps between forced spans. Full reference — mechanism, measured A/Bs, when it pays, prior art: [docs/grammar-draft.md](docs/grammar-draft.md). -- **True sampling** — temperature + nucleus, defaults tuned for int4 reality (0.7 / 0.90; the official 1.0 / 0.95 samples quantization noise from the tail). -- **Integer-dot kernels** (Q8_0-style int8 activations, AVX2 `maddubs`): int8 matmuls 1.4–2.5× faster (119 GFLOP/s measured), int4 1.8× in batch — routing decided per shape by measurement (int4 single-row stays f32: it measured slower). -- **MLA weight absorption** (DeepSeek trick) for decode: no per-token k/v reconstruction — the query absorbs `kv_b`, context is projected after attention. Validated exact: TF 32/32 and generation 20/20 with absorption forced everywhere. -- **Async expert readahead**: while one block of experts is being multiplied, the kernel is already reading the next (`WILLNEED`). -- **Quantization kernels**: int8 / packed int4 / packed int2, per-row scales, AVX2, dequant-on-use. Packing validated bit-identical to the int8 container. -- **DSA sparse attention** — GLM-5.2's lightning indexer, faithful to the reference `glm_moe_dsa` modeling: per-layer top-2048 causal key selection (full/shared indexer layers), auto-detected from the `out-idx-*` weights (`--indexer` converter mode, ~189 MB extracted from the FP8 repo). Validated exact: forcing the selection to keep every key reproduces dense attention token-for-token. `DSA=0` disables, `DSA_TOPK` overrides. -- **KV-cache persistence** — conversations reopen **warm** across engine restarts: serve mode appends the compressed MLA KV to `.coli_kv` after every turn (~182 KB/token, crash-safe) and resumes it at startup with zero re-prefill. Validated byte-identical to an uninterrupted session. `KVSAVE=0` disables. -- **Router-lookahead prefetch** (`PILOT=1`, experimental) — the next layer's routing is 71.6% predictable from the current layer's post-attention state (measured); a dedicated I/O thread prefetches those experts while the current layer computes. -- **Batch-union MoE**: in prefill (and MTP verification), each unique expert of the batch is read once and applied to every position that routes to it. -- **Byte-level BPE tokenizer in C** (GPT-2-style with Unicode-property regex, 320k merges). -- **RAM safety**: the expert cache is auto-sized from `MemAvailable` at startup — an honest peak projection (working set, KV, MTP row, reconstruction buffers) so the kernel OOM-killer never fires. -- **Offline FP8→int4 converter** (`c/tools/convert_fp8_to_int4.py`): downloads one shard at a time (~5 GB), dequants (128×128 block scales), requantizes to the engine's container, deletes the shard — the 756 GB FP8 checkpoint never needs to exist on disk at once. Resumable. +The engine is a single C file (`c/glm.c`) plus small headers. No BLAS, no Python +at runtime, no GPU required. -## Honest numbers (WSL2, 12 cores, 25 GB RAM, NVMe via VHDX) +## How it works -Detailed GPU experiment: [GLM-5.2 on 6x RTX 5090](docs/experiments/glm52-6x5090-2026-07-12.md) — full expert residency across VRAM+RAM reaches 6.84 tok/s single-request decode. +### The per-token path -| metric | value | -|---|---| -| model on disk (int4 container) | ~370 GB | -| resident RAM (dense, int4) | 9.9 GB | -| load time | ~30 s | -| peak RSS during chat | ~20 GB (auto-capped) | -| cold decode cost | ~11 GB disk reads/token (75 layers × 8 experts) | -| disk ceiling (this dev box's drive) | ~1 GB/s → ~0.05–0.1 tok/s cold | -| MTP speculation (int8 head) | 2.2–2.8 tok/forward measured ([#8](https://github.com/JustVugg/colibri/issues/8)) | +

+ route → union → place → overlap → learn +

-This is not fast. It is a 744B frontier-class model **answering correctly on a machine that costs less than one H100 fan**. Warm cache, pinned hot experts and MTP push the useful-response latency down considerably; the physics of the disk does the rest. +Every layer of every token walks the same five steps. The design goal is that +**placement only ever decides speed** — the router's decisions and the weights' +precision are the same whether an expert answered from VRAM or from disk. -### SSD note -Cold starts are heavy on random reads (~11 GB/token), but reads don't meaningfully wear an SSD — colibrì's streaming is read-only. The real concerns under heavy use are (1) **swap traffic** if the system runs out of RAM (writes do wear the drive — keep a sane `--ram` budget; colibrì's auto-budget is designed to stay clear of swap) and (2) **sustained thermals**: hours at full read duty cycle will heat cheaper drives. Monitor drive temperature and health. +### One memory hierarchy instead of one memory requirement -## Download the model +

+ VRAM / RAM / NVMe three-tier expert residency +

-A pre-converted **GLM-5.2 int4** model for colibrì is available on Hugging Face — **use the version with the int8 MTP heads** (matey-0's clone): +The same engine spans the whole range: on a 25 GB laptop everything streams from +disk (slow but correct); on a large host the entire expert set becomes resident +(`CUDA_EXPERT_GB=auto PIN_GB=all`) and disk drops out of the decode path +entirely. Between the tiers sits a **learning cache**: the engine records which +experts *your* workload routes to (`.coli_usage`, updated every turn) and pins +the hottest ones automatically — colibrì literally gets faster the more you use +it. On multi-socket hosts, `COLI_NUMA=1` interleaves the resident weights across +memory controllers ([#82](https://github.com/JustVugg/colibri/issues/82)). + +### Never wait for the disk twice + +Misses are expensive, so the engine spends most of its cleverness avoiding and +overlapping them: each expert's three matrices are stored adjacent and read in +one `pread`; a bounded async I/O pool (`PIPE=1`, default) loads missing experts +while resident ones compute; batched positions read each unique expert once +(**batch-union**); and a router-lookahead thread (`PILOT=1`) prefetches the next +layer's experts — routing is measurably **71.6% predictable one layer ahead**. +On GPUs, the resident pipeline (`COLI_CUDA_PIPE=2`) keeps the residual stream +on-device across layers so the CPU expert loop runs uninterrupted; on Apple +Silicon an experimental [Metal backend](docs/metal.md) does the batched expert +math on the unified-memory GPU. + +### Faithful model, compressed state + +The forward pass is validated **token-exact against a `transformers` oracle** +(teacher-forcing 32/32). MLA attention stores a compressed KV state — 576 +floats/token instead of 32,768 (**57× smaller**) — and persists it across +restarts (`.coli_kv`): conversations reopen warm with zero re-prefill, +byte-identical to an uninterrupted session. DSA sparse attention (GLM-5.2's +lightning indexer) is implemented faithfully and validated by forcing full-key +selection to reproduce dense attention exactly. + +### Speculative decoding, honestly + +GLM-5.2's native MTP head drafts tokens that the main model verifies in one +batched forward — 2.2–2.8 tokens/forward when it pays. Two hard-won rules ship +as defaults: the MTP head must be **int8** (int4 heads collapse to 0–4% +acceptance, [#8](https://github.com/JustVugg/colibri/issues/8)), and draft and +verify must compute **the same function** — `SPEC_PIN=1` pins both to one +kernel family ([#163](https://github.com/JustVugg/colibri/issues/163) is the +full forensic story). Grammar-forced drafts +([`GRAMMAR=file.gbnf`](docs/grammar-draft.md)) add ~free acceptance on +constrained JSON output. Whether speculation is a net win depends on your +cache temperature — measure, and use `DRAFT=0` when it doesn't pay. + +## What it achieves + +

+ measured decode speed by hardware class +

+ +Same engine, same int4 container — the hardware only changes where the experts +live. Highlights from the [full benchmark tables](docs/benchmarks.md): + +- **6× RTX 5090, full residency:** 5.8–6.8 tok/s decode, TTFT ~13 s + ([experiment log](docs/experiments/glm52-6x5090-2026-07-12.md)); +- **128 GB CPU-only desktop:** ~1.8 tok/s warm ([#200](https://github.com/JustVugg/colibri/issues/200)); +- **single RTX 5070 Ti laptop-class box:** 1.07 tok/s via the GPU-resident + pipeline ([#273](https://github.com/JustVugg/colibri/issues/273)); +- **25 GB dev box:** 0.05–0.1 tok/s cold — the proven floor where this project + started, and still the honest baseline. + +Quality is measured, not assumed: the int4 container's quantization cost and the +scale-granularity/rotation ablations live in +[docs/benchmarks.md](docs/benchmarks.md#quality-benchmark) and +[#108](https://github.com/JustVugg/colibri/issues/108)/[#81](https://github.com/JustVugg/colibri/issues/81). + +## Get started + +> **New here?** The [Quick Start guide](docs/quickstart.md) walks through +> install → build → model → first chat step by step for Linux, Windows, and +> macOS, with copy-paste commands and no assumed background. + +### 1. Get the model + +A pre-converted **GLM-5.2 int4** container is on Hugging Face — **use the +version with the int8 MTP heads**: **https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** -> ⚠️ **The MTP head must be int8.** The original mirror ([jlnsrk/GLM-5.2-colibri-int4](https://huggingface.co/jlnsrk/GLM-5.2-colibri-int4)) ships **int4** MTP heads, which give **0% draft acceptance** — speculation silently never engages and you lose the ~2× MTP lever. This is the single most common "why is MTP stuck at 0%?" report ([#8](https://github.com/JustVugg/colibri/issues/8), [#102](https://github.com/JustVugg/colibri/issues/102)). The int8 head gives the measured **39–59% acceptance**. matey-0's clone above is the original int4 model with the three `out-mtp-*` files already swapped to int8 — download that one and you're done. -> -> Check what you have: `ls -l /out-mtp-*` -> · **int8 (correct):** `3527131672 / 5366238584 / 1065950496` -> · **int4 (0% acceptance):** `1765523544 / 2686077736 / 536747200` — if you see these, replace just those three files from the int8 mirror. +> ⚠️ The original mirror ships int4 MTP heads → 0% draft acceptance +> ([#8](https://github.com/JustVugg/colibri/issues/8)). Check yours: +> `ls -l /out-mtp-*` — int8 (correct) is `3527131672 / 5366238584 / 1065950496`. -Download the repository and point `COLI_MODEL` to its directory: +Or convert from the FP8 source yourself — one resumable command that never needs +the full 756 GB on disk at once: ```bash -COLI_MODEL=/path/to/GLM-5.2-colibri-int4-with-int8-mtp ./coli chat +cd c && ./setup.sh # checks gcc/OpenMP, builds, self-tests +./coli convert --model /nvme/glm52_i4 # download+convert shard by shard (python, one-time) ``` -This skips the FP8 → int4 conversion step entirely. Thanks to DatPat for the original mirror and matey-0 for the int8-head clone. - -### Quick start +### 2. Run it ```bash -cd c -./setup.sh # checks gcc/OpenMP, builds, self-tests - -# ONE command does everything model-side: downloads GLM-5.2-FP8 shard by shard -# (never needs the full 756 GB at once), converts to the int4 container, then -# converts the MTP head for speculative decoding. Resumable at any point. -# Conversion (only) needs python with: pip install torch safetensors huggingface_hub numpy -./coli convert --model /nvme/glm52_i4 # ~400 GB free on a real ext4/NVMe path - -# chat — RAM budget, expert cache and MTP are all detected automatically: -COLI_MODEL=/nvme/glm52_i4 ./coli chat +COLI_MODEL=/nvme/glm52_i4 ./coli chat # RAM budget, cache and MTP auto-detected +COLI_MODEL=/nvme/glm52_i4 ./coli plan # inspect the planned VRAM/RAM/disk placement +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # read-only readiness check +./coli web --model /nvme/glm52_i4 # API + web dashboard on one port +./coli serve --model /nvme/glm52_i4 # OpenAI-compatible API only ``` -Inspect the planned storage hierarchy before loading the model: +The engine at runtime is pure C — python is only used by the one-time converter +and the optional API gateway. -```bash -COLI_MODEL=/nvme/glm52_i4 ./coli plan -COLI_MODEL=/nvme/glm52_i4 ./coli plan --gpu 0,1 --ram 128 --vram 48 --json +Prefer a `coli` command on your PATH? From a checkout, `pip install -e .` +registers it (the engine itself still lives in `c/` — this is an editable +install from the clone, not a standalone wheel). -# apply the bounded plan to the normal runner -COLI_MODEL=/nvme/glm52_i4 ./coli chat --auto-tier -``` +### 3. Go deeper -`coli plan` reads only safetensors headers and reports the model's exact dense/expert -footprint, runtime RAM reserve, safe expert-cache cap, and bounded VRAM hot tier. Its -versioned JSON output is intended to be shared by the CLI, API server, Web UI, and -desktop shell; it does not allocate model tensors or start inference. -`--auto-tier` applies the same plan to `chat`, `run`, `serve`, and benchmarks. It -sets the RAM budget and context immediately; the VRAM tier is enabled only when -the current `glm` binary is linked with CUDA. Explicit flags and environment -variables keep precedence over automatic values. - -Before loading the model, `coli doctor` performs a read-only readiness check and -explains whether the selected Disk/RAM/VRAM placement is runnable: - -```bash -COLI_MODEL=/nvme/glm52_i4 ./coli doctor -COLI_MODEL=/nvme/glm52_i4 ./coli doctor --gpu 0 --ram 128 --json -``` - -Doctor validates the model directory, config, tokenizer, safetensors headers, -engine executable, available RAM, requested NVIDIA devices, CUDA linkage, and the -same placement budget used by `coli plan`. It never starts `glm`, reads tensor -payloads, imports a model framework, or creates a CUDA context. The versioned JSON -report uses stable check IDs for automation. Warnings keep exit status 0; missing -requirements or an unsafe RAM projection return 1, while invalid CLI values return 2. - -The engine at runtime is pure C — python is only used by the one-time converter. - -### Windows 11 (native, no WSL) - -colibrì builds and runs natively on Windows 11 x86-64 with MinGW-w64. The port adds -a `_WIN32` compatibility layer in `c/compat.h` that maps POSIX I/O to the Windows API -(pread → ReadFile+OVERLAPPED, posix_fadvise no-op, aligned allocation, MoveFileEx rename, -GlobalMemoryStatusEx RAM detection). All platform differences stay in `compat.h`; the -engine source is unchanged. - -**Toolchain:** GCC via [winlibs](https://winlibs.com/) or MSYS2 MinGW-w64. Tested with -GCC 16.1.0 (x86_64-ucrt-posix-seh). - -```powershell -# One-time toolchain install (pick one): -scoop install mingw-winlibs # portable, no shell needed -# or: pacman -S mingw-w64-x86_64-gcc make # via MSYS2 - -# Build (from c/ directory): -make glm.exe # GLM-5.2 engine (static, no DLL dependencies) -make olmoe.exe # OLMoE engine (same shims) -make iobench.exe # disk I/O benchmark -make test-c # run C tests -make test-python # run Python tests (requires python) - -# AVX-VNNI: Intel Alder Lake+ (and Meteor Lake+) CPUs have a 128-bit int8 -# dot-product instruction (VPDPBUSD) the engine can use for ~1.3x faster -# quantized matmul. The x86-64-v3 default (portable AVX2) compiles it out; -# build for THIS machine to enable it: -make glm.exe ARCH=native # banner prints "idot: avx-vnni" - -# Verify (tiny model, 2.4 MB): -pip install torch transformers safetensors huggingface_hub -python tools/make_glm_oracle.py # generate tiny oracle -SNAP=./glm_tiny TF=1 ./glm.exe 64 16 16 # expect "32/32 positions" - -# Run with real model: -SNAP=D:\glm52_i4 ./glm.exe 64 4 16 # batch inference -python coli chat --model D:\glm52_i4 # interactive chat -python coli serve --model D:\glm52_i4 # OpenAI-compatible API -``` - -**Warmup (overnight cache priming):** the engine's expert cache learns from -your workload. The included `warmup.ps1` script runs `coli run` in a loop with -diverse prompts to build the `.coli_usage` histogram unattended, so the next -real session starts with a large, accurate hot-expert pin. Each run saves usage -atomically on clean completion. - -```powershell -.\warmup.ps1 -Rounds 1 -Ngen 32 # ~60-90 min, durable progress -``` - -**NVIDIA GPU (optional, via runtime DLL):** on Windows the engine is built with -MinGW gcc but CUDA kernels require MSVC + nvcc. The split is clean: build the -CUDA backend into a standalone `coli_cuda.dll` (nvcc + MSVC), then the host -`glm.exe` loads it at runtime via `LoadLibrary` (`c/backend_loader.c`). The host -never links cudart directly; if the DLL is absent the engine falls back to CPU -without error. - -```powershell -# Prerequisites: CUDA Toolkit + MSVC Build Tools (cl.exe) + nvcc on PATH. -# Build the DLL from a shell with the MSVC environment set (vcvars64.bat or -# "x64 Native Tools Command Prompt for VS"): -make cuda-dll CUDA_HOME="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" CUDA_ARCH=sm_120 - -# Build the host with the runtime loader (CUDA_DLL=1 adds -DCOLI_CUDA and -# links backend_loader.o instead of cudart): -make glm.exe CUDA_DLL=1 ARCH=native - -# Run with the GPU expert tier (8 GB VRAM budget here; scale to your free VRAM): -$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_EXPERT_GB="8" -python coli chat --model D:\glm52_i4 --topp 0.7 -``` - -The DLL exports 11 `extern "C"` symbols (`coli_cuda_init`, `coli_cuda_matmul`, -etc.); `backend_loader.c` resolves them via `GetProcAddress` on first use. -`ColiCudaTensor*` is opaque to the host (stored, never dereferenced), so the -MSVC-allocated struct is safe across the ABI boundary. `CUDA_ARCH` must match -your GPU's compute capability (e.g. `sm_120` for Blackwell / RTX 50-series, -`sm_89` for Ada / RTX 40-series). - -**Status:** Phase 1 complete (compiles, correct, static-linked). The Windows -GPU tier (runtime `coli_cuda.dll` via `LoadLibrary`) is implemented and -verified on RTX 50-series (sm_120). O_DIRECT (Phase 2) and full-model -validation against the transformers oracle remain separate workstreams. - -### OpenAI-compatible API - -`coli serve` keeps one model process loaded and exposes a text-only OpenAI-compatible -HTTP API. The gateway uses only the Python standard library; inference still runs in -the same dependency-free C engine. - -```bash -cd c -COLI_MODEL=/nvme/glm52_i4 COLI_API_KEY=local-secret ./coli serve \ - --host 127.0.0.1 --port 8000 --model-id glm-5.2-colibri - -curl http://127.0.0.1:8000/v1/chat/completions \ - -H 'Authorization: Bearer local-secret' \ - -H 'Content-Type: application/json' \ - -d '{ - "model": "glm-5.2-colibri", - "messages": [{"role": "user", "content": "Hello"}], - "stream": true - }' -``` - -Implemented endpoints are `GET /v1/models`, `GET /v1/models/{model}`, -`POST /v1/chat/completions`, and legacy `POST /v1/completions`. Chat and -completion requests support JSON responses, SSE streaming, usage counts, -`max_tokens`/`max_completion_tokens`, `temperature`, and `top_p`. The extension -`enable_thinking: true` enables GLM-5.2's reasoning block; the standard -`reasoning_effort` field also enables it unless set to `none`. - -The first version is deliberately text-only and serves one generation at a time: -the 744B model stays in one persistent process, so concurrent HTTP requests queue -instead of loading duplicate model copies. Tools, image/audio input, custom stop -sequences, log probabilities, and token penalties return an explicit error rather -than being silently ignored. The default bind address is localhost; set -`COLI_API_KEY` before exposing the server beyond the machine. - -Browser access from the Vite development server and Tauri local origins is enabled -by default. Repeat `--cors-origin https://your-ui.example` to allow another exact -origin, or use `--cors-origin '*'` only on a trusted local network. - -The engine owns one mutable KV context, so HTTP generation uses a bounded FIFO -admission queue instead of pretending to run unsafe parallel sequences. Configure it -with `--max-queue N` (default 8) and `--queue-timeout SECONDS` (default 300), or the -`COLI_MAX_QUEUE` / `COLI_QUEUE_TIMEOUT` environment variables. Saturated and timed-out -requests receive OpenAI-shaped HTTP 429 errors before streaming headers are sent. -`GET /health` exposes active/queued/completed/rejected counters, and successful -generation responses include `x-colibri-queue-wait-ms`. - -### Isolated KV contexts - -`coli serve --kv-slots N` allocates up to 16 independent sequence contexts. Requests -select one with the optional integer `cache_slot` field; ordinary OpenAI clients omit -it and keep the original slot 0 behavior. - -```json -{ - "model": "glm-5.2-colibri", - "messages": [{"role": "user", "content": "Continue this conversation"}], - "cache_slot": 1 -} -``` - -Each slot owns its token history, compressed MLA/DSA KV memory, MTP window, and -crash-safe persistence file (`.coli_kv`, `.coli_kv.1`, ...). The engine still executes -one sequence at a time; this establishes explicit KV ownership without pretending that -threaded HTTP is continuous batching. RAM admission accounts for every configured slot. -Use `COLI_KV_SLOTS=N` as the environment equivalent. Start with a small value: at the -default 4096-token context, every slot costs hundreds of MB. - -### Experimental Metal backend (Apple Silicon) - -On Apple Silicon the decode profile is matmul-bound, and unified memory removes the -PCIe copy tax that keeps CUDA's streaming experts on the CPU — so colibrì has an -opt-in Metal backend that runs the **routed-expert SwiGLU (batched, zero-copy from -the RAM slabs)**, the **fused decode attention** (full MLA layer in one command -buffer, S≤4), and **prefill's large GEMMs** on the GPU. Token-exact vs the CPU path. - -```bash -cd c -make glm METAL=1 # macOS only; no Xcode needed (shader compiles at runtime) -make metal-test # standalone kernel/attention correctness vs CPU reference -COLI_METAL=1 COLI_MODEL=/path/glm52_i4 ./coli chat --ram 96 -``` - -Measured on an M4 Max (128 GB, warm cache, MTP on): CPU 0.30 → Metal **0.42 tok/s (~1.4×)** -(best config adds `DIRECT=1`; ~3× vs this machine's first cold run). -Key design points: Metal's ~5 ms submit latency makes per-matmul dispatch a loss — -everything is batched into few command buffers per layer, and the resident experts' -GPU work is submitted *before* the missed experts' disk reads so I/O and compute -overlap. `COLI_METAL_GEMM_MIN` tunes the prefill GEMM row threshold (default 16). -Streaming, cache, MTP, DSA and the persistence formats are unchanged; every GPU -path falls back to the CPU per-block on any fault. Numerics are dequant→f32-MAC -(same as the CUDA tier); greedy outputs are byte-identical to the CPU engine. - -### Experimental resident CUDA backend - -colibrì includes an opt-in CUDA backend for model-resident tensors. Streaming -experts deliberately remain on the original CPU path for now: copying an expert -from NVMe to the GPU on every use would only replace the disk bottleneck with a -PCIe bottleneck. Resident quantized tensors are uploaded lazily once and reused. - -```bash -cd c -make cuda-test CUDA=1 # q8/q4/q2/f32 kernel correctness -make CUDA=1 -# optional dense-path experiment (hot experts are configured below) -COLI_CUDA=1 COLI_GPU=0 CUDA_DENSE=1 SNAP=/nvme/glm52_i4 ./glm 64 4 4 -``` - -Requirements: Linux, an NVIDIA driver, and a CUDA Toolkit under -`/usr/local/cuda` (override with `CUDA_HOME=/path/to/cuda`). `CUDA_ARCH=native` -builds for the GPU in the current machine; set an explicit architecture when -cross-compiling. Requesting CUDA with a CPU-only binary, an invalid device, or -an unavailable runtime fails at startup instead of silently falling back. - -The normal `make` build and runtime behavior are unchanged. CUDA defaults to an -expert-only accelerator. `CUDA_DENSE=1` additionally distributes resident -dense/attention projection tensors round-robin across the selected devices; -their projected footprint is reserved before the expert tier is placed. On six -RTX 5090s with a 150 GB expert tier, a warmed two-request/64-token GLM-5.2 run -improved from 1.650 to 2.157 aggregate tok/s (+30.8%) while retaining the full -expert tier. Treat this as an opt-in until the projected dense set and the 2 GB -per-device runtime reserve fit the target GPUs. -A measured `PIN` profile can promote its hottest experts into the persistent -VRAM tier while keeping the rest in RAM: - -```bash -STATS=stats.txt SNAP=/nvme/glm52_i4 ./glm 64 4 4 # collect routing frequencies first -COLI_CUDA=1 COLI_GPU=0 CUDA_EXPERT_GB=16 \ -PIN=stats.txt PIN_GB=160 SNAP=/nvme/glm52_i4 ./glm 64 4 4 -# multi-GPU expert tier, 150 GB total budget across six 32 GB devices -COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=150 \ -CUDA_DENSE=1 PIN=stats.txt PIN_GB=300 RAM_GB=226 \ -SNAP=/nvme/glm52_i4 ./glm 64 4 4 -# large-RAM host: fill safe VRAM, then keep every remaining expert in RAM -COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=auto \ -CUDA_DENSE=1 COLI_CUDA_ATTN=1 PIN=stats.txt PIN_GB=all RAM_GB=auto \ -SNAP=/nvme/glm52_i4 ./glm 64 4 4 -``` - -Selected experts are uploaded during startup, so capacity failures occur before -inference and the log reports their exact tensor footprint. The budget is clamped -against free VRAM after reserving the projected dense resident set and 2 GB of -runtime headroom per selected device. With `COLI_GPUS`, `CUDA_EXPERT_GB` is a -total budget across the device set; experts are assigned whole to the -least-loaded device that can hold them. Multi-GPU runs also default to -`PIN_FILL=1`: the measured hot set is placed first, then unused VRAM is filled -with zero-heat experts. `CUDA_RELEASE_HOST=1` (the multi-GPU default) releases -the RAM copy after a successful upload and reloads it from disk only if CUDA -later fails. Set either variable to `0` to restore the conservative behavior. -When host backing is released, placement is disjoint and staged: the hottest -prefix is loaded, uploaded to VRAM, and freed before the next-ranked suffix is -loaded into RAM. `PIN_GB` therefore describes the combined ranked set rather -than duplicate RAM and VRAM copies. On a 256 GB dual-socket host, moving from a -150 GB VRAM + 130 GB RAM placement to 150 GB VRAM + 150 GB RAM raised fixed-token -replay from 1.87 to 2.16 tok/s (+15.7%), reduced expert disk wait from 5.144s to -3.948s, and kept the projected RAM peak below `RAM_GB=226`. The cache cap adjusts -down automatically (54 to 40 in that run) so the larger pinned tier does not exceed -the process budget. Start lower on hosts with less available RAM. - -`CUDA_EXPERT_GB=auto` fills each selected device only up to its measured free -memory minus projected dense tensors and 2 GB of runtime headroom. `PIN_GB=all` -then loads every remaining routed expert into RAM, eliminating decode-time disk -misses when the host budget permits it. The regular `RAM_GB` guard still clamps -the per-layer working cache and rejects unsafe projections; this mode is intended -for dedicated high-memory inference hosts, not desktops running other workloads. -On a dedicated 251 GiB host with six RTX 5090s, this mode selected a 176.7 GB -VRAM expert tier and a 191.3 GB RAM tier (all 19,456 experts resident). The -mode also adapts the VRAM tier every 16 emitted tokens by swapping hot RAM -experts into existing GPU slots. A real 64-token greedy GLM-5.2 generation -measured **6.00 tok/s decode**, up from -2.20 tok/s end-to-end with the earlier 150 GB tier; expert hit rate was 100% -and disk wait was zero. Prompt prefill is reported separately. This is a -host-specific capacity result, not a portable default. - -Text-mode timing reports prefill separately from decode. The decode rate starts -after the prompt KV is built, so it is comparable to `REPLAY` throughput without -hiding time-to-first-token. -MTP speculation defaults off on CUDA because cold draft routes increase expert -traffic; an explicit `DRAFT=n` still overrides the default. - -On six RTX 5090 32 GB cards with GLM-5.2 int4, a 150 GB hot-first tier sustained -0.94 token/s over a 64-token varied prompt (87.8% expert hit rate), and reached -1.64 token/s on a warmed short prompt (99.3% hit rate). The same capacity filled -without routing heat managed only 0.29 token/s, so profile quality matters more -than raw VRAM capacity. These are single-run engineering measurements, not a -portable performance guarantee. - -Current limitations: devices use independent contexts and synchronous -host-staged activation copies—there is no P2P/NCCL dependency yet. Independent -expert groups execute concurrently across devices, but a single expert is not -sharded. The kernels are correctness-first custom kernels rather than -cuBLAS/Tensor Core kernels. - -For a reproducible backend A/B without the full checkpoint, generate the -deterministic 313M-parameter `glm_moe_dsa` fixture and run fixed-token replay: - -```bash -cd c -python tools/make_glm_bench_model.py --output /nvme/colibri-bench-medium --device cuda -python tools/benchmark_cuda_fixture.py --model /nvme/colibri-bench-medium --gpu 0 -``` - -The fixture has random weights and is not a language model. It exists only to -preserve the real MLA/MoE/streaming shapes and compare CPU streaming, dense-only -CUDA, CPU hot-store, and CUDA hot-expert execution with identical replay tokens. - -### Web interface - -`web/` contains a community-contributed browser UI (React + TypeScript, a pure -API client — it never touches the engine directly): - -```bash -cd web -npm ci && npm run dev # then point it at an OpenAI-compatible endpoint -``` - -It speaks the standard OpenAI Chat Completions protocol with SSE streaming, so it -works against the colibrì OpenAI-compatible server (in review, #21) or any other -compatible endpoint. Nothing leaves the endpoint you configure. The terminal -`coli chat` remains the first-class interface. - -Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens per answer (`:more` in chat continues a truncated one), `--repin N` adapt RAM/VRAM hot experts every N emitted tokens, `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `GRAMMAR=g.gbnf` grammar-forced drafts for constrained JSON/NDJSON output (`GRAMMAR_DRAFT=n` caps the forced span), `TF=1` teacher-forcing validation, `PILOT=1` router-lookahead disk prefetch (experimental — see below), `URING=1` Linux-only batched expert I/O (implies `PIPE=1`; also batches `PILOT_REAL`), `PIPE=0` disable the async expert-load pool (**default ON on Windows** — overlaps expert `pread` with the matmul so the CPU isn't idle waiting on the SSD; measured −18% disk service time), `RAM_GB=` claim more RAM for the expert cache than the conservative auto-detect (e.g. `RAM_GB=31` on a 32 GB host raises the cache cap and hit rate measurably), `CAP_RAISE=0` don't auto-grow the expert cache. - -### Resource policy - -`coli plan` reports the planned hot (VRAM), warm (RAM), and cold backing -(disk) tiers, the reason for each placement, and the expected bottleneck. The -default `--policy quality` and `--policy balanced` modes preserve checkpoint -quantization and router decisions unless `--topk` or `--topp` is passed; those -explicit lossy overrides print a warning and proceed. - -Auto-tier plans size OpenMP from physical cores and bind workers across cores. -Memory-bound quantized kernels can regress sharply when SMT siblings compete -for limited memory channels; explicit `OMP_*` settings always take precedence. - -```bash -coli plan --model /models/glm52_i4 --policy quality -coli run --auto-tier --policy quality "Explain MoE offloading" -# Explicit research-only router reduction: -coli run --policy experimental-fast --topk 4 "Benchmark prompt" -``` - -Disk is an immutable recovery source, not a normal decode target. If the plan -leaves cold expert bytes on disk, speed depends on cache hit rate; output -quality does not. - -Cold expert reads can use a deferred pipeline: resident RAM/VRAM experts execute -while missing experts are loaded in a bounded background I/O pool, then the -cold results join before the layer completes. The pool engages only under -`PIPE=1`; `PIPE_WORKERS=n` sets its worker count (default 8). Profiling reports -both disk service time and the smaller foreground-visible wait time so overlap -is explicit rather than credited as unexplained speedup. - -`--policy balanced` enables lossless live placement (`REPIN=64`). At safe -request boundaries, a per-layer LFRU score combines decaying session frequency -with recent access and replaces at most four sufficiently colder pinned -experts. `--policy quality` leaves live replacement off by default; `REPIN=0` -always disables it. Persistent `.coli_usage` history and session-local LFRU -state remain separate. - -For single-token q4 CPU experts, gate and up projections share one OpenMP -dispatch while retaining the same per-row AVX2/NEON arithmetic. This removes -one thread-team launch per RAM expert without activation requantization or a -lower-precision fallback. It is a stepping stone toward a persistent native -CPU expert pool, not a replacement for one. - -**The expert cache auto-sizes to your RAM** (since 2026-07-10): the engine now *raises* the LRU cap to fill your `--ram` budget instead of only lowering it. Before this fix a 128 GB machine ran with the same 8-experts/layer cache as a 16 GB one (issue #12) — **if you benchmarked colibrì before this date, rerun: your numbers were capped.** - -**Router-lookahead prefetch** (`PILOT=1`, experimental): GLM-5.2's expert routing is measurably predictable *ahead of time* — applying layer L+1's router to layer L's post-attention state recalls **71.6%** of the true top-8 (vs 41.3% for "same experts as last token"). `PILOT=1` uses this to issue next-layer expert readahead from a dedicated I/O thread while the current layer computes. On our dev box the disk is already ~80% saturated, so it measures neutral; on machines where compute and disk are balanced (like the Ryzen AI 9 in issue #12: 43% disk / 46% matmul) it should overlap real work — measurements welcome. - -**The learning cache**: the engine records which experts your usage actually routes to (`.coli_usage` next to the model, updated every turn) and at startup automatically pins the hottest ones in spare RAM. colibrì literally gets faster the more you use it. - -**Live tier adaptation** (`--repin N`, opt-in): at safe turn boundaries, a decaying -session heat map replaces cold pinned experts with hotter streamed experts. Replacement -loads the expert from disk into the existing RAM slot; GPU-backed slots immediately -refresh the same VRAM tier budget. A 25% hysteresis and a four-swap limit prevent tier -thrashing. Persistent `.coli_usage` remains the long-term signal and is not decayed. - -**Conversations reopen warm** (`.coli_kv`, since 2026-07-10): `coli chat` persists the compressed MLA KV-cache to disk after every turn (~182 KB/token, appended incrementally, crash-safe). Close the chat, reopen it tomorrow — the model still remembers the whole conversation and **zero re-prefill happens**: validated byte-identical to an uninterrupted session. `:reset` clears it, `KVSAVE=0` disables it. - -## Web dashboard - -One command serves the OpenAI-compatible API **and** the web console on the same port, then opens your browser when the engine is ready: - -```bash -cd web && npm install && npm run build # once -./coli web --model -``` - -What you get: - -- **Chat** with live metrics: a flashing token counter while generating, then tok/s, time-to-first-token, prompt→completion counts and queue wait; -- **Runtime panel**: your hardware (CPU, GPUs + VRAM, RAM, cores), the scheduler, and the live expert-tier bar — how many of the 19,456 experts sit in VRAM / RAM / disk right now; -- **Brain**: the whole model as a 76×256 cortex, one cell per expert. Colour = tier, brightness = routing heat, and the experts routed in each turn flash white and decay — you watch the model think. Hover any cell for its tier, heat and [measured topic affinity](https://github.com/JustVugg/colibri/issues/175) (specialists for code, Chinese, math, law… live in layers 11–22). - -The dashboard talks to the engine over two tiny protocol lines (`TIERS`, `EMAP`/`HITS`) and plain JSON endpoints — nothing heavier than the engine itself. - -## Got a better machine? Try it — here's what to expect - -colibrì was built on deliberately humble hardware (12 cores, 25 GB RAM, an older DRAM-less NVMe behind a WSL2 VHDX that measured ~1 GB/s random on *this* drive — note WSL2 VHDX is not inherently slow: a community 5090 box measured 10.5 GB/s O_DIRECT through one, [#101](https://github.com/JustVugg/colibri/issues/101)). **Every one of those constraints is a knob your machine can turn up.** The engine needs: Linux (or WSL2), macOS, or **Windows 11 natively (MinGW-w64)**; gcc with OpenMP, AVX2, ≥16 GB RAM, and the ~370 GB int4 model on a local NVMe (ext4/NTFS — never a network/9p mount). - -**How to test it, in order:** - -```bash -cd c && ./setup.sh # build + architecture self-test (expects 32/32) - -# 1) measure YOUR disk the way the engine uses it (parallel 19 MB random reads): -gcc -O2 -fopenmp iobench.c -o iobench -./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 0 # buffered, 8 threads -./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 1 # O_DIRECT (bypass cache) -# Caveat (#86): iobench reads a bounded ~1 GB shard, so buffered reads on a big-RAM box -# report the PAGE CACHE, not the disk. Use the O_DIRECT run (arg 1) for a true number, and -# run it on a shard you haven't touched this session (a prior buffered run caches its pages). -# On macOS there is no O_DIRECT — iobench uses F_NOCACHE, which stops *new* caching but can't -# evict pages a prior buffered run already resident-mapped, so a macOS "O_DIRECT" figure right -# after a buffered run still reads cache. Reboot or use a fresh shard for a real cold read. - -# 2) chat; watch the per-turn stats line (tok/s, expert hit-rate, RSS): -COLI_MODEL=/path/to/glm52_i4 ./coli chat - -# 3) record expert usage, then pin the hottest experts in your spare RAM: -STATS=stats.txt ./coli chat -PIN=stats.txt PIN_GB=20 ./coli chat # scale PIN_GB to your free RAM - -# 4) quality benchmarks (MMLU/HellaSwag/ARC): -./coli bench -``` - -**Back-of-envelope predictions** (decode is disk-bound: a cold token costs ~11.4 GB of expert reads; MTP speculation roughly halves the effective cost *once the cache is warm*; RAM turns cold reads into free cache hits): - -| machine | expected | +| topic | doc | |---|---| -| this dev box (WSL2 VHDX, ~1 GB/s, 25 GB RAM) | ~0.05–0.1 tok/s cold — proven baseline | -| native Linux, PCIe4 NVMe (~3–5 GB/s random), 32 GB | ~0.5–1 tok/s | -| PCIe5 NVMe or 2×NVMe RAID0 (~8–12 GB/s), 64 GB (PIN ~40 GB of hot experts) | ~2–4 tok/s | -| 128–256 GB RAM, 12 cores (hot experts cached) | ~2–4 tok/s — matmul-bound: ~80 GFLOP/token vs ~250 GFLOP/s of our AVX2 kernels | -| same RAM + 24–32 cores, or AVX-512/VNNI kernels | ~5–15 tok/s — interactive; kernel work is the multiplier | - -These are estimates, not measurements — if you run colibrì on serious hardware, **please open an issue with your numbers**: real datapoints from better machines are exactly what this project needs next. - -### Community benchmarks (measured) - -Real numbers from real machines, stock build (`setup.sh`, gcc 13), greedy decoding, `--ngen 32`, MTP active: - -| machine | disk (iobench, 19 MB × 64, 8 threads) | config | measured | -|---|---|---|---| -| Intel Core Ultra 7 270K Plus (24 threads) · WSL2 · 24 GB RAM · NVMe VHDX ([#2](https://github.com/JustVugg/colibri/issues/2)) | 1.96 GB/s buffered · 2.74 GB/s O_DIRECT | default | 0.07 tok/s · expert hit 3–4% · RSS 14.1 GB | -| 〃 | 〃 | `--topp 0.7` | **0.11 tok/s** · expert hit 11% · RSS 14.7 GB | -| Apple M5 Max (18 cores) · macOS · 128 GB unified · internal SSD ([#4](https://github.com/JustVugg/colibri/issues/4), [#5](https://github.com/JustVugg/colibri/issues/5)) | ~4 GB/s cold (the 14.2 GB/s reading was cache-influenced — see note) | default, MTP off | **1.06 tok/s** · expert hit 23% · RSS 21.8 GB | -| Apple M5 Max · macOS · 128 GB unified · 2 TB SSD · **Metal backend** ([#72](https://github.com/JustVugg/colibri/pull/72), [#87](https://github.com/JustVugg/colibri/issues/87)) | (macOS O_DIRECT figure unreliable — see note) | Metal on · `--ram 96` · 39.7 GB warm pin · MTP off | **1.83 tok/s** · expert hit 66% · warmed 1.11 → 1.83 over the run | -| 〃 · 46.9 GB pin (2.94M-selection history) · `--ram 110`, 1024-token run ([#103](https://github.com/JustVugg/colibri/issues/103)) | 〃 | Metal on (experts + attention) · MTP off | **2.06 tok/s** · hit 72.5% · coherent output · fastest datapoint yet (still on the pre-rebase Metal branch) | -| Mac Mini M4 Pro · macOS · **48 GB** unified · **Metal backend** ([#107](https://github.com/JustVugg/colibri/issues/107)) | 6.59 GB/s F_NOCACHE (fresh shard) | Metal on · `--ram 38` | **0.30 tok/s** (vs 0.18 CPU-only) — entry Apple Silicon on a third the RAM beats the 32-core 9950X row | -| Epyc 9654 ES · Linux · 4x16GB DDR5-4800-rdimm · Samsung PCIe Gen3 x4 NVME SSD | — | `MTP=1 DIRECT=1` | 0.31 tok/s · expert hit 35% · RSS 21.52 GB | -| Ryzen AI 9 HX 370 (Framework 13) · Arch Linux · 128 GB · WD SN850X, BTRFS zstd ([#12](https://github.com/JustVugg/colibri/issues/12)) | — | int8 MTP head · `--cap 32` · 46.7 GB auto-learned PIN | **0.37 tok/s** · expert hit 66% · MTP acceptance 52% (2.59 tok/fw) · RSS 105 GB | -| Ryzen 9 9950X (32 threads) · Linux · 123 GB · Crucial P3 QLC Gen3 ([#31](https://github.com/JustVugg/colibri/issues/31)) | 1.51 GB/s buffered | default, 2 runs from cold | 0.10 tok/s · hit 53% · profile 66% disk | -| 〃 same machine, model moved to a Samsung 9100 PRO PCIe 5.0 ([#31](https://github.com/JustVugg/colibri/issues/31)) | **8.81 GB/s** O_DIRECT | 〃 (usage history retained) | **0.28 tok/s** · hit 57% · profile flips: 32% disk / **57% matmul** | -| Ryzen AI Max+ 395 (Framework Desktop) · Ubuntu · 128 GB LPDDR5x · Intel Optane 905p PCIe 3.0 ([#39](https://github.com/JustVugg/colibri/issues/39)) | 3.27 GB/s buffered | int8 MTP head · fresh history (pure LRU, auto-raised cap 65) | 0.16 tok/s · hit 57% · profile 49% disk / 47% matmul | -| 〃 five runs later — learned pin 47.6 GB ([#39](https://github.com/JustVugg/colibri/issues/39)) | 〃 | `--temp 0.7 --topp 0.7` | **0.40 tok/s** · hit 71% · fastest non-Apple datapoint | -| Ryzen 7 9800X3D (16T) · WSL2 · 70 GB RAM · Samsung 9100 PRO PCIe 5.0 · RTX 5090 ([#101](https://github.com/JustVugg/colibri/issues/101)) | **10.51 GB/s** O_DIRECT | MTP off · learned pin 24 GB · hit 54% · OMP hot-team on | **0.41 tok/s** · disk-bound (36.5 s disk vs 24.0 s matmul) · **CUDA expert tier ≈ 0%** (AVX-512 CPU matches the 5090) · `--topp 0.7` → **0.52 tok/s** | -| EPYC 7443 (24C/48T, Zen3 AVX2) · Linux · **430 GB RAM** · NVMe RAID-Z1 via TrueNAS VM ([#104](https://github.com/JustVugg/colibri/issues/104)) | ~1 GB/s (VM overhead) | 77.5 GB pin · cap auto-raised to 194/layer · MTP off | **1.00 tok/s** · **hit 98%** · disk eliminated → **RAM-bandwidth + matmul bound** (no AVX-512/VNNI on Zen3) | -| Intel i5-12600K (10C/16T, AVX2) · **native Windows 11, no WSL** · 32 GB · MinGW GCC 16.1 ([#113](https://github.com/JustVugg/colibri/issues/113)) | buffered (no O_DIRECT on MinGW) | int8 MTP head · cold, small-RAM (cap ~2/layer) | **0.08 tok/s** · hit 3.7% · **MTP 57% acceptance** — first native-Windows datapoint, port validated | -| Ryzen 9 9950X3D2 (16C/32T, avx512-vnni) · native Linux · 121 GB · Samsung 9100 PRO **PCIe Gen5** · RTX 5090 (28 GB expert tier, 1475 pinned) ([#120](https://github.com/JustVugg/colibri/issues/120)) | **11.48 GB/s** O_DIRECT | `MTP=0 DIRECT=1 PIPE_WORKERS=16 PREFETCH=1` | **1.23 tok/s** · MTP-off wins disk-bound · fastest x86 datapoint yet | -| Ryzen AI Max+ 395 (Strix Halo, 16C/32T Zen5, avx512-vnni) · Arch Linux · 128 GB unified LPDDR5x · SK hynix P41 PCIe 4.0 ([#124](https://github.com/JustVugg/colibri/issues/124)) | — | `DIRECT=1 PIPE=1 --topp 0.7` · auto-pin | 0.06 cold → **1.10 tok/s** sustained · first Strix Halo / gfx1151 datapoint (unified memory: no discrete VRAM tier) | -| Intel Core Ultra 9 185H (16C/22T, avx-vnni) · **native Windows 11, no WSL** · 32 GB · Crucial P3 QLC NTFS · RTX 5070 Ti (unused) ([#128](https://github.com/JustVugg/colibri/issues/128)) | — | int8 MTP head · **with [#131](https://github.com/JustVugg/colibri/pull/131) (pipe + RAM fixes), warm cache, no GPU** | 0.03 cold → **0.5 tok/s** warm (~7-prompt warmup) · cache-warming on native Windows once the portability blockers are fixed — stock main hung on the `\r\n` READY sentinel before #131 | -| Dell Pro Max GB10 (DGX Spark: Grace 10×X925 + 10×A725, **aarch64 i8mm/sve2**) · Linux · 121 GB unified LPDDR5x · Dell OEM 4 TB NVMe · GB10 sm_121 ([#136](https://github.com/JustVugg/colibri/issues/136)) | **5.58 GB/s** O_DIRECT (NVIDIA-OEM unit in #76 was 10.74 — same platform, different SSD) | int8 MTP head · warm cache | 0.21 cold → **0.50 tok/s** warm · hit 83% · MTP 73% (3.20 tok/fw) · **matmul-bound** (matmul 130 s vs disk 58 s) — unified memory, CUDA placement tier neutral; the lever here is an i8mm compute kernel, not placement | - -Takeaways: with 24 GB of RAM the engine auto-caps the expert cache to 2 slots/layer, so decode stays cold even on a disk 2–2.7× faster than the dev box — **on small-RAM machines the RAM cap, not the disk, is the binding constraint**, exactly as the table above predicts; `--topp 0.7` alone bought a clean 1.6× end-to-end speedup. The M5 Max datapoint lands right on the table's second row: **~1 tok/s of a 744B model on a laptop SSD** — and its 14 GB/s disk shifts the bottleneck back to RAM budget and kernels. The Framework 13 rows are the cache thesis proven end-to-end on one machine: 0.29 → 0.37 tok/s (hit 28% → 66%, speculation finally engaging at 52% acceptance) just by giving the cache its RAM — int8 MTP head + a bigger cap + the learned pin. The cap part is now automatic (cap auto-raise, 2026-07-10). The 9950X pair is the cleanest bottleneck experiment yet — same machine, same history, only the disk swapped: ×5.8 disk bandwidth bought ×2.9 tokens, and the profile **flipped from 66% disk to 57% matmul**. But the crossover depends on the CPU kernel: the 9800X3D row ([#101](https://github.com/JustVugg/colibri/issues/101)) shows that with the OMP hot-team tuning on, the AVX-512 CPU matmul is fast enough that even a **10 GB/s NVMe stays disk-bound** — and there the **CUDA expert tier buys ≈ 0%**, because the CPU already matches the 5090 on expert matmul. The GPU tier earns its VRAM only when the CPU is the weak link, not by default. (Honest correction from #101: an earlier version of that report ran with the OMP tuning off, which manufactured a false matmul-bound crossover and a false +14% for CUDA — neither survived a clean re-run.) - -## Quality benchmark — help wanted - -**First measurement is in** ([#108](https://github.com/JustVugg/colibri/issues/108), thanks dnnspaul): the int4 container scored **62.5% mean acc_norm** on hellaswag/arc/mmlu (0-shot log-likelihood, n=40) — below the 85–95% published for full-precision GLM-5.2, but **the gap is not yet attributable to quantization.** Two confounds sit in the way: (1) 0-shot log-likelihood MC scoring badly underserves a *reasoning* model like GLM-5.2 (it never gets to think), so a large gap is expected even at fp16; (2) n=40 is ±14pp. The **decisive experiment** is the OLMoE fp16-vs-int4 A/B under this same harness (small enough to run both precisions) — that delta *is* the quantization cost with the scoring protocol cancelled out. Until it's run, 62.5% is a datapoint, not a verdict. - -The code is here and ready; one command runs it end to end (it auto-downloads the datasets on first use): - -```bash -cd c -pip install tokenizers datasets # in addition to the convert deps above -./coli bench # hellaswag, arc_challenge, mmlu — 40 questions each -./coli bench hellaswag --limit 200 # one task, more questions -./coli bench mmlu arc_challenge --ram 100 # pick tasks, set a RAM budget -``` - -It prints per-task accuracy (log-likelihood scoring, EleutherAI-harness style). **If you can run the OLMoE fp16-vs-int4 A/B (or a large-n GLM run), please open an issue with the numbers** — it's the measurement that turns 62.5% into either "int4 is fine, scoring artifact" or "quantization is the ceiling, grouped-scale is the priority." +| Benchmarks, community datapoints, quality measurements | [docs/benchmarks.md](docs/benchmarks.md) | +| Tuning knobs, policies, the learning cache, prefetch | [docs/tuning.md](docs/tuning.md) | +| Windows 11 native build (+ CUDA DLL) | [docs/windows.md](docs/windows.md) | +| CUDA backend, VRAM expert tier, full residency | [docs/cuda.md](docs/cuda.md) | +| Apple Silicon Metal backend | [docs/metal.md](docs/metal.md) | +| OpenAI-compatible API, KV slots, web dashboard | [docs/api.md](docs/api.md) | +| Grammar-forced drafts (structured output) | [docs/grammar-draft.md](docs/grammar-draft.md) | +| Environment variable inventory | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | ## Supporting the project -colibrì is a one-person project, written and tested entirely on a 12-core laptop with 25 GB of RAM — the numbers above are the ceiling of what I can measure at home. If this project is useful or interesting to you and you'd like to support its development (better test hardware translates *directly* into a faster engine for everyone: real NVMe scaling data, bigger pinned caches, int2/int3 quality sweeps on real benchmarks), you can: +colibrì started as a one-person project on a 12-core laptop with 25 GB of RAM; +today its numbers come from a community of real machines. If it's useful to you: - ⭐ star the repo and share it; -- 🐛 open issues with benchmark numbers from your hardware; -- 💬 reach out via GitHub issues if you'd like to sponsor development or donate hardware. - -Every contribution, from a datapoint to a disk, moves the ceiling. +- 🐛 open issues with benchmark numbers from your hardware — datapoints move + this project more than anything else; +- 💬 reach out via GitHub issues to sponsor development or donate hardware. ## Repo layout @@ -656,20 +229,20 @@ c/ ├── tools/ offline conversion, fixtures and benchmarks ├── scripts/ long-running conversion helpers └── tests/ dependency-free C and Python tests -web/ browser UI (pure OpenAI-API client, community-maintained) +web/ browser UI (pure OpenAI-API client) desktop/ Tauri v2 desktop shell wrapping the web UI +docs/ reference docs, experiments, media ``` The runtime path intentionally stays flat and readable: `glm.c` plus its small -headers. Auxiliary Python and shell tooling is grouped separately and is never a -runtime dependency of the engine. - -From the repository root, `make`, `make check`, and `make clean` delegate to the -engine Makefile. Existing commands run from `c/` continue to work unchanged. +headers. From the repository root, `make`, `make check`, and `make clean` +delegate to the engine Makefile. ## Why "colibrì" -The hummingbird weighs a few grams, hovers in place, and visits a thousand flowers a day. This engine keeps a 744-billion-parameter giant alive on hummingbird rations: 25 GB of RAM, twelve CPU cores, and a lot of disk patience. +The hummingbird weighs a few grams, hovers in place, and visits a thousand +flowers a day. This engine keeps a 744-billion-parameter giant alive on +hummingbird rations: 25 GB of RAM, twelve CPU cores, and a lot of disk patience. ## License diff --git a/README.zh-CN.md b/README.zh-CN.md new file mode 100644 index 0000000..352de33 --- /dev/null +++ b/README.zh-CN.md @@ -0,0 +1,237 @@ +

+ colibrì——小巧引擎，庞大模型 +

+ +

+ English · 简体中文 · 繁體中文 · Italiano +

+ +**小巧引擎，庞大模型。**只需约 25 GB 内存，就能在消费级电脑上运行 **GLM-5.2（744B 参数的 MoE）**——以零依赖的纯 C 实现，从磁盘流式加载专家。 + +Colibrì 是一套轻量、保持模型质量的 MoE 运行时，将 VRAM、RAM +与存储设备视为统一管理的内存层级。高速内存不足可能降低速度， +但默认策略**绝不会在未告知的情况下改变模型精度或路由语义**。 + +``` +$ ./coli chat + 🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU + ✓ ready in 32s · resident 9.9 GB + › ciao! + ◆ Ciao! 😊 Come posso aiutarti oggi? +``` + +## 实际运行效果 + +

+ colibrì 网页仪表盘——实时指标、硬件面板与专家存储层级 +

+

网页仪表盘（./coli web）：744B 模型达到 4 tok/s、TTFT 1.6 秒、磁盘读取 0—— +在 6× RTX 5090 上让所有专家常驻，并实时显示 token 指标、每轮耗时明细、 +VRAM／RAM／磁盘层级条，以及角落的实时迷你大脑。

+ +

+ 大脑页面——以实时皮层呈现 19,456 个专家 +

+

大脑（Brain）页面：将全部 19,456 个专家呈现为活的皮层——颜色代表存储层级， +亮度代表路由热度，每轮被路由到的专家都会闪白。将光标停在专家上，即可查看其 +实测主题亲和度。

+ +

+ 图谱页面——以 3D 星系呈现实测专家图谱 +

+

图谱（Atlas）页面：将实测专家图谱 +呈现为 3D 星系——共 13,260 个已分析专家，其中 1,041 个可复现的专门专家会按主题聚集 +（诗歌、法律、中文、SQL……）。位置取自实测路由亲和度，而非学习出的嵌入向量。拖拽即可旋转。

+ +## 核心概念 + +744B 的专家混合（Mixture-of-Experts）模型，每个 token 只会激活约 40B 参数—— +其中每个 token 之间会变动的只有约 11 GB（被路由到的专家）： + +

+ 每个 token 只会激活约 5.4% 的参数 +

+ +所以模型不必完整**装进**高速内存，而是需要正确**放置**： + +- **稠密部分**（注意力、共享专家、嵌入——约 17B 参数）以 int4 + **常驻 RAM**（约 9.9 GB）； +- **19,456 个路由专家**（75 个 MoE 层 × 256，加上 MTP head；每个在 int4 下约 19 MB） + **存放在磁盘**（约 370 GB），并**按需流式加载**，配合逐层 LRU 缓存、 + 会学习的热门专家固定存储区，以及可选的 VRAM 层级。 + +引擎是一个 C 主文件（`c/colibri.c`）加上若干头文件。不需要 BLAS， +运行时不需要 Python，也不需要 GPU。 + +## 工作原理 + +### 每个 token 的处理路径 + +

+ 路由 → 并集 → 放置 → 重叠执行 → 学习 +

+ +每个 token 的每一层都会经过相同的五个步骤。设计目标是让 +**放置只决定速度**——无论专家是从 VRAM 还是磁盘响应，路由器的决策与权重精度都完全相同。 + +### 统一内存层级，取代单一内存门槛 + +

+ VRAM／RAM／NVMe 三层专家常驻架构 +

+ +同一套引擎覆盖完整硬件范围：在 25 GB 笔记本上，一切都从磁盘流式加载 +（慢，但结果正确）；在大内存主机上，则可让整组专家常驻 +（`CUDA_EXPERT_GB=auto PIN_GB=all`），让磁盘完全退出解码路径。 +两端之间有一层**学习型缓存**：引擎会记录*你的*工作负载路由到哪些专家 +（`.coli_usage`，每轮更新），并自动固定最热门的专家——colibrì 确实会越用越快。 +在多路主机上，`COLI_NUMA=1` 会将常驻权重交错分配到各内存控制器 +（[#82](https://github.com/JustVugg/colibri/issues/82)）。 + +### 绝不为同一次磁盘读取等待两遍 + +缓存未命中的代价很高，因此引擎大部分的巧思都用来避免或重叠这些读取： +每个专家的三个矩阵相邻存储，并以一次 `pread` 读取；有界异步 I/O 池 +（`PIPE=1`，默认启用）会在常驻专家计算时加载缺失的专家；批量位置只读取每个 +不重复专家一次（**批量并集**）；路由前瞻线程（`PILOT=1`）则预取下一层专家—— +实测显示，路由结果提前一层时有 **71.6% 的可预测性**。 +在 GPU 上，常驻管线（`COLI_CUDA_PIPE=2`）让残差流跨层保留在设备端， +使 CPU 专家循环不中断；在 Apple Silicon 上，实验性的 +[Metal 后端](docs/metal.md)会用统一内存 GPU 执行批量专家运算。 + +### 忠实模型，压缩状态 + +前向传播已通过 `transformers` oracle 验证为**逐 token 完全一致** +（teacher-forcing 32/32）。MLA 注意力存储压缩后的 KV 状态——每个 token 为 576 个 +浮点数，而非 32,768 个（**缩小 57×**）——并跨重启持久保存 +（`.coli_kv`）：对话可暖启恢复，不需重新 prefill，结果与不中断的会话 +逐字节相同。DSA 稀疏注意力（GLM-5.2 的 lightning indexer）已忠实实现， +并通过强制选取所有 key，验证可精确复现稠密注意力。 + +### 诚实的推测解码 + +GLM-5.2 原生 MTP head 会起草 token，再由主模型以一次批量前向传播验证—— +条件合适时每次 forward 可产生 2.2–2.8 个 token。两条来之不易的规则已成为默认值： +MTP head 必须是 **int8**（int4 head 的接受率会崩塌到 0–4%，见 +[#8](https://github.com/JustVugg/colibri/issues/8)），且草稿与验证必须计算 +**相同函数**——`SPEC_PIN=1` 会把两者固定在同一 kernel family +（完整取证过程见 [#163](https://github.com/JustVugg/colibri/issues/163)）。 +语法强制草稿（[`GRAMMAR=file.gbnf`](docs/grammar-draft.md)）可在受限 JSON 输出中， +以近乎免费的代价提高接受率。推测解码是否带来净收益取决于缓存热度——请实测， +若不划算就使用 `DRAFT=0`。 + +## 实际成果 + +

+ 各硬件级别的实测解码速度 +

+ +同一套引擎、同一个 int4 容器——硬件只会改变专家的存放位置。 +[完整 benchmark 表格](docs/benchmarks.md)中的重点如下： + +- **6× RTX 5090，全部常驻：**解码 5.8–6.8 tok/s，TTFT 约 13 秒 + （[实验记录](docs/experiments/glm52-6x5090-2026-07-12.md)）； +- **128 GB、仅使用 CPU 的台式机：**热缓存后约 1.8 tok/s + （[#200](https://github.com/JustVugg/colibri/issues/200)）； +- **单张 RTX 5070 Ti 的笔记本级主机：**通过 GPU 常驻管线达到 1.07 tok/s + （[#273](https://github.com/JustVugg/colibri/issues/273)）； +- **25 GB 开发机：**冷启动 0.05–0.1 tok/s——这是项目起步时已证实的下限， + 也仍是诚实的基准。 + +质量来自测量，而非假设：int4 容器的量化损失，以及 scale granularity／rotation +消融实验，收录于 [docs/benchmarks.md](docs/benchmarks.md#quality-benchmark)、 +[#108](https://github.com/JustVugg/colibri/issues/108) 与 +[#81](https://github.com/JustVugg/colibri/issues/81)。 + +## 开始使用 + +### 1. 获取模型 + +Hugging Face 上已有预转换的 **GLM-5.2 int4** 容器——请务必使用 +**含 int8 MTP head 的版本**： + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +> ⚠️ 原始镜像使用 int4 MTP head → 草稿接受率为 0% +>（[#8](https://github.com/JustVugg/colibri/issues/8)）。请检查你的版本： +> `ls -l /out-mtp-*`——正确的 int8 大小为 `3527131672 / 5366238584 / 1065950496`。 + +你也可以自行从 FP8 源转换——只需一条可断点续传的命令，且任何时候都不需要 +在磁盘上同时存放完整的 756 GB： + +```bash +cd c && ./setup.sh # 检查 gcc/OpenMP、构建并运行自测 +./coli convert --model /nvme/glm52_i4 # 逐 shard 下载并转换（仅此一次需要 python） +``` + +### 2. 运行 + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli chat # 自动检测 RAM 预算、缓存与 MTP +COLI_MODEL=/nvme/glm52_i4 ./coli plan # 查看规划的 VRAM／RAM／磁盘配置 +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # 只读就绪检查 +./coli web --model /nvme/glm52_i4 # 在同一端口提供 API 与网页仪表盘 +./coli serve --model /nvme/glm52_i4 # 仅提供 OpenAI 兼容 API +``` + +引擎运行时是纯 C——python 只供一次性转换工具与可选的 API gateway 使用。 + +### 3. 深入了解 + +| 主题 | 文档 | +|---|---| +| Benchmark、社区实测数据、质量测量 | [docs/benchmarks.md](docs/benchmarks.md) | +| 调优选项、策略、学习型缓存、预取 | [docs/tuning.md](docs/tuning.md) | +| Windows 11 原生构建（含 CUDA DLL） | [docs/windows.md](docs/windows.md) | +| CUDA 后端、VRAM 专家层级、全部常驻 | [docs/cuda.md](docs/cuda.md) | +| Apple Silicon Metal 后端 | [docs/metal.md](docs/metal.md) | +| OpenAI 兼容 API、KV slots、网页仪表盘 | [docs/api.md](docs/api.md) | +| 语法强制草稿（结构化输出） | [docs/grammar-draft.md](docs/grammar-draft.md) | +| 环境变量完整清单 | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | + +## 支持项目 + +colibrì 最初由一人使用 12 核心、25 GB RAM 的笔记本开发； +如今它的数据来自社区中各种真实机器。如果这个项目对你有用： + +- ⭐ 为仓库加星并分享； +- 🐛 以 issue 提交你的硬件 benchmark 数据——实测数据比任何其他事都更能推动项目； +- 💬 若想赞助开发或捐赠硬件，请通过 GitHub issues 联系。 + +## 仓库结构 + +``` +Makefile 根目录构建／检查入口 +c/ +├── colibri.c 引擎主文件 +├── quant.h 量化 matmul 内核（SIMD 多架构） +├── sample.h 采样与 stop-set 管理 +├── kv_persist.h .coli_kv 磁盘持久化 +├── telemetry.h 仪表盘协议、统计与用量持久化 +├── st.h, tok.h, json.h 运行时头文件 +├── backend_cuda.* 可选的 CUDA 层级 +├── Makefile 构建与本地检查 +├── coli 用户界面 CLI +├── openai_server.py OpenAI 兼容 HTTP gateway +├── setup.sh 一条命令完成本地设置 +├── tools/ 离线转换、fixtures 与 benchmarks +├── scripts/ 长时间转换辅助工具 +└── tests/ 零依赖的 C 与 Python 测试 +web/ 浏览器 UI（纯 OpenAI API client） +desktop/ 封装网页 UI 的 Tauri v2 桌面 shell +docs/ 参考文档、实验与媒体文件 +``` + +运行时路径刻意保持扁平、易读：`colibri.c` 加上若干头文件。 +在仓库根目录执行 `make`、`make check` 与 `make clean`， +都会转发给引擎的 Makefile。 + +## 为什么叫"colibrì" + +蜂鸟只有几克重，能在原地悬停，并在一天内造访上千朵花。 +这套引擎只用蜂鸟般的配给，就能让 744B 参数的巨人运转： +25 GB RAM、十二个 CPU 核心，以及对磁盘的大量耐心。 + +## 许可证 + +Apache 2.0。GLM-5.2 权重由 Z.ai 以 MIT 许可发布。 diff --git a/README.zh-TW.md b/README.zh-TW.md new file mode 100644 index 0000000..2576eba --- /dev/null +++ b/README.zh-TW.md @@ -0,0 +1,237 @@ +

+ colibrì——小巧引擎，龐大模型 +

+ +

+ English · 简体中文 · 繁體中文 · Italiano +

+ +**小巧引擎，龐大模型。**只要約 25 GB 記憶體，就能在消費級電腦上執行 **GLM-5.2（744B 參數的 MoE）**——以零相依套件的純 C 實作，從硬碟串流載入專家。 + +Colibrì 是一套輕量、維持品質的 MoE 執行環境，將 VRAM、RAM +與儲存裝置視為統一管理的記憶體階層。高速記憶體不足可能降低速度， +但預設策略**絕不會在未告知的情況下改變模型精度或路由語意**。 + +``` +$ ./coli chat + 🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU + ✓ ready in 32s · resident 9.9 GB + › ciao! + ◆ Ciao! 😊 Come posso aiutarti oggi? +``` + +## 實際運行畫面 + +

+ colibrì 網頁儀表板——即時指標、硬體面板與專家儲存層級 +

+

網頁儀表板（./coli web）：744B 模型達到 4 tok/s、TTFT 1.6 秒、硬碟讀取 0—— +在 6× RTX 5090 上讓所有專家常駐，並即時顯示 token 指標、每輪耗時明細、 +VRAM／RAM／硬碟層級長條，以及角落的即時迷你大腦。

+ +

+ 大腦頁面——以即時皮質呈現 19,456 個專家 +

+

大腦（Brain）頁面：將全部 19,456 個專家呈現為活的皮質——顏色代表儲存層級， +亮度代表路由熱度，每輪被路由到的專家都會閃白。將游標停在專家上，即可查看其 +實測主題親和度。

+ +

+ 圖譜頁面——以 3D 星系呈現實測專家圖譜 +

+

圖譜（Atlas）頁面：將實測專家圖譜 +呈現為 3D 星系——共 13,260 個已分析專家，其中 1,041 個可重現的專門專家會按主題聚集 +（詩歌、法律、中文、SQL……）。位置取自實測路由親和度，而非學習出的嵌入向量。拖曳即可旋轉。

+ +## 核心概念 + +744B 的專家混合（Mixture-of-Experts）模型，每個 token 只會啟用約 40B 參數—— +其中每個 token 之間會變動的只有約 11 GB（被路由到的專家）： + +

+ 每個 token 只會啟用約 5.4% 的參數 +

+ +所以模型不必完整**放進**高速記憶體，而是需要正確**配置位置**： + +- **稠密部分**（注意力、共享專家、嵌入——約 17B 參數）以 int4 + **常駐 RAM**（約 9.9 GB）； +- **19,456 個路由專家**（75 個 MoE 層 × 256，加上 MTP head；每個在 int4 下約 19 MB） + **存放在硬碟**（約 370 GB），並**隨需串流載入**，搭配逐層 LRU 快取、 + 會學習的熱門專家固定儲存區，以及選用的 VRAM 層級。 + +引擎由主 C 檔（`c/colibri.c`）與多個標頭檔模組組成。不需要 BLAS， +執行階段不需要 Python，也不需要 GPU。 + +## 運作方式 + +### 每個 token 的處理路徑 + +

+ 路由 → 聯集 → 配置 → 重疊執行 → 學習 +

+ +每個 token 的每一層都會走過相同的五個步驟。設計目標是讓 +**配置只決定速度**——無論專家是從 VRAM 或硬碟回應，路由器的決策與權重精度都完全相同。 + +### 統一記憶體階層，取代單一記憶體門檻 + +

+ VRAM／RAM／NVMe 三層專家常駐架構 +

+ +同一套引擎涵蓋完整硬體範圍：在 25 GB 筆電上，一切都從硬碟串流載入 +（慢，但結果正確）；在大型主機上，則可讓整組專家常駐 +（`CUDA_EXPERT_GB=auto PIN_GB=all`），讓硬碟完全退出解碼路徑。 +兩端之間有一層**學習型快取**：引擎會記錄*你的*工作負載路由到哪些專家 +（`.coli_usage`，每輪更新），並自動固定最熱門的專家——colibrì 確實會越用越快。 +在多插槽主機上，`COLI_NUMA=1` 會將常駐權重交錯分配到各記憶體控制器 +（[#82](https://github.com/JustVugg/colibri/issues/82)）。 + +### 絕不為同一次硬碟讀取等待兩遍 + +快取未命中的成本很高，因此引擎大部分的巧思都用來避免或重疊處理這些讀取： +每個專家的三個矩陣相鄰儲存，並以一次 `pread` 讀取；有界非同步 I/O pool +（`PIPE=1`，預設啟用）會在常駐專家運算時載入缺少的專家；批次位置只讀取每個 +不重複專家一次（**批次聯集**）；路由前瞻執行緒（`PILOT=1`）則預先載入下一層專家—— +實測顯示，路由結果提前一層時有 **71.6% 的可預測性**。 +在 GPU 上，常駐管線（`COLI_CUDA_PIPE=2`）讓殘差流跨層保留在裝置端， +使 CPU 專家迴圈不中斷；在 Apple Silicon 上，實驗性的 +[Metal 後端](docs/metal.md)會用統一記憶體 GPU 執行批次專家運算。 + +### 忠實模型，壓縮狀態 + +前向傳遞已透過 `transformers` oracle 驗證為**逐 token 完全一致** +（teacher-forcing 32/32）。MLA 注意力儲存壓縮後的 KV 狀態——每個 token 為 576 個 +浮點數，而非 32,768 個（**縮小 57×**）——並跨重新啟動持久保存 +（`.coli_kv`）：對話可暖啟恢復，不需重新 prefill，結果與不中斷的工作階段 +逐位元組相同。DSA 稀疏注意力（GLM-5.2 的 lightning indexer）已忠實實作， +並透過強制選取所有 key，驗證可精確重現稠密注意力。 + +### 如實呈現推測式解碼 + +GLM-5.2 原生 MTP head 會起草 token，再由主模型以一次批次前向傳遞驗證—— +條件合適時每次 forward 可產生 2.2–2.8 個 token。兩條得來不易的規則已成為預設值： +MTP head 必須是 **int8**（int4 head 的接受率會崩落到 0–4%，見 +[#8](https://github.com/JustVugg/colibri/issues/8)），且草稿與驗證必須計算 +**相同函數**——`SPEC_PIN=1` 會把兩者固定在同一 kernel family +（完整鑑識過程見 [#163](https://github.com/JustVugg/colibri/issues/163)）。 +文法強制草稿（[`GRAMMAR=file.gbnf`](docs/grammar-draft.md)）可在受限 JSON 輸出中， +以近乎免費的成本提高接受率。推測式解碼是否帶來淨收益取決於快取熱度——請實測， +若不划算就使用 `DRAFT=0`。 + +## 實際成果 + +

+ 各硬體等級的實測解碼速度 +

+ +同一套引擎、同一個 int4 容器——硬體只會改變專家的存放位置。 +[完整 benchmark 表格](docs/benchmarks.md)中的重點如下： + +- **6× RTX 5090，全部常駐：**解碼 5.8–6.8 tok/s，TTFT 約 13 秒 + （[實驗紀錄](docs/experiments/glm52-6x5090-2026-07-12.md)）； +- **128 GB、僅使用 CPU 的桌上型電腦：**暖機後約 1.8 tok/s + （[#200](https://github.com/JustVugg/colibri/issues/200)）； +- **單張 RTX 5070 Ti 的筆電級電腦：**透過 GPU 常駐管線達到 1.07 tok/s + （[#273](https://github.com/JustVugg/colibri/issues/273)）； +- **25 GB 開發機：**冷啟動 0.05–0.1 tok/s——這是專案起步時已證實的下限， + 也仍是如實呈現的基準。 + +品質來自測量，而非假設：int4 容器的量化成本，以及 scale granularity／rotation +消融實驗，收錄於 [docs/benchmarks.md](docs/benchmarks.md#quality-benchmark)、 +[#108](https://github.com/JustVugg/colibri/issues/108) 與 +[#81](https://github.com/JustVugg/colibri/issues/81)。 + +## 開始使用 + +### 1. 取得模型 + +Hugging Face 上已有預先轉換的 **GLM-5.2 int4** 容器——請務必使用 +**含 int8 MTP head 的版本**： + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +> ⚠️ 原始鏡像使用 int4 MTP head → 草稿接受率為 0% +>（[#8](https://github.com/JustVugg/colibri/issues/8)）。請檢查你的版本： +> `ls -l /out-mtp-*`——正確的 int8 大小為 `3527131672 / 5366238584 / 1065950496`。 + +你也可以自行從 FP8 來源轉換——只需一條可續傳的指令，且任何時候都不需要 +在硬碟上同時存放完整的 756 GB： + +```bash +cd c && ./setup.sh # 檢查 gcc/OpenMP、建置並執行自我測試 +./coli convert --model /nvme/glm52_i4 # 逐 shard 下載並轉換（僅此一次需要 python） +``` + +### 2. 執行 + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli chat # 自動偵測 RAM 預算、快取與 MTP +COLI_MODEL=/nvme/glm52_i4 ./coli plan # 檢視規劃的 VRAM／RAM／硬碟配置 +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # 唯讀就緒檢查 +./coli web --model /nvme/glm52_i4 # 在同一個連接埠提供 API 與網頁儀表板 +./coli serve --model /nvme/glm52_i4 # 僅提供 OpenAI 相容 API +``` + +引擎執行階段是純 C——python 只供單次轉換工具與選用的 API gateway 使用。 + +### 3. 深入了解 + +| 主題 | 文件 | +|---|---| +| Benchmark、社群實測數據、品質測量 | [docs/benchmarks.md](docs/benchmarks.md) | +| 調校選項、策略、學習型快取、預先載入 | [docs/tuning.md](docs/tuning.md) | +| Windows 11 原生建置（含 CUDA DLL） | [docs/windows.md](docs/windows.md) | +| CUDA 後端、VRAM 專家層級、全部常駐 | [docs/cuda.md](docs/cuda.md) | +| Apple Silicon Metal 後端 | [docs/metal.md](docs/metal.md) | +| OpenAI 相容 API、KV slots、網頁儀表板 | [docs/api.md](docs/api.md) | +| 文法強制草稿（結構化輸出） | [docs/grammar-draft.md](docs/grammar-draft.md) | +| 環境變數完整清單 | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | + +## 支持專案 + +colibrì 最初是由一人使用 12 核心、25 GB RAM 的筆電開發； +如今它的數據來自社群中的各種真實機器。如果這個專案對你有用： + +- ⭐ 為儲存庫加星並分享； +- 🐛 以 issue 提交你的硬體 benchmark 數據——實測資料比任何其他事都更能推動專案； +- 💬 若想贊助開發或捐贈硬體，請透過 GitHub issues 聯絡。 + +## 儲存庫結構 + +``` +Makefile 根目錄建置／檢查入口 +c/ +├── colibri.c GLM 引擎主檔 +├── quant.h 量化 matmul kernel +├── sample.h 取樣與 stop-set +├── kv_persist.h .coli_kv 磁碟持久化 +├── telemetry.h 儀表板協定、統計 +├── st.h, tok.h, json.h 執行階段標頭檔 +├── backend_cuda.* 選用的 CUDA 層級 +├── Makefile 建置與本機檢查 +├── coli 使用者介面 CLI +├── openai_server.py OpenAI 相容 HTTP gateway +├── setup.sh 單一指令完成本機設定 +├── tools/ 離線轉換、fixtures 與 benchmarks +├── scripts/ 長時間轉換輔助工具 +└── tests/ 零相依套件的 C 與 Python 測試 +web/ 瀏覽器 UI（純 OpenAI API client） +desktop/ 包裝網頁 UI 的 Tauri v2 桌面 shell +docs/ 參考文件、實驗與媒體檔 +``` + +執行階段路徑刻意維持扁平、易讀：`colibri.c` 加上模組化標頭檔。 +在儲存庫根目錄執行 `make`、`make check` 與 `make clean`， +都會轉交給引擎的 Makefile。 + +## 為什麼叫做「colibrì」 + +蜂鳥只有幾公克重，能在原地懸停，並在一天內造訪上千朵花。 +這套引擎只用蜂鳥般的配給，就能讓 744B 參數的巨人運轉： +25 GB RAM、十二個 CPU 核心，以及對硬碟的大量耐心。 + +## 授權條款 + +Apache 2.0。GLM-5.2 權重由 Z.ai 以 MIT 授權發布。 diff --git a/c/.gitignore b/c/.gitignore index 782930b..d8e1f88 100644 --- a/c/.gitignore +++ b/c/.gitignore @@ -2,3 +2,4 @@ glm_tiny/ olmoe_hf/ olmoe_i4/ +.build-config diff --git a/c/Makefile b/c/Makefile index ccf0a3c..af17357 100644 --- a/c/Makefile +++ b/c/Makefile @@ -56,7 +56,7 @@ OMPL = endif CFLAGS = -O3 $(OMPC) -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function # Opt-in: ARCH=native appends -mcpu=native (arm64 clang uses -mcpu, not -march), -# which unlocks the i8mm SMMLA int8/int4 dot kernels in glm.c. ARCH unset -> +# which unlocks the i8mm SMMLA int8/int4 dot kernels in colibri.c. ARCH unset -> # no -mcpu, default build byte-identical. Apple clang knows apple-m4 / native. ifneq ($(ARCH),) CFLAGS += -mcpu=$(ARCH) @@ -70,7 +70,7 @@ else ifneq ($(IS_WIN),) # ARCH default = x86-64-v3 (portable binary with AVX2). For max speed on THIS # machine use ARCH=native: on AVX-VNNI CPUs (Intel Alder Lake+, Meteor Lake+) # it also unlocks the 128-bit VPDPBUSD int8/int4 dot kernel (dot_i8i8/dot_i4i8), -# which the x86-64-v3 baseline does not define. The #ifdef guards in glm.c mean +# which the x86-64-v3 baseline does not define. The #ifdef guards in colibri.c mean # a v3 build simply compiles out the VNNI path - safe on any x86-64. CC = gcc ARCH ?= x86-64-v3 @@ -166,7 +166,7 @@ else PYTHON ?= python3 endif CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) ifneq (,$(LINUX)) TEST_BINS += tests/test_uring$(EXE) endif @@ -207,16 +207,31 @@ LDFLAGS += -framework Metal -framework Foundation -lc++ METAL_OBJ = backend_metal.o endif -all: glm$(EXE) +all: colibri$(EXE) -# phony 'glm' → 'glm.exe' on Windows (so 'make glm' and 'coli build' work on every platform) -glm: glm$(EXE) +# phony targets — 'glm' kept for backward compatibility +colibri: colibri$(EXE) +glm: colibri$(EXE) -glm$(EXE): glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h $(CUDA_OBJ) $(METAL_OBJ) - $(CC) $(CFLAGS) glm.c $(CUDA_OBJ) $(METAL_OBJ) -o glm$(EXE) $(LDFLAGS) +# Config stamp: make only tracks file timestamps, not flag changes. Without this, +# `make colibri.exe CUDA_DLL=1` after a prior CPU-only build reports "up to date" +# and silently keeps the CPU-only binary (no CUDA loader) — a build that looks like +# it worked but isn't. We record the build-affecting flags in .build-config and +# rewrite it ONLY when they change (evaluated here at parse time, so the file's +# timestamp moves exactly when the config moves). The binary and CUDA/loader objects depend +# on it, so they relink on a config change and stay put otherwise. (#306) +BUILD_CONFIG := $(CC)|$(CFLAGS)|$(LDFLAGS)|CUDA=$(CUDA)|CUDA_DLL=$(CUDA_DLL)|ARCH=$(ARCH)|CUDA_ARCH=$(CUDA_ARCH)|METAL=$(METAL) +BUILD_CONFIG_OLD := $(shell cat .build-config 2>/dev/null) +ifneq "$(BUILD_CONFIG)" "$(BUILD_CONFIG_OLD)" +$(shell printf '%s' '$(BUILD_CONFIG)' > .build-config) +endif +.build-config: ; + +colibri$(EXE): colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h quant.h sample.h kv_persist.h telemetry.h $(CUDA_OBJ) $(METAL_OBJ) .build-config + $(CC) $(CFLAGS) colibri.c $(CUDA_OBJ) $(METAL_OBJ) -o colibri$(EXE) $(LDFLAGS) # Windows runtime loader object: resolves coli_cuda_* from coli_cuda.dll. -backend_loader.o: backend_loader.c backend_cuda.h compat.h +backend_loader.o: backend_loader.c backend_cuda.h compat.h .build-config $(CC) $(CFLAGS) -c backend_loader.c -o $@ # Windows CUDA DLL: compile backend_cuda.cu with nvcc (+MSVC cl.exe as host @@ -225,15 +240,15 @@ backend_loader.o: backend_loader.c backend_cuda.h compat.h # "x64 Native Tools Command Prompt"). COLI_CUDA_BUILDING_DLL enables # __declspec(dllexport) so the 15 API symbols are exported. cuda-dll: backend_cuda.cu backend_cuda.h - @command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } + @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } @command -v cl >/dev/null 2>&1 || { echo "cl.exe (MSVC) not in PATH — run vcvars64.bat first" >&2; exit 1; } - $(NVCC) $(NVCCFLAGS) -shared -DCOLI_CUDA_BUILDING_DLL \ + "$(NVCC)" $(NVCCFLAGS) -shared -DCOLI_CUDA_BUILDING_DLL \ -L"$(CUDA_HOME)/lib/x64" -lcudart \ backend_cuda.cu -o coli_cuda.dll -backend_cuda.o: backend_cuda.cu backend_cuda.h - @command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } - $(NVCC) $(NVCCFLAGS) -c backend_cuda.cu -o $@ +backend_cuda.o: backend_cuda.cu backend_cuda.h .build-config + @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } + "$(NVCC)" $(NVCCFLAGS) -c backend_cuda.cu -o $@ backend_metal.o: backend_metal.mm backend_metal.h $(METALXX) -c backend_metal.mm -o $@ @@ -243,13 +258,13 @@ metal-test: tests/test_backend_metal.mm backend_metal.mm backend_metal.h ./backend_metal_test cuda-test: backend_cuda.cu backend_cuda.h tests/test_backend_cuda.cu - @command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } - $(NVCC) $(NVCCFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) + @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } + "$(NVCC)" $(NVCCFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) ./backend_cuda_test$(EXE) cuda-bench: backend_cuda.cu backend_cuda.h tests/bench_tensor_core.cu - @command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } - $(NVCC) $(NVCCFLAGS) backend_cuda.cu tests/bench_tensor_core.cu -o backend_cuda_bench$(EXE) + @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } + "$(NVCC)" $(NVCCFLAGS) backend_cuda.cu tests/bench_tensor_core.cu -o backend_cuda_bench$(EXE) ./backend_cuda_bench$(EXE) olmoe$(EXE): olmoe.c st.h json.h compat.h @@ -271,7 +286,7 @@ PORTABLE_ARCH = native endif portable: - $(MAKE) glm$(EXE) ARCH=$(PORTABLE_ARCH) + $(MAKE) colibri$(EXE) ARCH=$(PORTABLE_ARCH) iobench$(EXE): iobench.c compat.h $(CC) $(CFLAGS) iobench.c -o iobench$(EXE) $(LDFLAGS) @@ -279,6 +294,9 @@ iobench$(EXE): iobench.c compat.h tests/test_json$(EXE): tests/test_json.c json.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_st_pread$(EXE): tests/test_st_pread.c st.h json.h compat.h + $(CC) $(CFLAGS) -DST_PREAD_CHUNK=7 $< -o $@ $(LDFLAGS) + tests/test_st$(EXE): tests/test_st.c st.h json.h compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) @@ -294,10 +312,30 @@ tests/test_schema_gbnf$(EXE): tests/test_schema_gbnf.c schema_gbnf.h grammar.h j tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_idot$(EXE): tests/test_idot.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_idot$(EXE): tests/test_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_stops$(EXE): tests/test_stops.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_topp$(EXE): tests/test_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +# bench_topp is a microbenchmark (old qsort vs new heap partial-select, #335), NOT a test +# gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_topp +tests/bench_topp$(EXE): tests/bench_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_sample_nan$(EXE): tests/test_sample_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c colibri.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_logit_nan$(EXE): tests/test_logit_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c @@ -306,7 +344,15 @@ tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c tests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_uring$(EXE): tests/test_uring.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_dsa_select$(EXE): tests/test_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +# bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356), +# NOT a test gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_dsa_select +tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_uring$(EXE): tests/test_uring.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) test-c: $(TEST_BINS) @@ -323,12 +369,12 @@ check: $(MAKE) portable $(MAKE) test -install: glm$(EXE) olmoe$(EXE) +install: colibri$(EXE) olmoe$(EXE) $(INSTALL) -d $(DESTDIR)$(BINDIR) $(INSTALL) -d $(DESTDIR)$(LIBEXECDIR) $(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)/tools $(INSTALL) -m 755 coli $(DESTDIR)$(BINDIR)/coli - $(INSTALL) -m 755 glm$(EXE) $(DESTDIR)$(LIBEXECDIR)/glm$(EXE) + $(INSTALL) -m 755 colibri$(EXE) $(DESTDIR)$(LIBEXECDIR)/colibri$(EXE) $(INSTALL) -m 755 olmoe$(EXE) $(DESTDIR)$(LIBEXECDIR)/olmoe$(EXE) $(INSTALL) -m 644 resource_plan.py doctor.py openai_server.py $(DESTDIR)$(LIBEXECDIR)/ $(INSTALL) -m 644 tools/*.py $(DESTDIR)$(LIBEXECDIR)/tools/ @@ -342,4 +388,4 @@ clean: bench: iobench$(EXE) @if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) — run: ./iobench$(EXE) "; fi -.PHONY: all glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench +.PHONY: all colibri glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu index 9ce142f..413e53b 100644 --- a/c/backend_cuda.cu +++ b/c/backend_cuda.cu @@ -8,12 +8,21 @@ #include #include +struct RaggedKVEntry { + const void *key; + const float *host_l,*host_r; + float *latent,*rope; + int length,capacity,K,R; +}; + struct ColiCudaTensor { void *weights; float *scales; size_t weight_bytes; int fmt, I, O, device; int tracked; + RaggedKVEntry ragged[512]; + int ragged_count; }; typedef struct { @@ -23,7 +32,7 @@ typedef struct { size_t x_cap, y_cap, gate_cap, up_cap; uint8_t *qx; float *qscale; size_t qx_cap, qscale_cap; - float *host_x,*host_y; size_t host_x_cap,host_y_cap; + float *host_x,*host_y,*host_kv; size_t host_x_cap,host_y_cap,host_kv_cap; float *aq,*al,*ar,*ac; size_t aq_cap,al_cap,ar_cap,ac_cap; float *pipe_buf[24]; size_t pipe_cap[24]; /* scratch persistenti del resident pipeline */ cudaStream_t stream; @@ -338,6 +347,49 @@ __global__ static void attention_absorb_batch_kernel(float *ctx,const float *q, ctx[((size_t)s*H+h)*V+v]=a*(fmt?wscale[row]:1.f);} } +/* Independent device-resident KV sequence per row. lengths selects the valid + * prefix; latent/rope point at paged caches updated by the host wrapper. */ +__global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q, + const float *const *latent,const float *const *rope,const int *lengths, + const void *weights,const float *wscale,int fmt,int S,int H,int Q,int R, + int V,int K,int T,float scale){ + int s=blockIdx.y,h=blockIdx.x,tid=threadIdx.x,nt=lengths[s],rbase=h*(Q+V); + if(s>=S||nt<1||nt>T)return; + extern __shared__ float sm[];float *qa=sm,*cl=qa+K,*scores=cl+K,*red=scores+T; + const float *qs=q+((size_t)s*H+h)*(Q+R); + const float *ls=latent[s],*rs=rope[s]; + for(int k=tid;k>1;n;n>>=1){if(tid>1;n;n>>=1){if(tid= bytes) return 1; if (*ptr) cudaFree(*ptr); @@ -406,16 +458,17 @@ extern "C" void coli_cuda_shutdown(void) { for(int b=0;b<24;b++) if(ctx->pipe_buf[b]) cudaFree(ctx->pipe_buf[b]); if (ctx->host_x) cudaFreeHost(ctx->host_x); if (ctx->host_y) cudaFreeHost(ctx->host_y); + if (ctx->host_kv) cudaFreeHost(ctx->host_kv); if (ctx->stream) cudaStreamDestroy(ctx->stream); if (ctx->group_desc) cudaFree(ctx->group_desc); ctx->x = ctx->y = ctx->gate = ctx->up = nullptr; ctx->qx=nullptr; ctx->qscale=nullptr; ctx->aq=ctx->al=ctx->ar=ctx->ac=nullptr; - ctx->host_x=ctx->host_y=nullptr;ctx->stream=nullptr; + ctx->host_x=ctx->host_y=ctx->host_kv=nullptr;ctx->stream=nullptr; ctx->x_cap = ctx->y_cap = ctx->gate_cap = ctx->up_cap = 0; ctx->qx_cap=ctx->qscale_cap=0; ctx->aq_cap=ctx->al_cap=ctx->ar_cap=ctx->ac_cap=0; - ctx->host_x_cap=ctx->host_y_cap=0; + ctx->host_x_cap=ctx->host_y_cap=ctx->host_kv_cap=0; ctx->group_desc=nullptr; ctx->group_desc_cap=0; } g_nctx = 0; @@ -770,6 +823,89 @@ extern "C" int coli_cuda_attention_project_batch(ColiCudaTensor *w,ColiCudaTenso return attention_absorb_batch_run(w,proj,out,q,latent,rope,S,H,Q,R,V,K,T,scale); } +extern "C" int coli_cuda_attention_project_ragged(ColiCudaTensor *w,ColiCudaTensor *proj, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int T,float scale){ + if(!w||!proj||!out||!q||!keys||!latent||!rope||!lengths||S<1||S>512||T<1||T>8192|| + H<1||Q<1||R<1||V<1||K<1||K>512||w->I!=K||w->O!=H*(Q+V)|| + proj->device!=w->device||proj->I!=H*V)return 0; + DeviceContext *dc=find_ctx(w->device); + if(!select_ctx(dc))return 0; + float **dl=(float**)std::malloc((size_t)S*sizeof(*dl)); + float **dr=(float**)std::malloc((size_t)S*sizeof(*dr)); + int *old=(int*)std::malloc((size_t)S*sizeof(*old)); + int *add=(int*)std::malloc((size_t)S*sizeof(*add)); + int *off=(int*)std::malloc((size_t)S*sizeof(*off));int packed_n=0; + if(!dl||!dr||!old||!add||!off){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;} + for(int s=0;sT){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;} + RaggedKVEntry *e=nullptr; + for(int i=0;iragged_count;i++)if(w->ragged[i].key==keys[s]){e=&w->ragged[i];break;} + if(!e){ + if(w->ragged_count>=512){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;} + e=&w->ragged[w->ragged_count++];std::memset(e,0,sizeof(*e));e->key=keys[s]; + } + if(e->K!=K||e->R!=R||e->host_l!=latent[s]||e->host_r!=rope[s]||lengths[s]length){ + if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope); + e->latent=e->rope=nullptr;e->length=e->capacity=0; + e->K=K;e->R=R;e->host_l=latent[s];e->host_r=rope[s]; + } + if(lengths[s]>e->capacity){ + int cap=(lengths[s]+63)&~63;float *nl=nullptr,*nr=nullptr; + if(!cuda_ok(cudaMalloc(&nl,(size_t)cap*K*sizeof(float)),"ragged KV latent page")|| + !cuda_ok(cudaMalloc(&nr,(size_t)cap*R*sizeof(float)),"ragged KV rope page")){ + if(nl)cudaFree(nl);if(nr)cudaFree(nr);std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0; + } + if(e->length){ + cudaMemcpyAsync(nl,e->latent,(size_t)e->length*K*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream); + cudaMemcpyAsync(nr,e->rope,(size_t)e->length*R*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream); + } + if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope); + e->latent=nl;e->rope=nr;e->capacity=cap; + } + dl[s]=e->latent;dr[s]=e->rope;old[s]=e->length;add[s]=lengths[s]-e->length; + off[s]=packed_n;packed_n+=add[s]*(K+R); + } + size_t qb=(size_t)S*H*(Q+R)*sizeof(float); + size_t cb=(size_t)S*H*V*sizeof(float),ob=(size_t)S*proj->O*sizeof(float); + size_t pb=(size_t)packed_n*sizeof(float); + size_t desc=(size_t)S*(2*sizeof(float*)+4*sizeof(int)); + int ok=reserve(&dc->aq,&dc->aq_cap,qb)&&reserve(&dc->ac,&dc->ac_cap,cb)&& + reserve(&dc->y,&dc->y_cap,ob)&&reserve_bytes(&dc->group_desc,&dc->group_desc_cap,desc)&& + (!pb||(reserve(&dc->al,&dc->al_cap,pb)&&reserve_pinned(&dc->host_kv,&dc->host_kv_cap,pb))); + char *db=(char*)dc->group_desc;float **ddl=(float**)db,**ddr=ddl+S; + int *dn=(int*)(ddr+S),*dold=dn+S,*dadd=dold+S,*doff=dadd+S; + if(ok&&pb){ + for(int s=0;shost_kv+off[s]; + std::memcpy(p,latent[s]+(size_t)old[s]*K,(size_t)add[s]*K*sizeof(float)); + std::memcpy(p+(size_t)add[s]*K,rope[s]+(size_t)old[s]*R,(size_t)add[s]*R*sizeof(float)); + } + ok=cuda_ok(cudaMemcpyAsync(dc->al,dc->host_kv,pb,cudaMemcpyHostToDevice,dc->stream),"ragged KV append upload"); + } + if(ok)ok=cuda_ok(cudaMemcpyAsync(dc->aq,q,qb,cudaMemcpyHostToDevice,dc->stream),"ragged q upload")&& + cuda_ok(cudaMemcpyAsync(ddl,dl,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged latent pointers")&& + cuda_ok(cudaMemcpyAsync(ddr,dr,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged rope pointers")&& + cuda_ok(cudaMemcpyAsync(dn,lengths,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged lengths upload")&& + cuda_ok(cudaMemcpyAsync(dold,old,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged old lengths")&& + cuda_ok(cudaMemcpyAsync(dadd,add,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append lengths")&& + cuda_ok(cudaMemcpyAsync(doff,off,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append offsets"); + if(ok&&pb)ragged_kv_append<<stream>>>(ddl,ddr,dc->al,dold,dadd,doff,K,R); + if(ok)for(int s=0;sragged_count;i++)if(w->ragged[i].key==keys[s]){w->ragged[i].length=lengths[s];break;} + } + std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);if(!ok)return 0; + size_t shared=(size_t)(2*K+T+256)*sizeof(float); + attention_absorb_ragged_kernel<<stream>>>(dc->ac,dc->aq,ddl,ddr, + dn,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale); + quant_matmul<<O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights, + proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I)); + return cuda_ok(cudaGetLastError(),"ragged attention launch")&& + cuda_ok(cudaMemcpyAsync(out,dc->y,ob,cudaMemcpyDeviceToHost,dc->stream),"ragged output download")&& + cuda_ok(cudaStreamSynchronize(dc->stream),"ragged attention synchronize"); +} + extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) { if (!tensor) return; DeviceContext *ctx = find_ctx(tensor->device); @@ -781,6 +917,10 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) { } if (tensor->weights) cudaFree(tensor->weights); if (tensor->scales) cudaFree(tensor->scales); + for(int i=0;iragged_count;i++){ + if(tensor->ragged[i].latent)cudaFree(tensor->ragged[i].latent); + if(tensor->ragged[i].rope)cudaFree(tensor->ragged[i].rope); + } std::free(tensor); } diff --git a/c/backend_cuda.h b/c/backend_cuda.h index acbe4bc..63c1f11 100644 --- a/c/backend_cuda.h +++ b/c/backend_cuda.h @@ -14,6 +14,7 @@ #define COLI_CUDA_DLLEXPORT #endif + #ifdef __cplusplus extern "C" { #endif @@ -92,6 +93,11 @@ COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,C const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale); +COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale); + COLI_CUDA_DLLEXPORT void coli_cuda_tensor_free(ColiCudaTensor *tensor); COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor); COLI_CUDA_DLLEXPORT int coli_cuda_tensor_device(const ColiCudaTensor *tensor); @@ -143,4 +149,3 @@ COLI_CUDA_DLLEXPORT int coli_cuda_pipe_sync(int device); #endif #endif - diff --git a/c/backend_loader.c b/c/backend_loader.c index b743d1b..eedbd50 100644 --- a/c/backend_loader.c +++ b/c/backend_loader.c @@ -61,6 +61,10 @@ typedef int (*fn_attention_absorb_batch)(ColiCudaTensor *kv_b,float *ctx,const f typedef int (*fn_attention_absorb_batch_dev)(ColiCudaTensor *kv_b_shard,float *ctx_dev, const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); typedef int (*fn_attention_absorb_kvdev)(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale); typedef int (*fn_attention_project_batch)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q,const float *latent, const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale); +typedef int (*fn_attention_project_ragged)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale); typedef int (*fn_attention_project_batch_dev)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); typedef int (*fn_attention_project_batch_dev_out)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); typedef int (*fn_pipe_add)(int device,float *x_dev,const float *t_dev,size_t n); @@ -107,6 +111,7 @@ static struct { fn_attention_absorb_batch_dev attention_absorb_batch_dev; fn_attention_absorb_kvdev attention_absorb_kvdev; fn_attention_project_batch attention_project_batch; + fn_attention_project_ragged attention_project_ragged; fn_attention_project_batch_dev attention_project_batch_dev; fn_attention_project_batch_dev_out attention_project_batch_dev_out; fn_pipe_add pipe_add; @@ -200,6 +205,7 @@ static int coli_cuda_load(void){ RESOLVE(attention_absorb_batch_dev, fn_attention_absorb_batch_dev) RESOLVE(attention_absorb_kvdev, fn_attention_absorb_kvdev) RESOLVE(attention_project_batch, fn_attention_project_batch) + RESOLVE(attention_project_ragged, fn_attention_project_ragged) RESOLVE(attention_project_batch_dev, fn_attention_project_batch_dev) RESOLVE(attention_project_batch_dev_out, fn_attention_project_batch_dev_out) RESOLVE(pipe_add, fn_pipe_add) @@ -342,6 +348,15 @@ int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,ColiCudaTensor *o_pro return g_cuda.attention_project_batch(kv_b, o_proj, out, q, latent, rope, S, H, Q, R, V, K, T, attention_scale); } +int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale){ + if(!coli_cuda_load()) return 0; + return g_cuda.attention_project_ragged(kv_b,o_proj,out,q,keys,latent,rope,lengths, + S,H,Q,R,V,K,max_t,attention_scale); +} + int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){ if(!g_cuda.available){ return 0; } return g_cuda.attention_project_batch_dev(kv_b, o_proj, out, q_dev, latent_dev, rope_dev, S, H, Q, R, V, K, T, scale); diff --git a/c/coli b/c/coli index 4c27415..d6728fe 100755 --- a/c/coli +++ b/c/coli @@ -20,7 +20,7 @@ Configuration through environment variables or flags (also valid after the subco --topp P adaptive expert top-p --topk N fixed top-k --ngen N maximum response tokens --cap N cache slots/layer """ -import os, sys, subprocess, argparse, json, time, signal, shutil, threading, re, codecs, tempfile, textwrap +import os, sys, subprocess, argparse, json, time, signal, shutil, threading, re, codecs, tempfile, textwrap, struct # The engine mmaps every shard (144+ files); macOS default RLIMIT_NOFILE is 256. if sys.platform != "win32": @@ -40,6 +40,8 @@ if sys.platform == "win32": except (AttributeError, OSError): pass HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +from version import __version__ as _version # Run-in-place (source checkout, "cd c && ./coli ..."): the engine, the # support modules (resource_plan.py, doctor.py, openai_server.py) and @@ -52,16 +54,20 @@ HERE = os.path.dirname(os.path.abspath(__file__)) # guess is right (e.g. a custom packaging layout). _EXE = ".exe" if sys.platform == "win32" else "" _LIBEXEC = os.path.join(os.path.dirname(HERE), "libexec", "colibri") +_here_colibri = os.path.join(HERE, "colibri" + _EXE) _here_glm = os.path.join(HERE, "glm" + _EXE) if os.environ.get("COLI_ENGINE"): GLM = os.environ["COLI_ENGINE"] TOOLS = os.path.join(os.path.dirname(GLM), "tools") +elif os.path.exists(_here_colibri): + GLM = _here_colibri + TOOLS = os.path.join(HERE, "tools") elif os.path.exists(_here_glm): GLM = _here_glm TOOLS = os.path.join(HERE, "tools") else: - GLM = os.path.join(_LIBEXEC, "glm" + _EXE) + GLM = os.path.join(_LIBEXEC, "colibri" + _EXE) TOOLS = os.path.join(_LIBEXEC, "tools") sys.path.insert(0, _LIBEXEC) # so `import resource_plan`, `doctor`, `openai_server` still resolve @@ -115,7 +121,7 @@ def sprite_lines(): def banner(sub=""): sp=sprite_lines() txt=[ - f"{C.teal}{C.b}colibrì{C.r} {C.dim}v1.0{C.r}", + f"{C.teal}{C.b}colibrì{C.r} {C.dim}v{_version}{C.r}", f"{C.dim}tiny engine, immense model{C.r}", f"{C.gray}GLM-5.2 · 744B MoE · int4 · streaming CPU{C.r}", f"{C.dgray}{sub}{C.r}" if sub else "", @@ -141,12 +147,24 @@ def need_model(model): sys.exit(f"{C.yel}engine is not built.{C.r} Run: coli build") def cuda_binary(): - if not os.path.exists(GLM) or sys.platform != "linux": return False - try: - linked=subprocess.run(["ldd",GLM],capture_output=True,text=True,timeout=3) - return any("libcudart" in line and "not found" not in line - for line in linked.stdout.splitlines()) - except (OSError,subprocess.SubprocessError): return False + if not os.path.exists(GLM): return False + if sys.platform == "linux": + try: + linked=subprocess.run(["ldd",GLM],capture_output=True,text=True,timeout=3) + return any("libcudart" in line and "not found" not in line + for line in linked.stdout.splitlines()) + except (OSError,subprocess.SubprocessError): return False + if sys.platform == "win32": + # Windows CUDA_DLL=1 builds never link libcudart directly: glm.exe loads + # coli_cuda.dll at runtime via LoadLibrary (backend_loader.c), so there's no + # import-table entry for ldd/dumpbin to see. Detect the COLI_CUDA build via a + # marker string baked into glm.c's #ifdef COLI_CUDA block instead, and require + # coli_cuda.dll to actually sit next to glm.exe (else CUDA init fails at startup). + try: + with open(GLM,"rb") as f: built=b"[CUDA] mode: routed experts" in f.read() + except OSError: return False + return built and os.path.exists(os.path.join(os.path.dirname(GLM),"coli_cuda.dll")) + return False def resource_request(a, env): ctx=a.ctx or int(env.get("CTX",4096)) @@ -227,13 +245,13 @@ def env_for(a): e["COLI_CUDA"]="0"; e.pop("CUDA_EXPERT_GB",None); e.pop("CUDA_DENSE",None) else: if not cuda_binary(): - sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} make glm CUDA=1 (this binary is CPU-only)") + sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} make colibri CUDA=1 (this binary is CPU-only)") e["COLI_CUDA"]="1" if a.gpu!="auto": e["COLI_GPUS"]=a.gpu e.setdefault("CUDA_DENSE","1") if a.vram and a.gpu!="none": if not cuda_binary(): - sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} make glm CUDA=1 (this binary is CPU-only)") + sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} make colibri CUDA=1 (this binary is CPU-only)") e["COLI_CUDA"]="1"; e["CUDA_EXPERT_GB"]=str(a.vram) return e @@ -407,8 +425,8 @@ def cmd_build(a): banner("build") if not os.path.exists(os.path.join(HERE, "Makefile")): sys.exit(f"{C.yel}coli build{C.r} only works from a source checkout (this is an installed copy).\n" - f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c glm.") - sys.exit(subprocess.call(["make","-C",HERE,"glm"])) + f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c colibri.") + sys.exit(subprocess.call(["make","-C",HERE,"colibri"])) def cmd_info(a): banner("info") @@ -480,13 +498,141 @@ def cmd_run(a): need_model(a.model) prompt=" ".join(a.prompt) if a.prompt else sys.exit('usage: coli run "your prompt"') banner("run") - # template ufficiale GLM-5.2: niente \n dopo i ruoli; = risposta diretta (nothink) - e=env_for(a); e["PROMPT"]=f"[gMASK]<|user|>{prompt}<|assistant|>" + # template ufficiale GLM-5.2: niente \n dopo i ruoli; = risposta diretta (nothink). + # THINK=1 lascia aperto, stessa convenzione del serve mode (glm.c). EN: THINK=1 leaves + # open so the engine emits its reasoning block; the default stays nothink. + tk="" if os.environ.get("THINK","0")=="1" else "" + e=env_for(a); e["PROMPT"]=f"[gMASK]<|user|>{prompt}<|assistant|>{tk}" sys.exit(subprocess.call([GLM, str(a.cap)], env=e)) +def server_probe(base, api_key=None, timeout=1.5): + """Is a coli serve alive at `base`? Returns its model_id, or None. + Probes /health then /v1/models — both cheap, neither touches the engine.""" + import urllib.request, urllib.error + def get(path): + req=urllib.request.Request(base.rstrip("/")+path) + if api_key: req.add_header("Authorization", f"Bearer {api_key}") + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read().decode("utf-8","replace")) + try: + if get("/health").get("status")!="ok": return None + data=get("/v1/models").get("data") or [] + return data[0]["id"] if data else None + except Exception: + return None + +def chat_attached(a, base, model_id): + """The chat REPL over HTTP against a running `coli serve`. + + Why this exists (the cold-chat cost, measured): spawning a private engine + pays 34-136 s of resident load on EVERY start, and begins with an empty + expert cache — hit rate 4% cold vs 55% warm, a ~10x on early decode. A + resident server pays load once and keeps the LRU warm across sessions; + its KV slots reuse the conversation prefix, so a continued chat skips + re-prefill too. The engine byte-protocol stays untouched — this is plain + OpenAI SSE over localhost, stdlib only.""" + import urllib.request + print(f" {C.grn}✦ attached{C.r} {C.dim}to {base} · model {model_id} · the engine stays warm after you quit{C.r}") + print(f" {C.dim}type and press Enter · Ctrl-C stops the answer · :reset starts a new conversation · :q exits{C.r}\n") + msgs=[] + w=term_w()-4 + while True: + if TTY: + print(f" {C.dgray}╭{'─'*w}╮{C.r}") + try: msg=input(f" {C.dgray}│{C.r} {C.teal}{C.b}›{C.r} ") + except EOFError: print(); break + print(f" {C.dgray}╰{'─'*w}╯{C.r}") + else: + try: msg=input() + except EOFError: break + msg=msg.strip() + if msg in (":q",":quit","exit"): break + if not msg: continue + if msg==":reset": msgs=[]; print(f" {C.dim}✦ new conversation{C.r}\n"); continue + msgs.append({"role":"user","content":msg}) + body=json.dumps({"model":model_id,"messages":msgs,"stream":True, + "max_tokens":a.ngen}).encode() + req=urllib.request.Request(base.rstrip("/")+"/v1/chat/completions", data=body, + headers={"Content-Type":"application/json"}) + if a.api_key: req.add_header("Authorization", f"Bearer {a.api_key}") + print(f"\n {C.teal}◆ colibrì{C.r}") + sp=Spinner("thinking…"); sp.start() + md=MDStream(" "); reply=[]; first=True; t0=time.time(); interrupted=False + try: + with urllib.request.urlopen(req) as r: + for raw in r: + line=raw.decode("utf-8","replace").strip() + if not line.startswith("data: "): continue + data=line[6:] + if data=="[DONE]": break + try: ev=json.loads(data) + except ValueError: continue + for ch in ev.get("choices",[]): + d=ch.get("delta",{}) + txt=d.get("content") + if not txt: continue # ping/ruolo/reasoning: non è testo + if first: sp.stop(); first=False + md.feed(txt); reply.append(txt) + except KeyboardInterrupt: + interrupted=True # il server annulla la richiesta alla disconnessione + except OSError as e: + sp.stop() + print(f"\n {C.yel}[server unreachable: {e}]{C.r}"); break + sp.stop(); md.close() + if reply: msgs.append({"role":"assistant","content":"".join(reply)}) + else: msgs.pop() # turno vuoto: non sporcare la history + el=time.time()-t0 + note=" · ⏹ interrupted" if interrupted else "" + print(f"\r {C.dgray}└─ ~{len(''.join(reply))//4} tok · {el:.0f}s{note}{C.r}\n") + print(f" {C.dim}goodbye — the engine keeps running for the next chat 🐦{C.r}") + +def kv_resume_notice(model_dir): + """SERVE mode silently resumes .coli_kv from disk (glm.c kv_disk_load): a chat + started today continues a conversation from days ago, with `first=0` so the + turn is appended WITHOUT the [gMASK] prefix. The engine does announce it + on stderr — but nothing here ever shows that: the drain thread's + p.stderr.read() blocks until EOF, so on a healthy start errlog is still empty + when the status lines are printed. The warning only appeared once the engine + DIED, which is exactly when it no longer mattered. + + Measured cost of the silence: a chat inherited 670 tokens of an old Italian + session ("il mio numero preferito e 7, ricordalo!"). Every later reply came + back in Italian, and "explain fibonacci in short" was answered about the + number 7 — the model was being coherent with a context nobody could see, and + it read as a quantization bug for a day. + + So say it here, in Python, from the file itself: no pipe, no thread, no + Windows deadlock risk (see the stderr comment below).""" + p=os.path.join(model_dir, ".coli_kv") + try: + with open(p,"rb") as f: + if f.read(8)!=b"COLIKV1\0": return + h=struct.unpack("<8i", f.read(32)) + n=h[6] + if n<1: return + age=time.time()-os.path.getmtime(p) + when=f"{age/86400:.0f}d ago" if age>86400 else f"{age/3600:.0f}h ago" if age>3600 else "just now" + print(f" {C.yel}↺ resuming a saved conversation: {n} tokens, last written {when}{C.r}") + print(f" {C.dgray} it steers tone, language and topic. :reset clears it · " + f"KVSAVE=0 disables saving · delete {p} to start clean{C.r}") + except (OSError, struct.error): pass + def cmd_chat(a): + # ATTACH: a running `coli serve` beats a private engine every time — the load + # (34-136 s) and the cache warmth survive between sessions. Explicit --attach + # wins; otherwise probe localhost quietly and use it if it's there. --no-attach + # forces the old behaviour. The probe costs ~1 ms when nothing is listening. + if not getattr(a,"no_attach",False): + base=getattr(a,"attach",None) or "http://127.0.0.1:8000" + mid=server_probe(base, getattr(a,"api_key",None)) + if mid: + banner(f"chat · {mid} · attached") + chat_attached(a, base, mid); return + if getattr(a,"attach",None): + sys.exit(f"--attach: no coli serve answering at {base} (start one with: coli serve --model )") need_model(a.model) banner(f"chat · {os.path.basename(a.model)} · ram {a.ram or '-'}GB · topp {a.topp or 'off'}") + kv_resume_notice(a.model) errlog=tempfile.NamedTemporaryFile(mode="w+", suffix=".log", delete=False) e=env_for(a); e["SERVE"]="1" # stderr -> PIPE, NOT stderr=errlog (file). On Windows/MinGW, pointing the @@ -619,12 +765,65 @@ def cmd_chat(a): except Exception: pass print(f" {C.teal}goodbye{C.r} {C.dim}— the hummingbird returns to its nest{C.r} 🐦\n") +def serve_pidfile(port): return os.path.join(tempfile.gettempdir(), f"coli-serve-{port}.pid") + def cmd_serve(a): need_model(a.model) + # pidfile: cosi' `coli stop` spegne tutto con un comando, senza pkill a mano. + # EN: pidfile so `coli stop` can shut everything down without manual pkill. + try: + with open(serve_pidfile(a.port),"w") as f: f.write(f"{os.getpid()} {a.model}\n") + except OSError: pass from openai_server import serve - serve(a.model, a.host, a.port, a.model_id, a.api_key, - a.cap,a.ngen,GLM,env_for(a),a.cors_origin, - a.max_queue,a.queue_timeout,a.kv_slots) + try: + serve(a.model, a.host, a.port, a.model_id, a.api_key, + a.cap,a.ngen,GLM,env_for(a),a.cors_origin, + a.max_queue,a.queue_timeout,a.kv_slots) + finally: + try: os.unlink(serve_pidfile(a.port)) + except OSError: pass + +def cmd_stop(a): + """Shut down a running `coli serve` AND its engine — one command, no pkill. + The engine re-execs itself for OMP tuning, so its process is named `exe`, + not `glm`: every `pkill -x glm` in history silently killed nothing (that is + how two 17+5 GB ghost engines OOM'd this box on 2026-07-16). This finds the + real processes: the pidfile first, then /proc by cmdline/environ — only + processes that are demonstrably ours (SERVE=1 + our SNAP, or `coli serve` + in the command line).""" + banner("stop") + targets=[] # (pid, descrizione) + pf=serve_pidfile(a.port) + try: + pid=int(open(pf).read().split()[0]) + os.kill(pid,0); targets.append((pid,f"coli serve (pidfile, port {a.port})")) + except (OSError,ValueError,IndexError): pass + for pd in os.listdir("/proc"): + if not pd.isdigit(): continue + pid=int(pd) + try: + cmd=open(f"/proc/{pd}/cmdline","rb").read().replace(b"\0",b" ").decode("utf-8","replace") + if "coli" in cmd and " serve" in cmd and pid!=os.getpid(): + if not any(p==pid for p,_ in targets): targets.append((pid,"coli serve (cmdline)")) + comm=open(f"/proc/{pd}/comm").read().strip() + if comm in ("colibri","glm","exe","olmoe"): + env=open(f"/proc/{pd}/environ","rb").read().replace(b"\0",b"\n").decode("utf-8","replace") + if "SERVE=1" in env: targets.append((pid,f"engine `{comm}` (SERVE=1)")) + except (OSError,PermissionError): continue + if not targets: + print(f" nothing running — no serve on port {a.port}, no SERVE engines"); return + for pid,desc in targets: print(f" {'would stop' if a.dry_run else 'stopping'} {pid}: {desc}") + if a.dry_run: return + for pid,_ in targets: + try: os.kill(pid, signal.SIGTERM) + except OSError: pass + time.sleep(2.0) + for pid,_ in targets: + try: os.kill(pid, signal.SIGKILL); print(f" {pid}: forced (SIGKILL)") + except OSError: pass # gia' morto: bene + try: os.unlink(pf) + except OSError: pass + print(f" {C.grn}✓ stopped{C.r} — RAM released") def cmd_web(a): """serve + open the dashboard in the browser once the API answers.""" @@ -712,6 +911,7 @@ def main(): common.add_argument("--topp", type=float, default=0); common.add_argument("--topk", type=int, default=0) common.add_argument("--temp", type=float, default=None) # temperatura token (0=greedy, default 1.0+nucleus .95) ap=argparse.ArgumentParser(prog="coli", parents=[common], description="colibrì — run GLM-5.2 locally") + ap.add_argument("--version", action="version", version=f"colibrì {_version}") sub=ap.add_subparsers(dest="cmd") sub.add_parser("build", parents=[common]); sub.add_parser("info", parents=[common]) pp=sub.add_parser("plan",parents=[common]) @@ -719,7 +919,14 @@ def main(): pd=sub.add_parser("doctor",parents=[common]) pd.add_argument("--json",action="store_true",help="emit a versioned JSON report") pr=sub.add_parser("run", parents=[common]); pr.add_argument("prompt", nargs="*") - sub.add_parser("chat", parents=[common]) + pc=sub.add_parser("chat", parents=[common]) + pc.add_argument("--attach", nargs="?", const="http://127.0.0.1:8000", default=None, + help="chat against a running `coli serve` instead of spawning an engine " + "(keeps the model loaded and the expert cache warm across chat sessions). " + "Bare --attach probes localhost:8000.") + pc.add_argument("--no-attach", action="store_true", + help="never auto-attach, always spawn a private engine") + pc.add_argument("--api-key", default=os.environ.get("COLI_API_KEY")) ps=sub.add_parser("serve", parents=[common]) ps.add_argument("--host",default="127.0.0.1"); ps.add_argument("--port",type=int,default=8000) ps.add_argument("--model-id",default=os.environ.get("COLI_MODEL_ID","glm-5.2-colibri")) @@ -728,6 +935,8 @@ def main(): ps.add_argument("--max-queue",type=int,default=int(os.environ.get("COLI_MAX_QUEUE","8"))) ps.add_argument("--queue-timeout",type=float,default=float(os.environ.get("COLI_QUEUE_TIMEOUT","300"))) ps.add_argument("--kv-slots",type=int,default=int(os.environ.get("COLI_KV_SLOTS","1"))) + pst=sub.add_parser("stop", parents=[common], help="shut down a running coli serve and its engine") + pst.add_argument("--port",type=int,default=8000); pst.add_argument("--dry-run",action="store_true") pw=sub.add_parser("web", parents=[common], help="serve + open the dashboard in a browser") for arg,kw in (("--host",dict(default="127.0.0.1")),("--port",dict(type=int,default=8000)), ("--model-id",dict(default=os.environ.get("COLI_MODEL_ID","glm-5.2-colibri"))), @@ -751,7 +960,7 @@ def main(): pc.add_argument("--no-mtp",action="store_true",help="skip the MTP head (no speculative drafts)") a=ap.parse_args() handler={"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"doctor":cmd_doctor, - "run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"bench":cmd_bench, + "run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"stop":cmd_stop,"bench":cmd_bench, "convert":cmd_convert,"web":cmd_web}.get(a.cmd) if handler: sys.exit(handler(a) or 0) banner(); print(__doc__) diff --git a/c/glm.c b/c/colibri.c similarity index 80% rename from c/glm.c rename to c/colibri.c index 3237c0a..bd91ddf 100644 --- a/c/glm.c +++ b/c/colibri.c @@ -34,9 +34,15 @@ #if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) #include #include /* mlock: inchioda le pagine in RAM / wire pages into RAM */ +#ifdef __linux__ +#include /* COLI_NUMA: mbind degli slab expert / expert-slab interleave */ +#endif #include /* fstat per mmap degli shard (COLI_MMAP) */ #include /* SIGINT = stop morbido del turno in serve mode */ #endif +#ifdef __linux__ +#include /* statfs: real fs-type check for the 9p warning (below) */ +#endif #if defined(_WIN32) && (defined(__x86_64__) || defined(__i386__)) #include /* hwinfo_emit: CPU brand string senza /proc */ #endif @@ -67,21 +73,6 @@ static int g_metal_gemm_min=16; /* COLI_METAL_GEMM_MIN: min rows to send a mat static const int *g_pre_idx; static const float *g_pre_w; static const int *g_pre_keff; static const float *g_pre_sh; /* output dello shared expert gia' calcolato su GPU */ #endif -#ifdef __AVX2__ -#include -static inline float hsum256(__m256 v){ /* somma orizzontale di 8 float */ - __m128 lo=_mm256_castps256_ps128(v), hi=_mm256_extractf128_ps(v,1); - lo=_mm_add_ps(lo,hi); __m128 sh=_mm_movehl_ps(lo,lo); lo=_mm_add_ps(lo,sh); - sh=_mm_shuffle_ps(lo,lo,1); lo=_mm_add_ss(lo,sh); return _mm_cvtss_f32(lo); -} -#elif defined(__ARM_NEON) -#include /* Apple Silicon / aarch64: kernel NEON */ -#elif defined(__VSX__) -#include /* POWER8+ (ppc64le): kernel VSX */ -#undef vector /* igiene: si usano __vector/__bool espliciti */ -#undef pixel -#undef bool -#endif #ifdef __APPLE__ #include /* host_statistics64: MemAvailable di macOS */ #endif @@ -177,6 +168,28 @@ typedef struct { uint32_t **eusage; /* contatori persistenti (per STATS/PIN) */ uint32_t **eheat; /* calore recente per promotion/demotion live */ uint32_t **elast, eaccess_clock; /* recency per LFRU session-local */ + /* DISK-CLASS: PRIVATE recency state, read only by expert_classify(). Private -- + * not the real elast/eaccess_clock -- kept fully separate so DISK-CLASS's bookkeeping + * can never read from or write into stock eviction state: every DISK-CLASS write lives + * inside its own need_classify/dc_on gate, so "byte-identical with PROF=0" is provable + * by construction instead of by argument. (Historical note: when this was first written, + * the Metal pre-routed FASE A path (g_pre_idx) never bumped the real elast/eaccess_clock + * -- on Metal decode the real clock froze at end of prefill, so REPIN's LRU tie-breaker + * ran on stale recency for the rest of the run. That was an upstream defect; it has since + * been reported and fixed (#417, cfcc742) -- FASE A now bumps the real clock too. The + * private clock is retained anyway: separation from stock state is the stronger property, + * independent of whether the real clock is correct.) elast_dc/eaccess_clock_dc tick in + * BOTH FASE A paths, under the same need_classify gate, at the same rate the real clock + * ticks on the CPU path (one per selected (position,expert)) -- so the + * COLI_DISKCLASS_WINDOW window keeps its meaning in every mode. elast_pre snapshots + * elast_dc just BEFORE this call's own bump (see the touched[] guard in FASE A) -- + * classifying against the live array would read the bump routing just made a few lines + * above the load that needed it, so a giant cold prefill burst would score every expert + * "just accessed" and get called warm. Recency alone (not eheat's access COUNT): a count + * never decays, so an expert hot early in a long session would keep reading "warm" long + * after it dropped out of the working set. Same shape/allocation as elast; NULL for dense + * layers. */ + uint32_t **elast_dc, **elast_pre, eaccess_clock_dc; /* DSA lightning indexer (attivo solo se i pesi out-idx-* sono presenti) */ int has_dsa; QT *ix_wq, *ix_wk, *ix_wp; /* per layer FULL: wq_b, wk, weights_proj */ @@ -190,12 +203,18 @@ typedef struct { uint64_t mtp_prop, mtp_acc; /* statistica acceptance */ int **eroute; int *enr; /* metodo C: routing dell'ULTIMO token per layer */ uint64_t eclock, hits, miss, ereq; + uint64_t hit_pin, hit_ecache; /* split di hits per tier (#336): pin vs LRU ecache */ uint64_t gpu_expert_calls; int gpu_expert_count; int64_t gpu_expert_bytes; uint64_t n_fw, n_emit; /* metodo E: forward di decode / token emessi */ uint64_t route_slots, route_swaps; /* CACHE_ROUTE: slots chosen / substituted vs true top-K */ uint64_t route_agree_hit, route_agree_tot; /* ROUTE_AGREE: |chosen ∩ true top-K| / K */ double route_kl_sum; uint64_t route_kl_n; /* mean KL(true||chosen) on gate mass */ - double t_edisk, t_ewait, t_emm, t_attn, t_kvb, t_head;/* profiling: dove va il tempo */ + double t_ewait, t_emm, t_ecpu, t_egpu, t_route, t_p2p, t_attn, t_kvb, t_head; + uint64_t n_p2p; /* P0 execution profile: tier split + residual hops */ + uint64_t cpu_expert_rows; int64_t cpu_expert_bytes; + /* profiling: dove va il tempo (wall del + * thread di compute; il servizio disco + * overlappato vive in g_edisk_ns) */ double t_aproj,t_acore,t_aout; /* attention breakdown */ int64_t resident_bytes; /* DISK_SPLIT=1: split dei DISK LOAD (miss LRU -> expert_load) per contesto e per tipo @@ -206,12 +225,21 @@ typedef struct { uint64_t bytes_mtp, bytes_main; /* byte letti da disco per tipo layer */ } Model; -static void usage_save(Model *m); /* cache che impara: definita accanto a stats_dump */ -static void tiers_emit(Model *m); -static void ehit_mark(Model *m, int layer, int eid); -static void emap_emit(Model *m); -static void hits_emit(Model *m); -static void hwinfo_emit(Model *m); +#include "quant.h" +static int g_no_fused_pair=0; +static int g_spec_pin=1; +static int g_spec_live=0; +static inline int spec_pinned(void){ return g_spec_pin && g_spec_live; } + +static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot); +static void matmul_qt(float *y, const float *x, QT *w, int S){ matmul_qt_ex(y,x,w,S,1); } + +static void expert_gate_up(float *g,float *u,const float *x,QT *wg,QT *wu,int S){ + if(!g_no_fused_pair&&!spec_pinned()&&S==1&&wg->fmt==2&&wu->fmt==2&&wg->I==wu->I&&wg->O==wu->O) + matmul_i4_pair(g,u,x,wg->q4,wg->s,wu->q4,wu->s,wg->I,wg->O); + else { matmul_qt(g,x,wg,S); matmul_qt(u,x,wu,S); } +} + static int g_repin; static uint64_t g_last_repin; #ifdef COLI_CUDA @@ -277,719 +305,125 @@ static double rss_gb(void){ struct rusage r; getrusage(RUSAGE_SELF,&r); return r.ru_maxrss/(1024.0*1024.0); /* Linux: in KB */ #endif } +/* ---- PROF=1: opt-in performance profile ---------------------------------- + * Records per-forward decode latency and expert-file bytes fetched, then + * reports percentiles, I/O totals, phase shares and a tuning verdict next to + * the existing PROFILE line. Additive only: with PROF unset the output of + * every mode stays byte-identical. */ +static int g_prof=0; +static _Atomic int64_t g_prof_io; /* bytes pread()/faulted from expert files */ +/* Disk service: wall time inside expert_load on whichever thread runs the read + * (PIPE I/O workers, OMP loaders, the speculative pilot). It overlaps compute, + * so it is NOT a wall-time phase — the stall the compute thread actually felt + * is m->t_ewait. Thread-seconds, so it can exceed wall time under parallel + * reads; wait << service means overlap/parallelism is hiding the reads, + * wait ~ service means the loads block the compute thread. */ +static _Atomic int64_t g_edisk_ns; +static double edisk_s(void){ return atomic_load_explicit(&g_edisk_ns,memory_order_relaxed)*1e-9; } +/* DISK-CLASS (PROF=1): per-load cold/warm classification against the engine's own + * recency state. Instrumentation only -- it never changes which fd serves a read (see + * expert_classify() and its call site in expert_load_impl; the fd choice expression is + * untouched by this feature). COLI_DISKCLASS_WINDOW is the recency window in ticks of the + * PRIVATE clock (m->eaccess_clock_dc -- NOT the real eaccess_clock; see elast_dc in Model + * for why DISK-CLASS keeps its own clock instead of reading the real one): one tick per + * selected (position,expert) in FASE A while classification is active, the same per-token + * rate the real clock has on the CPU path, so the window's meaning is unchanged. At or + * under the window = warm; 0 (default, unset) derives it from topk*n_layers*8 once the + * model config is known (main(), right after model_init) -- roughly "seen in the last ~8 + * tokens", generous on purpose (conservative-toward-warm: a load the page cache could have + * served that gets labeled cold overstates the cold class, the bucket this line exists to + * size). */ +static uint32_t g_direct_heat_ticks=0; +static int g_direct_heat_explicit=0; /* 1 if COLI_DISKCLASS_WINDOW was set (skip the auto-derive) */ +#define DC_COLD 0 +#define DC_WARM 1 +static _Atomic uint64_t g_dc_n[2]; /* [DC_COLD]/[DC_WARM]: loads classified */ +static _Atomic int64_t g_dc_bytes[2]; /* bytes read (weights + scales, matches g_prof_io) */ +static _Atomic int64_t g_dc_ns[2]; /* wall ns spent reading (thread-seconds, like g_edisk_ns) */ +static _Atomic uint64_t g_dc_direct_n[2]; /* subset of the above ACTUALLY served by the uncached fd */ +/* Busy-wall per class + combined: how much WALL time had >=1 classified load of the + * class in flight (thread-seconds / busy-wall = average concurrency; bytes / busy-wall + * = aggregate GB/s the disk actually delivered for that class -- the quantity the + * thread-second numbers alone can't answer: N slow overlapped reads can beat N fast + * serial ones in aggregate, and only wall-denominated rates see it). Transition scheme: + * 0->1 records a start, 1->0 accumulates (now - start). One dedicated mutex serializes + * the transition bookkeeping -- two short lock/unlock pairs per load against ms-scale + * reads; a CAS scheme would save nothing measurable and be harder to audit + * (correctness over cleverness). Only COMPLETED intervals are in the accumulators: an + * interval still open at report time is not counted (bounded by one read's duration -- + * noise at report granularity). */ +static pthread_mutex_t g_dc_wall_mx=PTHREAD_MUTEX_INITIALIZER; +static int g_dc_inflight[2], g_dc_inflight_all; /* guarded by g_dc_wall_mx */ +static double g_dc_wall_open[2], g_dc_wall_open_all; /* start of the open interval */ +static int64_t g_dc_wall_ns[2], g_dc_wall_all_ns; /* completed busy-wall ns */ +static void dc_wall_enter(int cls, double now){ + pthread_mutex_lock(&g_dc_wall_mx); + if(g_dc_inflight[cls]++==0) g_dc_wall_open[cls]=now; + if(g_dc_inflight_all++==0) g_dc_wall_open_all=now; + pthread_mutex_unlock(&g_dc_wall_mx); +} +static void dc_wall_exit(int cls, double now){ + pthread_mutex_lock(&g_dc_wall_mx); + if(--g_dc_inflight[cls]==0) g_dc_wall_ns[cls]+=(int64_t)((now-g_dc_wall_open[cls])*1e9); + if(--g_dc_inflight_all==0) g_dc_wall_all_ns +=(int64_t)((now-g_dc_wall_open_all)*1e9); + pthread_mutex_unlock(&g_dc_wall_mx); +} +static int dc_needed(void); /* fwd: defined with the classifier (needs g_prof) */ +static void dc_wall_read(int64_t out[2], int64_t *all){ /* mutex-consistent snapshot for the report */ + if(!dc_needed()){ out[0]=out[1]=0; *all=0; return; } /* off for the whole process => accumulators are + * provably zero (only dc_wall_exit writes them, + * only under dc_on): skip the lock, zero work. + * Needed because prof_base runs unconditionally + * at some call sites ("cheap enough to always"). */ + pthread_mutex_lock(&g_dc_wall_mx); + out[0]=g_dc_wall_ns[0]; out[1]=g_dc_wall_ns[1]; *all=g_dc_wall_all_ns; + pthread_mutex_unlock(&g_dc_wall_mx); +} +#define PROF_LAT_CAP 32768 +static double g_prof_lat[PROF_LAT_CAP]; /* per-forward decode wall clock (ring) */ +static uint64_t g_prof_nlat; /* forwards recorded (monotonic) */ +static void prof_lat(double s){ g_prof_lat[g_prof_nlat++ % PROF_LAT_CAP]=s; } +/* snapshot for windowed reports (serve mode: one report per turn) */ +typedef struct { + double edisk,ewait,emm,ecpu,egpu,route,p2p,attn,head; + int64_t io,cpu_bytes; uint64_t hits,miss,ereq,n_fw,n_emit,nlat,n_p2p,cpu_rows; + uint64_t hit_pin,hit_ecache; + uint64_t dc_n[2], dc_direct_n[2]; int64_t dc_bytes[2], dc_ns[2]; /* DISK-CLASS */ + int64_t dc_wall_ns[2], dc_wall_all_ns; /* busy-wall (per class + combined) */ +} ProfBase; +static void prof_base(Model *m, ProfBase *b){ + b->edisk=edisk_s(); b->ewait=m->t_ewait; b->emm=m->t_emm; + b->ecpu=m->t_ecpu; b->egpu=m->t_egpu; b->route=m->t_route; b->p2p=m->t_p2p; + b->attn=m->t_attn; b->head=m->t_head; + b->io=atomic_load_explicit(&g_prof_io,memory_order_relaxed); + b->hits=m->hits; b->miss=m->miss; b->ereq=m->ereq; + b->hit_pin=m->hit_pin; b->hit_ecache=m->hit_ecache; + b->n_fw=m->n_fw; b->n_emit=m->n_emit; b->nlat=g_prof_nlat; b->n_p2p=m->n_p2p; + b->cpu_bytes=m->cpu_expert_bytes;b->cpu_rows=m->cpu_expert_rows; + for(int i=0;i<2;i++){ + b->dc_n[i]=atomic_load_explicit(&g_dc_n[i],memory_order_relaxed); + b->dc_bytes[i]=atomic_load_explicit(&g_dc_bytes[i],memory_order_relaxed); + b->dc_ns[i]=atomic_load_explicit(&g_dc_ns[i],memory_order_relaxed); + b->dc_direct_n[i]=atomic_load_explicit(&g_dc_direct_n[i],memory_order_relaxed); + } + dc_wall_read(b->dc_wall_ns,&b->dc_wall_all_ns); +} + static float *falloc(int64_t n){ /* guardia anti-wrap (report PR #25): n assurdo da file modello ostili non deve * diventare una malloc piccola. Niente calloc: il memset nel percorso caldo costa. */ if(n<0 || (uint64_t)n > SIZE_MAX/sizeof(float)){ fprintf(stderr,"falloc: n=%lld is out of range\n",(long long)n); exit(1); } float *p=malloc((size_t)n*sizeof(float)); if(!p){fprintf(stderr,"OOM\n");exit(1);} return p; } -/* ---- Accumulatore int4->float a 512 bit / 512-bit int4->float accumulator ---- - * Stessa matematica lossless di matmul_i4 (nibble->f32, FMA), ma 32 pesi/iter su - * due catene FMA indipendenti. NON bit-identico al vecchio ordine: la riduzione - * ad albero accumula MENO errore della somma sequenziale (misurato 2-4x più - * vicino all'oracolo double sulle forme reali; perplexity invariata, +4-7% sul - * decode con routing CPU-heavy — vedi docs/experiments/glm52-6x5090-2026-07-12.md). - * EN: same lossless math as matmul_i4, 32 weights/iter on two independent FMA - * chains. Not bit-identical to the old order: tree reduction accumulates LESS - * rounding than sequential summation. I4_ACC512=0 restores the old order (A/B). */ -#if defined(__AVX512F__) && defined(__AVX512BW__) -static int g_i4_acc512=1; -static inline float dot_i4f_avx512(const uint8_t *w,const float *x,int I){ - const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8); - __m512 acc0=_mm512_setzero_ps(),acc1=_mm512_setzero_ps(); int i=0; - for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1))); - __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi); - __m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8)); - __m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8)); - acc0=_mm512_fmadd_ps(_mm512_loadu_ps(x+i),w0,acc0); - acc1=_mm512_fmadd_ps(_mm512_loadu_ps(x+i+16),w1,acc1); - } - return _mm512_reduce_add_ps(_mm512_add_ps(acc0,acc1)); -} -/* selftest contro il riferimento scalare (I4_ACC512_TEST=1): copre l'ordine dei - * nibble e ogni multiplo di 32. / selftest vs the scalar reference. */ -static int i4_acc512_selftest(void){ - enum { N=224 }; uint8_t w[(N+1)/2]; float x[N]; - for(int i=0;i>1]=(uint8_t)(q+8); - else w[i>>1]|=(uint8_t)((q+8)<<4); - x[i]=(float)(((i*29+7)%101)-50)/37.f; - } - for(int n=32;n<=N;n+=32){ - float ref=0; for(int i=0;i>1]>>((i&1)*4))&15)-8); - float got=dot_i4f_avx512(w,x,n),tol=2e-5f*(1.f+fabsf(ref)); - if(fabsf(got-ref)>tol){ fprintf(stderr,"AVX512 i4 selftest n=%d: %.9g != %.9g\n",n,got,ref); return 0; } - } - return 1; -} -#endif -/* y[S,O] = x[S,I] @ W^T, W[O,I] f32 */ -static void matmul(float *y, const float *x, const float *W, int S, int I, int O){ - #pragma omp parallel for schedule(static) - for (int o=0;o>1))); /* 8 byte=16 nibble */ - __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i nib=_mm_unpacklo_epi8(lo,hi); /* nibble in ordine */ - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } - a=hsum256(acc); -#elif defined(__ARM_NEON) - const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); - float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); - for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); /* 8 byte=16 nibble */ - uint8x8x2_t z=vzip_u8(vand_u8(by,m4), vshr_n_u8(by,4)); /* nibble in ordine */ - int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[0]),b8)); - int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[1]),b8)); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } - a=vaddvq_f32(vaddq_f32(ac0,ac1)); -#endif -#if defined(__AVX512F__) && defined(__AVX512BW__) - } -#endif - for(;i+1>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8; - a += xs[i]*(float)lo + xs[i+1]*(float)hi; } - if(i>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; } - y[(int64_t)s*O+o]=a*sc; } } -} -/* y[S,O] = x[S,I] @ W^T with W int4 packed (2/byte) + per-GROUP scales (fmt=4). - * Same nibble math as matmul_i4, but the scale changes every `gs` elements along I. - * The accumulator resets at each group boundary: dot(x[grp], w[grp]) * scale[grp]. - * gs MUST be a multiple of 16 (the AVX2 vector width). */ -static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const float *scale, - int S, int I, int O, int gs){ - int rb=(I+1)/2; int ng=(I+gs-1)/gs; - #pragma omp parallel for schedule(static) - for(int o=0;oI) glen=I-base; - float sc=scl[g]; - int i=base; -#ifdef __AVX2__ - const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8); - __m256 acc=_mm256_setzero_ps(); - for(; i+16<=base+glen; i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1))); - __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i nib=_mm_unpacklo_epi8(lo,hi); - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } - a+=hsum256(acc)*sc; -#endif - /* scalar tail for the group remainder */ - for(; i>1]; - a+=(xs[i]*(float)((int)(byte&0xF)-8)+xs[i+1]*(float)((int)(byte>>4)-8))*sc; } - else { uint8_t byte=w[i>>1]; a+=xs[i]*(float)((int)(byte&0xF)-8)*sc; } - } - } - y[(int64_t)s*O+o]=a; - } - } -} -/* Decode hot path for gate+up: same exact q4 dot products as matmul_i4, but one - * OpenMP dispatch covers both matrices. KTransformers uses persistent pools; - * this keeps colibri dependency-free while removing one team launch/expert. */ -static void matmul_i4_pair(float *yg, float *yu, const float *x, - const uint8_t *qg, const float *sg, - const uint8_t *qu, const float *su, int I, int O){ - int rb=(I+1)/2; - #pragma omp parallel for schedule(static) - for(int z=0;z<2*O;z++){ - int o=z>1))); - __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i nib=_mm_unpacklo_epi8(lo,hi); - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); } - a=hsum256(acc); -#elif defined(__ARM_NEON) - const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); - float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0); - for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); - uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4)); - int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8)); - int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8)); - ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); - ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); - ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); - ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } - a=vaddvq_f32(vaddq_f32(ac0,ac1)); -#endif -#if defined(__AVX512F__) && defined(__AVX512BW__) - } -#endif - for(;i+1>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); } - if(i>1]&15)-8); - (z=2) non calcolano la STESSA funzione. Tre interruttori dipendono da S: il gate - * int4-IDOT (S>=g_i4s — asimmetrico proprio dove g_i4s>1), la fusione gate+up solo-S==1, - * e la soglia righe del GEMM Metal. Con SPEC_PIN=1 (default) ogni forward emesso mentre - * i draft del modello sono attivi resta sulla famiglia di kernel di S=1: draft e verifica - * coincidono per costruzione. Prefill e decode non speculativo sono intoccati. - * EN: MTP acceptance collapses when the draft (S=1) and verify (S>=2) forwards do not - * compute the SAME function. Three switches are S-dependent: the int4 IDOT gate - * (S>=g_i4s — asymmetric exactly on ISAs where g_i4s>1), the S==1-only gate+up fusion, - * and the Metal GEMM row threshold. SPEC_PIN=1 (default) pins every forward issued - * while model drafts are live to the platform's S=1 kernel family, so draft and verify - * agree by construction; prefill and non-speculative decode are untouched. - * SPEC_PIN=0 restores the S-dependent gates (A/B). */ -static int g_spec_pin=1; -static int g_spec_live=0; /* set by spec_decode while drafts are live */ -static inline int spec_pinned(void){ return g_spec_pin && g_spec_live; } -static void expert_gate_up(float *g,float *u,const float *x,QT *wg,QT *wu,int S){ - if(!g_no_fused_pair&&!spec_pinned()&&S==1&&wg->fmt==2&&wu->fmt==2&&wg->I==wu->I&&wg->O==wu->O) - matmul_i4_pair(g,u,x,wg->q4,wg->s,wu->q4,wu->s,wg->I,wg->O); - else { matmul_qt(g,x,wg,S); matmul_qt(u,x,wu,S); } -} -/* y[S,O] = x[S,I] @ W^T con W int2 impacchettato (4 valori/byte) + scala[O]. nibble 2-bit -> [-2,1]. */ -static void matmul_i2(float *y, const float *x, const uint8_t *q2, const float *scale, int S, int I, int O){ - int rb=(I+3)/4; - #pragma omp parallel for schedule(static) - for (int o=0;o>2))); /* 4 byte=16 valori */ - __m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2); - __m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2); - __m128i lo=_mm_unpacklo_epi8(p0,p1), hi=_mm_unpacklo_epi8(p2,p3); - __m128i nib=_mm_unpacklo_epi16(lo,hi); /* 16 valori in ordine */ - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b2)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b2)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } - a=hsum256(acc); -#elif defined(__ARM_NEON) - const uint8x8_t m2v=vdup_n_u8(3); const int8x8_t b2v=vdup_n_s8(2); - float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); - for(;i+16<=I;i+=16){ uint32_t wd; memcpy(&wd, w+(i>>2), 4); /* 4 byte=16 valori */ - uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd)); - uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v)); - uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6)); - uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0])); - int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[0]),b2v)); /* 16 valori in ordine */ - int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[1]),b2v)); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } - a=vaddvq_f32(vaddq_f32(ac0,ac1)); -#endif - for(;i>2]; int sh=(i&3)*2; a += xs[i]*(float)((int)((byte>>sh)&3)-2); } - y[(int64_t)s*O+o]=a*sc; } } -} -/* ---- KERNEL INTERI (IDOT): attivazioni quantizzate a int8 per riga (absmax/127, - * stile Q8_0), prodotto scalare INTERO via maddubs/madd AVX2 — niente conversione - * f32 dei pesi nel ciclo caldo. ~2-3x sui matmul quantizzati; errore aggiunto ~0.3% - * RMS per matmul (attivazione int8), IDOT=0 torna al percorso f32 esatto. */ -#if defined(__AVX512VNNI__) && defined(__AVX512BW__) -#define IDOT_KERNEL "avx512-vnni" -#elif defined(__AVXVNNI__) && defined(__AVX2__) -#define IDOT_KERNEL "avx-vnni" -#elif defined(__AVX2__) -#define IDOT_KERNEL "avx2" -#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) -#define IDOT_KERNEL "neon-i8mm" -#elif defined(__ARM_NEON) -#define IDOT_KERNEL "neon" -#elif defined(__VSX__) -#define IDOT_KERNEL "vsx" -#else -#define IDOT_KERNEL "scalar" -#endif -static int g_idot=1; -#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) -static int g_i4s=1; /* SDOT presente: int4 IDOT conviene anche a S=1 (decode). Misurato - * su Apple M-series: +14%%, expert-matmul -16%%. EN: with SDOT, int4 - * IDOT pays even at S=1 (decode); measured on Apple M-series. */ -#elif defined(__VSX__) -static int g_i4s=1; /* POWER8 vec_msum: qui il fallback f32 e' SCALARE, quindi l'IDOT - * int4 conviene anche a S=1. Misurato su POWER8 S824 (vedi PR). - * EN: on VSX the f32 fallback is plain scalar C, so int4 IDOT - * pays even at S=1. Measured on a POWER8 S824 (see PR). */ -#else -static int g_i4s=2; /* senza SDOT / altrove: soglia originale (misura AVX2 dell'autore). - * EN: without SDOT / elsewhere: original threshold (author's AVX2). */ -#endif -static inline float qrow_i8(const float *x, int8_t *q, int I){ - float amax=0; for(int i=0;iamax)amax=a; } - float s=amax/127.f; if(s<1e-12f) s=1e-12f; float inv=1.f/s; - for(int i=0;i s32 directly, 64 bytes/iter, no 16-bit intermediate. - * AVX-512 has no vpsignb: |w| via abs, sign folded into x with a mask-negate - * (w==0 -> product 0 either way). |x|<=127 (qrow_i8), |w|<=128 as u8: each - * s32 lane adds <= 4*128*127, safe up to I=16384 like the AVX2 bound. */ - __m512i acc=_mm512_setzero_si512(); - for(;i+64<=I;i+=64){ - __m512i wv=_mm512_loadu_si512((const void*)(w+i)); - __m512i xv=_mm512_loadu_si512((const void*)(x+i)); - __mmask64 neg=_mm512_movepi8_mask(wv); - __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv); - acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs); - } - sum=_mm512_reduce_add_epi32(acc); -#elif defined(__AVXVNNI__) && defined(__AVX2__) - /* AVX-VNNI 128-bit: vpdpbusd u8*s8 -> s32, 16 byte/iter. Stesso trucco del - * segno della variante 512-bit: |w| via abs, segno piegato in x con maschera - * (w==0 -> product 0). __AVX2__ serve per _mm_sign_epi8 / abs. */ - __m128i acc=_mm_setzero_si128(); - for(;i+16<=I;i+=16){ - __m128i wv=_mm_loadu_si128((const __m128i*)(w+i)); - __m128i xv=_mm_loadu_si128((const __m128i*)(x+i)); - __m128i xs=_mm_sign_epi8(xv,wv); /* x * sign(w); _mm_sign zona __AVX2__ */ - acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(wv),xs); - } - sum=hsum128_i32(acc); -#elif defined(__AVX2__) - __m256i acc=_mm256_setzero_si256(); const __m256i ones=_mm256_set1_epi16(1); - for(;i+32<=I;i+=32){ - __m256i wv=_mm256_loadu_si256((const __m256i*)(w+i)); - __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); - __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); - acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); - } - sum=hsum256_i32(acc); -#elif defined(__ARM_NEON) - /* ARM: SDOT nativo se disponibile (Apple Silicon: sempre); altrimenti vmull/vpadal. - * Stesso bound anti-overflow del trucco AVX2: coppie <= 128*127*2 = 32512 < 32767. */ -#if defined(__ARM_FEATURE_DOTPROD) - /* 4 accumulatori indipendenti: SDOT ha latenza ~3-4 cicli, con un solo acc la - * catena seriale strozza il core a ~26 GB/s di pesi; con 4 lane indipendenti il - * dot diventa memory-bound (misurato su M4: 26 -> 63 GB/s per core, 2.4x). */ - int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); - for(;i+64<=I;i+=64){ - a0=vdotq_s32(a0,vld1q_s8(w+i), vld1q_s8(x+i)); - a1=vdotq_s32(a1,vld1q_s8(w+i+16),vld1q_s8(x+i+16)); - a2=vdotq_s32(a2,vld1q_s8(w+i+32),vld1q_s8(x+i+32)); - a3=vdotq_s32(a3,vld1q_s8(w+i+48),vld1q_s8(x+i+48)); - } - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - for(;i+16<=I;i+=16) acc=vdotq_s32(acc,vld1q_s8(w+i),vld1q_s8(x+i)); - sum=vaddvq_s32(acc); -#else - int32x4_t acc=vdupq_n_s32(0); - for(;i+16<=I;i+=16){ - int8x16_t wv=vld1q_s8(w+i), xv=vld1q_s8(x+i); - int16x8_t p=vmull_s8(vget_low_s8(wv),vget_low_s8(xv)); - p=vmlal_s8(p,vget_high_s8(wv),vget_high_s8(xv)); - acc=vpadalq_s16(acc,p); - } - sum=vaddvq_s32(acc); -#endif -#elif defined(__VSX__) - /* POWER8: vec_msum (s8 x u8 -> s32) somma i prodotti byte DIRETTAMENTE in lane - * s32, 16 byte/iter: il bound anti-saturazione a 16 bit di maddubs qui non serve. - * Stesso trucco del segno (|w| u8 per x*sign(w) s8), ma |w| via select+sub MODULO - * e non vec_abs: -128 deve diventare 128 u8, non saturare a 127. - * EN: vec_msum accumulates byte products straight into s32 lanes; |w| is built - * with a modulo subtract select instead of vec_abs so w=-128 wraps to 128 (u8) - * rather than saturating to 127. |x|<=127 from qrow_i8, so x negation is safe. */ - __vector signed int acc=vec_splats(0); - const __vector signed char vz=vec_splats((signed char)0); - for(;i+16<=I;i+=16){ - __vector signed char wv=vec_xl(0,(const signed char*)(w+i)); - __vector signed char xv=vec_xl(0,(const signed char*)(x+i)); - __vector __bool char neg=vec_cmplt(wv,vz); - __vector signed char xs=vec_sel(xv,vec_sub(vz,xv),neg); - __vector unsigned char wa=(__vector unsigned char)vec_sel(wv,vec_sub(vz,wv),neg); - acc=vec_msum(xs,wa,acc); - } - sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); -#endif - for(;i int8 [-8,7] al volo, poi stesso trucco */ -static inline int32_t dot_i4i8(const uint8_t *w4, const int8_t *x, int I){ - int32_t sum=0; int i=0; -#if defined(__AVX512VNNI__) && defined(__AVX512BW__) - /* 32 bytes = 64 nibbles -> int8 in [-8,7], one vpdpbusd per 64 values. - * 256-bit unpack leaves values in per-128-lane order [0-15][32-47]/[16-31][48-63]; - * dot pairing is order-invariant, so permute x's 128-bit blocks to match - * instead of re-ordering w (one vpermq per iter, off the critical unpack path). */ - const __m256i m4v=_mm256_set1_epi8(0x0F); - const __m512i b8v=_mm512_set1_epi8(8); - const __m512i xidx=_mm512_setr_epi64(0,1,4,5,2,3,6,7); - __m512i acc=_mm512_setzero_si512(); - for(;i+64<=I;i+=64){ - __m256i by=_mm256_loadu_si256((const __m256i*)(w4+(i>>1))); - __m256i lo=_mm256_and_si256(by,m4v), hi=_mm256_and_si256(_mm256_srli_epi16(by,4),m4v); - __m256i z0=_mm256_unpacklo_epi8(lo,hi), z1=_mm256_unpackhi_epi8(lo,hi); - __m512i wv=_mm512_sub_epi8(_mm512_inserti64x4(_mm512_castsi256_si512(z0),z1,1),b8v); - __m512i xv=_mm512_permutexvar_epi64(xidx,_mm512_loadu_si512((const void*)(x+i))); - __mmask64 neg=_mm512_movepi8_mask(wv); - __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv); - acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs); - } - sum=_mm512_reduce_add_epi32(acc); -#elif defined(__AVXVNNI__) && defined(__AVX2__) - /* AVX-VNNI 128-bit, int4: 16 byte = 32 nibble -> int8 [-8,7] in due half - * (n0/n1), ciascuno alimentato a un vpdpbusd da 16 byte. Stesso unpack - * 128-bit del ramo AVX2 sotto; 32 elementi/iter come li. */ - const __m128i m4=_mm_set1_epi8(0x0F); const __m128i b8=_mm_set1_epi8(8); - __m128i acc=_mm_setzero_si128(); - for(;i+32<=I;i+=32){ - __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); /* 16 byte = 32 nibble */ - __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); /* nibble in ordine */ - __m128i w0=_mm_sub_epi8(n0,b8), w1=_mm_sub_epi8(n1,b8); - __m128i x0=_mm_loadu_si128((const __m128i*)(x+i)); - __m128i x1=_mm_loadu_si128((const __m128i*)(x+i+16)); - acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0)); - acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1)); - } - sum=hsum128_i32(acc); -#elif defined(__AVX2__) - const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi8(8); - const __m256i ones=_mm256_set1_epi16(1); - __m256i acc=_mm256_setzero_si256(); - for(;i+32<=I;i+=32){ - __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); /* 16 byte = 32 nibble */ - __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); /* in ordine */ - __m256i wv=_mm256_sub_epi8(_mm256_set_m128i(n1,n0),b8); - __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); - __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); - acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); - } - sum=hsum256_i32(acc); -#elif defined(__ARM_NEON) - const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8); -#if defined(__ARM_FEATURE_DOTPROD) - /* 4 accumulatori indipendenti (vedi dot_i8i8): spezza la catena seriale su acc. - * Misurato su M4: 12.4 -> 29.9 GB/s di pesi per core (2.4x). */ - int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); - for(;i+64<=I;i+=64){ - uint8x16_t byA=vld1q_u8(w4+(i>>1)), byB=vld1q_u8(w4+(i>>1)+16); - uint8x16x2_t zA=vzipq_u8(vandq_u8(byA,m4q), vshrq_n_u8(byA,4)); /* nibble in ordine */ - uint8x16x2_t zB=vzipq_u8(vandq_u8(byB,m4q), vshrq_n_u8(byB,4)); - a0=vdotq_s32(a0,vsubq_s8(vreinterpretq_s8_u8(zA.val[0]),b8q),vld1q_s8(x+i)); - a1=vdotq_s32(a1,vsubq_s8(vreinterpretq_s8_u8(zA.val[1]),b8q),vld1q_s8(x+i+16)); - a2=vdotq_s32(a2,vsubq_s8(vreinterpretq_s8_u8(zB.val[0]),b8q),vld1q_s8(x+i+32)); - a3=vdotq_s32(a3,vsubq_s8(vreinterpretq_s8_u8(zB.val[1]),b8q),vld1q_s8(x+i+48)); - } - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - for(;i+32<=I;i+=32){ - uint8x16_t by=vld1q_u8(w4+(i>>1)); /* 16 byte = 32 nibble */ - uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); /* nibble in ordine */ - acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q),vld1q_s8(x+i)); - acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q),vld1q_s8(x+i+16)); - } - sum=vaddvq_s32(acc); -#else - int32x4_t acc=vdupq_n_s32(0); - for(;i+32<=I;i+=32){ - uint8x16_t by=vld1q_u8(w4+(i>>1)); /* 16 byte = 32 nibble */ - uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); /* nibble in ordine */ - int8x16_t w0=vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q); - int8x16_t w1=vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q); - int8x16_t x0=vld1q_s8(x+i), x1=vld1q_s8(x+i+16); - int16x8_t p=vmull_s8(vget_low_s8(w0),vget_low_s8(x0)); /* |w|<=8: nessun overflow */ - p=vmlal_s8(p,vget_high_s8(w0),vget_high_s8(x0)); - acc=vpadalq_s16(acc,p); - p=vmull_s8(vget_low_s8(w1),vget_low_s8(x1)); - p=vmlal_s8(p,vget_high_s8(w1),vget_high_s8(x1)); - acc=vpadalq_s16(acc,p); - } - sum=vaddvq_s32(acc); -#endif -#elif defined(__VSX__) - /* 16 byte = 32 nibble. vec_mergeh/vec_mergel su ppc64le (GCC) interallacciano come - * unpacklo/unpackhi x86 (verificato empiricamente su POWER8): i nibble escono in - * ordine di memoria. |w|<=8 dopo il -8, quindi stesso trucco del segno di dot_i8i8. - * EN: vec_mergeh/l on ppc64le interleave like x86 unpacklo/hi (verified on POWER8), - * so nibbles come out in memory order; then the same sign trick as dot_i8i8. */ - const __vector unsigned char m4v=vec_splats((unsigned char)0x0F); - const __vector unsigned char sh4=vec_splats((unsigned char)4); - const __vector signed char b8v=vec_splats((signed char)8); - const __vector signed char vz=vec_splats((signed char)0); - __vector signed int acc=vec_splats(0); - for(;i+32<=I;i+=32){ - __vector unsigned char by=vec_xl(0,w4+(i>>1)); /* 16 byte = 32 nibble */ - __vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4); - __vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v); - __vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v); - __vector signed char x0=vec_xl(0,(const signed char*)(x+i)); - __vector signed char x1=vec_xl(0,(const signed char*)(x+i+16)); - __vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz); - acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0), - (__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc); - acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1), - (__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc); - } - sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); -#endif - for(;i+1>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } - if(i>1]; sum+=((int)(b&0xF)-8)*x[i]; } - return sum; -} -#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) -/* SMMLA (i8mm): vmmlaq_s32 vede ogni int8x16_t come matrice 2x8 row-major (byte 0-7 = - * riga 0, byte 8-15 = riga 1) e accumula C += A*B^T nel 2x2 int32: lane0=a0.b0, - * lane1=a0.b1, lane2=a1.b0, lane3=a1.b1. vcombine di due mezze-righe costruisce la - * matrice: A = due righe di peso (o,o+1), B = due righe di attivazione (s,s+1), quindi - * meta' traffico pesi e doppio lavoro per istruzione a S>=2. EN: vmmlaq_s32 treats each - * int8x16_t as a 2x8 row-major matrix and does C += A*B^T on a 2x2 int32 tile; vcombine - * of vget_low/high halves builds the 2-row register from two weight/activation rows. */ -static inline int32x4_t mm_tile16(int32x4_t acc, int8x16_t wo, int8x16_t wo1, - int8x16_t xs, int8x16_t xs1){ - acc=vmmlaq_s32(acc, vcombine_s8(vget_low_s8(wo), vget_low_s8(wo1)), - vcombine_s8(vget_low_s8(xs), vget_low_s8(xs1))); - return vmmlaq_s32(acc, vcombine_s8(vget_high_s8(wo), vget_high_s8(wo1)), - vcombine_s8(vget_high_s8(xs), vget_high_s8(xs1))); -} -static void matmul_q_idot_mm(float *y, const int8_t *xq, const float *sx, const int8_t *q, - const float *scale, int S, int I, int O){ - #pragma omp parallel for schedule(static) - for(int o=0;o<(O&~1);o+=2){ - const int8_t *wo=q+(int64_t)o*I, *wo1=q+(int64_t)(o+1)*I; - float sc0=scale[o], sc1=scale[o+1]; - for(int s=0;s<(S&~1);s+=2){ - const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I; - /* 4 accumulatori indipendenti: una sola catena vmmla e' latency-bound. - * EN: 4 independent accumulators; a single vmmla chain is latency-bound. */ - int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0; - for(;i+64<=I;i+=64){ - a0=mm_tile16(a0,vld1q_s8(wo+i), vld1q_s8(wo1+i), vld1q_s8(xs+i), vld1q_s8(xs1+i)); - a1=mm_tile16(a1,vld1q_s8(wo+i+16),vld1q_s8(wo1+i+16),vld1q_s8(xs+i+16),vld1q_s8(xs1+i+16)); - a2=mm_tile16(a2,vld1q_s8(wo+i+32),vld1q_s8(wo1+i+32),vld1q_s8(xs+i+32),vld1q_s8(xs1+i+32)); - a3=mm_tile16(a3,vld1q_s8(wo+i+48),vld1q_s8(wo1+i+48),vld1q_s8(xs+i+48),vld1q_s8(xs1+i+48)); - } - for(;i+16<=I;i+=16) - a0=mm_tile16(a0,vld1q_s8(wo+i),vld1q_s8(wo1+i),vld1q_s8(xs+i),vld1q_s8(xs1+i)); - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); - int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); - for(;i>1)), byo1=vld1q_u8(wo1+(i>>1)); - uint8x16_t cyo=vld1q_u8(wo+(i>>1)+16), cyo1=vld1q_u8(wo1+(i>>1)+16); - uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); - uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); - uint8x16x2_t ko =vzipq_u8(vandq_u8(cyo, m4q), vshrq_n_u8(cyo, 4)); - uint8x16x2_t ko1=vzipq_u8(vandq_u8(cyo1,m4q), vshrq_n_u8(cyo1,4)); - a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), - vld1q_s8(xs+i), vld1q_s8(xs1+i)); - a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), - vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); - a2=mm_tile16(a2, vsubq_s8(vreinterpretq_s8_u8(ko.val[0]),b8q), - vsubq_s8(vreinterpretq_s8_u8(ko1.val[0]),b8q), - vld1q_s8(xs+i+32), vld1q_s8(xs1+i+32)); - a3=mm_tile16(a3, vsubq_s8(vreinterpretq_s8_u8(ko.val[1]),b8q), - vsubq_s8(vreinterpretq_s8_u8(ko1.val[1]),b8q), - vld1q_s8(xs+i+48), vld1q_s8(xs1+i+48)); - } - for(;i+32<=I;i+=32){ - uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1)); - uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); - uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); - a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), - vld1q_s8(xs+i), vld1q_s8(xs1+i)); - a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), - vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); - } - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); - int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); - for(;i+1>1], bo1=wo1[i>>1]; - int a0=(int)(bo&0xF)-8, a1=(int)(bo>>4)-8, b0=(int)(bo1&0xF)-8, b1=(int)(bo1>>4)-8; - int u0=xs[i],u1=xs[i+1],v0=xs1[i],v1=xs1[i+1]; - d00+=a0*u0+a1*u1; d01+=a0*v0+a1*v1; d10+=b0*u0+b1*u1; d11+=b0*v0+b1*v1; } - if(i>1], bo1=wo1[i>>1]; - int a0=(int)(bo&0xF)-8, b0=(int)(bo1&0xF)-8; - d00+=a0*xs[i]; d01+=a0*xs1[i]; d10+=b0*xs[i]; d11+=b0*xs1[i]; } - y[(int64_t)s*O+o] =(float)d00*sc0*sx[s]; - y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s]; - y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1]; - y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1]; - } - if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I; - y[(int64_t)s*O+o] =(float)dot_i4i8(wo, xs,I)*sc0*sx[s]; - y[(int64_t)s*O+(o+1)]=(float)dot_i4i8(wo1,xs,I)*sc1*sx[s]; } - } - if(O&1){ int o=O-1; const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o]; - #pragma omp parallel for schedule(static) - for(int s=0;s=2){ matmul_q_idot_mm(y,xq,sx,q,scale,S,I,O); return; } -#endif - #pragma omp parallel for schedule(static) - for(int o=0;o=2){ matmul_i4_idot_mm(y,xq,sx,q4,scale,S,I,O); return; } -#endif - #pragma omp parallel for schedule(static) - for(int o=0;og_qscratch.xq_cap){ - int8_t *p=realloc(g_qscratch.xq,xn); - if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); } - g_qscratch.xq=p; g_qscratch.xq_cap=xn; - } - if(sn>g_qscratch.sx_cap){ - float *p=realloc(g_qscratch.sx,sn*sizeof(float)); - if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); } - g_qscratch.sx=p; g_qscratch.sx_cap=sn; - } - *xq=g_qscratch.xq; *sx=g_qscratch.sx; -} - -/* allow_idot=0: forza il kernel int4/int8 ESATTO (attivazioni f32). Serve alle proiezioni di - * attenzione: sono sensibili alla quantizzazione int8 delle attivazioni dell'IDOT. Misurato su - * GLM-5.2 int4, 1023 token, log-lik -5040.33 (esatto) -> -5160.47 (IDOT) = +0.117 nat/token, - * ~+12% perplexity. Gli altri matmul del prefill (o_proj, kv_b, expert) tengono l'IDOT. - * EN: allow_idot=0 forces the EXACT int4/int8 kernel (f32 activations). The attention - * projections need it: IDOT's int8 activation quantization costs +0.117 nats/token there - * (~+12% perplexity), measured. Every other prefill matmul keeps IDOT as before. */ -static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot); -static void matmul_qt(float *y, const float *x, QT *w, int S){ matmul_qt_ex(y,x,w,S,1); } static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot){ #ifdef COLI_METAL - /* Large row-batches (prefill: kv_b reconstruction, o_proj, dense MLP, step_all logits) - * amortize Metal's ~5ms submit latency; small-S decode matmuls stay on CPU (NEON wins). - * Weights must be registered (all dense QT allocs are, via qalloc). */ if(g_metal_enabled && S>=g_metal_gemm_min && !spec_pinned() && (w->fmt==1||w->fmt==2) && !omp_in_parallel()){ const void *wp = w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4; if(coli_metal_gemm(y,x,wp,w->s,w->fmt,S,w->I,w->O)) return; } #endif #ifdef COLI_CUDA - /* The CUDA backend owns persistent copies only for model-resident tensors. - * Streaming expert slots are reused for different IDs and must never enter - * this cache. Nested OpenMP calls stay on CPU because each device context - * intentionally owns one synchronous scratch stream in this stage. */ if(g_cuda_enabled && w->cuda_eligible && !w->cuda_failed && !omp_in_parallel()){ const void *weights = w->fmt==0 ? (const void*)w->qf : w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4; @@ -1000,21 +434,7 @@ static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot) } #endif if(w->fmt==0){ matmul(y,x,w->qf,S,w->I,w->O); return; } - /* fmt=4: grouped int4 — always use the exact grouped kernel (no IDOT approximation, - * since the whole point of grouped scales is better quality). */ if(w->fmt==4){ matmul_i4_grouped(y,x,w->q4,w->s,S,w->I,w->O,w->gs); return; } - /* int8 IDOT vince sempre (1.4-2.5x). int4 IDOT: l'autore su AVX2 trovo' che a S=1 - * non ripaga (soglia S>=2); ma su ARM/SDOT il singolo token CONVIENE (vedi g_i4s / - * PR #9 per il gemello VNNI). Soglia configurabile con I4S. - * EN: int8 IDOT always wins (1.4-2.5x). int4 IDOT: on AVX2 the author found S=1 didn't - * pay (S>=2 gate); on ARM/SDOT single-token DOES pay (see g_i4s / PR #9 for the VNNI - * twin). Threshold configurable via I4S. */ - /* #163: sotto SPEC_PIN il gate int4-IDOT usa la decisione di S=1 per OGNI S, cosi' - * draft e verifica restano sulla stessa famiglia. (CUDA non e' toccato: la sua - * condizione non dipende da S, quindi e' gia' coerente tra draft e verifica.) - * EN: under SPEC_PIN the int4 IDOT gate uses the S=1 decision for EVERY S, so draft - * and verify stay in one family. (CUDA untouched: its condition is S-independent, - * hence already draft/verify-consistent.) */ if(allow_idot && g_idot && (w->fmt==1 || (w->fmt==2 && (spec_pinned() ? g_i4s<=1 : S>=g_i4s)))){ int I=w->I; int8_t *xq; float *sx; if(S<0 || I<0 || (size_t)S>SIZE_MAX/(size_t)(I?I:1)){ fprintf(stderr,"matmul_qt: shape overflow\n"); exit(1); } @@ -1029,49 +449,6 @@ static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot) else matmul_i4(y,x,w->q4,w->s,S,w->I,w->O); } -/* quantizza w[O,I] f32 -> int8 q[O,I] + scala[O] simmetrica per riga */ -static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits){ - int qmax=(1<<(bits-1))-1; - #pragma omp parallel for schedule(static) - for(int o=0;oamax)amax=a; } - float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; - int8_t *qr=q+(int64_t)o*I; - for(int i=0;iqmax)v=qmax; if(v<-qmax-1)v=-qmax-1; qr[i]=(int8_t)v; } - } -} -/* quantizza w[O,I] f32 -> int4 impacchettato (2/byte) + scala[O]. - * bits<=4: valori in [-qmax-1,qmax] stanno in un nibble [-8,7]; memorizzati come v+8 (0..15). */ -static void pack_int4(const float *w, uint8_t *q4, float *scale, int O, int I, int bits){ - int qmax=(1<<(bits-1))-1, rb=(I+1)/2; - #pragma omp parallel for schedule(static) - for(int o=0;oamax)amax=a; } - float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; - uint8_t *qr=q4+(int64_t)o*rb; - for(int i=0;iqmax)v0=qmax; if(v0<-8)v0=-8; - int v1=0; if(i+1qmax)v1=qmax; if(v1<-8)v1=-8; } - qr[i>>1] = (uint8_t)((v0+8) | ((v1+8)<<4)); - } - } -} - -/* quantizza w[O,I] f32 -> int2 impacchettato (4/byte) + scala[O]. valori nibble 2-bit in [-2,1]. */ -static void pack_int2(const float *w, uint8_t *q2, float *scale, int O, int I, int bits){ - int qmax=(1<<(bits-1))-1, rb=(I+3)/4; - #pragma omp parallel for schedule(static) - for(int o=0;oamax)amax=a; } - float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; - uint8_t *qr=q2+(int64_t)o*rb; - for(int i=0;iqmax)v=qmax; if(v<-2)v=-2; byte|=(uint8_t)((v+2)<<(k*2)); } - qr[i>>2]=byte; - } - } -} - static int g_nopack=0; /* NOPACK=1 -> tiene i valori <=4bit in contenitore int8 (per validare il packing) */ static int g_drop=0; /* DROP=1 -> scarta le pagine expart dopo l'uso. Default 0: le lascia in * page-cache (buff/cache, NON RSS) come L2 gratuito -> sfrutta lo @@ -1157,15 +534,57 @@ static int g_disk_split=0; /* DISK_SPLIT=1: contatori che spezzano i DISK LOAD ( * (int4), con i byte letti. Default OFF: a flag spento gli atomic * non vengono MAI toccati (zero overhead), le righe extra di stats * non vengono stampate. Solo misura: nessun effetto sull'output. */ + +#include "sample.h" +#include "kv_persist.h" +#include "telemetry.h" + /* Aligned allocator for dense QT weights/scales: under METAL, page-align + register so the * GPU reads them zero-copy (no upload duplicate). Plain malloc otherwise. */ +/* ---- COLI_NUMA=1 (#82): interleave the expert slabs across NUMA nodes ---- + * On multi-socket hosts first-touch parks nearly the whole pin+LRU on the loader + * thread's node (measured: node0 766MB free / node1 idle), and every far-socket + * core then streams weights over the interconnect. Interleaving ONLY the expert + * slabs recruits all memory controllers: +7%/-14% expert-matmul on 2 sockets, + * +40% on a 4-socket (#82). Blanket `numactl --interleave=all` is NOT equivalent: + * it also interleaves the CUDA pinned staging buffers and cost a 4-socket GPU host + * 10x (#82) — hence per-region mbind here and nothing else. Raw syscall, no libnuma + * dependency; MPOL_MF_MOVE migrates pages of reused heap chunks too. Linux-only, + * silent no-op elsewhere or on single-node hosts. */ +#ifdef __linux__ +static int g_numa_nodes=0; /* only touched under __linux__; off-Linux NUMA is a no-op */ +#endif +static void numa_slab_bind(void *p, size_t n){ +#ifdef __linux__ + if(g_numa_nodes<2 || !p || !n) return; + unsigned long mask=(1UL<=2) fprintf(stderr,"[NUMA] expert slabs interleaved across %d nodes\n",g_numa_nodes); + else fprintf(stderr,"[NUMA] single node: COLI_NUMA ignored\n"); +#endif +} + static void *qalloc(size_t n){ #ifdef COLI_METAL if(g_metal_enabled){ void *p; size_t r=(n+16383)&~(size_t)16383; if(posix_memalign(&p,16384,r)){fprintf(stderr,"OOM qalloc\n");exit(1);} coli_metal_register(p,r); return p; } #endif - return malloc(n); + void *p=malloc(n); + if(n>=(size_t)1<<20) numa_slab_bind(p,n); /* resident dense weights too (#82: attention/shared stream from RAM every token) */ + return p; } static float *qsalloc(int O){ return (float*)qalloc((size_t)O*sizeof(float)); } static int g_pilot_real=0;/* PILOT_REAL=1: il pilota fa LOAD VERI cross-layer dentro ecache[L+1] @@ -1250,7 +669,12 @@ static jval* cfg_root(const char *snap, char **arena){ char p[2048]; snprintf(p,sizeof(p),"%s/config.json",snap); FILE *f=fopen(p,"rb"); if(!f){perror(p);exit(1);} fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *b=malloc(n+1); size_t got=fread(b,1,n,f); b[got]=0; fclose(f); + /* SEC: config.json arriva dalla dir modello non fidata. Limita la dimensione + * (un file ostile enorme = OOM al load) e controlla la malloc: senza il NULL + * check, b[got]=0 su malloc fallita era un NULL-deref. */ + if(n<0 || n>(256L<<20)){ fprintf(stderr,"%s: size %ld out of range (0..256 MB)\n",p,n); exit(1); } + char *b=malloc((size_t)n+1); if(!b){ fprintf(stderr,"OOM reading %s (%ld bytes)\n",p,n); exit(1); } + size_t got=fread(b,1,(size_t)n,f); b[got]=0; fclose(f); if((long)got!=n) fprintf(stderr,"warning: short read on %s (%ld of %ld)\n",p,(long)got,n); return json_parse(b,arena); } @@ -1277,6 +701,36 @@ static void load_cfg(Cfg *c, const char *snap){ if(eo){ if(eo->t==J_NUM) c->stop_ids[c->n_stop++]=(int)eo->num; else if(eo->t==J_ARR) for(int i=0;ilen && c->n_stop<8;i++) c->stop_ids[c->n_stop++]=(int)eo->kids[i]->num; } + /* generation_config.json e' il file AUTOREVOLE per la generazione secondo HuggingFace: + * config.json ne porta spesso una copia legacy o parziale. Un tool di conversione che + * rigenera un config.json ridotto lascia il motore fermo su MENO stop del dovuto, e i + * token di controllo che restano finiscono stampati in chat come testo (woolcoxm, #298: + * "the stop token being printed to chat", verificato sui token id). Unione dei due: + * uno stop in piu' non fa danno, uno in meno si' -- e chi converte i pesi non siamo noi. + * EN: generation_config.json is HF's authority for generation; config.json often carries + * a partial legacy copy. Union both -- an extra stop is harmless, a missing one is not. */ + { char gp[2100]; snprintf(gp,sizeof(gp),"%s/generation_config.json",snap); + FILE *gf=fopen(gp,"rb"); /* assente = nessun problema: e' opzionale */ + if(gf){ + fseek(gf,0,SEEK_END); long gn=ftell(gf); fseek(gf,0,SEEK_SET); + char *gb = (gn>0 && gn<=(256L<<20)) ? malloc((size_t)gn+1) : NULL; /* SEC: cap + NULL check */ + if(gb){ + size_t gg=fread(gb,1,(size_t)gn,gf); gb[gg]=0; + char *ga=NULL; jval *gr=json_parse(gb,&ga); + jval *ge=gr?json_get(gr,"eos_token_id"):NULL; + if(ge){ + int add[8], na=0; + if(ge->t==J_NUM) add[na++]=(int)ge->num; + else if(ge->t==J_ARR) for(int i=0;ilen && na<8;i++) add[na++]=(int)ge->kids[i]->num; + for(int i=0;in_stop<8;i++){ + int dup=0; for(int j=0;jn_stop;j++) if(c->stop_ids[j]==add[i]) dup=1; + if(!dup) c->stop_ids[c->n_stop++]=add[i]; + } + } + free(ga); free(gb); + } + fclose(gf); + } } /* DSA lightning indexer: parametri + tipo per-layer (lista esplicita o formula freq/offset) */ c->index_topk=gi(r,"index_topk"); c->index_nh=gi(r,"index_n_heads"); c->index_hd=gi(r,"index_head_dim"); { jval *it=json_get(r,"indexer_types"); @@ -1330,6 +784,27 @@ static int detect_group_size(int O, int I, int64_t ns){ return 0; } +/* SEC: risolve e VALIDA il formato quantizzato di un tensore [O,I] letto da un + * container non fidato (mirror). L'inferenza precedente (`?1:?2:3`) cadeva su + * int2 per QUALSIASI conteggio byte non riconosciuto: un peso troppo corto + * diventava un int2 valido e il matmul leggeva oltre il buffer (O*I nibble a + * 4/byte). Qui i byte del peso devono corrispondere a un layout noto e i byte + * della scala alla cardinalita' attesa (O per-row, O*ng per-gruppo) — altrimenti + * si termina invece di sforare. Ritorna fmt (1/2/3/4) e scrive *gs. */ +static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns, int *gs){ + int64_t exp_i8=(int64_t)O*I, exp_i4=(int64_t)O*((I+1)/2), exp_i2=(int64_t)O*((I+3)/4); + int fmt = (nb==exp_i8)?1 : (nb==exp_i4)?2 : (nb==exp_i2)?3 : 0; + if(!fmt){ + fprintf(stderr,"%s: quantized weight is %lld bytes — no int8/int4/int2 layout for [%d,%d], refusing (untrusted container)\n", + name,(long long)nb,O,I); exit(1); } + *gs=0; + if(fmt==2){ int g=detect_group_size(O,I,ns); if(g>0){ fmt=4; *gs=g; } } + int64_t exp_scale = (fmt==4)? (int64_t)O*((I+*gs-1)/(*gs)) : (int64_t)O; /* in FLOAT */ + if(ns != exp_scale*4){ + fprintf(stderr,"%s: scale array is %lld bytes — expected %lld for [%d,%d] fmt=%d, refusing (untrusted container)\n", + name,(long long)ns,(long long)(exp_scale*4),O,I,fmt); exit(1); } + return fmt; +} /* costruisce un QT [O,I] dal disco in `t` (buffer riusabili tra chiamate). * - se esiste `name.qs`: pesi GIA' quantizzati nel container (U8 qdata + F32 scala) -> letti diretti * - altrimenti: tensore pieno (f32/bf16) -> quantizzato a runtime a `bits` (oracolo tiny / pesi pieni) @@ -1339,12 +814,11 @@ static void qt_from_disk(Model *m, const char *name, int O, int I, int bits, int if(st_has(&m->S,sn)){ int64_t nb=st_nbytes(&m->S,name); int64_t ns=st_nbytes(&m->S,sn); /* scale bytes (F32) */ - /* Detect int4-grouped (fmt=4): packed int4 weight bytes BUT scale array is - * larger than O*4 — the group size is derived from the scale-array size. */ - int fmt = (nb==(int64_t)O*I)?1 : (nb==(int64_t)O*((I+1)/2))?2 : 3; + /* fmt=4 int4-grouped: byte int4 ma scala > O*4 — gs deriva dalla scala. + * qt_resolve_fmt valida entrambi i conteggi contro [O,I] e termina se + * non fidati (SEC). */ int gs=0; - if(fmt==2) gs=detect_group_size(O,I,ns); - if(gs>0) fmt=4; + int fmt = qt_resolve_fmt(name,O,I,nb,ns,&gs); if(fmt==1){ if(t->fmt!=1||!t->q8){ t->fmt=1; t->O=O; t->I=I; t->gs=0; t->q8=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q8,drop); } else if(fmt==4){ int ng=(I+gs-1)/gs; if(t->fmt!=4||!t->q4){ t->fmt=4; t->O=O; t->I=I; t->gs=gs; t->q4=qalloc(nb); t->s=falloc((int64_t)O*ng); } @@ -1420,6 +894,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits m->pin=calloc(NR,sizeof(ESlot*)); m->npin=calloc(NR,sizeof(int)); m->eusage=calloc(NR,sizeof(uint32_t*)); m->eheat=calloc(NR,sizeof(uint32_t*)); m->elast=calloc(NR,sizeof(uint32_t*)); + m->elast_dc=calloc(NR,sizeof(uint32_t*)); m->elast_pre=calloc(NR,sizeof(uint32_t*)); m->kv=calloc(1,sizeof(KVState)); m->kv_start=m->kv->kv_start=calloc(NR,sizeof(int)); for(int i=0;in_layers;i++){ @@ -1464,6 +939,8 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t)); m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t)); m->elast[i]=calloc(c->n_experts,sizeof(uint32_t)); + m->elast_dc[i]=calloc(c->n_experts,sizeof(uint32_t)); + m->elast_pre[i]=calloc(c->n_experts,sizeof(uint32_t)); } #undef P } @@ -1476,12 +953,17 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits "self_attn.q_a_proj.weight","self_attn.q_b_proj.weight","self_attn.kv_a_proj_with_mqa.weight", "self_attn.kv_b_proj.weight","self_attn.o_proj.weight","mlp.gate.weight", "mlp.shared_experts.gate_proj.weight","mlp.shared_experts.down_proj.weight", - "mlp.experts.0.gate_proj.weight","mlp.experts.255.down_proj.weight"}; + "mlp.experts.0.gate_proj.weight"}; char mn[256]; m->has_mtp=1; for(unsigned q=0;qn_layers,req[q]); if(!st_has(&m->S,mn)){ m->has_mtp=0; break; } } + /* probe the LAST expert by index, not a fixed 255: REAP-pruned + * checkpoints have n_routed_experts < 256 and the MTP set stays complete, + * so a hardcoded expert.255 would spuriously report has_mtp=0 on them. */ + snprintf(mn,sizeof(mn),"model.layers.%d.mlp.experts.%d.down_proj.weight",c->n_layers,c->n_experts-1); + if(!st_has(&m->S,mn)) m->has_mtp=0; if(getenv("MTP") && atoi(getenv("MTP"))==0) m->has_mtp=0; if(m->has_mtp){ int i=c->n_layers; Layer *l=&m->mtpL; @@ -1510,6 +992,8 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t)); m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t)); m->elast[i]=calloc(c->n_experts,sizeof(uint32_t)); + m->elast_dc[i]=calloc(c->n_experts,sizeof(uint32_t)); + m->elast_pre[i]=calloc(c->n_experts,sizeof(uint32_t)); m->kv_start[i]=-1; /* KV MTP: parte dalla prima posizione di decode */ #undef PM } @@ -1542,7 +1026,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits c->index_topk, c->index_topk); } } - m->hlast=falloc(D); m->h_all=falloc((int64_t)64*D); + m->hlast=falloc(D); m->h_all=falloc((int64_t)512*D); /* byte della parte DENSA residente (embed+lm_head+attn+mlp densa+shared+norme) */ int64_t rb=qt_bytes(&m->embed)+qt_bytes(&m->lm_head); @@ -1630,15 +1114,49 @@ static int pread_full(int fd, void *buf, int64_t n, int64_t off, const char *tag while(gotelast_pre || !m->elast_pre[layer]) return DC_WARM; /* no snapshot: label as the safe class */ + uint32_t last_pre=m->elast_pre[layer][eid]; + if(last_pre==0) return DC_COLD; /* never touched before this call: certain cold */ + uint32_t age=m->eaccess_clock_dc-last_pre; /* ticks since last access, PRE this call's bump */ + return age>g_direct_heat_ticks ? DC_COLD : DC_WARM; /* '>' not '>=': ties lean warm */ +} +static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal, int demand){ #ifdef COLI_CUDA /* A live REPIN may reuse a GPU-enabled pinned slot for a different expert. * Keep its tier assignment, but invalidate the old device weights. */ @@ -1654,6 +1172,8 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){ qt_from_disk(m,nm[0],I,D,b,g_drop,&s->g); qt_from_disk(m,nm[1],I,D,b,g_drop,&s->u); qt_from_disk(m,nm[2],D,I,b,g_drop,&s->d); + atomic_fetch_add_explicit(&g_prof_io, + st_nbytes(&m->S,nm[0])+st_nbytes(&m->S,nm[1])+st_nbytes(&m->S,nm[2]),memory_order_relaxed); s->eid=eid; return 0; } st_tensor *tw[3], *tq[3]; @@ -1679,11 +1199,8 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){ QT *qt[3]={&s->g,&s->u,&s->d}; int OO[3]={I,I,D}, II[3]={D,D,I}; for(int k=0;k<3;k++){ int64_t nb=tw[k]->nbytes; - int fmt=(nb==(int64_t)OO[k]*II[k])?1:(nb==(int64_t)OO[k]*((II[k]+1)/2))?2:3; - /* detect grouped int4 (fmt=4): int4 weight bytes + larger scale array */ int gs=0; - if(fmt==2) gs=detect_group_size(OO[k],II[k],tq[k]->nbytes); - if(gs>0) fmt=4; + int fmt=qt_resolve_fmt(tw[k]->name,OO[k],II[k],nb,tq[k]->nbytes,&gs); qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL; qt[k]->q8=(int8_t*)((char*)bw[k]+tw[k]->off); qt[k]->q4=(uint8_t*)((char*)bw[k]+tw[k]->off); qt[k]->s=(float*)((char*)bq[k]+tq[k]->off); @@ -1709,6 +1226,7 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){ * leak locked pages for every GPU-tier expert). See pin_wire() below: it wires * the final resident set only, after GPU release has already nulled out the * pointers for anything that isn't genuinely RAM-tier. */ + atomic_fetch_add_explicit(&g_prof_io,(int64_t)(n+nq),memory_order_relaxed); } s->eid=eid; return 0; } @@ -1730,6 +1248,7 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){ compat_aligned_free(s->slab); if(posix_memalign((void**)&s->slab,4096,wtot+8192)){fprintf(stderr,"OOM slab\n"); if(fatal) exit(1); s->slab=NULL; s->slab_cap=0; return -1;} s->slab_cap=wtot+8192; + numa_slab_bind(s->slab,(size_t)s->slab_cap); #endif } if(!s->fslab || ftot > s->fslab_cap){ @@ -1761,14 +1280,25 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){ } } s->fslab_cap=ftot; + numa_slab_bind(s->fslab,(size_t)ftot*sizeof(float)); #endif } + /* DISK-CLASS: classify before the reads; computed unconditionally at dc_on sites so + * the timer (dc_t0) brackets exactly the read work, matching what the GB/s in the + * DISK-CLASS line describes. dc_on gates ALL of it off demand=0 call sites + * (pilot/repin/pin -- never classified, see the call sites) and off PROF=0 runs + * (dc_needed()) -- zero cost, zero behavior change there. The fd choice below is + * NOT influenced by the verdict: this is measurement only. */ + int dc_on = demand && dc_needed(); + int dc_cls = dc_on ? expert_classify(m,layer,eid) : DC_WARM; + double dc_t0 = dc_on ? now_s() : 0; + if(dc_on) dc_wall_enter(dc_cls,dc_t0); /* busy-wall open; EVERY exit path below must pair it */ int ord[3]={0,1,2}; /* ordina per offset nel file */ for(int a=0;a<3;a++) for(int bb=a+1;bb<3;bb++) if(tw[ord[bb]]->offoff){ int t=ord[a]; ord[a]=ord[bb]; ord[bb]=t; } int contig = tw[ord[0]]->fd==tw[ord[1]]->fd && tw[ord[1]]->fd==tw[ord[2]]->fd && tw[ord[0]]->off+tw[ord[0]]->nbytes==tw[ord[1]]->off && tw[ord[1]]->off+tw[ord[1]]->nbytes==tw[ord[2]]->off; - int64_t pos[3]; int done=0; + int64_t pos[3]; int done=0, dc_direct=0; if(contig){ int64_t off0=tw[ord[0]]->off; int dfd = g_direct ? st_direct_fd(&m->S, tw[ord[0]]->fd) : -1; @@ -1778,24 +1308,41 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){ ssize_t r=pread(dfd, s->slab, len, base); if(r>=need){ pos[ord[0]]=off0-base; pos[ord[1]]=pos[ord[0]]+tw[ord[0]]->nbytes; - pos[ord[2]]=pos[ord[1]]+tw[ord[1]]->nbytes; done=1; + pos[ord[2]]=pos[ord[1]]+tw[ord[1]]->nbytes; done=1; dc_direct=1; } } if(!done){ /* fallback bufferizzato */ - if(pread_full(tw[ord[0]]->fd, s->slab, wtot, off0, "pread expert")){ if(fatal) exit(1); return -1; } + if(pread_full(tw[ord[0]]->fd, s->slab, wtot, off0, "pread expert")){ if(fatal) exit(1); + if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */ + return -1; } pos[ord[0]]=0; pos[ord[1]]=tw[ord[0]]->nbytes; pos[ord[2]]=tw[ord[0]]->nbytes+tw[ord[1]]->nbytes; done=1; } } if(!done){ /* non contigui: 3 pread bufferizzate */ int64_t o=0; for(int a=0;a<3;a++){ int k=ord[a]; - if(pread_full(tw[k]->fd, s->slab+o, tw[k]->nbytes, tw[k]->off, "pread expert")){ if(fatal) exit(1); return -1; } + if(pread_full(tw[k]->fd, s->slab+o, tw[k]->nbytes, tw[k]->off, "pread expert")){ if(fatal) exit(1); + if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */ + return -1; } pos[k]=o; o+=tw[k]->nbytes; } } float *fp[3]; int64_t fo=0; /* scale (piccole) */ for(int k=0;k<3;k++){ - if(pread_full(tq[k]->fd, (char*)(s->fslab+fo), tq[k]->nbytes, tq[k]->off, "pread qs")){ if(fatal) exit(1); return -1; } + if(pread_full(tq[k]->fd, (char*)(s->fslab+fo), tq[k]->nbytes, tq[k]->off, "pread qs")){ if(fatal) exit(1); + if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */ + return -1; } fp[k]=s->fslab+fo; fo+=tq[k]->nbytes/4; } + atomic_fetch_add_explicit(&g_prof_io,wtot+fo*4,memory_order_relaxed); + if(dc_on){ /* DISK-CLASS accounting, see dc_needed() */ + double dc_t1=now_s(); /* one clock read for thread-ns AND the wall exit */ + int64_t bytes=wtot+fo*4; + atomic_fetch_add_explicit(&g_dc_n[dc_cls],1,memory_order_relaxed); + atomic_fetch_add_explicit(&g_dc_bytes[dc_cls],bytes,memory_order_relaxed); + atomic_fetch_add_explicit(&g_dc_ns[dc_cls],(int64_t)((dc_t1-dc_t0)*1e9),memory_order_relaxed); + dc_wall_exit(dc_cls,dc_t1); + if(dc_direct) /* which fd ACTUALLY served this class */ + atomic_fetch_add_explicit(&g_dc_direct_n[dc_cls],1,memory_order_relaxed); + } if(g_drop){ /* scarta subito le pagine: evita che la page * cache in pressione strangoli il throughput */ posix_fadvise(tw[ord[0]]->fd, tw[ord[0]]->off, wtot, POSIX_FADV_DONTNEED); @@ -1804,15 +1351,27 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){ QT *qt[3]={&s->g,&s->u,&s->d}; int OO[3]={I,I,D}, II[3]={D,D,I}; for(int k=0;k<3;k++){ int64_t nb=tw[k]->nbytes; - int fmt = (nb==(int64_t)OO[k]*II[k])?1 : (nb==(int64_t)OO[k]*((II[k]+1)/2))?2 : 3; int gs=0; - if(fmt==2) gs=detect_group_size(OO[k],II[k],tq[k]->nbytes); - if(gs>0) fmt=4; + int fmt=qt_resolve_fmt(tw[k]->name,OO[k],II[k],nb,tq[k]->nbytes,&gs); qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL; qt[k]->q8=(int8_t*)(s->slab+pos[k]); qt[k]->q4=s->slab+pos[k]; qt[k]->s=fp[k]; } s->eid=eid; return 0; } +/* Every expert read goes through here: time the whole load (pread/fault + + * bookkeeping) on the thread that runs it, into the disk-service counter. */ +static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal, int demand){ + /* `demand` marks a routing-driven demand-load (moe()'s PIPE/OMP miss path, where the + * pre-bump elast_pre snapshot moe() just wrote is valid) -- pass 0 from anywhere else + * (pilot speculative loads, repin, startup PIN loading): those never run through THIS + * call's own FASE A, so the snapshot either doesn't apply or was never written for + * them, and DISK-CLASS deliberately leaves them unclassified -- see expert_classify()'s + * call site. */ + double t0=now_s(); + int rc=expert_load_impl(m,layer,eid,s,fatal,demand); + atomic_fetch_add_explicit(&g_edisk_ns,(int64_t)((now_s()-t0)*1e9),memory_order_relaxed); + return rc; +} #ifdef __linux__ /* io_uring expert batches. One owner prepares all reads for a block, submits @@ -2059,7 +1618,7 @@ static void *pipe_worker(void *arg){ memory_order_acq_rel,memory_order_relaxed)){ int L =atomic_load_explicit(&p->layer,memory_order_relaxed); int eid=atomic_load_explicit(&p->eids[i],memory_order_relaxed); /* AFTER winning CAS */ - expert_load(p->m,L,eid,&p->m->ws[i],1); /* needed-now load: fatal on I/O error (matches serial path) */ + expert_load(p->m,L,eid,&p->m->ws[i],1,1); /* needed-now load: fatal on I/O error (matches serial path); demand=1: this IS moe()'s own miss path */ atomic_store_explicit(&p->ready[i],1,memory_order_release); } /* CAS failed → another worker advanced index (or gen advanced): re-loop */ @@ -2137,7 +1696,7 @@ static void expert_host_release(Model *m, ESlot *s){ m->resident_bytes-=bytes; if(m->resident_bytes<0) m->resident_bytes=0; } static void expert_host_ensure(Model *m, int layer, ESlot *s){ - if(!s->slab) expert_load(m,layer,s->eid,s,1); + if(!s->slab) expert_load(m,layer,s->eid,s,1,0); /* re-materializing a GPU-resident expert's host copy, not a routing miss: demand=0 */ } #endif @@ -2157,6 +1716,18 @@ static void expert_prefetch(Model *m, int layer, int eid){ static void qt_addrow(const QT *t, int row, float coef, float *acc){ int I=t->I; if(t->fmt==0){ const float *w=t->qf+(int64_t)row*I; for(int i=0;ifmt==4){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); + int gs=t->gs, ng=(I+gs-1)/gs; const float *scl=t->s+(int64_t)row*ng; + for(int i=0;i+1>1]; + acc[i] +=coef*scl[i/gs] *((int)(b&0xF)-8); + acc[i+1]+=coef*scl[(i+1)/gs]*((int)(b>>4)-8); } + if(I&1){ uint8_t b=w[I>>1]; acc[I-1]+=coef*scl[(I-1)/gs]*((int)(b&0xF)-8); } return; } float c=coef*t->s[row]; if(t->fmt==1){ const int8_t *w=t->q8+(int64_t)row*I; for(int i=0;ifmt==2){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); @@ -2175,6 +1746,13 @@ static void qt_matvec_rows(const QT *t, int r0, int n, const float *x, float *y) else if(t->fmt==2){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); float s=t->s[row]; float acc=0; for(int i=0;i+1>1]; acc+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } if(I&1){ uint8_t b=w[I>>1]; acc+=((int)(b&0xF)-8)*x[I-1]; } a=acc*s; } + else if(t->fmt==4){ /* per-gruppo, come matmul_i4_grouped / per-group, as matmul_i4_grouped */ + const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); + int gs=t->gs, ng=(I+gs-1)/gs; const float *scl=t->s+(int64_t)row*ng; + for(int g=0; g*gsI?I:base+gs; float acc=0; + for(int i=base;i>1]; + acc+=(float)((i&1)?((int)(b>>4)-8):((int)(b&0xF)-8))*x[i]; } + a+=(double)acc*scl[g]; } } else { const uint8_t *w=t->q4+(int64_t)row*((I+3)/4); float s=t->s[row]; float acc=0; for(int i=0;i>2]; acc+=((int)((b>>((i&3)*2))&3)-2)*x[i]; } a=acc*s; } y[j]=(float)a; @@ -2188,6 +1766,43 @@ static int g_dsa_force=0; /* DSA_FORCE=1: selezione sempre attiva (test: top-min static int cmp_fdesc(const void *a,const void *b){ float x=*(const float*)a, y=*(const float*)b; return xy?-1:0; } +/* PARTIAL SELECT (quickselect, Hoare partition, DESCending). After this call the k + * LARGEST elements of a[0..n) are in a[0..k) in unspecified order; the (k+1)-th and + * beyond are untouched-or-smaller. O(n) average, O(n^2) pathological (mitigated by + * median-of-three below) — and unlike a full qsort it never orders more than needed. + * + * Why this exists (#356): the DSA top-keep in attention_rows previously full-qsorted + * all nk context scores (O(nk log nk)) per layer per token just to read ONE value -- + * the keep-th largest (the threshold). quickselect finds that pivot in O(nk) average, + * and the position-order scans that build dst[] are unchanged, so the kept set is + * bit-identical. Mirrors the sampling-side fix in #335 (heap partial-select there). + * + * NOT a stable partition: callers must derive the threshold and then re-scan the + * ORIGINAL array (the DSA code does exactly this) rather than reading a[0..k). */ +static void partial_select_desc(float *a, int n, int k){ + if(k<=0) return; + if(k>=n) return; /* nothing to partition: all kept */ + int lo=0, hi=n-1; + while(lo>1); + if(a[mid]>a[lo]){ float t=a[lo]; a[lo]=a[mid]; a[mid]=t; } + if(a[hi]>a[lo]){ float t=a[lo]; a[lo]=a[hi]; a[hi]=t; } + if(a[mid]>a[hi]){ float t=a[hi]; a[hi]=a[mid]; a[mid]=t; } + float piv=a[hi]; + int i=lo, j=hi; + for(;;){ + while(a[i]>piv) i++; /* desc: large values go left */ + while(j>lo && a[j]=j) break; + float t=a[i]; a[i]=a[j]; a[j]=t; i++; if(i>j) break; j--; + } + /* partition point: a[lo..i) are all >= piv, a[i..hi] are all <= piv */ + if(k<=i-1) hi=i-1; /* the k-th largest is in the left partition */ + else lo=i; /* it's in the right partition */ + } +} + /* attenzione MLA con KV-cache compressa, su token nuovi x[S,hidden], pos_base = pos del primo */ /* kvs/pos describe a ragged decode batch: each row may belong to a different * sequence. NULL keeps the original contiguous, currently-bound KV path. */ @@ -2470,10 +2085,14 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p } isc[t]=a*wsc; } - /* top-keep: soglia via qsort desc, poi scan in ordine di posizione */ + /* top-keep: threshold via PARTIAL SELECT (#356), poi scan in ordine di posizione. + * Era un qsort completo su nk (O(nk log nk)); quickselect estrae solo il + * keep-esimo valore piu' grande in O(nk) medio. La soglia (= min del blocco + * dei keep maggiori) e' identica a tmp[keep-1] del vecchio qsort, quindi i + * due scan qui sotto costruiscono dst[] bit-identical. */ float *tmp=falloc(nk); memcpy(tmp,isc,nk*sizeof(float)); - qsort(tmp,nk,sizeof(float),cmp_fdesc); - float thr=tmp[keep-1]; + partial_select_desc(tmp,nk,keep); + float thr=tmp[0]; for(int t=1;tdsa_sel+(int64_t)s*dtopk, nd=0; for(int t=0;tthr) dst[nd++]=t; for(int t=0;tn_kv_b_shard>1){ + if(kvs&&g_cuda_enabled&&getenv("COLI_CUDA_ATTN")&&atoi(getenv("COLI_CUDA_ATTN"))&& + !dnsel&&l->kv_b.cuda_eligible&&l->o.cuda_eligible&& + qt_cuda_upload(&l->kv_b)&&qt_cuda_upload(&l->o)){ + const float **rl=malloc((size_t)S*sizeof(*rl)),**rr=malloc((size_t)S*sizeof(*rr)); + const void **rk=malloc((size_t)S*sizeof(*rk)); + int *rn=malloc((size_t)S*sizeof(*rn)); int mt=0; + if(rk&&rl&&rr&&rn){ + for(int s=0;skv_start[layer]; rn[s]=pos+1-st0; + rk[s]=kvs[s]; + rl[s]=coli_kv_row(kvs[s]->Lc[layer],st0,kvl); + rr[s]=coli_kv_row(kvs[s]->Rc[layer],st0,c->qk_rope); + if(rn[s]>mt)mt=rn[s]; + } + cuda_core=cuda_projected=coli_cuda_attention_project_ragged(l->kv_b.cuda,l->o.cuda, + out,Q,rk,rl,rr,rn,S,H,c->qk_nope,c->qk_rope,vh,kvl,mt,c->attn_scale); + } + free(rk);free(rl);free(rr);free(rn); + } else if(cuda_absorb&&l->n_kv_b_shard>1){ int n=l->n_kv_b_shard,st0=m->kv_start[layer],nt=pos_base+S-st0,ok=1; float *qs=falloc((int64_t)S*H*qh),*cs=falloc((int64_t)S*H*vh); for(int d=0;dc; int D=c->hidden, E=c->n_experts, K=c->topk, I=c->moe_inter; + /* DISK-CLASS: does THIS call need the pre-bump recency snapshot? Must agree with + * dc_needed() in expert_load_impl -- that's what reads what this writes. touched[] + * makes the write once-per-call: an expert routed by more than one position in a big + * batch (prefill's S) must snapshot the state from BEFORE this call started, not from + * an earlier position's bump within the SAME call (which would reintroduce the + * same-call contamination one position later). Unconditional VLA like the FASE B + * `seen[E]` below -- E is small, cost is noise. */ + int need_classify = dc_needed(); + unsigned char touched[E]; if(need_classify) memset(touched,0,(size_t)E); float *choice=falloc(E); int sI=c->moe_inter*c->n_shared; /* Rank buffer for CACHE_ROUTE max-rank selection (up to all E experts). */ @@ -2680,6 +2326,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int if(!rank_buf||!rank_w){ free(rank_buf); free(rank_w); rank_buf=NULL; rank_w=NULL; do_cache_route=0; } } /* ---- FASE A: routing di tutte le S posizioni ---- */ + double route_t0=g_prof?now_s():0; int *idxs=malloc((size_t)S*K*sizeof(int)); float *ws=malloc((size_t)S*K*sizeof(float)); int *keff=malloc(S*sizeof(int)); /* router in UN matmul batch: stessa matematica, via le S chiamate S=1 */ @@ -2695,7 +2342,21 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int for(int kk=0;kkeusage[layer][idxs[(int64_t)s*K+kk]]++; ehit_mark(m,layer,idxs[(int64_t)s*K+kk]); + if(need_classify){ /* DISK-CLASS private recency -- snapshot BEFORE this call's own bump, + * then tick. This path also bumps the REAL elast/eaccess_clock a few + * lines below (#417/cfcc742 fixed the once-missing bump on Metal + * decode) -- DISK-CLASS's own clock stays independent regardless of + * that fix, see elast_dc in Model. */ + int e=idxs[(int64_t)s*K+kk]; + if(!touched[e]){ m->elast_pre[layer][e]=m->elast_dc[layer][e]; touched[e]=1; } + m->elast_dc[layer][e]=++m->eaccess_clock_dc; + } if(m->eheat[layer][idxs[(int64_t)s*K+kk]]eheat[layer][idxs[(int64_t)s*K+kk]]++; + /* #417: la scorciatoia GPU-prerouted deve far avanzare l'orologio di recency + * come il percorso router completo (riga ~3055), altrimenti elast/eaccess_clock + * si congelano a fine prefill e il tie-breaker LFRU di REPIN gira su punteggi + * stantii durante il decode su Metal. */ + m->elast[layer][idxs[(int64_t)s*K+kk]]=++m->eaccess_clock; } for(int d=0;deusage[layer][idx[kk]]++; ehit_mark(m,layer,idx[kk]); + if(need_classify){ /* DISK-CLASS private recency -- snapshot BEFORE this call's own bump, + * then tick (same rate as the real clock below: one per (s,kk)) */ + if(!touched[idx[kk]]){ m->elast_pre[layer][idx[kk]]=m->elast_dc[layer][idx[kk]]; touched[idx[kk]]=1; } + m->elast_dc[layer][idx[kk]]=++m->eaccess_clock_dc; + } if(m->eheat[layer][idx[kk]]eheat[layer][idx[kk]]++; m->elast[layer][idx[kk]]=++m->eaccess_clock; } @@ -2829,6 +2495,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int for(int d=0;dt_route+=now_s()-route_t0; if(g_route_fp) g_route_call++; if(g_couple && cp_pred && S<=8) for(int s2=0;s2pin[layer]; - for(int z=0;znpin[layer];z++) if(P[z].eid==eid){ m->hits++; use[j]=&P[z]; break; } + for(int z=0;znpin[layer];z++) if(P[z].eid==eid){ m->hits++; m->hit_pin++; use[j]=&P[z]; break; } if(!use[j]){ ESlot *Sl=m->ecache[layer]; int nn=m->ecn[layer]; - for(int z=0;zhits++; Sl[z].used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); use[j]=&Sl[z]; break; } } + for(int z=0;zhits++; m->hit_ecache++; Sl[z].used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); use[j]=&Sl[z]; break; } } if(!use[j]){ qof[j]=nmiss; use[j]=&m->ws[nmiss]; missk[nmiss++]=j; m->miss++; if(g_disk_split){ if(m->ld_ctx==1) m->miss_draft++; else if(m->ld_ctx==2) m->miss_absorb++; } } } @@ -3011,11 +2678,12 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int double t0=now_s(); int eids[64]; for(int q=0;qt_edisk += now_s()-t0; /* dispatch only; real reads hide behind matmul */ + m->t_ewait += now_s()-t0; /* dispatch only; the reads overlap matmul and + * are timed as service inside expert_load */ } else { double t0=now_s(); /* ORIGINALE: blocking parallel load */ #pragma omp parallel for schedule(dynamic,1) - for(int q=0;qws[q],1); - m->t_ewait += now_s()-t0; } /* blocking: whole load stalls compute */ + for(int q=0;qws[q],1,1); /* demand=1: this IS the miss path */ + m->t_ewait += now_s()-t0; } /* compute thread blocked for the whole load */ } /* I/O ASINCRONO: readahead (WILLNEED) del blocco SUCCESSIVO mentre calcoliamo * questo — il kernel legge in background, le pread dopo trovano cache calda */ @@ -3044,7 +2712,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int * correct (and free) when a subset falls back to the CPU. */ if(g_pipe && nmiss){ double tw=now_s(); for(int q=0;qt_ewait += now_s()-tw; } /* blocking: drain stalled compute */ + m->t_ewait += now_s()-tw; } MB_BUILD(1, 0); /* missed experts, now loaded */ if(nbb>0){ double t0=now_s(); @@ -3068,7 +2736,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int * Stays ABOVE the METAL skip: a subset that fell back to the CPU still needs its * slot drained here, and under METAL the block-level drain above already ran (this * spin is then a no-op). */ - if(g_pipe && qof[j]>=0){ double tw=now_s(); pipe_wait(qof[j]); m->t_ewait += now_s()-tw; } /* blocking */ + if(g_pipe && qof[j]>=0){ double tw=now_s(); pipe_wait(qof[j]); m->t_ewait += now_s()-tw; } #ifdef COLI_METAL /* skip the subsets already computed on GPU */ if(g_metal_enabled && ((is_miss[j] && !cpu_miss) || (!is_miss[j] && !cpu_res))) continue; @@ -3094,7 +2762,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int coli_cuda_expert_mlp(e->g.cuda,e->u.cuda,e->d.cuda,hh,xg,nr)){ for(int r=0;rt_emm+=now_s()-t0; continue; + double dt=now_s()-t0;m->t_emm+=dt;if(g_prof)m->t_egpu+=dt;continue; } if(!e->slab) expert_host_ensure(m,layer,e); #endif @@ -3103,7 +2771,9 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int matmul_qt(hh, gg, &e->d, nr); for(int r=0;rt_emm += now_s()-t0; + double dt=now_s()-t0;m->t_emm+=dt;if(g_prof){m->t_ecpu+=dt; + m->cpu_expert_bytes+=qt_bytes(&e->g)+qt_bytes(&e->u)+qt_bytes(&e->d); + m->cpu_expert_rows+=(uint64_t)nr;} } #ifdef COLI_CUDA ColiCudaTensor *dev_g[COLI_CUDA_MAX_DEVICES][64],*dev_u[COLI_CUDA_MAX_DEVICES][64]; @@ -3111,6 +2781,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int int dev_rows[COLI_CUDA_MAX_DEVICES][64],dev_which[COLI_CUDA_MAX_DEVICES][64]; int dev_nc[COLI_CUDA_MAX_DEVICES]={0},dev_total[COLI_CUDA_MAX_DEVICES]={0}; int dev_off[COLI_CUDA_MAX_DEVICES]={0},dev_ok[COLI_CUDA_MAX_DEVICES]={0}; + double dev_time[COLI_CUDA_MAX_DEVICES]={0}; for(int di=0;dig.cuda_device==g_cuda_devices[di]) dev_total[di]+=group_n[q]; for(int di=1;di1) schedule(static) - for(int di=0;dig.cuda,e->u.cuda,e->d.cuda,hh,xg,nr)){ expert_host_ensure(m,layer,e); expert_gate_up(gg,uu,xg,&e->g,&e->u,nr); for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z]; matmul_qt(hh,gg,&e->d,nr); + if(g_prof){m->cpu_expert_bytes+=qt_bytes(&e->g)+qt_bytes(&e->u)+qt_bytes(&e->d); + m->cpu_expert_rows+=(uint64_t)nr;} } + if(g_prof)m->t_ecpu+=now_s()-tc; } float *src=dev_ok[di]?group_y+(int64_t)off*D:hh; for(int r=0;rmx)mx=dev_time[di];m->t_egpu+=mx;} m->t_emm+=now_s()-tg; #endif /* No drain barrier: the per-expert pipe_wait(qof[j]) above (issued for every @@ -3309,7 +2988,7 @@ static void pilot_realload(Model *m, int layer, int eid){ g_pilot_inflight[layer]++; pthread_mutex_unlock(&g_pilot_mx); - int rc=expert_load(m,layer,eid,dst,0); /* pread VERO — fuori dal lock, sovrapposto al compute; fatal=0: un errore su una speculazione NON deve uccidere il server */ + int rc=expert_load(m,layer,eid,dst,0,0); /* pread VERO — fuori dal lock, sovrapposto al compute; fatal=0: un errore su una speculazione NON deve uccidere il server; demand=0: speculative, never classified */ pthread_mutex_lock(&g_pilot_mx); if(rc==0){ @@ -3756,7 +3435,11 @@ static void layers_forward_rows(Model *m, float *x, int S, int pos_base, float *dst=coli_cuda_pipe_scratch(dev,15,xb); if(dst){ if(x_dev_on<0) ok=coli_cuda_pipe_upload(dev,dst,x,xb); - else if(x_dev_on!=dev){ ok=coli_cuda_pipe_peer_copy(dev,dst,x_dev_on,x_dev,xb); } + else if(x_dev_on!=dev){ + double tp=g_prof?now_s():0; + ok=coli_cuda_pipe_peer_copy(dev,dst,x_dev_on,x_dev,xb); + if(g_prof){m->t_p2p+=now_s()-tp;m->n_p2p++;} + } else dst=x_dev; if(ok){ x_dev=dst; x_dev_on=dev; @@ -3852,7 +3535,7 @@ static float *step_all(Model *m, const int *ids, int S, int pos_base){ float *x=falloc((int64_t)S*D); for(int s=0;sh_all) memcpy(m->h_all, x, (int64_t)S*D*sizeof(float)); /* hidden di TUTTE le pos (S<=64) */ + if(m->h_all) memcpy(m->h_all, x, (int64_t)S*D*sizeof(float)); /* hidden di TUTTE le pos (S<=512) */ if(m->hlast) memcpy(m->hlast, x+(int64_t)(S-1)*D, D*sizeof(float)); float *lo=falloc((int64_t)S*c->vocab), *row=falloc(D); for(int s=0;sfinal_norm, D, c->eps); @@ -3865,8 +3548,8 @@ static float *step_all(Model *m, const int *ids, int S, int pos_base){ static float *step_decode_batch(Model *m, const DecodeRow *rows, int S){ Cfg *c=&m->c; int D=c->hidden; /* Ragged KV currently uses MLA absorption; the stack kernel is sized to 512. */ - if(!rows || S<1 || S>64 || c->kv_lora>512) return NULL; - KVState *kvs[64]; int positions[64]; + if(!rows || S<1 || S>512 || c->kv_lora>512) return NULL; + KVState *kvs[512]; int positions[512]; float *x=falloc((int64_t)S*D); for(int s=0;sLc || !rows[s].kv->Rc || !rows[s].kv->kv_start || @@ -3981,9 +3664,9 @@ static void mtp_absorb(Model *m, const int *next_ids, const float *x, int S, int free(hx); free(cat); free(e); free(hn); free(hf); free(nrm); free(tmp); } -static inline int argmax_v(const float *lo, int V){ - int b=0; float bv=lo[0]; for(int i=1;ibv){bv=lo[i];b=i;} return b; -} + +static void repin_pass_limit(Model *m,int limit); +static void repin_pass(Model *m){ repin_pass_limit(m,16); } /* ---- METODO F: draft grammaticale (#48) ---- * gr_feed consuma i byte di ogni token EMESSO e tiene il walker in sync con l'output; @@ -4065,73 +3748,6 @@ static int grammar_draft(int *draft, int cap){ * Rejection sampling di Leviathan: accetta il draft x_d con prob p(x_d); al rifiuto * ricampiona da p con x_d azzerato e rinormalizzato. La distribuzione risultante e' * ESATTAMENTE p: la speculazione resta invisibile all'output anche col sampling. */ -static uint64_t g_rng=0x9E3779B97F4A7C15ULL; -static inline double rndu(void){ g_rng^=g_rng<<13; g_rng^=g_rng>>7; g_rng^=g_rng<<17; - return (double)(g_rng>>11)*(1.0/9007199254740992.0); } -static float *g_pbuf=NULL; static int *g_pidx=NULL; /* buffer riusati (decode single-thread) */ -static int cmp_pdesc(const void *a,const void *b){ - float pa=g_pbuf[*(const int*)a], pb=g_pbuf[*(const int*)b]; - return papb ? -1 : 0; } -/* costruisce in g_pbuf la distribuzione target: softmax(lo/temp) troncata a top-p g_nuc */ -static void dist_build(const float *lo, int V){ - if(!g_pbuf){ g_pbuf=falloc(V); g_pidx=malloc(V*sizeof(int)); } - float mx=lo[0]; for(int i=1;imx) mx=lo[i]; - double s=0; float invt=1.f/(g_temp>1e-4f?g_temp:1e-4f); - for(int i=0;i0 && g_nuc<1.f){ - for(int i=0;i=g_nuc){ keep=i+1; break; } } - double s2=0; for(int i=keep;i=0 -> quel token e' escluso (rinormalizzando al volo) */ -static int dist_sample(int V, int ban){ - double z = 1.0 - (ban>=0 ? g_pbuf[ban] : 0.0); if(z<=1e-12) z=1e-12; - double u = rndu()*z, cum=0; - for(int i=0;i=u) return i; } - for(int i=V-1;i>=0;i--) if(i!=ban && g_pbuf[i]>0) return i; - return 0; -} -/* prossimo token dai logits: greedy se g_temp<=0, altrimenti sampling. - * ban = token escluso perche' rifiutato dalla verifica speculativa precedente. */ -static int pick_tok(const float *lo, int V, int ban){ - if(g_temp<=0) return argmax_v(lo,V); - dist_build(lo,V); - return dist_sample(V,ban); -} - -/* stop-set attivo (popolato da run_text/run_serve dal config; vuoto in validazione, - * dove si genera un numero fisso di token da confrontare con l'oracolo) */ -static int g_stop[9], g_nstop=0; -static void repin_pass_limit(Model *m,int limit); -static void repin_pass(Model *m){ repin_pass_limit(m,16); } -static inline int is_stop(int t){ for(int i=0;in_stop;i++) g_stop[g_nstop++]=c->stop_ids[i]; - if(tok_eos>=0 && !is_stop(tok_eos)) g_stop[g_nstop++]=tok_eos; - /* In serve mode (API), keep only <|endoftext|> as a stop token. The tokens - * <|user|> and <|observation|> are role markers that the Python server - * handles — keeping them as stop tokens causes the engine to halt - * prematurely when the model tries to emit blocks, because - * int4-quantized logits can be noisy enough that argmax picks a stop-token - * ID instead of the correct tool-call token. (#401) */ - if(getenv("SERVE")){ - int kept=0; - for(int i=0;in_fw++; + if(g_prof) prof_lat(now_s()-tf0); int k=0; /* verifica: accetta finche' coincide */ if(g>0 && getenv("MTP_DEBUG")){ int veri=argmax_v(lo,V); fprintf(stderr,"[mtpdbg] draft0=%d verified=%d %s\n", draft[0], veri, draft[0]==veri?"HIT":"miss"); } @@ -4281,18 +3899,6 @@ static void forward_all(Model *m, const int *ids, int S, int *pred){ } /* log-prob (log-softmax) del token target dato il vettore di logit; *am=1 se e' l'argmax */ -static double logprob_target(const float *lo, int V, int target, int *am){ - float mx=lo[0]; int best=0; for(int i=1;imx){mx=lo[i];best=i;} } - double se=0; for(int i=0;i .. " (T=ctxlen+contlen) * output: riga " " per richiesta. @@ -4360,12 +3966,17 @@ static void generate(Model *m, const int *prompt, int np, int n_new, int *out){ } static void profile_print(Model *m, double elapsed){ - double accounted=m->t_edisk+m->t_ewait+m->t_emm+m->t_attn+m->t_head; + double accounted=m->t_ewait+m->t_emm+m->t_attn+m->t_head; printf("PROFILE: expert-disk %.3fs service / %.3fs wait | expert-matmul %.3fs | attention %.3fs " "(including kvb %.3fs) | lm_head %.3fs | other %.3fs\n", - m->t_edisk,m->t_ewait,m->t_emm,m->t_attn,m->t_kvb,m->t_head,elapsed-accounted); + edisk_s(),m->t_ewait,m->t_emm,m->t_attn,m->t_kvb,m->t_head,elapsed-accounted); printf("ATTENTION: projection/RoPE %.3fs | score-softmax-value %.3fs | output projection %.3fs\n", m->t_aproj,m->t_acore,m->t_aout); + if(g_prof)printf("P0-EXEC: routed CPU %.3fs / %.2f GB/s (%llu row) | routed GPU critical %.3fs | router %.3fs | residual P2P %.3fs / %llu hop | orchestration %.3fs\n", + m->t_ecpu,m->t_ecpu>0?m->cpu_expert_bytes/1e9/m->t_ecpu:0.0, + (unsigned long long)m->cpu_expert_rows,m->t_egpu,m->t_route,m->t_p2p,(unsigned long long)m->n_p2p, + elapsed-m->t_ewait-m->t_emm-m->t_attn-m->t_head-m->t_route-m->t_p2p>0? + elapsed-m->t_ewait-m->t_emm-m->t_attn-m->t_head-m->t_route-m->t_p2p:0); #ifdef COLI_METAL if(g_metal_enabled){ uint64_t ok=0,fb=0,ex=0; double su=0,gp=0,sc=0; coli_metal_moe_counts(&ok,&fb,&ex); coli_metal_moe_times(&su,&gp,&sc); @@ -4378,8 +3989,123 @@ static void profile_print(Model *m, double elapsed){ } static void profile_reset(Model *m){ - m->t_edisk=m->t_ewait=m->t_emm=m->t_attn=m->t_kvb=m->t_head=0; + m->t_ewait=m->t_emm=m->t_attn=m->t_kvb=m->t_head=0; + m->t_ecpu=m->t_egpu=m->t_route=m->t_p2p=0;m->n_p2p=0; + m->cpu_expert_bytes=0;m->cpu_expert_rows=0; m->t_aproj=m->t_acore=m->t_aout=0; + atomic_store_explicit(&g_edisk_ns,0,memory_order_relaxed); +} + +/* PROF=1 report: forward-latency percentiles, expert I/O totals, phase shares + * of wall time, and a plain-language verdict naming the knob most likely to + * move tok/s on THIS machine with THIS config. `b` marks the window start + * (serve mode reports per turn; batch modes snapshot right after reset). */ +static int prof_cmp_d(const void *a,const void *b){ + double x=*(const double*)a, y=*(const double*)b; return (x>y)-(xc; if(elapsed<1e-9) elapsed=1e-9; + uint64_t nw=g_prof_nlat-b->nlat; if(nw>PROF_LAT_CAP) nw=PROF_LAT_CAP; /* ring keeps the tail */ + uint64_t nfw=m->n_fw-b->n_fw, nem=m->n_emit-b->n_emit; + if(nw){ + double *v=malloc((size_t)nw*sizeof(double)); + if(v){ + for(uint64_t i=0;i=32 && p99>3*p50) + fprintf(f,"[PROF] tail: p99 is %.1fx p50 — the slow forwards are cold-cache expert loads; " + "a warm-up turn or a pinned hot-store (PIN / AUTOPIN history) shrinks them\n",p99/p50); + free(v); + } + } + int64_t io=atomic_load_explicit(&g_prof_io,memory_order_relaxed)-b->io; + uint64_t dh=m->hits-b->hits, dm=m->miss-b->miss, dq=m->ereq-b->ereq; + double hitp=(dh+dm)?100.0*dh/(dh+dm):100.0; + double eb=(double)expert_bytes_probe(m,m->ebits); + int pinned=0,lru=0; + for(int i=0;i<=c->n_layers;i++){ if(m->npin)pinned+=m->npin[i]; if(m->ecn)lru+=m->ecn[i]; } + double io_w=m->t_ewait-b->ewait; /* stall the compute thread felt */ + double io_svc=edisk_s()-b->edisk; /* read service on the loading threads (overlaps compute) */ + uint64_t dhp=m->hit_pin-b->hit_pin, dhe=m->hit_ecache-b->hit_ecache; /* split #336 */ + fprintf(f,"[PROF] expert I/O: %.3f GB fetched (%.1f MB/token, %.2f GB/s over the run%s) | " + "hit %.1f%% (%llu pin + %llu lru / %llu load) | %.1f loads/token | %.1fs read service / %.1fs felt wait\n", + io/1e9, tokens>0?io/1e6/tokens:0.0, io/1e9/elapsed, + g_mmap?"; COLI_MMAP=1: page cache may serve part":"", + hitp,(unsigned long long)dhp,(unsigned long long)dhe,(unsigned long long)dm, tokens>0?(double)dq/tokens:0.0, + io_svc,io_w); + /* DISK-CLASS: per-load cold/warm classification vs. which fd ACTUALLY served it. + * Three per-class rates, labeled to keep the units unambiguous (ambiguous units + * mislead -- measured lesson): GB/s-thread = bytes / thread-seconds (per-read + * service rate, same convention as the read-service line above); GB/s-wall = + * bytes / busy-wall (aggregate rate the disk actually delivered while >=1 load of + * the class was in flight); avg-conc = thread-seconds / busy-wall (mean overlap + * depth). disk-busy = combined busy-wall (either class in flight) as a share of + * the profile window. */ + { + uint64_t dcn[2], dcdn[2]; int64_t dcbytes[2], dcns[2], dcwall[2], wnow[2], dcwall_all, wall_now; + dc_wall_read(wnow,&wall_now); + for(int i=0;i<2;i++){ + dcn[i]=atomic_load_explicit(&g_dc_n[i],memory_order_relaxed)-b->dc_n[i]; + dcbytes[i]=atomic_load_explicit(&g_dc_bytes[i],memory_order_relaxed)-b->dc_bytes[i]; + dcns[i]=atomic_load_explicit(&g_dc_ns[i],memory_order_relaxed)-b->dc_ns[i]; + dcdn[i]=atomic_load_explicit(&g_dc_direct_n[i],memory_order_relaxed)-b->dc_direct_n[i]; + dcwall[i]=wnow[i]-b->dc_wall_ns[i]; + } + dcwall_all=wall_now-b->dc_wall_all_ns; + if(dcn[DC_COLD]+dcn[DC_WARM]){ + double ct=dcns[DC_COLD]>0?(double)dcbytes[DC_COLD]/dcns[DC_COLD]:0.0; /* bytes/ns = GB/s */ + double wt=dcns[DC_WARM]>0?(double)dcbytes[DC_WARM]/dcns[DC_WARM]:0.0; + double cw=dcwall[DC_COLD]>0?(double)dcbytes[DC_COLD]/dcwall[DC_COLD]:0.0; + double ww=dcwall[DC_WARM]>0?(double)dcbytes[DC_WARM]/dcwall[DC_WARM]:0.0; + double cc=dcwall[DC_COLD]>0?(double)dcns[DC_COLD]/dcwall[DC_COLD]:0.0; + double wc=dcwall[DC_WARM]>0?(double)dcns[DC_WARM]/dcwall[DC_WARM]:0.0; + fprintf(f,"[PROF] DISK-CLASS (recency split): cold %llu x %.2f GB @ %.2f GB/s-thread, wall %.1fs @ %.2f GB/s-wall, avg-conc %.1f (direct %llu/%llu) | " + "warm %llu x %.2f GB @ %.2f GB/s-thread, wall %.1fs @ %.2f GB/s-wall, avg-conc %.1f (direct %llu/%llu) | " + "disk-busy %.1fs (%.0f%% of window)\n", + (unsigned long long)dcn[DC_COLD],dcbytes[DC_COLD]/1e9,ct,dcwall[DC_COLD]/1e9,cw,cc, + (unsigned long long)dcdn[DC_COLD],(unsigned long long)dcn[DC_COLD], + (unsigned long long)dcn[DC_WARM],dcbytes[DC_WARM]/1e9,wt,dcwall[DC_WARM]/1e9,ww,wc, + (unsigned long long)dcdn[DC_WARM],(unsigned long long)dcn[DC_WARM], + dcwall_all/1e9,100.0*(dcwall_all/1e9)/elapsed); + } + } + fprintf(f,"[PROF] resident experts: %d pinned (%.1f GB) + %d in LRU (%.1f GB, cap %d/layer)\n", + pinned,pinned*eb/1e9,lru,lru*eb/1e9,m->ecap); + double emm=m->t_emm-b->emm, ecpu=m->t_ecpu-b->ecpu, egpu=m->t_egpu-b->egpu; + double route=m->t_route-b->route,p2p=m->t_p2p-b->p2p; + uint64_t np2p=m->n_p2p-b->n_p2p; + int64_t cpu_bytes=m->cpu_expert_bytes-b->cpu_bytes; + uint64_t cpu_rows=m->cpu_expert_rows-b->cpu_rows; + double attn=m->t_attn-b->attn, head=m->t_head-b->head; + double other=elapsed-io_w-emm-attn-head-route-p2p; if(other<0) other=0; + double f_io=io_w/elapsed, f_emm=emm/elapsed, f_attn=attn/elapsed; + fprintf(f,"[PROF] time shares: expert-I/O %.0f%% | expert-matmul %.0f%% | attention %.0f%% | lm_head %.0f%% | other %.0f%%\n", + 100*f_io,100*f_emm,100*f_attn,100*head/elapsed,100*other/elapsed); + double slow=ecpu>egpu?ecpu:egpu,fast=ecpu0?cpu_bytes/1e9/ecpu:0.0,(unsigned long long)cpu_rows, + egpu,fast>1e-9?slow/fast:0.0,route,p2p,(unsigned long long)np2p, + np2p?p2p*1e3/np2p:0.0,other); + if(f_io>=0.30){ + fprintf(f,"[PROF] verdict: I/O-bound — %.0f%% of the time waits on expert reads (hit %.0f%%).",100*f_io,hitp); + if(hitp<90) fprintf(f," More cache is the lever: raise RAM_GB (or add RAM)."); + else fprintf(f," The cache is already warm — the routed working set streams from disk; a faster disk or a bigger pinned tier (PIN_GB) is the lever."); + if(!g_pipe) fprintf(f," Try PIPE=1 (overlap reads with matmul)."); + if(!g_direct) fprintf(f," On NVMe try DIRECT=1."); + fprintf(f,"\n"); + } else if(f_emm>=0.40){ + fprintf(f,"[PROF] verdict: compute-bound in expert matmuls (%.0f%%) — more cores/threads help; keep IDOT=1, or move hot experts to a GPU tier (COLI_CUDA / COLI_METAL).%s\n", + 100*f_emm, g_mmap?" Note: with COLI_MMAP=1 page-fault I/O is accounted inside matmul.":""); + } else if(f_attn>=0.35){ + fprintf(f,"[PROF] verdict: attention-bound (%.0f%%) — context length is the cost (DSA %s). A lower CTX helps if the workload allows.\n", + 100*f_attn, m->has_dsa?"on":"not available for this model"); + } else { + fprintf(f,"[PROF] verdict: balanced — no phase dominates (I/O %.0f%%, matmul %.0f%%, attention %.0f%%); this config is a reasonable fit for this machine.\n", + 100*f_io,100*f_emm,100*f_attn); + } } /* Fixed-token decode benchmark: prefill all but the prompt's last token, then @@ -4389,16 +4115,20 @@ static void run_replay(Model *m, const int *full, int nfull, int np){ if(np<2||nfull<=np){ fprintf(stderr,"REPLAY requires a non-empty prompt and continuation\n"); return; } kv_alloc(m,nfull+2); float *logit=step(m,full,np-1,0); free(logit); - m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; + m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; m->hit_pin=m->hit_ecache=0; profile_reset(m); + ProfBase pb; prof_base(m,&pb); double t0=now_s(); int steps=0; for(int i=np-1;in_fw++; m->n_emit++; } } double dt=now_s()-t0, tot=m->hits+m->miss; printf("REPLAY decode: %d tokens in %.3fs | %.2f tok/s | expert hit %.1f%%\n", steps,dt,steps/dt,tot?100.0*m->hits/tot:0.0); profile_print(m,dt); + if(g_prof) prof_report(m,&pb,dt,steps,stdout); #ifdef COLI_CUDA if(m->gpu_expert_count) printf("CUDA expert tier: %d resident experts (%.2f GB) | %llu calls served from VRAM\n", m->gpu_expert_count,m->gpu_expert_bytes/1e9,(unsigned long long)m->gpu_expert_calls); @@ -4412,7 +4142,7 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ Cfg *c=&m->c; char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap); Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); - stops_arm(&m->c, eos); + stops_arm_tok(&m->c, eos, &T); grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */ if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della * distribuzione int4 e' rumore di quantizzazione */ @@ -4435,10 +4165,11 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ } prefill_t=now_s()-prefill_t; printf("PROFILO PREFILL (%.2fs):\n",prefill_t); profile_print(m,prefill_t); - m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; + m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; m->hit_pin=m->hit_ecache=0; m->n_emit=m->n_fw=0; g_last_repin=0; profile_reset(m); + ProfBase pb; prof_base(m,&pb); double t=now_s(); EmitStream es={&T,m,t,0,0}; grammar_reset(); @@ -4447,8 +4178,9 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ double tot=m->hits+m->miss; int nsp=0; for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; printf("\n---\nprefill %d tokens in %.2fs | decode %d tokens in %.2fs (%.2f tok/s) | " - "expert hit rate %.1f%% | RSS %.2f GB", - np,prefill_t,produced,dt,produced/dt,tot?100.0*m->hits/tot:0.0,rss_gb()); + "expert hit rate %.1f%% (pin %.1f%% + lru %.1f%%) | RSS %.2f GB", /* split #336: quale tier serve gli hit */ + np,prefill_t,produced,dt,produced/dt,tot?100.0*m->hits/tot:0.0, + tot?100.0*m->hit_pin/tot:0.0, tot?100.0*m->hit_ecache/tot:0.0, rss_gb()); if(g_cache_route && m->route_slots) printf(" | swap %.1f%% (%llu/%llu)", 100.0*m->route_swaps/m->route_slots, @@ -4482,6 +4214,7 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ if(g_cuda_enabled) cuda_stats_print(); #endif profile_print(m,dt); + if(g_prof) prof_report(m,&pb,dt,produced,stdout); if(g_pilot_real) printf("PILOT_REAL: %ld load cross-layer completati, %ld scartati (main gia' sul layer) | PILOT_K=%d\n", (long)atomic_load_explicit(&g_pilot_loads,memory_order_relaxed), (long)atomic_load_explicit(&g_pilot_drops,memory_order_relaxed), g_pilot_k); @@ -4556,7 +4289,68 @@ static int repin_pick(Model *m, RepinCand *out, int maxc){ } return nb; } +/* ---- RSS GUARD (#403) ----------------------------------------------------- + * La proiezione di cap_for_ram e' una STIMA: sul GB10 (#403) le generazioni + * lunghe l'hanno sforata di ~40 GB (proiettato 74.4, reale 115.6 -> 3 kill del + * kernel). La run D dell'issue prova che un cap piu' basso CONTIENE la crescita: + * questa guardia lo fa da sola, sull'RSS MISURATO invece che sul proiettato. + * Al safe point (stessa sede di repin: nessun moe in volo), ogni ~16 token + * emessi: se l'RSS supera il budget, svuota gli slot LRU meno usati e abbassa + * ecap perche' non ricrescano. Gli slab sono >128KB (mmap'd da glibc): la free + * restituisce le pagine al kernel subito, quindi l'RSS scende davvero. + * Lo slot NON viene compattato via: resta al suo posto con eid=-1/used=0 (primo + * candidato al riuso), perche' con PILOT_REAL il worker tiene puntatori dentro + * ecache[] durante i suoi pread e uno spostamento li invaliderebbe; per lo + * stesso motivo gli slot eid<0 (riservati/in caricamento) non si toccano e la + * selezione avviene sotto g_pilot_mx. resident_bytes resta invariato: gli slot + * LRU non sono mai contati li' (solo pin e densa). + * EN: evict = free the slab in place (eid=-1, used=0, never compact: PILOT_REAL + * EN: holds pointers into ecache[] across its preads), skip eid<0 reservations, + * EN: select under g_pilot_mx. RSS_GUARD_GB= forces an explicit ceiling. */ +static double g_ram_budget_gb=0; /* budget risolto, scritto da cap_for_ram */ +static uint64_t g_rssg_last=0; +static void rss_guard(Model *m){ + double lim = getenv("RSS_GUARD_GB") ? atof(getenv("RSS_GUARD_GB")) : g_ram_budget_gb; + if(lim<=0) return; + if(m->n_emit - g_rssg_last < 16) return; + g_rssg_last = m->n_emit; + double rss=rss_gb(); + if(rss <= lim*1.02+0.3) return; /* tolleranza: 2% + 300MB */ + Cfg *c=&m->c; + int64_t need=(int64_t)((rss-lim)*1e9), freed=0; int dropped=0; + for(int pass=0; pass<8 && freedn_layers && freedecache || !m->ecache[l]) continue; + pthread_mutex_lock(&g_pilot_mx); + int nn=m->ecn[l], lru=-1; + for(int z=0;zecache[l][z]; + if(cand->eid<0 || !cand->slab) continue; + if(lru<0 || cand->usedecache[l][lru].used) lru=z; + } + if(lru<0){ pthread_mutex_unlock(&g_pilot_mx); continue; } + ESlot *s=&m->ecache[l][lru]; + s->eid=-1; /* nascosto: nessun hit/evict altrui */ + pthread_mutex_unlock(&g_pilot_mx); + int64_t sb=s->slab_cap + s->fslab_cap*4; +#ifdef COLI_METAL + if(s->slab && g_metal_enabled) coli_metal_unregister(s->slab); +#endif + compat_aligned_free(s->slab); free(s->fslab); + s->slab=NULL; s->fslab=NULL; s->slab_cap=s->fslab_cap=0; + QT *q[3]={&s->g,&s->u,&s->d}; + for(int k=0;k<3;k++){ q[k]->qf=NULL; q[k]->q8=NULL; q[k]->q4=NULL; q[k]->s=NULL; } + s->used=0; /* primo candidato al riuso */ + freed += sb; dropped++; + } + if(m->ecap>2) m->ecap--; /* il tetto scende: niente ricrescita */ + } + if(dropped) + fprintf(stderr,"[RAM-GUARD] RSS %.1f GB over the %.1f GB budget (#403): " + "dropped %d cached experts, cap -> %d\n", rss, lim, dropped, m->ecap); +} static void repin_pass_limit(Model *m,int limit){ + rss_guard(m); /* #403: il budget si fa rispettare sull'RSS MISURATO */ if(g_repin<=0) return; if(m->n_emit - g_last_repin < (uint64_t)g_repin) return; g_last_repin = m->n_emit; @@ -4614,7 +4408,7 @@ static void repin_pass_limit(Model *m,int limit){ +(int64_t)coli_cuda_tensor_bytes(s->d.cuda) : 0; #endif double t0=now_s(); - expert_load(m,cd[b].l,cd[b].eid,s,1); /* disk -> RAM, same resident slot */ + expert_load(m,cd[b].l,cd[b].eid,s,1,0); /* disk -> RAM, same resident slot; demand=0: repin, never classified */ const char *tier="RAM"; #ifdef COLI_CUDA if(gpu){ /* refresh the same VRAM slot now, not lazily */ @@ -4646,117 +4440,6 @@ static void repin_pass_limit(Model *m,int limit){ * si appendono SOLO le posizioni nuove e si riscrive nrec per ultimo: un crash a meta' * append lascia nrec vecchio = file coerente. La riga KV del layer MTP non si salva: * al resume kv_start=-1 e la finestra di draft riparte da sola. */ -static int g_kvsave=1; -#define KV_MAGIC "COLIKV1\0" -static void kv_hdr(Model *m, int32_t *h, int nrec){ - Cfg *c=&m->c; int nic=0; - for(int i=0;in_layers;i++) if(m->Ic && m->Ic[i]) nic++; - h[0]=c->n_layers; h[1]=c->kv_lora; h[2]=c->qk_rope; - h[3]=m->has_dsa?c->index_hd:0; h[4]=nic; h[5]=c->vocab; h[6]=nrec; h[7]=0; -} -/* Bytes of one on-disk record: [tok i32][Lc+Rc per layer][Ic per DSA layer]. - * Layout matches what kv_disk_append writes and kv_disk_load reads. */ -static int64_t kv_rec_bytes(Model *m){ - Cfg *c=&m->c; - int64_t rec = 4 + (int64_t)c->n_layers*(c->kv_lora+c->qk_rope)*4; - if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) rec+=(int64_t)c->index_hd*4; - return rec; -} -/* Open the persistent handle lazily; write the header if the file is new. After - * this returns successfully, k->disk_fp is valid for the engine's lifetime and - * positioned at end-of-header (nrec==0 case) or wherever the caller seeks. */ -static int kv_disk_open(Model *m){ - KVState *k=m->kv; - if(k->disk_fp) return 1; - k->disk_fp=fopen(k->disk_path,"r+b"); - if(!k->disk_fp){ /* not there yet -> create + header */ - k->disk_fp=fopen(k->disk_path,"wb"); - if(!k->disk_fp) return 0; - int32_t h[8]; kv_hdr(m,h,0); - fwrite(KV_MAGIC,1,8,k->disk_fp); fwrite(h,4,8,k->disk_fp); - fflush(k->disk_fp); - fclose(k->disk_fp); - k->disk_fp=fopen(k->disk_path,"r+b"); /* reopen r+b for append */ - if(!k->disk_fp) return 0; - } - return 1; -} -static void kv_disk_truncate(Model *m, int nrec){ - if(!g_kvsave) return; - KVState *k=m->kv; - if(k->disk_fp){ fclose(k->disk_fp); k->disk_fp=NULL; } /* drop to shrink on disc */ - FILE *f=fopen(k->disk_path,"r+b"); - if(!f){ k->disk_nrec=0; return; } - k->disk_nrec=nrec; - int32_t nr=nrec; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); - fflush(f); fclose(f); -} -static void kv_disk_reset(Model *m){ kv_disk_truncate(m,0); } -static void kv_disk_append(Model *m, const int *hist, int len){ - KVState *k=m->kv; - if(!g_kvsave || len<=k->disk_nrec) return; - Cfg *c=&m->c; - if(!kv_disk_open(m)) return; - FILE *f=k->disk_fp; - int64_t rec = kv_rec_bytes(m); - /* grow the contiguous staging buffer if the record is larger (#1 batching) */ - if(rec > k->disk_buf_cap){ - uint8_t *nb=realloc(k->disk_buf, rec); - if(!nb) return; /* OOM: skip this turn, retry next */ - k->disk_buf=nb; k->disk_buf_cap=rec; - } - fseek(f, 8+8*4 + (int64_t)k->disk_nrec*rec, SEEK_SET); - for(int p=k->disk_nrec;pdisk_buf; /* pack token + every layer into one record */ - *(int32_t*)b = hist[p]; b+=4; - for(int i=0;in_layers;i++){ - memcpy(b, m->Lc[i]+(int64_t)p*c->kv_lora, (size_t)c->kv_lora*4); b+=c->kv_lora*4; - memcpy(b, m->Rc[i]+(int64_t)p*c->qk_rope,(size_t)c->qk_rope*4); b+=c->qk_rope*4; - } - if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]){ - memcpy(b, m->Ic[i]+(int64_t)p*c->index_hd, (size_t)c->index_hd*4); b+=c->index_hd*4; - } - fwrite(k->disk_buf, 1, (size_t)rec, f); /* one fwrite per position (was ~157) */ - } - fflush(f); /* dati prima, contatore poi */ - int32_t nr=len; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); - fflush(f); /* persist the counter too */ - k->disk_nrec=len; -} -static int kv_disk_load(Model *m, int *hist, int maxctx){ - if(!g_kvsave) return 0; - KVState *k=m->kv; - Cfg *c=&m->c; - FILE *f=fopen(k->disk_path,"rb"); if(!f) return 0; - char mg[8]; int32_t h[8], w[8]; kv_hdr(m,w,0); - if(fread(mg,1,8,f)!=8 || memcmp(mg,KV_MAGIC,8) || fread(h,4,8,f)!=8 || - h[0]!=w[0]||h[1]!=w[1]||h[2]!=w[2]||h[3]!=w[3]||h[4]!=w[4]||h[5]!=w[5]){ - fprintf(stderr,"[KV] ignoring .coli_kv from a different model or version\n"); fclose(f); return 0; } - int nrec=h[6]; - if(nrec<1){ fclose(f); return 0; } - if(nrec>=maxctx-8-g_draft){ - fprintf(stderr,"[KV] saved conversation (%d tokens) exceeds the context: starting over\n",nrec); - fclose(f); return 0; } - double t0=now_s(); - for(int p=0;pn_layers;i++){ - if(fread(m->Lc[i]+(int64_t)p*c->kv_lora, 4, c->kv_lora, f)!=(size_t)c->kv_lora || - fread(m->Rc[i]+(int64_t)p*c->qk_rope, 4, c->qk_rope, f)!=(size_t)c->qk_rope){ nrec=p; goto out; } - } - if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) - if(fread(m->Ic[i]+(int64_t)p*c->index_hd, 4, c->index_hd, f)!=(size_t)c->index_hd){ nrec=p; goto out; } - } -out: - fclose(f); - if(nrec>0){ - if(m->has_mtp) m->kv_start[c->n_layers]=-1; /* la finestra MTP riparte da sola */ - fprintf(stderr,"[KV] resumed conversation from disk: %d tokens in %.1fs (no re-prefill)\n", - nrec, now_s()-t0); - } - k->disk_nrec=nrec; - return nrec; -} typedef struct { KVState kv; int *hist, len, first; } ServeCtx; static double kv_pool_bytes(Model *m, int max_ctx); @@ -4788,6 +4471,8 @@ typedef struct { float temp, top_p; double started; uint64_t hits0, miss0; + ProfBase pb; /* phase-time window start (same convention as hits0): + feeds the PROF protocol line and the PROF=1 report */ } ServeReq; static void mux_data(Tok *T, unsigned long long id, int token){ @@ -4804,10 +4489,26 @@ static void mux_done(Model *m, ServeCtx *sc, ServeReq *r){ tiers_emit(m); emap_emit(m); hits_emit(m); + /* PROF: per-turn phase timings for the dashboard profiling page — + * "PROF ". + * edisk = disk service (expert_load wall on the reading threads, overlaps + * compute); ewait = the stall the compute thread felt — only ewait belongs + * in a wall-time breakdown. With KV_SLOTS>1 concurrent slots share the + * batched forwards, so the shares describe the whole engine over the + * window, not the single request (same convention as the STAT hit% below). */ + printf("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %llu\n",dt, + r->prompt_tokens,r->emitted, + edisk_s()-r->pb.edisk,m->t_ewait-r->pb.ewait,m->t_emm-r->pb.emm, + m->t_attn-r->pb.attn,m->t_head-r->pb.head, + (unsigned long long)(m->n_fw-r->pb.n_fw)); printf("DONE %llu STAT %d %.2f %.1f %.2f %d %d\n",r->id,r->emitted, r->emitted/dt,(dh+dm)>0?100.0*dh/(dh+dm):0.0,rss_gb(), r->prompt_tokens,r->length_limited); fflush(stdout); kv_bind(m,&sc->kv); kv_disk_append(m,sc->hist,sc->len); + /* PROF window = this request's lifetime; with KV_SLOTS>1 concurrent slots + * share the batched forwards, so the shares describe the engine, not the + * single request (same convention as the STAT hit%% above). */ + if(g_prof) prof_report(m,&r->pb,dt,r->emitted,stderr); r->active=0; } @@ -4870,6 +4571,7 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, int nctx, ServeReq *r=&req[sub.slot]; memset(r,0,sizeof(*r)); r->id=sub.id; r->maximum=sub.max_tokens; r->temp=sub.temperature; r->top_p=sub.top_p; r->prompt_tokens=nt; r->started=now_s(); r->hits0=m->hits; r->miss0=m->miss; + prof_base(m,&r->pb); /* a few loads: cheap enough to always track */ int room=maxctx-sc->len-1; if(r->maximum>room){r->maximum=room; r->length_limited=1;} g_temp=r->temp; g_nuc=r->top_p; int next=pick_tok(logit,m->c.vocab,-1); free(logit); @@ -4882,11 +4584,11 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, int nctx, static void run_serve_mux(Model *m, const char *snap){ char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap); - Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); stops_arm(&m->c,eos); + Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); stops_arm_tok(&m->c,eos,&T); g_draft=0; /* one scheduler owns every forward; MTP/speculation is not ragged-safe */ int maxctx=getenv("CTX")?atoi(getenv("CTX")):4096; int nctx=getenv("KV_SLOTS")?atoi(getenv("KV_SLOTS")):1; - if(nctx<1||nctx>16){fprintf(stderr,"KV_SLOTS deve essere tra 1 e 16\n");exit(2);} + if(nctx<1||nctx>512){fprintf(stderr,"KV_SLOTS must be between 1 and 512\n");exit(2);} g_kvsave=getenv("KVSAVE")?atoi(getenv("KVSAVE")):1; KVState *initial=m->kv; free(initial->kv_start); free(initial); ServeCtx *ctx=calloc(nctx,sizeof(*ctx)); ServeReq *req=calloc(nctx,sizeof(*req)); @@ -4942,12 +4644,14 @@ static void run_serve_mux(Model *m, const char *snap){ } active=0; for(int i=0;in_fw++; + if(g_prof) prof_lat(now_s()-tf0); for(int s=0;slen++; g_temp=r->temp; g_nuc=r->top_p; @@ -4983,7 +4687,7 @@ static void run_serve(Model *m, const char *snap){ char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap); Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); - stops_arm(&m->c, eos); + stops_arm_tok(&m->c, eos, &T); grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */ if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della * distribuzione int4 e' rumore di quantizzazione */ @@ -5017,6 +4721,7 @@ static void run_serve(Model *m, const char *snap){ if(len<1){ printf("\x01\x01" "END" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f\n", rss_gb()); fflush(stdout); continue; } int cur=ngen; if(len+cur+g_draft+2>=maxctx) cur=maxctx-len-g_draft-2; uint64_t h0=m->hits, ms0=m->miss; double tt0=now_s(); + ProfBase pb; if(g_prof) prof_base(m,&pb); float *logit=step(m,hist+len-1,1,len-1); EmitStream es={&T,m,now_s(),0,1}; int prod=0; @@ -5026,7 +4731,9 @@ static void run_serve(Model *m, const char *snap){ double dh=(double)(m->hits-h0), dm=(double)(m->miss-ms0); printf("\n\x01\x01" "END" "\x01\x01\n"); printf("STAT %d %.2f %.1f %.2f\n", prod, prod/tdt, (dh+dm)>0?100.0*dh/(dh+dm):0.0, rss_gb()); - fflush(stdout); kv_disk_append(m,hist,len); repin_pass(m); continue; } /* RFC: re-pin a caldo tra i turni / live re-pin between turns */ + fflush(stdout); + if(g_prof) prof_report(m,&pb,tdt,prod,stderr); /* per-turn window; stdout is the framed protocol */ + kv_disk_append(m,hist,len); repin_pass(m); continue; } /* RFC: re-pin a caldo tra i turni / live re-pin between turns */ if(nr<1){ printf("\x01\x01" "END" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f\n", rss_gb()); fflush(stdout); continue; } /* API mode: an exact, length-prefixed prompt. Unlike the interactive * line protocol this accepts newlines. The tokenized prompt is matched @@ -5094,6 +4801,7 @@ static void run_serve(Model *m, const char *snap){ uint64_t agh0=m->route_agree_hit, agt0=m->route_agree_tot; uint64_t kln0=m->route_kl_n; double kls0=m->route_kl_sum; double tt0=now_s(); + ProfBase pb; if(g_prof) prof_base(m,&pb); float *logit; if(k>0){ logit=step(m,hist+len,k,len); len+=k; } else logit=step(m,hist+len-1,1,len-1); /* prompt identico/prefisso: rigenera i logits */ @@ -5120,6 +4828,7 @@ static void run_serve(Model *m, const char *snap){ agree_pct,kl_mean); printf("\n"); fflush(stdout); + if(g_prof) prof_report(m,&pb,tdt,prod,stderr); /* per-turn window; stdout is the framed protocol */ free(raw); g_temp=base_temp; g_nuc=base_nuc; usage_save(m); /* la cache che impara: storia aggiornata a ogni turno */ kv_disk_append(m,hist,len); /* KV su disco: il prossimo avvio riparte da qui */ @@ -5141,207 +4850,7 @@ static int *read_arr(jval*o,const char*k,int*n){ if(!r){ fprintf(stderr,"OOM read_arr\n"); exit(1); } for(int i=0;ilen;i++) r[i]=(int)a->kids[i]->num; *n=a->len; return r; } -/* byte residenti di un tensore [O,I] al numero di bit dato (specchio di qt_bytes) */ -static int64_t tbytes(int O,int I,int bits){ - if(bits>=16) return (int64_t)O*I*4; - if(bits>=5) return (int64_t)O*I + (int64_t)O*4; - return (int64_t)O*((I+1)/2) + (int64_t)O*4; -} -/* byte VERI di un expert: dal container se pre-quantizzato, altrimenti stima da ebits */ -static int64_t expert_bytes_probe(Model *m, int ebits){ - Cfg *c=&m->c; int64_t eb=0; char nm[256]; - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.gate_proj.weight",c->first_dense); - if(st_nbytes(&m->S,nm)>0){ - const char *suf[3]={"gate_proj","up_proj","down_proj"}; - for(int k=0;k<3;k++){ - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight",c->first_dense,suf[k]); - eb+=st_nbytes(&m->S,nm); - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight.qs",c->first_dense,suf[k]); - int64_t q=st_nbytes(&m->S,nm); if(q>0) eb+=q; - } - } - if(eb<=0) eb = tbytes(c->moe_inter,c->hidden,ebits)*2 + tbytes(c->hidden,c->moe_inter,ebits); - return eb; -} - -/* TIERS: fotografia della piramide expert per la dashboard web — - * "TIERS " sul canale di protocollo. - * ram = pinnati non-VRAM + LRU corrente; disk = tutto il resto. */ -/* BRAIN MAP (dashboard): per-turn expert hit bitmap + full residency/heat map. - * g_ehit[layer][eid]=1 quando l'expert viene instradato in questo turno; - * hits_emit lo serializza e lo azzera. emap_emit fotografa tier+heat di TUTTI. */ -static uint8_t **g_ehit; -static void ehit_mark(Model *m, int layer, int eid){ - if(!g_ehit){ Cfg *c=&m->c; - g_ehit=calloc(c->n_layers+1,sizeof(uint8_t*)); - for(int i=0;i<=c->n_layers;i++) g_ehit[i]=calloc(c->n_experts,1); - } - g_ehit[layer][eid]=1; -} - -/* HWINFO: hardware snapshot for the web dashboard — emitted once at READY. */ -static void hwinfo_emit(Model *m){ - Cfg *c=&m->c; (void)c; /* silence -Wunused on builds without /proc (#148 report) */ - /* CPU */ - char cpu[256]=""; -#ifdef _WIN32 - /* niente /proc su Windows: brand string via CPUID (0x80000002..4), zero - * dipendenze extra. La dashboard mostrava "0 GB RAM / 0 cores" perche' - * tutto questo blocco era solo-Linux mentre il ramo CUDA funzionava. */ -#if defined(__x86_64__) || defined(__i386__) - { unsigned int r[12]={0}; unsigned int *w=r; - for(unsigned int f=0x80000002u; f<=0x80000004u; f++,w+=4) - __get_cpuid(f,&w[0],&w[1],&w[2],&w[3]); - char *b=(char*)r; b[47]=0; while(*b==' ')b++; - snprintf(cpu,sizeof(cpu),"%s",b); } -#endif -#else - FILE *ci=fopen("/proc/cpuinfo","r"); - if(ci){ char ln[256]; - while(fgets(ln,sizeof(ln),ci)) if(!strncmp(ln,"model name",10)){ - char *p=strchr(ln,':'); if(p){ p++; while(*p==' ')p++; - int n=(int)strlen(p); if(n>0&&p[n-1]=='\n')p[--n]=0; - snprintf(cpu,sizeof(cpu),"%s",p); } break; } - fclose(ci); } -#endif - int cores=0; -#ifdef _WIN32 - { SYSTEM_INFO si; GetSystemInfo(&si); cores=(int)si.dwNumberOfProcessors; } -#elif defined(_SC_NPROCESSORS_ONLN) - cores=(int)sysconf(_SC_NPROCESSORS_ONLN); -#endif - /* RAM */ - double ram_total=0,ram_avail=0; -#ifdef _WIN32 - compat_meminfo(&ram_total,&ram_avail); /* GlobalMemoryStatusEx, gia' in compat.h */ -#else - FILE *mi=fopen("/proc/meminfo","r"); - if(mi){ char ln[256]; double mt=0,ma=0; - while(fgets(ln,sizeof(ln),mi)){ - if(sscanf(ln,"MemTotal: %lf",&mt)==1) ram_total=mt/1e6; - if(sscanf(ln,"MemAvailable: %lf",&ma)==1) ram_avail=ma/1e6; - } fclose(mi); } -#endif - /* GPU */ - int ngpu=0; double vram_total=0; - char gpu_name[128]=""; -#ifdef COLI_CUDA - ngpu=g_cuda_ndev; vram_total=m->gpu_expert_bytes/1e9; - for(int i=0;i0){ - /* We don't have a device-name API; parse from the init log line stored in stderr. - * Simpler: just read the nvidia driver sysfs or use a fixed label. */ - snprintf(gpu_name,sizeof(gpu_name),"CUDA device x%d",g_cuda_ndev); - } -#endif - printf("HWINFO %d %.1f %.1f %d %.1f %s|%s\n", - cores,ram_total,ram_avail,ngpu,vram_total,cpu,gpu_name); - fflush(stdout); -} - -static void tiers_emit(Model *m){ - Cfg *c=&m->c; int nsp=0; - for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; - int total=(nsp+(m->has_mtp?1:0))*c->n_experts; - int pinned=0,lru=0; - for(int i=0;i<=c->n_layers;i++){ pinned+=m->npin?m->npin[i]:0; lru+=m->ecn?m->ecn[i]:0; } - int vram=0; double vram_gb=0; -#ifdef COLI_CUDA - vram=m->gpu_expert_count; vram_gb=m->gpu_expert_bytes/1e9; -#endif - int ram=pinned-vram+lru; if(ram<0) ram=0; - int disk=total-vram-ram; if(disk<0) disk=0; - double eb=(double)expert_bytes_probe(m,m->ebits); - printf("TIERS %d %d %d %.2f %.2f\n",vram,ram,disk,vram_gb,ram*eb/1e9); - fflush(stdout); -} - -/* EMAP: 1 byte/expert (2bit tier: 0=disk 1=RAM 2=VRAM | 6bit heat log2-bucket), - * righe = layer sparsi in ordine (+MTP se presente), colonne = n_experts. Hex. */ -static void emap_emit(Model *m){ - Cfg *c=&m->c; - int rows=0; - for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; - int has_mtp = m->has_mtp && m->eusage[c->n_layers]; - if(has_mtp) rows++; - int cols=c->n_experts; - char *hex=malloc((size_t)rows*cols*2+1); int w=0; - for(int i=0;i<=c->n_layers;i++){ - int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); - if(!is_row) continue; - for(int e=0;epin[i]; - for(int z=0;znpin[i];z++) if(P[z].eid==e){ -#ifdef COLI_CUDA - tier = P[z].g.cuda?2:1; -#else - tier = 1; -#endif - break; } - if(!tier && m->ecache && m->ecache[i]) - for(int z=0;zecn[i];z++) if(m->ecache[i][z].eid==e){ tier=1; break; } - uint32_t u = m->eusage[i]?m->eusage[i][e]:0; - int heat=0; while(u){ heat++; u>>=1; } if(heat>63) heat=63; - int b=(tier<<6)|heat; - hex[w++]="0123456789abcdef"[b>>4]; hex[w++]="0123456789abcdef"[b&15]; - } - } - hex[w]=0; - printf("EMAP %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); -} - -/* HITS: bitmap 1bit/expert (stesso ordine di EMAP), poi azzera per il turno dopo. */ -static void hits_emit(Model *m){ - Cfg *c=&m->c; if(!g_ehit) return; - int rows=0; - for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; - int has_mtp = m->has_mtp && m->eusage[c->n_layers]; - if(has_mtp) rows++; - int cols=c->n_experts, nb=(rows*cols+7)/8; - uint8_t *bm=calloc(nb,1); int bit=0; - for(int i=0;i<=c->n_layers;i++){ - int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); - if(!is_row) continue; - for(int e=0;e>3]|=1<<(bit&7); g_ehit[i][e]=0; } - } - char *hex=malloc((size_t)nb*2+1); int w=0; - for(int b=0;b>4]; hex[w++]="0123456789abcdef"[bm[b]&15]; } - hex[w]=0; - printf("HITS %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); free(bm); -} - -/* scarica su file l'istogramma d'uso degli expert: righe "layer eid count" (per PIN). - * Include la riga MTP (layer n_layers). Scrittura atomica (tmp+rename): viene chiamata - * anche a ogni turno di serve e il processo puo' morire in qualsiasi momento. */ -static void stats_dump_q(Model *m, const char *path, int quiet){ - char tmp[2100]; snprintf(tmp,sizeof(tmp),"%s.tmp",path); - FILE *f=fopen(tmp,"w"); if(!f){ if(!quiet) perror(tmp); return; } - Cfg *c=&m->c; int64_t tot=0, nz=0; - for(int i=0;i<=c->n_layers;i++){ if(!m->eusage[i]) continue; - for(int e=0;en_experts;e++) if(m->eusage[i][e]){ fprintf(f,"%d %d %u\n",i,e,m->eusage[i][e]); tot+=m->eusage[i][e]; nz++; } } - fclose(f); rename(tmp,path); - if(!quiet) fprintf(stderr,"[STATS] %lld selections across %lld distinct experts -> %s\n",(long long)tot,(long long)nz,path); -} -static void stats_dump(Model *m, const char *path){ stats_dump_q(m,path,0); } - -/* CACHE CHE IMPARA: istogramma d'uso PERSISTENTE in /.coli_usage. - * Caricato all'avvio (i contatori ripartono dalla storia), salvato a ogni turno: - * piu' usi colibri', meglio l'auto-pin conosce i TUOI expert caldi. */ -static char g_usage_path[2100]=""; -static int64_t usage_load(Model *m, const char *path){ - FILE *f=fopen(path,"r"); if(!f) return 0; - Cfg *c=&m->c; int l,e; uint32_t cnt; int64_t tot=0; - while(fscanf(f,"%d %d %u",&l,&e,&cnt)==3) - if(l>=0&&l<=c->n_layers&&e>=0&&en_experts&&m->eusage[l]){ m->eusage[l][e]+=cnt; tot+=cnt; } - fclose(f); return tot; -} -static void usage_save(Model *m){ if(g_usage_path[0]) stats_dump_q(m,g_usage_path,1); } +/* telemetry, stats, usage persistence — moved to telemetry.h */ /* HOT-STORE ("il redis del colibri'"): carica in RAM, UNA VOLTA e per sempre, i top expert * per frequenza d'uso misurata (file STATS di un run precedente), entro un budget in GB. @@ -5484,6 +4993,29 @@ static void pin_load(Model *m, const char *statspath, double gb){ double avail=expert_avail(m,ram_env,m->ebits,est_ctx); npin=avail>0?(int)(avail/eb):0; } else npin=(int)(gb*1e9/eb); +#ifdef COLI_CUDA + /* The VRAM budget must be known BEFORE npin is finalized: with + * CUDA_RELEASE_HOST the VRAM-ranked prefix's host slabs are freed right + * after upload, so those slots must NOT consume the RAM pin budget. + * Before this, a 6-GPU host lost its top ~9k ranked experts from the RAM + * count and pinned only the leftovers (measured: 9,280 VRAM + 1,721 RAM + * on a box whose RAM fits ~11k — the cold tail then paid disk forever). */ + double remaining[COLI_CUDA_MAX_DEVICES]={0}, placed_b[COLI_CUDA_MAX_DEVICES]={0}; + int placed_n[COLI_CUDA_MAX_DEVICES]={0}, gpu_prefix=0, prefix_est=0; + double budget=g_cuda_expert_gb*1e9, safe_total=0; + if(g_cuda_enabled&&(g_cuda_expert_gb>0||g_cuda_expert_auto)) for(int i=0;isafe_total) budget=safe_total; + if(g_cuda_enabled&&g_cuda_release_host&&budget>0){ + prefix_est=(int)(budget/eb)+g_cuda_ndev; + npin+=prefix_est; /* additive: prefix RAM is returned after upload */ + } +#endif if(npin>n) npin=n; if(npin<1){ free(r); return; } int *cnt_l=calloc(c->n_layers+1,sizeof(int)); /* +1: riga MTP */ @@ -5494,18 +5026,7 @@ static void pin_load(Model *m, const char *statspath, double gb){ for(int i=0;i<=c->n_layers;i++) m->npin[i]=cnt_l[i]; double t0=now_s(); #ifdef COLI_CUDA - double remaining[COLI_CUDA_MAX_DEVICES]={0}, placed_b[COLI_CUDA_MAX_DEVICES]={0}; - int placed_n[COLI_CUDA_MAX_DEVICES]={0}, gpu_prefix=0; - double budget=g_cuda_expert_gb*1e9, safe_total=0; - if(g_cuda_enabled&&(g_cuda_expert_gb>0||g_cuda_expert_auto)) for(int i=0;isafe_total) budget=safe_total; - if(g_cuda_enabled&&g_cuda_release_host&&budget>0){ gpu_prefix=(int)(budget/eb)+g_cuda_ndev; if(gpu_prefix>npin)gpu_prefix=npin; } + if(prefix_est>0){ gpu_prefix=prefix_est; if(gpu_prefix>npin) gpu_prefix=npin; } #else int gpu_prefix=0; #endif @@ -5513,7 +5034,7 @@ static void pin_load(Model *m, const char *statspath, double gb){ * released before the disjoint RAM-ranked suffix is allocated. */ #pragma omp parallel for schedule(dynamic,1) for(int a=0;a<(gpu_prefix?gpu_prefix:npin);a++) - expert_load(m,r[a].l,r[a].e,&m->pin[r[a].l][slot_of[a]],1); + expert_load(m,r[a].l,r[a].e,&m->pin[r[a].l][slot_of[a]],1,0); /* startup pin load; demand=0, never classified */ m->resident_bytes+=(int64_t)(gpu_prefix?gpu_prefix:npin)*eb; #ifdef COLI_CUDA if(g_cuda_enabled && budget>0){ @@ -5557,7 +5078,7 @@ static void pin_load(Model *m, const char *statspath, double gb){ if(gpu_prefix>0&&gpu_prefixpin[r[a].l][slot_of[a]],1); + expert_load(m,r[a].l,r[a].e,&m->pin[r[a].l][slot_of[a]],1,0); /* startup pin load; demand=0, never classified */ m->resident_bytes+=(int64_t)(npin-gpu_prefix)*eb; } fprintf(stderr,"[PIN] placement: %d VRAM + %d RAM expert (%.1f GB warm) in %.0fs da %s\n", @@ -5625,6 +5146,7 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){ if(auto_b){ ram_gb = g_mem_avail_boot*0.88; /* misurata PRIMA del load: il residente gia' * allocato viene sottratto sotto, non due volte */ if(ram_gb<4){ fprintf(stderr,"[RAM] MemAvailable is unreadable or too low; assuming 8 GB\n"); ram_gb=8; } } + g_ram_budget_gb = ram_gb; /* #403: la RSS-guard usa il budget RISOLTO */ /* slack ONESTO, non forfettario (l'OOM del 2026-07-04 veniva da qui): * ws[64] slab del working-set (si materializzano TUTTI nel prefill batch-union), * KV cache a max_ctx, kvb_all della ricostruzione k/v in attention, @@ -5714,9 +5236,9 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){ * self-test, and would "generate" from "$P$G". So on Windows a PROMPT carrying * cmd's $-metacodes is ignored; set COLI_PROMPT to pass a real prompt from cmd. */ static const char *coli_user_prompt(void){ - const char *p = getenv("COLI_PROMPT"); + const char *p = getenv_utf8("COLI_PROMPT"); if(p) return p; - p = getenv("PROMPT"); + p = getenv_utf8("PROMPT"); #ifdef _WIN32 if(p) for(const char *q=p; q[0]; q++) if(q[0]=='$' && q[1] && strchr("ABCDEFGHLNPQSTV_+|$", q[1]&~0x20)){ p=NULL; break; } @@ -5724,6 +5246,34 @@ static const char *coli_user_prompt(void){ return p; } +/* PROF=1 startup header: one self-describing block so a saved log answers + * "what machine, what config" when comparing runs after changing RAM_GB, + * knobs, or moving to another host. */ +static void prof_config(Model *m, double ram_env, int est_ctx){ + Cfg *c=&m->c; + char cpu[256]; int cores; double rt,ra; + hw_probe(cpu,sizeof(cpu),&cores,&rt,&ra); + const char *backend="CPU"; +#ifdef COLI_CUDA + if(g_cuda_enabled) backend="CUDA"; +#endif +#ifdef COLI_METAL + if(g_metal_enabled) backend="Metal"; +#endif + int nsp=0; for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; + int rows=nsp+(m->has_mtp?2:0); /* stessa convenzione di cap_for_ram (MTP int8 = 2x) */ + int pinned=0; for(int i=0;i<=c->n_layers;i++) if(m->npin) pinned+=m->npin[i]; + double eb=(double)expert_bytes_probe(m,m->ebits); + fprintf(stderr,"[PROF] machine: %s | %d cores (%d omp threads) | RAM %.1f GB total, %.1f GB available | backend %s\n", + cpu[0]?cpu:"unknown CPU",cores,omp_get_max_threads(),rt,ra,backend); + fprintf(stderr,"[PROF] config: RAM_GB=%s%.1f CTX=%d | expert cache cap %d/layer (up to %.1f GB) | pinned %d (%.1f GB) | " + "DRAFT=%d PIPE=%d DIRECT=%d MMAP=%d IDOT=%d DSA=%s PILOT=%d CACHE_ROUTE=%d\n", + ram_env<=0?"auto ":"",ram_env<=0?g_mem_avail_boot*0.88:ram_env,est_ctx, + m->ecap,(double)m->ecap*rows*eb/1e9,pinned,pinned*eb/1e9, + g_draft,g_pipe,g_direct,g_mmap,g_idot, + (m->has_dsa&&c->index_topk)?"on":"off",g_pilot,g_cache_route); +} + int main(int argc, char **argv){ /* ---- Permanent OpenMP hot-thread tuning. The per-expert matmul regions are * tiny and back-to-back; with the default passive wait policy libgomp parks @@ -5748,6 +5298,14 @@ int main(int argc, char **argv){ !getenv("COLI_CUDA") && !getenv("COLI_METAL")){ setenv("OMP_WAIT_POLICY","active",0); /* keep the team hot across the tiny per-expert matmul regions */ setenv("GOMP_SPINCOUNT","200000",0); /* spin briefly, then yield so long disk waits don't burn a core */ + /* LLVM libomp (clang builds: FreeBSD cc, macOS, some Linux setups) does not + * read GOMP_*: with OMP_WAIT_POLICY=active it sets KMP_BLOCKTIME=infinite, + * so the idle team SPINS FOREVER once generation ends — a serve-mode engine + * parked on stdin burns ~100% x nthreads (#341, measured 3000% on FreeBSD). + * 200 ms of blocktime keeps the team hot across back-to-back expert matmuls + * and lets it sleep at the prompt. libgomp ignores KMP_*; overwrite=0 keeps + * the user's own setting authoritative. */ + setenv("KMP_BLOCKTIME","200",0); setenv("OMP_PROC_BIND","close",0); /* pack the team onto adjacent cores for cache locality */ setenv("OMP_DYNAMIC","FALSE",0); /* fixed team size: no per-region thread-count churn */ setenv("COLI_OMP_TUNED","1",1); @@ -5762,6 +5320,9 @@ int main(int argc, char **argv){ perror("[OMP] execv self-reexec failed, running untuned"); #endif } +#ifdef _WIN32 + _setmode(fileno(stdout), O_BINARY); +#endif #if defined(__AVX512F__) && defined(__AVX512BW__) if(getenv("I4_ACC512")) g_i4_acc512=atoi(getenv("I4_ACC512"))!=0; if(getenv("I4_ACC512_TEST")){ @@ -5775,6 +5336,7 @@ int main(int argc, char **argv){ g_prefetch = getenv("PREFETCH")?atoi(getenv("PREFETCH")):0; g_mmap = getenv("COLI_MMAP")?atoi(getenv("COLI_MMAP")):0; if(g_mmap) fprintf(stderr,"[MMAP] expert = viste zero-copy nei file (page cache = cache)\n"); + numa_init(); /* COLI_NUMA=1: expert-slab interleave (#82) */ g_topk = getenv("TOPK")?atoi(getenv("TOPK")):0; g_topp = getenv("TOPP")?atof(getenv("TOPP")):0; /* EXPERT_BUDGET e' sotto quarantena: la finestra operativa e' misurata VUOTA. @@ -5855,6 +5417,8 @@ int main(int argc, char **argv){ g_pipe_nw = getenv("PIPE_WORKERS")?atoi(getenv("PIPE_WORKERS")):8; /* I/O worker threads */ if(g_pipe_nw<1) g_pipe_nw=1; g_direct = getenv("DIRECT")?atoi(getenv("DIRECT")):0; + { const char *dh=getenv("COLI_DISKCLASS_WINDOW"); /* DISK-CLASS recency window, see its declaration */ + if(dh){ g_direct_heat_ticks=(uint32_t)strtoul(dh,NULL,10); g_direct_heat_explicit=1; } } g_uring = getenv("URING")?atoi(getenv("URING")):0; if(g_uring){ #ifdef __linux__ @@ -5897,8 +5461,9 @@ int main(int argc, char **argv){ int cap = argc>1?atoi(argv[1]):64; int ebits= argc>2?atoi(argv[2]):8; int dbits= argc>3?atoi(argv[3]):ebits; - if(getenv("SERVE") && (kv_slot_count()<1 || kv_slot_count()>16)){ - fprintf(stderr,"KV_SLOTS must be between 1 and 16\n"); return 2; + int kv_limit=(getenv("SERVE_BATCH")&&atoi(getenv("SERVE_BATCH")))?512:16; + if(getenv("SERVE") && (kv_slot_count()<1 || kv_slot_count()>kv_limit)){ + fprintf(stderr,"KV_SLOTS must be between 1 and %d\n",kv_limit); return 2; } #ifdef COLI_CUDA if(getenv("COLI_CUDA") && atoi(getenv("COLI_CUDA"))){ @@ -5952,6 +5517,19 @@ int main(int argc, char **argv){ printf("== GLM C engine (glm_moe_dsa), cache=%d experts/layer | experts@%d-bit dense@%d-bit | idot: " IDOT_KERNEL " ==\n", cap, ebits, dbits); g_mem_avail_boot = mem_available_gb(); Model m; double t0=now_s(); model_init(&m,snap,cap,ebits,dbits); + if(!g_direct_heat_explicit){ /* COLI_DISKCLASS_WINDOW default, needs m.c (topk/n_layers) */ + /* CURRENT-STATE CALIBRATION: the "8" multiplier (recency window ~= the last 8 + * tokens' worth of routing) is a measured-config constant, not a derived truth. + * Coordinates: measured 2026-07, macOS 26.5, M5 Max 128 GB, base caa49f7, + * GLM-5.2 int4 (topk=8, n_layers=78). On that setup it splits the classes + * cleanly (decode cold share ~71% of classified bytes, 1061.85/1491.49 GB); + * other models, page-cache pressures, or disks may want a different window -- + * COLI_DISKCLASS_WINDOW overrides, and the DISK-CLASS line itself is the + * tuning instrument. */ + uint32_t nl=(uint32_t)(m.c.n_layers>0?m.c.n_layers:1), k=(uint32_t)(m.c.topk>0?m.c.topk:1); + g_direct_heat_ticks = k*nl*8u; /* ~last 8 tokens' worth of routing, see the declaration */ + if(!g_direct_heat_ticks) g_direct_heat_ticks=1; + } if(g_draft<0){ #ifdef COLI_CUDA /* MTP is disabled under CUDA by default: cold (streaming) experts still @@ -5969,14 +5547,32 @@ int main(int argc, char **argv){ #endif } if(getenv("DSA_TOPK")) m.c.index_topk=atoi(getenv("DSA_TOPK")); /* override per test */ + /* Il path MUX (SERVE_BATCH=1, cioe' `coli serve`) forza g_draft=0 sotto — + * la speculazione non e' ragged-safe nel batch multi-slot. Segnalarlo QUI, + * altrimenti "MTP active (draft=8)" mentirebbe: il messaggio e' stampato + * prima della scelta del path (run_serve_mux, sotto), e con DRAFT=8 diceva + * "active" per poi disabilitarlo in silenzio (#358, LordMZTE). */ + int mux_will_disable_mtp = getenv("SERVE") && getenv("SERVE_BATCH") && atoi(getenv("SERVE_BATCH")); + int eff_draft = mux_will_disable_mtp ? 0 : g_draft; printf("loaded in %.2fs | resident dense: %.2f MB | layers=%d experts=%d | MTP %s (draft=%d)\n", now_s()-t0, m.resident_bytes/(1024.0*1024.0), m.c.n_layers, m.c.n_experts, - m.has_mtp?"ACTIVE":"absent", g_draft); + m.has_mtp?(mux_will_disable_mtp?"DISABLED (multiplexed serve)":"ACTIVE"):"absent", eff_draft); /* anche su stderr: e' il canale che le UI (coli) mostrano all'utente */ - fprintf(stderr,"[MTP] %s (draft=%d)\n", m.has_mtp?"active: native speculative decoding":"absent", g_draft); - if(!strncmp(snap,"/mnt/",5)) - fprintf(stderr,"WARNING: the model is on %s (slow 9p/Windows filesystem; fadvise is ineffective).\n" - " Keep it on ext4 (for example, /home/...) for memory efficiency and speed.\n", snap); + if(mux_will_disable_mtp && m.has_mtp) + fprintf(stderr,"[MTP] disabled in multiplexed serve (SERVE_BATCH=1): speculation is not " + "ragged-safe across KV slots. Single-client interactive use (`coli chat`) keeps MTP.\n"); + else + fprintf(stderr,"[MTP] %s (draft=%d)\n", m.has_mtp?"active: native speculative decoding":"absent", eff_draft); +#ifdef __linux__ + { /* Only warn for a GENUINE 9p mount (WSL Windows drives, magic 0x01021997), where + * fadvise is a no-op. The old check was `snap` starting with "/mnt/", which + * false-positives on native-Linux ZFS/ext4/xfs/NFS mounts that also live under /mnt. */ + struct statfs sfb; + if(statfs(snap,&sfb)==0 && (unsigned long)sfb.f_type==0x01021997UL) + fprintf(stderr,"WARNING: the model is on %s (9p/Windows filesystem; fadvise is ineffective).\n" + " Keep it on a native Linux fs (ext4/xfs/zfs) for memory efficiency and speed.\n", snap); + } +#endif /* HOT-STORE: PIN= [PIN_GB=g] -> top expert per frequenza fissi in RAM. * Va PRIMA di cap_for_ram: i pinnati contano nel residente. */ if(getenv("PIN")){ @@ -6028,7 +5624,9 @@ int main(int argc, char **argv){ } /* SEMPRE: senza clamp la LRU cresce fino a cap*76 layer = decine di GB -> OOM-kill. * RAM_GB assente o <=0 = budget automatico da MemAvailable. */ - cap_for_ram(&m, ram_env, ebits, est_ctx); } + cap_for_ram(&m, ram_env, ebits, est_ctx); + g_prof = getenv("PROF")?atoi(getenv("PROF")):0; /* PROF=1: opt-in performance profile */ + if(g_prof) prof_config(&m, ram_env, est_ctx); } const char *stats=getenv("STATS"); /* STATS= -> istogramma uso expert a fine run */ /* modo scoring per benchmark: SCORE= -> log-likelihood per riga */ @@ -6105,6 +5703,7 @@ int main(int argc, char **argv){ return 0; } int *out=malloc((np+n_new)*sizeof(int)); + ProfBase pb; prof_base(&m,&pb); double t=now_s(); generate(&m,prompt,np,n_new,out); double dt=now_s()-t; int match=0; printf("\nReference (oracle): "); for(int i=np;i #include +#include /* _mkdir (for the mkdtemp shim below) */ #include #include #include @@ -143,6 +144,12 @@ static inline int compat_fadvise(int fd, off_t off, off_t len, int advice){ * Thread-safe (no shared seek position). Gestisce offset >4 GB e chunking * per letture >2 GB (anche se i tensori individuali sono nell'ordine dei * MB-centinaia di MB, il wrapper e' robusto per ogni taglia). */ +/* Ultimo GetLastError() di una ReadFile fallita, per thread: il chiamante + * (pread_full in glm.c) lo stampa accanto a strerror. Senza questo, OGNI + * fallimento Windows collassa in "EIO -> Input/output error" e la diagnosi + * dal campo diventa un tirare a indovinare (#307: tre giri di ipotesi tra + * tre persone perche' il codice vero non compariva da nessuna parte). */ +static __thread DWORD compat_pread_lasterr __attribute__((unused)); static inline ssize_t compat_pread(int fd, void *buf, size_t n, off_t off){ intptr_t osfh = _get_osfhandle(fd); if(osfh == -1 || osfh == -2){ errno = EBADF; return -1; } @@ -158,6 +165,7 @@ static inline ssize_t compat_pread(int fd, void *buf, size_t n, off_t off){ if(!ReadFile(h, (char*)buf + total, chunk32, &rd, &ov)){ DWORD err = GetLastError(); if(err == ERROR_HANDLE_EOF) break; /* past EOF → return bytes read (0 if none, matching POSIX pread) */ + compat_pread_lasterr = err; /* preserva il codice VERO per il report (#307) */ if(err == ERROR_INVALID_HANDLE || err == ERROR_INVALID_FUNCTION) errno = EBADF; else errno = EIO; return -1; @@ -237,7 +245,9 @@ static inline int compat_rename(const char *old, const char *new){ /* --- rss_gb: getrusage -> GetProcessMemoryInfo --- * ru_maxrss in KB (come Linux): rss_gb() divide per 1e6 → GB corretti. */ #include -#pragma comment(lib, "psapi.lib") +#ifdef _MSC_VER +#pragma comment(lib, "psapi.lib") /* MSVC: link psapi; MinGW/GCC uses -lpsapi */ +#endif struct rusage { long ru_maxrss; }; #define RUSAGE_SELF 0 static inline int getrusage(int who, struct rusage *r){ @@ -304,8 +314,52 @@ static inline int compat_setenv(const char *name, const char *value, int overwri } #define setenv(name,value,overwrite) compat_setenv(name,value,overwrite) +/* --- getenv_utf8: read an env var as UTF-8, not through the ANSI codepage --- + * Plain getenv()/_environ are populated by the CRT from the ANSI-codepage view + * of the process environment block, not UTF-8. A parent that hands the child a + * Unicode value via CreateProcessW's wide env block (e.g. Python's subprocess + * module, which coli uses to pass the chat prompt) round-trips correctly only + * through GetEnvironmentVariableW; going through narrow getenv() re-encodes it + * via CP_ACP first, so any non-ASCII prompt text (Cyrillic, CJK, ...) comes out + * corrupted before the byte-level tokenizer ever sees it. Read the wide value + * directly and convert straight to UTF-8, bypassing the ANSI codepage entirely. + * Returned buffer is intentionally leaked: called a handful of times at + * startup, lives for the process. */ +static inline const char *compat_getenv_utf8(const char *name){ + wchar_t wname[64]; + if(MultiByteToWideChar(CP_UTF8, 0, name, -1, wname, 64) <= 0) return getenv(name); + DWORD need = GetEnvironmentVariableW(wname, NULL, 0); + if(!need) return NULL; + wchar_t *wval = (wchar_t*)malloc(need * sizeof(wchar_t)); + if(!wval) return NULL; + GetEnvironmentVariableW(wname, wval, need); + int blen = WideCharToMultiByte(CP_UTF8, 0, wval, -1, NULL, 0, NULL, NULL); + char *val = blen>0 ? (char*)malloc((size_t)blen) : NULL; + if(val) WideCharToMultiByte(CP_UTF8, 0, wval, -1, val, blen, NULL, NULL); + free(wval); + return val; +} +#define getenv_utf8(name) compat_getenv_utf8(name) + +/* --- mkdtemp -> _mktemp + _mkdir (POSIX mkdtemp assente su Windows) --- + * Test binaries (test_stops.c) create a scratch dir in the CWD via a + * "name_XXXXXX" template; POSIX mkdtemp fills the X's and mkdirs 0700. The + * Windows CRT has _mktemp (in-place, same XXXXXX contract) so we compose it. + * Returns the template pointer on success, NULL on failure — matching POSIX. */ +static inline char *compat_mkdtemp(char *tmpl){ + if(!tmpl) return NULL; + if(!_mktemp(tmpl)) return NULL; /* fills the trailing X's in place */ + if(_mkdir(tmpl) != 0) return NULL; /* EEXIST is impossible post-_mktemp */ + return tmpl; +} +#define mkdtemp(tmpl) compat_mkdtemp(tmpl) + #endif /* _WIN32 */ +#ifndef getenv_utf8 +#define getenv_utf8(name) getenv(name) +#endif + /* --- compat_aligned_free su piattaforme diverse da Windows --- * Su Linux/macOS, posix_memalign usa free() normale. */ #ifndef compat_aligned_free diff --git a/c/doctor.py b/c/doctor.py index 4cb5a42..cbe9c87 100644 --- a/c/doctor.py +++ b/c/doctor.py @@ -19,16 +19,33 @@ def _check(identifier, status, summary, **details): def cuda_linkage(engine_path): """Return CUDA linkage state without loading the executable or CUDA runtime.""" - if not Path(engine_path).is_file() or os.name != "posix": + engine = Path(engine_path) + if not engine.is_file(): return {"linked": False, "missing": False} - try: - result = subprocess.run(["ldd", str(engine_path)], capture_output=True, text=True, - timeout=3, check=False) - except (OSError, subprocess.SubprocessError): - return {"linked": False, "missing": False} - lines = [line for line in result.stdout.splitlines() if "libcudart" in line] - return {"linked": any("not found" not in line for line in lines), - "missing": any("not found" in line for line in lines)} + if os.name == "posix": + try: + result = subprocess.run(["ldd", str(engine)], capture_output=True, text=True, + timeout=3, check=False) + except (OSError, subprocess.SubprocessError): + return {"linked": False, "missing": False} + lines = [line for line in result.stdout.splitlines() if "libcudart" in line] + return {"linked": any("not found" not in line for line in lines), + "missing": any("not found" in line for line in lines)} + if sys.platform == "win32": + # Windows CUDA_DLL=1 builds never link libcudart directly: glm.exe loads + # coli_cuda.dll at runtime via LoadLibrary (backend_loader.c), so there's no + # import-table entry for ldd/dumpbin to see. Detect the COLI_CUDA build via a + # marker string baked into glm.c's #ifdef COLI_CUDA block instead, and require + # coli_cuda.dll to actually sit next to glm.exe (else CUDA init fails at startup). + try: + built = b"[CUDA] mode: routed experts" in engine.read_bytes() + except OSError: + return {"linked": False, "missing": False} + if not built: + return {"linked": False, "missing": False} + dll_present = (engine.parent / "coli_cuda.dll").is_file() + return {"linked": dll_present, "missing": not dll_present} + return {"linked": False, "missing": False} def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *, diff --git a/c/kv_persist.h b/c/kv_persist.h new file mode 100644 index 0000000..51289a8 --- /dev/null +++ b/c/kv_persist.h @@ -0,0 +1,121 @@ +/* kv_persist.h — .coli_kv on-disk KV cache persistence. + * Conversations reopen warm across engine restarts: the compressed MLA KV-cache + * is appended incrementally after every turn, crash-safe (nrec written last). + * Include after Model/KVState/Cfg are defined; requires now_s() and g_draft. */ +#ifndef KV_PERSIST_H +#define KV_PERSIST_H + +static int g_kvsave=1; +#define KV_MAGIC "COLIKV1\0" + +static void kv_hdr(Model *m, int32_t *h, int nrec){ + Cfg *c=&m->c; int nic=0; + for(int i=0;in_layers;i++) if(m->Ic && m->Ic[i]) nic++; + h[0]=c->n_layers; h[1]=c->kv_lora; h[2]=c->qk_rope; + h[3]=m->has_dsa?c->index_hd:0; h[4]=nic; h[5]=c->vocab; h[6]=nrec; h[7]=0; +} + +static int64_t kv_rec_bytes(Model *m){ + Cfg *c=&m->c; + int64_t rec = 4 + (int64_t)c->n_layers*(c->kv_lora+c->qk_rope)*4; + if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) rec+=(int64_t)c->index_hd*4; + return rec; +} + +static int kv_disk_open(Model *m){ + KVState *k=m->kv; + if(k->disk_fp) return 1; + k->disk_fp=fopen(k->disk_path,"r+b"); + if(!k->disk_fp){ + k->disk_fp=fopen(k->disk_path,"wb"); + if(!k->disk_fp) return 0; + int32_t h[8]; kv_hdr(m,h,0); + fwrite(KV_MAGIC,1,8,k->disk_fp); fwrite(h,4,8,k->disk_fp); + fflush(k->disk_fp); + fclose(k->disk_fp); + k->disk_fp=fopen(k->disk_path,"r+b"); + if(!k->disk_fp) return 0; + } + return 1; +} + +static void kv_disk_truncate(Model *m, int nrec){ + if(!g_kvsave) return; + KVState *k=m->kv; + if(k->disk_fp){ fclose(k->disk_fp); k->disk_fp=NULL; } + FILE *f=fopen(k->disk_path,"r+b"); + if(!f){ k->disk_nrec=0; return; } + k->disk_nrec=nrec; + int32_t nr=nrec; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); + fflush(f); fclose(f); +} + +static void kv_disk_reset(Model *m){ kv_disk_truncate(m,0); } + +static void kv_disk_append(Model *m, const int *hist, int len){ + KVState *k=m->kv; + if(!g_kvsave || len<=k->disk_nrec) return; + Cfg *c=&m->c; + if(!kv_disk_open(m)) return; + FILE *f=k->disk_fp; + int64_t rec = kv_rec_bytes(m); + if(rec > k->disk_buf_cap){ + uint8_t *nb=realloc(k->disk_buf, rec); + if(!nb) return; + k->disk_buf=nb; k->disk_buf_cap=rec; + } + fseek(f, 8+8*4 + (int64_t)k->disk_nrec*rec, SEEK_SET); + for(int p=k->disk_nrec;pdisk_buf; + *(int32_t*)b = hist[p]; b+=4; + for(int i=0;in_layers;i++){ + memcpy(b, m->Lc[i]+(int64_t)p*c->kv_lora, (size_t)c->kv_lora*4); b+=c->kv_lora*4; + memcpy(b, m->Rc[i]+(int64_t)p*c->qk_rope,(size_t)c->qk_rope*4); b+=c->qk_rope*4; + } + if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]){ + memcpy(b, m->Ic[i]+(int64_t)p*c->index_hd, (size_t)c->index_hd*4); b+=c->index_hd*4; + } + fwrite(k->disk_buf, 1, (size_t)rec, f); + } + fflush(f); + int32_t nr=len; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); + fflush(f); + k->disk_nrec=len; +} + +static int kv_disk_load(Model *m, int *hist, int maxctx){ + if(!g_kvsave) return 0; + KVState *k=m->kv; + Cfg *c=&m->c; + FILE *f=fopen(k->disk_path,"rb"); if(!f) return 0; + char mg[8]; int32_t h[8], w[8]; kv_hdr(m,w,0); + if(fread(mg,1,8,f)!=8 || memcmp(mg,KV_MAGIC,8) || fread(h,4,8,f)!=8 || + h[0]!=w[0]||h[1]!=w[1]||h[2]!=w[2]||h[3]!=w[3]||h[4]!=w[4]||h[5]!=w[5]){ + fprintf(stderr,"[KV] ignoring .coli_kv from a different model or version\n"); fclose(f); return 0; } + int nrec=h[6]; + if(nrec<1){ fclose(f); return 0; } + if(nrec>=maxctx-8-g_draft){ + fprintf(stderr,"[KV] saved conversation (%d tokens) exceeds the context: starting over\n",nrec); + fclose(f); return 0; } + double t0=now_s(); + for(int p=0;pn_layers;i++){ + if(fread(m->Lc[i]+(int64_t)p*c->kv_lora, 4, c->kv_lora, f)!=(size_t)c->kv_lora || + fread(m->Rc[i]+(int64_t)p*c->qk_rope, 4, c->qk_rope, f)!=(size_t)c->qk_rope){ nrec=p; goto out; } + } + if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) + if(fread(m->Ic[i]+(int64_t)p*c->index_hd, 4, c->index_hd, f)!=(size_t)c->index_hd){ nrec=p; goto out; } + } +out: + fclose(f); + if(nrec>0){ + if(m->has_mtp) m->kv_start[c->n_layers]=-1; + fprintf(stderr,"[KV] resumed conversation from disk: %d tokens in %.1fs (no re-prefill)\n", + nrec, now_s()-t0); + } + k->disk_nrec=nrec; + return nrec; +} + +#endif /* KV_PERSIST_H */ diff --git a/c/olmoe.c b/c/olmoe.c index 5923bde..840b59a 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -5,6 +5,15 @@ * Densa (embed, attn, router, norme, lm_head) residente in RAM (float32). * Expert letti dal disco on-demand via pread+fadvise(DONTNEED), cache LRU per-layer. * Matmul multi-thread con OpenMP (niente BLAS). + * + * ENV VARS: + * PILOT=0/1/2/3 : 0=no prefetch, 1=1-layer lookahead, 2=2-layer, 3=3-layer lookahead + * HOT=N : pin top-N hot experts per layer permanently (never evict) + * WARMUP=N : tokens before hot pinning activates (default 5) + * WIDE=N : prefetch top-K*N candidates (default 1, try 2 or 3) + * SMOOTH=F : EMA coefficient for routing momentum (default 0.3, range 0.0-0.95) + * CONF_LIMIT=F : cumulative gate probability threshold for prefetch cutoff (default 0.92) + * (expert queue is sorted by eid for SSD read locality) */ #define _GNU_SOURCE #include @@ -12,11 +21,22 @@ #include #include #include +#include #if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) #include +#include #endif #include "st.h" +#ifdef _WIN32 +#include +#define sleep_ms(ms) Sleep(ms) +#else +#define sleep_ms(ms) usleep((ms) * 1000) +#endif + + + /* ---------- config ---------- */ typedef struct { int hidden, n_layers, n_heads, n_kv_heads, head_dim; @@ -33,22 +53,61 @@ typedef struct { * Ogni weight [out,in] tenuto come int8 (per-riga) + scala float per riga. * Cosi' la RAM-cache scende da 4 byte/param (f32) a 1 byte/param: e' il * meccanismo che fa stare GLM-5.2 nei 15 GB. dequant-on-use nel matmul. */ -typedef struct { int eid; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot; +/* pinned=1 means this slot is strongly preferred to keep (hot expert); it will + * not be evicted during normal LRU eviction, but may be displaced under extreme + * cache pressure when all slots are pinned or in-flight. */ +typedef struct { int eid; int pinned; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot; typedef struct { Slot *slots; int n, cap; } LCache; typedef struct { Cfg c; shards S; - int quant_bits; /* bit di quantizzazione degli expert (2..8); storage int8, niente f32 (#134) */ + int quant_bits; float *embed, *lm_head, *final_norm; Layer *L; LCache *cache; /* [n_layers] */ uint64_t clock, hits, miss; - /* kv-cache per-layer: K,V come [H * maxT * head_dim] */ float **K, **V; int kv_len, max_t; double dense_load_s; + /* IMPROVEMENT 2: expert frequency heatmap */ + uint32_t *freq; + int freq_token_count, hot_pinned, hot_n, warmup_tokens; + int token_count; + /* PREDICTION IMPROVEMENT A: per-layer EMA of gate logits across tokens. + * momentum_logits[l*E .. (l+1)*E-1] = EMA of gate outputs for layer l. + * Used exclusively by the PILOT prefetcher to stabilise routing predictions + * across tokens; does NOT affect actual MoE routing (pr is unchanged). */ + float *momentum_logits; /* [n_layers * n_experts], EMA of gate logits */ + float pilot_smooth; /* SMOOTH env: EMA coefficient 0.0-0.9 (default 0.3) */ + uint8_t *is_pinned; /* [n_layers * n_experts], 1 if expert is globally pinned */ + uint8_t *is_queued; /* [n_layers * n_experts], 1 if expert is currently in the prefetch queue */ + float pilot_conf_limit; /* CONF_LIMIT env: cumulative gate probability threshold (e.g. 0.92) */ } Model; +static pthread_mutex_t g_pilot_mx = PTHREAD_MUTEX_INITIALIZER; +static struct { int l, e; } pilot_q[4096]; +static volatile unsigned pilot_r = 0, pilot_w = 0; +static Model *pilot_m = NULL; +static int g_pilot = 0; +static int g_wide = 1; /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */ + +static void pilot_prefetch(Model *m, int lnext, const float *x, int S); +static void *pilot_worker(void *arg); +static void ensure_pilot_worker_started(Model *m); +static void slot_ensure_allocated(Model *m, Slot *s); + +static void ensure_pilot_worker_started(Model *m) { + if (!pilot_m) { + pilot_m = m; + pthread_t t; + if (pthread_create(&t, NULL, pilot_worker, NULL) != 0) { + fprintf(stderr, "Error: Failed to create pilot prefetch worker thread\n"); + exit(1); + } + pthread_detach(t); + } +} + /* ---------- utility ---------- */ static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec*1e-9; } #if defined(__APPLE__) @@ -210,51 +269,224 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { #undef LD } m->cache = calloc(c->n_layers, sizeof(LCache)); - for (int i = 0; i < c->n_layers; i++) { m->cache[i].cap = cap; m->cache[i].slots = calloc(cap, sizeof(Slot)); } + for (int i = 0; i < c->n_layers; i++) { + m->cache[i].cap = cap; + m->cache[i].slots = calloc(cap, sizeof(Slot)); + } + /* IMPROVEMENT 2: frequency heatmap for hot expert pinning */ + m->freq = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint32_t)); + m->hot_pinned = 0; m->freq_token_count = 0; + m->hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0; + m->warmup_tokens = getenv("WARMUP") ? atoi(getenv("WARMUP")) : 5; + m->token_count = 0; + /* PREDICTION A: routing momentum — EMA of gate logits across tokens. + * Initialized to zero; first token sets EMA = fresh logits. */ + m->momentum_logits = calloc((size_t)c->n_layers * c->n_experts, sizeof(float)); + float sv = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f; + if (sv < 0.f) sv = 0.f; if (sv > 0.95f) sv = 0.95f; + m->pilot_smooth = sv; + m->is_pinned = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t)); + m->is_queued = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t)); + float cl = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f; + if (cl < 0.1f) cl = 0.1f; if (cl > 1.0f) cl = 1.0f; + m->pilot_conf_limit = cl; m->dense_load_s = now_s() - t0; + + // Persistent Hot Pinning: try to load hot_pinned.bin + char pinpath[512]; + snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap); + FILE *pinf = fopen(pinpath, "rb"); + if (pinf) { + size_t expected_size = (size_t)c->n_layers * c->n_experts; + if (fread(m->is_pinned, 1, expected_size, pinf) == expected_size) { + m->hot_pinned = 1; + printf("[HOT] Loaded persistent pinning from %s\n", pinpath); + + if (g_pilot) { + ensure_pilot_worker_started(m); + for (int l = 0; l < c->n_layers; l++) { + for (int e = 0; e < c->n_experts; e++) { + if (m->is_pinned[l * c->n_experts + e]) { + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w - r < 4096) { + pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = e; + pthread_mutex_lock(&g_pilot_mx); + m->is_queued[l * c->n_experts + e] = 1; + pthread_mutex_unlock(&g_pilot_mx); + __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE); + } + } + } + } + printf("[HOT] Pre-loading pinned experts into cache...\n"); + double t_wait = now_s(); + while (1) { + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE); + if (r == w) break; + sleep_ms(2); + } + printf("[HOT] Pre-loaded in %.1fs!\n", now_s() - t_wait); + } + } + fclose(pinf); + } } -/* legge un weight dal disco (streaming) e lo quantizza in q[O,I]+scale[O]. - * Container pre-quantizzato (convert_olmoe.py: int8 + scale f32 in "name.qs"): - * lettura raw diretta — meta' I/O e zero quantize_rows a runtime. Prima di - * questa patch il container int8 causava SIGBUS (st_read_f32 su tensori I8). */ -static void load_expert_w(Model *m, const char *name, int8_t *q, float *scale, int O, int I, float *tmp) { - st_tensor *t = st_find(&m->S, name); - if (t && t->dtype == 3) { /* I8/U8: container colibri */ - char qs[300]; snprintf(qs, sizeof(qs), "%s.qs", name); - st_read_raw(&m->S, name, q, 1); - st_read_f32(&m->S, qs, scale, 1); - return; +static void slot_ensure_allocated(Model *m, Slot *s) { + if (s->g) return; + Cfg *c = &m->c; + int64_t ng = (int64_t)c->inter * c->hidden; + int64_t nd = (int64_t)c->hidden * c->inter; + int8_t *w_block = malloc(ng + ng + nd); + if (!w_block) { + fprintf(stderr, "Error: Out of memory allocating slot weights block\n"); + exit(1); } - st_read_f32(&m->S, name, tmp, 1); /* pread + fadvise DONTNEED */ - quantize_rows(tmp, q, scale, O, I, m->quant_bits); + s->g = w_block; + s->u = w_block + ng; + s->d = w_block + ng + ng; + float *s_block = falloc(c->inter + c->inter + c->hidden); + s->gs = s_block; + s->us = s_block + c->inter; + s->ds = s_block + c->inter + c->inter; + s->pinned = 0; +} + +static void load_expert_merged(Model *m, int layer, int eid, Slot *s) { + char nm[256], qsnm[256]; + snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.merged_weight", layer, eid); + snprintf(qsnm, sizeof(qsnm), "model.layers.%d.mlp.experts.%d.qs", layer, eid); + st_read_raw(&m->S, nm, s->g, 1); + st_read_f32(&m->S, qsnm, s->gs, 0); /* scales are F32; use typed reader for dtype safety */ } /* ---------- cache expert: ritorna i pesi quantizzati (q+scale) da cache o disco ---------- */ static void expert_get(Model *m, int layer, int eid, Slot **out) { LCache *lc = &m->cache[layer]; + pthread_mutex_lock(&g_pilot_mx); for (int i = 0; i < lc->n; i++) if (lc->slots[i].eid == eid) { - m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; return; + m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; + pthread_mutex_unlock(&g_pilot_mx); + return; } m->miss++; Cfg *c = &m->c; - int64_t ng = (int64_t)c->inter * c->hidden, nd = (int64_t)c->hidden * c->inter; Slot *s; if (lc->n < lc->cap) { s = &lc->slots[lc->n++]; - s->g = malloc(ng); s->u = malloc(ng); s->d = malloc(nd); - s->gs = falloc(c->inter); s->us = falloc(c->inter); s->ds = falloc(c->hidden); - } else { int lru = 0; for (int i = 1; i < lc->n; i++) if (lc->slots[i].used < lc->slots[lru].used) lru = i; s = &lc->slots[lru]; } - float *tmp = falloc(ng > nd ? ng : nd); - char nm[256]; - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.gate_proj.weight",layer,eid); load_expert_w(m,nm,s->g,s->gs,c->inter,c->hidden,tmp); - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.up_proj.weight", layer,eid); load_expert_w(m,nm,s->u,s->us,c->inter,c->hidden,tmp); - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.down_proj.weight",layer,eid); load_expert_w(m,nm,s->d,s->ds,c->hidden,c->inter,tmp); - free(tmp); - s->eid = eid; s->used = ++m->clock; + slot_ensure_allocated(m, s); + } else { + /* LRU eviction — skip pinned and in-flight (eid==-1) slots */ + int lru = -1; + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue; + if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; + } + if (lru < 0) { + /* All slots are pinned or in-flight; find oldest non-in-flight slot + * (may be pinned, but never select one currently being loaded). */ + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].eid < 0) continue; /* never evict in-flight */ + if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; + } + } + if (lru < 0) lru = 0; /* absolute last resort: all in-flight, evict slot 0 */ + s = &lc->slots[lru]; + s->pinned = 0; + } + s->eid = -1; + s->used = ++m->clock; + pthread_mutex_unlock(&g_pilot_mx); + + load_expert_merged(m, layer, eid, s); + + pthread_mutex_lock(&g_pilot_mx); + s->eid = eid; + s->pinned = m->is_pinned[layer * c->n_experts + eid]; + s->used = ++m->clock; *out = s; + pthread_mutex_unlock(&g_pilot_mx); } +/* ---------- IMPROVEMENT 2: pin top-N hot experts per layer ---------- */ +static void pin_hot_experts(Model *m) { + Cfg *c = &m->c; + if (m->hot_n <= 0 || m->hot_pinned) return; + m->hot_pinned = 1; + + int is_dynamic = (m->hot_n >= 100); + double thresh = is_dynamic ? (double)m->hot_n / 1000.0 : 0.0; + + int pinned_total = 0; + for (int l = 0; l < c->n_layers; l++) { + uint32_t *freq_l = m->freq + (int64_t)l * c->n_experts; + + uint64_t layer_total = 0; + for (int e = 0; e < c->n_experts; e++) layer_total += freq_l[e]; + if (layer_total == 0) continue; + + int max_pin = m->cache[l].cap - 8; + if (max_pin < 4) max_pin = 4; + + int hn = is_dynamic ? max_pin : (m->hot_n < c->n_experts ? m->hot_n : c->n_experts); + if (hn > 256) hn = 256; + int hot_eids[256]; + int actual_hn = 0; + + for (int k = 0; k < hn; k++) { + int best = -1; uint32_t bv = 0; + for (int e = 0; e < c->n_experts; e++) { + int already = 0; + for (int j = 0; j < k; j++) if (hot_eids[j] == e) { already = 1; break; } + if (!already && freq_l[e] > bv) { bv = freq_l[e]; best = e; } + } + if (best < 0 || bv == 0) break; + if (is_dynamic && bv < thresh * layer_total) break; + hot_eids[k] = best; + actual_hn++; + } + + for (int k = 0; k < actual_hn; k++) { + int eid = hot_eids[k]; + m->is_pinned[l * c->n_experts + eid] = 1; + + LCache *lc = &m->cache[l]; + int found = 0; + pthread_mutex_lock(&g_pilot_mx); + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].eid == eid) { lc->slots[i].pinned = 1; found = 1; break; } + } + pthread_mutex_unlock(&g_pilot_mx); + if (!found && g_pilot > 0) { + /* Only enqueue when the prefetch worker is active (PILOT>0). */ + ensure_pilot_worker_started(m); + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + int gidx = l * c->n_experts + eid; + pthread_mutex_lock(&g_pilot_mx); + int already = m->is_queued[gidx]; + if (!already && w - r < 4096) { + pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = eid; + m->is_queued[gidx] = 1; + __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE); + } + pthread_mutex_unlock(&g_pilot_mx); + } + pinned_total++; + } + } + if (is_dynamic) { + printf("[HOT] Dynamic Pinned %d experts total (thresh=%.1f%%) after %d warmup tokens\n", + pinned_total, thresh * 100.0, m->freq_token_count); + } else { + printf("[HOT] Pinned %d experts (top-%d/layer) after %d warmup tokens\n", + pinned_total, m->hot_n, m->freq_token_count); + } +} + + /* ---------- RoPE su un vettore di una testa (head_dim) a posizione assoluta pos ---------- */ static void rope_head(float *x, int pos, const Cfg *c) { int h = c->head_dim / 2; @@ -325,6 +557,19 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { float *g = falloc(I), *u = falloc(I), *hh = falloc(D); for (int s = 0; s < S; s++) { float *pr = logits + (int64_t)s*E; + if (m->momentum_logits && m->pilot_smooth > 0.f) { + float *ema = m->momentum_logits + (int64_t)layer * E; + int is_zero = 1; + for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } + if (is_zero) { + for (int e = 0; e < E; e++) ema[e] = pr[e]; + } else { + for (int e = 0; e < E; e++) { + ema[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e]; + } + } + } + softmax_row(pr, E); /* top-K indici (selezione parziale) */ int idx[64]; float val[64]; @@ -337,6 +582,11 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { idx[kk] = best; val[kk] = bv; } if (c->norm_topk) { float sm=0; for(int kk=0;kkhot_pinned && m->freq) { + uint32_t *freq_l = m->freq + (int64_t)layer * E; + for (int kk = 0; kk < K; kk++) if (idx[kk] >= 0) freq_l[idx[kk]]++; + } const float *xs = x + (int64_t)s*D; for (int kk = 0; kk < K; kk++) { Slot *e; expert_get(m, layer, idx[kk], &e); @@ -352,9 +602,20 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { free(logits); free(g); free(u); free(hh); } -/* un passo: token nuovi ids[S] a posizione pos_base. Ritorna logits dell'ultimo token (malloc'd). */ static float *step(Model *m, const int *ids, int S, int pos_base) { Cfg *c = &m->c; int D = c->hidden; + if (g_pilot && m->token_count > 0) { + /* Flush stale prefetch requests: clear is_queued so pilot_realload + * will skip any entries still sitting in pilot_q for the previous + * token. We deliberately do NOT move pilot_w backwards; that would + * break the ring-buffer invariant (pilot_r could exceed pilot_w if + * the worker consumed an entry concurrently). The worker will drain + * the stale slots harmlessly because pilot_realload already exits + * early when the expert is already cached or is_queued is clear. */ + pthread_mutex_lock(&g_pilot_mx); + memset(m->is_queued, 0, (size_t)c->n_layers * c->n_experts); + pthread_mutex_unlock(&g_pilot_mx); + } float *x = falloc((int64_t)S*D); for (int s = 0; s < S; s++) memcpy(x + (int64_t)s*D, m->embed + (int64_t)ids[s]*D, D*sizeof(float)); float *nrm = falloc((int64_t)S*D), *tmp = falloc((int64_t)S*D); @@ -363,12 +624,26 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->in_ln, D, c->eps); attention(m, l, i, nrm, S, pos_base, tmp); for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j]; + /* IMPROVEMENT 1: PILOT=1 -> 1-layer lookahead */ + if (g_pilot >= 1 && S <= 8 && i + 1 < c->n_layers) + pilot_prefetch(m, i + 1, x, S); for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->post_ln, D, c->eps); moe(m, l, i, nrm, S, tmp); for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j]; + + /* PREDICTION IMPROVEMENT C (Residual gate trick): + * PILOT=2 -> prefetch layer i+2 using completed state x (containing MoE residual). */ + if (g_pilot >= 2 && S <= 8 && i + 2 < c->n_layers) + pilot_prefetch(m, i + 2, x, S); + if (g_pilot >= 3 && S <= 8 && i + 3 < c->n_layers) + pilot_prefetch(m, i + 3, x, S); + } + /* count actual tokens processed (S>1 during prefill) */ + m->token_count += S; m->freq_token_count += S; + if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens) + pin_hot_experts(m); m->kv_len = pos_base + S; - /* solo l'ultimo token -> logits */ float *last = falloc(D); rmsnorm_row(last, x + (int64_t)(S-1)*D, m->final_norm, D, c->eps); float *logit = falloc(c->vocab); @@ -377,6 +652,192 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { return logit; } +static void pilot_realload(Model *m, int layer, int eid) { + LCache *lc = &m->cache[layer]; + Cfg *c = &m->c; + + pthread_mutex_lock(&g_pilot_mx); + /* Early-exit if entry was flushed (is_queued cleared) while waiting. */ + if (!m->is_queued[layer * c->n_experts + eid]) { + pthread_mutex_unlock(&g_pilot_mx); + return; + } + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].eid == eid) { + m->is_queued[layer * c->n_experts + eid] = 0; + pthread_mutex_unlock(&g_pilot_mx); + return; + } + } + Slot *s; + if (lc->n < lc->cap) { + s = &lc->slots[lc->n++]; + slot_ensure_allocated(m, s); + } else { + /* LRU eviction — skip pinned and in-flight (eid==-1) slots */ + int lru = -1; + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue; + if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; + } + if (lru < 0) { + m->is_queued[layer * c->n_experts + eid] = 0; + pthread_mutex_unlock(&g_pilot_mx); + return; /* all pinned/in-flight, skip */ + } + s = &lc->slots[lru]; s->pinned = 0; + } + s->eid = -1; s->used = ++m->clock; + pthread_mutex_unlock(&g_pilot_mx); + + load_expert_merged(m, layer, eid, s); + + pthread_mutex_lock(&g_pilot_mx); + s->eid = eid; + s->pinned = m->is_pinned[layer * c->n_experts + eid]; + s->used = ++m->clock; + m->is_queued[layer * c->n_experts + eid] = 0; + pthread_mutex_unlock(&g_pilot_mx); +} + +static void *pilot_worker(void *arg) { + (void)arg; + while (1) { + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE); + if (r == w) { + sleep_ms(1); + continue; + } + int layer = pilot_q[r & 4095].l; + int eid = pilot_q[r & 4095].e; + pilot_realload(pilot_m, layer, eid); + __atomic_store_n(&pilot_r, r + 1, __ATOMIC_RELEASE); + } + return NULL; +} + +static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { + if (lnext < 0 || lnext >= m->c.n_layers) return; + Cfg *c = &m->c; int D = c->hidden, E = c->n_experts; + ensure_pilot_worker_started(m); + float *logits = falloc((int64_t)S * E); + Layer *l = &m->L[lnext]; + + // PREDICTION IMPROVEMENT B: Apply RMSNorm to x using destination layer's post_ln + // This scales inputs to the distribution expected by l->gate. + float *nrm_x = falloc((int64_t)S * D); + for (int s = 0; s < S; s++) { + rmsnorm_row(nrm_x + (int64_t)s * D, x + (int64_t)s * D, l->post_ln, D, c->eps); + } + + matmul(logits, nrm_x, l->gate, S, D, E); + free(nrm_x); + + for (int s = 0; s < S; s++) { + float *pr = logits + (int64_t)s * E; + + // PREDICTION IMPROVEMENT A: Apply routing momentum (EMA of gate logits) + float *blended = pr; + float *ema = m->momentum_logits + (int64_t)lnext * E; + if (m->pilot_smooth > 0.f) { + blended = falloc(E); + int is_zero = 1; + for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } + if (is_zero) { + for (int e = 0; e < E; e++) { + ema[e] = pr[e]; + blended[e] = pr[e]; + } + } else { + for (int e = 0; e < E; e++) { + blended[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e]; + ema[e] = blended[e]; // update EMA + } + } + } + + int cand = 0; + int idx[128]; + + float max_logit = -1e30f; + for (int e = 0; e < E; e++) { if (blended[e] > max_logit) max_logit = blended[e]; } + float *exps = falloc(E); + float sum_exps = 0.f; + for (int e = 0; e < E; e++) { + exps[e] = expf(blended[e] - max_logit); + sum_exps += exps[e]; + } + + float cum_sum = 0.f; + int min_cand = c->topk; + int max_cand = c->topk * g_wide; + if (max_cand < min_cand) max_cand = min_cand; + if (max_cand > 128) max_cand = 128; /* idx[] buffer bound */ + if (max_cand > E) max_cand = E; + + for (int kk = 0; kk < max_cand; kk++) { + int best = -1; float bv = -1.f; + for (int e = 0; e < E; e++) { + int taken = 0; for (int j = 0; j < kk; j++) if (idx[j] == e) { taken=1; break; } + if (!taken && exps[e] > bv) { bv = exps[e]; best = e; } + } + if (best < 0) break; + idx[kk] = best; + cum_sum += bv; + cand++; + if (cum_sum >= m->pilot_conf_limit * sum_exps && cand >= min_cand) { + break; + } + } + free(exps); + + if (blended != pr) free(blended); + + /* IMPROVEMENT 5: sort candidates by eid for sequential SSD read locality */ + for (int a = 0; a < cand-1; a++) + for (int b = a+1; b < cand; b++) + if (idx[b] >= 0 && (idx[a] < 0 || idx[a] > idx[b])) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; } + + for (int kk = 0; kk < cand; kk++) { + int eid = idx[kk]; + if (eid < 0) continue; + int found = 0; + pthread_mutex_lock(&g_pilot_mx); + LCache *lc = &m->cache[lnext]; + for (int z = 0; z < lc->n; z++) { + if (lc->slots[z].eid == eid) { found = 1; break; } + } + pthread_mutex_unlock(&g_pilot_mx); + if (!found) { + int gidx = lnext * E + eid; + pthread_mutex_lock(&g_pilot_mx); + int already_queued = m->is_queued[gidx]; + if (!already_queued) { + m->is_queued[gidx] = 1; + } + pthread_mutex_unlock(&g_pilot_mx); + + if (!already_queued) { + unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w2 - r2 < 4096) { + pilot_q[w2 & 4095].l = lnext; + pilot_q[w2 & 4095].e = eid; + __atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE); + } else { + pthread_mutex_lock(&g_pilot_mx); + m->is_queued[gidx] = 0; + pthread_mutex_unlock(&g_pilot_mx); + } + } + } + } + } + free(logits); +} + + /* generazione greedy. prompt[np] -> riempie out[np+n_new] */ static void generate(Model *m, const int *prompt, int np, int n_new, int *out) { Cfg *c = &m->c; @@ -400,6 +861,37 @@ static void generate(Model *m, const int *prompt, int np, int n_new, int *out) { } } +/* teacher-forced NLL of full_ids[np..nfull): feed the REFERENCE token at each step + * (never the argmax), accumulate -log softmax(logits)[next_ref]. A loss meter for + * throughput experiments: same engine path as decode, so hit rate/speed stay + * comparable, but quality is measured as perplexity instead of exact-match. + * Cross-checked vs HF transformers bf16 on identical token ids: engine (int8 + * experts) 12.11 ppl vs reference 12.25 (#108). Enabled by PPL=1. */ +static int tf_nll(Model *m, const int *full, int nfull, int np, double *nll_out) { + Cfg *c = &m->c; + m->max_t = nfull; + m->K = calloc(c->n_layers, sizeof(float*)); m->V = calloc(c->n_layers, sizeof(float*)); + for (int i = 0; i < c->n_layers; i++) { + m->K[i] = falloc((int64_t)c->n_heads * m->max_t * c->head_dim); + m->V[i] = falloc((int64_t)c->n_heads * m->max_t * c->head_dim); + } + double nll = 0; int scored = 0; + float *logit = step(m, full, np, 0); /* prefill on the prompt */ + for (int i = np; i < nfull; i++) { + /* log softmax(logit)[full[i]] without materializing the softmax */ + float mx = logit[0]; for (int v = 1; v < c->vocab; v++) if (logit[v] > mx) mx = logit[v]; + double Z = 0; for (int v = 0; v < c->vocab; v++) Z += exp((double)logit[v] - mx); + nll += -((double)logit[full[i]] - mx - log(Z)); + scored++; + free(logit); logit = NULL; + if (i == nfull - 1) break; + logit = step(m, &full[i], 1, i); /* teacher forcing */ + } + if (logit) free(logit); + *nll_out = nll / scored; + return scored; +} + /* ---------- lettura ref.json ---------- */ static int *read_int_array(jval *o, const char *key, int *n_out) { jval *a = json_get(o, key); @@ -411,25 +903,48 @@ static int *read_int_array(jval *o, const char *key, int *n_out) { int main(int argc, char **argv) { const char *snap = getenv("SNAP"); if (!snap) { fprintf(stderr, "set SNAP=\n"); return 1; } - int cap = argc > 1 ? atoi(argv[1]) : 16; - int bits = argc > 2 ? atoi(argv[2]) : 8; - if (bits < 2 || bits > 8) { /* expert storage is int8_t: bits>8 truncates in quantize_rows (#134). f32 mode is not implemented here — int8 is already token-exact vs the oracle. */ - fprintf(stderr, "quant_bits must be 2..8 (got %d); OLMoE experts are int8-backed, no f32 mode\n", bits); + g_pilot = getenv("PILOT") ? atoi(getenv("PILOT")) : 0; + g_wide = getenv("WIDE") ? atoi(getenv("WIDE")) : 1; + if (g_wide < 1) g_wide = 1; + if (g_wide > 4) g_wide = 4; + int hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0; + int cap = argc > 1 ? atoi(argv[1]) : 16; + int bits = argc > 2 ? atoi(argv[2]) : 8; + if (bits < 2 || bits > 8) { + fprintf(stderr, "quant_bits must be 2..8 (got %d)\n", bits); return 1; } const char *refpath = argc > 3 ? argv[3] : "ref.json"; - FILE *f = fopen(refpath, "rb"); if(!f){perror(refpath);return 1;} + float smooth = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f; + float conf = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f; + + printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d smooth=%.2f conf=%.2f ==\n", + cap, bits, g_pilot, g_wide, hot_n, smooth, conf); + + FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; } fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *buf=malloc(n+1); if(fread(buf,1,n,f)!=(size_t)n){} buf[n]=0; fclose(f); + char *buf=malloc(n+1); if (fread(buf,1,n,f)!=(size_t)n) {} buf[n]=0; fclose(f); char *arena=NULL; jval *ref = json_parse(buf, &arena); int np, nfull; int *prompt = read_int_array(ref,"prompt_ids",&np); int *full = read_int_array(ref,"full_ids",&nfull); int n_new = nfull - np; - printf("== Streaming C engine, cache = %d experts/layer, experts @ %d-bit ==\n", cap, bits); Model m; model_init(&m, snap, cap, bits); printf("resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb()); + if (getenv("PPL") && atoi(getenv("PPL")) == 1) { /* loss-meter mode: teacher-forced NLL */ + double nll; double t = now_s(); + int scored = tf_nll(&m, full, nfull, np, &nll); + double dt = now_s() - t; + double tot = m.hits + m.miss; + printf("TF-NLL: %.4f nats/token over %d tokens | ppl = %.2f\n", nll, scored, exp(nll)); + printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0, + (unsigned long long)m.hits, (unsigned long long)m.miss); + printf("Speed: %.2f tok/s (%.1fs for %d tokens) | PEAK RSS: %.2f GB\n", scored/dt, dt, scored, rss_gb()); + free(buf); free(arena); + return 0; + } + int *out = malloc((np + n_new) * sizeof(int)); double t = now_s(); generate(&m, prompt, np, n_new, out); @@ -443,6 +958,26 @@ int main(int argc, char **argv) { printf("\nPEAK RSS: %.2f GB\n", rss_gb()); printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hits, (unsigned long long)m.miss); + + + // Persistent Hot Pinning: save dynamic pinning if newly created + if (m.hot_pinned) { + char pinpath[512]; + snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap); + FILE *pinf_chk = fopen(pinpath, "rb"); + if (!pinf_chk) { + FILE *pinf_save = fopen(pinpath, "wb"); + if (pinf_save) { + size_t expected_size = (size_t)m.c.n_layers * m.c.n_experts; + fwrite(m.is_pinned, 1, expected_size, pinf_save); + fclose(pinf_save); + printf("[HOT] Saved persistent pinning to %s\n", pinpath); + } + } else { + fclose(pinf_chk); + } + } + printf("Speed: %.2f tok/s (%.1fs for %d tokens)\n", n_new/dt, dt, n_new); free(buf); free(arena); return 0; diff --git a/c/openai_server.py b/c/openai_server.py index d8703a8..be21d91 100644 --- a/c/openai_server.py +++ b/c/openai_server.py @@ -27,6 +27,7 @@ HERE = Path(__file__).resolve().parent END = b"\x01\x01END\x01\x01\n" READY = b"\x01\x01READY\x01\x01\n" MAX_BODY = 4 << 20 +PROFILE_TURNS = 120 # rolling window of per-turn PROF snapshots kept for /profile DEFAULT_CORS_ORIGINS = ( "http://127.0.0.1:8000", "http://localhost:8000", @@ -270,6 +271,14 @@ def parse_tool_calls(reply, tools=None): salvaged.append(name) calls.append({"id": "call_" + uuid.uuid4().hex[:24], "type": "function", "function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)}}) + if tools and not calls and re.search(r"||", reply): + # Diagnosi per la #401: il client ha dichiarato i tools e il modello ha PROVATO la + # sintassi, ma il parse rigoroso non ha agganciato nulla (tipico output int4 storpiato). + # EN: #401 field diagnosis: tools were declared and the model attempted the syntax, + # EN: but the strict parse matched nothing (typically quantization-mangled output). + sys.stderr.write("[api] tools declared and tool-call markers present, but no call " + "parsed -- output may be quantization-mangled; try COLI_TOOL_SALVAGE=1\n") + sys.stderr.flush() text = _BOX_RE.sub("", reply) if THINK_CLOSE in text: text = text.split(THINK_CLOSE, 1)[1] @@ -426,7 +435,10 @@ def generation_options(body, limit): maximum = body.get("max_tokens") maximum_param = "max_tokens" if maximum is None: - maximum = min(256, limit) + # Client omitted max_tokens: honor the operator's configured budget (--max-tokens / + # --ngen), not an arbitrary 256 — `coli serve --ngen 32768` must mean 32768 (#382). + # Generation still ends at EOS, so this is a cap, not a target. + maximum = limit temperature = body.get("temperature") top_p = body.get("top_p") temperature = 0.7 if temperature is None else temperature @@ -494,6 +506,8 @@ class Engine: self.emap = None self.hits = None self.hits_seq = 0 # latest "TIERS" snapshot from the engine + self.profile = collections.deque(maxlen=PROFILE_TURNS) # per-turn phase timings + self.profile_seq = 0 read_engine_turn(self.process.stdout, READY, lambda _: None) self.dispatcher = threading.Thread(target=self._dispatch_stdout, name="colibri-stdout", daemon=True) @@ -571,6 +585,20 @@ class Engine: elif kind == "HITS" and len(fields) == 4: self.hits = fields[3] self.hits_seq += 1 + elif kind == "PROF" and len(fields) >= 10: + # per-turn phase timings: where the engine spent this turn's wall time + self.profile.append({ + "wall_s": float(fields[1]), + "prompt_tokens": int(fields[2]), + "completion_tokens": int(fields[3]), + "expert_disk_s": float(fields[4]), + "expert_wait_s": float(fields[5]), + "expert_matmul_s": float(fields[6]), + "attention_s": float(fields[7]), + "lm_head_s": float(fields[8]), + "forwards": int(fields[9]), + }) + self.profile_seq += 1 elif kind == "TIERS" and len(fields) >= 6: self.tiers = {"vram": int(fields[1]), "ram": int(fields[2]), "disk": int(fields[3]), "vram_gb": float(fields[4]), @@ -803,6 +831,12 @@ class APIHandler(BaseHTTPRequestHandler): payload["seq"] = eng.hits_seq self.send_json(200, payload, request_id) return + if path == "/profile": + eng = self.server.engine + payload = {"seq": getattr(eng, "profile_seq", 0) if eng else 0, + "turns": list(getattr(eng, "profile", ()) or ()) if eng else []} + self.send_json(200, payload, request_id) + return if self.serve_static(path): return self.require_auth() diff --git a/c/quant.h b/c/quant.h new file mode 100644 index 0000000..928de96 --- /dev/null +++ b/c/quant.h @@ -0,0 +1,672 @@ +/* quant.h — quantized matmul kernels (header-only, all functions static). + * Multi-architecture SIMD: AVX2 / AVX-512 / AVX-VNNI / ARM NEON / NEON-SDOT / + * NEON-i8mm / POWER VSX. Pure compute — no Model or QT dependency. */ +#ifndef COLI_QUANT_H +#define COLI_QUANT_H + +#include +#include +#include +#include +#include + +#ifdef _OPENMP +#include +#endif + +/* ---- SIMD includes -------------------------------------------------------- */ +#ifdef __AVX2__ +#include +static inline float hsum256(__m256 v){ + __m128 lo=_mm256_castps256_ps128(v), hi=_mm256_extractf128_ps(v,1); + lo=_mm_add_ps(lo,hi); __m128 sh=_mm_movehl_ps(lo,lo); lo=_mm_add_ps(lo,sh); + sh=_mm_shuffle_ps(lo,lo,1); lo=_mm_add_ss(lo,sh); return _mm_cvtss_f32(lo); +} +static inline int hsum256_i32(__m256i v){ + __m128i lo=_mm256_castsi256_si128(v), hi=_mm256_extracti128_si256(v,1); + lo=_mm_add_epi32(lo,hi); lo=_mm_hadd_epi32(lo,lo); lo=_mm_hadd_epi32(lo,lo); + return _mm_cvtsi128_si32(lo); +} +#endif +#if defined(__AVXVNNI__) && defined(__AVX2__) +static inline int hsum128_i32(__m128i v){ + v=_mm_hadd_epi32(v,v); v=_mm_hadd_epi32(v,v); return _mm_cvtsi128_si32(v); +} +#endif +#ifdef __ARM_NEON +#include +#endif +#ifdef __VSX__ +#include +#undef vector +#undef pixel +#undef bool +#endif + +/* ---- AVX-512 int4->float accumulator -------------------------------------- */ +#if defined(__AVX512F__) && defined(__AVX512BW__) +static int g_i4_acc512=1; +static inline float dot_i4f_avx512(const uint8_t *w,const float *x,int I){ + const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8); + __m512 acc0=_mm512_setzero_ps(),acc1=_mm512_setzero_ps(); int i=0; + for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi); + __m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8)); + __m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8)); + acc0=_mm512_fmadd_ps(_mm512_loadu_ps(x+i),w0,acc0); + acc1=_mm512_fmadd_ps(_mm512_loadu_ps(x+i+16),w1,acc1); + } + return _mm512_reduce_add_ps(_mm512_add_ps(acc0,acc1)); +} +static int i4_acc512_selftest(void){ + enum { N=224 }; uint8_t w[(N+1)/2]; float x[N]; + for(int i=0;i>1]=(uint8_t)(q+8); + else w[i>>1]|=(uint8_t)((q+8)<<4); + x[i]=(float)(((i*29+7)%101)-50)/37.f; + } + for(int n=32;n<=N;n+=32){ + float ref=0; for(int i=0;i>1]>>((i&1)*4))&15)-8); + float got=dot_i4f_avx512(w,x,n),tol=2e-5f*(1.f+fabsf(ref)); + if(fabsf(got-ref)>tol){ fprintf(stderr,"AVX512 i4 selftest n=%d: %.9g != %.9g\n",n,got,ref); return 0; } + } + return 1; +} +#endif + +/* ---- y[S,O] = x[S,I] @ W^T, W[O,I] f32 ---------------------------------- */ +static void matmul(float *y, const float *x, const float *W, int S, int I, int O){ + #pragma omp parallel for schedule(static) + for (int o=0;o>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i nib=_mm_unpacklo_epi8(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } + a=hsum256(acc); +#elif defined(__ARM_NEON) + const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); + float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); + for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); + uint8x8x2_t z=vzip_u8(vand_u8(by,m4), vshr_n_u8(by,4)); + int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[0]),b8)); + int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[1]),b8)); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); +#endif +#if defined(__AVX512F__) && defined(__AVX512BW__) + } +#endif + for(;i+1>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8; + a += xs[i]*(float)lo + xs[i+1]*(float)hi; } + if(i>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; } + y[(int64_t)s*O+o]=a*sc; } } +} + +/* ---- y[S,O] = x[S,I] @ W^T, W int4 packed + per-GROUP scales (fmt=4) ----- */ +static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const float *scale, + int S, int I, int O, int gs){ + int rb=(I+1)/2; int ng=(I+gs-1)/gs; + #pragma omp parallel for schedule(static) + for(int o=0;oI) glen=I-base; + float sc=scl[g]; + int i=base; +#ifdef __AVX2__ + const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8); + __m256 acc=_mm256_setzero_ps(); + for(; i+16<=base+glen; i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i nib=_mm_unpacklo_epi8(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } + a+=hsum256(acc)*sc; +#endif + for(; i>1]; + a+=(xs[i]*(float)((int)(byte&0xF)-8)+xs[i+1]*(float)((int)(byte>>4)-8))*sc; } + else { uint8_t byte=w[i>>1]; a+=xs[i]*(float)((int)(byte&0xF)-8)*sc; } + } + } + y[(int64_t)s*O+o]=a; + } + } +} + +/* ---- fused gate+up: one OMP dispatch for both matrices -------------------- */ +static void matmul_i4_pair(float *yg, float *yu, const float *x, + const uint8_t *qg, const float *sg, + const uint8_t *qu, const float *su, int I, int O){ + int rb=(I+1)/2; + #pragma omp parallel for schedule(static) + for(int z=0;z<2*O;z++){ + int o=z>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i nib=_mm_unpacklo_epi8(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); } + a=hsum256(acc); +#elif defined(__ARM_NEON) + const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); + float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0); + for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); + uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4)); + int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8)); + int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8)); + ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); +#endif +#if defined(__AVX512F__) && defined(__AVX512BW__) + } +#endif + for(;i+1>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); } + if(i>1]&15)-8); + (z>2))); + __m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2); + __m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2); + __m128i lo=_mm_unpacklo_epi8(p0,p1), hi=_mm_unpacklo_epi8(p2,p3); + __m128i nib=_mm_unpacklo_epi16(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b2)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b2)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } + a=hsum256(acc); +#elif defined(__ARM_NEON) + const uint8x8_t m2v=vdup_n_u8(3); const int8x8_t b2v=vdup_n_s8(2); + float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); + for(;i+16<=I;i+=16){ uint32_t wd; memcpy(&wd, w+(i>>2), 4); + uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd)); + uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v)); + uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6)); + uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0])); + int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[0]),b2v)); + int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[1]),b2v)); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); +#endif + for(;i>2]; int sh=(i&3)*2; a += xs[i]*(float)((int)((byte>>sh)&3)-2); } + y[(int64_t)s*O+o]=a*sc; } } +} + +/* ---- IDOT: integer dot kernels (int8-quantized activations) --------------- */ +#if defined(__AVX512VNNI__) && defined(__AVX512BW__) +#define IDOT_KERNEL "avx512-vnni" +#elif defined(__AVXVNNI__) && defined(__AVX2__) +#define IDOT_KERNEL "avx-vnni" +#elif defined(__AVX2__) +#define IDOT_KERNEL "avx2" +#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) +#define IDOT_KERNEL "neon-i8mm" +#elif defined(__ARM_NEON) +#define IDOT_KERNEL "neon" +#elif defined(__VSX__) +#define IDOT_KERNEL "vsx" +#else +#define IDOT_KERNEL "scalar" +#endif +static int g_idot=1; +#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) +static int g_i4s=1; +#elif defined(__VSX__) +static int g_i4s=1; +#else +static int g_i4s=2; +#endif + +static inline float qrow_i8(const float *x, int8_t *q, int I){ + float amax=0; for(int i=0;iamax)amax=a; } + float s=amax/127.f; if(s<1e-12f) s=1e-12f; float inv=1.f/s; + for(int i=0;i>1))); + __m256i lo=_mm256_and_si256(by,m4v), hi=_mm256_and_si256(_mm256_srli_epi16(by,4),m4v); + __m256i z0=_mm256_unpacklo_epi8(lo,hi), z1=_mm256_unpackhi_epi8(lo,hi); + __m512i wv=_mm512_sub_epi8(_mm512_inserti64x4(_mm512_castsi256_si512(z0),z1,1),b8v); + __m512i xv=_mm512_permutexvar_epi64(xidx,_mm512_loadu_si512((const void*)(x+i))); + __mmask64 neg=_mm512_movepi8_mask(wv); + __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv); + acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs); + } + sum=_mm512_reduce_add_epi32(acc); +#elif defined(__AVXVNNI__) && defined(__AVX2__) + const __m128i m4=_mm_set1_epi8(0x0F); const __m128i b8=_mm_set1_epi8(8); + __m128i acc=_mm_setzero_si128(); + for(;i+32<=I;i+=32){ + __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); + __m128i w0=_mm_sub_epi8(n0,b8), w1=_mm_sub_epi8(n1,b8); + __m128i x0=_mm_loadu_si128((const __m128i*)(x+i)); + __m128i x1=_mm_loadu_si128((const __m128i*)(x+i+16)); + acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0)); + acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1)); + } + sum=hsum128_i32(acc); +#elif defined(__AVX2__) + const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi8(8); + const __m256i ones=_mm256_set1_epi16(1); + __m256i acc=_mm256_setzero_si256(); + for(;i+32<=I;i+=32){ + __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); + __m256i wv=_mm256_sub_epi8(_mm256_set_m128i(n1,n0),b8); + __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); + __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); + acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); + } + sum=hsum256_i32(acc); +#elif defined(__ARM_NEON) + const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8); +#if defined(__ARM_FEATURE_DOTPROD) + int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); + for(;i+64<=I;i+=64){ + uint8x16_t byA=vld1q_u8(w4+(i>>1)), byB=vld1q_u8(w4+(i>>1)+16); + uint8x16x2_t zA=vzipq_u8(vandq_u8(byA,m4q), vshrq_n_u8(byA,4)); + uint8x16x2_t zB=vzipq_u8(vandq_u8(byB,m4q), vshrq_n_u8(byB,4)); + a0=vdotq_s32(a0,vsubq_s8(vreinterpretq_s8_u8(zA.val[0]),b8q),vld1q_s8(x+i)); + a1=vdotq_s32(a1,vsubq_s8(vreinterpretq_s8_u8(zA.val[1]),b8q),vld1q_s8(x+i+16)); + a2=vdotq_s32(a2,vsubq_s8(vreinterpretq_s8_u8(zB.val[0]),b8q),vld1q_s8(x+i+32)); + a3=vdotq_s32(a3,vsubq_s8(vreinterpretq_s8_u8(zB.val[1]),b8q),vld1q_s8(x+i+48)); + } + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + for(;i+32<=I;i+=32){ + uint8x16_t by=vld1q_u8(w4+(i>>1)); + uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); + acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q),vld1q_s8(x+i)); + acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q),vld1q_s8(x+i+16)); + } + sum=vaddvq_s32(acc); +#else + int32x4_t acc=vdupq_n_s32(0); + for(;i+32<=I;i+=32){ + uint8x16_t by=vld1q_u8(w4+(i>>1)); + uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); + int8x16_t w0=vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q); + int8x16_t w1=vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q); + int8x16_t x0=vld1q_s8(x+i), x1=vld1q_s8(x+i+16); + int16x8_t p=vmull_s8(vget_low_s8(w0),vget_low_s8(x0)); + p=vmlal_s8(p,vget_high_s8(w0),vget_high_s8(x0)); + acc=vpadalq_s16(acc,p); + p=vmull_s8(vget_low_s8(w1),vget_low_s8(x1)); + p=vmlal_s8(p,vget_high_s8(w1),vget_high_s8(x1)); + acc=vpadalq_s16(acc,p); + } + sum=vaddvq_s32(acc); +#endif +#elif defined(__VSX__) + const __vector unsigned char m4v=vec_splats((unsigned char)0x0F); + const __vector unsigned char sh4=vec_splats((unsigned char)4); + const __vector signed char b8v=vec_splats((signed char)8); + const __vector signed char vz=vec_splats((signed char)0); + __vector signed int acc=vec_splats(0); + for(;i+32<=I;i+=32){ + __vector unsigned char by=vec_xl(0,w4+(i>>1)); + __vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4); + __vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v); + __vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v); + __vector signed char x0=vec_xl(0,(const signed char*)(x+i)); + __vector signed char x1=vec_xl(0,(const signed char*)(x+i+16)); + __vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz); + acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0), + (__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc); + acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1), + (__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc); + } + sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); +#endif + for(;i+1>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } + if(i>1]; sum+=((int)(b&0xF)-8)*x[i]; } + return sum; +} + +/* ---- ARM i8mm SMMLA tiled kernels ---------------------------------------- */ +#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) +static inline int32x4_t mm_tile16(int32x4_t acc, int8x16_t wo, int8x16_t wo1, + int8x16_t xs, int8x16_t xs1){ + acc=vmmlaq_s32(acc, vcombine_s8(vget_low_s8(wo), vget_low_s8(wo1)), + vcombine_s8(vget_low_s8(xs), vget_low_s8(xs1))); + return vmmlaq_s32(acc, vcombine_s8(vget_high_s8(wo), vget_high_s8(wo1)), + vcombine_s8(vget_high_s8(xs), vget_high_s8(xs1))); +} +static void matmul_q_idot_mm(float *y, const int8_t *xq, const float *sx, const int8_t *q, + const float *scale, int S, int I, int O){ + #pragma omp parallel for schedule(static) + for(int o=0;o<(O&~1);o+=2){ + const int8_t *wo=q+(int64_t)o*I, *wo1=q+(int64_t)(o+1)*I; + float sc0=scale[o], sc1=scale[o+1]; + for(int s=0;s<(S&~1);s+=2){ + const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I; + int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0; + for(;i+64<=I;i+=64){ + a0=mm_tile16(a0,vld1q_s8(wo+i), vld1q_s8(wo1+i), vld1q_s8(xs+i), vld1q_s8(xs1+i)); + a1=mm_tile16(a1,vld1q_s8(wo+i+16),vld1q_s8(wo1+i+16),vld1q_s8(xs+i+16),vld1q_s8(xs1+i+16)); + a2=mm_tile16(a2,vld1q_s8(wo+i+32),vld1q_s8(wo1+i+32),vld1q_s8(xs+i+32),vld1q_s8(xs1+i+32)); + a3=mm_tile16(a3,vld1q_s8(wo+i+48),vld1q_s8(wo1+i+48),vld1q_s8(xs+i+48),vld1q_s8(xs1+i+48)); + } + for(;i+16<=I;i+=16) + a0=mm_tile16(a0,vld1q_s8(wo+i),vld1q_s8(wo1+i),vld1q_s8(xs+i),vld1q_s8(xs1+i)); + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); + int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); + for(;i>1)), byo1=vld1q_u8(wo1+(i>>1)); + uint8x16_t cyo=vld1q_u8(wo+(i>>1)+16), cyo1=vld1q_u8(wo1+(i>>1)+16); + uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); + uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); + uint8x16x2_t ko =vzipq_u8(vandq_u8(cyo, m4q), vshrq_n_u8(cyo, 4)); + uint8x16x2_t ko1=vzipq_u8(vandq_u8(cyo1,m4q), vshrq_n_u8(cyo1,4)); + a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), + vld1q_s8(xs+i), vld1q_s8(xs1+i)); + a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), + vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); + a2=mm_tile16(a2, vsubq_s8(vreinterpretq_s8_u8(ko.val[0]),b8q), + vsubq_s8(vreinterpretq_s8_u8(ko1.val[0]),b8q), + vld1q_s8(xs+i+32), vld1q_s8(xs1+i+32)); + a3=mm_tile16(a3, vsubq_s8(vreinterpretq_s8_u8(ko.val[1]),b8q), + vsubq_s8(vreinterpretq_s8_u8(ko1.val[1]),b8q), + vld1q_s8(xs+i+48), vld1q_s8(xs1+i+48)); + } + for(;i+32<=I;i+=32){ + uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1)); + uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); + uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); + a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), + vld1q_s8(xs+i), vld1q_s8(xs1+i)); + a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), + vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); + } + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); + int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); + for(;i+1>1], bo1=wo1[i>>1]; + int a0=(int)(bo&0xF)-8, a1=(int)(bo>>4)-8, b0=(int)(bo1&0xF)-8, b1=(int)(bo1>>4)-8; + int u0=xs[i],u1=xs[i+1],v0=xs1[i],v1=xs1[i+1]; + d00+=a0*u0+a1*u1; d01+=a0*v0+a1*v1; d10+=b0*u0+b1*u1; d11+=b0*v0+b1*v1; } + if(i>1], bo1=wo1[i>>1]; + int a0=(int)(bo&0xF)-8, b0=(int)(bo1&0xF)-8; + d00+=a0*xs[i]; d01+=a0*xs1[i]; d10+=b0*xs[i]; d11+=b0*xs1[i]; } + y[(int64_t)s*O+o] =(float)d00*sc0*sx[s]; + y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s]; + y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1]; + y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1]; + } + if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I; + y[(int64_t)s*O+o] =(float)dot_i4i8(wo, xs,I)*sc0*sx[s]; + y[(int64_t)s*O+(o+1)]=(float)dot_i4i8(wo1,xs,I)*sc1*sx[s]; } + } + if(O&1){ int o=O-1; const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o]; + #pragma omp parallel for schedule(static) + for(int s=0;s=2){ matmul_q_idot_mm(y,xq,sx,q,scale,S,I,O); return; } +#endif + #pragma omp parallel for schedule(static) + for(int o=0;o=2){ matmul_i4_idot_mm(y,xq,sx,q4,scale,S,I,O); return; } +#endif + #pragma omp parallel for schedule(static) + for(int o=0;og_qscratch.xq_cap){ + int8_t *p=realloc(g_qscratch.xq,xn); + if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); } + g_qscratch.xq=p; g_qscratch.xq_cap=xn; + } + if(sn>g_qscratch.sx_cap){ + float *p=realloc(g_qscratch.sx,sn*sizeof(float)); + if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); } + g_qscratch.sx=p; g_qscratch.sx_cap=sn; + } + *xq=g_qscratch.xq; *sx=g_qscratch.sx; +} + +/* ---- f32 -> quantized packing --------------------------------------------- */ +static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits){ + int qmax=(1<<(bits-1))-1; + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; + int8_t *qr=q+(int64_t)o*I; + for(int i=0;iqmax)v=qmax; if(v<-qmax-1)v=-qmax-1; qr[i]=(int8_t)v; } + } +} +static void pack_int4(const float *w, uint8_t *q4, float *scale, int O, int I, int bits){ + int qmax=(1<<(bits-1))-1, rb=(I+1)/2; + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; + uint8_t *qr=q4+(int64_t)o*rb; + for(int i=0;iqmax)v0=qmax; if(v0<-8)v0=-8; + int v1=0; if(i+1qmax)v1=qmax; if(v1<-8)v1=-8; } + qr[i>>1] = (uint8_t)((v0+8) | ((v1+8)<<4)); + } + } +} +static void pack_int2(const float *w, uint8_t *q2, float *scale, int O, int I, int bits){ + int qmax=(1<<(bits-1))-1, rb=(I+3)/4; + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; + uint8_t *qr=q2+(int64_t)o*rb; + for(int i=0;iqmax)v=qmax; if(v<-2)v=-2; byte|=(uint8_t)((v+2)<<(k*2)); } + qr[i>>2]=byte; + } + } +} + +#endif /* COLI_QUANT_H */ diff --git a/c/ref_olmoe_real.json b/c/ref_olmoe_real.json new file mode 100644 index 0000000..b271e07 --- /dev/null +++ b/c/ref_olmoe_real.json @@ -0,0 +1,28 @@ +{ + "prompt_ids": [ + 510, + 5347, + 273, + 6181, + 310 + ], + "full_ids": [ + 510, + 5347, + 273, + 6181, + 310, + 7785, + 15, + 187, + 187, + 510, + 3565, + 3448, + 273, + 6181, + 310, + 5112, + 15 + ] +} \ No newline at end of file diff --git a/c/resource_plan.py b/c/resource_plan.py index 6b3f2fe..15ce02a 100644 --- a/c/resource_plan.py +++ b/c/resource_plan.py @@ -203,6 +203,70 @@ def physical_cpu_count(): return os.cpu_count() or 1 +def cpu_socket_count(): + """Return the number of physical CPU sockets visible to this process.""" + if not sys.platform.startswith("linux"): + return 1 + try: + result = subprocess.run(["lscpu", "-p=socket"], text=True, + capture_output=True, check=True, timeout=5) + sockets = {int(line) for line in result.stdout.splitlines() + if line and not line.startswith("#")} + if sockets: + return len(sockets) + except (OSError, ValueError, subprocess.SubprocessError): + pass + return 1 + + +def _auto_tune(bottleneck_class, projected_hit, gpus, cpu_sockets, plan_has_metal): + """Derive tuning knobs from the bottleneck classification.""" + tune = {} + has_gpu = bool(gpus) + n_gpu = len(gpus) + + # MTP: costs more than it saves when compute-bound (#389 measured 42% loss) + if bottleneck_class == "compute": + tune["DRAFT"] = {"value": "0", + "reason": "compute-bound: MTP batch overhead exceeds yield"} + elif bottleneck_class == "disk" and projected_hit < 0.90: + tune["DRAFT"] = {"value": "0", + "reason": "low hit rate: MTP widens expert union, adds disk reads"} + # otherwise leave DRAFT unset (engine default: auto) + + # PIPE: resident pipeline mode depends on GPU count + if has_gpu and n_gpu == 1: + tune["COLI_CUDA_PIPE"] = {"value": "1", + "reason": "single GPU: S=1 pipeline gate"} + elif has_gpu and n_gpu > 1: + tune["COLI_CUDA_PIPE"] = {"value": "2", + "reason": "multi-GPU: residual stays on-device across layers"} + elif not has_gpu and bottleneck_class == "disk": + tune["PIPE"] = {"value": "1", + "reason": "overlap disk reads with resident expert compute"} + + # NUMA: selective interleave for GPU hosts, blanket hint for CPU-only + if cpu_sockets > 1 and has_gpu: + tune["COLI_NUMA"] = {"value": "1", + "reason": "multi-socket + GPU: interleave expert slabs, protect DMA buffers"} + elif cpu_sockets > 1 and not has_gpu: + tune["COLI_NUMA"] = {"value": "1", + "reason": "multi-socket CPU-only: interleave expert slabs across nodes"} + tune["_numa_hint"] = "numactl --interleave=all may perform better on CPU-only hosts" + + # OMP: kill hot-thread spin when GPU/Metal owns the power budget + if plan_has_metal: + tune["COLI_NO_OMP_TUNE"] = {"value": "1", + "reason": "Metal: OMP spin-wait steals GPU power budget"} + + # PIN: fully resident if RAM allows and no GPU tier competes + if projected_hit >= 0.99 and not has_gpu: + tune["PIN_GB"] = {"value": "all", + "reason": "enough RAM for full expert residency"} + + return tune + + POLICIES = { "quality": {"preserve_quantization": True, "preserve_router": True}, "balanced": {"preserve_quantization": True, "preserve_router": True}, @@ -212,11 +276,12 @@ POLICIES = { def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, available_memory=None, available_disk=None, gpus=None, - policy="quality", physical_cpus=None): + policy="quality", physical_cpus=None, cpu_sockets=None): if policy not in POLICIES: raise ValueError(f"unknown policy: {policy}") info = analyze_model(model) physical_cpus = physical_cpu_count() if physical_cpus is None else physical_cpus + cpu_sockets = cpu_socket_count() if cpu_sockets is None else cpu_sockets cfg = info["config"] available_memory = memory_available() if available_memory is None else available_memory if available_disk is None: @@ -273,12 +338,28 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, if cold_bytes: warnings.append("cold expert misses may reach disk; normal decode speed depends on hit rate") + total_expert = info["expert_bytes"] + resident_expert = hot_bytes + warm_bytes + projected_hit = resident_expert / total_expert if total_expert else 1.0 + if cold_bytes: bottleneck = "disk expert misses" - elif warm_bytes: - bottleneck = "CPU expert compute and RAM bandwidth" + bottleneck_class = "disk" + elif warm_bytes and gpus: + bottleneck = "CPU expert tail and GPU compute" + bottleneck_class = "mixed" + elif projected_hit >= 0.99: + if gpus: + bottleneck = "GPU compute and interconnect" + else: + bottleneck = "CPU expert compute (fully resident)" + bottleneck_class = "compute" else: - bottleneck = "GPU compute and interconnect" + bottleneck = "CPU expert compute and RAM bandwidth" + bottleneck_class = "memory" + + tune = _auto_tune(bottleneck_class, projected_hit, gpus, cpu_sockets, + plan_has_metal=False) return { "version": 2, @@ -286,6 +367,7 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, "quality_preserving": policy != "experimental-fast"}, "model": {key: value for key, value in info.items() if key != "config"}, "cpu": {"physical_cores": max(1, int(physical_cpus)), + "sockets": max(1, int(cpu_sockets)), "thread_policy": "physical-cores"}, "tiers": { "disk": {"role": "cold-backing", "model_bytes": info["model_bytes"], @@ -299,6 +381,9 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, "expert_capacity": vram_experts, "requires_host_backing": False}, }, "expected_bottleneck": bottleneck, + "bottleneck_class": bottleneck_class, + "projected_hit_rate": round(projected_hit, 4), + "tune": tune, "decisions": [ {"target": "VRAM", "reason": "profile-ranked hot experts"}, {"target": "RAM", "reason": "warm experts execute on CPU without quality loss"}, @@ -318,6 +403,11 @@ def environment_for_plan(plan, env=None, cuda_enabled=True): # ("Affinity not supported on this configuration"): non impostarle li'. result.setdefault("OMP_PROC_BIND", "spread") result.setdefault("OMP_PLACES", "cores") + tune = plan.get("tune", {}) + for key, entry in tune.items(): + if key.startswith("_"): + continue + result.setdefault(key, entry["value"]) if plan["policy"]["name"] == "balanced": result.setdefault("REPIN", "64") ram = plan["tiers"]["ram"] @@ -364,5 +454,18 @@ def format_plan(plan): else: lines.append("VRAM no NVIDIA device detected · CPU path") lines.append(f"limit {plan['expected_bottleneck']}") + hit = plan.get("projected_hit_rate", 0) + lines.append(f"hit {hit:.0%} projected expert residency") + tune = plan.get("tune", {}) + if tune: + lines.append("") + lines.append("auto-tune:") + for key, entry in tune.items(): + if key.startswith("_"): + continue + lines.append(f" {key}={entry['value']:12s} {entry['reason']}") + hint = tune.get("_numa_hint") + if hint: + lines.append(f" hint: {hint}") lines.extend(f"warn {warning}" for warning in plan["warnings"]) return "\n".join(lines) diff --git a/c/sample.h b/c/sample.h new file mode 100644 index 0000000..414c621 --- /dev/null +++ b/c/sample.h @@ -0,0 +1,153 @@ +/* sample.h — sampling (temperature + nucleus) and stop-set management. + * Header-only: all functions are static — include from the main engine file. */ +#ifndef SAMPLE_H +#define SAMPLE_H + +#include +#include +#include +#include "tok.h" + +/* ---- RNG (xorshift64*) -------------------------------------------------- */ +static uint64_t g_rng = 0x9E3779B97F4A7C15ULL; +static inline double rndu(void){ + g_rng ^= g_rng << 13; g_rng ^= g_rng >> 7; g_rng ^= g_rng << 17; + return (double)(g_rng >> 11) * (1.0 / 9007199254740992.0); +} + +/* ---- argmax over a float vector ----------------------------------------- */ +static inline int argmax_v(const float *lo, int V){ + int b=-1; float bv=-INFINITY; + for(int i=0;ibv){ bv=x; b=i; } } + return b<0?0:b; +} + +/* ---- distribution buffers (reused, single-threaded decode) --------------- */ +static float *g_pbuf = NULL; +static int *g_pidx = NULL; + +/* sift-down on max-heap in h[0..n), key = g_pbuf[h[i]] (#335: partial top-p). + * "hole" variant: carries the root value and deposits only at the end, so + * heapify is O(V) and each pop is O(log n) without qsort on the full vocab. */ +static void topp_siftdown(int *h, int n, int i){ + int iv = h[i]; float kv = g_pbuf[iv]; + for (;;) { + int l = 2*i + 1; + if (l >= n) break; + int b = l; if (l+1 < n && g_pbuf[h[l+1]] > g_pbuf[h[l]]) b = l+1; + if (g_pbuf[h[b]] <= kv) break; + h[i] = h[b]; i = b; + } + h[i] = iv; +} + +/* build the target distribution in g_pbuf: softmax(lo/temp) truncated to + * top-p g_nuc. Invariant: g_pbuf stays indexed by token-id (never reordered); + * the truncated tail is zeroed (dist_sample reads by id directly). + * Requires: g_temp, g_nuc, falloc() — declared in the main engine file. */ +static void dist_build(const float *lo, int V){ + if (!g_pbuf) { g_pbuf = falloc(V); g_pidx = malloc(V * sizeof(int)); } + int mxi = -1; float mx = 0; + for (int i = 0; i < V; i++) + if (isfinite(lo[i]) && (mxi < 0 || lo[i] > mx)) { mx = lo[i]; mxi = i; } + double s = 0; float invt = 1.f / (g_temp > 1e-4f ? g_temp : 1e-4f); + if (mxi >= 0) { + for (int i = 0; i < V; i++) { + g_pbuf[i] = isfinite(lo[i]) ? expf((lo[i] - mx) * invt) : 0.f; + s += g_pbuf[i]; + } + } + if (mxi < 0 || !isfinite(s) || s <= 0.0) { + static int warned = 0; + if (!warned) { warned = 1; fprintf(stderr, + "[SAMPLE] warning: non-finite logits (NaN/Inf) — falling back to argmax; " + "output may be degraded. This usually means a numerical blow-up upstream.\n"); } + int a = (mxi >= 0) ? mxi : 0; + for (int i = 0; i < V; i++) g_pbuf[i] = 0.f; + g_pbuf[a] = 1.f; + return; + } + for (int i = 0; i < V; i++) g_pbuf[i] /= (float)s; + if (g_nuc > 0 && g_nuc < 1.f) { + for (int i = 0; i < V; i++) g_pidx[i] = i; + for (int i = V/2-1; i >= 0; i--) topp_siftdown(g_pidx, V, i); + double s2 = 0, cum = 0; int out = V; + do { + int root = g_pidx[0]; + g_pidx[0] = g_pidx[--out]; g_pidx[out] = root; + s2 += g_pbuf[root]; cum += g_pbuf[root]; + if (out > 0) topp_siftdown(g_pidx, out, 0); + } while (cum < g_nuc && out > 0); + for (int i = 0; i < out; i++) g_pbuf[g_pidx[i]] = 0; + float s2f = (float)s2; + for (int i = out; i < V; i++) g_pbuf[g_pidx[i]] /= s2f; + } +} + +/* sample from g_pbuf; ban>=0 excludes that token (renormalizing on the fly) */ +static int dist_sample(int V, int ban){ + double z = 1.0 - (ban >= 0 ? g_pbuf[ban] : 0.0); + if (z <= 1e-12) z = 1e-12; + double u = rndu() * z, cum = 0; + for (int i = 0; i < V; i++) { if (i == ban) continue; cum += g_pbuf[i]; if (cum >= u) return i; } + for (int i = V-1; i >= 0; i--) if (i != ban && g_pbuf[i] > 0) return i; + return 0; +} + +/* next token from logits: greedy if g_temp<=0, sampling otherwise. + * ban = token excluded because it was rejected by speculative verification. */ +static int pick_tok(const float *lo, int V, int ban){ + if (g_temp <= 0) return argmax_v(lo, V); + dist_build(lo, V); + return dist_sample(V, ban); +} + +/* ---- stop set ----------------------------------------------------------- */ +static int g_stop[64], g_nstop = 0; +static inline int is_stop(int t){ + for (int i = 0; i < g_nstop; i++) if (t == g_stop[i]) return 1; + return 0; +} +/* T=NULL -> config stops only (validation/oracle, where the tokenizer is not needed). */ +static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){ + g_nstop = 0; + for (int i = 0; i < c->n_stop && g_nstop < 64; i++) g_stop[g_nstop++] = c->stop_ids[i]; + if (tok_eos >= 0 && !is_stop(tok_eos) && g_nstop < 64) g_stop[g_nstop++] = tok_eos; + int nsp = 0; + if (T) for (int id = 0; id < T->n_ids && g_nstop < 64; id++) + if (T->id_special[id] && !is_stop(id)) { g_stop[g_nstop++] = id; nsp++; } + /* #401: in serve mode keep ONLY <|endoftext|>. Role markers <|user|>/<|observation|> + * (config stops + tokenizer special set) are boundaries the Python server owns; as + * hard stops they cut generation the moment the model opens a block, + * because int4 argmax noise picks a stop-token ID over the correct '<' token. */ + if (getenv("SERVE") && tok_eos >= 0) { + int kept = 0; + for (int i = 0; i < g_nstop; i++) if (g_stop[i] == tok_eos) g_stop[kept++] = g_stop[i]; + if (kept < g_nstop) fprintf(stderr, "[stop] serve mode: filtered %d non-EOS stop tokens (tool-call safety, #401)\n", g_nstop - kept); + g_nstop = kept; nsp = 0; + } + fprintf(stderr, "[stop] %d stop tokens:", g_nstop); + for (int i = 0; i < g_nstop; i++) fprintf(stderr, " %d", g_stop[i]); + if (nsp) fprintf(stderr, " (%d from the tokenizer's special set)", nsp); + fprintf(stderr, "\n"); +} +static void stops_arm(const Cfg *c, int tok_eos){ stops_arm_tok(c, tok_eos, NULL); } + +/* ---- log-prob of a target token given the logit vector ------------------- */ +static double logprob_target(const float *lo, int V, int target, int *am){ + float mx = lo[0]; int best = 0; + for (int i = 1; i < V; i++) if (lo[i] > mx) { mx = lo[i]; best = i; } + double se = 0; + for (int i = 0; i < V; i++) se += exp((double)lo[i] - mx); + if (am) *am = (best == target); + return (double)(lo[target] - mx) - log(se); +} + +/* "glm" in model_type, case-insensitive */ +static int mt_is_glm(const char *s){ + if (s) for (; *s; s++) + if ((s[0]|32) == 'g' && (s[1]|32) == 'l' && (s[2]|32) == 'm') return 1; + return 0; +} + +#endif /* SAMPLE_H */ diff --git a/c/setup.sh b/c/setup.sh index e31e60a..dcb851d 100755 --- a/c/setup.sh +++ b/c/setup.sh @@ -32,11 +32,11 @@ esac # 2) build: nativa (veloce, per QUESTA macchina). Per un binario da distribuire: make portable echo " building (ARCH=${ARCH:-native})…" -make -s glm ARCH="${ARCH:-native}" +make -s colibri ARCH="${ARCH:-native}" # 3) self-test sull'oracolo tiny, se presente if [ -d glm_tiny ] && [ -f ref_glm.json ]; then - r=$(SNAP=./glm_tiny TF=1 ./glm 64 16 16 2>/dev/null | grep -oE "[0-9]+/[0-9]+ positions" || true) + r=$(SNAP=./glm_tiny TF=1 ./colibri 64 16 16 2>/dev/null | grep -oE "[0-9]+/[0-9]+ positions" || true) echo " engine self-test: ${r:-?} (expected 32/32)" fi diff --git a/c/st.h b/c/st.h index 6b4a710..a2c2c3f 100644 --- a/c/st.h +++ b/c/st.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -104,6 +105,38 @@ static int st_direct_fd(shards *S, int fd) { } /* indicizza tutti i model-*.safetensors in snap_dir */ +/* pread completo: chunk-loop (una singola pread si ferma a ~2^31 byte su Linux + * — i tensori bf16 grandi la superano), riprova su EINTR e riporta un errore + * ONESTO: perror stampava "Success" su una short-read (errno resta 0), lo + * stesso sintomo corretto in glm.c per #236. ST_PREAD_CHUNK e' sovrascrivibile + * per i test. EN: full pread — chunk loop (one pread caps at ~2^31 bytes and + * big bf16 tensors exceed it), EINTR retry, honest short-read errors. + * Exits on failure, like every st.h reader. */ +#ifndef ST_PREAD_CHUNK +#define ST_PREAD_CHUNK (1u << 30) +#endif +static void st_pread_full(int fd, void *buf, int64_t n, int64_t off, const char *tag) { + char *p = (char *)buf; + int64_t got = 0; + while (got < n) { + int64_t want = n - got; + if (want > (int64_t)ST_PREAD_CHUNK) want = ST_PREAD_CHUNK; + ssize_t r = pread(fd, p + got, (size_t)want, off + got); + if (r < 0) { + if (errno == EINTR) continue; + fprintf(stderr, "%s: %s (off %lld, %lld/%lld bytes)\n", tag, strerror(errno), + (long long)off, (long long)got, (long long)n); + exit(1); + } + if (r == 0) { + fprintf(stderr, "%s: short read at EOF (off %lld, %lld/%lld bytes) — truncated file?\n", + tag, (long long)off, (long long)got, (long long)n); + exit(1); + } + got += r; + } +} + static void st_init(shards *S, const char *snap_dir) { memset(S, 0, sizeof(*S)); S->cap = 4096; S->t = calloc(S->cap, sizeof(st_tensor)); @@ -128,7 +161,7 @@ static void st_init(shards *S, const char *snap_dir) { if (fstat(fd, &sst) != 0) { perror("fstat shard"); exit(1); } int64_t fsz = (int64_t)sst.st_size; uint64_t hlen; - if (pread(fd, &hlen, 8, 0) != 8) { perror("pread hlen"); exit(1); } + st_pread_full(fd, &hlen, 8, 0, "pread hlen"); /* file malevolo/troncato: hlen deve stare nel file dopo gli 8 byte di * prefisso e sotto il tetto. Senza questo bound hlen+1 puo' andare in * overflow (malloc(0) e poi hdr[hlen]=0 fuori limiti) o forzare una @@ -138,7 +171,7 @@ static void st_init(shards *S, const char *snap_dir) { files[fi], (unsigned long long)hlen, (long long)fsz); exit(1); } char *hdr = malloc(hlen + 1); if (!hdr) { perror("malloc safetensors header"); exit(1); } - if (pread(fd, hdr, hlen, 8) != (ssize_t)hlen) { perror("pread hdr"); exit(1); } + st_pread_full(fd, hdr, (int64_t)hlen, 8, "pread hdr"); hdr[hlen] = 0; int64_t data_start = 8 + (int64_t)hlen; char *arena = NULL; @@ -167,7 +200,19 @@ static void st_init(shards *S, const char *snap_dir) { if (a0 < 0 || b0 < a0 || data_start + b0 > fsz) { fprintf(stderr, "%s: tensor '%s' data_offsets [%lld,%lld] out of file bounds (%lld)\n", files[fi], name, (long long)a0, (long long)b0, (long long)fsz); exit(1); } - int64_t numel = 1; for (int k = 0; k < shp->len; k++) numel *= (int64_t)shp->kids[k]->num; + /* SEC: lo shape viene da un file non fidato (mirror). Senza il guard + * di overflow, uno shape tipo [65535,65535,65535,...] fa avvolgere + * numel a un valore piccolo/negativo che poi passerebbe il cross-check + * numel*esz==nbytes in st_read_f32, riaprendo l'OOB. */ + int64_t numel = 1; int bad_shape = 0; + for (int k = 0; k < shp->len; k++) { + int64_t d = (int64_t)shp->kids[k]->num; + if (d < 0 || (d != 0 && numel > INT64_MAX / d)) { bad_shape = 1; break; } + numel *= d; + } + if (bad_shape) { + fprintf(stderr, "%s: tensor '%s' shape overflows int64 — refusing (hostile or corrupt file)\n", + files[fi], name); exit(1); } if (S->n == S->cap) { S->cap *= 2; S->t = realloc(S->t, S->cap*sizeof(st_tensor)); } st_tensor *t = &S->t[S->n++]; t->name = strdup(name); t->fd = fd; t->off = data_start + a0; @@ -216,9 +261,18 @@ static void st_prefetch(shards *S, const char *name) { static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) { st_tensor *t = st_find(S, name); if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); } + /* SEC: numel viene dallo shape, nbytes dagli offset — due campi indipendenti + * del file. Se non concordano, la memcpy F32 (nbytes) o i loop BF16/F16 + * (numel elementi da un raw di soli nbytes) sforano il buffer del chiamante, + * che e' dimensionato sul config, non sul file. Il chiamante che alloca su + * st_numel resta coerente; questo blocca l'ingresso ostile a monte. */ + int esz = (t->dtype == 2) ? 4 : 2; + if (t->numel < 0 || t->numel > t->nbytes / esz || t->numel * (int64_t)esz != t->nbytes) { + fprintf(stderr, "%s: tensor '%s' shape/bytes mismatch (numel %lld, %lld bytes, dtype %d) — refusing (hostile or corrupt file)\n", + name, name, (long long)t->numel, (long long)t->nbytes, t->dtype); exit(1); } void *raw = malloc(t->nbytes); if (!raw) { fprintf(stderr, "malloc %lld bytes for tensor %s failed\n", (long long)t->nbytes, name); exit(1); } - if (pread(t->fd, raw, t->nbytes, t->off) != t->nbytes) { perror("pread data"); exit(1); } + st_pread_full(t->fd, raw, t->nbytes, t->off, "pread data"); if (t->dtype == 2) { memcpy(out, raw, t->nbytes); } else if (t->dtype == 0) { @@ -243,7 +297,7 @@ static int64_t st_nbytes(shards *S, const char *name) { static void st_read_raw(shards *S, const char *name, void *out, int drop) { st_tensor *t = st_find(S, name); if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); } - if (pread(t->fd, out, t->nbytes, t->off) != t->nbytes) { perror("pread raw"); exit(1); } + st_pread_full(t->fd, out, t->nbytes, t->off, "pread raw"); if (drop) posix_fadvise(t->fd, t->off, t->nbytes, POSIX_FADV_DONTNEED); } @@ -256,7 +310,7 @@ static void st_read_slice_f32(shards *S, const char *name, int64_t elem_off, int int esz = (t->dtype == 2) ? 4 : 2; int64_t boff = t->off + elem_off * esz, nb = n_elems * esz; void *raw = malloc(nb); - if (pread(t->fd, raw, nb, boff) != nb) { perror("pread slice"); exit(1); } + st_pread_full(t->fd, raw, nb, boff, "pread slice"); if (t->dtype == 2) memcpy(out, raw, nb); else if (t->dtype == 0) { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = bf16_to_f32(p[i]); } else { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = f16_to_f32(p[i]); } diff --git a/c/telemetry.h b/c/telemetry.h new file mode 100644 index 0000000..3c3fe85 --- /dev/null +++ b/c/telemetry.h @@ -0,0 +1,189 @@ +/* telemetry.h — dashboard protocol lines, stats/usage persistence, hardware probe. + * Include after Model/Cfg/QT/ESlot/shards and st.h are defined; requires + * qt_bytes(), now_s(), rss_gb(), edisk_s(), and the g_cuda_* globals (ifdef). */ +#ifndef TELEMETRY_H +#define TELEMETRY_H + +static int64_t tbytes(int O,int I,int bits){ + if(bits>=16) return (int64_t)O*I*4; + if(bits>=5) return (int64_t)O*I + (int64_t)O*4; + return (int64_t)O*((I+1)/2) + (int64_t)O*4; +} + +static int64_t expert_bytes_probe(Model *m, int ebits){ + Cfg *c=&m->c; int64_t eb=0; char nm[256]; + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.gate_proj.weight",c->first_dense); + if(st_nbytes(&m->S,nm)>0){ + const char *suf[3]={"gate_proj","up_proj","down_proj"}; + for(int k=0;k<3;k++){ + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight",c->first_dense,suf[k]); + eb+=st_nbytes(&m->S,nm); + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight.qs",c->first_dense,suf[k]); + int64_t q=st_nbytes(&m->S,nm); if(q>0) eb+=q; + } + } + if(eb<=0) eb = tbytes(c->moe_inter,c->hidden,ebits)*2 + tbytes(c->hidden,c->moe_inter,ebits); + return eb; +} + +/* BRAIN MAP: per-turn expert hit bitmap for the dashboard. */ +static uint8_t **g_ehit; +static void ehit_mark(Model *m, int layer, int eid){ + if(!g_ehit){ Cfg *c=&m->c; + g_ehit=calloc(c->n_layers+1,sizeof(uint8_t*)); + for(int i=0;i<=c->n_layers;i++) g_ehit[i]=calloc(c->n_experts,1); + } + g_ehit[layer][eid]=1; +} + +/* CPU model + cores + RAM (GB); empty/zero where unavailable. */ +static void hw_probe(char *cpu, size_t cn, int *cores, double *ram_total, double *ram_avail){ + cpu[0]=0; +#ifdef _WIN32 +#if defined(__x86_64__) || defined(__i386__) + { unsigned int r[12]={0}; unsigned int *w=r; + for(unsigned int f=0x80000002u; f<=0x80000004u; f++,w+=4) + __get_cpuid(f,&w[0],&w[1],&w[2],&w[3]); + char *b=(char*)r; b[47]=0; while(*b==' ')b++; + snprintf(cpu,cn,"%s",b); } +#endif +#else + FILE *ci=fopen("/proc/cpuinfo","r"); + if(ci){ char ln[256]; + while(fgets(ln,sizeof(ln),ci)) if(!strncmp(ln,"model name",10)){ + char *p=strchr(ln,':'); if(p){ p++; while(*p==' ')p++; + int n=(int)strlen(p); if(n>0&&p[n-1]=='\n')p[--n]=0; + snprintf(cpu,cn,"%s",p); } break; } + fclose(ci); } +#endif + *cores=0; +#ifdef _WIN32 + { SYSTEM_INFO si; GetSystemInfo(&si); *cores=(int)si.dwNumberOfProcessors; } +#elif defined(_SC_NPROCESSORS_ONLN) + *cores=(int)sysconf(_SC_NPROCESSORS_ONLN); +#endif + *ram_total=*ram_avail=0; +#ifdef _WIN32 + compat_meminfo(ram_total,ram_avail); +#else + FILE *mi=fopen("/proc/meminfo","r"); + if(mi){ char ln[256]; double mt=0,ma=0; + while(fgets(ln,sizeof(ln),mi)){ + if(sscanf(ln,"MemTotal: %lf",&mt)==1) *ram_total=mt/1e6; + if(sscanf(ln,"MemAvailable: %lf",&ma)==1) *ram_avail=ma/1e6; + } fclose(mi); } +#endif +} + +static void hwinfo_emit(Model *m){ + Cfg *c=&m->c; (void)c; + char cpu[256]; int cores; double ram_total,ram_avail; + hw_probe(cpu,sizeof(cpu),&cores,&ram_total,&ram_avail); + int ngpu=0; double vram_total=0; + char gpu_name[128]=""; +#ifdef COLI_CUDA + ngpu=g_cuda_ndev; vram_total=m->gpu_expert_bytes/1e9; + for(int i=0;i0) + snprintf(gpu_name,sizeof(gpu_name),"CUDA device x%d",g_cuda_ndev); +#endif + printf("HWINFO %d %.1f %.1f %d %.1f %s|%s\n", + cores,ram_total,ram_avail,ngpu,vram_total,cpu,gpu_name); + fflush(stdout); +} + +static void tiers_emit(Model *m){ + Cfg *c=&m->c; int nsp=0; + for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; + int total=(nsp+(m->has_mtp?1:0))*c->n_experts; + int pinned=0,lru=0; + for(int i=0;i<=c->n_layers;i++){ pinned+=m->npin?m->npin[i]:0; lru+=m->ecn?m->ecn[i]:0; } + int vram=0; double vram_gb=0; +#ifdef COLI_CUDA + vram=m->gpu_expert_count; vram_gb=m->gpu_expert_bytes/1e9; +#endif + int ram=pinned-vram+lru; if(ram<0) ram=0; + int disk=total-vram-ram; if(disk<0) disk=0; + double eb=(double)expert_bytes_probe(m,m->ebits); + printf("TIERS %d %d %d %.2f %.2f\n",vram,ram,disk,vram_gb,ram*eb/1e9); + fflush(stdout); +} + +static void emap_emit(Model *m){ + Cfg *c=&m->c; + int rows=0; + for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; + int has_mtp = m->has_mtp && m->eusage[c->n_layers]; + if(has_mtp) rows++; + int cols=c->n_experts; + char *hex=malloc((size_t)rows*cols*2+1); int w=0; + for(int i=0;i<=c->n_layers;i++){ + int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); + if(!is_row) continue; + for(int e=0;epin[i]; + for(int z=0;znpin[i];z++) if(P[z].eid==e){ +#ifdef COLI_CUDA + tier = P[z].g.cuda?2:1; +#else + tier = 1; +#endif + break; } + if(!tier && m->ecache && m->ecache[i]) + for(int z=0;zecn[i];z++) if(m->ecache[i][z].eid==e){ tier=1; break; } + uint32_t u = m->eusage[i]?m->eusage[i][e]:0; + int heat=0; while(u){ heat++; u>>=1; } if(heat>63) heat=63; + int b=(tier<<6)|heat; + hex[w++]="0123456789abcdef"[b>>4]; hex[w++]="0123456789abcdef"[b&15]; + } + } + hex[w]=0; + printf("EMAP %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); +} + +static void hits_emit(Model *m){ + Cfg *c=&m->c; if(!g_ehit) return; + int rows=0; + for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; + int has_mtp = m->has_mtp && m->eusage[c->n_layers]; + if(has_mtp) rows++; + int cols=c->n_experts, nb=(rows*cols+7)/8; + uint8_t *bm=calloc(nb,1); int bit=0; + for(int i=0;i<=c->n_layers;i++){ + int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); + if(!is_row) continue; + for(int e=0;e>3]|=1<<(bit&7); g_ehit[i][e]=0; } + } + char *hex=malloc((size_t)nb*2+1); int w=0; + for(int b=0;b>4]; hex[w++]="0123456789abcdef"[bm[b]&15]; } + hex[w]=0; + printf("HITS %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); free(bm); +} + +static void stats_dump_q(Model *m, const char *path, int quiet){ + char tmp[2100]; snprintf(tmp,sizeof(tmp),"%s.tmp",path); + FILE *f=fopen(tmp,"w"); if(!f){ if(!quiet) perror(tmp); return; } + Cfg *c=&m->c; int64_t tot=0, nz=0; + for(int i=0;i<=c->n_layers;i++){ if(!m->eusage[i]) continue; + for(int e=0;en_experts;e++) if(m->eusage[i][e]){ fprintf(f,"%d %d %u\n",i,e,m->eusage[i][e]); tot+=m->eusage[i][e]; nz++; } } + fclose(f); rename(tmp,path); + if(!quiet) fprintf(stderr,"[STATS] %lld selections across %lld distinct experts -> %s\n",(long long)tot,(long long)nz,path); +} +static void stats_dump(Model *m, const char *path){ stats_dump_q(m,path,0); } + +static char g_usage_path[2100]=""; +static int64_t usage_load(Model *m, const char *path){ + FILE *f=fopen(path,"r"); if(!f) return 0; + Cfg *c=&m->c; int l,e; uint32_t cnt; int64_t tot=0; + while(fscanf(f,"%d %d %u",&l,&e,&cnt)==3) + if(l>=0&&l<=c->n_layers&&e>=0&&en_experts&&m->eusage[l]){ m->eusage[l][e]+=cnt; tot+=cnt; } + fclose(f); return tot; +} +static void usage_save(Model *m){ if(g_usage_path[0]) stats_dump_q(m,g_usage_path,1); } + +#endif /* TELEMETRY_H */ diff --git a/c/tests/bench_dsa_select.c b/c/tests/bench_dsa_select.c new file mode 100644 index 0000000..676e4c2 --- /dev/null +++ b/c/tests/bench_dsa_select.c @@ -0,0 +1,165 @@ +/* Microbenchmark: old (full-qsort) vs new (quickselect partial-select) DSA top-keep. + * + * This is NOT a unit test -- test_dsa_select.c proves correctness. This measures the + * headline claim of #356: that replacing the O(nk log nk) qsort over all nk context + * scores with an O(nk) partial_select_desc is materially faster per call, which is + * the win the issue was opened for -- and that the win GROWS with context length + * (because quickselect is linear average, qsort is n-log-n). + * + * It re-implements the OLD top-keep inline (qsort + threshold + scans) on a private + * buffer so the A/B runs in one process, same inputs, same warm caches -- a controlled + * comparison. It calls the REAL (new) partial_select_desc via the include-glm.c + * pattern, replicating the production threshold derivation + scans. + * + * Methodology (chosen to be honest, not to flatter the change): + * - keep = 2048 (the real GLM-5.2 index_topk), nk swept across context lengths from + * the 2049 activation boundary up to 65536 (a long conversation). + * - Three score shapes: (a) realistic peaked -- a few hot keys, long tail, the shape + * real DSA attention scores take; (b) uniform random -- no structure; (c) a plateau + * of ties, to exercise the boundary-membership path. + * - Each (shape, nk) is timed over N_REPEAT=2000 iterations, with the scores frozen + * so both algorithms do IDENTICAL work. We report median ns/call and the new/old + * ratio. A warmup pass primes caches before timing. + * + * Run: make tests/bench_dsa_select && ./tests/bench_dsa_select (not in TEST_BINS) + */ +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main + +#include +#include + +/* ---- the OLD algorithm, verbatim from dev before #356, on a private buffer ---- */ +static int cmp_pdesc_old(const void *a, const void *b){ + float x=*(const float*)a, y=*(const float*)b; return xy?-1:0; } +static void keep_old(const float *isc, int nk, int keep, int *dst, int *nd_out){ + float *tmp=malloc((size_t)nk*sizeof(float)); + memcpy(tmp,isc,(size_t)nk*sizeof(float)); + qsort(tmp,(size_t)nk,sizeof(float),cmp_pdesc_old); + float thr=tmp[keep-1]; int nd=0; + for(int t=0;tthr) dst[nd++]=t; + for(int t=0;t=0 && ts[b]>k){ ts[b+1]=ts[b]; b--; } ts[b+1]=k; } + free(dst); + return ts[N_REPEAT/2]; +} + +/* Sort an array of doubles ascending (median-of-medians aggregation below). */ +static void dsort(double *a, int n){ + for(int s=1;s=0 && a[b]>k){ a[b+1]=a[b]; b--; } a[b+1]=k; } +} + +/* the NEW algorithm calls the real partial_select_desc + replicates the production + * threshold derivation and position scans. */ +static void keep_new(const float *isc, int nk, int keep, int *dst, int *nd_out){ + float *tmp=malloc((size_t)nk*sizeof(float)); + memcpy(tmp,isc,(size_t)nk*sizeof(float)); + partial_select_desc(tmp,nk,keep); + float thr=tmp[0]; for(int t=1;tthr) dst[nd++]=t; + for(int t=0;t> 17; brng ^= brng << 5; + return (double)(brng >> 8) * (1.0 / 16777216.0); } +static void fill_realistic(float *isc, int nk){ /* few hot, long distinct tail */ + for(int i=0;i boundary path */ + for(int i=0;i +#include + +/* ---- the OLD algorithm, verbatim from dev before #354, on a private buffer ---- */ +static float *s_pbuf; static int *s_pidx; static double *s_ref; +static int cmp_pdesc_old(const void *a, const void *b){ + double pa = s_ref[*(const int*)a], pb = s_ref[*(const int*)b]; + return pa < pb ? 1 : pa > pb ? -1 : 0; } +static void dist_build_old(const float *lo, int V, double temp, double nuc){ + double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i]; + double s = 0, invt = 1.0 / (temp > 1e-4 ? temp : 1e-4); + for (int i = 0; i < V; i++){ s_ref[i] = exp((lo[i]-mx)*invt); s += s_ref[i]; } + for (int i = 0; i < V; i++) s_ref[i] /= s; + if (nuc > 0 && nuc < 1.0){ + for (int i = 0; i < V; i++) s_pidx[i] = i; + qsort(s_pidx, V, sizeof(int), cmp_pdesc_old); + double cum = 0; int keep = V; + for (int i = 0; i < V; i++){ cum += s_ref[s_pidx[i]]; if (cum >= nuc){ keep = i+1; break; } } + double s2 = 0; + for (int i = keep; i < V; i++) s_ref[s_pidx[i]] = 0; + for (int i = 0; i < keep; i++) s2 += s_ref[s_pidx[i]]; + for (int i = 0; i < keep; i++) s_ref[s_pidx[i]] /= s2; + } + (void)s_pbuf; +} + +/* ---- timing: median of N_REPEAT runs in ns/call, sorted ascending ---- */ +#define N_REPEAT 2000 +static double bench_ns(void (*fn)(const float*,int,double,double), + const float *lo, int V, double temp, double nuc){ + static double ts[N_REPEAT]; + for (int r = 0; r < N_REPEAT; r++){ + double t0 = now_s(); + fn(lo, V, temp, nuc); + ts[r] = (now_s() - t0) * 1e9; + } + /* insertion sort the N_REPEAT samples (small), take median */ + for (int a = 1; a < N_REPEAT; a++){ double k = ts[a]; int b = a-1; + while (b >= 0 && ts[b] > k){ ts[b+1] = ts[b]; b--; } ts[b+1] = k; } + return ts[N_REPEAT/2]; +} + +/* the NEW algorithm is the real dist_build, but it writes g_pbuf (not a private buf). + * Wrap it so the bench signature matches, and set the globals it reads. */ +static void dist_build_new(const float *lo, int V, double temp, double nuc){ + g_temp = (float)temp; g_nuc = (float)nuc; + dist_build(lo, V); +} + +/* deterministic logit fill for three shapes */ +static uint32_t brng = 0xA5A5A5A5u; +static double brand(void){ brng ^= brng << 13; brng ^= brng >> 17; brng ^= brng << 5; + return (double)(brng >> 8) * (1.0 / 16777216.0); } +static void fill_realistic(float *lo, int V){ /* one hot, exponential tail -- like real logits */ + for (int i = 0; i < V; i++) lo[i] = (float)(-4.0 * brand() - (double)i * 0.0001); + lo[0] = 6.f; lo[V/50] = 4.f; lo[V/200] = 3.f; +} +static void fill_uniform(float *lo, int V){ /* worst case for the heap: max pop count */ + for (int i = 0; i < V; i++) lo[i] = 0.f; +} +static void fill_plateau(float *lo, int V){ /* ties: blocks of equal value */ + for (int i = 0; i < V; i++) lo[i] = (float)(-(double)(i / 50)); +} + +int main(void){ + int V = 151936; + float *lo = malloc((size_t)V * sizeof(float)); + s_ref = malloc((size_t)V * sizeof(double)); + s_pidx = malloc((size_t)V * sizeof(int)); + /* force the new dist_build to allocate g_pbuf/g_pidx at full V once */ + g_temp = 0.7f; g_nuc = 0.9f; dist_build(lo, V); + + double temp = 0.7; + struct { const char *name; void (*fill)(float*,int); } shapes[] = { + { "realistic", fill_realistic }, + { "uniform", fill_uniform }, + { "plateau", fill_plateau }, + }; + double nucs[] = { 0.5, 0.9, 0.95, 0.99 }; + + printf("bench_topp: top-p truncation, old (qsort) vs new (heap) V=%d temp=%.2f\n", V, temp); + printf("%-12s %6s %14s %14s %9s %9s\n", "shape", "nuc", "old ns/call", "new ns/call", "speedup", "keep"); + printf("-----------------------------------------------------------------------------\n"); + + for (size_t sh = 0; sh < sizeof(shapes)/sizeof(shapes[0]); sh++){ + shapes[sh].fill(lo, V); + for (size_t ni = 0; ni < sizeof(nucs)/sizeof(nucs[0]); ni++){ + double nuc = nucs[ni]; + /* warmup both paths so caches/branch predictors are primed */ + for (int w = 0; w < 50; w++){ dist_build_old(lo, V, temp, nuc); dist_build_new(lo, V, temp, nuc); } + double t_old = bench_ns(dist_build_old, lo, V, temp, nuc); + double t_new = bench_ns(dist_build_new, lo, V, temp, nuc); + /* keep count = non-zero entries the new path leaves (== old's keep) */ + int keep = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) keep++; + printf("%-12s %6.2f %14.0f %14.0f %8.2fx %9d\n", + shapes[sh].name, nuc, t_old, t_new, t_old / t_new, keep); + } + printf("\n"); + } + + printf("bench_topp: done (lower ns is better; speedup = old/new)\n"); + free(lo); free(s_ref); free(s_pidx); + return 0; +} diff --git a/c/tests/test_benchmark_cuda_fixture.py b/c/tests/test_benchmark_cuda_fixture.py index a704052..3feaf11 100644 --- a/c/tests/test_benchmark_cuda_fixture.py +++ b/c/tests/test_benchmark_cuda_fixture.py @@ -1,11 +1,12 @@ import unittest -from tools.benchmark_cuda_fixture import parse_output +from tools.benchmark_cuda_fixture import parse_output, parse_p0 SAMPLE = """ REPLAY decode: 4 tokens | 12.34 tok/s PROFILE: expert-disk 1.25s | expert-matmul 2.50s | attention 0.75s | lm_head 0.10s | other -0.05s +P0-EXEC: routed CPU 1.200s / 123.40 GB/s (456 row) | routed GPU critical 0.150s | router 0.200s | residual P2P 0.030s / 75 hop | orchestration 0.100s """ @@ -19,6 +20,9 @@ class ParseOutputTest(unittest.TestCase): with self.assertRaisesRegex(RuntimeError, "benchmark output missing"): parse_output("REPLAY decode: 4 tokens | 12.34 tok/s", "engine failed") + def test_extracts_p0_profile(self): + self.assertEqual(parse_p0(SAMPLE), [1.2, 123.4, 456.0, 0.15, 0.2, 0.03, 75.0, 0.1]) + if __name__ == "__main__": unittest.main() diff --git a/c/tests/test_dsa_select.c b/c/tests/test_dsa_select.c new file mode 100644 index 0000000..ef3fd7b --- /dev/null +++ b/c/tests/test_dsa_select.c @@ -0,0 +1,223 @@ +/* DSA top-keep partial-select: the quickselect rewrite (#356) must produce a + * BIT-IDENTICAL kept-position set to the old full-vocab qsort, for every score + * shape the attention indexer can see. + * + * Why this test exists (#356): attention_rows() selects the top-`keep` context + * keys (index_topk=2048 on GLM-5.2) to attend to. It previously did this by + * full-qsorting all `nk` scores (O(nk log nk)) and reading tmp[keep-1] as the + * threshold. It now does a partial_select_desc (quickselect, O(nk) average) and + * takes the threshold as the min of the selected top-keep block. The contract is + * subtle but STRONGER than test_topp's: + * + * The two position-order scans that build dst[] -- + * for t: if isc[t] > thr -> keep (strictly above threshold) + * for t: if isc[t] == thr -> keep (ties, in position order) + * -- are UNCHANGED by the rewrite. So if the threshold value is identical, + * the kept-position set is identical element-by-element (not just as a + * multiset, which is all the unstable sampling heap in #335 could promise). + * + * Strategy: drive the REAL partial_select_desc (via the include-glm.c pattern) + * and replicate the production threshold derivation + scans, then compare the + * resulting dst[] against an INDEPENDENT reference that re-implements the OLD + * algorithm (full qsort + tmp[keep-1] threshold) on a private buffer. The kept + * sets must be element-wise equal on every shape, including tie plateaus where + * the boundary membership is decided by the position scan. + * + * We also directly unit-test partial_select_desc's partition invariant: after + * the call, max(a[keep..n)) <= min(a[0..keep)) -- i.e. the keep largest really + * did land in the prefix. This catches a broken quickselect even before the + * end-to-end comparison. + * + * In-memory only (no scratch files), so it builds clean on the Windows MinGW CI + * job without the unmerged compat shim. */ +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main + +#include + +static int g_nfails = 0; + +#define FAIL(fmt, ...) do { \ + fprintf(stderr, " FAIL [%s nk=%d keep=%d shape=%s]: " fmt "\n", \ + label, nk, keep, shape_name, ##__VA_ARGS__); \ + g_nfails++; \ + return; \ +} while (0) + +/* ---- independent reference: the OLD algorithm (full qsort + tmp[keep-1]) ---- */ +/* qsort comparator matching the production cmp_fdesc exactly (desc, unstable). */ +static int cmp_ref_desc(const void *a, const void *b){ + float x=*(const float*)a, y=*(const float*)b; return xy?-1:0; } + +/* Reproduce the OLD glm.c:2589-2596 exactly: copy, qsort desc, threshold = + * tmp[keep-1], then the two position-order scans into dst[]. Returns nd. */ +static int keep_old(const float *isc, int nk, int keep, int *dst){ + float *tmp=malloc((size_t)nk*sizeof(float)); + memcpy(tmp,isc,(size_t)nk*sizeof(float)); + qsort(tmp,(size_t)nk,sizeof(float),cmp_ref_desc); + float thr=tmp[keep-1]; + int nd=0; + for(int t=0;tthr) dst[nd++]=t; + for(int t=0;tthr) dst[nd++]=t; + for(int t=0;t= every element of tmp[keep..n). + * (>=, not >: equal values may sit on either side of the partition boundary, + * which is fine -- the threshold is the MIN of the prefix, and the position + * scan handles ties.) */ + float top_min=INFINITY, tail_max=-INFINITY; + for(int i=0;itail_max) tail_max=tmp[i]; + if(!(top_min >= tail_max)) + FAIL("partition invariant violated: top_min=%.9g < tail_max=%.9g", top_min, tail_max); + free(tmp); +} + +/* ---- end-to-end: old vs new kept-set must be element-wise identical ---- */ +static void check_case(const char *label, int nk, int keep, const char *shape_name, + const float *isc){ + int *da=malloc((size_t)nk*sizeof(int)); + int *db=malloc((size_t)nk*sizeof(int)); + int na=keep_old(isc,nk,keep,da); + int nb=keep_new(isc,nk,keep,db); + + /* 1. both keep exactly `keep` positions (the contract: keep the top-keep by + * count). A count mismatch is a real bug, not a tie artifact. */ + if(na!=keep) FAIL("old kept %d, expected %d (old path is the reference)", na, keep); + if(nb!=keep) FAIL("new kept %d, expected %d", nb, keep); + if(na!=nb) FAIL("keep-count mismatch: old=%d new=%d", na, nb); + + /* 2. element-wise identical dst[]. This is the strong contract: because the + * threshold is derived identically and the position-order scans are byte- + * for-byte the same, the kept SET and its ORDER must match exactly. (This + * is what makes #356 cleaner than #335, which was multiset-only.) */ + int first_diff=-1; + for(int i=0;i=0) + FAIL("kept-set differs at index %d: old dst[%d]=%d new dst[%d]=%d", + first_diff, first_diff, da[first_diff], first_diff, db[first_diff]); + + /* 3. also check the partition invariant directly (catches a subtly broken + * quickselect even if the threshold happened to come out right). */ + check_partition(label,isc,nk,keep,shape_name); + + free(da); free(db); + printf(" ok [nk=%d keep=%d shape=%s]\n", nk, keep, shape_name); +} +#undef FAIL + +/* deterministic xorshift32 RNG (matches the test_i4_grouped.c / test_topp.c convention) */ +static uint32_t rng_state = 0x12345678u; +static uint32_t xr(void){ rng_state ^= rng_state << 13; rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; return rng_state; } +static double frand(void){ return (xr() >> 8) * (1.0 / 16777216.0); } /* [0,1) */ + +/* fill scores for a given shape. Shapes stress the threshold boundary and the + * quickselect's median-of-three pivot differently. */ +static void fill_shape(float *isc, int nk, int shape){ + switch(shape){ + case 0: /* uniform random distinct (no ties): the clean contract case */ + for(int i=0;i3) isc[nk/3]=1.f; if(nk>2) isc[nk/2]=0.5f; break; + case 2: /* strictly decreasing geometric (no ties): sorted input -- worst case + * for a naive quickselect; median-of-three must handle it */ + for(int i=0;i boundary membership decided + * entirely by the position scan (exercises the ==thr path) */ + for(int i=0;i degenerate threshold, all kept + * via the ==thr scan; quickselect must not infinite-loop or corrupt */ + for(int i=0;i=n / k==1 / k==n edges. */ + int nks[] = {1, 2, 8, 64, 2049, 4097, 8193}; + int keeps[] = {1, 8, 256, 1024, 2048}; + int n_shapes = 6; + + int cases = 0; + for(size_t ni=0; nin, but skip the plateau/geometric edge if nk<7) */ + fill_shape(isc,nk,shape); + for(size_t ki=0; kink) continue; /* keep<=nk invariant of the production code */ + if(keep<=0) continue; + char label[40]; snprintf(label,sizeof(label),"nk[%zu]/keep[%zu]/shape[%d]",ni,ki,shape); + const char *sn=(const char*[]){"random","peaked","decreasing","increasing","plateau","all-equal"}[shape]; + check_case(label,nk,keep,sn,isc); + cases++; + } + } + free(isc); + } + + /* edge: keep == nk (nothing to partition; both paths keep everything) */ + { + int nk=100, keep=100; float isc[100]; + for(int i=0;i every kept slot is a tie; + * the position scan must pick positions 0..keep-1 deterministically */ + { + int nk=1000, keep=500; float isc[1000]; + for(int i=0;i positions 0..%d]\n",keep,keep-1); cases++; + free(db); + } + + printf("\ntest_dsa_select: %d cases run, %d failure(s)\n", cases, g_nfails); + if(g_nfails){ printf("test_dsa_select: FAIL\n"); return 1; } + printf("test_dsa_select: ok\n"); + return 0; +} diff --git a/c/tests/test_i4_grouped.c b/c/tests/test_i4_grouped.c new file mode 100644 index 0000000..04e8d88 --- /dev/null +++ b/c/tests/test_i4_grouped.c @@ -0,0 +1,201 @@ +/* Exactness test for the grouped-int4 kernel (fmt=4, one f32 scale per `gs` + * elements along I) against a plain-C reference that dequantizes and multiplies + * in double. + * + * Why this test exists: matmul_i4_grouped is the REFERENCE the CUDA fmt=4 path + * (#298) is expected to reproduce, and it had no test of its own. Debugging a + * backend against an unverified oracle means two moving targets. Anyone porting + * fmt=4 to a new backend can now diff against a kernel that is known exact here. + * + * Covers: I a clean multiple of gs, I with a partial last group (the `glen` + * clamp), odd I (the nibble tail), gs larger than I (single group), and the + * nibble edges 0 and 15 (which decode to -8 and +7 — an offset encoding, NOT + * two's complement; getting this backwards is silent and looks like noise). + * + * FP note: the kernel sums each group in f32 (AVX2 accumulator + scalar tail) + * while the reference sums in double, so we compare against a relative epsilon + * rather than bit-exactly. The tolerance is tight enough that a wrong scale + * index, a wrong group boundary or a swapped nibble cannot hide under it — + * those are O(1) relative errors, not O(1e-6). */ +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main + +static uint32_t rng_state=0xC0FFEEu; +static uint32_t xr(void){ rng_state^=rng_state<<13; rng_state^=rng_state>>17; rng_state^=rng_state<<5; return rng_state; } +static float frand(void){ return (float)((int)(xr()%2001)-1000)/1000.0f; } + +/* Reference: dequantize nibble -> (v-8)*scale[group], accumulate in double. + * Deliberately the dumbest possible expression of the format. */ +static void ref_grouped(double *y, double *mag, const float *x, const uint8_t *q4, + const float *scale, int S, int I, int O, int gs){ + int rb=(I+1)/2, ng=(I+gs-1)/gs; + for(int o=0;o>1]; + int nib=(i&1)?(int)(byte>>4):(int)(byte&0xF); + double term=(double)xs[i] * (double)(nib-8) * (double)scl[i/gs]; + a += term; m += fabs(term); + } + y[(int64_t)s*O+o]=a; + /* Sum of |terms|: the scale the f32 rounding error actually lives on. + * Comparing against |result| instead would flag pure cancellation -- + * a dot product of signed terms can land near zero, and then a 1e-6 + * absolute error reads as a 1e-3 relative one. That is the accumulator's + * precision, not a kernel defect. A wrong scale index or group boundary + * shifts the result by a fraction OF THE TERMS, so it is caught here. */ + mag[(int64_t)s*O+o]=m; + } + } +} + +static int check(const char *name, int S, int I, int O, int gs, int fill_edges){ + int rb=(I+1)/2, ng=(I+gs-1)/gs; + uint8_t *q4=malloc((size_t)O*rb); + float *scale=malloc((size_t)O*ng*sizeof(float)); + float *x=malloc((size_t)S*I*sizeof(float)); + float *y=malloc((size_t)S*O*sizeof(float)); + double *yr=malloc((size_t)S*O*sizeof(double)); + double *ym=malloc((size_t)S*O*sizeof(double)); + if(!q4||!scale||!x||!y||!yr||!ym){ fprintf(stderr,"%s: OOM\n",name); return 1; } + + for(size_t i=0;i<(size_t)O*rb;i++) q4[i]=(uint8_t)(xr()&0xFF); + if(fill_edges){ + /* nibble extremes: 0x0F -> +7, 0x00 -> -8. A two's-complement misread + * turns 15 into -1 instead of +7 and the error is data-dependent noise. */ + for(size_t i=0;i<(size_t)O*rb && i<64;i++) q4[i]=(i&1)?0x00:0xFF; + } + /* scales span a few orders of magnitude: a wrong group index shows up big */ + for(int i=0;i1e-30 ? d/ym[i] : d; /* error relative to the summed magnitude */ + if(rel>worst) worst=rel; + if(rel>1e-6){ + if(bad<3) fprintf(stderr,"%s: [%d] got %.9g want %.9g (|terms| %.3g, rel %.3g)\n", + name,i,(double)y[i],yr[i],ym[i],rel); + bad++; + } + } + free(q4);free(scale);free(x);free(y);free(yr);free(ym); + if(bad){ fprintf(stderr,"%s: FAIL (%d/%d mismatched, worst rel %.3g)\n",name,bad,S*O,worst); return 1; } + printf(" %-42s ok (S=%d I=%d O=%d gs=%d ng=%d, worst rel %.2g)\n",name,S,I,O,gs,ng,worst); + return 0; +} + +/* matmul_i4_grouped_pair (fused gate+up, #298) reads x once instead of twice. + * Checked two ways, because "identical" is only true where it can be: + * + * - Correctness, always: both outputs must match the double reference within + * the same magnitude-relative epsilon as the unfused kernel. + * - Bit-exactness, only when I % gs == 0: then every group is covered by the + * AVX2 body, whose accumulation order is identical to the unfused kernel, so + * the results agree to the last bit. This is the shape the real g64 + * checkpoints have (I = 2048 / 6144, gs = 64), i.e. the production path. + * + * With a PARTIAL last group the group tail falls to scalar code, and the + * compiler is free to contract/reassociate the fused body differently from the + * single-matrix one. The results then differ by ~1e-7 -- rounding, not logic + * (which of gate/up "differs" is arbitrary, the tell that it is FP luck). + * Demanding bit-exactness there would report a compiler artifact as a bug. */ +#ifdef COLI_HAVE_GROUPED_PAIR +static int check_pair(const char *name, int S, int I, int O, int gs){ + int rb=(I+1)/2, ng=(I+gs-1)/gs; + uint8_t *qg=malloc((size_t)O*rb), *qu=malloc((size_t)O*rb); + float *sg=malloc((size_t)O*ng*sizeof(float)), *su=malloc((size_t)O*ng*sizeof(float)); + float *x=malloc((size_t)S*I*sizeof(float)); + float *yg=malloc((size_t)S*O*sizeof(float)), *yu=malloc((size_t)S*O*sizeof(float)); + float *rg=malloc((size_t)S*O*sizeof(float)), *ru=malloc((size_t)S*O*sizeof(float)); + double *dg=malloc((size_t)S*O*sizeof(double)), *du=malloc((size_t)S*O*sizeof(double)); + double *mg=malloc((size_t)S*O*sizeof(double)), *mu=malloc((size_t)S*O*sizeof(double)); + if(!qg||!qu||!sg||!su||!x||!yg||!yu||!rg||!ru||!dg||!du||!mg||!mu){ fprintf(stderr,"%s: OOM\n",name); return 1; } + + for(size_t i=0;i<(size_t)O*rb;i++){ qg[i]=(uint8_t)(xr()&0xFF); qu[i]=(uint8_t)(xr()&0xFF); } + for(int i=0;i1e-30 ? fabs((double)yg[i]-dg[i])/mg[i] : fabs((double)yg[i]-dg[i]); + double eu = mu[i]>1e-30 ? fabs((double)yu[i]-du[i])/mu[i] : fabs((double)yu[i]-du[i]); + if(eg>worst) worst=eg; + if(eu>worst) worst=eu; + if(eg>1e-6||eu>1e-6){ + if(bad<3) fprintf(stderr,"%s: [%d] gate %.9g/%.9g up %.9g/%.9g (rel %.3g/%.3g)\n", + name,i,(double)yg[i],dg[i],(double)yu[i],du[i],eg,eu); + bad++; + } + if(yg[i]!=rg[i]||yu[i]!=ru[i]) exact=0; + } + /* Aligned shapes run entirely through the AVX2 body: same order as unfused, + * so bit-exactness is a real invariant there and worth asserting. */ + if(I%gs==0 && !exact){ + fprintf(stderr,"%s: FAIL fused != unfused bitwise on an ALIGNED shape " + "(no scalar tail runs here; the orders must match)\n",name); + bad++; + } + free(qg);free(qu);free(sg);free(su);free(x);free(yg);free(yu);free(rg);free(ru); + free(dg);free(du);free(mg);free(mu); + if(bad){ fprintf(stderr,"%s: FAIL (%d mismatched)\n",name,bad); return 1; } + printf(" %-42s ok (S=%d I=%d O=%d gs=%d, worst rel %.2g%s)\n",name,S,I,O,gs,worst, + I%gs==0?", bit-exact vs unfused":""); + return 0; +} +#endif + +int main(void){ + int fail=0; + printf("test_i4_grouped: matmul_i4_grouped vs plain-C dequant reference\n"); + + /* the shape the g64 checkpoints actually use */ + fail|=check("gs=64, I multiple of gs", 2, 512, 8, 64, 0); + fail|=check("gs=64, single row single token", 1, 128, 1, 64, 0); + fail|=check("gs=64, nibble edges (0x00/0xFF)", 1, 256, 4, 64, 1); + + /* partial last group: glen clamp, the classic off-by-one */ + fail|=check("gs=64, partial last group (I=200)", 2, 200, 4, 64, 0); + fail|=check("gs=64, I just over a group (I=65)", 1, 65, 3, 64, 0); + fail|=check("gs=64, I one under a group (I=63)", 1, 63, 3, 64, 0); + + /* odd I: the scalar nibble tail (i+1 == I) */ + fail|=check("gs=64, odd I (I=201)", 2, 201, 4, 64, 0); + fail|=check("gs=16, odd I (I=33)", 1, 33, 2, 16, 0); + + /* gs > I: everything in one group */ + fail|=check("gs=128 > I=64 (single group)", 1, 64, 4, 128, 0); + + /* the other documented group size */ + fail|=check("gs=128, I multiple of gs", 2, 512, 4, 128, 0); + + /* batch: S>1 exercises the per-s inner loop against a shared scale row */ + fail|=check("gs=64, batch S=8", 8, 320, 6, 64, 0); + +#ifdef COLI_HAVE_GROUPED_PAIR + printf("test_i4_grouped: matmul_i4_grouped_pair (fused gate+up) vs two separate calls\n"); + fail|=check_pair("pair: gs=64, I multiple of gs", 2, 512, 8, 64); + fail|=check_pair("pair: gs=64, partial last group",2, 200, 4, 64); + fail|=check_pair("pair: gs=64, odd I (I=201)", 2, 201, 4, 64); + fail|=check_pair("pair: gs=64, decode S=1", 1, 320, 6, 64); + fail|=check_pair("pair: gs=128, I=512", 2, 512, 4, 128); +#endif + + if(fail){ printf("test_i4_grouped: FAIL\n"); return 1; } + printf("test_i4_grouped: ok\n"); + return 0; +} diff --git a/c/tests/test_idot.c b/c/tests/test_idot.c index b7109d4..343535c 100644 --- a/c/tests/test_idot.c +++ b/c/tests/test_idot.c @@ -7,7 +7,7 @@ * (sign-trick kernels must treat |−128| as 128 unsigned, not saturate to 127), * and random data at qrow_i8's contract (|x| <= 127, w full int8 range). */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static uint32_t rng_state=0x12345678u; diff --git a/c/tests/test_kv_alloc.c b/c/tests/test_kv_alloc.c index 40ec00d..d080c08 100644 --- a/c/tests/test_kv_alloc.c +++ b/c/tests/test_kv_alloc.c @@ -4,7 +4,7 @@ * arrays twice on the second call -> allocator abort. No model file needed: * the CPU path of kv_alloc only reads c->n_layers/kv_lora/qk_rope. */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main int main(void){ diff --git a/c/tests/test_logit_nan.c b/c/tests/test_logit_nan.c new file mode 100644 index 0000000..de6dc2d --- /dev/null +++ b/c/tests/test_logit_nan.c @@ -0,0 +1,70 @@ +/* Regression for non-finite-logit poisoning of sampling. + * + * A single NaN or +Inf in the logits (a bad streamed expert tile, or an fp + * overflow in the matmul at a low-RAM eviction boundary) used to make softmax + * produce an all-NaN g_pbuf; dist_sample then never satisfied cum>=u and fell + * through to return token 0 — so the engine silently emitted an unbroken run of + * token 0 with no error, on the DEFAULT serve path (TEMP>0, 0 +#include +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main + +static int approx1(double x){ return x > 0.999 && x < 1.001; } + +int main(void){ + /* --- argmax_v must skip NaN (greedy decode + speculative-verify paths) --- */ + { float lo[8]={NAN,1.f,5.f,2.f,NAN,-3.f,4.f,0.f}; + assert(argmax_v(lo,8)==2 && "pick max finite (idx2=5.0), not NaN-pinned idx0"); } + { float lo[8]={3.f,INFINITY,1.f,2.f,0.f,-1.f,2.5f,1.5f}; + assert(argmax_v(lo,8)==1 && "pick the +Inf position"); } + { float lo[8]; for(int i=0;i<8;i++) lo[i]=NAN; + assert(argmax_v(lo,8)==0 && "all-NaN: no crash, defined fallback"); } + + g_temp=0.7f; g_nuc=0.9f; /* the default serve/chat sampling path */ + + /* --- dist_build: a NaN logit must yield a finite one-hot, not all-NaN --- */ + { float lo[8]={0.5f,1.f,NAN,8.f,0.2f,-1.f,0.f,0.3f}; /* max finite = idx3 (8.0) */ + dist_build(lo,8); + double sum=0; int nan=0; + for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; } + assert(!nan && "g_pbuf must be finite after a NaN logit"); + assert(approx1(sum) && "g_pbuf must normalize to 1"); + assert(approx1(g_pbuf[3]) && "mass must land on the max finite logit (idx3)"); + assert(dist_sample(8,-1)==3 && "sampler emits the finite argmax, not token 0"); } + + /* --- NaN at index 0: poisons the max scan itself (the old mx=lo[0] seed), + * the failure mode that starts before the sum (review note on #369) --- */ + { float lo[8]={NAN,1.f,0.5f,6.f,0.2f,-1.f,0.f,0.3f}; /* max finite = idx3 (6.0) */ + dist_build(lo,8); + double sum=0; int nan=0; + for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; } + assert(!nan && "NaN at lo[0] must not poison via the mx seed"); + assert(approx1(sum) && "still normalizes to 1"); + assert(dist_sample(8,-1)==3 && "emits the max finite logit, not token 0"); } + + /* --- all-NaN logits: worst case — must stay finite, no crash --- */ + { float lo[8]; for(int i=0;i<8;i++) lo[i]=NAN; + dist_build(lo,8); + for(int i=0;i<8;i++) assert(g_pbuf[i]==g_pbuf[i] && "all-NaN: g_pbuf stays finite"); } + + /* --- regression: clean logits still produce a valid distribution --- */ + { float lo[8]={0.1f,0.2f,3.0f,0.4f,0.5f,0.6f,0.7f,0.8f}; /* peak = idx2 */ + dist_build(lo,8); + double sum=0; int nan=0; + for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; } + assert(!nan && "clean softmax stays finite"); + assert(approx1(sum) && "clean softmax must sum to 1"); + assert(g_pbuf[2]>=g_pbuf[0] && "peak token keeps the most mass"); } + + printf("OK test_logit_nan: argmax_v NaN-skip + dist_build finite-collapse\n"); + return 0; +} diff --git a/c/tests/test_makefile_platform.py b/c/tests/test_makefile_platform.py index 3c72880..50b4c7a 100644 --- a/c/tests/test_makefile_platform.py +++ b/c/tests/test_makefile_platform.py @@ -35,7 +35,7 @@ class MakefilePlatformTests(unittest.TestCase): env["PATH"] = "" result = subprocess.run( - [MAKE, "--no-print-directory", "-B", "-n", "glm"], + [MAKE, "--no-print-directory", "-B", "-n", "colibri"], cwd=C_DIR, env=env, text=True, @@ -43,7 +43,7 @@ class MakefilePlatformTests(unittest.TestCase): check=True, ) - self.assertIn("-o glm.exe", result.stdout) + self.assertIn("-o colibri.exe", result.stdout) self.assertIn("-fopenmp", result.stdout) self.assertIn("-static", result.stdout) diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py index f8aec86..4cda9ef 100644 --- a/c/tests/test_openai_server.py +++ b/c/tests/test_openai_server.py @@ -380,6 +380,25 @@ class DispatcherTest(unittest.TestCase): engine.close() self.assertEqual(chunks, ["é"]) + def test_records_profile_snapshots_from_prof_lines(self): + def respond(process, frame): + request_id = frame.split()[1] + process.stdout.feed(b"DATA " + request_id + b" 2\nok\n") + process.stdout.feed(b"PROF 2.500 7 12 0.400 0.100 0.900 0.600 0.200 15\n") + process.stdout.feed(b"DONE " + request_id + b" STAT 12 4.8 0 1.0 7 0\n") + + process = FakeProcess(respond) + with patch("openai_server.subprocess.Popen", return_value=process): + engine = Engine("glm", "model") + engine.generate("hello", 16, 0.7, 0.9, lambda _: None) + engine.close() + self.assertEqual(engine.profile_seq, 1) + self.assertEqual(list(engine.profile), [{ + "wall_s": 2.5, "prompt_tokens": 7, "completion_tokens": 12, + "expert_disk_s": 0.4, "expert_wait_s": 0.1, "expert_matmul_s": 0.9, + "attention_s": 0.6, "lm_head_s": 0.2, "forwards": 15, + }]) + def test_cancels_generation_after_consumer_disconnects(self): request_id = None @@ -443,6 +462,20 @@ class HTTPTest(unittest.TestCase): self.assertIn("queued", scheduler) self.assertEqual(health["kv_slots"], 2) + def test_profile_reports_recent_turns_without_auth(self): + with urlopen(self.base + "/profile", timeout=2) as response: + self.assertEqual(json.load(response), {"seq": 0, "turns": []}) + turn = {"wall_s": 2.5, "prompt_tokens": 7, "completion_tokens": 12, + "expert_disk_s": 0.4, "expert_wait_s": 0.1, "expert_matmul_s": 0.9, + "attention_s": 0.6, "lm_head_s": 0.2, "forwards": 15} + self.engine.profile = [turn] + self.engine.profile_seq = 1 + try: + with urlopen(self.base + "/profile", timeout=2) as response: + self.assertEqual(json.load(response), {"seq": 1, "turns": [turn]}) + finally: + del self.engine.profile, self.engine.profile_seq + def test_browser_preflight(self): request = Request(self.base + "/v1/chat/completions", method="OPTIONS", headers={ "Origin": "http://localhost:5173", diff --git a/c/tests/test_openai_tools_e2e.py b/c/tests/test_openai_tools_e2e.py new file mode 100644 index 0000000..d921843 --- /dev/null +++ b/c/tests/test_openai_tools_e2e.py @@ -0,0 +1,182 @@ +"""End-to-end tool-calling test for the OpenAI gateway (#401). + +Unlike the unit tests in test_openai_server.py (which call parse_tool_calls / +render_chat directly), this suite runs openai_server.py as a real subprocess +against a mock engine that speaks the actual SERVE wire protocol +(READY / SUBMIT / DATA / DONE), then talks to it over real HTTP. It pins down +the full path a coding client exercises: tool declaration rendering, marker +suppression in streamed deltas (across chunk boundaries), tool_calls in both +response shapes, and the <|observation|> round trip. +""" +import json +import os +import socket +import subprocess +import sys +import tempfile +import unittest +import urllib.request +from pathlib import Path + +SERVER = Path(__file__).resolve().parent.parent / "openai_server.py" +MODEL_ID = "glm-5.2-colibri" + +# Mock engine: replies are keyed on the prompt so one process covers every case. +# Prompts received are appended to MOCK_LOG for assertions on the rendering. +MOCK_ENGINE = r'''#!/usr/bin/env python3 +import sys, os +out, inp = sys.stdout.buffer, sys.stdin.buffer +out.write(b"\x01\x01READY\x01\x01\n" + b"STAT 0 0 0 0 0\n"); out.flush() + +def reply(rid, text, chunks=1): + data = text.encode("utf-8") + n = max(1, len(data) // chunks) + for i in range(0, len(data), n): + part = data[i:i+n] + out.write(("DATA %s %d\n" % (rid, len(part))).encode() + part + b"\n"); out.flush() + out.write(("DONE %s STAT %d 1.0 50.0 10.0 42 0\n" % (rid, len(text.split()))).encode()) + out.flush() + +while True: + line = inp.readline() + if not line: break + f = line.decode().strip().split() + if not f or f[0] != "SUBMIT": continue + rid, plen = f[1], int(f[3]) + prompt = inp.read(plen).decode("utf-8", "replace"); inp.read(1) + with open(os.environ["MOCK_LOG"], "a") as log: + log.write(prompt + "\n\x00\n") + if "" in prompt: + reply(rid, "25 degrees and sunny in Rome.") + elif "weather in Rome" in prompt: + reply(rid, "get_weathercity" + "Rome") + elif "weather in Milan" in prompt: + # split across many tiny DATA chunks: streamed marker suppression must + # hold even when a marker straddles a chunk boundary + reply(rid, "Checking. get_weathercity" + "Milan", chunks=20) + else: + reply(rid, "Hello from the mock engine.") +''' + +TOOLS = [{"type": "function", "function": { + "name": "get_weather", + "description": "Current weather for a city", + "parameters": {"type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"]}}}] + + +@unittest.skipUnless(os.name == "posix", + "the mock engine is a shebang script the gateway execs directly; " + "Windows CreateProcess cannot run it. The gateway logic under test " + "is platform-independent and covered by the POSIX CI jobs.") +class ToolCallingE2E(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + mock = Path(cls.tmp.name) / "mock_engine.py" + mock.write_text(MOCK_ENGINE) + mock.chmod(0o755) + cls.mock_log = Path(cls.tmp.name) / "prompts.log" + cls.mock_log.touch() + with socket.socket() as probe: # free port, then hand it to the server + probe.bind(("127.0.0.1", 0)) + cls.port = probe.getsockname()[1] + env = dict(os.environ, MOCK_LOG=str(cls.mock_log)) + cls.server = subprocess.Popen( + [sys.executable, str(SERVER), "--model", cls.tmp.name, + "--engine", str(mock), "--port", str(cls.port)], + env=env, stderr=subprocess.DEVNULL) + cls.base = f"http://127.0.0.1:{cls.port}/v1" + for _ in range(100): + try: + urllib.request.urlopen(cls.base + "/models", timeout=2) + return + except OSError: + if cls.server.poll() is not None: + raise RuntimeError("gateway exited during startup") + import time + time.sleep(0.1) + raise RuntimeError("gateway did not come up") + + @classmethod + def tearDownClass(cls): + cls.server.terminate() + cls.server.wait(timeout=5) + cls.tmp.cleanup() + + def post(self, body, stream=False): + req = urllib.request.Request( + self.base + "/chat/completions", json.dumps(body).encode(), + {"Content-Type": "application/json"}) + resp = urllib.request.urlopen(req, timeout=30) + if not stream: + return json.loads(resp.read()) + events = [] + for raw in resp: + line = raw.decode().strip() + if line.startswith("data: ") and line != "data: [DONE]": + events.append(json.loads(line[6:])) + return events + + def test_tool_call_non_stream(self): + r = self.post({"model": MODEL_ID, "tools": TOOLS, + "messages": [{"role": "user", + "content": "What is the weather in Rome?"}]}) + choice = r["choices"][0] + self.assertEqual(choice["finish_reason"], "tool_calls") + calls = choice["message"]["tool_calls"] + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0]["function"]["name"], "get_weather") + self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"city": "Rome"}) + self.assertNotIn("", choice["message"].get("content") or "") + + def test_tool_call_streamed_markers_suppressed(self): + events = self.post({"model": MODEL_ID, "tools": TOOLS, "stream": True, + "messages": [{"role": "user", + "content": "What is the weather in Milan?"}]}, + stream=True) + deltas = [e["choices"][0]["delta"] for e in events if e["choices"]] + text = "".join(d.get("content") or "" for d in deltas) + self.assertNotIn("", text) + self.assertNotIn("", text) + calls = [d["tool_calls"] for d in deltas if d.get("tool_calls")] + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0][0]["function"]["name"], "get_weather") + self.assertEqual(json.loads(calls[0][0]["function"]["arguments"]), + {"city": "Milan"}) + finish = [e["choices"][0]["finish_reason"] for e in events + if e["choices"] and e["choices"][0].get("finish_reason")] + self.assertEqual(finish, ["tool_calls"]) + + def test_tool_result_round_trip(self): + r = self.post({"model": MODEL_ID, "tools": TOOLS, "messages": [ + {"role": "user", "content": "What is the weather in Rome?"}, + {"role": "assistant", "content": None, "tool_calls": [ + {"id": "call_x", "type": "function", + "function": {"name": "get_weather", + "arguments": "{\"city\": \"Rome\"}"}}]}, + {"role": "tool", "tool_call_id": "call_x", + "content": "25 degrees, sunny"}]}) + choice = r["choices"][0] + self.assertEqual(choice["finish_reason"], "stop") + self.assertFalse(choice["message"].get("tool_calls")) + self.assertIn("25 degrees", choice["message"]["content"]) + rendered = self.mock_log.read_text().split("\x00")[-2] + self.assertIn("<|observation|>25 degrees, sunny", + rendered) + self.assertIn("# Tools", rendered) + self.assertIn('"get_weather"', rendered) + + def test_no_tools_plain_text(self): + r = self.post({"model": MODEL_ID, + "messages": [{"role": "user", "content": "Hi!"}]}) + choice = r["choices"][0] + self.assertEqual(choice["finish_reason"], "stop") + self.assertIn("mock engine", choice["message"]["content"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_ragged_attention.cu b/c/tests/test_ragged_attention.cu new file mode 100644 index 0000000..dd1e2f8 --- /dev/null +++ b/c/tests/test_ragged_attention.cu @@ -0,0 +1,39 @@ +#include "../backend_cuda.h" + +#include +#include +#include + +int main(){ + int dev=0;if(!coli_cuda_init(&dev,1))return 77; + constexpr int S=3,H=2,Q=2,R=1,V=2,K=3,D=H*V,O=3,T=3; + std::vector w(H*(Q+V)*K),p(O*D),q(S*H*(Q+R)); + for(size_t i=0;i> l(S),r(S); + const float *lp[S],*rp[S]; + const void *keys[S]; + for(int s=0;s0` e' falso su NaN ovunque). + * Ora la distribuzione degenere ripiega sull'argmax dei logit FINITI e avvisa una volta. + * + * Il test verifica: (a) con logit sani il campionamento resta corretto; (b) con NaN/+Inf + * iniettato il token scelto e' l'argmax dei FINITI (mai 0 per default), su ogni posizione + * del NaN inclusa lo[0]; (c) nessun NaN sopravvive in g_pbuf. */ +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main +#include + +static uint32_t rs=0x1234abcd; static uint32_t xr(){rs^=rs<<13;rs^=rs>>17;rs^=rs<<5;return rs;} + +static int pbuf_has_nan(int V){ for(int i=0;i 0 e nessun NaN nel buffer */ + for(int i=0;ipv){pv=g_pbuf[i];picked=i;} + if(b<2 && picked!=amax){ /* NaN e +Inf: delta esatto su amax */ + printf(" FAIL: %s @ %d -> picked %d, atteso argmax finito %d\n",bn[b],at,picked,amax); fail=1; + } + } + } + if(!fail) printf(" NaN/+Inf iniettato: argmax dei finiti vince, mai 0/NaN ok\n"); + + /* (c) caso estremo: TUTTI non finiti -> non deve crashare, buffer valido */ + for(int i=0;i +#include +#include +#ifndef _WIN32 +#include +#include +#endif + +#include "../st.h" + +#define CHECK(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "%s:%d: check failed: %s\n", __FILE__, __LINE__, #condition); \ + return 1; \ + } \ +} while (0) + +static void write_snap(const char *dir, int truncate_bytes) { + char path[512]; + snprintf(path, sizeof(path), "%s/model.safetensors", dir); + unsigned char data[96]; + for (int i = 0; i < 96; i++) data[i] = (unsigned char)(i * 7 + 3); + const char *hdr = "{\"t\":{\"dtype\":\"U8\",\"shape\":[96],\"data_offsets\":[0,96]}}"; + uint64_t hlen = strlen(hdr); + FILE *f = fopen(path, "wb"); + fwrite(&hlen, 8, 1, f); + fwrite(hdr, 1, hlen, f); + fwrite(data, 1, (size_t)(96 - truncate_bytes), f); + fclose(f); +} + +int main(void) { + /* relative to the CWD, per test_stops: MinGW .exe files resolve Windows + * paths and "/tmp" is not one */ + char dir[] = "test_st_pread_XXXXXX"; + if (!mkdtemp(dir)) { perror("mkdtemp"); return 1; } + + /* 1) chunk loop: 96-byte tensor read 7 bytes at a time, content exact */ + write_snap(dir, 0); + shards S; st_init(&S, dir); + unsigned char out[96] = {0}; + st_read_raw(&S, "t", out, 0); + for (int i = 0; i < 96; i++) CHECK(out[i] == (unsigned char)(i * 7 + 3)); + +#ifndef _WIN32 + /* 2) shard truncated AFTER st_init (init validates static bounds, so the + * pread path only fires when the file shrinks underneath a live handle): + * child must exit(1) with an honest message, not perror's "Success" */ + char shard[512]; snprintf(shard, sizeof(shard), "%s/model.safetensors", dir); + struct stat sb; CHECK(stat(shard, &sb) == 0); + CHECK(truncate(shard, sb.st_size - 40) == 0); + int pipefd[2]; CHECK(pipe(pipefd) == 0); + pid_t pid = fork(); CHECK(pid >= 0); + if (pid == 0) { + dup2(pipefd[1], 2); close(pipefd[0]); close(pipefd[1]); + unsigned char buf[96]; + st_read_raw(&S, "t", buf, 0); /* inherited handles; must exit(1) inside */ + _exit(42); /* reaching here = bug */ + } + close(pipefd[1]); + char err[512] = {0}; + ssize_t n = read(pipefd[0], err, sizeof(err)-1); (void)n; + close(pipefd[0]); + int status = 0; waitpid(pid, &status, 0); + CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 1); + CHECK(strstr(err, "short read") != NULL); + CHECK(strstr(err, "Success") == NULL); +#else + /* fork/pipe/truncate are POSIX; Windows still runs the chunk-loop check */ + printf("test_st_pread: truncation subtest skipped on Windows\n"); +#endif + + char cmd[600]; +#ifdef _WIN32 + snprintf(cmd, sizeof(cmd), "rmdir /s /q %s", dir); +#else + snprintf(cmd, sizeof(cmd), "rm -rf %s", dir); +#endif + if (system(cmd)) {} + printf("test_st_pread: chunk loop + honest truncation error: ok\n"); + return 0; +} diff --git a/c/tests/test_stops.c b/c/tests/test_stops.c new file mode 100644 index 0000000..5e7e9d2 --- /dev/null +++ b/c/tests/test_stops.c @@ -0,0 +1,136 @@ +/* Stop-token arming: the engine must not depend on a converted checkpoint's + * config.json being complete. + * + * Why this test exists (#298, #307): GLM-5.2 declares THREE eos ids + * (<|endoftext|>, <|user|>, <|observation|>). A conversion tool that rewrites + * config.json with a reduced eos_token_id list leaves the engine stopping on + * fewer tokens than the model actually emits -- and the ones it misses get + * DETOKENIZED AND PRINTED INTO THE CHAT as literal text, with generation + * continuing past the real end of the turn. @woolcoxm hit exactly this on a + * g64 checkpoint and confirmed it by comparing token ids. + * + * Two independent defenses, one test each: + * 1. eos_token_id is unioned from generation_config.json (HuggingFace's + * authority for generation) as well as config.json. + * 2. every added-token the TOKENIZER marks "special":true is a stop, whatever + * the configs say. Those are control tokens (<|user|>, , ...) and are + * never legitimate content. / are "special":false and + * must NOT be swept up -- they are real output. + * + * Defense 2 is what makes this robust against checkpoints we don't control: + * even with BOTH configs mutilated, a control token cannot leak into a reply. */ +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main + +static const char *TOKJSON = +"{\"model\":{\"vocab\":{\"a\":0,\"b\":1,\"c\":2},\"merges\":[[\"a\",\"b\"]]}," +" \"added_tokens\":[" +" {\"id\":100,\"content\":\"<|endoftext|>\",\"special\":true}," +" {\"id\":101,\"content\":\"<|user|>\",\"special\":true}," +" {\"id\":102,\"content\":\"<|observation|>\",\"special\":true}," +" {\"id\":103,\"content\":\"<|assistant|>\",\"special\":true}," +" {\"id\":104,\"content\":\"\",\"special\":true}," +" {\"id\":110,\"content\":\"\",\"special\":false}," +" {\"id\":111,\"content\":\"\",\"special\":false}" +"]}"; + +/* minimal config.json that survives load_cfg's range validation */ +static void write_cfg(const char *dir, const char *fname, const char *eos_json){ + char p[512]; snprintf(p,sizeof(p),"%s/%s",dir,fname); + FILE *f=fopen(p,"w"); if(!f){ perror(p); exit(1); } + fprintf(f,"{\"hidden_size\":64,\"num_hidden_layers\":2,\"num_attention_heads\":4," + "\"n_routed_experts\":8,\"num_experts_per_tok\":2,\"moe_intermediate_size\":32," + "\"intermediate_size\":64,\"first_k_dense_replace\":1,\"q_lora_rank\":0," + "\"kv_lora_rank\":16,\"qk_nope_head_dim\":8,\"qk_rope_head_dim\":8," + "\"v_head_dim\":8,\"n_shared_experts\":1,\"vocab_size\":200," + "\"n_group\":1,\"topk_group\":1,\"rope_theta\":10000.0"); + if(eos_json) fprintf(f,",\"eos_token_id\":%s",eos_json); + fprintf(f,"}\n"); fclose(f); +} +static void write_tok(const char *dir){ + char p[512]; snprintf(p,sizeof(p),"%s/tokenizer.json",dir); + FILE *f=fopen(p,"w"); if(!f){ perror(p); exit(1); } + fputs(TOKJSON,f); fclose(f); +} +static void rm_file(const char *dir, const char *fname){ + char p[512]; snprintf(p,sizeof(p),"%s/%s",dir,fname); remove(p); +} + +static int expect(const char *what, int id, int want){ + int got=is_stop(id); + if(got!=want){ fprintf(stderr," FAIL %s: token %d stop=%d, expected %d\n",what,id,got,want); return 1; } + return 0; +} + +int main(void){ + int fail=0; + /* Relative to the CWD, like test_compat_direct's TMPF — NOT "/tmp/...". + * These binaries are built by MinGW into native Windows .exe files, which + * resolve Windows paths: "/tmp" is not one, so mkdtemp there fails ENOENT + * and the whole `make check` goes red on the windows job (and only there). */ + char dir[]="test_stops_XXXXXX"; + if(!mkdtemp(dir)){ perror("mkdtemp"); return 1; } + write_tok(dir); + char tkp[512]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",dir); + Tok T; tok_load(&T,tkp); + printf("test_stops: stop arming vs incomplete checkpoint metadata\n"); + + /* 1. the tokenizer's "special" flag must survive loading, and must NOT + * swallow /, which are real content. */ + if(!T.id_special[101]){ fprintf(stderr," FAIL <|user|> not marked special\n"); fail=1; } + if(!T.id_special[104]){ fprintf(stderr," FAIL not marked special\n"); fail=1; } + if(T.id_special[110]){ fprintf(stderr," FAIL wrongly marked special\n"); fail=1; } + if(T.id_special[111]){ fprintf(stderr," FAIL wrongly marked special\n"); fail=1; } + if(!fail) printf(" tokenizer: special flag parsed, / excluded ok\n"); + + /* 2. config.json mutilated to one eos, generation_config.json intact: + * the union must recover all three. This is @woolcoxm's case. */ + { Cfg c; memset(&c,0,sizeof c); + write_cfg(dir,"config.json","[100]"); + write_cfg(dir,"generation_config.json","[100,101,102]"); + load_cfg(&c,dir); + if(c.n_stop!=3){ fprintf(stderr," FAIL union: expected 3 stops, got %d\n",c.n_stop); fail=1; } + else printf(" config eos=[100] + generation_config=[100,101,102] -> 3 stops ok\n"); } + + /* 3. no generation_config.json at all: must not crash, config alone wins */ + { Cfg c; memset(&c,0,sizeof c); + write_cfg(dir,"config.json","[100,101,102]"); + rm_file(dir,"generation_config.json"); + load_cfg(&c,dir); + if(c.n_stop!=3){ fprintf(stderr," FAIL no-genconfig: expected 3, got %d\n",c.n_stop); fail=1; } + else printf(" generation_config.json absent -> config alone, no crash ok\n"); } + + /* 4. BOTH configs mutilated: the tokenizer's special set must still stop the + * turn on every control token, and must still leave alone. */ + { Cfg c; memset(&c,0,sizeof c); + write_cfg(dir,"config.json","[100]"); + write_cfg(dir,"generation_config.json","[100]"); + load_cfg(&c,dir); + stops_arm_tok(&c,-1,&T); + fail|=expect("endoftext (config)", 100,1); + fail|=expect("<|user|> (tokenizer)", 101,1); + fail|=expect("<|observation|>", 102,1); + fail|=expect("<|assistant|>", 103,1); + fail|=expect("", 104,1); + fail|=expect(" must NOT stop",110,0); + fail|=expect(" must NOT stop",111,0); + fail|=expect("plain token must NOT stop",2,0); + if(!fail) printf(" both configs mutilated -> tokenizer still stops all 5 control tokens ok\n"); } + + /* 5. T=NULL (validation/oracle path): config stops only, unchanged behaviour */ + { Cfg c; memset(&c,0,sizeof c); + write_cfg(dir,"config.json","[100]"); + rm_file(dir,"generation_config.json"); + load_cfg(&c,dir); + stops_arm_tok(&c,-1,NULL); + fail|=expect("T=NULL: config stop",100,1); + fail|=expect("T=NULL: no tokenizer sweep",101,0); + if(!fail) printf(" T=NULL -> config stops only (validation path untouched) ok\n"); } + + rm_file(dir,"config.json"); rm_file(dir,"generation_config.json"); rm_file(dir,"tokenizer.json"); + rmdir(dir); + if(fail){ printf("test_stops: FAIL\n"); return 1; } + printf("test_stops: ok\n"); + return 0; +} diff --git a/c/tests/test_topp.c b/c/tests/test_topp.c new file mode 100644 index 0000000..49f45d6 --- /dev/null +++ b/c/tests/test_topp.c @@ -0,0 +1,280 @@ +/* Top-p (nucleus) truncation in dist_build: the partial-select rewrite (#335) must be + * indistinguishable from the old full-vocab qsort for every shape dist_sample can see. + * + * Why this test exists (#335): dist_build() previously qsort-ed the entire 151936-entry + * vocab on every sampled token to find the few-hundred-token head whose cumulative mass + * reaches g_nuc. It now heapifies (O(V)) and pops only the head (k * O(log V)). The win + * is structural; the risk is a silent sampling-distribution change, because the contract + * is subtle: + * + * dist_sample() iterates g_pbuf[0..V-1] BY TOKEN ID and sums probabilities directly. + * So dist_build MUST leave g_pbuf indexed by id (never reordered) AND must zero every + * truncated tail entry -- merely excluding the tail from the head would leave mass on + * it and the sampled distribution would drift with no crash and no error. + * + * Strategy: drive the REAL dist_build (via the test_stops.c include-glm.c pattern) on a + * sweep of distributions and g_nuc values, and compare against an INDEPENDENT reference + * that re-implements the OLD algorithm (full qsort + zero-tail + renorm) in double on a + * private buffer. On shapes with no ties the renormalized head must be BIT-IDENTICAL to + * the reference (the issue's stated invariant: s2 accumulates in the same descending + * order). On tie shapes, where the unstable qsort already left ordering unspecified, we + * check multiset equality instead. Every shape also checks: exact-zero tails, head sums + * to 1.0, and a sane keep-count. + * + * No scratch files: the test runs entirely in memory (no mkdtemp), so it builds clean on + * the Windows MinGW CI job without the unmerged compat shim (#352). */ +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main + +#include + +static int g_nfails = 0; + +/* pointer set by ref_build so cmp_ref_desc can read the current reference buffer + * (the qsort comparator gets no user-data argument in C). */ +static const double *g_ref_p = NULL; + +#define FAIL(fmt, ...) do { \ + fprintf(stderr, " FAIL [%s V=%d nuc=%.3f shape=%s]: " fmt "\n", \ + label, V, nuc, shape_name, ##__VA_ARGS__); \ + g_nfails++; \ + return; \ +} while (0) + +/* ---- independent reference: the OLD algorithm, in double, on a private buffer ------- */ +/* Stable qsort by descending probability (ties broken by ascending index, which makes + * the reference deterministic regardless of the production comparator). */ +static int cmp_ref_desc(const void *a, const void *b){ + double pa = ((const double *)g_ref_p)[*(const int*)a]; + double pb = ((const double *)g_ref_p)[*(const int*)b]; + if (pa < pb) return 1; + if (pa > pb) return -1; + /* tie -> lower index first (stable, unlike the production comparator) */ + return *(const int*)a - *(const int*)b; +} + +/* Build the reference distribution into out[0..V-1] (indexed by token id), mirroring the + * old dist_build: softmax(lo/temp) truncated to top-p nuc, tail zeroed, head renormalized. + * Returns the keep-count through *keep_out. */ +static void ref_build(const float *lo, int V, double temp, double nuc, + double *out, int *pidx, int *keep_out){ + double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i]; + double s = 0, invt = 1.0 / (temp > 1e-4 ? temp : 1e-4); + for (int i = 0; i < V; i++){ out[i] = exp((lo[i]-mx)*invt); s += out[i]; } + for (int i = 0; i < V; i++) out[i] /= s; + + if (nuc > 0 && nuc < 1.0){ + for (int i = 0; i < V; i++) pidx[i] = i; + qsort(pidx, V, sizeof(int), cmp_ref_desc); + double cum = 0; int keep = V; + for (int i = 0; i < V; i++){ cum += out[pidx[i]]; if (cum >= nuc){ keep = i+1; break; } } + double s2 = 0; + for (int i = keep; i < V; i++) out[pidx[i]] = 0; + for (int i = 0; i < keep; i++) s2 += out[pidx[i]]; + for (int i = 0; i < keep; i++) out[pidx[i]] /= s2; + *keep_out = keep; + } else { + *keep_out = V; + } +} + +/* count how many production g_pbuf entries are non-zero == the head size */ +static int head_count(int V){ + int n = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) n++; return n; +} + +/* Run one case: load logits into g_pbuf via the real dist_build, compare to reference. + * shape_name is for diagnostics only. */ +static void check_case(const char *label, int V, double nuc, const char *shape_name, + const float *lo){ + /* reference on a private buffer */ + double *ref = malloc((size_t)V * sizeof(double)); + int *ridx = malloc((size_t)V * sizeof(int)); + int ref_keep = 0; + g_ref_p = ref; /* cmp_ref_desc reads this */ + ref_build(lo, V, g_temp, nuc, ref, ridx, &ref_keep); + + /* production: drive the real dist_build (writes the global g_pbuf) */ + g_nuc = (float)nuc; + dist_build(lo, V); + + int got_keep = head_count(V); + + /* 1. keep-count must match the reference exactly. The partial select and the old + * qsort keep the same NUMBER of tokens by construction (same cumulative-mass rule); + * a count divergence is a real bug, not a tie artifact. */ + if (got_keep != ref_keep) + FAIL("keep-count mismatch: got %d, ref %d", got_keep, ref_keep); + + /* 2. Detect ties across the WHOLE pre-truncation distribution, not just the kept set. + * A tie at the head/tail boundary makes which-side-a-token-lands-on interchangeable: + * both algorithms keep the right count but may keep different MEMBERS. So any input + * with a duplicated softmax value needs the relaxed multiset comparison below. We + * detect this on the reference softmax (pre-truncation) by sorting all V values. */ + int has_ties = 0; + { + double *all = malloc((size_t)V * sizeof(double)); + /* reconstruct the pre-truncation softmax the same way ref_build does */ + double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i]; + double s = 0, invt = 1.0 / (g_temp > 1e-4 ? g_temp : 1e-4); + for (int i = 0; i < V; i++){ all[i] = exp((lo[i]-mx)*invt); s += all[i]; } + for (int i = 0; i < V; i++) all[i] /= s; + for (int a = 1; a < V; a++){ double k = all[a]; int b = a-1; + while (b >= 0 && all[b] > k){ all[b+1] = all[b]; b--; } all[b+1] = k; } + for (int a = 1; a < V; a++) if (all[a] == all[a-1]){ has_ties = 1; break; } + free(all); + } + + if (has_ties){ + /* Multiset equality of the non-zero (head) values. Ties make membership + * interchangeable, so we compare sorted value-multisets, not id-aligned values. + * Tolerance is 1e-6 relative -- the engine uses float arithmetic, the reference + * double, so sub-ULP noise is expected (matches test_i4_grouped.c's convention). */ + double *got = malloc((size_t)ref_keep * sizeof(double)); + int gm = 0; + for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) got[gm++] = (double)g_pbuf[i]; + if (gm != ref_keep) + FAIL("tie-shape head size mismatch: got %d non-zero, ref %d", gm, ref_keep); + for (int a = 1; a < gm; a++){ double k = got[a]; int b = a-1; + while (b >= 0 && got[b] > k){ got[b+1] = got[b]; b--; } got[b+1] = k; } + double *rsort = malloc((size_t)ref_keep * sizeof(double)); + int rm = 0; + for (int i = 0; i < V; i++) if (ref[i] != 0.0) rsort[rm++] = ref[i]; + for (int a = 1; a < rm; a++){ double k = rsort[a]; int b = a-1; + while (b >= 0 && rsort[b] > k){ rsort[b+1] = rsort[b]; b--; } rsort[b+1] = k; } + int mm = 0; double worst = 0; + for (int i = 0; i < gm; i++){ + double d = fabs(got[i] - rsort[i]); + double rel = rsort[i] > 1e-30 ? d / rsort[i] : d; + if (rel > worst) worst = rel; + if (rel > 1e-6) mm++; + } + free(got); free(rsort); + if (mm) FAIL("tie-shape multiset mismatch: %d/%d head values differ beyond 1e-6 rel (worst %.3g)", + mm, ref_keep, worst); + } else { + /* No ties anywhere: membership is forced, so compare id-aligned head values. The + * engine computes in float (g_pbuf /= (float)s2) while the reference uses double, + * so the comparison is relative-tolerance (1e-6), not bit-exact -- the partial + * select and qsort accumulate s2 in the same descending order, so any difference + * is pure float-rounding noise, not an ordering bug. */ + int bad = 0; int first_id = -1; float gv = 0, rv = 0; double worst = 0; + for (int i = 0; i < V; i++){ + if (ref[i] == 0.0) continue; /* tail */ + float want = (float)ref[i]; + double d = fabs((double)g_pbuf[i] - (double)want); + double rel = fabs((double)want) > 1e-30 ? d / fabs((double)want) : d; + if (rel > worst) worst = rel; + if (rel > 1e-6){ + bad++; if (first_id < 0){ first_id = i; gv = g_pbuf[i]; rv = want; } + if (bad > 3) break; + } + } + if (bad) + FAIL("head not within 1e-6 rel of reference: %d entries differ (first id %d: got %.9g want %.9g, worst %.3g)", + bad, first_id, (double)gv, (double)rv, worst); + } + + /* 3. head must renormalize to 1.0 (within float epsilon) */ + double sum = 0; for (int i = 0; i < V; i++) sum += g_pbuf[i]; + if (fabs(sum - 1.0) > 1e-5) + FAIL("head does not sum to 1.0: sum=%.12g (keep=%d)", sum, got_keep); + + free(ref); free(ridx); + printf(" ok [V=%d nuc=%.3f shape=%s keep=%d%s sum=%.10f]\n", + V, nuc, shape_name, got_keep, has_ties ? " (ties)" : "", sum); +} +#undef FAIL + +/* deterministic xorshift32 RNG (matches the test_i4_grouped.c convention) */ +static uint32_t rng_state = 0x12345678u; +static uint32_t xr(void){ rng_state ^= rng_state << 13; rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; return rng_state; } +static double frand(void){ return (xr() >> 8) * (1.0 / 16777216.0); } /* [0,1) */ + +/* fill logits for a given shape. Shapes chosen to stress the comparator and the head/tail + * boundary differently. */ +static void fill_shape(float *lo, int V, int shape){ + switch (shape){ + case 0: /* uniform -> every token equal probability -> massive tie plateau */ + for (int i = 0; i < V; i++) lo[i] = 0.f; break; + case 1: /* peaked: one dominant token, rest small and distinct (no ties). + * The fixed hot-spots are clamped to V-1 so small V (incl. V=1) doesn't + * write out of bounds and corrupt heap metadata on the later free(lo). */ + for (int i = 0; i < V; i++) lo[i] = (float)(-1.0 - frand()*4.0); + lo[0] = 3.f; lo[V/3 comparator tie handling */ + for (int i = 0; i < V; i++) lo[i] = (float)(-(double)(i / 7)); /* 7-wide plateaus */ + break; + case 4: /* sharp-tail: a few hot, then a long flat floor (small tie at the floor). + * Hot count is min(12,V) so V<12 (incl. V=1) stays in bounds. */ + for (int i = 0; i < V; i++) lo[i] = -8.f; + { int hot = V<12 ? V : 12; for (int i = 0; i < hot; i++) lo[i] = (float)(2.0 - frand()); } break; + } +} + +int main(void){ + /* sizes: small for exhaustive tie detection up to near-production scale */ + int sizes[] = {1, 2, 8, 64, 257, 1519}; /* 1519 ~= V/100 of GLM-5.2 */ + double nucs[] = {0.001, 0.5, 0.9, 0.999}; /* tight -> almost-everything */ + int n_shapes = 5; + + /* temperature used by dist_build: pick a normal serving value */ + g_temp = 0.7f; + + int cases = 0; + for (size_t si = 0; si < sizeof(sizes)/sizeof(sizes[0]); si++){ + int V = sizes[si]; + /* dist_build allocates g_pbuf/g_pidx ONCE and reuses them (single-V invariant in + * real serving, where V is the constant model vocab). This sweep varies V, so free + * and force a reallocation per size -- otherwise a later, larger V would overflow + * the buffer sized for the first (smallest) V. */ + free(g_pbuf); g_pbuf = NULL; free(g_pidx); g_pidx = NULL; + float *lo = malloc((size_t)V * sizeof(float)); + for (int shape = 0; shape < n_shapes; shape++){ + fill_shape(lo, V, shape); + for (size_t ni = 0; ni < sizeof(nucs)/sizeof(nucs[0]); ni++){ + char label[32]; snprintf(label, sizeof(label), "size[%zu]/shape[%d]", si, shape); + const char *sn = (const char*[]){"uniform","peaked","geometric","plateau","sharptail"}[shape]; + check_case(label, V, nucs[ni], sn, lo); + cases++; + } + } + free(lo); + } + + /* guard-off path: g_nuc >= 1 must skip truncation entirely (full softmax kept) */ + { + int V = 256; float lo[256]; + for (int i = 0; i < V; i++) lo[i] = (float)(frand()*4 - 2); + g_nuc = 1.0f; dist_build(lo, V); + int nz = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) nz++; + if (nz != V){ fprintf(stderr, " FAIL [guard-off nuc=1.0]: %d/%d entries kept, expected all\n", nz, V); g_nfails++; } + else printf(" ok [guard-off nuc=1.0 keep=%d]\n", nz); + cases++; + + g_nuc = 0.0f; dist_build(lo, V); + nz = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) nz++; + if (nz != V){ fprintf(stderr, " FAIL [guard-off nuc=0.0]: %d/%d entries kept, expected all\n", nz, V); g_nfails++; } + else printf(" ok [guard-off nuc=0.0 keep=%d]\n", nz); + cases++; + } + + /* extreme tie edge case: V=1, single token -> keep=1 regardless of nuc */ + { + float lo[1] = {5.f}; + g_nuc = 0.5f; dist_build(lo, 1); + if (g_pbuf[0] == 0.f || !(fabs((double)g_pbuf[0] - 1.0) < 1e-6)){ + fprintf(stderr, " FAIL [V=1]: g_pbuf[0]=%.9g, expected 1.0\n", (double)g_pbuf[0]); g_nfails++; + } else printf(" ok [V=1 keep=1]\n"); + cases++; + } + + printf("\ntest_topp: %d cases run, %d failure(s)\n", cases, g_nfails); + if (g_nfails){ printf("test_topp: FAIL\n"); return 1; } + printf("test_topp: ok\n"); + return 0; +} diff --git a/c/tests/test_uring.c b/c/tests/test_uring.c index e64b9f2..625c79b 100644 --- a/c/tests/test_uring.c +++ b/c/tests/test_uring.c @@ -6,7 +6,7 @@ #include #include #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static int fail(const char *s){ fprintf(stderr,"FAIL: %s\n",s); return 1; } diff --git a/c/tok.h b/c/tok.h index f61011e..7d1140b 100644 --- a/c/tok.h +++ b/c/tok.h @@ -42,6 +42,11 @@ typedef struct { hmap vocab; /* stringa byte-level -> id */ hmap merges; /* "left\0right" -> rank */ char **id2str; int *id_added; int n_ids; /* id -> stringa; id_added=1 se added-token (output letterale) */ + int *id_special; /* 1 = added-token con "special":true nel tokenizer: + * token di CONTROLLO (<|user|>, <|assistant|>, , ...), + * mai contenuto legittimo di una risposta. Distinto da + * id_added, che copre anche / ("special" + * false), i quali sono testo vero e vanno renderizzati. */ Special *sp; int nsp; /* added tokens, ordinati per lunghezza decrescente */ uint32_t byte2cp[256]; int byte2cp_len[256]; char byte2str[256][3]; int16_t cp2byte[1024]; @@ -106,6 +111,7 @@ static void tok_load(Tok *T, const char *path){ T->n_ids=maxid+1; T->id2str=calloc(T->n_ids,sizeof(char*)); T->id_added=calloc(T->n_ids,sizeof(int)); + T->id_special=calloc(T->n_ids,sizeof(int)); /* vocab: stringa -> id (capacita' potenza di 2, ~2-3x) */ int vc=1; while(vc < vocab->len*2) vc<<=1; @@ -133,6 +139,8 @@ static void tok_load(Tok *T, const char *path){ char *content=json_get(a,"content")->str; int id=(int)json_get(a,"id")->num; T->sp[i].str=content; T->sp[i].len=(int)strlen(content); T->sp[i].id=id; T->id2str[id]=content; T->id_added[id]=1; + jval *sf=json_get(a,"special"); /* "special": true/false */ + if(sf && sf->t==J_BOOL && sf->boolean) T->id_special[id]=1; } qsort(T->sp,T->nsp,sizeof(Special),cmp_sp_len); /* match piu' lungo per primo */ } diff --git a/c/tools/benchmark_cuda_fixture.py b/c/tools/benchmark_cuda_fixture.py index bced18e..26512dd 100644 --- a/c/tools/benchmark_cuda_fixture.py +++ b/c/tools/benchmark_cuda_fixture.py @@ -17,6 +17,12 @@ PROFILE_RE = re.compile( r"\| attention ([0-9.]+)s .* lm_head ([0-9.]+)s \| other ([0-9.-]+)s" ) PROFILE_KEYS = ("disk", "expert_matmul", "attention", "lm_head", "other") +P0_RE = re.compile( + r"P0-EXEC: routed CPU ([0-9.]+)s / ([0-9.]+) GB/s \(([0-9]+) row\) \| routed GPU critical ([0-9.]+)s \| " + r"router ([0-9.]+)s \| residual P2P ([0-9.]+)s / ([0-9]+) hop \| orchestration ([0-9.]+)s" +) +P0_KEYS = ("routed_cpu", "routed_cpu_gb_s", "routed_cpu_rows", "routed_gpu_critical", + "router", "p2p", "p2p_hops", "orchestration") def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]: @@ -30,11 +36,20 @@ def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]: return float(speed.group(1)), [disk] + [float(value) for value in rest] -def execute(engine: str, env: dict[str, str]) -> tuple[float, list[float]]: +def parse_p0(stdout: str) -> list[float]: + """Extract the optional PROF=1 execution-layer breakdown.""" + row = P0_RE.search(stdout) + if not row: + raise RuntimeError("benchmark output missing P0-EXEC profile") + return [float(value) for value in row.groups()] + + +def execute(engine: str, env: dict[str, str]) -> tuple[float, list[float], list[float]]: run = subprocess.run( [engine, "4", "4", "4"], env=env, text=True, capture_output=True, check=True ) - return parse_output(run.stdout, run.stderr) + speed, profile = parse_output(run.stdout, run.stderr) + return speed, profile, parse_p0(run.stdout) def main() -> None: @@ -63,6 +78,8 @@ def main() -> None: OMP_NUM_THREADS=str(args.threads), OMP_PROC_BIND="spread", OMP_PLACES="cores", + DRAFT="0", + PROF="1", ) execute(args.engine, base | {"STATS": str(stats)}) @@ -86,13 +103,15 @@ def main() -> None: execute(args.engine, base | extra) # warm-up speeds = {name: [] for name in modes} profiles = {name: [] for name in modes} + p0_profiles = {name: [] for name in modes} names = list(modes) for run_index in range(args.runs): order = names[run_index % len(names):] + names[:run_index % len(names)] for name in order: - speed, profile = execute(args.engine, base | modes[name]) + speed, profile, p0 = execute(args.engine, base | modes[name]) speeds[name].append(speed) profiles[name].append(profile) + p0_profiles[name].append(p0) result = {} for name in names: @@ -103,6 +122,10 @@ def main() -> None: key: statistics.median(row[index] for row in profiles[name]) for index, key in enumerate(PROFILE_KEYS) }, + "median_p0": { + key: statistics.median(row[index] for row in p0_profiles[name]) + for index, key in enumerate(P0_KEYS) + }, } print(json.dumps(result, indent=2)) diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py index 382125e..eac51f9 100644 --- a/c/tools/convert_fp8_to_int4.py +++ b/c/tools/convert_fp8_to_int4.py @@ -286,6 +286,17 @@ def main(): # testa MTP a int4 = acceptance ~0-4% (misurato, issue #8): il draft sbaglia sempre # e la speculazione non parte mai. A int8: 39-59%, 2.2-2.8 token/forward. a.ebits = 8 if (a.mtp or a.indexer) else 4 + if a.mtp and a.ebits < 8 and a.group_size <= 0: + # Non solo lossy: eh_proj ha ~20-30x di asimmetria di scala fra le due meta' di + # colonna, quindi l'int4 per-riga (UNA scala per riga) arrotonda a ZERO l'intera + # meta' embedding -> il draft non vede il token -> acceptance ~0% (issue #8). + # EN: not merely lossy: eh_proj has ~20-30x column-scale asymmetry, so per-row + # EN: int4 rounds its ENTIRE embedding half to exact zeros -> the draft cannot + # EN: see the input token -> ~0% acceptance (issue #8). A container converted + # EN: this way is repairable in place with tools/repair_mtp_int8.py. + print(f"WARNING: --mtp with --ebits {a.ebits} and per-row scales ZEROES eh_proj's " + "embedding half -> MTP acceptance ~0% (issue #8). Use the default --ebits 8, " + "or add --group-size 128 for group-scaled int4.") if a.xbits is None: a.xbits = a.ebits # Build per-type bits map. If a type-specific arg is set, use it; otherwise the @@ -299,6 +310,16 @@ def main(): if bits_map: print(f"[MIXED] precision map: " + ", ".join(f"{k}={v}bit" for k,v in sorted(bits_map.items()))) + # Il PIANO risolto, PRIMA di toccare qualunque cosa (#383): --mtp/--indexer cambiano il + # default di ebits a 8 (testa int4 = acceptance ~0%, issue #8) e il ramo grouped e' + # gated su bits<=4 — combinazioni sorprendenti devono mostrarsi al secondo 1 di un job + # da ore, non nel size-check dopo. EN: print the RESOLVED plan before doing anything. + mode = "MTP head only" if a.mtp else "DSA indexer only" if a.indexer else "main model" + grp = f"grouped gs={a.group_size} (fmt=4)" if (a.group_size and a.ebits <= 4) else \ + (f"PER-ROW (grouped branch needs bits<=4; ebits={a.ebits} disables it)" if a.group_size else "per-row") + print(f"[PLAN] mode: {mode} | source: {'local ' + a.indir if a.indir else 'download ' + a.repo} | " + f"experts {a.ebits}-bit, embed/lm_head {a.io_bits}-bit, x {a.xbits}-bit | {grp}") + if a.selftest_nvfp4: import torch # 1) LUT e2m1: i 16 codici devono decodificare esattamente ai valori attesi. @@ -379,14 +400,99 @@ def main(): if a.indir: # conversione locale (test) shards = sorted(glob.glob(os.path.join(a.indir, "*.safetensors"))) from safetensors.numpy import save_file + # #383: se l'indice c'e', i passaggi --mtp/--indexer convertono SOLO gli shard + # che contengono i tensori richiesti (3 invece di scandire tutti i 141 — ogni + # scansione a vuoto apre comunque uno shard da 5 GB). Senza indice: scansione + # completa come prima. + # EN: #383: when the index is present, the --mtp/--indexer passes convert ONLY + # the shards that hold the requested tensors (3 instead of scanning all 141 — + # every empty scan still opens a 5 GB shard). Without the index: full scan as + # before. + if a.mtp or a.indexer: + idxp = os.path.join(a.indir, "model.safetensors.index.json") + if os.path.exists(idxp): + wmap = json.load(open(idxp))["weight_map"] + if a.mtp: + want = {v for k, v in wmap.items() if k.startswith(f"model.layers.{a.n_layers}.")} + else: + want = {v for k, v in wmap.items() if "indexer" in k and 0 <= layer_idx(k) < a.n_layers} + keep = [sp for sp in shards if os.path.basename(sp) in want] + print(f"[PLAN] index: {len(keep)}/{len(shards)} local shard(s) hold the requested tensors") + shards = keep + # BUG #355: questo ramo ignorava --mtp/--indexer. Con --mtp scriveva + # out-NNNNN (gli STESSI nomi di una conversione normale) in ebits=8 e + # keep_mtp=False -> il "secondo passaggio MTP" nella stessa outdir + # SOVRASCRIVEVA il container gia' finito con una riconversione int8 + # completa, in silenzio (137/141 shard distrutti prima di accorgersene). + # Ora il ramo locale rispecchia il download path: prefisso corretto, + # flag passate, shard vuoti saltati. + prefix = "out-mtp-" if a.mtp else "out-idx-" if a.indexer else "out-" + # RIPRESA (#383): i nomi out-NNNNN contano gli shard EMESSI, non l'indice di + # input (gli shard senza tensori rilevanti non producono file), quindi "il + # file esiste" non basta per saltare il lavoro gia' fatto. Un manifest + # sidecar ricorda input -> output (o "vuoto") e con quali parametri: la + # ripresa salta solo cio' che combacia, e parametri diversi sulla stessa + # outdir vengono rifiutati invece di mescolare container (il modo #355). + # EN: RESUME (#383): out-NNNNN names count EMITTED shards, not the input + # EN: index (shards with no relevant tensors emit no file), so "the file + # EN: exists" is not enough to skip completed work. A sidecar manifest + # EN: records input -> output (or "empty") plus the conversion parameters: + # EN: resume skips only what matches, and different parameters on the same + # EN: outdir are refused instead of mixing containers (the #355 failure mode). + params = {"ebits": a.ebits, "io_bits": a.io_bits, "xbits": a.xbits, + "group_size": a.group_size, "n_layers": a.n_layers, "bits_map": bits_map} + prog_path = os.path.join(a.outdir, f".{prefix}progress.json") + prog = {} + if os.path.exists(prog_path): + try: prog = json.loads(open(prog_path).read()) + except (OSError, ValueError): prog = {} + if prog and prog.get("params") != params: + print(f"ERROR: {prog_path} records a conversion with {prog.get('params')};\n" + f" this run uses {params}. Refusing to mix conversions in the same " + f"outdir — use a fresh --outdir (or delete the manifest and the " + f"{prefix}*.safetensors shards to redo).") + return + done = prog.setdefault("shards", {}); prog["params"] = params + n = 0; fresh = 0; skipped = 0 for i, sp in enumerate(shards): - out = {}; convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits, group_size=a.group_size, bits_map=bits_map) - save_file(out, os.path.join(a.outdir, f"out-{i:05d}.safetensors")) - # copia config + tokenizer - for fn in ["config.json"]: - src = os.path.join(a.indir, fn) - if os.path.exists(src): shutil.copy(src, a.outdir) - print(f"converted {len(shards)} shards -> {a.outdir}") + key = os.path.basename(sp) + prev = done.get(key) # None = mai visto; "" = visto, vuoto; nome = emesso + if prev is not None and (prev == "" or os.path.exists(os.path.join(a.outdir, prev))): + if prev: n += 1 + skipped += 1 + continue + out = {} + convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits, + keep_mtp=a.mtp, keep_idx=a.indexer, + group_size=a.group_size, bits_map=bits_map) + if not out: # shard senza MTP/idx: niente file (come il download path) + done[key] = "" + else: + name = f"{prefix}{n:05d}.safetensors" + save_file(out, os.path.join(a.outdir, name)) + done[key] = name; n += 1; fresh += 1 + tmp_prog = prog_path + ".tmp" # scrittura atomica: una ripresa non vede mai un manifest mezzo scritto + with open(tmp_prog, "w") as f: json.dump(prog, f, indent=1) # EN: atomic write: a resume never sees a half-written manifest + os.replace(tmp_prog, prog_path) + if skipped: print(f"[RESUME] {skipped} shard(s) already done in {a.outdir}, skipped") + # metadati per la conversione principale: gli stessi quattro file del download + # path — senza tokenizer.json chat/serve non partono. I passaggi mtp/idx vanno + # nella stessa outdir di un container gia' completo di metadati. + # EN: metadata for the main pass: the same four files as the download path — + # EN: chat/serve won't start without tokenizer.json. The mtp/idx passes target + # EN: an outdir whose container already has its metadata. + if not a.mtp and not a.indexer: + copied, missing = [], [] + for fn in ["config.json", "tokenizer.json", "tokenizer_config.json", "generation_config.json"]: + src = os.path.join(a.indir, fn) + if os.path.exists(src): shutil.copy(src, a.outdir); copied.append(fn) + else: missing.append(fn) + print(f"[META] copied from {a.indir}: {', '.join(copied) if copied else 'nothing'}") + if missing: + print(f"[META] WARNING: not found in {a.indir}: {', '.join(missing)}" + + (" — chat/serve need tokenizer.json" if "tokenizer.json" in missing else "")) + tag = "MTP" if a.mtp else "indexer" if a.indexer else "main" + print(f"converted {fresh} {tag} shard(s), {n} in container -> {a.outdir} ({prefix}NNNNN)") return # reale: scarica shard per shard, converte, cancella diff --git a/c/tools/convert_olmoe.py b/c/tools/convert_olmoe.py index dd45806..dc25f6a 100644 --- a/c/tools/convert_olmoe.py +++ b/c/tools/convert_olmoe.py @@ -3,7 +3,10 @@ Downloads or converts a local OLMoE checkpoint (e.g., allenai/OLMoE-1B-7B-0125-Instruct). Dense weights stay as-is (engine reads BF16/F16 → F32 on load). -Expert weights get row-wise int8 quantization with float32 scales. +Expert weights get row-wise symmetric quantization to --ebits bits (default 4) +with float32 scales. Storage stays one value per int8 byte regardless of bits, +matching the engine's expert layout (olmoe.c quantize_rows) — for 4 bits the +values are simply confined to [-8, 7] with scales computed against qmax=7. Usage: python tools/convert_olmoe.py --repo allenai/OLMoE-1B-7B-0125-Instruct --out ./olmoe_i4 @@ -29,12 +32,21 @@ except ImportError as exc: EXPERT_KEY_RE = r"model\.layers\.\d+\.mlp\.experts\.\d+\.(gate_proj|up_proj|down_proj)\.weight" -def quantize_row(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: - """Row-wise int8 quantization. Returns (int8_weights, float32_scales).""" +def quantize_row(w: torch.Tensor, bits: int = 8) -> tuple[torch.Tensor, torch.Tensor]: + """Row-wise symmetric quantization to `bits` (2..8). + + Returns (int8_weights, float32_scales). Storage is one value per int8 byte + for every bit width — the engine dequantizes as q*scale and never assumes + the full int8 range — mirroring olmoe.c quantize_rows(): + qmax = 2**(bits-1) - 1 (8 -> 127, 4 -> 7, 2 -> 1) + scale = amax(|w|, row) / qmax + q = clamp(round(w / scale), -qmax-1, qmax) + """ + qmax = (1 << (bits - 1)) - 1 w_f32 = w.float() row_max = w_f32.abs().amax(dim=1, keepdim=True).clamp(min=1e-12) - scales = row_max / 127.0 - q = (w_f32 / scales).round().clamp(-128, 127).to(torch.int8) + scales = row_max / qmax + q = (w_f32 / scales).round().clamp(-qmax - 1, qmax).to(torch.int8) return q, scales.squeeze(1) @@ -50,9 +62,12 @@ def main(): src.add_argument("--model", help="Local HF checkpoint directory") ap.add_argument("--out", required=True, help="Output directory for int4 model") ap.add_argument("--ebits", type=int, default=4, - help="Expert quant bits (4 or 8, default 4)") + help="Expert quant bits (2..8, default 4)") args = ap.parse_args() + if not 2 <= args.ebits <= 8: # storage is int8_t; engine rejects the same range (olmoe.c) + sys.exit(f"--ebits must be 2..8 (got {args.ebits})") + if args.repo: from huggingface_hub import snapshot_download from huggingface_hub.errors import LocalEntryNotFoundError @@ -96,7 +111,7 @@ def main(): for name, tensor in tensors.items(): if is_expert_weight(name): expert_count += 1 - q, scales = quantize_row(tensor) + q, scales = quantize_row(tensor, args.ebits) total_expert_f32 += tensor.numel() * tensor.element_size() total_expert_q += q.numel() * 1 + scales.numel() * 4 out_tensors[name] = q @@ -109,7 +124,7 @@ def main(): ratio = total_expert_q / max(total_expert_f32, 1) * 100 print(f"ok") - print(f"\nDone. {expert_count} expert tensors quantized.") + print(f"\nDone. {expert_count} expert tensors quantized to int{args.ebits}.") print(f"Expert storage: {total_expert_f32/1e9:.1f} GB -> {total_expert_q/1e9:.1f} GB ({ratio:.0f}%)") print(f"Model ready at: {out}") print(f"\nRun: SNAP={out} ./olmoe.exe 32 4 16") diff --git a/c/tools/convert_olmoe_merged.py b/c/tools/convert_olmoe_merged.py new file mode 100644 index 0000000..883b8df --- /dev/null +++ b/c/tools/convert_olmoe_merged.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Convert OLMoE HuggingFace checkpoint to colibri merged int8 format. + +Consolidates gate_proj, up_proj, and down_proj into a single merged tensor per expert. +This allows olmoe.c to load an expert in a single disk read call instead of 3. + +Usage: + python tools/convert_olmoe_merged.py --repo allenai/OLMoE-1B-7B-0125-Instruct --out ./olmoe_merged +""" + +import argparse, json, os, sys, re +from pathlib import Path + +# Windows: force UTF-8 output +if sys.platform == "win32": + for s in (sys.stdout, sys.stderr): + try: s.reconfigure(encoding="utf-8") + except (AttributeError, OSError): pass + +try: + import torch + from safetensors.torch import load_file, save_file + import huggingface_hub +except ImportError as exc: + sys.exit(f"Missing dependencies: {exc}. Install: pip install torch safetensors huggingface_hub") + +EXPERT_KEY_RE = r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight" + +def quantize_row(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Row-wise int8 quantization. Returns (int8_weights, float32_scales).""" + w_f32 = w.float() + row_max = w_f32.abs().amax(dim=1, keepdim=True).clamp(min=1e-12) + scales = row_max / 127.0 + q = (w_f32 / scales).round().clamp(-128, 127).to(torch.int8) + return q, scales.squeeze(1) + +def main(): + ap = argparse.ArgumentParser(description="Convert OLMoE HF checkpoint -> colibri merged int8") + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("--repo", help="HuggingFace repo ID") + src.add_argument("--model", help="Local HF checkpoint directory") + ap.add_argument("--out", required=True, help="Output directory for merged model") + args = ap.parse_args() + + if args.repo: + from huggingface_hub import snapshot_download + from huggingface_hub.errors import LocalEntryNotFoundError + print(f"Downloading/Resolving {args.repo}...") + try: + src_dir = snapshot_download(args.repo, local_files_only=True, max_workers=4) + except LocalEntryNotFoundError: + src_dir = None + if src_dir is None or not any(Path(src_dir).glob("*.safetensors")): + print("Downloading safetensors...") + src_dir = snapshot_download(args.repo, max_workers=4) + else: + src_dir = args.model + + src = Path(src_dir) + if not src.is_dir(): + sys.exit(f"Model directory not found: {src}") + if not (src / "config.json").is_file(): + sys.exit(f"config.json missing in {src}") + + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + + # Copy config.json + import shutil + shutil.copy2(src / "config.json", out / "config.json") + print(f"config.json -> {out}") + + # Process safetensors + shards = sorted(src.glob("*.safetensors")) + if not shards: + sys.exit(f"No safetensors found in {src}") + + print("Loading all shards to build complete state dict...") + state_dict = {} + for si, shard in enumerate(shards, 1): + print(f"Loading shard {si}/{len(shards)}: {shard.name}...") + tensors = load_file(str(shard)) + state_dict.update(tensors) + + # Gather experts + experts = {} + for name in list(state_dict.keys()): + m = re.match(EXPERT_KEY_RE, name) + if m: + layer_idx, expert_idx, proj = m.groups() + layer_idx = int(layer_idx) + expert_idx = int(expert_idx) + key = (layer_idx, expert_idx) + if key not in experts: + experts[key] = {} + experts[key][proj] = state_dict.pop(name) + + print(f"Found {len(experts)} experts to merge.") + + # Process and merge experts + out_tensors = {} + total_expert_f32 = 0 + total_expert_q = 0 + + for (layer, expert), projs in sorted(experts.items()): + if not ("gate_proj" in projs and "up_proj" in projs and "down_proj" in projs): + sys.exit(f"Missing projection for layer {layer} expert {expert}!") + + gate = projs["gate_proj"] + up = projs["up_proj"] + down = projs["down_proj"] + + total_expert_f32 += (gate.numel() + up.numel() + down.numel()) * gate.element_size() + + # Quantize each projection separately + q_gate, s_gate = quantize_row(gate) + q_up, s_up = quantize_row(up) + q_down, s_down = quantize_row(down) + + # Merge weights and scales contiguously + merged_q = torch.cat([q_gate.flatten(), q_up.flatten(), q_down.flatten()]) + merged_scales = torch.cat([s_gate, s_up, s_down]) + + total_expert_q += merged_q.numel() * 1 + merged_scales.numel() * 4 + + # Save to output + out_tensors[f"model.layers.{layer}.mlp.experts.{expert}.merged_weight"] = merged_q + out_tensors[f"model.layers.{layer}.mlp.experts.{expert}.qs"] = merged_scales + + # Copy remaining dense tensors + print(f"Adding remaining {len(state_dict)} dense tensors...") + out_tensors.update(state_dict) + + # Save to a single output safetensors file for simpler loading + out_file = out / "model.safetensors" + print(f"Saving merged safetensors model to {out_file}...") + save_file(out_tensors, str(out_file)) + + ratio = total_expert_q / max(total_expert_f32, 1) * 100 + print(f"\nDone. {len(experts)} experts successfully merged and saved.") + print(f"Expert storage: {total_expert_f32/1e9:.1f} GB -> {total_expert_q/1e9:.1f} GB ({ratio:.0f}%)") + print(f"Model ready at: {out}") + +if __name__ == "__main__": + main() diff --git a/c/tools/make_glm_oracle.py b/c/tools/make_glm_oracle.py index b623faf..5e16f51 100644 --- a/c/tools/make_glm_oracle.py +++ b/c/tools/make_glm_oracle.py @@ -17,6 +17,45 @@ EN: computed AFTER the FP8 round-trip, so the reference matches exactly what the EN: ingests. Default: bf16 (original oracle unchanged).""" import json, sys, argparse from pathlib import Path + +# --- Version gate (must run BEFORE the heavy `from transformers import ...`, +# which triggers transformers' lazy-loading and can in turn reset the in-memory +# __version__ attribute; importlib.metadata reads the installed package version +# directly and is immune to that). --- +# +# GLM-5.2's MLA attention uses interleaved (DeepSeek-style) RoPE, which is what +# the C engine implements. transformers < 5.11.0 applied split-half (Llama-style) +# RoPE in GlmMoeDsa* instead; an oracle built on those versions drifts and the +# engine then scores 25/32 instead of the documented 32/32 (issue #281). Weights +# come out identical across versions — only the forward pass differs — so there +# is no safe "partial" run: a too-old transformers silently produces an invalid +# ref_glm.json. Hard-fail rather than warn. EN: same. +import transformers +from importlib.metadata import version as _pkg_version, PackageNotFoundError + +_MIN_TRANSFORMERS = (5, 11) +def _tf_version_tuple(): + try: + v = _pkg_version("transformers") # authoritative: installed dist metadata + except PackageNotFoundError: + v = getattr(transformers, "__version__", "0") # fallback (editable/src installs) + out = [] + for part in v.split(".")[:2]: # major.minor only + out.append(int("".join(c for c in part if c.isdigit()) or "0")) + while len(out) < 2: + out.append(0) + return tuple(out[:2]) + +_tf_ver = _tf_version_tuple() +if _tf_ver < _MIN_TRANSFORMERS: + sys.exit( + f"\nERROR: make_glm_oracle.py requires transformers >= " + f"{'.'.join(map(str, _MIN_TRANSFORMERS))}.0 (found {_tf_ver[0]}.{_tf_ver[1]}). " + f"GLM-5.2 MLA uses interleaved RoPE; older versions apply split-half RoPE " + f"and silently produce an oracle the engine scores 25/32 against (issue #281). " + f"Upgrade: pip install -U 'transformers>=5.11'\n" + ) + import torch from transformers import GlmMoeDsaConfig, GlmMoeDsaForCausalLM @@ -115,6 +154,7 @@ print("tf_pred:", tf_pred) sd = model.state_dict() unfuse_experts(sd) +Path("glm_tiny").mkdir(parents=True, exist_ok=True) # safetensors/json won't create the dir themselves if args.fp8: n_fp8, n_tot = save_fp8_safetensors(sd, "glm_tiny/model.safetensors") print(f"\nsaved FP8: {n_fp8} e4m3 tensors (+{n_tot - n_fp8} scale_inv sidecars / f32) " diff --git a/c/tools/make_olmoe_real_oracle.py b/c/tools/make_olmoe_real_oracle.py new file mode 100644 index 0000000..1e638e4 --- /dev/null +++ b/c/tools/make_olmoe_real_oracle.py @@ -0,0 +1,70 @@ +"""Generate reference token IDs for the real OLMoE-1B-7B model. + +Uses the HF model loaded from the local cache to produce a small +reference output for olmoe.exe validation. Saves to ref_olmoe_real.json. + +Usage: python tools/make_olmoe_real_oracle.py +""" +import json +import sys +from pathlib import Path + +if sys.platform == "win32": + for s in (sys.stdout, sys.stderr): + try: + s.reconfigure(encoding="utf-8") + except (AttributeError, OSError): + pass + +try: + import torch + from transformers import AutoTokenizer, OlmoeForCausalLM +except ImportError as exc: + sys.exit(f"Missing deps: {exc}. Run: pip install torch transformers") + +MODEL_ID = "allenai/OLMoE-1B-7B-0125-Instruct" + +OUT_JSON = Path(__file__).resolve().parent.parent / "ref_olmoe_real.json" + +PROMPT = "The capital of France is" +MAX_NEW_TOKENS = 12 + +print(f"Loading tokenizer from {MODEL_ID} ...") +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) + +print("Encoding prompt ...") +enc = tokenizer(PROMPT, return_tensors="pt") +prompt_ids = enc["input_ids"][0].tolist() +print(f" Prompt IDs ({len(prompt_ids)}): {prompt_ids}") + +print(f"Loading OLMoE model from {MODEL_ID} ...") +print(" (this will use ~14 GB RAM — please be patient)") +model = OlmoeForCausalLM.from_pretrained( + MODEL_ID, + torch_dtype=torch.bfloat16, + device_map="cpu", + low_cpu_mem_usage=True, +) +model.eval() +print(" Model loaded!") + +print(f"Generating {MAX_NEW_TOKENS} tokens ...") +with torch.no_grad(): + out = model.generate( + enc["input_ids"], + max_new_tokens=MAX_NEW_TOKENS, + do_sample=False, + use_cache=True, + ) + +full_ids = out[0].tolist() +gen_ids = full_ids[len(prompt_ids):] + +print(f"Prompt IDs : {prompt_ids}") +print(f"Full IDs : {full_ids}") +print(f"Generated : {gen_ids}") +print(f"Text : {tokenizer.decode(gen_ids, skip_special_tokens=True)!r}") + +payload = {"prompt_ids": prompt_ids, "full_ids": full_ids} +OUT_JSON.write_text(json.dumps(payload, indent=2)) +print(f"\nSaved reference to {OUT_JSON}") diff --git a/c/tools/quant_ablation.py b/c/tools/quant_ablation.py index 689643a..dd08715 100644 --- a/c/tools/quant_ablation.py +++ b/c/tools/quant_ablation.py @@ -118,7 +118,7 @@ def quantize_param(w, bits, group, rot=False, e8=""): def _grid_or_e8(x, bits, group, e8): if e8: - return _quant_e8(x.float(), group, ball=(e8 == "-e8")) + return _quant_e8(x.float(), group, bits, ball=(e8 == "-e8")) return _quant_last_dim(x, bits, group) @@ -169,8 +169,15 @@ def _e8_ball(y, r2=10.0): return p -def _quant_e8(x, group, ball): - """Blocks of 8 along the input dim; per-group scale by MSE search over RMS multiples.""" +_E8_R2_REPORTED = set() +def _e8_radius(bits): + # E8 lattice: points within |p|^2<=r2 grow ~r2^4, so +1 bit (x256 codebook) needs r2 x4. + # Anchor: r2=10 is the ~2^16 E8P ball (2 bits over 8 dims). Scale from there. + return 10.0 * (4.0 ** (bits - 2)) + +def _quant_e8(x, group, bits, ball): + """Blocks of 8 along the input dim; per-group scale by MSE search over RMS multiples. + ball=True clamps to the rate-scaled E8 ball for `bits`; ball=False is the unbounded ideal.""" if x.shape[-1] % 8: raise SystemExit(f"-e8 needs input dim divisible by 8 (got {x.shape[-1]})") g = group or x.shape[-1] @@ -183,7 +190,11 @@ def _quant_e8(x, group, ball): for k in (0.5, 0.7, 0.9, 1.1, 1.4, 1.8, 2.4): s = rms * k yb = (xg / s).reshape(-1, g // 8, 8) - p = _e8_ball(yb) if ball else _e8_nearest(yb) + p = _e8_ball(yb, _e8_radius(bits)) if ball else _e8_nearest(yb) + if ball and bits not in _E8_R2_REPORTED: + _E8_R2_REPORTED.add(bits) + import sys as _sys + _sys.stderr.write(f"[e8] bits={bits}: ball r2={_e8_radius(bits):.1f}\n") out = (p.reshape(-1, g) * s) err = (out - xg).pow(2).sum(-1, keepdim=True) if best_err is None: diff --git a/c/tools/repair_mtp_int8.py b/c/tools/repair_mtp_int8.py new file mode 100644 index 0000000..695e99f --- /dev/null +++ b/c/tools/repair_mtp_int8.py @@ -0,0 +1,229 @@ +"""Repair an existing colibri int4 container whose MTP head was quantized at int4. + +WHY THIS EXISTS + `model.layers..eh_proj.weight` [D, 2D] multiplies the MTP concat + [embedding_norm ; hidden_norm], and its two column halves differ in scale by + ~20-30x per row (embedding-half absmax ~0.05, hidden-half ~1.5 on GLM-5.2). + Per-row int4 uses ONE scale (= absmax/7) per row, so every embedding-half + weight lands below half a quantization step and np.rint rounds the ENTIRE + embedding half to exact zeros (packed bytes 0x88). The MTP head then drafts + garbage: acceptance ~0% (issue #8 measured 0-4% at int4; 39-59% at int8 — + which is why `convert_fp8_to_int4.py --mtp` defaults to --ebits 8). + + A container converted (or downloaded) with an int4 MTP head does not need a + full re-conversion: this script re-downloads ONLY the affected dense tensors + (~355 MB of HTTP range reads against the FP8 source repo), requantizes them + at int8 with the converter's exact math, and patches the local shards in + place. Originals are kept beside as *.bak-int4. + +WHAT IT TOUCHES + The MTP layer's dense tensors only (eh_proj, q_a/q_b/kv_a/kv_b/o_proj, + shared_experts.*): the ones that stream into RAM once and stay resident. + Routed experts (model.layers..mlp.experts.*) are NOT touched — they are + statistically like the main layers' experts and int4 is acceptable there. + The engine auto-detects int8 vs int4 by blob size (qt_from_disk), so no + engine or config change is needed. Cost: ~+133 MB on disk / resident RAM. + +USAGE + python3 tools/repair_mtp_int8.py --snap /path/to/glm52_i4 # repair + python3 tools/repair_mtp_int8.py --snap /path/to/glm52_i4 --dry-run # inspect only + + --source-repo defaults to zai-org/GLM-5.2-FP8 (the checkpoint the public + int4 containers were converted from). Requires numpy and network access; + no torch, no HF token (public repo, anonymous range reads). +""" +import argparse, glob, json, os, ssl, struct, sys, urllib.request +import numpy as np + +# macOS python.org builds ship no CA bundle: use certifi when available (Linux +# system Pythons generally have working system certs and skip this). +try: + import certifi + _SSL_CTX = ssl.create_default_context(cafile=certifi.where()) +except ImportError: + _SSL_CTX = ssl.create_default_context() + + +# ---------- HTTP range reads against the source repo ---------- +def http_range(url, start, length, tries=5): + req = urllib.request.Request(url, headers={"User-Agent": "colibri-mtp-repair", + "Range": f"bytes={start}-{start+length-1}"}) + for attempt in range(tries): + try: + with urllib.request.urlopen(req, timeout=30, context=_SSL_CTX) as r: + data = r.read() + if len(data) == length: + return data + except KeyboardInterrupt: + raise + except Exception as ex: + if attempt == tries - 1: + raise RuntimeError(f"range read failed for {url}: {ex}") + raise RuntimeError(f"short range read for {url}") + + +class SourceRepo: + def __init__(self, repo, revision="main"): + self.base = f"https://huggingface.co/{repo}/resolve/{revision}/" + with urllib.request.urlopen(self.base + "model.safetensors.index.json", timeout=30, context=_SSL_CTX) as r: + self.wmap = json.loads(r.read())["weight_map"] + self._hdr = {} + + def _shard_header(self, shard): + if shard not in self._hdr: + n = struct.unpack("> 3) & 0xF + mant = (b & 7).astype(np.float64) + v = (sign * np.where(e > 0, (1 + mant / 8) * np.exp2(e.astype(np.float64) - 7), + mant / 8 * np.exp2(-6.0))).reshape(m["shape"]) + sn = name + "_scale_inv" + sshard = self.wmap[sn] + shdr, sbase = self._shard_header(sshard) + sm = shdr[sn] + so0, so1 = sm["data_offsets"] + sc = np.frombuffer(http_range(self.base + sshard, sbase + so0, so1 - so0), + dtype=np.float32).reshape(sm["shape"]) + O, I = m["shape"] + scf = np.repeat(np.repeat(sc, 128, axis=0)[:O], 128, axis=1)[:, :I] + return (v * scf).astype(np.float32) + raise ValueError(f"{name}: unsupported source dtype {m['dtype']}") + + +# ---------- quantization: identical to convert_fp8_to_int4.quant_int8 ---------- +def quant_int8(w): + amax = np.abs(w).max(axis=1, keepdims=True) + s = np.maximum(amax / 127, 1e-8) + q = np.clip(np.rint(w / s), -128, 127).astype(np.int8) + return q.reshape(-1).view(np.uint8).copy(), s[:, 0].astype(np.float32) + + +# ---------- local safetensors IO (no deps; preserves byte-identity of untouched tensors) ---------- +def read_shard(path): + with open(path, "rb") as fh: + n = struct.unpack(" [tensor names to repair] + already_ok, skipped_experts = [], 0 + for f in sorted(glob.glob(os.path.join(a.snap, "*.safetensors"))): + with open(f, "rb") as fh: + n = struct.unpack(" ..." line +m = re.search(r"C engine\s*:\s*([\d ]+)", r1.stdout) +if not m: + sys.exit("Could not parse 'C engine :' line from output") +gen_ids = [int(x) for x in m.group(1).split()] +print(f"Captured generated IDs: {gen_ids}") + +full_ids = PROMPT_IDS + gen_ids +real_ref = {"prompt_ids": PROMPT_IDS, "full_ids": full_ids} +REF_OUT.write_text(json.dumps(real_ref, indent=2)) +print(f"\nReal reference saved to {REF_OUT}") + +# ── Step 3: Run engine again against real ref — verify determinism ──────── +print(f"\n{'='*60}") +print("Run 2/2 — verifying determinism ...") +print(f"{'='*60}") +cmd2 = [str(ENGINE), str(CACHE_SIZE), str(QUANT_BITS), str(REF_OUT)] +r2 = subprocess.run(cmd2, env=env, capture_output=True, text=True, cwd=str(HERE)) +print(r2.stdout) +if r2.returncode != 0: + print("STDERR:", r2.stderr, file=sys.stderr) + sys.exit(r2.returncode) + +if "Matching tokens: 12/12" in r2.stdout or f"Matching tokens: {MAX_NEW}/{MAX_NEW}" in r2.stdout: + print("✓ Engine is DETERMINISTIC — same output on both runs!") +else: + m2 = re.search(r"Matching tokens: (\d+)/(\d+)", r2.stdout) + if m2: + print(f"⚠ Partial match: {m2.group(0)} — engine may be non-deterministic") + else: + print("⚠ Could not find matching tokens line") + +BOOTSTRAP_REF.unlink(missing_ok=True) diff --git a/c/version.py b/c/version.py new file mode 100644 index 0000000..9951ca1 --- /dev/null +++ b/c/version.py @@ -0,0 +1,3 @@ +"""Single source of truth for the colibrì version number.""" + +__version__ = "1.0.0" diff --git a/colibri/__init__.py b/colibri/__init__.py new file mode 100644 index 0000000..77f7061 --- /dev/null +++ b/colibri/__init__.py @@ -0,0 +1,5 @@ +"""colibrì — tiny engine, immense model.""" + +from colibri._version import __version__ + +__all__ = ["__version__"] diff --git a/colibri/_version.py b/colibri/_version.py new file mode 100644 index 0000000..174442a --- /dev/null +++ b/colibri/_version.py @@ -0,0 +1,23 @@ +"""Version accessor for the pip package. + +The single source of truth is c/version.py (#394): coli --version and the +GitHub Release workflow read it, so the pip metadata must read the SAME file +instead of carrying a second literal that would drift on the first bump. + +From a checkout (the supported install: `pip install -e .`) the file is read +directly. From an installed wheel c/ is not on disk, so fall back to the +package metadata that setuptools baked at build time from that same file. +""" +from pathlib import Path + +try: + _ns = {} + exec((Path(__file__).resolve().parent.parent / "c" / "version.py").read_text(), _ns) + __version__ = _ns["__version__"] +except OSError: + from importlib.metadata import PackageNotFoundError, version + + try: + __version__ = version("colibri-engine") + except PackageNotFoundError: + __version__ = "0.0.0+unknown" diff --git a/colibri/cli.py b/colibri/cli.py new file mode 100644 index 0000000..3f19234 --- /dev/null +++ b/colibri/cli.py @@ -0,0 +1,30 @@ +"""Entry point for `coli` when installed via pip. + +Delegates to the original c/coli script which handles all subcommands. +This wrapper exists so `pip install colibri-engine` creates a `coli` console +script that works without the user having to add c/ to PATH manually. +""" + +import os +import sys +import runpy + + +def main(): + here = os.path.dirname(os.path.abspath(__file__)) + engine_dir = os.path.join(os.path.dirname(here), "c") + coli_script = os.path.join(engine_dir, "coli") + + if not os.path.exists(coli_script): + sys.exit( + "colibri engine directory not found.\n" + "Install from source: git clone + pip install -e ." + ) + + sys.path.insert(0, engine_dir) + sys.argv[0] = coli_script + runpy.run_path(coli_script, run_name="__main__") + + +if __name__ == "__main__": + main() diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index d345b7b..9f0d749 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -2,7 +2,7 @@ Reference for the environment variables read by the colibrì engine. -**Generated from `upstream/dev @ 6d3ed7e`** by scanning every `getenv()` site in `c/glm.c`. Defaults and behavior are taken from the source; see [MAINTAINING-DOCS.md](MAINTAINING-DOCS.md) to regenerate this after the code changes. +**Generated from `dev @ d5327e2`** by scanning every `getenv()` site in `c/glm.c` and the other C sources (`c/olmoe.c`, `c/backend_cuda.cu`, `c/backend_metal.mm`). Defaults and behavior are taken from the source; see [MAINTAINING-DOCS.md](MAINTAINING-DOCS.md) to regenerate this after the code changes. ## Which program reads these? @@ -43,6 +43,7 @@ Format: `VAR` — default — effect. | `URING` | `0` (off) | Linux-only queued expert I/O. `URING=1` implies `PIPE=1`, forces cold reads through io-wq (`IOSQE_ASYNC`), replaces blocking loader pthreads and spin waits with batched SQEs/CQEs, and batches `PILOT_REAL` loads on a separate ring. Use `DIRECT=1` for cold NVMe to avoid page-cache copy/readahead limits. Fails clearly if the kernel denies io_uring; incompatible with `COLI_MMAP=1`. | | `DIRECT` | `0` (off) | Use `O_DIRECT`/unbuffered reads for expert slabs. Helps sustained NVMe; keeps the zero-copy GPU path. | | `COLI_NO_OMP_TUNE` | off | **Kill-switch** for the OpenMP hot-thread tuning (`OMP_WAIT_POLICY=active` spin + proc-bind). Set `=1` when the CPU is mostly waiting on the GPU (Metal) so spin doesn't steal the shared power budget. | +| `COLI_NUMA` | auto in generated plans on multi-socket Linux; otherwise off | `COLI_NUMA=1` selectively interleaves large expert and dense slabs across NUMA nodes via `mbind` (raw syscall, no libnuma). Helps multi-socket hosts (+7–40% expert matmul); silent no-op on single-node or non-Linux. Explicit `COLI_NUMA=0` overrides the generated plan. | | `MLOCK` | `-1` (auto: on for macOS) | Wire the streamed expert cache into physical RAM (`mlock`) to dodge the memory compressor. `0` off, `1` force. | | `CAP_RAISE` | `1` (on) | Let the engine raise the expert-cache cap above `topk` when RAM allows (bigger batches). `0` fixes the cap. | | `PREFETCH` | `0` | Prefetch depth for streamed experts. | @@ -54,15 +55,26 @@ Format: `VAR` — default — effect. | `PILOT` | `0` (off) | Router-piloted cross-layer expert prefetch. | | `PILOT_REAL` | `0` (off) | Value-preserving real cross-layer prefetch loads (`PILOT_REAL=1` opts in). | | `PILOT_K` | `6` if `PILOT_REAL` else `8` | Number of experts the pilot prefetches per step. | +| `PILOT_TWO` | `0` (off) | Two-step shared-expert-corrected router prediction for the pilot. | +| `COUPLE` | unset | Path to a coupling-score file driving cross-layer expert prefetch (#176). When set, `couple_load` reads it. | +| `COUPLE_K` | `8` | Top-K coupled experts per layer when `COUPLE` is set. | +| `COUPLE_D` | `1` | Coupling lookahead depth (`1` or `2`) when `COUPLE` is set. | | `CACHE_ROUTE` | `0` (off) | Opt-in max-rank cache-aware MoE routing (pin∪LRU prefer within top-M). See [CACHE_ROUTE.md](CACHE_ROUTE.md). | | `ROUTE_J` | `2` | Sacred top ranks always taken when `CACHE_ROUTE=1`. | | `ROUTE_M` | `12` | Max-rank window for resident preference when `CACHE_ROUTE=1`. | | `ROUTE_P` | `0` | Cumulative mass window for CACHE_ROUTE (`0` = fixed M). | | `ROUTE_ALPHA` | `1` | Scale gate mass of substituted experts before renorm (`1` = off). | | `ROUTE_AGREE` | auto | Overlap% + KL vs true top-K; auto-on when `CACHE_ROUTE=1`. | +| `ROUTE_TRACE` | unset | If set to a path, logs every routing decision there (testing/analysis). | | `ABSORB` | `-1` (auto: absorbed for S≤4) | MLA attention absorption mode. | | `IDOT` | `1` | Integer dot-product kernel. `IDOT=0` uses exact f32 kernels (for A/B numerical checks). | | `COLI_POLICY` | `quality` | Resource policy: `quality`, `balanced`, or `experimental-fast`. | +| `PROF` | `0` (off) | Performance profile: a startup header (machine + effective config), then per run — or per turn in serve mode, on stderr — forward-latency percentiles (p50/p90/p99/max), expert-I/O totals and cache-tier fill, phase shares of wall time, and a verdict naming the knob most likely to help on this machine. Output is additive; `PROF` unset changes nothing. | +| `COLI_NO_FUSED_PAIR` | `0` (off) | `=1` disables the fused-pair matmul kernel. | +| `DISK_SPLIT` | `0` (off) | `=1` splits the reported disk-load time across the draft/absorb/forward phases in stats. | +| `I4S` | unset | Engage the int4 `IDOT` kernel only for batch `S>=` (testing). | +| `SPEC_PIN` | `1` (on) | Speculation gate mode. `0` reverts to the legacy S-dependent speculation gates (#163). | +| `COLI_RAM_OVERCOMMIT` | off | `=1` overrides the "projected peak > MemAvailable → exit(2)" guard so a run that risks kernel OOM-kill is allowed to proceed. | --- @@ -76,7 +88,22 @@ Format: `VAR` — default — effect. | `CUDA_EXPERT_GB` | `0` | VRAM budget (GB) for caching experts on the GPU. | | `CUDA_RELEASE_HOST` | auto (`1` if >1 device) | Release host-side copies after upload. | | `COLI_CUDA_ATTN` | off | Run S≤4 attention on the GPU. | +| `COLI_CUDA_ATTN_SHARD` | off | `=1` splits KV-b heads across devices during attention load (multi-GPU). | | `COLI_CUDA_PROFILE` | off | Emit CUDA timing. | +| `COLI_CUDA_PIPE` | `0` (off) | `1` engages the multi-step attention pipeline; `2` enables the pipe2 path. | +| `COLI_CUDA_PIPE_SHARD` | off | `=1` runs the multi-device P2P head-shard attention path (opt-in for NVLink topologies; serializes ~95 MB/layer over a star PCIe topology). | +| `COLI_CUDA_PIPE_S_MIN` | `1` single-GPU, `8` multi-GPU | Minimum prefill batch S to engage the pipe2 CUDA path. | +| `COLI_CUDA_MTP` | `0` (off) | `=1` opts into MTP speculation under CUDA (off by default: cold streaming experts run on CPU where the fused-pair/IDOT kernels diverge in FP order, collapsing draft acceptance, #163/#292). | +| `COLI_CUDA_ASYNC` | on | `=0` forces synchronous `cudaMemcpy` instead of async + pinned host staging. | +| `COLI_CUDA_DUAL_PROJ` | on | `=0` issues gate+up as two separate launches instead of one fused `grouped_hidden_w4_dual`. | +| `COLI_CUDA_W4_PACKED` | on | `=0` disables the grouped packed-int4 path. | +| `COLI_CUDA_TC_INT4` | off | `=1` uses the W4A4 WMMA Tensor Core path (when all expert tensors are int4 and dims divide). | +| `COLI_CUDA_TC_MIN_ROWS` | `8` | Min rows-per-expert to engage the W4A4 Tensor Core path. | +| `COLI_CUDA_TC_W4A16` | off | `=1` uses the lossless W4A16 Tensor Core path (compute capability ≥7). | +| `COLI_CUDA_TC_W4A16_MIN` | `16` | Per-expert row threshold above which W4A16 TC tiles dispatch (smaller batches fall back to the naive kernel). | +| `COLI_CUDA_SHARED_W4A16` | off | `=1` uploads shared-expert weights and runs the shared-MLP W4A16 Tensor Core kernel. | +| `COLI_CUDA_SHARED_W4A16_MIN_ROWS` | `32` | Min row count to engage the shared-MLP W4A16 kernel. | +| `COLI_METAL_UNTRACKED` | off (Metal only) | `=1` sets `MTLResourceHazardTrackingModeUntracked` on Metal buffers (reduces hazard-tracking overhead). | --- @@ -88,8 +115,11 @@ These are for testing, benchmarking, or internal use — not part of the everyda |---|---|---| | `SPEC` | `1` | Speculative decoding on/off. | | `DRAFT` | `-1` (auto: 3 with MTP, else 0) | Number of speculative draft tokens per step. | -| `GRAMMAR` | unset | Path to a GBNF grammar file to constrain generation. | +| `GRAMMAR` | unset | Path to a GBNF grammar file to constrain generation. Takes precedence over `SCHEMA`. | +| `SCHEMA` | unset | Path to a JSON-Schema file compiled to GBNF to constrain generation (consulted only when `GRAMMAR` is empty). | | `GRAMMAR_DRAFT` | unset | Max grammar-forced draft span length. | +| `EXPERT_BUDGET` | `0` (off) | Cap experts loaded per layer (MoE-Spec). **Quarantined:** silently forced to `0` unless `EXPERT_BUDGET_EXPERIMENTAL` is set — every tested value is either no faster or incoherent (issue #303). | +| `EXPERT_BUDGET_EXPERIMENTAL` | unset | Setting it (any value) allows `EXPERT_BUDGET>0` to actually take effect (expect garbage, #294). | | `DSA` | on | Dynamic Sparse Attention indexer. `DSA=0` disables. | | `DSA_FORCE` | `0` | Force the DSA path on. | | `DSA_TOPK` | model value | Override the DSA index top-k (testing). | @@ -100,11 +130,15 @@ These are for testing, benchmarking, or internal use — not part of the everyda | `PIN_FILL` | `0` | Fill the pinned store even without usage data. | | `MTP_DEBUG` / `MTP_PRENORM` / `MTP_SWAP` | off | MTP head debugging / ablations. | | `STATS` | unset | Write an expert-usage histogram to `STATS=` at end of run. | +| `TOKENS` | unset | If set, dumps generated token ids to stderr for A/B comparison. | | `SCORE` | unset | Scoring/eval mode over `SCORE=`. | +| `SCORE_PREFIX` | on | If unset or `≠0`, prepends `[gMASK]` to scoring contexts (GLM-family only). | +| `REPIN_VERBOSE` | off | If set, prints per-swap `[REPIN]` diagnostics during VRAM repin. | | `REF` / `REF_FORCE` | `ref_glm.json` | Reference-output comparison mode. | | `REPLAY` | unset | Replay mode. | | `TF` | unset | Teacher-forcing mode. | | `CHAT_TEMPLATE` | `1` | Apply the GLM chat template (`0` = raw prompt). | +| `PPL` | off (`olmoe.c` only) | `PPL=1` enters teacher-forced NLL/perplexity meter mode in the OLMoE sister engine. | --- @@ -135,7 +169,7 @@ These are read by the Python programs (not the `glm` engine), so they don't appe - `SNAP` — model snapshot directory (required by `glm`; set from `--model`). - `SERVE`, `SERVE_BATCH` — select serve / batched-serve mode. -- `PROMPT` — one-shot text mode. +- `PROMPT` — one-shot text mode (the engine also honors `COLI_PROMPT`, preferred cross-platform; `PROMPT` is ignored on Windows if it contains cmd.exe `$`-metacharacters). - `COLI_OMP_TUNED` — internal sentinel guarding the OMP re-exec (see `COLI_NO_OMP_TUNE`); not user-facing. --- diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..62010bd --- /dev/null +++ b/docs/api.md @@ -0,0 +1,186 @@ +# OpenAI-compatible API, KV contexts & web UI + +## `coli serve` + +`coli serve` keeps one model process loaded and exposes a text-only +OpenAI-compatible HTTP API. The gateway uses only the Python standard library; +inference still runs in the same dependency-free C engine. + +```bash +cd c +COLI_MODEL=/nvme/glm52_i4 COLI_API_KEY=local-secret ./coli serve \ + --host 127.0.0.1 --port 8000 --model-id glm-5.2-colibri + +curl http://127.0.0.1:8000/v1/chat/completions \ + -H 'Authorization: Bearer local-secret' \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "glm-5.2-colibri", + "messages": [{"role": "user", "content": "Hello"}], + "stream": true + }' +``` + +Implemented endpoints are `GET /v1/models`, `GET /v1/models/{model}`, +`POST /v1/chat/completions`, and legacy `POST /v1/completions`. Chat and +completion requests support JSON responses, SSE streaming, usage counts, +`max_tokens`/`max_completion_tokens`, `temperature`, and `top_p`. The extension +`enable_thinking: true` enables GLM-5.2's reasoning block; the standard +`reasoning_effort` field also enables it unless set to `none`. + +The server is deliberately text-only and serves one generation at a time: the +744B model stays in one persistent process, so concurrent HTTP requests queue +instead of loading duplicate model copies. Tools, image/audio input, custom +stop sequences, log probabilities, and token penalties return an explicit error +rather than being silently ignored. The default bind address is localhost; set +`COLI_API_KEY` before exposing the server beyond the machine. + +Browser access from the Vite development server and Tauri local origins is +enabled by default. Repeat `--cors-origin https://your-ui.example` to allow +another exact origin, or use `--cors-origin '*'` only on a trusted local +network. + +The engine owns its KV contexts, so HTTP generation uses a bounded FIFO +admission queue instead of pretending to run unsafe parallel sequences. +Configure it with `--max-queue N` (default 8) and `--queue-timeout SECONDS` +(default 300), or the `COLI_MAX_QUEUE` / `COLI_QUEUE_TIMEOUT` environment +variables. Saturated and timed-out requests receive OpenAI-shaped HTTP 429 +errors before streaming headers are sent. `GET /health` exposes +active/queued/completed/rejected counters, and successful generation responses +include `x-colibri-queue-wait-ms`. + +## Connect a coding CLI or editor + +The API is OpenAI-compatible, so most coding CLIs and editor extensions work by +pointing them at Colibri as an *OpenAI-compatible* provider. Three settings: + +- **Base URL** — `http://localhost:8000/v1` +- **Model** — `glm-5.2-colibri` (or whatever you pass to `--model-id`) +- **API key** — any non-empty string, e.g. `local` + +Colibri needs **no** API key by default, but many clients refuse to start without +one — give them any dummy value. The key is only enforced if you set `COLI_API_KEY`. + +Smoke-test the endpoint first (no key needed unless you set one): + +```bash +curl http://127.0.0.1:8000/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"glm-5.2-colibri","messages":[{"role":"user","content":"hi"}]}' +``` + +**aider** + +```bash +export OPENAI_API_BASE=http://localhost:8000/v1 +export OPENAI_API_KEY=local +aider --model openai/glm-5.2-colibri # the openai/ prefix routes to OPENAI_API_BASE +``` + +**crush** — add a provider to `crush.json` (`~/.config/crush/crush.json`, or +`%USERPROFILE%\AppData\Local\crush\crush.json` on Windows): + +```json +{ + "$schema": "https://charm.land/crush.json", + "providers": { + "colibri": { + "name": "Colibri", + "type": "openai-compat", + "base_url": "http://localhost:8000/v1/", + "api_key": "local", + "models": [ + { "name": "GLM-5.2 (Colibri)", "id": "glm-5.2-colibri", + "context_window": 131072, "default_max_tokens": 1024 } + ] + } + } +} +``` + +The `"api_key": "local"` dummy is what satisfies clients that demand a key. +`context_window` is only the client's budget display — set it to whatever your +KV configuration actually allows. + +**Continue, Cline / Roo, `llm`, the OpenAI SDKs, …** — set the provider's base +URL to `http://localhost:8000/v1`, the model to `glm-5.2-colibri`, and any dummy +key (`OPENAI_API_KEY` / `OPENAI_BASE_URL` for env-based tools). + +> **Set your expectations before connecting an agentic CLI.** Two costs dominate, +> and the first one is invisible until you know it's there: +> +> 1. **Prefill.** Coding agents (crush, aider in repo-map mode, Cline, …) send a +> large system prompt plus tool definitions — often 10–20k tokens — *before +> your first word*. Prefill on the CPU-streaming path runs at a few tokens per +> second (it is attention-bound, see #153), so a 15k-token agent preamble is +> **an hour of silent "thinking" before the first output token**. The client +> looks hung; it isn't. Smoke-test with the tiny `curl` above first — if that +> answers in about a minute, the pipeline works and what you're paying for is +> prompt size. +> 2. **Decode.** Roughly 1 tok/s for a large model, so multi-turn agent loops +> (which re-pay the growing context every turn) compound the cost. +> +> Practical guidance: single surgical asks with a short context work; iterative +> agent sessions against a disk-streaming 744B model do not resemble a hosted +> API and mostly won't be worth the wait. If your client lets you trim or disable +> its system preamble and tool catalog, do it. + +## Isolated KV contexts + +`coli serve --kv-slots N` allocates up to 16 independent sequence contexts. +Requests select one with the optional integer `cache_slot` field; ordinary +OpenAI clients omit it and keep the original slot 0 behavior. + +```json +{ + "model": "glm-5.2-colibri", + "messages": [{"role": "user", "content": "Continue this conversation"}], + "cache_slot": 1 +} +``` + +Each slot owns its token history, compressed MLA/DSA KV memory, MTP window, and +crash-safe persistence file (`.coli_kv`, `.coli_kv.1`, ...). The engine matches +each request's tokenized prompt against the slot's history and reuses the common +KV prefix, so stateless HTTP turns keep their cache across requests and even +across engine restarts. Use `COLI_KV_SLOTS=N` as the environment equivalent. +Start small: at the default 4096-token context, every slot costs hundreds of MB. + +## Web dashboard + +One command serves the OpenAI-compatible API **and** the web console on the +same port, then opens your browser when the engine is ready: + +```bash +cd web && npm install && npm run build # once +./coli web --model +``` + +What you get: + +- **Chat** with live metrics: a flashing token counter while generating, then + tok/s, time-to-first-token, prompt→completion counts and queue wait; +- **Runtime panel**: your hardware (CPU, GPUs + VRAM, RAM, cores), the + scheduler, and the live expert-tier bar — how many of the 19,456 experts sit + in VRAM / RAM / disk right now; +- **Brain**: the whole model as a 76×256 cortex, one cell per expert. Colour = + tier, brightness = routing heat, and the experts routed in each turn flash + white and decay — you watch the model think. Hover any cell for its tier, + heat and [measured topic affinity](https://github.com/JustVugg/colibri/issues/175); +- **Atlas**: the measured expert atlas as a 3-D galaxy (publish `experts.json` + from `tools/expert_atlas/analyze.py --web`). + +The dashboard talks to the engine over a small line protocol and plain JSON +endpoints — nothing heavier than the engine itself. `web/` is a pure OpenAI-API +client (React + TypeScript) and also works against any other compatible +endpoint; the terminal `coli chat` remains the first-class interface. + +The layout is responsive down to phone widths, and the sidebar carries the full +telemetry stack — hardware, scheduler, tier bar, per-turn time breakdown, tok/s +trend and per-GPU expert counts: + +

+ the dashboard on a phone-sized viewport +    + the telemetry sidebar +

diff --git a/docs/benchmarks.md b/docs/benchmarks.md new file mode 100644 index 0000000..09d3752 --- /dev/null +++ b/docs/benchmarks.md @@ -0,0 +1,144 @@ +# Benchmarks & measured numbers + +Everything on this page is a measurement, not a promise. If you run colibrì on +hardware not listed here, **please open an issue with your numbers** — real +datapoints are what move this project. + +## Reference numbers (the original dev box: WSL2, 12 cores, 25 GB RAM, NVMe via VHDX) + +Detailed GPU experiment: [GLM-5.2 on 6× RTX 5090](experiments/glm52-6x5090-2026-07-12.md) — +full expert residency across VRAM+RAM reaches **6.84 tok/s** single-request decode. + +| metric | value | +|---|---| +| model on disk (int4 container) | ~370 GB | +| resident RAM (dense, int4) | 9.9 GB | +| load time | ~30 s | +| peak RSS during chat | ~20 GB (auto-capped) | +| cold decode cost | ~11 GB disk reads/token (75 layers × 8 experts) | +| disk ceiling (this dev box's drive) | ~1 GB/s → ~0.05–0.1 tok/s cold | +| MTP speculation (int8 head) | 2.2–2.8 tok/forward measured ([#8](https://github.com/JustVugg/colibri/issues/8)) | + +This is not fast. It is a 744B frontier-class model **answering correctly on a +machine that costs less than one H100 fan**. Warm cache, pinned hot experts and +MTP push the useful-response latency down considerably; the physics of the disk +does the rest. + +### SSD note + +Cold starts are heavy on random reads (~11 GB/token), but reads don't +meaningfully wear an SSD — colibrì's streaming is read-only. The real concerns +under heavy use are (1) **swap traffic** if the system runs out of RAM (writes +do wear the drive — keep a sane `--ram` budget; colibrì's auto-budget is designed +to stay clear of swap) and (2) **sustained thermals**: hours at full read duty +cycle will heat cheaper drives. Monitor drive temperature and health. + +## Test your machine, in order + +```bash +cd c && ./setup.sh # build + architecture self-test (expects 32/32) + +# 1) measure YOUR disk the way the engine uses it (parallel 19 MB random reads): +gcc -O2 -fopenmp iobench.c -o iobench +./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 0 # buffered, 8 threads +./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 1 # O_DIRECT (bypass cache) +# Caveat (#86): iobench reads a bounded ~1 GB shard, so buffered reads on a big-RAM box +# report the PAGE CACHE, not the disk. Use the O_DIRECT run (arg 1) for a true number, and +# run it on a shard you haven't touched this session (a prior buffered run caches its pages). +# On macOS there is no O_DIRECT — iobench uses F_NOCACHE, which stops *new* caching but can't +# evict pages a prior buffered run already resident-mapped, so a macOS "O_DIRECT" figure right +# after a buffered run still reads cache. Reboot or use a fresh shard for a real cold read. + +# 2) chat; watch the per-turn stats line (tok/s, expert hit-rate, RSS): +COLI_MODEL=/path/to/glm52_i4 ./coli chat + +# 3) record expert usage, then pin the hottest experts in your spare RAM: +STATS=stats.txt ./coli chat +PIN=stats.txt PIN_GB=20 ./coli chat # scale PIN_GB to your free RAM + +# 4) quality benchmarks (MMLU/HellaSwag/ARC): +./coli bench +``` + +## Back-of-envelope predictions + +Decode is disk-bound: a cold token costs ~11.4 GB of expert reads; MTP +speculation roughly halves the effective cost *once the cache is warm*; RAM +turns cold reads into free cache hits. + +| machine | expected | +|---|---| +| the dev box (WSL2 VHDX, ~1 GB/s, 25 GB RAM) | ~0.05–0.1 tok/s cold — proven baseline | +| native Linux, PCIe4 NVMe (~3–5 GB/s random), 32 GB | ~0.5–1 tok/s | +| PCIe5 NVMe or 2×NVMe RAID0 (~8–12 GB/s), 64 GB (PIN ~40 GB of hot experts) | ~2–4 tok/s | +| 128–256 GB RAM, 12 cores (hot experts cached) | ~2–4 tok/s — matmul-bound: ~80 GFLOP/token vs ~250 GFLOP/s of our AVX2 kernels | +| same RAM + 24–32 cores, or AVX-512/VNNI kernels | ~5–15 tok/s — interactive; kernel work is the multiplier | + +These are estimates, not measurements. + +## Community benchmarks (measured) + +Real numbers from real machines, stock build (`setup.sh`, gcc 13), greedy decoding, `--ngen 32`, MTP active: + +| machine | disk (iobench, 19 MB × 64, 8 threads) | config | measured | +|---|---|---|---| +| Intel Core Ultra 7 270K Plus (24 threads) · WSL2 · 24 GB RAM · NVMe VHDX ([#2](https://github.com/JustVugg/colibri/issues/2)) | 1.96 GB/s buffered · 2.74 GB/s O_DIRECT | default | 0.07 tok/s · expert hit 3–4% · RSS 14.1 GB | +| 〃 | 〃 | `--topp 0.7` | **0.11 tok/s** · expert hit 11% · RSS 14.7 GB | +| Apple M5 Max (18 cores) · macOS · 128 GB unified · internal SSD ([#4](https://github.com/JustVugg/colibri/issues/4), [#5](https://github.com/JustVugg/colibri/issues/5)) | ~4 GB/s cold (the 14.2 GB/s reading was cache-influenced — see note) | default, MTP off | **1.06 tok/s** · expert hit 23% · RSS 21.8 GB | +| Apple M5 Max · macOS · 128 GB unified · 2 TB SSD · **Metal backend** ([#72](https://github.com/JustVugg/colibri/pull/72), [#87](https://github.com/JustVugg/colibri/issues/87)) | (macOS O_DIRECT figure unreliable — see note) | Metal on · `--ram 96` · 39.7 GB warm pin · MTP off | **1.83 tok/s** · expert hit 66% · warmed 1.11 → 1.83 over the run | +| 〃 · 46.9 GB pin (2.94M-selection history) · `--ram 110`, 1024-token run ([#103](https://github.com/JustVugg/colibri/issues/103)) | 〃 | Metal on (experts + attention) · MTP off | **2.06 tok/s** · hit 72.5% · coherent output | +| Mac Mini M4 Pro · macOS · **48 GB** unified · **Metal backend** ([#107](https://github.com/JustVugg/colibri/issues/107)) | 6.59 GB/s F_NOCACHE (fresh shard) | Metal on · `--ram 38` | **0.30 tok/s** (vs 0.18 CPU-only) | +| Epyc 9654 ES · Linux · 4x16GB DDR5-4800-rdimm · Samsung PCIe Gen3 x4 NVME SSD | — | `MTP=1 DIRECT=1` | 0.31 tok/s · expert hit 35% · RSS 21.52 GB | +| Ryzen AI 9 HX 370 (Framework 13) · Arch Linux · 128 GB · WD SN850X, BTRFS zstd ([#12](https://github.com/JustVugg/colibri/issues/12)) | — | int8 MTP head · `--cap 32` · 46.7 GB auto-learned PIN | **0.37 tok/s** · expert hit 66% · MTP acceptance 52% (2.59 tok/fw) · RSS 105 GB | +| Ryzen 9 9950X (32 threads) · Linux · 123 GB · Crucial P3 QLC Gen3 ([#31](https://github.com/JustVugg/colibri/issues/31)) | 1.51 GB/s buffered | default, 2 runs from cold | 0.10 tok/s · hit 53% · profile 66% disk | +| 〃 same machine, model moved to a Samsung 9100 PRO PCIe 5.0 ([#31](https://github.com/JustVugg/colibri/issues/31)) | **8.81 GB/s** O_DIRECT | 〃 (usage history retained) | **0.28 tok/s** · hit 57% · profile flips: 32% disk / **57% matmul** | +| Ryzen AI Max+ 395 (Framework Desktop) · Ubuntu · 128 GB LPDDR5x · Intel Optane 905p PCIe 3.0 ([#39](https://github.com/JustVugg/colibri/issues/39)) | 3.27 GB/s buffered | int8 MTP head · fresh history (pure LRU, auto-raised cap 65) | 0.16 tok/s · hit 57% · profile 49% disk / 47% matmul | +| 〃 five runs later — learned pin 47.6 GB ([#39](https://github.com/JustVugg/colibri/issues/39)) | 〃 | `--temp 0.7 --topp 0.7` | **0.40 tok/s** · hit 71% | +| Ryzen 7 9800X3D (16T) · WSL2 · 70 GB RAM · Samsung 9100 PRO PCIe 5.0 · RTX 5090 ([#101](https://github.com/JustVugg/colibri/issues/101)) | **10.51 GB/s** O_DIRECT | MTP off · learned pin 24 GB · hit 54% · OMP hot-team on | **0.41 tok/s** · disk-bound (36.5 s disk vs 24.0 s matmul) · **CUDA expert tier ≈ 0%** (AVX-512 CPU matches the 5090) · `--topp 0.7` → **0.52 tok/s** | +| EPYC 7443 (24C/48T, Zen3 AVX2) · Linux · **430 GB RAM** · NVMe RAID-Z1 via TrueNAS VM ([#104](https://github.com/JustVugg/colibri/issues/104)) | ~1 GB/s (VM overhead) | 77.5 GB pin · cap auto-raised to 194/layer · MTP off | **1.00 tok/s** · **hit 98%** · disk eliminated → **RAM-bandwidth + matmul bound** | +| Intel i5-12600K (10C/16T, AVX2) · **native Windows 11, no WSL** · 32 GB · MinGW GCC 16.1 ([#113](https://github.com/JustVugg/colibri/issues/113)) | buffered (no O_DIRECT on MinGW) | int8 MTP head · cold, small-RAM (cap ~2/layer) | **0.08 tok/s** · hit 3.7% · **MTP 57% acceptance** — first native-Windows datapoint | +| Ryzen 9 9950X3D2 (16C/32T, avx512-vnni) · native Linux · 121 GB · Samsung 9100 PRO **PCIe Gen5** · RTX 5090 (28 GB expert tier, 1475 pinned) ([#120](https://github.com/JustVugg/colibri/issues/120)) | **11.48 GB/s** O_DIRECT | `MTP=0 DIRECT=1 PIPE_WORKERS=16 PREFETCH=1` | **1.23 tok/s** | +| Ryzen AI Max+ 395 (Strix Halo, 16C/32T Zen5, avx512-vnni) · Arch Linux · 128 GB unified LPDDR5x · SK hynix P41 PCIe 4.0 ([#124](https://github.com/JustVugg/colibri/issues/124)) | — | `DIRECT=1 PIPE=1 --topp 0.7` · auto-pin | 0.06 cold → **1.10 tok/s** sustained · later **1.83 tok/s** on current dev with `DIRECT=1 PIPE=1 PILOT_REAL=1 PILOT_TWO=1` ([#200](https://github.com/JustVugg/colibri/issues/200)) | +| Intel Core Ultra 9 185H (16C/22T, avx-vnni) · **native Windows 11, no WSL** · 32 GB · Crucial P3 QLC NTFS · RTX 5070 Ti ([#128](https://github.com/JustVugg/colibri/issues/128), [#273](https://github.com/JustVugg/colibri/issues/273)) | — | int8 MTP head · warm cache · GPU-resident pipeline at decode | 0.03 cold → 0.5 warm CPU → **1.07 tok/s** with the pipe2 decode gate (#274) | +| Dell Pro Max GB10 (DGX Spark: Grace, **aarch64 i8mm/sve2**) · Linux · 121 GB unified LPDDR5x · GB10 sm_121 ([#136](https://github.com/JustVugg/colibri/issues/136), [#161](https://github.com/JustVugg/colibri/issues/161)) | **5.58 GB/s** O_DIRECT | int8 MTP head · warm cache | 0.50 tok/s warm · **2.4 tok/s full-k8**, **3.33 tok/s** with `CACHE_ROUTE` (#199) | +| **6 × RTX 5090 · dual Xeon Silver 4510 · 251 GB** (author's rig, [experiment log](experiments/glm52-6x5090-2026-07-12.md)) | NVMe | `CUDA_EXPERT_GB=auto PIN_GB=all` full residency · `COLI_CUDA_PIPE=2 TC_W4A16` · DRAFT=0 | **5.8–6.8 tok/s** decode · TTFT ~13 s · hit 89–100% | + +### Takeaways + +With 24 GB of RAM the engine auto-caps the expert cache to 2 slots/layer, so +decode stays cold even on a fast disk — **on small-RAM machines the RAM cap, not +the disk, is the binding constraint**; `--topp 0.7` alone bought a clean 1.6× +end-to-end speedup. The 9950X pair is the cleanest bottleneck experiment: same +machine, same history, only the disk swapped — ×5.8 disk bandwidth bought ×2.9 +tokens, and the profile **flipped from 66% disk to 57% matmul**. But the +crossover depends on the CPU kernel: with OMP hot-team tuning on, an AVX-512 CPU +can match an RTX 5090 on expert matmul ([#101](https://github.com/JustVugg/colibri/issues/101)), +so **the GPU tier earns its VRAM only when the CPU is the weak link**. On +multi-socket hosts, NUMA placement is a further lever: interleaving the resident +weights across nodes measured **+13% (2-socket) and +40% (4-socket CPU-only)** +([#82](https://github.com/JustVugg/colibri/issues/82)). On a 2-socket Xeon Silver +4510 host with 6× RTX 5090, selective `COLI_NUMA=1` raised effective CPU-expert +bandwidth from **42.42 to 58.26/65.89 GB/s** and greedy decode from **7.66 to +9.02/9.17 tok/s** (64 tokens, `TEMP=0 DRAFT=0`, byte-identical output). Do not +blanket-interleave a GPU host: it also spreads DMA staging pages and has measured +up to a 10× regression; generated plans enable only the selective slab policy. + +## Quality benchmark + +**Measured** ([#108](https://github.com/JustVugg/colibri/issues/108)): the int4 +container scored **62.5% mean acc_norm** on hellaswag/arc/mmlu (0-shot +log-likelihood, n=40) — but 0-shot MC scoring underserves a reasoning model, and +the OLMoE fp16-vs-int4 A/B under the same harness measured the pure quantization +cost at **-8.2pp**, concentrated on the hardest task (per-row int4 scales erode +the small logit margins hard questions depend on — grouped scales recover ~63% +of that loss, see [#225](https://github.com/JustVugg/colibri/issues/225)). The +scale-granularity/rotation/lattice ablation lives in +`tools/quant_ablation.py` ([#81](https://github.com/JustVugg/colibri/issues/81)). + +```bash +cd c +pip install tokenizers datasets +./coli bench # hellaswag, arc_challenge, mmlu — 40 questions each +./coli bench hellaswag --limit 200 # one task, more questions +./coli bench mmlu arc_challenge --ram 100 # pick tasks, set a RAM budget +``` diff --git a/docs/cuda.md b/docs/cuda.md new file mode 100644 index 0000000..fab477d --- /dev/null +++ b/docs/cuda.md @@ -0,0 +1,104 @@ +# CUDA backend (Linux) + +colibrì includes an opt-in CUDA backend for model-resident tensors. Streaming +experts deliberately remain on the original CPU path: copying an expert from +NVMe to the GPU on every use would only replace the disk bottleneck with a PCIe +bottleneck. Resident quantized tensors are uploaded lazily once and reused. + +```bash +cd c +make cuda-test CUDA=1 # q8/q4/q2/f32 kernel correctness +make CUDA=1 +# optional dense-path experiment (hot experts are configured below) +COLI_CUDA=1 COLI_GPU=0 CUDA_DENSE=1 SNAP=/nvme/glm52_i4 ./glm 64 4 4 +``` + +Requirements: Linux, an NVIDIA driver, and a CUDA Toolkit under +`/usr/local/cuda` (override with `CUDA_HOME=/path/to/cuda`). +`CUDA_ARCH=native` builds for the GPU in the current machine. Requesting CUDA +with a CPU-only binary, an invalid device, or an unavailable runtime fails at +startup instead of silently falling back. For Windows, see +[windows.md](windows.md) (runtime DLL path). + +## The VRAM expert tier + +A measured `PIN` profile promotes its hottest experts into a persistent VRAM +tier while keeping the rest in RAM: + +```bash +STATS=stats.txt SNAP=/nvme/glm52_i4 ./glm 64 4 4 # collect routing frequencies first +COLI_CUDA=1 COLI_GPU=0 CUDA_EXPERT_GB=16 \ +PIN=stats.txt PIN_GB=160 SNAP=/nvme/glm52_i4 ./glm 64 4 4 + +# multi-GPU expert tier, 150 GB total budget across six 32 GB devices +COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=150 \ +CUDA_DENSE=1 PIN=stats.txt PIN_GB=300 RAM_GB=226 \ +SNAP=/nvme/glm52_i4 ./glm 64 4 4 + +# large-RAM host: fill safe VRAM, then keep every remaining expert in RAM +COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=auto \ +CUDA_DENSE=1 COLI_CUDA_ATTN=1 PIN=stats.txt PIN_GB=all RAM_GB=auto \ +SNAP=/nvme/glm52_i4 ./glm 64 4 4 +``` + +Selected experts are uploaded during startup, so capacity failures occur before +inference. The budget is clamped against free VRAM after reserving the projected +dense resident set and 2 GB of runtime headroom per device. With `COLI_GPUS`, +`CUDA_EXPERT_GB` is a total budget across the device set; experts are assigned +whole to the least-loaded device that can hold them. Multi-GPU runs default to +`PIN_FILL=1` (measured hot set first, then unused VRAM filled with zero-heat +experts) and `CUDA_RELEASE_HOST=1` (RAM copy released after upload, reloaded +from disk only if CUDA later fails). + +`CUDA_EXPERT_GB=auto` fills each device up to measured free memory minus +projected dense tensors and headroom. `PIN_GB=all` then loads the remaining +routed experts into RAM **up to the `--ram` budget** (it clamps — [#229](https://github.com/JustVugg/colibri/issues/229)), +eliminating decode-time disk misses when capacity permits. This mode is intended +for dedicated high-memory inference hosts. + +### Full-residency reference result (6× RTX 5090, 251 GiB host) + +`CUDA_EXPERT_GB=auto PIN_GB=all` selected a 176.7 GB VRAM tier + 191.3 GB RAM +tier (all 19,456 experts resident), adapting the VRAM tier every 16 tokens. +With the GPU-resident pipeline (`COLI_CUDA_PIPE=2`) and Tensor-Core W4A16 +dispatch (`COLI_CUDA_TC_W4A16=1`), 96-token greedy decode measured +**5.8–6.8 tok/s** (TTFT ~13 s; 1571-token prefill ~122 s then 4.2 tok/s). +Full experiment log: [experiments/glm52-6x5090-2026-07-12.md](experiments/glm52-6x5090-2026-07-12.md). +These are host-specific capacity results, not portable defaults. + +## The GPU-resident pipeline (`COLI_CUDA_PIPE`) + +`COLI_CUDA_PIPE=2` keeps the residual stream on-device across layers: rmsnorms, +residual adds, router GEMMs and the shared expert run on the GPU while the CPU +expert loop runs uninterrupted, with batched attention and grouped expert +uploads at prefill. On a single-GPU host this also pays at decode (S=1): +**+49%** measured on a 5070 Ti ([#273](https://github.com/JustVugg/colibri/issues/273)/#274); +on multi-GPU hosts the per-layer P2P hops cancel the gain, so the decode gate is +device-count aware. `COLI_CUDA_TC_W4A16=1` enables Tensor-Core int4×fp16 mixed +dispatch for batched rows (pays at ≥16 rows). + +## Notes and limitations + +- Text-mode timing reports prefill separately from decode. +- MTP speculation defaults off on CUDA (cold draft routes increase expert + traffic); explicit `DRAFT=n` overrides. Since #294, `SPEC_PIN=1` keeps + draft/verify kernels consistent when speculation is on. +- Devices use independent contexts; a single expert is not sharded. Kernels are + correctness-first custom kernels. +- Profile quality matters more than raw VRAM capacity: the same 150 GB tier + measured 0.94–1.64 tok/s hot-first vs 0.29 tok/s filled without routing heat. +- The GPU tier earns its VRAM only when the CPU is the weak link — a tuned + AVX-512 CPU can match a 5090 on expert matmul + ([#101](https://github.com/JustVugg/colibri/issues/101)). + +## Reproducible backend A/B without the full checkpoint + +```bash +cd c +python tools/make_glm_bench_model.py --output /nvme/colibri-bench-medium --device cuda +python tools/benchmark_cuda_fixture.py --model /nvme/colibri-bench-medium --gpu 0 +``` + +The 313M-parameter fixture has random weights and is not a language model. It +preserves the real MLA/MoE/streaming shapes to compare CPU streaming, dense-only +CUDA, CPU hot-store, and CUDA hot-expert execution with identical replay tokens. diff --git a/docs/media/colibri-atlas.png b/docs/media/colibri-atlas.png new file mode 100644 index 0000000..5e5e76e Binary files /dev/null and b/docs/media/colibri-atlas.png differ diff --git a/docs/media/colibri-brain.png b/docs/media/colibri-brain.png index 0cd6c90..9ac5676 100644 Binary files a/docs/media/colibri-brain.png and b/docs/media/colibri-brain.png differ diff --git a/docs/media/colibri-dashboard.png b/docs/media/colibri-dashboard.png index 123fd3e..d51aa08 100644 Binary files a/docs/media/colibri-dashboard.png and b/docs/media/colibri-dashboard.png differ diff --git a/docs/media/colibri-metrics.png b/docs/media/colibri-metrics.png new file mode 100644 index 0000000..853d686 Binary files /dev/null and b/docs/media/colibri-metrics.png differ diff --git a/docs/media/colibri-mobile.png b/docs/media/colibri-mobile.png new file mode 100644 index 0000000..5c1c63c Binary files /dev/null and b/docs/media/colibri-mobile.png differ diff --git a/docs/media/ladder.png b/docs/media/ladder.png new file mode 100644 index 0000000..cd272fc Binary files /dev/null and b/docs/media/ladder.png differ diff --git a/docs/media/sparse.png b/docs/media/sparse.png new file mode 100644 index 0000000..0d7a4a1 Binary files /dev/null and b/docs/media/sparse.png differ diff --git a/docs/media/tiers.png b/docs/media/tiers.png new file mode 100644 index 0000000..9452587 Binary files /dev/null and b/docs/media/tiers.png differ diff --git a/docs/media/token-path.png b/docs/media/token-path.png new file mode 100644 index 0000000..4aebd39 Binary files /dev/null and b/docs/media/token-path.png differ diff --git a/docs/metal.md b/docs/metal.md new file mode 100644 index 0000000..3c94e5c --- /dev/null +++ b/docs/metal.md @@ -0,0 +1,30 @@ +# Metal backend (Apple Silicon, experimental) + +On Apple Silicon the decode profile is matmul-bound, and unified memory removes +the PCIe copy tax that keeps CUDA's streaming experts on the CPU — so colibrì +has an opt-in Metal backend that runs the **routed-expert SwiGLU (batched, +zero-copy from the RAM slabs)**, the **fused decode attention** (full MLA layer +in one command buffer, S≤4), and **prefill's large GEMMs** on the GPU. +Token-exact vs the CPU path. + +```bash +cd c +make glm METAL=1 # macOS only; no Xcode needed (shader compiles at runtime) +make metal-test # standalone kernel/attention correctness vs CPU reference +COLI_METAL=1 COLI_MODEL=/path/glm52_i4 ./coli chat --ram 96 +``` + +Measured on an M4 Max (128 GB, warm cache, MTP on): CPU 0.30 → Metal +**0.42 tok/s (~1.4×)** (best config adds `DIRECT=1`; ~3× vs this machine's +first cold run). An M5 Max with a 46.9 GB learned pin reached **2.06 tok/s** +([#103](https://github.com/JustVugg/colibri/issues/103); see also the +[M5 Max performance report](METAL-M5MAX-PERF-REPORT.md)). + +Key design points: Metal's ~5 ms submit latency makes per-matmul dispatch a +loss — everything is batched into few command buffers per layer, and the +resident experts' GPU work is submitted *before* the missed experts' disk reads +so I/O and compute overlap. `COLI_METAL_GEMM_MIN` tunes the prefill GEMM row +threshold (default 16). Streaming, cache, MTP, DSA and the persistence formats +are unchanged; every GPU path falls back to the CPU per-block on any fault. +Numerics are dequant→f32-MAC (same as the CUDA tier); greedy outputs are +byte-identical to the CPU engine. diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 0000000..7d6a9bf --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,164 @@ +# Quick Start — from zero to a running model + +A step-by-step guide for first-time users on **Linux**, **Windows**, and **macOS**. +No prior experience with C, CUDA, or model conversion is assumed. If you get +stuck, `./coli doctor` (below) tells you exactly what's missing. + +> **What you're setting up:** colibrì runs a very large Mixture-of-Experts model +> (e.g. GLM-5.2, 744B parameters) on a normal machine by streaming the model's +> experts from disk instead of needing them all in RAM. The engine is a single +> C program; Python is only used once, to prepare the model files. + +--- + +## 0. What you need first (prerequisites) + +| | Minimum | Recommended | +|---|---|---| +| **RAM** | ~16 GB | 24 GB+ | +| **Free disk** | ~380 GB for the int4 model | a fast NVMe SSD (streaming speed = your token speed) | +| **OS** | Linux, Windows 10/11, or macOS | any | +| **Tools** | a C compiler + `make` + `git` + `python3` | — | + +You do **not** need a GPU. A GPU only helps if you have one; the engine runs +CPU-only by default. + +--- + +## 1. Install the build tools + +### Linux (Ubuntu / Debian) + +```bash +sudo apt update +sudo apt install -y build-essential git python3 +``` + +`build-essential` gives you `gcc`, `make`, and OpenMP (libgomp) — everything the +engine needs. + +### Windows + +You have two options. + +**Option A — download a prebuilt binary (no compiler needed).** +Grab the latest `colibri--windows-x86_64.zip` from the +[Releases page](https://github.com/JustVugg/colibri/releases), unzip it, and +skip to [step 3](#3-get-the-model). Python 3 (from +[python.org](https://www.python.org/downloads/)) is still needed if you want to +convert a model yourself. + +**Option B — build from source with MSYS2.** +Install [MSYS2](https://www.msys2.org/), open the **UCRT64** shell, and run: + +```bash +pacman -S --needed mingw-w64-ucrt-x86_64-gcc make git python +``` + +### macOS + +```bash +xcode-select --install # C compiler (clang) +brew install libomp git python # OpenMP for multithreading +``` + +--- + +## 2. Get the code and build the engine + +```bash +git clone https://github.com/JustVugg/colibri.git +cd colibri/c +./setup.sh +``` + +`setup.sh` checks your compiler and OpenMP, builds the engine, and runs a tiny +self-test. When it prints: + +``` +engine self-test: 32/32 (expected 32/32) +``` + +the engine is working correctly. (On Windows Option A you already have the +binary — you can skip this step.) + +--- + +## 3. Get the model + +You have two paths. + +### Easiest — download a ready-made int4 container + +A pre-converted **GLM-5.2 int4** model is on Hugging Face. **Use the version +with the int8 MTP heads** (the plain int4 heads disable speculative decoding — +see [#8](https://github.com/JustVugg/colibri/issues/8)): + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +Download it into a folder on a fast disk, e.g. `/nvme/glm52_i4` (Linux/macOS) or +`D:\glm52_i4` (Windows). It is about **372 GB**, so make sure you have the space. + +### Or convert it yourself from the FP8 source + +One resumable command downloads and converts the model shard by shard, so it +never needs the full ~756 GB on disk at once: + +```bash +./coli convert --model /nvme/glm52_i4 +``` + +This step uses Python and runs only once. Safe to interrupt and re-run — it +resumes where it left off. + +--- + +## 4. Run it + +Point `COLI_MODEL` at the folder from step 3 and start chatting: + +```bash +# Linux / macOS +COLI_MODEL=/nvme/glm52_i4 ./coli chat + +# Windows (UCRT64 shell) +COLI_MODEL=/d/glm52_i4 ./coli chat +``` + +Useful first commands: + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # read-only check: is everything ready? +COLI_MODEL=/nvme/glm52_i4 ./coli plan # shows where the model will live (RAM/disk/GPU) +COLI_MODEL=/nvme/glm52_i4 ./coli chat --topp 0.85 # faster: reads less from disk, same quality +``` + +> **Tip:** `--topp 0.85` is worth adding on a disk-bound machine — it reads +> fewer expert bytes per token with no quality loss, which directly means more +> tokens per second. + +--- + +## 5. What to expect + +- **First launch loads the resident weights** (~10 GB) — this takes a moment. +- **Speed depends on your disk.** The experts stream from storage, so a fast + NVMe SSD is the single biggest factor in tokens/second. On a slow or shared + disk, generation can be well under 1 token/second — that's expected, and it's + the honest cost of running a 744B model on a small machine. +- **It's still the full model.** Placement only changes speed, never the model's + answers or precision. + +If something doesn't work, run `./coli doctor` — it reports exactly what's +missing (compiler, model files, permissions) and how to fix it. + +--- + +## Where to go next + +| Topic | Doc | +|---|---| +| Windows native build (and CUDA DLL) | [docs/windows.md](windows.md) | +| Tuning: cache, prefetch, speculation | [docs/tuning.md](tuning.md) | +| OpenAI-compatible API + web dashboard | [docs/api.md](api.md) | +| Every environment variable | [docs/ENVIRONMENT.md](ENVIRONMENT.md) | diff --git a/docs/serve_protocol.md b/docs/serve_protocol.md new file mode 100644 index 0000000..b0eb61e --- /dev/null +++ b/docs/serve_protocol.md @@ -0,0 +1,107 @@ +# The serve protocols — engine ⇄ server wire format + +The engine speaks two line-oriented protocols over stdin/stdout. Both are plain text +plus byte-counted payload frames; every outbound line is written with a trailing +`fflush`, one line per write. On Windows both ends of the pipe are switched to binary +mode at startup — the CRT's CRLF translation otherwise corrupts the sentinels and +stalls byte-counted reads (#195). + +| protocol | entry | selected by | used by | +|---|---|---|---| +| **mux** (continuous batching, up to 16 KV slots) | `run_serve_mux` | `SERVE_BATCH=1` | `openai_server.py`, `coli web` | +| **legacy** (single slot, interactive) | `run_serve` | `SERVE=1` (without `SERVE_BATCH`) | `coli chat` | + +This document is the reference for the **mux** protocol; the legacy protocol is +summarized at the end. Line formats below are quoted from the emitting `printf`s in +`glm.c` — if this document and the code disagree, the code wins and this file needs a PR. + +## Startup handshake (engine → server) + +``` +\x01\x01READY\x01\x01 +STAT 0 0.00 0.0 +HWINFO | +TIERS +EMAP +``` + +The server must not send requests before `READY`. `HWINFO`/`TIERS`/`EMAP` are +telemetry (see below) and may grow — **servers must ignore line kinds they do not +recognize**; that is the protocol's forward-compatibility rule. + +## Requests (server → engine) + +``` +SUBMIT \n\n +CANCEL \n +``` + +- `id` — non-zero u64, unique among in-flight requests. +- `slot` — KV slot index, `0 … KV_SLOTS-1` (`KV_SLOTS` env, 1–16, default 1). A slot + holds one conversation's KV; the engine matches the tokenized payload against the + slot's history and reuses the common prefix (truncate-and-extend), so stateless + HTTP turns keep their cache. +- `bytes` — exact byte length of `payload` (UTF-8, may contain newlines). The engine + reads exactly that many bytes after the header line, then one trailing `\n`. +- `payload` — the fully rendered prompt (the server owns the chat template). +- EOF on stdin = graceful shutdown: in-flight requests finish first. + +Prefill is serial; decode is continuously batched — every active slot contributes +one row per forward. + +## Responses (engine → server) + +Per request, in order: + +``` +DATA \n\n # a decoded token's text; repeated +TOPK 5 ... ×5 # candidates for the sampled token (SERVE_TOPK=1) +HITS # ~every 6 tokens: routed-expert bitmap since last HITS +REPIN # live re-pin swap events, as they happen +... +DONE STAT +``` + +Errors replace the stream: `ERROR ` with codes `BAD_FRAME`, `BAD_REQUEST`, +`SLOT_BUSY`, `DUPLICATE_ID`, `EMPTY_PROMPT`, `NOT_FOUND` (CANCEL of unknown id), +`CANCELLED`. A `CANCEL` is acknowledged by `ERROR CANCELLED` after the slot's KV +is persisted. + +Immediately before each `DONE` the engine emits a telemetry block for the finished +turn: `HWINFO`, `PERF`, `ENTROPY`, `GPUS`, `TIERS`, `EMAP`, `HITS` (formats below). +`.coli_usage` is persisted at every turn end, not only at exit. + +## Telemetry lines + +| line | format | meaning | +|---|---|---| +| `TIERS` | `TIERS ` | expert count per tier + resident bytes | +| `HWINFO` | `HWINFO \|` | host snapshot (GBs are floats) | +| `EMAP` | `EMAP ` | one byte per expert, row-major over `rows×cols` (sparse layers +MTP × experts): `byte = (tier<<6) \| heat` — 2-bit tier (0 disk / 1 RAM / 2 VRAM), 6-bit log₂-bucketed usage heat | +| `HITS` | `HITS ` | 1 bit per expert, experts routed since the previous `HITS` | +| `PERF` | `PERF
` | this turn's PROFILO deltas, seconds | +| `ENTROPY` | `ENTROPY

…` | per-sparse-layer routing entropy of the turn, bits | +| `GPUS` | `GPUS ( )×n` | per-device VRAM + resident expert count (CUDA builds) | +| `TOPK` | `TOPK 5 ( )×5` | token text hex-encoded so the line stays line-shaped | +| `REPIN` | `REPIN ` | one line per hot-store swap (`REPIN=n` mode) | + +All telemetry is advisory: servers render what they know and skip the rest. + +## HTTP surface (`openai_server.py`) + +- `POST /v1/chat/completions` — OpenAI-compatible; streaming responses emit one extra + SSE frame `data: {"colibri": {stats, perf, topk, entropy, gpus, repin}}` immediately + before `data: [DONE]`; non-streaming responses attach the same object as a + `"colibri"` field. +- `GET /experts` — the latest `EMAP`/`HITS` state: `{rows, cols, map, hits, seq, + gpus, entropy, repin}`. +- `GET /*` — static hosting of `web/dist` (SPA fallback, path-traversal-safe), plus + `experts.json` if published there (the measured expert atlas, #175/#218). + +## Legacy protocol (`run_serve`, `coli chat`) + +Interactive lines are prompts; control frames: `\x02RESET` (clear history), +`\x02MORE` (continue an NGEN-truncated answer), and +`\x02PROMPT [kv_slot]\n\n` +(the pre-mux API mode). Responses are raw text terminated by `\x01\x01END\x01\x01` +plus a `STAT` line. New integrations should use the mux protocol. diff --git a/docs/tuning.md b/docs/tuning.md new file mode 100644 index 0000000..c95eca8 --- /dev/null +++ b/docs/tuning.md @@ -0,0 +1,110 @@ +# Tuning & runtime knobs + +Everything here is opt-in; the defaults are chosen so a plain `./coli chat` +is safe on any machine. See also [SETTINGS.md](SETTINGS.md) and +[ENVIRONMENT.md](ENVIRONMENT.md) for the full variable inventory. + +## The knobs that matter most + +| knob | what it does | +|---|---| +| `--temp T` | token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy) | +| `--topp 0.7` | adaptive expert top-p (30–40% less disk; lossy — prints a warning) | +| `--ngen N` | max tokens per answer (`:more` in chat continues a truncated one) | +| `--repin N` | adapt RAM/VRAM hot experts every N emitted tokens | +| `RAM_GB=` | claim more RAM for the expert cache than the conservative auto-detect | +| `PIN=stats PIN_GB=g` | pin the hottest experts from a measured usage profile | +| `DRAFT=n` | MTP draft depth (0 disables speculation) | +| `GRAMMAR=g.gbnf` | grammar-forced drafts for constrained JSON/NDJSON output ([docs](grammar-draft.md)) | +| `THINK=1` | enable GLM-5.2's reasoning block | +| `PILOT=1` | router-lookahead disk prefetch (see below) | +| `URING=1` | Linux-only batched expert I/O (implies `PIPE=1`) | +| `PIPE=0` | disable the async expert-load pool (default ON — overlaps `pread` with matmul, −18% disk service) | +| `DIRECT=1` | O_DIRECT expert reads (measured **+65%** alone on a Strix Halo, [#200](https://github.com/JustVugg/colibri/issues/200)) | +| `COLI_NUMA=1` | interleave resident weights across NUMA nodes on multi-socket hosts ([#82](https://github.com/JustVugg/colibri/issues/82)) | +| `CACHE_ROUTE=1` | cache-aware max-rank routing (opt-in, [#199](https://github.com/JustVugg/colibri/issues/199)) | +| `AUTOPIN=0` | disable the learning cache's auto-pin | +| `CAP_RAISE=0` | don't auto-grow the expert cache | +| `KVSAVE=0` | disable KV-cache persistence | +| `TF=1` | teacher-forcing validation | + +## Resource policy + +`coli plan` reports the planned hot (VRAM), warm (RAM), and cold backing (disk) +tiers, the reason for each placement, and the expected bottleneck. The default +`--policy quality` and `--policy balanced` modes preserve checkpoint quantization +and router decisions unless `--topk` or `--topp` is passed; those explicit lossy +overrides print a warning and proceed. + +Auto-tier plans size OpenMP from physical cores and bind workers across cores. +Memory-bound quantized kernels can regress sharply when SMT siblings compete for +limited memory channels; explicit `OMP_*` settings always take precedence. + +```bash +coli plan --model /models/glm52_i4 --policy quality +coli run --auto-tier --policy quality "Explain MoE offloading" +# Explicit research-only router reduction: +coli run --policy experimental-fast --topk 4 "Benchmark prompt" +``` + +Disk is an immutable recovery source, not a normal decode target. If the plan +leaves cold expert bytes on disk, speed depends on cache hit rate; output quality +does not. + +Cold expert reads can use a deferred pipeline: resident RAM/VRAM experts execute +while missing experts are loaded in a bounded background I/O pool, then the cold +results join before the layer completes. The pool engages only under `PIPE=1`; +`PIPE_WORKERS=n` sets its worker count (default 8). Profiling reports both disk +service time and the smaller foreground-visible wait time so overlap is explicit. + +`--policy balanced` enables lossless live placement (`REPIN=64`). At safe request +boundaries, a per-layer LFRU score combines decaying session frequency with recent +access and replaces at most four sufficiently colder pinned experts. `--policy +quality` leaves live replacement off by default; `REPIN=0` always disables it. + +## The learning cache + +The engine records which experts your usage actually routes to (`.coli_usage` +next to the model, updated every turn) and at startup automatically pins the +hottest ones in spare RAM — colibrì literally gets faster the more you use it. +`PIN=auto` seeds the pin directly from the live usage history +([#301](https://github.com/JustVugg/colibri/pull/301)). + +**The expert cache auto-sizes to your RAM** (since 2026-07-10): the engine +*raises* the LRU cap to fill your `--ram` budget instead of only lowering it. +If you benchmarked colibrì before that date, rerun — your numbers were capped. + +**Live tier adaptation** (`--repin N`, opt-in): at safe turn boundaries, a +decaying session heat map replaces cold pinned experts with hotter streamed +experts. A 25% hysteresis and a four-swap limit prevent tier thrashing. +Persistent `.coli_usage` remains the long-term signal and is not decayed. + +## Router-lookahead prefetch (`PILOT=1`, experimental) + +GLM-5.2's expert routing is measurably predictable *ahead of time* — applying +layer L+1's router to layer L's post-attention state recalls **71.6%** of the +true top-8 (vs 41.3% for "same experts as last token"). `PILOT=1` issues +next-layer expert readahead from a dedicated I/O thread while the current layer +computes. `PILOT_REAL=1` moves the prefetched loads off the critical path +(measured +11pp hit rate on a big-cache host), and `PILOT_TWO=1` folds the +computed shared-expert into the prediction (+3% recall, +[#200](https://github.com/JustVugg/colibri/issues/200)). On disk-saturated +hosts hint-only PILOT can be net negative — measure on yours. + +## Speculation and reproducibility + +Speculative decoding requires that the draft and verify paths compute the same +function — `SPEC_PIN=1` (default since [#294](https://github.com/JustVugg/colibri/pull/294)) +pins every forward issued while drafts are live to the platform's S=1 kernel +family. For byte-exact reproducibility across runs: `DRAFT=0`, plus `IDOT=0 +COLI_CUDA=0` if you also want kernel-family/GPU independence. Acceptance +percentages are not comparable across engine versions under `--topp` +([#163](https://github.com/JustVugg/colibri/issues/163) has the full story). + +## Conversations reopen warm + +`coli chat` persists the compressed MLA KV-cache to disk after every turn +(`.coli_kv`, ~182 KB/token, appended incrementally, crash-safe). Close the chat, +reopen it tomorrow — the model still remembers the whole conversation and **zero +re-prefill happens**: validated byte-identical to an uninterrupted session. +`:reset` clears it, `KVSAVE=0` disables it. diff --git a/docs/windows.md b/docs/windows.md new file mode 100644 index 0000000..ecb81e7 --- /dev/null +++ b/docs/windows.md @@ -0,0 +1,172 @@ +# Windows 11 native install — a complete walkthrough (no WSL) + +A start-to-finish, reproducible path from a fresh Windows 11 machine to GLM-5.2 generating tokens, with the GPU tier. Every step and every failure mode below was hit and verified on real hardware: Core Ultra 9 285K (AVX-VNNI) / RTX 5080 (sm_120) / 128 GB RAM / Windows 11 24H2 (issue #306). Steps are ordered so the long downloads run while you build. + +## 0. What you need + +| Piece | Why | Get it | +|---|---|---| +| git, Python 3 | clone + `coli` launcher | winget / python.org | +| MinGW-w64 gcc + make | builds the engine (MSVC can't) | `scoop install mingw-winlibs`, MSYS2, or portable **w64devkit** (no admin, unzip and go) | +| CUDA Toolkit ≥ 12.8 | GPU tier; ≥12.8 required for Blackwell/sm_120 | `winget install Nvidia.CUDA` | +| MSVC Build Tools (C++ workload) | nvcc's host compiler for the CUDA DLL | `winget install Microsoft.VisualStudio.2022.BuildTools` + "Desktop development with C++" | +| ~400 GB free on a local NVMe | the int4 model (~370–384 GB) | NTFS is fine; **never** a network mount | + +RAM: 16 GB minimum, more = bigger expert cache = faster. The build itself needs none of the CUDA/MSVC pieces — do the CPU build first, add the GPU tier later. + +## 1. Start the model download first (it's the long pole) + +```powershell +python -m pip install -U "huggingface_hub[hf_transfer]" +$env:HF_HUB_ENABLE_HF_TRANSFER = "1" +hf download --local-dir D:\glm52_i4 +``` + +Use the container recommended in the README (with **int8 MTP heads** — int4 heads silently give 0% draft acceptance). The download is resumable: if it stops, rerun the same command. Expect hours; everything below fits inside them. + +## 2. Build the engine (CPU) + +From a normal PowerShell, in the repo's `c\` directory: + +```powershell +make glm.exe ARCH=native # ARCH=native unlocks AVX-VNNI on Alder Lake+/Arrow Lake +make iobench.exe # disk benchmark, useful before committing to the download +``` + +Warnings about `#pragma comment` and unused variables are normal (MSVC-isms gcc ignores). The engine banner should print `idot: avx-vnni` on VNNI-capable CPUs — if it says avx2, you built without `ARCH=native`. + +### ⚠️ Smart App Control will block your fresh binary + +On Windows 11 machines with **Smart App Control** enforced (`VerifiedAndReputablePolicyState = 1`), running your self-compiled `glm.exe` fails with: + +``` +Program 'glm.exe' failed to run: An Application Control policy has blocked this file +``` + +This is not Defender and not Mark-of-the-Web — SAC blocks *all* unsigned, unknown binaries, which includes anything you compile yourself. **Fix:** Windows Security → App & browser control → Smart App Control settings → **Off**, then **reboot** (the policy only reloads on restart). Note SAC is one-way: re-enabling later requires resetting Windows. If the settings page is missing, the registry equivalent is setting `HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy\VerifiedAndReputablePolicyState` to `0` (admin PowerShell), then rebooting. Check your current state before touching anything: + +```powershell +(Get-ItemProperty "HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy").VerifiedAndReputablePolicyState +# 0 = off, 1 = enforced, 2 = evaluation +``` + +## 3. Build the CUDA DLL (GPU tier) + +nvcc needs MSVC as host compiler, so this one step must run from a shell with the MSVC environment: open **"x64 Native Tools Command Prompt for VS 2022"** from the Start menu (plain PowerShell will fail the `cl` check). Then: + +```cmd +make cuda-dll CUDA_ARCH=sm_120 # match your GPU: sm_120 Blackwell, sm_89 Ada, ... +make glm.exe CUDA_DLL=1 ARCH=native # relink host with the runtime loader +``` + +Two pitfalls, both fixed on current `dev` (#314) but worth knowing on older checkouts: + +- **Spaces in `CUDA_HOME`** (`C:\Program Files\...`) used to break the recipe → fixed; nvcc now comes from PATH and `"$(NVCC)"` is quoted. +- **`make glm.exe CUDA_DLL=1` after a CPU-only build** used to report `up to date` and silently keep the CPU-only binary (GPU tier never engages, no error). Current `dev` has a build-config stamp that forces the relink. On older trees: delete `glm.exe` first. + +Sanity check: first GPU run should print `[CUDA] device 0: , ... sm_XX` and `[CUDA] mode: routed experts + resident dense tensors`. + +## 4. First run + +```powershell +cd \c +$env:OMP_NUM_THREADS = "" +python coli run "Explain what a mixture-of-experts model is." --model D:\glm52_i4 --ngen 48 +``` + +The first run is cold — expect the profile to be dominated by `expert-disk` while the cache warms; hit rate climbs run over run. GPU tier on top: + +```powershell +$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_DENSE="1"; $env:CUDA_EXPERT_GB="4" +python coli run "..." --model D:\glm52_i4 --ngen 64 +``` + +Size `CUDA_EXPERT_GB` so dense (~10 GB) + experts + working set stays under your VRAM. Note MTP speculation is off by default under CUDA (#293, float-accumulation divergence between draft and verify) — `COLI_CUDA_MTP=1` opts back in. + +## 5. Reference numbers from this walkthrough's hardware + +285K / RTX 5080 / 128 GB / NVMe at 5.85 GB/s random-read (19 MB blocks, `iobench`): 0.26 tok/s cold CPU → 0.30 warm CPU (MTP 2.2–2.3 tok/forward) → 0.42 tok/s GPU tier + auto-pin, expert hit 66%, ~65% of wall time in expert-disk. Disk-bound is the expected shape at ~25% expert residency — a faster disk and more RAM move the floor, the GPU moves the compute. + +## Quick failure index + +| Symptom | Cause | Fix | +|---|---|---| +| `An Application Control policy has blocked this file` | Smart App Control | §2 — turn SAC off + **reboot** | +| `cuda-dll ... Error 1` immediately | old tree: spaced CUDA_HOME / MSVC rejects `-Wextra` | update to current `dev` (#314) | +| `glm.exe is up to date` but GPU never engages | old tree: stale CPU-only binary | update to `dev`, or delete `glm.exe` and rebuild | +| `cl.exe (MSVC) not in PATH` | built from plain PowerShell | use the x64 Native Tools prompt | +| `nvcc fatal: unsupported gpu architecture 'sm_120'` | CUDA < 12.8 | install CUDA 12.8+ | +| MTP `0% (0/0)` on CPU path | int4 MTP heads in the container | use the int8-MTP container | +| MTP `draft=0` under CUDA | intended default since #293 | `COLI_CUDA_MTP=1` to opt in | + +--- + +## Reference: build flags & warmup + +# AVX-VNNI: Intel Alder Lake+ (and Meteor Lake+) CPUs have a 128-bit int8 +# dot-product instruction (VPDPBUSD) the engine can use for ~1.3x faster +# quantized matmul. The x86-64-v3 default (portable AVX2) compiles it out; +# build for THIS machine to enable it: +make glm.exe ARCH=native # banner prints "idot: avx-vnni" + +# Verify (tiny model, 2.4 MB): +pip install torch transformers safetensors huggingface_hub +python tools/make_glm_oracle.py # generate tiny oracle +SNAP=./glm_tiny TF=1 ./glm.exe 64 16 16 # expect "32/32 positions" + +# Run with real model: +SNAP=D:\glm52_i4 ./glm.exe 64 4 16 # batch inference +python coli chat --model D:\glm52_i4 # interactive chat +python coli serve --model D:\glm52_i4 # OpenAI-compatible API +``` + +> Windows Store's `python` alias stub is the single most common native-Windows +> trap: install real Python (python.org or `winget install Python.Python.3.12`) +> or disable the alias under *Settings → Apps → App execution aliases*. + +## Warmup (overnight cache priming) + +The engine's expert cache learns from your workload. The included `warmup.ps1` +script runs `coli run` in a loop with diverse prompts to build the +`.coli_usage` histogram unattended, so the next real session starts with a +large, accurate hot-expert pin. Each run saves usage atomically on clean +completion. + +```powershell +.\warmup.ps1 -Rounds 1 -Ngen 32 # ~60-90 min, durable progress +``` + +## NVIDIA GPU (optional, via runtime DLL) + +On Windows the engine is built with MinGW gcc but CUDA kernels require MSVC + +nvcc. The split is clean: build the CUDA backend into a standalone +`coli_cuda.dll` (nvcc + MSVC), then the host `glm.exe` loads it at runtime via +`LoadLibrary` (`c/backend_loader.c`). The host never links cudart directly; if +the DLL is absent the engine falls back to CPU without error. + +```powershell +# Prerequisites: CUDA Toolkit + MSVC Build Tools (cl.exe) + nvcc on PATH. +# Build the DLL from a shell with the MSVC environment set (vcvars64.bat or +# "x64 Native Tools Command Prompt for VS"): +make cuda-dll CUDA_HOME="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" CUDA_ARCH=sm_120 + +# Build the host with the runtime loader (CUDA_DLL=1 adds -DCOLI_CUDA and +# links backend_loader.o instead of cudart): +make glm.exe CUDA_DLL=1 ARCH=native + +# Run with the GPU expert tier (8 GB VRAM budget here; scale to your free VRAM): +$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_EXPERT_GB="8" +python coli chat --model D:\glm52_i4 --topp 0.7 +``` + +The DLL exports the full `extern "C"` surface (including the #111 pipeline ABI); +`backend_loader.c` resolves symbols via `GetProcAddress` on first use. +`ColiCudaTensor*` is opaque to the host (stored, never dereferenced), so the +MSVC-allocated struct is safe across the ABI boundary. `CUDA_ARCH` must match +your GPU's compute capability (e.g. `sm_120` for Blackwell / RTX 50-series, +`sm_89` for Ada / RTX 40-series). A one-shot `build_cuda.bat` wrapper is also +available. + +**Measured on a single RTX 5070 Ti + Core Ultra 9 (32 GB RAM):** CPU-only 0.63 +→ CUDA attention+dense 0.72 → **1.07 tok/s** with the GPU-resident pipeline at +decode ([#273](https://github.com/JustVugg/colibri/issues/273), merged in #274). diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..9b788fa --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1784160687, + "narHash": "sha256-iYL/bixrb6FlHFu/gIuBYzq6c6lM5AAXsXNSWXtIgQc=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "4382ed2b7a6839d4280a9b386db49cbc5907414d", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-26.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix index 368b0a7..584a088 100644 --- a/flake.nix +++ b/flake.nix @@ -26,7 +26,9 @@ version = "1.0"; src = ./.; - nativeBuildInputs = [ pkgs.makeWrapper ]; + # python3 is needed by checkPhase: `make test-c` shells out to + # `python3 tools/run_tests.py` (see c/Makefile, PYTHON ?= python3). + nativeBuildInputs = [ pkgs.makeWrapper pkgs.python3 ]; buildInputs = [ pkgs.gcc @@ -44,18 +46,29 @@ installPhase = '' runHook preInstall - mkdir -p $out/bin - cp c/glm $out/bin/glm - # Wrap coli (the Python CLI) so it finds the right python and the engine - mkdir -p $out/share/colibri - cp c/coli $out/share/colibri/coli - chmod +x $out/share/colibri/coli - cp -r c/tools $out/share/colibri/tools + # Self-contained layout under $out/lib/colibri that mirrors the + # source tree `coli` runs in (see the path-resolution logic at the + # top of c/coli): the engine, the coli CLI script, the support + # modules it imports (openai_server.py, resource_plan.py, + # doctor.py), and tools/ all sit next to each other. + mkdir -p $out/lib/colibri/tools $out/bin + cp c/glm $out/lib/colibri/glm + cp c/coli $out/lib/colibri/coli + chmod +x $out/lib/colibri/coli + cp c/openai_server.py c/resource_plan.py c/doctor.py $out/lib/colibri/ + cp -r c/tools/* $out/lib/colibri/tools/ + # $out/bin holds the user-facing entry points. + ln -s ../lib/colibri/glm $out/bin/glm + + # Wrap coli: point it at the bundled engine (COLI_ENGINE) so it is + # found by default, and at the module dir (PYTHONPATH) so + # `import openai_server` / `resource_plan` / `doctor` resolve. makeWrapper ${pythonEnv}/bin/python $out/bin/coli \ - --add-flags "$out/share/colibri/coli" \ - --set PYTHONPATH "${pythonEnv}/${pkgs.python3.sitePackages}" + --add-flags "$out/lib/colibri/coli" \ + --set-default COLI_ENGINE "$out/lib/colibri/glm" \ + --set PYTHONPATH "$out/lib/colibri:${pythonEnv}/${pkgs.python3.sitePackages}" runHook postInstall ''; diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..18f8f85 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,53 @@ +[build-system] +requires = ["setuptools>=68.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "colibri-engine" +dynamic = ["version"] +description = "Tiny engine, immense model — run GLM-5.2 (744B MoE) locally" +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.10" +authors = [ + {name = "JustVugg"}, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Science/Research", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] + +[project.optional-dependencies] +convert = [ + "numpy", + "huggingface_hub", +] +oracle = [ + "torch>=2.0", + "transformers>=4.40", + "safetensors", +] +bench = [ + "tokenizers", + "datasets", +] + +[project.scripts] +coli = "colibri.cli:main" + +[project.urls] +Homepage = "https://github.com/JustVugg/colibri" +Issues = "https://github.com/JustVugg/colibri/issues" + +[tool.setuptools.dynamic] +version = {attr = "colibri._version.__version__"} + +[tool.setuptools.packages.find] +where = ["."] +include = ["colibri*"] diff --git a/web/README.md b/web/README.md index c867d01..fe502ae 100644 --- a/web/README.md +++ b/web/README.md @@ -17,9 +17,14 @@ npm test npm run build ``` +Besides Chat and Brain, the **Profiling** tab charts where the engine spent +each turn's wall time (I/O wait, expert matmul, attention, LM head) from the +server's `/profile` endpoint — a rolling window of per-turn `PROF` snapshots +emitted by the engine. + The test suite stays browser-light: API requests use a mocked `fetch`, while runtime capability and storage behavior are covered through pure helpers. It -checks that `/health` is resolved next to (not below) the OpenAI `/v1` prefix, +checks that `/health` and `/profile` are resolved next to (not below) the OpenAI `/v1` prefix, supports both boolean and numeric `scheduler.active` responses, and sends the colibrì-specific `cache_slot` field only when KV-slot support was advertised. diff --git a/web/src/App.tsx b/web/src/App.tsx index d4e0e41..39c9f3b 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -9,6 +9,7 @@ import { Database, Feather, Gauge, + Globe, HardDrive, KeyRound, Layers, @@ -31,8 +32,10 @@ import { Textarea } from "@/components/ui/textarea" import { getHealth, listModels, streamChat, type ChatMessage, type HealthResponse, type StreamChatResult } from "@/lib/api" import { activeRequests, supportsCacheSlots } from "@/lib/runtime" import { Brain } from "./Brain" +import { Profiling } from "./Profiling" import { persistPublicSettings, stored } from "@/lib/storage" import { cn } from "@/lib/utils" +import { useLocale } from "./i18n" const message = (role: ChatMessage["role"], content: string): ChatMessage => { let id: string @@ -41,15 +44,12 @@ const message = (role: ChatMessage["role"], content: string): ChatMessage => { } export default function App() { - // When the page is served by the engine itself (coli web), same-origin is the - // right default: no CORS, no manual endpoint editing. The Vite dev server - // (port 5173) keeps the classic default. + const { t, locale, setLocale, locales } = useLocale() + const servedByEngine = typeof window !== "undefined" && window.location.port !== "5173" && window.location.protocol.startsWith("http") const defaultBase = servedByEngine ? `${window.location.origin}/v1` : "http://127.0.0.1:8000/v1" const [baseUrl, setBaseUrl] = useState(() => { const saved = stored(localStorage, "colibri.baseUrl", defaultBase) - // migrate: a stored FACTORY default pointing at another origin would trip CORS - // when the page is engine-served — upgrade it to same-origin once. if (servedByEngine && saved === "http://127.0.0.1:8000/v1" && defaultBase !== saved) return defaultBase return saved }) @@ -73,7 +73,7 @@ export default function App() { const [totalTokens, setTotalTokens] = useState({ prompt: 0, completion: 0 }) const [connecting, setConnecting] = useState(false) const [connected, setConnected] = useState(false) - const [view, setView] = useState<"chat" | "brain">("chat") + const [view, setView] = useState<"chat" | "brain" | "profiling">("chat") const [error, setError] = useState("") const autoConnected = useRef(false) const abortRef = useRef(null) @@ -119,7 +119,7 @@ export default function App() { const result = await getHealth(baseUrl, apiKey) if (!disposed) { setHealth(result); setHealthError("") } } catch (cause) { - if (!disposed) setHealthError(cause instanceof Error ? cause.message : "Runtime metrics unavailable") + if (!disposed) setHealthError(cause instanceof Error ? cause.message : "status.runtimeUnavailable") } } const timer = window.setInterval(() => void poll(), 5000) @@ -154,13 +154,13 @@ export default function App() { } catch (cause) { if (!controller.signal.aborted) { setHealth(null) - setHealthError(cause instanceof Error ? cause.message : "Runtime metrics unavailable") + setHealthError(cause instanceof Error ? cause.message : "status.runtimeUnavailable") } } } catch (cause) { if (controller.signal.aborted) return setConnected(false) - setError(cause instanceof Error ? cause.message : "Could not reach the server.") + setError(cause instanceof Error ? cause.message : "status.serverError") } finally { if (probeRef.current === controller) { probeRef.current = null; setConnecting(false) } } @@ -226,7 +226,7 @@ export default function App() { if (controller.signal.aborted) { updateMessages((current) => current.filter((item) => item.id !== assistant.id || item.content)) } else { - setError(cause instanceof Error ? cause.message : "Generation failed.") + setError(cause instanceof Error ? cause.message : "status.generationFailed") updateMessages((current) => current.filter((item) => item.id !== assistant.id || item.content)) } } finally { @@ -240,22 +240,22 @@ export default function App() {
-
ACTIVE MODEL{model}
+
{t("topbar.activeModel")}{model}
- - + + +
- {loading && tokenCount > 0 ? {tokenCount} tokens : null} - {!loading && tokPerSec != null ? {tokPerSec.toFixed(1)} tok/s : null} + {loading && tokenCount > 0 ? {t("topbar.tokens", { n: tokenCount })} : null} + {!loading && tokPerSec != null ? {t("topbar.tokPerSec", { n: tokPerSec.toFixed(1) })} : null} {!loading && ttft != null ? TTFT {(ttft/1000).toFixed(1)}s : null} {!loading && lastRun?.usage ? {lastRun.usage.prompt_tokens}→{lastRun.usage.completion_tokens} : null} {lastRun?.queueWaitMs != null ? queue {Math.round(lastRun.queueWaitMs)}ms : null} - slot {cacheSlot + 1} - + {t("topbar.slot", { n: cacheSlot + 1 })} +
- {view === "brain" ? : <> + {view === "brain" ? + : view === "profiling" ? : <>
{!messages.length ? (
- COLIBRÌ ENGINE -

Ask the giant.
Keep the machine yours.

-

Connect to a local colibrì server and stream responses directly from your hardware. Nothing leaves the endpoint you choose.

+ {t("hero.title")} +

{t("hero.subtitle")}
{t("hero.tagline")}

+

{t("hero.description")}

- {["Explain how expert routing works", "Write a small C benchmark", "Compare RAM and VRAM caching"].map((item) => )} + {[t("prompts.routing"), t("prompts.benchmark"), t("prompts.caching")].map((item) => )}
) : ( @@ -344,7 +354,7 @@ export default function App() { {messages.map((item) => (
{item.role === "user" ? "Y" : }
-
{item.role === "user" ? "You" : "colibrì"}
{item.content || }
+
{item.role === "user" ? t("chat.you") : t("chat.colibri")}
{item.content || }
))}
@@ -353,10 +363,10 @@ export default function App() {
- {error &&
{error}
} + {error &&
{t(error)}
}
-