diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..f911652 --- /dev/null +++ b/.clang-format @@ -0,0 +1,10 @@ +BasedOnStyle: LLVM +IndentWidth: 4 +ColumnLimit: 120 +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: AllIfsAndElse +AllowShortLoopsOnASingleLine: true +BreakBeforeBraces: Attach +PointerAlignment: Right +SpaceAfterCStyleCast: false +SortIncludes: false diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..e0acabb --- /dev/null +++ b/.editorconfig @@ -0,0 +1,24 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 + +[*.{c,h,cu}] +indent_size = 4 + +[*.{ts,tsx,js,json,css}] +indent_size = 2 + +[Makefile] +indent_style = tab + +[*.yml] +indent_size = 2 + +[*.md] +trim_trailing_whitespace = false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72b7149..f213363 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Build glm - run: cd c && make glm + - name: Build colibri + run: cd c && make colibri - name: C test suite run: cd c && make test-c @@ -105,13 +105,13 @@ jobs: make cuda-dll CUDA_ARCH=sm_80 test -f coli_cuda.dll || { echo "cuda-dll reported success but produced no DLL" >&2; exit 1; } echo "coli_cuda.dll built (MSVC host)" - - name: make glm CUDA_DLL=1 (host links backend_loader, not cudart) + - name: make colibri CUDA_DLL=1 (host links backend_loader, not cudart) shell: msys2 {0} run: | cd c - make glm CUDA_DLL=1 - test -f glm.exe || { echo "glm CUDA_DLL=1 reported success but produced no exe" >&2; exit 1; } - echo "glm.exe built against the DLL loader" + make colibri CUDA_DLL=1 + test -f colibri.exe || { echo "colibri CUDA_DLL=1 reported success but produced no exe" >&2; exit 1; } + echo "colibri.exe built against the DLL loader" web: name: Web UI diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..e471fb9 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,104 @@ +name: Release + +on: + push: + tags: ['v*'] + +permissions: + contents: write + +jobs: + build: + strategy: + matrix: + include: + - os: ubuntu-latest + name: linux-x86_64 + ext: "" + make_args: "ARCH=x86-64-v3" + - os: macos-latest + name: macos-arm64 + ext: "" + make_args: "" + - os: windows-latest + name: windows-x86_64 + ext: ".exe" + make_args: "" + # GitHub shells are format strings: 'msys2 {0}', never bare 'msys2' + # (the v1.0.0 tag build failed on exactly this). Same as ci.yml. + shell: "msys2 {0}" + runs-on: ${{ matrix.os }} + defaults: + run: + shell: ${{ matrix.shell || 'bash' }} + steps: + - uses: actions/checkout@v4 + + - if: matrix.os == 'windows-latest' + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: false + # inherit: the Package step calls the runner's 7z.exe, which lives on + # the Windows PATH; the default minimal path would not see it. + path-type: inherit + install: >- + make + mingw-w64-ucrt-x86_64-gcc + + - if: matrix.os == 'macos-latest' + run: brew install libomp + + - name: Build engine + run: | + cd c + make glm ${{ matrix.make_args }} + ls -lh glm${{ matrix.ext }} + + - name: Package + run: | + TAG=${GITHUB_REF#refs/tags/} + mkdir -p dist + cp c/glm${{ matrix.ext }} dist/colibri-${TAG}-${{ matrix.name }}${{ matrix.ext }} + cp c/coli dist/ + cp c/version.py dist/ + cp c/openai_server.py dist/ + cp c/resource_plan.py dist/ + cp c/doctor.py dist/ + cp LICENSE dist/ + cd dist + if [ "${{ matrix.ext }}" = ".exe" ]; then + 7z a colibri-${TAG}-${{ matrix.name }}.zip * + else + tar czf colibri-${TAG}-${{ matrix.name }}.tar.gz * + fi + + - uses: actions/upload-artifact@v4 + with: + name: colibri-${{ matrix.name }} + path: dist/colibri-*.* + + release: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + path: artifacts + merge-multiple: true + + - name: Create GitHub Release + env: + GH_TOKEN: ${{ github.token }} + run: | + TAG=${GITHUB_REF#refs/tags/} + # Extract the latest version section from CHANGELOG + NOTES=$(awk "/^## \\[${TAG#v}\\]/{found=1;next} /^## \\[/{if(found)exit} found{print}" CHANGELOG.md) + if [ -z "$NOTES" ]; then + NOTES="Release ${TAG}" + fi + gh release create "$TAG" artifacts/* \ + --title "colibrì ${TAG}" \ + --notes "$NOTES" diff --git a/.github/workflows/site.yml b/.github/workflows/site.yml new file mode 100644 index 0000000..160cd61 --- /dev/null +++ b/.github/workflows/site.yml @@ -0,0 +1,36 @@ +name: Deploy website + +# Publishes site/ to GitHub Pages. One-time repo setup: +# Settings → Pages → Build and deployment → Source: "GitHub Actions". +# Custom domain later: add site/CNAME with the bare domain, point DNS +# (A/AAAA to GitHub Pages IPs or CNAME to .github.io), done. + +on: + push: + branches: [main] + paths: ['site/**', '.github/workflows/site.yml'] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: true + +jobs: + deploy: + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/checkout@v4 + - uses: actions/configure-pages@v5 + - uses: actions/upload-pages-artifact@v3 + with: + path: site + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 981e740..b3e1193 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ desktop/src-tauri/target/ desktop/src-tauri/gen/ # binari compilati (si rigenerano con make / coli build) +c/colibri +c/colibri.exe c/glm c/glm.exe c/olmoe @@ -68,3 +70,7 @@ c/tests/test_decode_batch c/tests/test_i4_acc512 c/tests/test_idot c/tests/test_uring +olmoe_merged/ +olmoe_i4/ +c/olmoe_merged/ +c/olmoe_i4/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6177966 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,53 @@ +# Changelog + +All notable changes to colibrì are documented here. +Format follows [Keep a Changelog](https://keepachangelog.com/). + +## [1.0.0] — 2026-07-19 + +First tagged release. The engine has been running in production since late June +2026; this tag marks the baseline for semantic versioning going forward. + +### Highlights + +- **GLM-5.2 (744B MoE)** runs on ~25 GB RAM in pure C, streaming experts from disk +- **Three-tier placement**: VRAM (hot) / RAM (warm) / NVMe (cold), with a learning + cache that pins your workload's hottest experts automatically +- **CUDA backend**: multi-GPU expert tier, dense tensor distribution, batched + ragged attention, resident pipeline (`COLI_CUDA_PIPE=2`) +- **Metal backend** (Apple Silicon): batched expert SwiGLU + fused decode attention + on unified memory GPU +- **MTP speculation**: native GLM-5.2 draft heads, grammar-forced drafts, kernel- + pinned verification (`SPEC_PIN=1`) +- **OpenAI-compatible API**: `coli serve` with SSE streaming, KV slots, bounded + queue, web dashboard (`coli web`) +- **Web UI**: chat with live metrics, expert cortex brain page, profiling breakdown, + expert atlas 3-D galaxy +- **Cross-platform**: Linux, macOS, Windows 11 (native MinGW), PowerPC; CI on all three +- **Auto-tune**: `coli plan --auto-tier` classifies the bottleneck and derives + MTP/PIPE/NUMA/PIN settings with explanations + +### Engine + +- Token-exact validation against `transformers` oracle (teacher-forcing 32/32) +- Compressed MLA KV cache (576 floats/token, 57× smaller), persisted across + restarts (`.coli_kv`, zero re-prefill) +- DSA sparse attention (lightning indexer), faithfully implemented +- Router-lookahead prefetch (`PILOT=1`, 71.6% predictive) +- Async expert I/O pool (`PIPE=1`), io_uring batching (`URING=1`) +- NUMA-aware expert placement (`COLI_NUMA=1`, +13–40% on multi-socket) +- AVX2 / AVX-512 / AVX-VNNI / ARM NEON / NEON-i8mm / POWER VSX kernels +- int4 / int8 / int2 / grouped-int4 (fmt=4) quantization formats + +### Tools + +- `coli convert` — FP8→int4 one-shard-at-a-time converter +- `coli doctor` — read-only setup diagnostics +- `coli plan` — resource planner with auto-tune prescription +- `coli bench` — MMLU / HellaSwag / ARC quality benchmarks +- Expert atlas (`tools/analyze.py --web`) — measured topic affinity for 19,456 experts + +### Community + +- 30+ hardware datapoints in the benchmark tracker +- Contributions from 20+ authors across engine, docs, tooling, and ports diff --git a/README.it.md b/README.it.md new file mode 100644 index 0000000..76e3c21 --- /dev/null +++ b/README.it.md @@ -0,0 +1,260 @@ +

+ colibrì — motore piccolo, modello immenso +

+ +

+ English · 简体中文 · 繁體中文 · Italiano +

+ +**Motore piccolo, modello immenso.** Esegui **GLM-5.2 (744 miliardi di parametri, MoE)** su un computer consumer con ~25 GB di RAM — in C puro, zero dipendenze, caricando gli expert dal disco in streaming. + +Colibrì è un runtime MoE leggero e che preserva la qualità: tratta VRAM, RAM e +disco come un'unica gerarchia di memoria gestita. Se la memoria veloce non basta +il modello rallenta, ma la policy predefinita **non cambia mai silenziosamente la +precisione del modello né la semantica del router**. + +``` +$ ./coli chat + 🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU + ✓ ready in 32s · resident 9.9 GB + › ciao! + ◆ Ciao! 😊 Come posso aiutarti oggi? +``` + +## Guardalo in azione + +

+ dashboard web di colibrì — metriche live, pannello hardware, livelli degli expert +

+

La dashboard web (./coli web): un modello da 744B a 4 tok/s, TTFT 1.6 s, disco 0 — +residenza completa degli expert su 6× RTX 5090, con metriche token in tempo reale, breakdown dei tempi per turno, +la barra dei livelli VRAM/RAM/disco e il mini-cervello live nell'angolo.

+ +

+ la pagina Brain — 19.456 expert come una corteccia vivente +

+

La pagina Brain: tutti i 19.456 expert come una corteccia vivente — il colore indica +il livello di archiviazione, la luminosità il calore di routing, e ogni expert instradato in un turno +lampeggia bianco. Passando il cursore si vede l'affinità +tematica misurata dell'expert.

+ +

+ la pagina Atlas — l'atlante misurato degli expert come una galassia 3D +

+

La pagina Atlas: l'atlante +misurato degli expert come una galassia 3D — 13.260 expert caratterizzati, 1.041 specialisti +replicabili che si raggruppano per argomento (poesia, legge, cinese, SQL…). La posizione deriva +dall'affinità di routing misurata, non da un embedding appreso. Trascinare per ruotare.

+ +## L'idea + +Un modello Mixture-of-Experts da 744B attiva solo ~40B parametri per token — e +solo ~11 GB di quelli cambiano da un token all'altro (gli expert instradati): + +

+ solo ~5.4% dei parametri è attivo per token +

+ +Il modello non ha bisogno di *stare* in memoria veloce — ha bisogno di essere +**piazzato**: + +- la **parte densa** (attenzione, expert condivisi, embedding — ~17B parametri) + resta **residente in RAM a int4** (~9.9 GB); +- i **19.456 expert instradati** (75 layer MoE × 256 + la testa MTP, ~19 MB + ciascuno a int4) stanno **su disco** (~370 GB) e vengono **caricati on demand + in streaming**, con una cache LRU per layer, un hot-store pinnato che impara, + e un livello VRAM opzionale. + +Il motore è un singolo file C (`c/colibri.c`) più header piccoli. Niente BLAS, +niente Python a runtime, niente GPU obbligatoria. + +## Come funziona + +### Il percorso di ogni token + +

+ instrada → unione → piazza → sovrapponi → impara +

+ +Ogni layer di ogni token percorre gli stessi cinque passi. L'obiettivo +progettuale è che **il piazzamento decide solo la velocità** — le decisioni +del router e la precisione dei pesi sono identiche sia che un expert risponda +dalla VRAM sia dal disco. + +### Una gerarchia di memoria, non un requisito di memoria + +

+ residenza expert a tre livelli: VRAM / RAM / NVMe +

+ +Lo stesso motore copre l'intero spettro: su un portatile da 25 GB tutto viene +caricato dal disco in streaming (lento, ma corretto); su un host grande l'intero +set di expert diventa residente (`CUDA_EXPERT_GB=auto PIN_GB=all`) e il disco +esce completamente dal percorso di decode. Tra i livelli c'è una **cache che +impara**: il motore registra quali expert il *tuo* carico di lavoro instrada +(`.coli_usage`, aggiornato a ogni turno) e fissa automaticamente i più caldi — +colibrì diventa letteralmente più veloce man mano che lo usi. Sugli host +multi-socket, `COLI_NUMA=1` interlaccia i pesi residenti tra i controller di +memoria ([#82](https://github.com/JustVugg/colibri/issues/82)). + +### Mai aspettare il disco due volte + +I miss nella cache costano caro, quindi il motore investe la maggior parte +della sua astuzia per evitarli e sovrapporli: le tre matrici di ogni expert sono +memorizzate contigue e lette con un unico `pread`; un pool I/O asincrono +limitato (`PIPE=1`, attivo per default) carica gli expert mancanti mentre quelli +residenti calcolano; le posizioni in batch leggono ogni expert unico una sola +volta (**batch-union**); un thread di lookahead del router (`PILOT=1`) fa il +prefetch degli expert del layer successivo — il routing è misurabilmente +**prevedibile al 71.6% un layer in anticipo**. Sulle GPU, la pipeline residente +(`COLI_CUDA_PIPE=2`) mantiene il flusso residuo on-device tra i layer, così il +loop CPU degli expert procede senza interruzioni; su Apple Silicon un backend +[Metal](docs/metal.md) sperimentale esegue la matmul batch degli expert sulla +GPU a memoria unificata. + +### Modello fedele, stato compresso + +Il forward pass è validato **token-esatto contro un oracle `transformers`** +(teacher-forcing 32/32). L'attenzione MLA memorizza uno stato KV compresso — 576 +float/token invece di 32.768 (**57× più piccolo**) — e lo persiste tra i +riavvii (`.coli_kv`): le conversazioni riaprono "calde", senza alcun re-prefill, +byte-identiche a una sessione ininterrotta. L'attenzione sparsa DSA (il +lightning indexer di GLM-5.2) è implementata fedelmente e validata forzando la +selezione di tutte le chiavi per riprodurre esattamente l'attenzione densa. + +### Decodifica speculativa, onestamente + +La testa MTP nativa di GLM-5.2 propone token che il modello principale verifica +in un unico forward batch — 2.2–2.8 token/forward quando conviene. Due regole +conquistate a caro prezzo sono i default: la testa MTP deve essere **int8** (le +teste int4 crollano al 0–4% di accettazione, +[#8](https://github.com/JustVugg/colibri/issues/8)), e draft e verifica devono +calcolare **la stessa funzione** — `SPEC_PIN=1` fissa entrambi sulla stessa +famiglia di kernel ([#163](https://github.com/JustVugg/colibri/issues/163) +contiene l'intera indagine forense). I draft forzati da grammatica +([`GRAMMAR=file.gbnf`](docs/grammar-draft.md)) aggiungono accettazione quasi +gratuita sull'output JSON vincolato. Se la speculazione conviene dipende dalla +temperatura della cache — misura, e usa `DRAFT=0` quando non paga. + +## Cosa ottiene + +

+ velocità di decode misurata per classe hardware +

+ +Stesso motore, stesso container int4 — cambia solo dove risiedono gli expert. +Punti salienti dalle [tabelle benchmark complete](docs/benchmarks.md): + +- **6× RTX 5090, residenza completa:** 5.8–6.8 tok/s in decode, TTFT ~13 s + ([log dell'esperimento](docs/experiments/glm52-6x5090-2026-07-12.md)); +- **desktop solo-CPU da 128 GB:** ~1.8 tok/s a cache calda + ([#200](https://github.com/JustVugg/colibri/issues/200)); +- **singola RTX 5070 Ti, classe laptop:** 1.07 tok/s tramite la pipeline + GPU-residente ([#273](https://github.com/JustVugg/colibri/issues/273)); +- **macchina di sviluppo da 25 GB:** 0.05–0.1 tok/s a freddo — il punto di + partenza dimostrato da cui è nato il progetto, e ancora oggi la baseline onesta. + +La qualità è misurata, non presunta: il costo di quantizzazione del container +int4 e le ablazioni su granularità delle scale e rotazione sono in +[docs/benchmarks.md](docs/benchmarks.md#quality-benchmark) e +[#108](https://github.com/JustVugg/colibri/issues/108)/[#81](https://github.com/JustVugg/colibri/issues/81). + +## Per iniziare + +### 1. Scarica il modello + +Un container **GLM-5.2 int4** pre-convertito è su Hugging Face — **usa la +versione con le teste MTP int8**: + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +> ⚠️ Il mirror originale contiene teste MTP int4 → accettazione dei draft allo 0% +> ([#8](https://github.com/JustVugg/colibri/issues/8)). Verifica la tua versione: +> `ls -l /out-mtp-*` — int8 (corretto) è `3527131672 / 5366238584 / 1065950496`. + +Oppure converti tu stesso dalla sorgente FP8 — un unico comando riprendibile che +non richiede mai i 756 GB completi su disco contemporaneamente: + +```bash +cd c && ./setup.sh # verifica gcc/OpenMP, compila, autotest +./coli convert --model /nvme/glm52_i4 # scarica e converti shard per shard (python, una tantum) +``` + +### 2. Esegui + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli chat # budget RAM, cache e MTP rilevati automaticamente +COLI_MODEL=/nvme/glm52_i4 ./coli plan # mostra il piazzamento pianificato VRAM/RAM/disco +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # controllo di idoneità (sola lettura) +./coli web --model /nvme/glm52_i4 # API + dashboard web sulla stessa porta +./coli serve --model /nvme/glm52_i4 # solo API compatibile OpenAI +``` + +Il motore a runtime è puro C — python si usa solo per il convertitore (una tantum) +e per il gateway API opzionale. + +### 3. Approfondisci + +| argomento | documento | +|---|---| +| Benchmark, dati dalla comunità, misurazioni di qualità | [docs/benchmarks.md](docs/benchmarks.md) | +| Parametri di tuning, policy, cache che impara, prefetch | [docs/tuning.md](docs/tuning.md) | +| Build nativa su Windows 11 (con CUDA DLL) | [docs/windows.md](docs/windows.md) | +| Backend CUDA, livello expert in VRAM, residenza completa | [docs/cuda.md](docs/cuda.md) | +| Backend Metal per Apple Silicon | [docs/metal.md](docs/metal.md) | +| API compatibile OpenAI, KV slot, dashboard web | [docs/api.md](docs/api.md) | +| Draft forzati da grammatica (output strutturato) | [docs/grammar-draft.md](docs/grammar-draft.md) | +| Inventario delle variabili d'ambiente | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | + +## Sostenere il progetto + +colibrì è nato come progetto di una sola persona su un portatile con 12 core +e 25 GB di RAM; oggi i suoi numeri arrivano da una comunità di macchine reali. +Se ti è utile: + +- ⭐ metti una stella al repository e condividilo; +- 🐛 apri issue con i numeri di benchmark del tuo hardware — i datapoint + fanno avanzare questo progetto più di qualsiasi altra cosa; +- 💬 contattaci via GitHub issues per sponsorizzare lo sviluppo o donare hardware. + +## Struttura del repository + +``` +Makefile punto d'ingresso root per build/check +c/ +├── colibri.c motore principale +├── quant.h kernel matmul quantizzati (SIMD multi-architettura) +├── sample.h campionamento, RNG, set di stop +├── kv_persist.h persistenza KV su disco (.coli_kv) +├── telemetry.h protocollo dashboard, statistiche, usage +├── st.h, tok.h, json.h header di runtime +├── backend_cuda.* livello CUDA opzionale +├── Makefile build e check locali +├── coli CLI utente +├── openai_server.py gateway HTTP compatibile OpenAI +├── setup.sh setup locale in un solo comando +├── tools/ conversione offline, fixture e benchmark +├── scripts/ helper per conversioni lunghe +└── tests/ test C e Python senza dipendenze +web/ UI browser (puro client API OpenAI) +desktop/ shell desktop Tauri v2 che racchiude la web UI +docs/ documentazione di riferimento, esperimenti, media +``` + +Il percorso a runtime resta intenzionalmente piatto e leggibile: `colibri.c` +più i suoi header. Dalla radice del repository, `make`, `make check` e +`make clean` delegano al Makefile del motore. + +## Perché "colibrì" + +Il colibrì pesa pochi grammi, sta sospeso nel vuoto e visita un migliaio di +fiori al giorno. Questo motore tiene in vita un gigante da 744 miliardi di +parametri con le razioni di un colibrì: 25 GB di RAM, dodici core CPU e +tanta pazienza col disco. + +Il nome è rimasto in italiano perché questa è la lingua in cui è stato scritto +il primo prototipo — i commenti nel codice lo testimoniano ancora. + +## Licenza + +Apache 2.0. I pesi di GLM-5.2 sono rilasciati da Z.ai sotto licenza MIT. diff --git a/README.md b/README.md index 4045e9a..152274f 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ colibrì — tiny engine, immense model

+

+ English · 简体中文 · 繁體中文 · Italiano +

+ **Tiny engine, immense model.** Run **GLM-5.2 (744B-parameter MoE)** on a consumer machine with ~25 GB of RAM — in pure C, with zero dependencies, by streaming experts from disk. Colibrì is a lightweight, quality-preserving MoE runtime that treats VRAM, RAM, @@ -40,6 +44,16 @@ brightness is routing heat, and every expert routed in a turn flashes white. Hov as a 3-D galaxy — 13,260 characterised experts, 1,041 replicated specialists clustering by topic (poetry, law, Chinese, SQL…). Position is measured routing affinity, not a learned embedding. Drag to spin.

+## The vision + +Frontier models should not be sealed inside datacenters. colibrì exists so that +**anyone curious enough can open one up**: run a 744B-parameter mind on hardware +you already own, watch every expert fire in real time, and change the code that +does it. Not renting intelligence behind an API — *holding* it: probing it, +measuring it, improving it. Every optimisation in this project started with +someone measuring something on their own machine; the engine is deliberately +small enough that the next one can come from you. + ## The idea A 744B Mixture-of-Experts model activates only ~40B parameters per token — and @@ -57,6 +71,18 @@ So the model doesn't need to *fit* in fast memory — it needs to be **placed**: at int4) live **on disk** (~370 GB) and are **streamed on demand**, with a per-layer LRU cache, a learned pinned hot-store, and an optional VRAM tier. +Think of the core algorithm as **a JIT, but for weights**. A compiler JIT never +compiles the whole program — it watches what actually runs and compiles the hot +paths, just in time. colibrì makes the same bet about a 744B parameter space: +parameters are not resident state to be held, they are **data to be staged** +across a heterogeneous storage hierarchy (VRAM / RAM / NVMe), exactly when the +router proves they are needed. Measured routing heat decides which experts earn +which tier, the router runs a layer ahead so prefetch hides the staging latency, +and — like a JIT — the engine learns your workload: the more you run, the hotter +the right experts get. It works because routing has measurable structure (see +the [expert atlas](https://github.com/JustVugg/colibri/issues/175)) — and +structure is cacheable. + The engine is a single C file (`c/glm.c`) plus small headers. No BLAS, no Python at runtime, no GPU required. @@ -147,6 +173,10 @@ scale-granularity/rotation ablations live in ## Get started +> **New here?** The [Quick Start guide](docs/quickstart.md) walks through +> install → build → model → first chat step by step for Linux, Windows, and +> macOS, with copy-paste commands and no assumed background. + ### 1. Get the model A pre-converted **GLM-5.2 int4** container is on Hugging Face — **use the @@ -179,6 +209,17 @@ COLI_MODEL=/nvme/glm52_i4 ./coli doctor # read-only readiness check The engine at runtime is pure C — python is only used by the one-time converter and the optional API gateway. +**On Windows?** You don't need to build. Download the +`colibri--windows-x86_64.zip` from +[Releases](https://github.com/JustVugg/colibri/releases), unzip it, rename +`colibri-*-windows-x86_64.exe` → `glm.exe` (so the `coli` launcher finds the +engine), install [Python 3](https://www.python.org/downloads/), then run +`coli chat`. Full walkthrough in the [Quick Start guide](docs/quickstart.md#windows). + +Prefer a `coli` command on your PATH? From a checkout, `pip install -e .` +registers it (the engine itself still lives in `c/` — this is an editable +install from the clone, not a standalone wheel). + ### 3. Go deeper | topic | doc | @@ -192,6 +233,18 @@ and the optional API gateway. | Grammar-forced drafts (structured output) | [docs/grammar-draft.md](docs/grammar-draft.md) | | Environment variable inventory | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | +## What's next + +- **Algorithmic research is active.** The current hierarchy is LRU + a learned + pin set; the next step is under way — smarter placement and scheduling, + overlap of CPU and GPU expert execution, and routing-aware speculation. + Everything lands the way this project always works: measured, reviewed, and + merged in the open. +- **More open models.** The tiering algorithm is model-agnostic: any MoE with + routed experts can be staged the same way. GLM-5.2 and OLMoE run today; + support for more open-weight families — **Kimi K2** (Moonshot AI), + **Qwen3 MoE** (Alibaba), **MiniMax** — is on the roadmap. + ## Supporting the project colibrì started as a one-person project on a 12-core laptop with 25 GB of RAM; @@ -232,6 +285,14 @@ The hummingbird weighs a few grams, hovers in place, and visits a thousand flowers a day. This engine keeps a 744-billion-parameter giant alive on hummingbird rations: 25 GB of RAM, twelve CPU cores, and a lot of disk patience. +## Acknowledgements + +colibrì is an engine; the minds it runs are a gift. Thank you to the teams +releasing frontier-class weights in the open — **Z.ai** (GLM), **Moonshot AI** +(Kimi), **Alibaba Qwen**, **MiniMax**, and **Allen AI** (OLMoE) — and to every +contributor who benchmarked, bisected, replicated an atlas run, or sent a patch. +This project is proof of what open weights make possible. + ## License Apache 2.0. GLM-5.2 weights are released by Z.ai under MIT. diff --git a/README.zh-CN.md b/README.zh-CN.md new file mode 100644 index 0000000..352de33 --- /dev/null +++ b/README.zh-CN.md @@ -0,0 +1,237 @@ +

+ colibrì——小巧引擎，庞大模型 +

+ +

+ English · 简体中文 · 繁體中文 · Italiano +

+ +**小巧引擎，庞大模型。**只需约 25 GB 内存，就能在消费级电脑上运行 **GLM-5.2（744B 参数的 MoE）**——以零依赖的纯 C 实现，从磁盘流式加载专家。 + +Colibrì 是一套轻量、保持模型质量的 MoE 运行时，将 VRAM、RAM +与存储设备视为统一管理的内存层级。高速内存不足可能降低速度， +但默认策略**绝不会在未告知的情况下改变模型精度或路由语义**。 + +``` +$ ./coli chat + 🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU + ✓ ready in 32s · resident 9.9 GB + › ciao! + ◆ Ciao! 😊 Come posso aiutarti oggi? +``` + +## 实际运行效果 + +

+ colibrì 网页仪表盘——实时指标、硬件面板与专家存储层级 +

+

网页仪表盘（./coli web）：744B 模型达到 4 tok/s、TTFT 1.6 秒、磁盘读取 0—— +在 6× RTX 5090 上让所有专家常驻，并实时显示 token 指标、每轮耗时明细、 +VRAM／RAM／磁盘层级条，以及角落的实时迷你大脑。

+ +

+ 大脑页面——以实时皮层呈现 19,456 个专家 +

+

大脑（Brain）页面：将全部 19,456 个专家呈现为活的皮层——颜色代表存储层级， +亮度代表路由热度，每轮被路由到的专家都会闪白。将光标停在专家上，即可查看其 +实测主题亲和度。

+ +

+ 图谱页面——以 3D 星系呈现实测专家图谱 +

+

图谱（Atlas）页面：将实测专家图谱 +呈现为 3D 星系——共 13,260 个已分析专家，其中 1,041 个可复现的专门专家会按主题聚集 +（诗歌、法律、中文、SQL……）。位置取自实测路由亲和度，而非学习出的嵌入向量。拖拽即可旋转。

+ +## 核心概念 + +744B 的专家混合（Mixture-of-Experts）模型，每个 token 只会激活约 40B 参数—— +其中每个 token 之间会变动的只有约 11 GB（被路由到的专家）： + +

+ 每个 token 只会激活约 5.4% 的参数 +

+ +所以模型不必完整**装进**高速内存，而是需要正确**放置**： + +- **稠密部分**（注意力、共享专家、嵌入——约 17B 参数）以 int4 + **常驻 RAM**（约 9.9 GB）； +- **19,456 个路由专家**（75 个 MoE 层 × 256，加上 MTP head；每个在 int4 下约 19 MB） + **存放在磁盘**（约 370 GB），并**按需流式加载**，配合逐层 LRU 缓存、 + 会学习的热门专家固定存储区，以及可选的 VRAM 层级。 + +引擎是一个 C 主文件（`c/colibri.c`）加上若干头文件。不需要 BLAS， +运行时不需要 Python，也不需要 GPU。 + +## 工作原理 + +### 每个 token 的处理路径 + +

+ 路由 → 并集 → 放置 → 重叠执行 → 学习 +

+ +每个 token 的每一层都会经过相同的五个步骤。设计目标是让 +**放置只决定速度**——无论专家是从 VRAM 还是磁盘响应，路由器的决策与权重精度都完全相同。 + +### 统一内存层级，取代单一内存门槛 + +

+ VRAM／RAM／NVMe 三层专家常驻架构 +

+ +同一套引擎覆盖完整硬件范围：在 25 GB 笔记本上，一切都从磁盘流式加载 +（慢，但结果正确）；在大内存主机上，则可让整组专家常驻 +（`CUDA_EXPERT_GB=auto PIN_GB=all`），让磁盘完全退出解码路径。 +两端之间有一层**学习型缓存**：引擎会记录*你的*工作负载路由到哪些专家 +（`.coli_usage`，每轮更新），并自动固定最热门的专家——colibrì 确实会越用越快。 +在多路主机上，`COLI_NUMA=1` 会将常驻权重交错分配到各内存控制器 +（[#82](https://github.com/JustVugg/colibri/issues/82)）。 + +### 绝不为同一次磁盘读取等待两遍 + +缓存未命中的代价很高，因此引擎大部分的巧思都用来避免或重叠这些读取： +每个专家的三个矩阵相邻存储，并以一次 `pread` 读取；有界异步 I/O 池 +（`PIPE=1`，默认启用）会在常驻专家计算时加载缺失的专家；批量位置只读取每个 +不重复专家一次（**批量并集**）；路由前瞻线程（`PILOT=1`）则预取下一层专家—— +实测显示，路由结果提前一层时有 **71.6% 的可预测性**。 +在 GPU 上，常驻管线（`COLI_CUDA_PIPE=2`）让残差流跨层保留在设备端， +使 CPU 专家循环不中断；在 Apple Silicon 上，实验性的 +[Metal 后端](docs/metal.md)会用统一内存 GPU 执行批量专家运算。 + +### 忠实模型，压缩状态 + +前向传播已通过 `transformers` oracle 验证为**逐 token 完全一致** +（teacher-forcing 32/32）。MLA 注意力存储压缩后的 KV 状态——每个 token 为 576 个 +浮点数，而非 32,768 个（**缩小 57×**）——并跨重启持久保存 +（`.coli_kv`）：对话可暖启恢复，不需重新 prefill，结果与不中断的会话 +逐字节相同。DSA 稀疏注意力（GLM-5.2 的 lightning indexer）已忠实实现， +并通过强制选取所有 key，验证可精确复现稠密注意力。 + +### 诚实的推测解码 + +GLM-5.2 原生 MTP head 会起草 token，再由主模型以一次批量前向传播验证—— +条件合适时每次 forward 可产生 2.2–2.8 个 token。两条来之不易的规则已成为默认值： +MTP head 必须是 **int8**（int4 head 的接受率会崩塌到 0–4%，见 +[#8](https://github.com/JustVugg/colibri/issues/8)），且草稿与验证必须计算 +**相同函数**——`SPEC_PIN=1` 会把两者固定在同一 kernel family +（完整取证过程见 [#163](https://github.com/JustVugg/colibri/issues/163)）。 +语法强制草稿（[`GRAMMAR=file.gbnf`](docs/grammar-draft.md)）可在受限 JSON 输出中， +以近乎免费的代价提高接受率。推测解码是否带来净收益取决于缓存热度——请实测， +若不划算就使用 `DRAFT=0`。 + +## 实际成果 + +

+ 各硬件级别的实测解码速度 +

+ +同一套引擎、同一个 int4 容器——硬件只会改变专家的存放位置。 +[完整 benchmark 表格](docs/benchmarks.md)中的重点如下： + +- **6× RTX 5090，全部常驻：**解码 5.8–6.8 tok/s，TTFT 约 13 秒 + （[实验记录](docs/experiments/glm52-6x5090-2026-07-12.md)）； +- **128 GB、仅使用 CPU 的台式机：**热缓存后约 1.8 tok/s + （[#200](https://github.com/JustVugg/colibri/issues/200)）； +- **单张 RTX 5070 Ti 的笔记本级主机：**通过 GPU 常驻管线达到 1.07 tok/s + （[#273](https://github.com/JustVugg/colibri/issues/273)）； +- **25 GB 开发机：**冷启动 0.05–0.1 tok/s——这是项目起步时已证实的下限， + 也仍是诚实的基准。 + +质量来自测量，而非假设：int4 容器的量化损失，以及 scale granularity／rotation +消融实验，收录于 [docs/benchmarks.md](docs/benchmarks.md#quality-benchmark)、 +[#108](https://github.com/JustVugg/colibri/issues/108) 与 +[#81](https://github.com/JustVugg/colibri/issues/81)。 + +## 开始使用 + +### 1. 获取模型 + +Hugging Face 上已有预转换的 **GLM-5.2 int4** 容器——请务必使用 +**含 int8 MTP head 的版本**： + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +> ⚠️ 原始镜像使用 int4 MTP head → 草稿接受率为 0% +>（[#8](https://github.com/JustVugg/colibri/issues/8)）。请检查你的版本： +> `ls -l /out-mtp-*`——正确的 int8 大小为 `3527131672 / 5366238584 / 1065950496`。 + +你也可以自行从 FP8 源转换——只需一条可断点续传的命令，且任何时候都不需要 +在磁盘上同时存放完整的 756 GB： + +```bash +cd c && ./setup.sh # 检查 gcc/OpenMP、构建并运行自测 +./coli convert --model /nvme/glm52_i4 # 逐 shard 下载并转换（仅此一次需要 python） +``` + +### 2. 运行 + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli chat # 自动检测 RAM 预算、缓存与 MTP +COLI_MODEL=/nvme/glm52_i4 ./coli plan # 查看规划的 VRAM／RAM／磁盘配置 +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # 只读就绪检查 +./coli web --model /nvme/glm52_i4 # 在同一端口提供 API 与网页仪表盘 +./coli serve --model /nvme/glm52_i4 # 仅提供 OpenAI 兼容 API +``` + +引擎运行时是纯 C——python 只供一次性转换工具与可选的 API gateway 使用。 + +### 3. 深入了解 + +| 主题 | 文档 | +|---|---| +| Benchmark、社区实测数据、质量测量 | [docs/benchmarks.md](docs/benchmarks.md) | +| 调优选项、策略、学习型缓存、预取 | [docs/tuning.md](docs/tuning.md) | +| Windows 11 原生构建（含 CUDA DLL） | [docs/windows.md](docs/windows.md) | +| CUDA 后端、VRAM 专家层级、全部常驻 | [docs/cuda.md](docs/cuda.md) | +| Apple Silicon Metal 后端 | [docs/metal.md](docs/metal.md) | +| OpenAI 兼容 API、KV slots、网页仪表盘 | [docs/api.md](docs/api.md) | +| 语法强制草稿（结构化输出） | [docs/grammar-draft.md](docs/grammar-draft.md) | +| 环境变量完整清单 | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | + +## 支持项目 + +colibrì 最初由一人使用 12 核心、25 GB RAM 的笔记本开发； +如今它的数据来自社区中各种真实机器。如果这个项目对你有用： + +- ⭐ 为仓库加星并分享； +- 🐛 以 issue 提交你的硬件 benchmark 数据——实测数据比任何其他事都更能推动项目； +- 💬 若想赞助开发或捐赠硬件，请通过 GitHub issues 联系。 + +## 仓库结构 + +``` +Makefile 根目录构建／检查入口 +c/ +├── colibri.c 引擎主文件 +├── quant.h 量化 matmul 内核（SIMD 多架构） +├── sample.h 采样与 stop-set 管理 +├── kv_persist.h .coli_kv 磁盘持久化 +├── telemetry.h 仪表盘协议、统计与用量持久化 +├── st.h, tok.h, json.h 运行时头文件 +├── backend_cuda.* 可选的 CUDA 层级 +├── Makefile 构建与本地检查 +├── coli 用户界面 CLI +├── openai_server.py OpenAI 兼容 HTTP gateway +├── setup.sh 一条命令完成本地设置 +├── tools/ 离线转换、fixtures 与 benchmarks +├── scripts/ 长时间转换辅助工具 +└── tests/ 零依赖的 C 与 Python 测试 +web/ 浏览器 UI（纯 OpenAI API client） +desktop/ 封装网页 UI 的 Tauri v2 桌面 shell +docs/ 参考文档、实验与媒体文件 +``` + +运行时路径刻意保持扁平、易读：`colibri.c` 加上若干头文件。 +在仓库根目录执行 `make`、`make check` 与 `make clean`， +都会转发给引擎的 Makefile。 + +## 为什么叫"colibrì" + +蜂鸟只有几克重，能在原地悬停，并在一天内造访上千朵花。 +这套引擎只用蜂鸟般的配给，就能让 744B 参数的巨人运转： +25 GB RAM、十二个 CPU 核心，以及对磁盘的大量耐心。 + +## 许可证 + +Apache 2.0。GLM-5.2 权重由 Z.ai 以 MIT 许可发布。 diff --git a/README.zh-TW.md b/README.zh-TW.md new file mode 100644 index 0000000..2576eba --- /dev/null +++ b/README.zh-TW.md @@ -0,0 +1,237 @@ +

+ colibrì——小巧引擎，龐大模型 +

+ +

+ English · 简体中文 · 繁體中文 · Italiano +

+ +**小巧引擎，龐大模型。**只要約 25 GB 記憶體，就能在消費級電腦上執行 **GLM-5.2（744B 參數的 MoE）**——以零相依套件的純 C 實作，從硬碟串流載入專家。 + +Colibrì 是一套輕量、維持品質的 MoE 執行環境，將 VRAM、RAM +與儲存裝置視為統一管理的記憶體階層。高速記憶體不足可能降低速度， +但預設策略**絕不會在未告知的情況下改變模型精度或路由語意**。 + +``` +$ ./coli chat + 🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU + ✓ ready in 32s · resident 9.9 GB + › ciao! + ◆ Ciao! 😊 Come posso aiutarti oggi? +``` + +## 實際運行畫面 + +

+ colibrì 網頁儀表板——即時指標、硬體面板與專家儲存層級 +

+

網頁儀表板（./coli web）：744B 模型達到 4 tok/s、TTFT 1.6 秒、硬碟讀取 0—— +在 6× RTX 5090 上讓所有專家常駐，並即時顯示 token 指標、每輪耗時明細、 +VRAM／RAM／硬碟層級長條，以及角落的即時迷你大腦。

+ +

+ 大腦頁面——以即時皮質呈現 19,456 個專家 +

+

大腦（Brain）頁面：將全部 19,456 個專家呈現為活的皮質——顏色代表儲存層級， +亮度代表路由熱度，每輪被路由到的專家都會閃白。將游標停在專家上，即可查看其 +實測主題親和度。

+ +

+ 圖譜頁面——以 3D 星系呈現實測專家圖譜 +

+

圖譜（Atlas）頁面：將實測專家圖譜 +呈現為 3D 星系——共 13,260 個已分析專家，其中 1,041 個可重現的專門專家會按主題聚集 +（詩歌、法律、中文、SQL……）。位置取自實測路由親和度，而非學習出的嵌入向量。拖曳即可旋轉。

+ +## 核心概念 + +744B 的專家混合（Mixture-of-Experts）模型，每個 token 只會啟用約 40B 參數—— +其中每個 token 之間會變動的只有約 11 GB（被路由到的專家）： + +

+ 每個 token 只會啟用約 5.4% 的參數 +

+ +所以模型不必完整**放進**高速記憶體，而是需要正確**配置位置**： + +- **稠密部分**（注意力、共享專家、嵌入——約 17B 參數）以 int4 + **常駐 RAM**（約 9.9 GB）； +- **19,456 個路由專家**（75 個 MoE 層 × 256，加上 MTP head；每個在 int4 下約 19 MB） + **存放在硬碟**（約 370 GB），並**隨需串流載入**，搭配逐層 LRU 快取、 + 會學習的熱門專家固定儲存區，以及選用的 VRAM 層級。 + +引擎由主 C 檔（`c/colibri.c`）與多個標頭檔模組組成。不需要 BLAS， +執行階段不需要 Python，也不需要 GPU。 + +## 運作方式 + +### 每個 token 的處理路徑 + +

+ 路由 → 聯集 → 配置 → 重疊執行 → 學習 +

+ +每個 token 的每一層都會走過相同的五個步驟。設計目標是讓 +**配置只決定速度**——無論專家是從 VRAM 或硬碟回應，路由器的決策與權重精度都完全相同。 + +### 統一記憶體階層，取代單一記憶體門檻 + +

+ VRAM／RAM／NVMe 三層專家常駐架構 +

+ +同一套引擎涵蓋完整硬體範圍：在 25 GB 筆電上，一切都從硬碟串流載入 +（慢，但結果正確）；在大型主機上，則可讓整組專家常駐 +（`CUDA_EXPERT_GB=auto PIN_GB=all`），讓硬碟完全退出解碼路徑。 +兩端之間有一層**學習型快取**：引擎會記錄*你的*工作負載路由到哪些專家 +（`.coli_usage`，每輪更新），並自動固定最熱門的專家——colibrì 確實會越用越快。 +在多插槽主機上，`COLI_NUMA=1` 會將常駐權重交錯分配到各記憶體控制器 +（[#82](https://github.com/JustVugg/colibri/issues/82)）。 + +### 絕不為同一次硬碟讀取等待兩遍 + +快取未命中的成本很高，因此引擎大部分的巧思都用來避免或重疊處理這些讀取： +每個專家的三個矩陣相鄰儲存，並以一次 `pread` 讀取；有界非同步 I/O pool +（`PIPE=1`，預設啟用）會在常駐專家運算時載入缺少的專家；批次位置只讀取每個 +不重複專家一次（**批次聯集**）；路由前瞻執行緒（`PILOT=1`）則預先載入下一層專家—— +實測顯示，路由結果提前一層時有 **71.6% 的可預測性**。 +在 GPU 上，常駐管線（`COLI_CUDA_PIPE=2`）讓殘差流跨層保留在裝置端， +使 CPU 專家迴圈不中斷；在 Apple Silicon 上，實驗性的 +[Metal 後端](docs/metal.md)會用統一記憶體 GPU 執行批次專家運算。 + +### 忠實模型，壓縮狀態 + +前向傳遞已透過 `transformers` oracle 驗證為**逐 token 完全一致** +（teacher-forcing 32/32）。MLA 注意力儲存壓縮後的 KV 狀態——每個 token 為 576 個 +浮點數，而非 32,768 個（**縮小 57×**）——並跨重新啟動持久保存 +（`.coli_kv`）：對話可暖啟恢復，不需重新 prefill，結果與不中斷的工作階段 +逐位元組相同。DSA 稀疏注意力（GLM-5.2 的 lightning indexer）已忠實實作， +並透過強制選取所有 key，驗證可精確重現稠密注意力。 + +### 如實呈現推測式解碼 + +GLM-5.2 原生 MTP head 會起草 token，再由主模型以一次批次前向傳遞驗證—— +條件合適時每次 forward 可產生 2.2–2.8 個 token。兩條得來不易的規則已成為預設值： +MTP head 必須是 **int8**（int4 head 的接受率會崩落到 0–4%，見 +[#8](https://github.com/JustVugg/colibri/issues/8)），且草稿與驗證必須計算 +**相同函數**——`SPEC_PIN=1` 會把兩者固定在同一 kernel family +（完整鑑識過程見 [#163](https://github.com/JustVugg/colibri/issues/163)）。 +文法強制草稿（[`GRAMMAR=file.gbnf`](docs/grammar-draft.md)）可在受限 JSON 輸出中， +以近乎免費的成本提高接受率。推測式解碼是否帶來淨收益取決於快取熱度——請實測， +若不划算就使用 `DRAFT=0`。 + +## 實際成果 + +

+ 各硬體等級的實測解碼速度 +

+ +同一套引擎、同一個 int4 容器——硬體只會改變專家的存放位置。 +[完整 benchmark 表格](docs/benchmarks.md)中的重點如下： + +- **6× RTX 5090，全部常駐：**解碼 5.8–6.8 tok/s，TTFT 約 13 秒 + （[實驗紀錄](docs/experiments/glm52-6x5090-2026-07-12.md)）； +- **128 GB、僅使用 CPU 的桌上型電腦：**暖機後約 1.8 tok/s + （[#200](https://github.com/JustVugg/colibri/issues/200)）； +- **單張 RTX 5070 Ti 的筆電級電腦：**透過 GPU 常駐管線達到 1.07 tok/s + （[#273](https://github.com/JustVugg/colibri/issues/273)）； +- **25 GB 開發機：**冷啟動 0.05–0.1 tok/s——這是專案起步時已證實的下限， + 也仍是如實呈現的基準。 + +品質來自測量，而非假設：int4 容器的量化成本，以及 scale granularity／rotation +消融實驗，收錄於 [docs/benchmarks.md](docs/benchmarks.md#quality-benchmark)、 +[#108](https://github.com/JustVugg/colibri/issues/108) 與 +[#81](https://github.com/JustVugg/colibri/issues/81)。 + +## 開始使用 + +### 1. 取得模型 + +Hugging Face 上已有預先轉換的 **GLM-5.2 int4** 容器——請務必使用 +**含 int8 MTP head 的版本**： + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +> ⚠️ 原始鏡像使用 int4 MTP head → 草稿接受率為 0% +>（[#8](https://github.com/JustVugg/colibri/issues/8)）。請檢查你的版本： +> `ls -l /out-mtp-*`——正確的 int8 大小為 `3527131672 / 5366238584 / 1065950496`。 + +你也可以自行從 FP8 來源轉換——只需一條可續傳的指令，且任何時候都不需要 +在硬碟上同時存放完整的 756 GB： + +```bash +cd c && ./setup.sh # 檢查 gcc/OpenMP、建置並執行自我測試 +./coli convert --model /nvme/glm52_i4 # 逐 shard 下載並轉換（僅此一次需要 python） +``` + +### 2. 執行 + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli chat # 自動偵測 RAM 預算、快取與 MTP +COLI_MODEL=/nvme/glm52_i4 ./coli plan # 檢視規劃的 VRAM／RAM／硬碟配置 +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # 唯讀就緒檢查 +./coli web --model /nvme/glm52_i4 # 在同一個連接埠提供 API 與網頁儀表板 +./coli serve --model /nvme/glm52_i4 # 僅提供 OpenAI 相容 API +``` + +引擎執行階段是純 C——python 只供單次轉換工具與選用的 API gateway 使用。 + +### 3. 深入了解 + +| 主題 | 文件 | +|---|---| +| Benchmark、社群實測數據、品質測量 | [docs/benchmarks.md](docs/benchmarks.md) | +| 調校選項、策略、學習型快取、預先載入 | [docs/tuning.md](docs/tuning.md) | +| Windows 11 原生建置（含 CUDA DLL） | [docs/windows.md](docs/windows.md) | +| CUDA 後端、VRAM 專家層級、全部常駐 | [docs/cuda.md](docs/cuda.md) | +| Apple Silicon Metal 後端 | [docs/metal.md](docs/metal.md) | +| OpenAI 相容 API、KV slots、網頁儀表板 | [docs/api.md](docs/api.md) | +| 文法強制草稿（結構化輸出） | [docs/grammar-draft.md](docs/grammar-draft.md) | +| 環境變數完整清單 | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | + +## 支持專案 + +colibrì 最初是由一人使用 12 核心、25 GB RAM 的筆電開發； +如今它的數據來自社群中的各種真實機器。如果這個專案對你有用： + +- ⭐ 為儲存庫加星並分享； +- 🐛 以 issue 提交你的硬體 benchmark 數據——實測資料比任何其他事都更能推動專案； +- 💬 若想贊助開發或捐贈硬體，請透過 GitHub issues 聯絡。 + +## 儲存庫結構 + +``` +Makefile 根目錄建置／檢查入口 +c/ +├── colibri.c GLM 引擎主檔 +├── quant.h 量化 matmul kernel +├── sample.h 取樣與 stop-set +├── kv_persist.h .coli_kv 磁碟持久化 +├── telemetry.h 儀表板協定、統計 +├── st.h, tok.h, json.h 執行階段標頭檔 +├── backend_cuda.* 選用的 CUDA 層級 +├── Makefile 建置與本機檢查 +├── coli 使用者介面 CLI +├── openai_server.py OpenAI 相容 HTTP gateway +├── setup.sh 單一指令完成本機設定 +├── tools/ 離線轉換、fixtures 與 benchmarks +├── scripts/ 長時間轉換輔助工具 +└── tests/ 零相依套件的 C 與 Python 測試 +web/ 瀏覽器 UI（純 OpenAI API client） +desktop/ 包裝網頁 UI 的 Tauri v2 桌面 shell +docs/ 參考文件、實驗與媒體檔 +``` + +執行階段路徑刻意維持扁平、易讀：`colibri.c` 加上模組化標頭檔。 +在儲存庫根目錄執行 `make`、`make check` 與 `make clean`， +都會轉交給引擎的 Makefile。 + +## 為什麼叫做「colibrì」 + +蜂鳥只有幾公克重，能在原地懸停，並在一天內造訪上千朵花。 +這套引擎只用蜂鳥般的配給，就能讓 744B 參數的巨人運轉： +25 GB RAM、十二個 CPU 核心，以及對硬碟的大量耐心。 + +## 授權條款 + +Apache 2.0。GLM-5.2 權重由 Z.ai 以 MIT 授權發布。 diff --git a/c/Makefile b/c/Makefile index db94a9a..b19ee3e 100644 --- a/c/Makefile +++ b/c/Makefile @@ -56,11 +56,16 @@ OMPL = endif CFLAGS = -O3 $(OMPC) -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function # Opt-in: ARCH=native appends -mcpu=native (arm64 clang uses -mcpu, not -march), -# which unlocks the i8mm SMMLA int8/int4 dot kernels in glm.c. ARCH unset -> +# which unlocks the i8mm SMMLA int8/int4 dot kernels in colibri.c. ARCH unset -> # no -mcpu, default build byte-identical. Apple clang knows apple-m4 / native. +# For older X86_64 Macs (for example, Mac Pro 2019) we need to use -march ifneq ($(ARCH),) +ifneq (,$(X86_64)) +CFLAGS += -march=$(ARCH) +else CFLAGS += -mcpu=$(ARCH) endif +endif LDFLAGS = -lm $(OMPL) EXE = else ifneq ($(IS_WIN),) @@ -70,7 +75,7 @@ else ifneq ($(IS_WIN),) # ARCH default = x86-64-v3 (portable binary with AVX2). For max speed on THIS # machine use ARCH=native: on AVX-VNNI CPUs (Intel Alder Lake+, Meteor Lake+) # it also unlocks the 128-bit VPDPBUSD int8/int4 dot kernel (dot_i8i8/dot_i4i8), -# which the x86-64-v3 baseline does not define. The #ifdef guards in glm.c mean +# which the x86-64-v3 baseline does not define. The #ifdef guards in colibri.c mean # a v3 build simply compiles out the VNNI path - safe on any x86-64. CC = gcc ARCH ?= x86-64-v3 @@ -166,7 +171,7 @@ else PYTHON ?= python3 endif CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_pipe_block$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_pipe_block$(EXE) ifneq (,$(LINUX)) TEST_BINS += tests/test_uring$(EXE) endif @@ -207,17 +212,18 @@ LDFLAGS += -framework Metal -framework Foundation -lc++ METAL_OBJ = backend_metal.o endif -all: glm$(EXE) +all: colibri$(EXE) -# phony 'glm' → 'glm.exe' on Windows (so 'make glm' and 'coli build' work on every platform) -glm: glm$(EXE) +# phony targets — 'glm' kept for backward compatibility +colibri: colibri$(EXE) +glm: colibri$(EXE) # Config stamp: make only tracks file timestamps, not flag changes. Without this, -# `make glm.exe CUDA_DLL=1` after a prior CPU-only build reports "up to date" and -# silently keeps the CPU-only binary (no CUDA loader) — a build that looks like it -# worked but isn't. We record the build-affecting flags in .build-config and rewrite -# it ONLY when they change (evaluated here at parse time, so the file's timestamp -# moves exactly when the config moves). glm.exe and the CUDA/loader objects depend +# `make colibri.exe CUDA_DLL=1` after a prior CPU-only build reports "up to date" +# and silently keeps the CPU-only binary (no CUDA loader) — a build that looks like +# it worked but isn't. We record the build-affecting flags in .build-config and +# rewrite it ONLY when they change (evaluated here at parse time, so the file's +# timestamp moves exactly when the config moves). The binary and CUDA/loader objects depend # on it, so they relink on a config change and stay put otherwise. (#306) BUILD_CONFIG := $(CC)|$(CFLAGS)|$(LDFLAGS)|CUDA=$(CUDA)|CUDA_DLL=$(CUDA_DLL)|ARCH=$(ARCH)|CUDA_ARCH=$(CUDA_ARCH)|METAL=$(METAL) BUILD_CONFIG_OLD := $(shell cat .build-config 2>/dev/null) @@ -226,8 +232,8 @@ $(shell printf '%s' '$(BUILD_CONFIG)' > .build-config) endif .build-config: ; -glm$(EXE): glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h $(CUDA_OBJ) $(METAL_OBJ) .build-config - $(CC) $(CFLAGS) glm.c $(CUDA_OBJ) $(METAL_OBJ) -o glm$(EXE) $(LDFLAGS) +colibri$(EXE): colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h quant.h sample.h kv_persist.h telemetry.h $(CUDA_OBJ) $(METAL_OBJ) .build-config + $(CC) $(CFLAGS) colibri.c $(CUDA_OBJ) $(METAL_OBJ) -o colibri$(EXE) $(LDFLAGS) # Windows runtime loader object: resolves coli_cuda_* from coli_cuda.dll. backend_loader.o: backend_loader.c backend_cuda.h compat.h .build-config @@ -272,8 +278,9 @@ olmoe$(EXE): olmoe.c st.h json.h compat.h # Use a baseline that matches the compiler target. macOS already targets a # portable baseline when ARCH is empty; forcing the x86 value there breaks # Apple Silicon. Unknown targets use native rather than an invalid x86 flag. +# Intel Macs need -march for vector instructions ifneq (,$(DARWIN)) -PORTABLE_ARCH = +PORTABLE_ARCH = $(if $(X86_64),x86-64-v3,) else ifneq (,$(AARCH64)) PORTABLE_ARCH = armv8-a else ifneq (,$(PPC64)) @@ -285,7 +292,7 @@ PORTABLE_ARCH = native endif portable: - $(MAKE) glm$(EXE) ARCH=$(PORTABLE_ARCH) + $(MAKE) colibri$(EXE) ARCH=$(PORTABLE_ARCH) iobench$(EXE): iobench.c compat.h $(CC) $(CFLAGS) iobench.c -o iobench$(EXE) $(LDFLAGS) @@ -293,6 +300,11 @@ iobench$(EXE): iobench.c compat.h tests/test_json$(EXE): tests/test_json.c json.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_tok_o200k$(EXE): tests/test_tok_o200k.c tok.h tok_unicode.h tok_unicode_o200k.h json.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_st_pread$(EXE): tests/test_st_pread.c st.h json.h compat.h + $(CC) $(CFLAGS) -DST_PREAD_CHUNK=7 $< -o $@ $(LDFLAGS) + tests/test_st$(EXE): tests/test_st.c st.h json.h compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) @@ -308,16 +320,30 @@ tests/test_schema_gbnf$(EXE): tests/test_schema_gbnf.c schema_gbnf.h grammar.h j tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_idot$(EXE): tests/test_idot.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_idot$(EXE): tests/test_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_stops$(EXE): tests/test_stops.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_stops$(EXE): tests/test_stops.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_topp$(EXE): tests/test_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +# bench_topp is a microbenchmark (old qsort vs new heap partial-select, #335), NOT a test +# gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_topp +tests/bench_topp$(EXE): tests/bench_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_sample_nan$(EXE): tests/test_sample_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c colibri.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_logit_nan$(EXE): tests/test_logit_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c @@ -326,7 +352,15 @@ tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c tests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) -tests/test_uring$(EXE): tests/test_uring.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h +tests/test_dsa_select$(EXE): tests/test_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +# bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356), +# NOT a test gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_dsa_select +tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_uring$(EXE): tests/test_uring.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_pipe_block$(EXE): tests/test_pipe_block.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h @@ -340,18 +374,41 @@ test-python: test: test-c test-python +# --- Efficiency / regression suite (issue: "test the program for inefficiencies") --- +# The tiny-model assertions live in test_inefficiency.py and run as part of +# test-python (they're discovered by the test_*.py glob). These targets are +# convenience entry points; the opt-in full-model report is NEVER in `make test`. +# +# make efficiency tiny-model asserted regression tests (CPU; part of test-python) +# make efficiency-cuda the CUDA-path tests (requires a CUDA build — see below) +# make efficiency-report opt-in full-model 🟢/🔴 diagnostic, never fails CI +# +# CUDA build (Windows): the CUDA tests need a host built with -DCOLI_CUDA plus +# the runtime DLL. Do this FIRST — note CUDA_DLL=1 on BOTH the host and the +# rule below, or `make glm.exe` will rebuild a CPU-only host and overwrite it: +# make clean && make glm.exe CUDA_DLL=1 && make cuda-dll +# The tests auto-skip with a clear message if the host is CPU-only. +efficiency: test-python + $(PYTHON) -m unittest tests.test_inefficiency -v + +efficiency-cuda: + $(PYTHON) -m unittest tests.test_inefficiency.TinyCudaEfficiencyTest -v + +efficiency-report: + $(PYTHON) tests/test_efficiency_report.py + # Local validation: one portable CPU build and dependency-free tests. check: $(MAKE) clean $(MAKE) portable $(MAKE) test -install: glm$(EXE) olmoe$(EXE) +install: colibri$(EXE) olmoe$(EXE) $(INSTALL) -d $(DESTDIR)$(BINDIR) $(INSTALL) -d $(DESTDIR)$(LIBEXECDIR) $(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)/tools $(INSTALL) -m 755 coli $(DESTDIR)$(BINDIR)/coli - $(INSTALL) -m 755 glm$(EXE) $(DESTDIR)$(LIBEXECDIR)/glm$(EXE) + $(INSTALL) -m 755 colibri$(EXE) $(DESTDIR)$(LIBEXECDIR)/colibri$(EXE) $(INSTALL) -m 755 olmoe$(EXE) $(DESTDIR)$(LIBEXECDIR)/olmoe$(EXE) $(INSTALL) -m 644 resource_plan.py doctor.py openai_server.py $(DESTDIR)$(LIBEXECDIR)/ $(INSTALL) -m 644 tools/*.py $(DESTDIR)$(LIBEXECDIR)/tools/ @@ -365,4 +422,4 @@ clean: bench: iobench$(EXE) @if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) — run: ./iobench$(EXE) "; fi -.PHONY: all glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench +.PHONY: all colibri glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu index 9ce142f..413e53b 100644 --- a/c/backend_cuda.cu +++ b/c/backend_cuda.cu @@ -8,12 +8,21 @@ #include #include +struct RaggedKVEntry { + const void *key; + const float *host_l,*host_r; + float *latent,*rope; + int length,capacity,K,R; +}; + struct ColiCudaTensor { void *weights; float *scales; size_t weight_bytes; int fmt, I, O, device; int tracked; + RaggedKVEntry ragged[512]; + int ragged_count; }; typedef struct { @@ -23,7 +32,7 @@ typedef struct { size_t x_cap, y_cap, gate_cap, up_cap; uint8_t *qx; float *qscale; size_t qx_cap, qscale_cap; - float *host_x,*host_y; size_t host_x_cap,host_y_cap; + float *host_x,*host_y,*host_kv; size_t host_x_cap,host_y_cap,host_kv_cap; float *aq,*al,*ar,*ac; size_t aq_cap,al_cap,ar_cap,ac_cap; float *pipe_buf[24]; size_t pipe_cap[24]; /* scratch persistenti del resident pipeline */ cudaStream_t stream; @@ -338,6 +347,49 @@ __global__ static void attention_absorb_batch_kernel(float *ctx,const float *q, ctx[((size_t)s*H+h)*V+v]=a*(fmt?wscale[row]:1.f);} } +/* Independent device-resident KV sequence per row. lengths selects the valid + * prefix; latent/rope point at paged caches updated by the host wrapper. */ +__global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q, + const float *const *latent,const float *const *rope,const int *lengths, + const void *weights,const float *wscale,int fmt,int S,int H,int Q,int R, + int V,int K,int T,float scale){ + int s=blockIdx.y,h=blockIdx.x,tid=threadIdx.x,nt=lengths[s],rbase=h*(Q+V); + if(s>=S||nt<1||nt>T)return; + extern __shared__ float sm[];float *qa=sm,*cl=qa+K,*scores=cl+K,*red=scores+T; + const float *qs=q+((size_t)s*H+h)*(Q+R); + const float *ls=latent[s],*rs=rope[s]; + for(int k=tid;k>1;n;n>>=1){if(tid>1;n;n>>=1){if(tid= bytes) return 1; if (*ptr) cudaFree(*ptr); @@ -406,16 +458,17 @@ extern "C" void coli_cuda_shutdown(void) { for(int b=0;b<24;b++) if(ctx->pipe_buf[b]) cudaFree(ctx->pipe_buf[b]); if (ctx->host_x) cudaFreeHost(ctx->host_x); if (ctx->host_y) cudaFreeHost(ctx->host_y); + if (ctx->host_kv) cudaFreeHost(ctx->host_kv); if (ctx->stream) cudaStreamDestroy(ctx->stream); if (ctx->group_desc) cudaFree(ctx->group_desc); ctx->x = ctx->y = ctx->gate = ctx->up = nullptr; ctx->qx=nullptr; ctx->qscale=nullptr; ctx->aq=ctx->al=ctx->ar=ctx->ac=nullptr; - ctx->host_x=ctx->host_y=nullptr;ctx->stream=nullptr; + ctx->host_x=ctx->host_y=ctx->host_kv=nullptr;ctx->stream=nullptr; ctx->x_cap = ctx->y_cap = ctx->gate_cap = ctx->up_cap = 0; ctx->qx_cap=ctx->qscale_cap=0; ctx->aq_cap=ctx->al_cap=ctx->ar_cap=ctx->ac_cap=0; - ctx->host_x_cap=ctx->host_y_cap=0; + ctx->host_x_cap=ctx->host_y_cap=ctx->host_kv_cap=0; ctx->group_desc=nullptr; ctx->group_desc_cap=0; } g_nctx = 0; @@ -770,6 +823,89 @@ extern "C" int coli_cuda_attention_project_batch(ColiCudaTensor *w,ColiCudaTenso return attention_absorb_batch_run(w,proj,out,q,latent,rope,S,H,Q,R,V,K,T,scale); } +extern "C" int coli_cuda_attention_project_ragged(ColiCudaTensor *w,ColiCudaTensor *proj, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int T,float scale){ + if(!w||!proj||!out||!q||!keys||!latent||!rope||!lengths||S<1||S>512||T<1||T>8192|| + H<1||Q<1||R<1||V<1||K<1||K>512||w->I!=K||w->O!=H*(Q+V)|| + proj->device!=w->device||proj->I!=H*V)return 0; + DeviceContext *dc=find_ctx(w->device); + if(!select_ctx(dc))return 0; + float **dl=(float**)std::malloc((size_t)S*sizeof(*dl)); + float **dr=(float**)std::malloc((size_t)S*sizeof(*dr)); + int *old=(int*)std::malloc((size_t)S*sizeof(*old)); + int *add=(int*)std::malloc((size_t)S*sizeof(*add)); + int *off=(int*)std::malloc((size_t)S*sizeof(*off));int packed_n=0; + if(!dl||!dr||!old||!add||!off){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;} + for(int s=0;sT){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;} + RaggedKVEntry *e=nullptr; + for(int i=0;iragged_count;i++)if(w->ragged[i].key==keys[s]){e=&w->ragged[i];break;} + if(!e){ + if(w->ragged_count>=512){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;} + e=&w->ragged[w->ragged_count++];std::memset(e,0,sizeof(*e));e->key=keys[s]; + } + if(e->K!=K||e->R!=R||e->host_l!=latent[s]||e->host_r!=rope[s]||lengths[s]length){ + if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope); + e->latent=e->rope=nullptr;e->length=e->capacity=0; + e->K=K;e->R=R;e->host_l=latent[s];e->host_r=rope[s]; + } + if(lengths[s]>e->capacity){ + int cap=(lengths[s]+63)&~63;float *nl=nullptr,*nr=nullptr; + if(!cuda_ok(cudaMalloc(&nl,(size_t)cap*K*sizeof(float)),"ragged KV latent page")|| + !cuda_ok(cudaMalloc(&nr,(size_t)cap*R*sizeof(float)),"ragged KV rope page")){ + if(nl)cudaFree(nl);if(nr)cudaFree(nr);std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0; + } + if(e->length){ + cudaMemcpyAsync(nl,e->latent,(size_t)e->length*K*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream); + cudaMemcpyAsync(nr,e->rope,(size_t)e->length*R*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream); + } + if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope); + e->latent=nl;e->rope=nr;e->capacity=cap; + } + dl[s]=e->latent;dr[s]=e->rope;old[s]=e->length;add[s]=lengths[s]-e->length; + off[s]=packed_n;packed_n+=add[s]*(K+R); + } + size_t qb=(size_t)S*H*(Q+R)*sizeof(float); + size_t cb=(size_t)S*H*V*sizeof(float),ob=(size_t)S*proj->O*sizeof(float); + size_t pb=(size_t)packed_n*sizeof(float); + size_t desc=(size_t)S*(2*sizeof(float*)+4*sizeof(int)); + int ok=reserve(&dc->aq,&dc->aq_cap,qb)&&reserve(&dc->ac,&dc->ac_cap,cb)&& + reserve(&dc->y,&dc->y_cap,ob)&&reserve_bytes(&dc->group_desc,&dc->group_desc_cap,desc)&& + (!pb||(reserve(&dc->al,&dc->al_cap,pb)&&reserve_pinned(&dc->host_kv,&dc->host_kv_cap,pb))); + char *db=(char*)dc->group_desc;float **ddl=(float**)db,**ddr=ddl+S; + int *dn=(int*)(ddr+S),*dold=dn+S,*dadd=dold+S,*doff=dadd+S; + if(ok&&pb){ + for(int s=0;shost_kv+off[s]; + std::memcpy(p,latent[s]+(size_t)old[s]*K,(size_t)add[s]*K*sizeof(float)); + std::memcpy(p+(size_t)add[s]*K,rope[s]+(size_t)old[s]*R,(size_t)add[s]*R*sizeof(float)); + } + ok=cuda_ok(cudaMemcpyAsync(dc->al,dc->host_kv,pb,cudaMemcpyHostToDevice,dc->stream),"ragged KV append upload"); + } + if(ok)ok=cuda_ok(cudaMemcpyAsync(dc->aq,q,qb,cudaMemcpyHostToDevice,dc->stream),"ragged q upload")&& + cuda_ok(cudaMemcpyAsync(ddl,dl,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged latent pointers")&& + cuda_ok(cudaMemcpyAsync(ddr,dr,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged rope pointers")&& + cuda_ok(cudaMemcpyAsync(dn,lengths,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged lengths upload")&& + cuda_ok(cudaMemcpyAsync(dold,old,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged old lengths")&& + cuda_ok(cudaMemcpyAsync(dadd,add,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append lengths")&& + cuda_ok(cudaMemcpyAsync(doff,off,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append offsets"); + if(ok&&pb)ragged_kv_append<<stream>>>(ddl,ddr,dc->al,dold,dadd,doff,K,R); + if(ok)for(int s=0;sragged_count;i++)if(w->ragged[i].key==keys[s]){w->ragged[i].length=lengths[s];break;} + } + std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);if(!ok)return 0; + size_t shared=(size_t)(2*K+T+256)*sizeof(float); + attention_absorb_ragged_kernel<<stream>>>(dc->ac,dc->aq,ddl,ddr, + dn,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale); + quant_matmul<<O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights, + proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I)); + return cuda_ok(cudaGetLastError(),"ragged attention launch")&& + cuda_ok(cudaMemcpyAsync(out,dc->y,ob,cudaMemcpyDeviceToHost,dc->stream),"ragged output download")&& + cuda_ok(cudaStreamSynchronize(dc->stream),"ragged attention synchronize"); +} + extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) { if (!tensor) return; DeviceContext *ctx = find_ctx(tensor->device); @@ -781,6 +917,10 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) { } if (tensor->weights) cudaFree(tensor->weights); if (tensor->scales) cudaFree(tensor->scales); + for(int i=0;iragged_count;i++){ + if(tensor->ragged[i].latent)cudaFree(tensor->ragged[i].latent); + if(tensor->ragged[i].rope)cudaFree(tensor->ragged[i].rope); + } std::free(tensor); } diff --git a/c/backend_cuda.h b/c/backend_cuda.h index acbe4bc..63c1f11 100644 --- a/c/backend_cuda.h +++ b/c/backend_cuda.h @@ -14,6 +14,7 @@ #define COLI_CUDA_DLLEXPORT #endif + #ifdef __cplusplus extern "C" { #endif @@ -92,6 +93,11 @@ COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,C const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale); +COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale); + COLI_CUDA_DLLEXPORT void coli_cuda_tensor_free(ColiCudaTensor *tensor); COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor); COLI_CUDA_DLLEXPORT int coli_cuda_tensor_device(const ColiCudaTensor *tensor); @@ -143,4 +149,3 @@ COLI_CUDA_DLLEXPORT int coli_cuda_pipe_sync(int device); #endif #endif - diff --git a/c/backend_loader.c b/c/backend_loader.c index b743d1b..eedbd50 100644 --- a/c/backend_loader.c +++ b/c/backend_loader.c @@ -61,6 +61,10 @@ typedef int (*fn_attention_absorb_batch)(ColiCudaTensor *kv_b,float *ctx,const f typedef int (*fn_attention_absorb_batch_dev)(ColiCudaTensor *kv_b_shard,float *ctx_dev, const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); typedef int (*fn_attention_absorb_kvdev)(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale); typedef int (*fn_attention_project_batch)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q,const float *latent, const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale); +typedef int (*fn_attention_project_ragged)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale); typedef int (*fn_attention_project_batch_dev)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); typedef int (*fn_attention_project_batch_dev_out)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); typedef int (*fn_pipe_add)(int device,float *x_dev,const float *t_dev,size_t n); @@ -107,6 +111,7 @@ static struct { fn_attention_absorb_batch_dev attention_absorb_batch_dev; fn_attention_absorb_kvdev attention_absorb_kvdev; fn_attention_project_batch attention_project_batch; + fn_attention_project_ragged attention_project_ragged; fn_attention_project_batch_dev attention_project_batch_dev; fn_attention_project_batch_dev_out attention_project_batch_dev_out; fn_pipe_add pipe_add; @@ -200,6 +205,7 @@ static int coli_cuda_load(void){ RESOLVE(attention_absorb_batch_dev, fn_attention_absorb_batch_dev) RESOLVE(attention_absorb_kvdev, fn_attention_absorb_kvdev) RESOLVE(attention_project_batch, fn_attention_project_batch) + RESOLVE(attention_project_ragged, fn_attention_project_ragged) RESOLVE(attention_project_batch_dev, fn_attention_project_batch_dev) RESOLVE(attention_project_batch_dev_out, fn_attention_project_batch_dev_out) RESOLVE(pipe_add, fn_pipe_add) @@ -342,6 +348,15 @@ int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,ColiCudaTensor *o_pro return g_cuda.attention_project_batch(kv_b, o_proj, out, q, latent, rope, S, H, Q, R, V, K, T, attention_scale); } +int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale){ + if(!coli_cuda_load()) return 0; + return g_cuda.attention_project_ragged(kv_b,o_proj,out,q,keys,latent,rope,lengths, + S,H,Q,R,V,K,max_t,attention_scale); +} + int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){ if(!g_cuda.available){ return 0; } return g_cuda.attention_project_batch_dev(kv_b, o_proj, out, q_dev, latent_dev, rope_dev, S, H, Q, R, V, K, T, scale); diff --git a/c/coli b/c/coli index 4c27415..b00a1cd 100755 --- a/c/coli +++ b/c/coli @@ -20,7 +20,7 @@ Configuration through environment variables or flags (also valid after the subco --topp P adaptive expert top-p --topk N fixed top-k --ngen N maximum response tokens --cap N cache slots/layer """ -import os, sys, subprocess, argparse, json, time, signal, shutil, threading, re, codecs, tempfile, textwrap +import os, sys, subprocess, argparse, json, time, signal, shutil, threading, re, codecs, tempfile, textwrap, struct # The engine mmaps every shard (144+ files); macOS default RLIMIT_NOFILE is 256. if sys.platform != "win32": @@ -40,6 +40,8 @@ if sys.platform == "win32": except (AttributeError, OSError): pass HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +from version import __version__ as _version # Run-in-place (source checkout, "cd c && ./coli ..."): the engine, the # support modules (resource_plan.py, doctor.py, openai_server.py) and @@ -52,16 +54,20 @@ HERE = os.path.dirname(os.path.abspath(__file__)) # guess is right (e.g. a custom packaging layout). _EXE = ".exe" if sys.platform == "win32" else "" _LIBEXEC = os.path.join(os.path.dirname(HERE), "libexec", "colibri") +_here_colibri = os.path.join(HERE, "colibri" + _EXE) _here_glm = os.path.join(HERE, "glm" + _EXE) if os.environ.get("COLI_ENGINE"): GLM = os.environ["COLI_ENGINE"] TOOLS = os.path.join(os.path.dirname(GLM), "tools") +elif os.path.exists(_here_colibri): + GLM = _here_colibri + TOOLS = os.path.join(HERE, "tools") elif os.path.exists(_here_glm): GLM = _here_glm TOOLS = os.path.join(HERE, "tools") else: - GLM = os.path.join(_LIBEXEC, "glm" + _EXE) + GLM = os.path.join(_LIBEXEC, "colibri" + _EXE) TOOLS = os.path.join(_LIBEXEC, "tools") sys.path.insert(0, _LIBEXEC) # so `import resource_plan`, `doctor`, `openai_server` still resolve @@ -115,7 +121,7 @@ def sprite_lines(): def banner(sub=""): sp=sprite_lines() txt=[ - f"{C.teal}{C.b}colibrì{C.r} {C.dim}v1.0{C.r}", + f"{C.teal}{C.b}colibrì{C.r} {C.dim}v{_version}{C.r}", f"{C.dim}tiny engine, immense model{C.r}", f"{C.gray}GLM-5.2 · 744B MoE · int4 · streaming CPU{C.r}", f"{C.dgray}{sub}{C.r}" if sub else "", @@ -141,12 +147,24 @@ def need_model(model): sys.exit(f"{C.yel}engine is not built.{C.r} Run: coli build") def cuda_binary(): - if not os.path.exists(GLM) or sys.platform != "linux": return False - try: - linked=subprocess.run(["ldd",GLM],capture_output=True,text=True,timeout=3) - return any("libcudart" in line and "not found" not in line - for line in linked.stdout.splitlines()) - except (OSError,subprocess.SubprocessError): return False + if not os.path.exists(GLM): return False + if sys.platform == "linux": + try: + linked=subprocess.run(["ldd",GLM],capture_output=True,text=True,timeout=3) + return any("libcudart" in line and "not found" not in line + for line in linked.stdout.splitlines()) + except (OSError,subprocess.SubprocessError): return False + if sys.platform == "win32": + # Windows CUDA_DLL=1 builds never link libcudart directly: glm.exe loads + # coli_cuda.dll at runtime via LoadLibrary (backend_loader.c), so there's no + # import-table entry for ldd/dumpbin to see. Detect the COLI_CUDA build via a + # marker string baked into glm.c's #ifdef COLI_CUDA block instead, and require + # coli_cuda.dll to actually sit next to glm.exe (else CUDA init fails at startup). + try: + with open(GLM,"rb") as f: built=b"[CUDA] mode: routed experts" in f.read() + except OSError: return False + return built and os.path.exists(os.path.join(os.path.dirname(GLM),"coli_cuda.dll")) + return False def resource_request(a, env): ctx=a.ctx or int(env.get("CTX",4096)) @@ -219,6 +237,43 @@ def env_for(a): gpu=f" · VRAM {format_bytes(vt['budget_bytes'])}" if has_cuda and vt["devices"] else " · CPU" print(f" {C.dim}[PLAN] RAM {format_bytes(rt['budget_bytes'])} · cap {rt['cache_slots_per_layer']}/layer{gpu}{C.r}",file=sys.stderr) else: + # Windows: a bare `coli chat` (no --gpu/--vram/--auto-tier) used to ALWAYS + # run CPU-only, even on a CUDA build with a GPU present — cuda_binary() + # returned False on Windows (see above), and nothing set COLI_CUDA without + # an explicit flag. Now that detection works, auto-enable the GPU when one + # is detected so `coli chat` Just Works. Scoped to Windows: Linux already + # has working detection + the explicit-flag UX, and changing bare-chat + # semantics there is out of scope. Falls back to CPU with a warning if + # nvidia-smi is missing (discover_gpus can't size VRAM without it). + if (sys.platform == "win32" and a.gpu is None and not a.vram): + if cuda_binary(): + from resource_plan import discover_gpus, build_plan, environment_for_plan, format_bytes + gpus = discover_gpus() + if gpus: + e["COLI_CUDA"]="1" + e.setdefault("COLI_GPUS", ",".join(str(g["index"]) for g in gpus)) + # Reuse the planner so the expert-tier VRAM budget is the real + # free VRAM minus the 2 GB reserve — not a guess. Same machinery + # as --auto-tier, just without requiring the user to pass it. + ram,ctx,devices,vram_req = resource_request(a, e) + try: + plan=build_plan(a.model,ram,ctx,devices,vram_req,policy=a.policy) + e.update(environment_for_plan(plan,e,cuda_enabled=True)) + vt=plan["tiers"]["vram"] + names=",".join(g["name"].strip() for g in gpus) + print(f" {C.dim}[GPU] auto-enabled CUDA · {names} · " + f"{format_bytes(vt['budget_bytes'])} expert tier{C.r}", file=sys.stderr) + except (OSError,ValueError,json.JSONDecodeError) as error: + # Plan failed (e.g. model dir unreadable): don't block the + # run, just leave the unsized COLI_CUDA=1 and let the engine + # pick its own budget. Engine handles a missing budget. + print(f" {C.yel}[GPU] auto-enable: could not size VRAM ({error}); " + f"using engine default{C.r}", file=sys.stderr) + else: + print(f" {C.yel}[GPU] coli_cuda.dll present but nvidia-smi not found on PATH " + f"(cannot size VRAM); running CPU-only. Add nvidia-smi to PATH or pass " + f"--vram N to enable CUDA.{C.r}", file=sys.stderr) + # else: CPU build (no coli_cuda.dll) — stay silent, CPU is correct. # --gpu/--vram SENZA --auto-tier: prima venivano ignorati in silenzio e il run # partiva CPU-only senza alcun avviso — benchmark "GPU" pubblicati per errore (#121). if a.gpu is not None: @@ -227,13 +282,13 @@ def env_for(a): e["COLI_CUDA"]="0"; e.pop("CUDA_EXPERT_GB",None); e.pop("CUDA_DENSE",None) else: if not cuda_binary(): - sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} make glm CUDA=1 (this binary is CPU-only)") + sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} make colibri CUDA=1 (this binary is CPU-only)") e["COLI_CUDA"]="1" if a.gpu!="auto": e["COLI_GPUS"]=a.gpu e.setdefault("CUDA_DENSE","1") if a.vram and a.gpu!="none": if not cuda_binary(): - sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} make glm CUDA=1 (this binary is CPU-only)") + sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} make colibri CUDA=1 (this binary is CPU-only)") e["COLI_CUDA"]="1"; e["CUDA_EXPERT_GB"]=str(a.vram) return e @@ -407,8 +462,8 @@ def cmd_build(a): banner("build") if not os.path.exists(os.path.join(HERE, "Makefile")): sys.exit(f"{C.yel}coli build{C.r} only works from a source checkout (this is an installed copy).\n" - f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c glm.") - sys.exit(subprocess.call(["make","-C",HERE,"glm"])) + f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c colibri.") + sys.exit(subprocess.call(["make","-C",HERE,"colibri"])) def cmd_info(a): banner("info") @@ -480,13 +535,141 @@ def cmd_run(a): need_model(a.model) prompt=" ".join(a.prompt) if a.prompt else sys.exit('usage: coli run "your prompt"') banner("run") - # template ufficiale GLM-5.2: niente \n dopo i ruoli; = risposta diretta (nothink) - e=env_for(a); e["PROMPT"]=f"[gMASK]<|user|>{prompt}<|assistant|>" + # template ufficiale GLM-5.2: niente \n dopo i ruoli; = risposta diretta (nothink). + # THINK=1 lascia aperto, stessa convenzione del serve mode (glm.c). EN: THINK=1 leaves + # open so the engine emits its reasoning block; the default stays nothink. + tk="" if os.environ.get("THINK","0")=="1" else "" + e=env_for(a); e["PROMPT"]=f"[gMASK]<|user|>{prompt}<|assistant|>{tk}" sys.exit(subprocess.call([GLM, str(a.cap)], env=e)) +def server_probe(base, api_key=None, timeout=1.5): + """Is a coli serve alive at `base`? Returns its model_id, or None. + Probes /health then /v1/models — both cheap, neither touches the engine.""" + import urllib.request, urllib.error + def get(path): + req=urllib.request.Request(base.rstrip("/")+path) + if api_key: req.add_header("Authorization", f"Bearer {api_key}") + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read().decode("utf-8","replace")) + try: + if get("/health").get("status")!="ok": return None + data=get("/v1/models").get("data") or [] + return data[0]["id"] if data else None + except Exception: + return None + +def chat_attached(a, base, model_id): + """The chat REPL over HTTP against a running `coli serve`. + + Why this exists (the cold-chat cost, measured): spawning a private engine + pays 34-136 s of resident load on EVERY start, and begins with an empty + expert cache — hit rate 4% cold vs 55% warm, a ~10x on early decode. A + resident server pays load once and keeps the LRU warm across sessions; + its KV slots reuse the conversation prefix, so a continued chat skips + re-prefill too. The engine byte-protocol stays untouched — this is plain + OpenAI SSE over localhost, stdlib only.""" + import urllib.request + print(f" {C.grn}✦ attached{C.r} {C.dim}to {base} · model {model_id} · the engine stays warm after you quit{C.r}") + print(f" {C.dim}type and press Enter · Ctrl-C stops the answer · :reset starts a new conversation · :q exits{C.r}\n") + msgs=[] + w=term_w()-4 + while True: + if TTY: + print(f" {C.dgray}╭{'─'*w}╮{C.r}") + try: msg=input(f" {C.dgray}│{C.r} {C.teal}{C.b}›{C.r} ") + except EOFError: print(); break + print(f" {C.dgray}╰{'─'*w}╯{C.r}") + else: + try: msg=input() + except EOFError: break + msg=msg.strip() + if msg in (":q",":quit","exit"): break + if not msg: continue + if msg==":reset": msgs=[]; print(f" {C.dim}✦ new conversation{C.r}\n"); continue + msgs.append({"role":"user","content":msg}) + body=json.dumps({"model":model_id,"messages":msgs,"stream":True, + "max_tokens":a.ngen}).encode() + req=urllib.request.Request(base.rstrip("/")+"/v1/chat/completions", data=body, + headers={"Content-Type":"application/json"}) + if a.api_key: req.add_header("Authorization", f"Bearer {a.api_key}") + print(f"\n {C.teal}◆ colibrì{C.r}") + sp=Spinner("thinking…"); sp.start() + md=MDStream(" "); reply=[]; first=True; t0=time.time(); interrupted=False + try: + with urllib.request.urlopen(req) as r: + for raw in r: + line=raw.decode("utf-8","replace").strip() + if not line.startswith("data: "): continue + data=line[6:] + if data=="[DONE]": break + try: ev=json.loads(data) + except ValueError: continue + for ch in ev.get("choices",[]): + d=ch.get("delta",{}) + txt=d.get("content") + if not txt: continue # ping/ruolo/reasoning: non è testo + if first: sp.stop(); first=False + md.feed(txt); reply.append(txt) + except KeyboardInterrupt: + interrupted=True # il server annulla la richiesta alla disconnessione + except OSError as e: + sp.stop() + print(f"\n {C.yel}[server unreachable: {e}]{C.r}"); break + sp.stop(); md.close() + if reply: msgs.append({"role":"assistant","content":"".join(reply)}) + else: msgs.pop() # turno vuoto: non sporcare la history + el=time.time()-t0 + note=" · ⏹ interrupted" if interrupted else "" + print(f"\r {C.dgray}└─ ~{len(''.join(reply))//4} tok · {el:.0f}s{note}{C.r}\n") + print(f" {C.dim}goodbye — the engine keeps running for the next chat 🐦{C.r}") + +def kv_resume_notice(model_dir): + """SERVE mode silently resumes .coli_kv from disk (glm.c kv_disk_load): a chat + started today continues a conversation from days ago, with `first=0` so the + turn is appended WITHOUT the [gMASK] prefix. The engine does announce it + on stderr — but nothing here ever shows that: the drain thread's + p.stderr.read() blocks until EOF, so on a healthy start errlog is still empty + when the status lines are printed. The warning only appeared once the engine + DIED, which is exactly when it no longer mattered. + + Measured cost of the silence: a chat inherited 670 tokens of an old Italian + session ("il mio numero preferito e 7, ricordalo!"). Every later reply came + back in Italian, and "explain fibonacci in short" was answered about the + number 7 — the model was being coherent with a context nobody could see, and + it read as a quantization bug for a day. + + So say it here, in Python, from the file itself: no pipe, no thread, no + Windows deadlock risk (see the stderr comment below).""" + p=os.path.join(model_dir, ".coli_kv") + try: + with open(p,"rb") as f: + if f.read(8)!=b"COLIKV1\0": return + h=struct.unpack("<8i", f.read(32)) + n=h[6] + if n<1: return + age=time.time()-os.path.getmtime(p) + when=f"{age/86400:.0f}d ago" if age>86400 else f"{age/3600:.0f}h ago" if age>3600 else "just now" + print(f" {C.yel}↺ resuming a saved conversation: {n} tokens, last written {when}{C.r}") + print(f" {C.dgray} it steers tone, language and topic. :reset clears it · " + f"KVSAVE=0 disables saving · delete {p} to start clean{C.r}") + except (OSError, struct.error): pass + def cmd_chat(a): + # ATTACH: a running `coli serve` beats a private engine every time — the load + # (34-136 s) and the cache warmth survive between sessions. Explicit --attach + # wins; otherwise probe localhost quietly and use it if it's there. --no-attach + # forces the old behaviour. The probe costs ~1 ms when nothing is listening. + if not getattr(a,"no_attach",False): + base=getattr(a,"attach",None) or "http://127.0.0.1:8000" + mid=server_probe(base, getattr(a,"api_key",None)) + if mid: + banner(f"chat · {mid} · attached") + chat_attached(a, base, mid); return + if getattr(a,"attach",None): + sys.exit(f"--attach: no coli serve answering at {base} (start one with: coli serve --model )") need_model(a.model) banner(f"chat · {os.path.basename(a.model)} · ram {a.ram or '-'}GB · topp {a.topp or 'off'}") + kv_resume_notice(a.model) errlog=tempfile.NamedTemporaryFile(mode="w+", suffix=".log", delete=False) e=env_for(a); e["SERVE"]="1" # stderr -> PIPE, NOT stderr=errlog (file). On Windows/MinGW, pointing the @@ -619,12 +802,65 @@ def cmd_chat(a): except Exception: pass print(f" {C.teal}goodbye{C.r} {C.dim}— the hummingbird returns to its nest{C.r} 🐦\n") +def serve_pidfile(port): return os.path.join(tempfile.gettempdir(), f"coli-serve-{port}.pid") + def cmd_serve(a): need_model(a.model) + # pidfile: cosi' `coli stop` spegne tutto con un comando, senza pkill a mano. + # EN: pidfile so `coli stop` can shut everything down without manual pkill. + try: + with open(serve_pidfile(a.port),"w") as f: f.write(f"{os.getpid()} {a.model}\n") + except OSError: pass from openai_server import serve - serve(a.model, a.host, a.port, a.model_id, a.api_key, - a.cap,a.ngen,GLM,env_for(a),a.cors_origin, - a.max_queue,a.queue_timeout,a.kv_slots) + try: + serve(a.model, a.host, a.port, a.model_id, a.api_key, + a.cap,a.ngen,GLM,env_for(a),a.cors_origin, + a.max_queue,a.queue_timeout,a.kv_slots) + finally: + try: os.unlink(serve_pidfile(a.port)) + except OSError: pass + +def cmd_stop(a): + """Shut down a running `coli serve` AND its engine — one command, no pkill. + The engine re-execs itself for OMP tuning, so its process is named `exe`, + not `glm`: every `pkill -x glm` in history silently killed nothing (that is + how two 17+5 GB ghost engines OOM'd this box on 2026-07-16). This finds the + real processes: the pidfile first, then /proc by cmdline/environ — only + processes that are demonstrably ours (SERVE=1 + our SNAP, or `coli serve` + in the command line).""" + banner("stop") + targets=[] # (pid, descrizione) + pf=serve_pidfile(a.port) + try: + pid=int(open(pf).read().split()[0]) + os.kill(pid,0); targets.append((pid,f"coli serve (pidfile, port {a.port})")) + except (OSError,ValueError,IndexError): pass + for pd in os.listdir("/proc"): + if not pd.isdigit(): continue + pid=int(pd) + try: + cmd=open(f"/proc/{pd}/cmdline","rb").read().replace(b"\0",b" ").decode("utf-8","replace") + if "coli" in cmd and " serve" in cmd and pid!=os.getpid(): + if not any(p==pid for p,_ in targets): targets.append((pid,"coli serve (cmdline)")) + comm=open(f"/proc/{pd}/comm").read().strip() + if comm in ("colibri","glm","exe","olmoe"): + env=open(f"/proc/{pd}/environ","rb").read().replace(b"\0",b"\n").decode("utf-8","replace") + if "SERVE=1" in env: targets.append((pid,f"engine `{comm}` (SERVE=1)")) + except (OSError,PermissionError): continue + if not targets: + print(f" nothing running — no serve on port {a.port}, no SERVE engines"); return + for pid,desc in targets: print(f" {'would stop' if a.dry_run else 'stopping'} {pid}: {desc}") + if a.dry_run: return + for pid,_ in targets: + try: os.kill(pid, signal.SIGTERM) + except OSError: pass + time.sleep(2.0) + for pid,_ in targets: + try: os.kill(pid, signal.SIGKILL); print(f" {pid}: forced (SIGKILL)") + except OSError: pass # gia' morto: bene + try: os.unlink(pf) + except OSError: pass + print(f" {C.grn}✓ stopped{C.r} — RAM released") def cmd_web(a): """serve + open the dashboard in the browser once the API answers.""" @@ -712,6 +948,7 @@ def main(): common.add_argument("--topp", type=float, default=0); common.add_argument("--topk", type=int, default=0) common.add_argument("--temp", type=float, default=None) # temperatura token (0=greedy, default 1.0+nucleus .95) ap=argparse.ArgumentParser(prog="coli", parents=[common], description="colibrì — run GLM-5.2 locally") + ap.add_argument("--version", action="version", version=f"colibrì {_version}") sub=ap.add_subparsers(dest="cmd") sub.add_parser("build", parents=[common]); sub.add_parser("info", parents=[common]) pp=sub.add_parser("plan",parents=[common]) @@ -719,7 +956,14 @@ def main(): pd=sub.add_parser("doctor",parents=[common]) pd.add_argument("--json",action="store_true",help="emit a versioned JSON report") pr=sub.add_parser("run", parents=[common]); pr.add_argument("prompt", nargs="*") - sub.add_parser("chat", parents=[common]) + pc=sub.add_parser("chat", parents=[common]) + pc.add_argument("--attach", nargs="?", const="http://127.0.0.1:8000", default=None, + help="chat against a running `coli serve` instead of spawning an engine " + "(keeps the model loaded and the expert cache warm across chat sessions). " + "Bare --attach probes localhost:8000.") + pc.add_argument("--no-attach", action="store_true", + help="never auto-attach, always spawn a private engine") + pc.add_argument("--api-key", default=os.environ.get("COLI_API_KEY")) ps=sub.add_parser("serve", parents=[common]) ps.add_argument("--host",default="127.0.0.1"); ps.add_argument("--port",type=int,default=8000) ps.add_argument("--model-id",default=os.environ.get("COLI_MODEL_ID","glm-5.2-colibri")) @@ -728,6 +972,8 @@ def main(): ps.add_argument("--max-queue",type=int,default=int(os.environ.get("COLI_MAX_QUEUE","8"))) ps.add_argument("--queue-timeout",type=float,default=float(os.environ.get("COLI_QUEUE_TIMEOUT","300"))) ps.add_argument("--kv-slots",type=int,default=int(os.environ.get("COLI_KV_SLOTS","1"))) + pst=sub.add_parser("stop", parents=[common], help="shut down a running coli serve and its engine") + pst.add_argument("--port",type=int,default=8000); pst.add_argument("--dry-run",action="store_true") pw=sub.add_parser("web", parents=[common], help="serve + open the dashboard in a browser") for arg,kw in (("--host",dict(default="127.0.0.1")),("--port",dict(type=int,default=8000)), ("--model-id",dict(default=os.environ.get("COLI_MODEL_ID","glm-5.2-colibri"))), @@ -751,7 +997,7 @@ def main(): pc.add_argument("--no-mtp",action="store_true",help="skip the MTP head (no speculative drafts)") a=ap.parse_args() handler={"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"doctor":cmd_doctor, - "run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"bench":cmd_bench, + "run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"stop":cmd_stop,"bench":cmd_bench, "convert":cmd_convert,"web":cmd_web}.get(a.cmd) if handler: sys.exit(handler(a) or 0) banner(); print(__doc__) diff --git a/c/glm.c b/c/colibri.c similarity index 81% rename from c/glm.c rename to c/colibri.c index 6bd66f4..1f4aa5b 100644 --- a/c/glm.c +++ b/c/colibri.c @@ -40,6 +40,9 @@ #include /* fstat per mmap degli shard (COLI_MMAP) */ #include /* SIGINT = stop morbido del turno in serve mode */ #endif +#ifdef __linux__ +#include /* statfs: real fs-type check for the 9p warning (below) */ +#endif #if defined(_WIN32) && (defined(__x86_64__) || defined(__i386__)) #include /* hwinfo_emit: CPU brand string senza /proc */ #endif @@ -70,21 +73,6 @@ static int g_metal_gemm_min=16; /* COLI_METAL_GEMM_MIN: min rows to send a mat static const int *g_pre_idx; static const float *g_pre_w; static const int *g_pre_keff; static const float *g_pre_sh; /* output dello shared expert gia' calcolato su GPU */ #endif -#ifdef __AVX2__ -#include -static inline float hsum256(__m256 v){ /* somma orizzontale di 8 float */ - __m128 lo=_mm256_castps256_ps128(v), hi=_mm256_extractf128_ps(v,1); - lo=_mm_add_ps(lo,hi); __m128 sh=_mm_movehl_ps(lo,lo); lo=_mm_add_ps(lo,sh); - sh=_mm_shuffle_ps(lo,lo,1); lo=_mm_add_ss(lo,sh); return _mm_cvtss_f32(lo); -} -#elif defined(__ARM_NEON) -#include /* Apple Silicon / aarch64: kernel NEON */ -#elif defined(__VSX__) -#include /* POWER8+ (ppc64le): kernel VSX */ -#undef vector /* igiene: si usano __vector/__bool espliciti */ -#undef pixel -#undef bool -#endif #ifdef __APPLE__ #include /* host_statistics64: MemAvailable di macOS */ #endif @@ -180,6 +168,28 @@ typedef struct { uint32_t **eusage; /* contatori persistenti (per STATS/PIN) */ uint32_t **eheat; /* calore recente per promotion/demotion live */ uint32_t **elast, eaccess_clock; /* recency per LFRU session-local */ + /* DISK-CLASS: PRIVATE recency state, read only by expert_classify(). Private -- + * not the real elast/eaccess_clock -- kept fully separate so DISK-CLASS's bookkeeping + * can never read from or write into stock eviction state: every DISK-CLASS write lives + * inside its own need_classify/dc_on gate, so "byte-identical with PROF=0" is provable + * by construction instead of by argument. (Historical note: when this was first written, + * the Metal pre-routed FASE A path (g_pre_idx) never bumped the real elast/eaccess_clock + * -- on Metal decode the real clock froze at end of prefill, so REPIN's LRU tie-breaker + * ran on stale recency for the rest of the run. That was an upstream defect; it has since + * been reported and fixed (#417, cfcc742) -- FASE A now bumps the real clock too. The + * private clock is retained anyway: separation from stock state is the stronger property, + * independent of whether the real clock is correct.) elast_dc/eaccess_clock_dc tick in + * BOTH FASE A paths, under the same need_classify gate, at the same rate the real clock + * ticks on the CPU path (one per selected (position,expert)) -- so the + * COLI_DISKCLASS_WINDOW window keeps its meaning in every mode. elast_pre snapshots + * elast_dc just BEFORE this call's own bump (see the touched[] guard in FASE A) -- + * classifying against the live array would read the bump routing just made a few lines + * above the load that needed it, so a giant cold prefill burst would score every expert + * "just accessed" and get called warm. Recency alone (not eheat's access COUNT): a count + * never decays, so an expert hot early in a long session would keep reading "warm" long + * after it dropped out of the working set. Same shape/allocation as elast; NULL for dense + * layers. */ + uint32_t **elast_dc, **elast_pre, eaccess_clock_dc; /* DSA lightning indexer (attivo solo se i pesi out-idx-* sono presenti) */ int has_dsa; QT *ix_wq, *ix_wk, *ix_wp; /* per layer FULL: wq_b, wk, weights_proj */ @@ -193,12 +203,16 @@ typedef struct { uint64_t mtp_prop, mtp_acc; /* statistica acceptance */ int **eroute; int *enr; /* metodo C: routing dell'ULTIMO token per layer */ uint64_t eclock, hits, miss, ereq; + uint64_t hit_pin, hit_ecache; /* split di hits per tier (#336): pin vs LRU ecache */ uint64_t gpu_expert_calls; int gpu_expert_count; int64_t gpu_expert_bytes; uint64_t n_fw, n_emit; /* metodo E: forward di decode / token emessi */ uint64_t route_slots, route_swaps; /* CACHE_ROUTE: slots chosen / substituted vs true top-K */ uint64_t route_agree_hit, route_agree_tot; /* ROUTE_AGREE: |chosen ∩ true top-K| / K */ double route_kl_sum; uint64_t route_kl_n; /* mean KL(true||chosen) on gate mass */ - double t_ewait, t_emm, t_attn, t_kvb, t_head;/* profiling: dove va il tempo (wall del + double t_ewait, t_emm, t_ecpu, t_egpu, t_route, t_p2p, t_attn, t_kvb, t_head; + uint64_t n_p2p; /* P0 execution profile: tier split + residual hops */ + uint64_t cpu_expert_rows; int64_t cpu_expert_bytes; + /* profiling: dove va il tempo (wall del * thread di compute; il servizio disco * overlappato vive in g_edisk_ns) */ double t_aproj,t_acore,t_aout; /* attention breakdown */ @@ -211,13 +225,21 @@ typedef struct { uint64_t bytes_mtp, bytes_main; /* byte letti da disco per tipo layer */ } Model; -static void usage_save(Model *m); /* cache che impara: definita accanto a stats_dump */ -static void tiers_emit(Model *m); -static void ehit_mark(Model *m, int layer, int eid); -static void emap_emit(Model *m); -static void hits_emit(Model *m); -static void hwinfo_emit(Model *m); -static int64_t expert_bytes_probe(Model *m, int ebits); /* PROF: tier sizes in the report */ +#include "quant.h" +static int g_no_fused_pair=0; +static int g_spec_pin=1; +static int g_spec_live=0; +static inline int spec_pinned(void){ return g_spec_pin && g_spec_live; } + +static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot); +static void matmul_qt(float *y, const float *x, QT *w, int S){ matmul_qt_ex(y,x,w,S,1); } + +static void expert_gate_up(float *g,float *u,const float *x,QT *wg,QT *wu,int S){ + if(!g_no_fused_pair&&!spec_pinned()&&S==1&&wg->fmt==2&&wu->fmt==2&&wg->I==wu->I&&wg->O==wu->O) + matmul_i4_pair(g,u,x,wg->q4,wg->s,wu->q4,wu->s,wg->I,wg->O); + else { matmul_qt(g,x,wg,S); matmul_qt(u,x,wu,S); } +} + static int g_repin; static uint64_t g_last_repin; #ifdef COLI_CUDA @@ -298,21 +320,93 @@ static _Atomic int64_t g_prof_io; /* bytes pread()/faulted from e * wait ~ service means the loads block the compute thread. */ static _Atomic int64_t g_edisk_ns; static double edisk_s(void){ return atomic_load_explicit(&g_edisk_ns,memory_order_relaxed)*1e-9; } +/* DISK-CLASS (PROF=1): per-load cold/warm classification against the engine's own + * recency state. Instrumentation only -- it never changes which fd serves a read (see + * expert_classify() and its call site in expert_load_impl; the fd choice expression is + * untouched by this feature). COLI_DISKCLASS_WINDOW is the recency window in ticks of the + * PRIVATE clock (m->eaccess_clock_dc -- NOT the real eaccess_clock; see elast_dc in Model + * for why DISK-CLASS keeps its own clock instead of reading the real one): one tick per + * selected (position,expert) in FASE A while classification is active, the same per-token + * rate the real clock has on the CPU path, so the window's meaning is unchanged. At or + * under the window = warm; 0 (default, unset) derives it from topk*n_layers*8 once the + * model config is known (main(), right after model_init) -- roughly "seen in the last ~8 + * tokens", generous on purpose (conservative-toward-warm: a load the page cache could have + * served that gets labeled cold overstates the cold class, the bucket this line exists to + * size). */ +static uint32_t g_direct_heat_ticks=0; +static int g_direct_heat_explicit=0; /* 1 if COLI_DISKCLASS_WINDOW was set (skip the auto-derive) */ +#define DC_COLD 0 +#define DC_WARM 1 +static _Atomic uint64_t g_dc_n[2]; /* [DC_COLD]/[DC_WARM]: loads classified */ +static _Atomic int64_t g_dc_bytes[2]; /* bytes read (weights + scales, matches g_prof_io) */ +static _Atomic int64_t g_dc_ns[2]; /* wall ns spent reading (thread-seconds, like g_edisk_ns) */ +static _Atomic uint64_t g_dc_direct_n[2]; /* subset of the above ACTUALLY served by the uncached fd */ +/* Busy-wall per class + combined: how much WALL time had >=1 classified load of the + * class in flight (thread-seconds / busy-wall = average concurrency; bytes / busy-wall + * = aggregate GB/s the disk actually delivered for that class -- the quantity the + * thread-second numbers alone can't answer: N slow overlapped reads can beat N fast + * serial ones in aggregate, and only wall-denominated rates see it). Transition scheme: + * 0->1 records a start, 1->0 accumulates (now - start). One dedicated mutex serializes + * the transition bookkeeping -- two short lock/unlock pairs per load against ms-scale + * reads; a CAS scheme would save nothing measurable and be harder to audit + * (correctness over cleverness). Only COMPLETED intervals are in the accumulators: an + * interval still open at report time is not counted (bounded by one read's duration -- + * noise at report granularity). */ +static pthread_mutex_t g_dc_wall_mx=PTHREAD_MUTEX_INITIALIZER; +static int g_dc_inflight[2], g_dc_inflight_all; /* guarded by g_dc_wall_mx */ +static double g_dc_wall_open[2], g_dc_wall_open_all; /* start of the open interval */ +static int64_t g_dc_wall_ns[2], g_dc_wall_all_ns; /* completed busy-wall ns */ +static void dc_wall_enter(int cls, double now){ + pthread_mutex_lock(&g_dc_wall_mx); + if(g_dc_inflight[cls]++==0) g_dc_wall_open[cls]=now; + if(g_dc_inflight_all++==0) g_dc_wall_open_all=now; + pthread_mutex_unlock(&g_dc_wall_mx); +} +static void dc_wall_exit(int cls, double now){ + pthread_mutex_lock(&g_dc_wall_mx); + if(--g_dc_inflight[cls]==0) g_dc_wall_ns[cls]+=(int64_t)((now-g_dc_wall_open[cls])*1e9); + if(--g_dc_inflight_all==0) g_dc_wall_all_ns +=(int64_t)((now-g_dc_wall_open_all)*1e9); + pthread_mutex_unlock(&g_dc_wall_mx); +} +static int dc_needed(void); /* fwd: defined with the classifier (needs g_prof) */ +static void dc_wall_read(int64_t out[2], int64_t *all){ /* mutex-consistent snapshot for the report */ + if(!dc_needed()){ out[0]=out[1]=0; *all=0; return; } /* off for the whole process => accumulators are + * provably zero (only dc_wall_exit writes them, + * only under dc_on): skip the lock, zero work. + * Needed because prof_base runs unconditionally + * at some call sites ("cheap enough to always"). */ + pthread_mutex_lock(&g_dc_wall_mx); + out[0]=g_dc_wall_ns[0]; out[1]=g_dc_wall_ns[1]; *all=g_dc_wall_all_ns; + pthread_mutex_unlock(&g_dc_wall_mx); +} #define PROF_LAT_CAP 32768 static double g_prof_lat[PROF_LAT_CAP]; /* per-forward decode wall clock (ring) */ static uint64_t g_prof_nlat; /* forwards recorded (monotonic) */ static void prof_lat(double s){ g_prof_lat[g_prof_nlat++ % PROF_LAT_CAP]=s; } /* snapshot for windowed reports (serve mode: one report per turn) */ typedef struct { - double edisk,ewait,emm,attn,head; - int64_t io; uint64_t hits,miss,ereq,n_fw,n_emit,nlat; + double edisk,ewait,emm,ecpu,egpu,route,p2p,attn,head; + int64_t io,cpu_bytes; uint64_t hits,miss,ereq,n_fw,n_emit,nlat,n_p2p,cpu_rows; + uint64_t hit_pin,hit_ecache; + uint64_t dc_n[2], dc_direct_n[2]; int64_t dc_bytes[2], dc_ns[2]; /* DISK-CLASS */ + int64_t dc_wall_ns[2], dc_wall_all_ns; /* busy-wall (per class + combined) */ } ProfBase; static void prof_base(Model *m, ProfBase *b){ b->edisk=edisk_s(); b->ewait=m->t_ewait; b->emm=m->t_emm; + b->ecpu=m->t_ecpu; b->egpu=m->t_egpu; b->route=m->t_route; b->p2p=m->t_p2p; b->attn=m->t_attn; b->head=m->t_head; b->io=atomic_load_explicit(&g_prof_io,memory_order_relaxed); b->hits=m->hits; b->miss=m->miss; b->ereq=m->ereq; - b->n_fw=m->n_fw; b->n_emit=m->n_emit; b->nlat=g_prof_nlat; + b->hit_pin=m->hit_pin; b->hit_ecache=m->hit_ecache; + b->n_fw=m->n_fw; b->n_emit=m->n_emit; b->nlat=g_prof_nlat; b->n_p2p=m->n_p2p; + b->cpu_bytes=m->cpu_expert_bytes;b->cpu_rows=m->cpu_expert_rows; + for(int i=0;i<2;i++){ + b->dc_n[i]=atomic_load_explicit(&g_dc_n[i],memory_order_relaxed); + b->dc_bytes[i]=atomic_load_explicit(&g_dc_bytes[i],memory_order_relaxed); + b->dc_ns[i]=atomic_load_explicit(&g_dc_ns[i],memory_order_relaxed); + b->dc_direct_n[i]=atomic_load_explicit(&g_dc_direct_n[i],memory_order_relaxed); + } + dc_wall_read(b->dc_wall_ns,&b->dc_wall_all_ns); } static float *falloc(int64_t n){ @@ -321,713 +415,15 @@ static float *falloc(int64_t n){ if(n<0 || (uint64_t)n > SIZE_MAX/sizeof(float)){ fprintf(stderr,"falloc: n=%lld is out of range\n",(long long)n); exit(1); } float *p=malloc((size_t)n*sizeof(float)); if(!p){fprintf(stderr,"OOM\n");exit(1);} return p; } -/* ---- Accumulatore int4->float a 512 bit / 512-bit int4->float accumulator ---- - * Stessa matematica lossless di matmul_i4 (nibble->f32, FMA), ma 32 pesi/iter su - * due catene FMA indipendenti. NON bit-identico al vecchio ordine: la riduzione - * ad albero accumula MENO errore della somma sequenziale (misurato 2-4x più - * vicino all'oracolo double sulle forme reali; perplexity invariata, +4-7% sul - * decode con routing CPU-heavy — vedi docs/experiments/glm52-6x5090-2026-07-12.md). - * EN: same lossless math as matmul_i4, 32 weights/iter on two independent FMA - * chains. Not bit-identical to the old order: tree reduction accumulates LESS - * rounding than sequential summation. I4_ACC512=0 restores the old order (A/B). */ -#if defined(__AVX512F__) && defined(__AVX512BW__) -static int g_i4_acc512=1; -static inline float dot_i4f_avx512(const uint8_t *w,const float *x,int I){ - const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8); - __m512 acc0=_mm512_setzero_ps(),acc1=_mm512_setzero_ps(); int i=0; - for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1))); - __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi); - __m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8)); - __m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8)); - acc0=_mm512_fmadd_ps(_mm512_loadu_ps(x+i),w0,acc0); - acc1=_mm512_fmadd_ps(_mm512_loadu_ps(x+i+16),w1,acc1); - } - return _mm512_reduce_add_ps(_mm512_add_ps(acc0,acc1)); -} -/* selftest contro il riferimento scalare (I4_ACC512_TEST=1): copre l'ordine dei - * nibble e ogni multiplo di 32. / selftest vs the scalar reference. */ -static int i4_acc512_selftest(void){ - enum { N=224 }; uint8_t w[(N+1)/2]; float x[N]; - for(int i=0;i>1]=(uint8_t)(q+8); - else w[i>>1]|=(uint8_t)((q+8)<<4); - x[i]=(float)(((i*29+7)%101)-50)/37.f; - } - for(int n=32;n<=N;n+=32){ - float ref=0; for(int i=0;i>1]>>((i&1)*4))&15)-8); - float got=dot_i4f_avx512(w,x,n),tol=2e-5f*(1.f+fabsf(ref)); - if(fabsf(got-ref)>tol){ fprintf(stderr,"AVX512 i4 selftest n=%d: %.9g != %.9g\n",n,got,ref); return 0; } - } - return 1; -} -#endif -/* y[S,O] = x[S,I] @ W^T, W[O,I] f32 */ -static void matmul(float *y, const float *x, const float *W, int S, int I, int O){ - #pragma omp parallel for schedule(static) - for (int o=0;o>1))); /* 8 byte=16 nibble */ - __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i nib=_mm_unpacklo_epi8(lo,hi); /* nibble in ordine */ - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } - a=hsum256(acc); -#elif defined(__ARM_NEON) - const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); - float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); - for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); /* 8 byte=16 nibble */ - uint8x8x2_t z=vzip_u8(vand_u8(by,m4), vshr_n_u8(by,4)); /* nibble in ordine */ - int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[0]),b8)); - int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[1]),b8)); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } - a=vaddvq_f32(vaddq_f32(ac0,ac1)); -#endif -#if defined(__AVX512F__) && defined(__AVX512BW__) - } -#endif - for(;i+1>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8; - a += xs[i]*(float)lo + xs[i+1]*(float)hi; } - if(i>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; } - y[(int64_t)s*O+o]=a*sc; } } -} -/* y[S,O] = x[S,I] @ W^T with W int4 packed (2/byte) + per-GROUP scales (fmt=4). - * Same nibble math as matmul_i4, but the scale changes every `gs` elements along I. - * The accumulator resets at each group boundary: dot(x[grp], w[grp]) * scale[grp]. - * gs MUST be a multiple of 16 (the AVX2 vector width). */ -static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const float *scale, - int S, int I, int O, int gs){ - int rb=(I+1)/2; int ng=(I+gs-1)/gs; - #pragma omp parallel for schedule(static) - for(int o=0;oI) glen=I-base; - float sc=scl[g]; - int i=base; -#ifdef __AVX2__ - const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8); - __m256 acc=_mm256_setzero_ps(); - for(; i+16<=base+glen; i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1))); - __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i nib=_mm_unpacklo_epi8(lo,hi); - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } - a+=hsum256(acc)*sc; -#endif - /* scalar tail for the group remainder */ - for(; i>1]; - a+=(xs[i]*(float)((int)(byte&0xF)-8)+xs[i+1]*(float)((int)(byte>>4)-8))*sc; } - else { uint8_t byte=w[i>>1]; a+=xs[i]*(float)((int)(byte&0xF)-8)*sc; } - } - } - y[(int64_t)s*O+o]=a; - } - } -} -/* Decode hot path for gate+up: same exact q4 dot products as matmul_i4, but one - * OpenMP dispatch covers both matrices. KTransformers uses persistent pools; - * this keeps colibri dependency-free while removing one team launch/expert. */ -static void matmul_i4_pair(float *yg, float *yu, const float *x, - const uint8_t *qg, const float *sg, - const uint8_t *qu, const float *su, int I, int O){ - int rb=(I+1)/2; - #pragma omp parallel for schedule(static) - for(int z=0;z<2*O;z++){ - int o=z>1))); - __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i nib=_mm_unpacklo_epi8(lo,hi); - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); } - a=hsum256(acc); -#elif defined(__ARM_NEON) - const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); - float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0); - for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); - uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4)); - int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8)); - int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8)); - ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); - ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); - ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); - ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } - a=vaddvq_f32(vaddq_f32(ac0,ac1)); -#endif -#if defined(__AVX512F__) && defined(__AVX512BW__) - } -#endif - for(;i+1>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); } - if(i>1]&15)-8); - (z=2) non calcolano la STESSA funzione. Tre interruttori dipendono da S: il gate - * int4-IDOT (S>=g_i4s — asimmetrico proprio dove g_i4s>1), la fusione gate+up solo-S==1, - * e la soglia righe del GEMM Metal. Con SPEC_PIN=1 (default) ogni forward emesso mentre - * i draft del modello sono attivi resta sulla famiglia di kernel di S=1: draft e verifica - * coincidono per costruzione. Prefill e decode non speculativo sono intoccati. - * EN: MTP acceptance collapses when the draft (S=1) and verify (S>=2) forwards do not - * compute the SAME function. Three switches are S-dependent: the int4 IDOT gate - * (S>=g_i4s — asymmetric exactly on ISAs where g_i4s>1), the S==1-only gate+up fusion, - * and the Metal GEMM row threshold. SPEC_PIN=1 (default) pins every forward issued - * while model drafts are live to the platform's S=1 kernel family, so draft and verify - * agree by construction; prefill and non-speculative decode are untouched. - * SPEC_PIN=0 restores the S-dependent gates (A/B). */ -static int g_spec_pin=1; -static int g_spec_live=0; /* set by spec_decode while drafts are live */ -static inline int spec_pinned(void){ return g_spec_pin && g_spec_live; } -static void expert_gate_up(float *g,float *u,const float *x,QT *wg,QT *wu,int S){ - if(!g_no_fused_pair&&!spec_pinned()&&S==1&&wg->fmt==2&&wu->fmt==2&&wg->I==wu->I&&wg->O==wu->O) - matmul_i4_pair(g,u,x,wg->q4,wg->s,wu->q4,wu->s,wg->I,wg->O); - else { matmul_qt(g,x,wg,S); matmul_qt(u,x,wu,S); } -} -/* y[S,O] = x[S,I] @ W^T con W int2 impacchettato (4 valori/byte) + scala[O]. nibble 2-bit -> [-2,1]. */ -static void matmul_i2(float *y, const float *x, const uint8_t *q2, const float *scale, int S, int I, int O){ - int rb=(I+3)/4; - #pragma omp parallel for schedule(static) - for (int o=0;o>2))); /* 4 byte=16 valori */ - __m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2); - __m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2); - __m128i lo=_mm_unpacklo_epi8(p0,p1), hi=_mm_unpacklo_epi8(p2,p3); - __m128i nib=_mm_unpacklo_epi16(lo,hi); /* 16 valori in ordine */ - __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b2)); - __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b2)); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); - acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } - a=hsum256(acc); -#elif defined(__ARM_NEON) - const uint8x8_t m2v=vdup_n_u8(3); const int8x8_t b2v=vdup_n_s8(2); - float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); - for(;i+16<=I;i+=16){ uint32_t wd; memcpy(&wd, w+(i>>2), 4); /* 4 byte=16 valori */ - uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd)); - uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v)); - uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6)); - uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0])); - int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[0]),b2v)); /* 16 valori in ordine */ - int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[1]),b2v)); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); - ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); - ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } - a=vaddvq_f32(vaddq_f32(ac0,ac1)); -#endif - for(;i>2]; int sh=(i&3)*2; a += xs[i]*(float)((int)((byte>>sh)&3)-2); } - y[(int64_t)s*O+o]=a*sc; } } -} -/* ---- KERNEL INTERI (IDOT): attivazioni quantizzate a int8 per riga (absmax/127, - * stile Q8_0), prodotto scalare INTERO via maddubs/madd AVX2 — niente conversione - * f32 dei pesi nel ciclo caldo. ~2-3x sui matmul quantizzati; errore aggiunto ~0.3% - * RMS per matmul (attivazione int8), IDOT=0 torna al percorso f32 esatto. */ -#if defined(__AVX512VNNI__) && defined(__AVX512BW__) -#define IDOT_KERNEL "avx512-vnni" -#elif defined(__AVXVNNI__) && defined(__AVX2__) -#define IDOT_KERNEL "avx-vnni" -#elif defined(__AVX2__) -#define IDOT_KERNEL "avx2" -#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) -#define IDOT_KERNEL "neon-i8mm" -#elif defined(__ARM_NEON) -#define IDOT_KERNEL "neon" -#elif defined(__VSX__) -#define IDOT_KERNEL "vsx" -#else -#define IDOT_KERNEL "scalar" -#endif -static int g_idot=1; -#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) -static int g_i4s=1; /* SDOT presente: int4 IDOT conviene anche a S=1 (decode). Misurato - * su Apple M-series: +14%%, expert-matmul -16%%. EN: with SDOT, int4 - * IDOT pays even at S=1 (decode); measured on Apple M-series. */ -#elif defined(__VSX__) -static int g_i4s=1; /* POWER8 vec_msum: qui il fallback f32 e' SCALARE, quindi l'IDOT - * int4 conviene anche a S=1. Misurato su POWER8 S824 (vedi PR). - * EN: on VSX the f32 fallback is plain scalar C, so int4 IDOT - * pays even at S=1. Measured on a POWER8 S824 (see PR). */ -#else -static int g_i4s=2; /* senza SDOT / altrove: soglia originale (misura AVX2 dell'autore). - * EN: without SDOT / elsewhere: original threshold (author's AVX2). */ -#endif -static inline float qrow_i8(const float *x, int8_t *q, int I){ - float amax=0; for(int i=0;iamax)amax=a; } - float s=amax/127.f; if(s<1e-12f) s=1e-12f; float inv=1.f/s; - for(int i=0;i s32 directly, 64 bytes/iter, no 16-bit intermediate. - * AVX-512 has no vpsignb: |w| via abs, sign folded into x with a mask-negate - * (w==0 -> product 0 either way). |x|<=127 (qrow_i8), |w|<=128 as u8: each - * s32 lane adds <= 4*128*127, safe up to I=16384 like the AVX2 bound. */ - __m512i acc=_mm512_setzero_si512(); - for(;i+64<=I;i+=64){ - __m512i wv=_mm512_loadu_si512((const void*)(w+i)); - __m512i xv=_mm512_loadu_si512((const void*)(x+i)); - __mmask64 neg=_mm512_movepi8_mask(wv); - __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv); - acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs); - } - sum=_mm512_reduce_add_epi32(acc); -#elif defined(__AVXVNNI__) && defined(__AVX2__) - /* AVX-VNNI 128-bit: vpdpbusd u8*s8 -> s32, 16 byte/iter. Stesso trucco del - * segno della variante 512-bit: |w| via abs, segno piegato in x con maschera - * (w==0 -> product 0). __AVX2__ serve per _mm_sign_epi8 / abs. */ - __m128i acc=_mm_setzero_si128(); - for(;i+16<=I;i+=16){ - __m128i wv=_mm_loadu_si128((const __m128i*)(w+i)); - __m128i xv=_mm_loadu_si128((const __m128i*)(x+i)); - __m128i xs=_mm_sign_epi8(xv,wv); /* x * sign(w); _mm_sign zona __AVX2__ */ - acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(wv),xs); - } - sum=hsum128_i32(acc); -#elif defined(__AVX2__) - __m256i acc=_mm256_setzero_si256(); const __m256i ones=_mm256_set1_epi16(1); - for(;i+32<=I;i+=32){ - __m256i wv=_mm256_loadu_si256((const __m256i*)(w+i)); - __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); - __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); - acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); - } - sum=hsum256_i32(acc); -#elif defined(__ARM_NEON) - /* ARM: SDOT nativo se disponibile (Apple Silicon: sempre); altrimenti vmull/vpadal. - * Stesso bound anti-overflow del trucco AVX2: coppie <= 128*127*2 = 32512 < 32767. */ -#if defined(__ARM_FEATURE_DOTPROD) - /* 4 accumulatori indipendenti: SDOT ha latenza ~3-4 cicli, con un solo acc la - * catena seriale strozza il core a ~26 GB/s di pesi; con 4 lane indipendenti il - * dot diventa memory-bound (misurato su M4: 26 -> 63 GB/s per core, 2.4x). */ - int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); - for(;i+64<=I;i+=64){ - a0=vdotq_s32(a0,vld1q_s8(w+i), vld1q_s8(x+i)); - a1=vdotq_s32(a1,vld1q_s8(w+i+16),vld1q_s8(x+i+16)); - a2=vdotq_s32(a2,vld1q_s8(w+i+32),vld1q_s8(x+i+32)); - a3=vdotq_s32(a3,vld1q_s8(w+i+48),vld1q_s8(x+i+48)); - } - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - for(;i+16<=I;i+=16) acc=vdotq_s32(acc,vld1q_s8(w+i),vld1q_s8(x+i)); - sum=vaddvq_s32(acc); -#else - int32x4_t acc=vdupq_n_s32(0); - for(;i+16<=I;i+=16){ - int8x16_t wv=vld1q_s8(w+i), xv=vld1q_s8(x+i); - int16x8_t p=vmull_s8(vget_low_s8(wv),vget_low_s8(xv)); - p=vmlal_s8(p,vget_high_s8(wv),vget_high_s8(xv)); - acc=vpadalq_s16(acc,p); - } - sum=vaddvq_s32(acc); -#endif -#elif defined(__VSX__) - /* POWER8: vec_msum (s8 x u8 -> s32) somma i prodotti byte DIRETTAMENTE in lane - * s32, 16 byte/iter: il bound anti-saturazione a 16 bit di maddubs qui non serve. - * Stesso trucco del segno (|w| u8 per x*sign(w) s8), ma |w| via select+sub MODULO - * e non vec_abs: -128 deve diventare 128 u8, non saturare a 127. - * EN: vec_msum accumulates byte products straight into s32 lanes; |w| is built - * with a modulo subtract select instead of vec_abs so w=-128 wraps to 128 (u8) - * rather than saturating to 127. |x|<=127 from qrow_i8, so x negation is safe. */ - __vector signed int acc=vec_splats(0); - const __vector signed char vz=vec_splats((signed char)0); - for(;i+16<=I;i+=16){ - __vector signed char wv=vec_xl(0,(const signed char*)(w+i)); - __vector signed char xv=vec_xl(0,(const signed char*)(x+i)); - __vector __bool char neg=vec_cmplt(wv,vz); - __vector signed char xs=vec_sel(xv,vec_sub(vz,xv),neg); - __vector unsigned char wa=(__vector unsigned char)vec_sel(wv,vec_sub(vz,wv),neg); - acc=vec_msum(xs,wa,acc); - } - sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); -#endif - for(;i int8 [-8,7] al volo, poi stesso trucco */ -static inline int32_t dot_i4i8(const uint8_t *w4, const int8_t *x, int I){ - int32_t sum=0; int i=0; -#if defined(__AVX512VNNI__) && defined(__AVX512BW__) - /* 32 bytes = 64 nibbles -> int8 in [-8,7], one vpdpbusd per 64 values. - * 256-bit unpack leaves values in per-128-lane order [0-15][32-47]/[16-31][48-63]; - * dot pairing is order-invariant, so permute x's 128-bit blocks to match - * instead of re-ordering w (one vpermq per iter, off the critical unpack path). */ - const __m256i m4v=_mm256_set1_epi8(0x0F); - const __m512i b8v=_mm512_set1_epi8(8); - const __m512i xidx=_mm512_setr_epi64(0,1,4,5,2,3,6,7); - __m512i acc=_mm512_setzero_si512(); - for(;i+64<=I;i+=64){ - __m256i by=_mm256_loadu_si256((const __m256i*)(w4+(i>>1))); - __m256i lo=_mm256_and_si256(by,m4v), hi=_mm256_and_si256(_mm256_srli_epi16(by,4),m4v); - __m256i z0=_mm256_unpacklo_epi8(lo,hi), z1=_mm256_unpackhi_epi8(lo,hi); - __m512i wv=_mm512_sub_epi8(_mm512_inserti64x4(_mm512_castsi256_si512(z0),z1,1),b8v); - __m512i xv=_mm512_permutexvar_epi64(xidx,_mm512_loadu_si512((const void*)(x+i))); - __mmask64 neg=_mm512_movepi8_mask(wv); - __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv); - acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs); - } - sum=_mm512_reduce_add_epi32(acc); -#elif defined(__AVXVNNI__) && defined(__AVX2__) - /* AVX-VNNI 128-bit, int4: 16 byte = 32 nibble -> int8 [-8,7] in due half - * (n0/n1), ciascuno alimentato a un vpdpbusd da 16 byte. Stesso unpack - * 128-bit del ramo AVX2 sotto; 32 elementi/iter come li. */ - const __m128i m4=_mm_set1_epi8(0x0F); const __m128i b8=_mm_set1_epi8(8); - __m128i acc=_mm_setzero_si128(); - for(;i+32<=I;i+=32){ - __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); /* 16 byte = 32 nibble */ - __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); /* nibble in ordine */ - __m128i w0=_mm_sub_epi8(n0,b8), w1=_mm_sub_epi8(n1,b8); - __m128i x0=_mm_loadu_si128((const __m128i*)(x+i)); - __m128i x1=_mm_loadu_si128((const __m128i*)(x+i+16)); - acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0)); - acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1)); - } - sum=hsum128_i32(acc); -#elif defined(__AVX2__) - const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi8(8); - const __m256i ones=_mm256_set1_epi16(1); - __m256i acc=_mm256_setzero_si256(); - for(;i+32<=I;i+=32){ - __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); /* 16 byte = 32 nibble */ - __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); - __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); /* in ordine */ - __m256i wv=_mm256_sub_epi8(_mm256_set_m128i(n1,n0),b8); - __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); - __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); - acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); - } - sum=hsum256_i32(acc); -#elif defined(__ARM_NEON) - const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8); -#if defined(__ARM_FEATURE_DOTPROD) - /* 4 accumulatori indipendenti (vedi dot_i8i8): spezza la catena seriale su acc. - * Misurato su M4: 12.4 -> 29.9 GB/s di pesi per core (2.4x). */ - int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); - for(;i+64<=I;i+=64){ - uint8x16_t byA=vld1q_u8(w4+(i>>1)), byB=vld1q_u8(w4+(i>>1)+16); - uint8x16x2_t zA=vzipq_u8(vandq_u8(byA,m4q), vshrq_n_u8(byA,4)); /* nibble in ordine */ - uint8x16x2_t zB=vzipq_u8(vandq_u8(byB,m4q), vshrq_n_u8(byB,4)); - a0=vdotq_s32(a0,vsubq_s8(vreinterpretq_s8_u8(zA.val[0]),b8q),vld1q_s8(x+i)); - a1=vdotq_s32(a1,vsubq_s8(vreinterpretq_s8_u8(zA.val[1]),b8q),vld1q_s8(x+i+16)); - a2=vdotq_s32(a2,vsubq_s8(vreinterpretq_s8_u8(zB.val[0]),b8q),vld1q_s8(x+i+32)); - a3=vdotq_s32(a3,vsubq_s8(vreinterpretq_s8_u8(zB.val[1]),b8q),vld1q_s8(x+i+48)); - } - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - for(;i+32<=I;i+=32){ - uint8x16_t by=vld1q_u8(w4+(i>>1)); /* 16 byte = 32 nibble */ - uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); /* nibble in ordine */ - acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q),vld1q_s8(x+i)); - acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q),vld1q_s8(x+i+16)); - } - sum=vaddvq_s32(acc); -#else - int32x4_t acc=vdupq_n_s32(0); - for(;i+32<=I;i+=32){ - uint8x16_t by=vld1q_u8(w4+(i>>1)); /* 16 byte = 32 nibble */ - uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); /* nibble in ordine */ - int8x16_t w0=vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q); - int8x16_t w1=vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q); - int8x16_t x0=vld1q_s8(x+i), x1=vld1q_s8(x+i+16); - int16x8_t p=vmull_s8(vget_low_s8(w0),vget_low_s8(x0)); /* |w|<=8: nessun overflow */ - p=vmlal_s8(p,vget_high_s8(w0),vget_high_s8(x0)); - acc=vpadalq_s16(acc,p); - p=vmull_s8(vget_low_s8(w1),vget_low_s8(x1)); - p=vmlal_s8(p,vget_high_s8(w1),vget_high_s8(x1)); - acc=vpadalq_s16(acc,p); - } - sum=vaddvq_s32(acc); -#endif -#elif defined(__VSX__) - /* 16 byte = 32 nibble. vec_mergeh/vec_mergel su ppc64le (GCC) interallacciano come - * unpacklo/unpackhi x86 (verificato empiricamente su POWER8): i nibble escono in - * ordine di memoria. |w|<=8 dopo il -8, quindi stesso trucco del segno di dot_i8i8. - * EN: vec_mergeh/l on ppc64le interleave like x86 unpacklo/hi (verified on POWER8), - * so nibbles come out in memory order; then the same sign trick as dot_i8i8. */ - const __vector unsigned char m4v=vec_splats((unsigned char)0x0F); - const __vector unsigned char sh4=vec_splats((unsigned char)4); - const __vector signed char b8v=vec_splats((signed char)8); - const __vector signed char vz=vec_splats((signed char)0); - __vector signed int acc=vec_splats(0); - for(;i+32<=I;i+=32){ - __vector unsigned char by=vec_xl(0,w4+(i>>1)); /* 16 byte = 32 nibble */ - __vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4); - __vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v); - __vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v); - __vector signed char x0=vec_xl(0,(const signed char*)(x+i)); - __vector signed char x1=vec_xl(0,(const signed char*)(x+i+16)); - __vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz); - acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0), - (__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc); - acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1), - (__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc); - } - sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); -#endif - for(;i+1>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } - if(i>1]; sum+=((int)(b&0xF)-8)*x[i]; } - return sum; -} -#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) -/* SMMLA (i8mm): vmmlaq_s32 vede ogni int8x16_t come matrice 2x8 row-major (byte 0-7 = - * riga 0, byte 8-15 = riga 1) e accumula C += A*B^T nel 2x2 int32: lane0=a0.b0, - * lane1=a0.b1, lane2=a1.b0, lane3=a1.b1. vcombine di due mezze-righe costruisce la - * matrice: A = due righe di peso (o,o+1), B = due righe di attivazione (s,s+1), quindi - * meta' traffico pesi e doppio lavoro per istruzione a S>=2. EN: vmmlaq_s32 treats each - * int8x16_t as a 2x8 row-major matrix and does C += A*B^T on a 2x2 int32 tile; vcombine - * of vget_low/high halves builds the 2-row register from two weight/activation rows. */ -static inline int32x4_t mm_tile16(int32x4_t acc, int8x16_t wo, int8x16_t wo1, - int8x16_t xs, int8x16_t xs1){ - acc=vmmlaq_s32(acc, vcombine_s8(vget_low_s8(wo), vget_low_s8(wo1)), - vcombine_s8(vget_low_s8(xs), vget_low_s8(xs1))); - return vmmlaq_s32(acc, vcombine_s8(vget_high_s8(wo), vget_high_s8(wo1)), - vcombine_s8(vget_high_s8(xs), vget_high_s8(xs1))); -} -static void matmul_q_idot_mm(float *y, const int8_t *xq, const float *sx, const int8_t *q, - const float *scale, int S, int I, int O){ - #pragma omp parallel for schedule(static) - for(int o=0;o<(O&~1);o+=2){ - const int8_t *wo=q+(int64_t)o*I, *wo1=q+(int64_t)(o+1)*I; - float sc0=scale[o], sc1=scale[o+1]; - for(int s=0;s<(S&~1);s+=2){ - const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I; - /* 4 accumulatori indipendenti: una sola catena vmmla e' latency-bound. - * EN: 4 independent accumulators; a single vmmla chain is latency-bound. */ - int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0; - for(;i+64<=I;i+=64){ - a0=mm_tile16(a0,vld1q_s8(wo+i), vld1q_s8(wo1+i), vld1q_s8(xs+i), vld1q_s8(xs1+i)); - a1=mm_tile16(a1,vld1q_s8(wo+i+16),vld1q_s8(wo1+i+16),vld1q_s8(xs+i+16),vld1q_s8(xs1+i+16)); - a2=mm_tile16(a2,vld1q_s8(wo+i+32),vld1q_s8(wo1+i+32),vld1q_s8(xs+i+32),vld1q_s8(xs1+i+32)); - a3=mm_tile16(a3,vld1q_s8(wo+i+48),vld1q_s8(wo1+i+48),vld1q_s8(xs+i+48),vld1q_s8(xs1+i+48)); - } - for(;i+16<=I;i+=16) - a0=mm_tile16(a0,vld1q_s8(wo+i),vld1q_s8(wo1+i),vld1q_s8(xs+i),vld1q_s8(xs1+i)); - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); - int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); - for(;i>1)), byo1=vld1q_u8(wo1+(i>>1)); - uint8x16_t cyo=vld1q_u8(wo+(i>>1)+16), cyo1=vld1q_u8(wo1+(i>>1)+16); - uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); - uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); - uint8x16x2_t ko =vzipq_u8(vandq_u8(cyo, m4q), vshrq_n_u8(cyo, 4)); - uint8x16x2_t ko1=vzipq_u8(vandq_u8(cyo1,m4q), vshrq_n_u8(cyo1,4)); - a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), - vld1q_s8(xs+i), vld1q_s8(xs1+i)); - a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), - vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); - a2=mm_tile16(a2, vsubq_s8(vreinterpretq_s8_u8(ko.val[0]),b8q), - vsubq_s8(vreinterpretq_s8_u8(ko1.val[0]),b8q), - vld1q_s8(xs+i+32), vld1q_s8(xs1+i+32)); - a3=mm_tile16(a3, vsubq_s8(vreinterpretq_s8_u8(ko.val[1]),b8q), - vsubq_s8(vreinterpretq_s8_u8(ko1.val[1]),b8q), - vld1q_s8(xs+i+48), vld1q_s8(xs1+i+48)); - } - for(;i+32<=I;i+=32){ - uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1)); - uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); - uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); - a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), - vld1q_s8(xs+i), vld1q_s8(xs1+i)); - a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), - vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), - vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); - } - int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); - int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); - int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); - for(;i+1>1], bo1=wo1[i>>1]; - int a0=(int)(bo&0xF)-8, a1=(int)(bo>>4)-8, b0=(int)(bo1&0xF)-8, b1=(int)(bo1>>4)-8; - int u0=xs[i],u1=xs[i+1],v0=xs1[i],v1=xs1[i+1]; - d00+=a0*u0+a1*u1; d01+=a0*v0+a1*v1; d10+=b0*u0+b1*u1; d11+=b0*v0+b1*v1; } - if(i>1], bo1=wo1[i>>1]; - int a0=(int)(bo&0xF)-8, b0=(int)(bo1&0xF)-8; - d00+=a0*xs[i]; d01+=a0*xs1[i]; d10+=b0*xs[i]; d11+=b0*xs1[i]; } - y[(int64_t)s*O+o] =(float)d00*sc0*sx[s]; - y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s]; - y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1]; - y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1]; - } - if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I; - y[(int64_t)s*O+o] =(float)dot_i4i8(wo, xs,I)*sc0*sx[s]; - y[(int64_t)s*O+(o+1)]=(float)dot_i4i8(wo1,xs,I)*sc1*sx[s]; } - } - if(O&1){ int o=O-1; const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o]; - #pragma omp parallel for schedule(static) - for(int s=0;s=2){ matmul_q_idot_mm(y,xq,sx,q,scale,S,I,O); return; } -#endif - #pragma omp parallel for schedule(static) - for(int o=0;o=2){ matmul_i4_idot_mm(y,xq,sx,q4,scale,S,I,O); return; } -#endif - #pragma omp parallel for schedule(static) - for(int o=0;og_qscratch.xq_cap){ - int8_t *p=realloc(g_qscratch.xq,xn); - if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); } - g_qscratch.xq=p; g_qscratch.xq_cap=xn; - } - if(sn>g_qscratch.sx_cap){ - float *p=realloc(g_qscratch.sx,sn*sizeof(float)); - if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); } - g_qscratch.sx=p; g_qscratch.sx_cap=sn; - } - *xq=g_qscratch.xq; *sx=g_qscratch.sx; -} - -/* allow_idot=0: forza il kernel int4/int8 ESATTO (attivazioni f32). Serve alle proiezioni di - * attenzione: sono sensibili alla quantizzazione int8 delle attivazioni dell'IDOT. Misurato su - * GLM-5.2 int4, 1023 token, log-lik -5040.33 (esatto) -> -5160.47 (IDOT) = +0.117 nat/token, - * ~+12% perplexity. Gli altri matmul del prefill (o_proj, kv_b, expert) tengono l'IDOT. - * EN: allow_idot=0 forces the EXACT int4/int8 kernel (f32 activations). The attention - * projections need it: IDOT's int8 activation quantization costs +0.117 nats/token there - * (~+12% perplexity), measured. Every other prefill matmul keeps IDOT as before. */ -static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot); -static void matmul_qt(float *y, const float *x, QT *w, int S){ matmul_qt_ex(y,x,w,S,1); } static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot){ #ifdef COLI_METAL - /* Large row-batches (prefill: kv_b reconstruction, o_proj, dense MLP, step_all logits) - * amortize Metal's ~5ms submit latency; small-S decode matmuls stay on CPU (NEON wins). - * Weights must be registered (all dense QT allocs are, via qalloc). */ if(g_metal_enabled && S>=g_metal_gemm_min && !spec_pinned() && (w->fmt==1||w->fmt==2) && !omp_in_parallel()){ const void *wp = w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4; if(coli_metal_gemm(y,x,wp,w->s,w->fmt,S,w->I,w->O)) return; } #endif #ifdef COLI_CUDA - /* The CUDA backend owns persistent copies only for model-resident tensors. - * Streaming expert slots are reused for different IDs and must never enter - * this cache. Nested OpenMP calls stay on CPU because each device context - * intentionally owns one synchronous scratch stream in this stage. */ if(g_cuda_enabled && w->cuda_eligible && !w->cuda_failed && !omp_in_parallel()){ const void *weights = w->fmt==0 ? (const void*)w->qf : w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4; @@ -1038,21 +434,7 @@ static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot) } #endif if(w->fmt==0){ matmul(y,x,w->qf,S,w->I,w->O); return; } - /* fmt=4: grouped int4 — always use the exact grouped kernel (no IDOT approximation, - * since the whole point of grouped scales is better quality). */ if(w->fmt==4){ matmul_i4_grouped(y,x,w->q4,w->s,S,w->I,w->O,w->gs); return; } - /* int8 IDOT vince sempre (1.4-2.5x). int4 IDOT: l'autore su AVX2 trovo' che a S=1 - * non ripaga (soglia S>=2); ma su ARM/SDOT il singolo token CONVIENE (vedi g_i4s / - * PR #9 per il gemello VNNI). Soglia configurabile con I4S. - * EN: int8 IDOT always wins (1.4-2.5x). int4 IDOT: on AVX2 the author found S=1 didn't - * pay (S>=2 gate); on ARM/SDOT single-token DOES pay (see g_i4s / PR #9 for the VNNI - * twin). Threshold configurable via I4S. */ - /* #163: sotto SPEC_PIN il gate int4-IDOT usa la decisione di S=1 per OGNI S, cosi' - * draft e verifica restano sulla stessa famiglia. (CUDA non e' toccato: la sua - * condizione non dipende da S, quindi e' gia' coerente tra draft e verifica.) - * EN: under SPEC_PIN the int4 IDOT gate uses the S=1 decision for EVERY S, so draft - * and verify stay in one family. (CUDA untouched: its condition is S-independent, - * hence already draft/verify-consistent.) */ if(allow_idot && g_idot && (w->fmt==1 || (w->fmt==2 && (spec_pinned() ? g_i4s<=1 : S>=g_i4s)))){ int I=w->I; int8_t *xq; float *sx; if(S<0 || I<0 || (size_t)S>SIZE_MAX/(size_t)(I?I:1)){ fprintf(stderr,"matmul_qt: shape overflow\n"); exit(1); } @@ -1067,49 +449,6 @@ static void matmul_qt_ex(float *y, const float *x, QT *w, int S, int allow_idot) else matmul_i4(y,x,w->q4,w->s,S,w->I,w->O); } -/* quantizza w[O,I] f32 -> int8 q[O,I] + scala[O] simmetrica per riga */ -static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits){ - int qmax=(1<<(bits-1))-1; - #pragma omp parallel for schedule(static) - for(int o=0;oamax)amax=a; } - float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; - int8_t *qr=q+(int64_t)o*I; - for(int i=0;iqmax)v=qmax; if(v<-qmax-1)v=-qmax-1; qr[i]=(int8_t)v; } - } -} -/* quantizza w[O,I] f32 -> int4 impacchettato (2/byte) + scala[O]. - * bits<=4: valori in [-qmax-1,qmax] stanno in un nibble [-8,7]; memorizzati come v+8 (0..15). */ -static void pack_int4(const float *w, uint8_t *q4, float *scale, int O, int I, int bits){ - int qmax=(1<<(bits-1))-1, rb=(I+1)/2; - #pragma omp parallel for schedule(static) - for(int o=0;oamax)amax=a; } - float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; - uint8_t *qr=q4+(int64_t)o*rb; - for(int i=0;iqmax)v0=qmax; if(v0<-8)v0=-8; - int v1=0; if(i+1qmax)v1=qmax; if(v1<-8)v1=-8; } - qr[i>>1] = (uint8_t)((v0+8) | ((v1+8)<<4)); - } - } -} - -/* quantizza w[O,I] f32 -> int2 impacchettato (4/byte) + scala[O]. valori nibble 2-bit in [-2,1]. */ -static void pack_int2(const float *w, uint8_t *q2, float *scale, int O, int I, int bits){ - int qmax=(1<<(bits-1))-1, rb=(I+3)/4; - #pragma omp parallel for schedule(static) - for(int o=0;oamax)amax=a; } - float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; - uint8_t *qr=q2+(int64_t)o*rb; - for(int i=0;iqmax)v=qmax; if(v<-2)v=-2; byte|=(uint8_t)((v+2)<<(k*2)); } - qr[i>>2]=byte; - } - } -} - static int g_nopack=0; /* NOPACK=1 -> tiene i valori <=4bit in contenitore int8 (per validare il packing) */ static int g_drop=0; /* DROP=1 -> scarta le pagine expart dopo l'uso. Default 0: le lascia in * page-cache (buff/cache, NON RSS) come L2 gratuito -> sfrutta lo @@ -1195,6 +534,11 @@ static int g_disk_split=0; /* DISK_SPLIT=1: contatori che spezzano i DISK LOAD ( * (int4), con i byte letti. Default OFF: a flag spento gli atomic * non vengono MAI toccati (zero overhead), le righe extra di stats * non vengono stampate. Solo misura: nessun effetto sull'output. */ + +#include "sample.h" +#include "kv_persist.h" +#include "telemetry.h" + /* Aligned allocator for dense QT weights/scales: under METAL, page-align + register so the * GPU reads them zero-copy (no upload duplicate). Plain malloc otherwise. */ /* ---- COLI_NUMA=1 (#82): interleave the expert slabs across NUMA nodes ---- @@ -1207,7 +551,9 @@ static int g_disk_split=0; /* DISK_SPLIT=1: contatori che spezzano i DISK LOAD ( * 10x (#82) — hence per-region mbind here and nothing else. Raw syscall, no libnuma * dependency; MPOL_MF_MOVE migrates pages of reused heap chunks too. Linux-only, * silent no-op elsewhere or on single-node hosts. */ -static int g_numa_nodes=0; +#ifdef __linux__ +static int g_numa_nodes=0; /* only touched under __linux__; off-Linux NUMA is a no-op */ +#endif static void numa_slab_bind(void *p, size_t n){ #ifdef __linux__ if(g_numa_nodes<2 || !p || !n) return; @@ -1323,7 +669,12 @@ static jval* cfg_root(const char *snap, char **arena){ char p[2048]; snprintf(p,sizeof(p),"%s/config.json",snap); FILE *f=fopen(p,"rb"); if(!f){perror(p);exit(1);} fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *b=malloc(n+1); size_t got=fread(b,1,n,f); b[got]=0; fclose(f); + /* SEC: config.json arriva dalla dir modello non fidata. Limita la dimensione + * (un file ostile enorme = OOM al load) e controlla la malloc: senza il NULL + * check, b[got]=0 su malloc fallita era un NULL-deref. */ + if(n<0 || n>(256L<<20)){ fprintf(stderr,"%s: size %ld out of range (0..256 MB)\n",p,n); exit(1); } + char *b=malloc((size_t)n+1); if(!b){ fprintf(stderr,"OOM reading %s (%ld bytes)\n",p,n); exit(1); } + size_t got=fread(b,1,(size_t)n,f); b[got]=0; fclose(f); if((long)got!=n) fprintf(stderr,"warning: short read on %s (%ld of %ld)\n",p,(long)got,n); return json_parse(b,arena); } @@ -1362,8 +713,9 @@ static void load_cfg(Cfg *c, const char *snap){ FILE *gf=fopen(gp,"rb"); /* assente = nessun problema: e' opzionale */ if(gf){ fseek(gf,0,SEEK_END); long gn=ftell(gf); fseek(gf,0,SEEK_SET); - if(gn>0){ - char *gb=malloc(gn+1); size_t gg=fread(gb,1,gn,gf); gb[gg]=0; + char *gb = (gn>0 && gn<=(256L<<20)) ? malloc((size_t)gn+1) : NULL; /* SEC: cap + NULL check */ + if(gb){ + size_t gg=fread(gb,1,(size_t)gn,gf); gb[gg]=0; char *ga=NULL; jval *gr=json_parse(gb,&ga); jval *ge=gr?json_get(gr,"eos_token_id"):NULL; if(ge){ @@ -1432,6 +784,27 @@ static int detect_group_size(int O, int I, int64_t ns){ return 0; } +/* SEC: risolve e VALIDA il formato quantizzato di un tensore [O,I] letto da un + * container non fidato (mirror). L'inferenza precedente (`?1:?2:3`) cadeva su + * int2 per QUALSIASI conteggio byte non riconosciuto: un peso troppo corto + * diventava un int2 valido e il matmul leggeva oltre il buffer (O*I nibble a + * 4/byte). Qui i byte del peso devono corrispondere a un layout noto e i byte + * della scala alla cardinalita' attesa (O per-row, O*ng per-gruppo) — altrimenti + * si termina invece di sforare. Ritorna fmt (1/2/3/4) e scrive *gs. */ +static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns, int *gs){ + int64_t exp_i8=(int64_t)O*I, exp_i4=(int64_t)O*((I+1)/2), exp_i2=(int64_t)O*((I+3)/4); + int fmt = (nb==exp_i8)?1 : (nb==exp_i4)?2 : (nb==exp_i2)?3 : 0; + if(!fmt){ + fprintf(stderr,"%s: quantized weight is %lld bytes — no int8/int4/int2 layout for [%d,%d], refusing (untrusted container)\n", + name,(long long)nb,O,I); exit(1); } + *gs=0; + if(fmt==2){ int g=detect_group_size(O,I,ns); if(g>0){ fmt=4; *gs=g; } } + int64_t exp_scale = (fmt==4)? (int64_t)O*((I+*gs-1)/(*gs)) : (int64_t)O; /* in FLOAT */ + if(ns != exp_scale*4){ + fprintf(stderr,"%s: scale array is %lld bytes — expected %lld for [%d,%d] fmt=%d, refusing (untrusted container)\n", + name,(long long)ns,(long long)(exp_scale*4),O,I,fmt); exit(1); } + return fmt; +} /* costruisce un QT [O,I] dal disco in `t` (buffer riusabili tra chiamate). * - se esiste `name.qs`: pesi GIA' quantizzati nel container (U8 qdata + F32 scala) -> letti diretti * - altrimenti: tensore pieno (f32/bf16) -> quantizzato a runtime a `bits` (oracolo tiny / pesi pieni) @@ -1441,12 +814,11 @@ static void qt_from_disk(Model *m, const char *name, int O, int I, int bits, int if(st_has(&m->S,sn)){ int64_t nb=st_nbytes(&m->S,name); int64_t ns=st_nbytes(&m->S,sn); /* scale bytes (F32) */ - /* Detect int4-grouped (fmt=4): packed int4 weight bytes BUT scale array is - * larger than O*4 — the group size is derived from the scale-array size. */ - int fmt = (nb==(int64_t)O*I)?1 : (nb==(int64_t)O*((I+1)/2))?2 : 3; + /* fmt=4 int4-grouped: byte int4 ma scala > O*4 — gs deriva dalla scala. + * qt_resolve_fmt valida entrambi i conteggi contro [O,I] e termina se + * non fidati (SEC). */ int gs=0; - if(fmt==2) gs=detect_group_size(O,I,ns); - if(gs>0) fmt=4; + int fmt = qt_resolve_fmt(name,O,I,nb,ns,&gs); if(fmt==1){ if(t->fmt!=1||!t->q8){ t->fmt=1; t->O=O; t->I=I; t->gs=0; t->q8=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q8,drop); } else if(fmt==4){ int ng=(I+gs-1)/gs; if(t->fmt!=4||!t->q4){ t->fmt=4; t->O=O; t->I=I; t->gs=gs; t->q4=qalloc(nb); t->s=falloc((int64_t)O*ng); } @@ -1522,6 +894,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits m->pin=calloc(NR,sizeof(ESlot*)); m->npin=calloc(NR,sizeof(int)); m->eusage=calloc(NR,sizeof(uint32_t*)); m->eheat=calloc(NR,sizeof(uint32_t*)); m->elast=calloc(NR,sizeof(uint32_t*)); + m->elast_dc=calloc(NR,sizeof(uint32_t*)); m->elast_pre=calloc(NR,sizeof(uint32_t*)); m->kv=calloc(1,sizeof(KVState)); m->kv_start=m->kv->kv_start=calloc(NR,sizeof(int)); for(int i=0;in_layers;i++){ @@ -1566,6 +939,8 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t)); m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t)); m->elast[i]=calloc(c->n_experts,sizeof(uint32_t)); + m->elast_dc[i]=calloc(c->n_experts,sizeof(uint32_t)); + m->elast_pre[i]=calloc(c->n_experts,sizeof(uint32_t)); } #undef P } @@ -1578,12 +953,17 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits "self_attn.q_a_proj.weight","self_attn.q_b_proj.weight","self_attn.kv_a_proj_with_mqa.weight", "self_attn.kv_b_proj.weight","self_attn.o_proj.weight","mlp.gate.weight", "mlp.shared_experts.gate_proj.weight","mlp.shared_experts.down_proj.weight", - "mlp.experts.0.gate_proj.weight","mlp.experts.255.down_proj.weight"}; + "mlp.experts.0.gate_proj.weight"}; char mn[256]; m->has_mtp=1; for(unsigned q=0;qn_layers,req[q]); if(!st_has(&m->S,mn)){ m->has_mtp=0; break; } } + /* probe the LAST expert by index, not a fixed 255: REAP-pruned + * checkpoints have n_routed_experts < 256 and the MTP set stays complete, + * so a hardcoded expert.255 would spuriously report has_mtp=0 on them. */ + snprintf(mn,sizeof(mn),"model.layers.%d.mlp.experts.%d.down_proj.weight",c->n_layers,c->n_experts-1); + if(!st_has(&m->S,mn)) m->has_mtp=0; if(getenv("MTP") && atoi(getenv("MTP"))==0) m->has_mtp=0; if(m->has_mtp){ int i=c->n_layers; Layer *l=&m->mtpL; @@ -1612,6 +992,8 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t)); m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t)); m->elast[i]=calloc(c->n_experts,sizeof(uint32_t)); + m->elast_dc[i]=calloc(c->n_experts,sizeof(uint32_t)); + m->elast_pre[i]=calloc(c->n_experts,sizeof(uint32_t)); m->kv_start[i]=-1; /* KV MTP: parte dalla prima posizione di decode */ #undef PM } @@ -1644,7 +1026,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits c->index_topk, c->index_topk); } } - m->hlast=falloc(D); m->h_all=falloc((int64_t)64*D); + m->hlast=falloc(D); m->h_all=falloc((int64_t)512*D); /* byte della parte DENSA residente (embed+lm_head+attn+mlp densa+shared+norme) */ int64_t rb=qt_bytes(&m->embed)+qt_bytes(&m->lm_head); @@ -1732,15 +1114,49 @@ static int pread_full(int fd, void *buf, int64_t n, int64_t off, const char *tag while(gotelast_pre || !m->elast_pre[layer]) return DC_WARM; /* no snapshot: label as the safe class */ + uint32_t last_pre=m->elast_pre[layer][eid]; + if(last_pre==0) return DC_COLD; /* never touched before this call: certain cold */ + uint32_t age=m->eaccess_clock_dc-last_pre; /* ticks since last access, PRE this call's bump */ + return age>g_direct_heat_ticks ? DC_COLD : DC_WARM; /* '>' not '>=': ties lean warm */ +} +static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal, int demand){ #ifdef COLI_CUDA /* A live REPIN may reuse a GPU-enabled pinned slot for a different expert. * Keep its tier assignment, but invalidate the old device weights. */ @@ -1783,11 +1199,8 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ QT *qt[3]={&s->g,&s->u,&s->d}; int OO[3]={I,I,D}, II[3]={D,D,I}; for(int k=0;k<3;k++){ int64_t nb=tw[k]->nbytes; - int fmt=(nb==(int64_t)OO[k]*II[k])?1:(nb==(int64_t)OO[k]*((II[k]+1)/2))?2:3; - /* detect grouped int4 (fmt=4): int4 weight bytes + larger scale array */ int gs=0; - if(fmt==2) gs=detect_group_size(OO[k],II[k],tq[k]->nbytes); - if(gs>0) fmt=4; + int fmt=qt_resolve_fmt(tw[k]->name,OO[k],II[k],nb,tq[k]->nbytes,&gs); qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL; qt[k]->q8=(int8_t*)((char*)bw[k]+tw[k]->off); qt[k]->q4=(uint8_t*)((char*)bw[k]+tw[k]->off); qt[k]->s=(float*)((char*)bq[k]+tq[k]->off); @@ -1870,12 +1283,22 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ numa_slab_bind(s->fslab,(size_t)ftot*sizeof(float)); #endif } + /* DISK-CLASS: classify before the reads; computed unconditionally at dc_on sites so + * the timer (dc_t0) brackets exactly the read work, matching what the GB/s in the + * DISK-CLASS line describes. dc_on gates ALL of it off demand=0 call sites + * (pilot/repin/pin -- never classified, see the call sites) and off PROF=0 runs + * (dc_needed()) -- zero cost, zero behavior change there. The fd choice below is + * NOT influenced by the verdict: this is measurement only. */ + int dc_on = demand && dc_needed(); + int dc_cls = dc_on ? expert_classify(m,layer,eid) : DC_WARM; + double dc_t0 = dc_on ? now_s() : 0; + if(dc_on) dc_wall_enter(dc_cls,dc_t0); /* busy-wall open; EVERY exit path below must pair it */ int ord[3]={0,1,2}; /* ordina per offset nel file */ for(int a=0;a<3;a++) for(int bb=a+1;bb<3;bb++) if(tw[ord[bb]]->offoff){ int t=ord[a]; ord[a]=ord[bb]; ord[bb]=t; } int contig = tw[ord[0]]->fd==tw[ord[1]]->fd && tw[ord[1]]->fd==tw[ord[2]]->fd && tw[ord[0]]->off+tw[ord[0]]->nbytes==tw[ord[1]]->off && tw[ord[1]]->off+tw[ord[1]]->nbytes==tw[ord[2]]->off; - int64_t pos[3]; int done=0; + int64_t pos[3]; int done=0, dc_direct=0; if(contig){ int64_t off0=tw[ord[0]]->off; int dfd = g_direct ? st_direct_fd(&m->S, tw[ord[0]]->fd) : -1; @@ -1885,25 +1308,41 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ ssize_t r=pread(dfd, s->slab, len, base); if(r>=need){ pos[ord[0]]=off0-base; pos[ord[1]]=pos[ord[0]]+tw[ord[0]]->nbytes; - pos[ord[2]]=pos[ord[1]]+tw[ord[1]]->nbytes; done=1; + pos[ord[2]]=pos[ord[1]]+tw[ord[1]]->nbytes; done=1; dc_direct=1; } } if(!done){ /* fallback bufferizzato */ - if(pread_full(tw[ord[0]]->fd, s->slab, wtot, off0, "pread expert")){ if(fatal) exit(1); return -1; } + if(pread_full(tw[ord[0]]->fd, s->slab, wtot, off0, "pread expert")){ if(fatal) exit(1); + if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */ + return -1; } pos[ord[0]]=0; pos[ord[1]]=tw[ord[0]]->nbytes; pos[ord[2]]=tw[ord[0]]->nbytes+tw[ord[1]]->nbytes; done=1; } } if(!done){ /* non contigui: 3 pread bufferizzate */ int64_t o=0; for(int a=0;a<3;a++){ int k=ord[a]; - if(pread_full(tw[k]->fd, s->slab+o, tw[k]->nbytes, tw[k]->off, "pread expert")){ if(fatal) exit(1); return -1; } + if(pread_full(tw[k]->fd, s->slab+o, tw[k]->nbytes, tw[k]->off, "pread expert")){ if(fatal) exit(1); + if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */ + return -1; } pos[k]=o; o+=tw[k]->nbytes; } } float *fp[3]; int64_t fo=0; /* scale (piccole) */ for(int k=0;k<3;k++){ - if(pread_full(tq[k]->fd, (char*)(s->fslab+fo), tq[k]->nbytes, tq[k]->off, "pread qs")){ if(fatal) exit(1); return -1; } + if(pread_full(tq[k]->fd, (char*)(s->fslab+fo), tq[k]->nbytes, tq[k]->off, "pread qs")){ if(fatal) exit(1); + if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */ + return -1; } fp[k]=s->fslab+fo; fo+=tq[k]->nbytes/4; } atomic_fetch_add_explicit(&g_prof_io,wtot+fo*4,memory_order_relaxed); + if(dc_on){ /* DISK-CLASS accounting, see dc_needed() */ + double dc_t1=now_s(); /* one clock read for thread-ns AND the wall exit */ + int64_t bytes=wtot+fo*4; + atomic_fetch_add_explicit(&g_dc_n[dc_cls],1,memory_order_relaxed); + atomic_fetch_add_explicit(&g_dc_bytes[dc_cls],bytes,memory_order_relaxed); + atomic_fetch_add_explicit(&g_dc_ns[dc_cls],(int64_t)((dc_t1-dc_t0)*1e9),memory_order_relaxed); + dc_wall_exit(dc_cls,dc_t1); + if(dc_direct) /* which fd ACTUALLY served this class */ + atomic_fetch_add_explicit(&g_dc_direct_n[dc_cls],1,memory_order_relaxed); + } if(g_drop){ /* scarta subito le pagine: evita che la page * cache in pressione strangoli il throughput */ posix_fadvise(tw[ord[0]]->fd, tw[ord[0]]->off, wtot, POSIX_FADV_DONTNEED); @@ -1912,10 +1351,8 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ QT *qt[3]={&s->g,&s->u,&s->d}; int OO[3]={I,I,D}, II[3]={D,D,I}; for(int k=0;k<3;k++){ int64_t nb=tw[k]->nbytes; - int fmt = (nb==(int64_t)OO[k]*II[k])?1 : (nb==(int64_t)OO[k]*((II[k]+1)/2))?2 : 3; int gs=0; - if(fmt==2) gs=detect_group_size(OO[k],II[k],tq[k]->nbytes); - if(gs>0) fmt=4; + int fmt=qt_resolve_fmt(tw[k]->name,OO[k],II[k],nb,tq[k]->nbytes,&gs); qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL; qt[k]->q8=(int8_t*)(s->slab+pos[k]); qt[k]->q4=s->slab+pos[k]; qt[k]->s=fp[k]; } @@ -1923,9 +1360,15 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal){ } /* Every expert read goes through here: time the whole load (pread/fault + * bookkeeping) on the thread that runs it, into the disk-service counter. */ -static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){ +static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal, int demand){ + /* `demand` marks a routing-driven demand-load (moe()'s PIPE/OMP miss path, where the + * pre-bump elast_pre snapshot moe() just wrote is valid) -- pass 0 from anywhere else + * (pilot speculative loads, repin, startup PIN loading): those never run through THIS + * call's own FASE A, so the snapshot either doesn't apply or was never written for + * them, and DISK-CLASS deliberately leaves them unclassified -- see expert_classify()'s + * call site. */ double t0=now_s(); - int rc=expert_load_impl(m,layer,eid,s,fatal); + int rc=expert_load_impl(m,layer,eid,s,fatal,demand); atomic_fetch_add_explicit(&g_edisk_ns,(int64_t)((now_s()-t0)*1e9),memory_order_relaxed); return rc; } @@ -2192,7 +1635,7 @@ static void *pipe_worker(void *arg){ memory_order_acq_rel,memory_order_relaxed)){ int L =atomic_load_explicit(&p->layer,memory_order_relaxed); int eid=atomic_load_explicit(&p->eids[i],memory_order_relaxed); /* AFTER winning CAS */ - expert_load(p->m,L,eid,&p->m->ws[i],1); /* needed-now load: fatal on I/O error (matches serial path) */ + expert_load(p->m,L,eid,&p->m->ws[i],1,1); /* needed-now load: fatal on I/O error (matches serial path); demand=1: this IS moe()'s own miss path */ atomic_store_explicit(&p->ready[i],1,memory_order_release); if(g_pipe_block){ /* wake a main thread parked in pipe_wait */ pthread_mutex_lock(&p->mx); @@ -2287,7 +1730,7 @@ static void expert_host_release(Model *m, ESlot *s){ m->resident_bytes-=bytes; if(m->resident_bytes<0) m->resident_bytes=0; } static void expert_host_ensure(Model *m, int layer, ESlot *s){ - if(!s->slab) expert_load(m,layer,s->eid,s,1); + if(!s->slab) expert_load(m,layer,s->eid,s,1,0); /* re-materializing a GPU-resident expert's host copy, not a routing miss: demand=0 */ } #endif @@ -2307,6 +1750,18 @@ static void expert_prefetch(Model *m, int layer, int eid){ static void qt_addrow(const QT *t, int row, float coef, float *acc){ int I=t->I; if(t->fmt==0){ const float *w=t->qf+(int64_t)row*I; for(int i=0;ifmt==4){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); + int gs=t->gs, ng=(I+gs-1)/gs; const float *scl=t->s+(int64_t)row*ng; + for(int i=0;i+1>1]; + acc[i] +=coef*scl[i/gs] *((int)(b&0xF)-8); + acc[i+1]+=coef*scl[(i+1)/gs]*((int)(b>>4)-8); } + if(I&1){ uint8_t b=w[I>>1]; acc[I-1]+=coef*scl[(I-1)/gs]*((int)(b&0xF)-8); } return; } float c=coef*t->s[row]; if(t->fmt==1){ const int8_t *w=t->q8+(int64_t)row*I; for(int i=0;ifmt==2){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); @@ -2325,6 +1780,13 @@ static void qt_matvec_rows(const QT *t, int r0, int n, const float *x, float *y) else if(t->fmt==2){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); float s=t->s[row]; float acc=0; for(int i=0;i+1>1]; acc+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } if(I&1){ uint8_t b=w[I>>1]; acc+=((int)(b&0xF)-8)*x[I-1]; } a=acc*s; } + else if(t->fmt==4){ /* per-gruppo, come matmul_i4_grouped / per-group, as matmul_i4_grouped */ + const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); + int gs=t->gs, ng=(I+gs-1)/gs; const float *scl=t->s+(int64_t)row*ng; + for(int g=0; g*gsI?I:base+gs; float acc=0; + for(int i=base;i>1]; + acc+=(float)((i&1)?((int)(b>>4)-8):((int)(b&0xF)-8))*x[i]; } + a+=(double)acc*scl[g]; } } else { const uint8_t *w=t->q4+(int64_t)row*((I+3)/4); float s=t->s[row]; float acc=0; for(int i=0;i>2]; acc+=((int)((b>>((i&3)*2))&3)-2)*x[i]; } a=acc*s; } y[j]=(float)a; @@ -2338,6 +1800,43 @@ static int g_dsa_force=0; /* DSA_FORCE=1: selezione sempre attiva (test: top-min static int cmp_fdesc(const void *a,const void *b){ float x=*(const float*)a, y=*(const float*)b; return xy?-1:0; } +/* PARTIAL SELECT (quickselect, Hoare partition, DESCending). After this call the k + * LARGEST elements of a[0..n) are in a[0..k) in unspecified order; the (k+1)-th and + * beyond are untouched-or-smaller. O(n) average, O(n^2) pathological (mitigated by + * median-of-three below) — and unlike a full qsort it never orders more than needed. + * + * Why this exists (#356): the DSA top-keep in attention_rows previously full-qsorted + * all nk context scores (O(nk log nk)) per layer per token just to read ONE value -- + * the keep-th largest (the threshold). quickselect finds that pivot in O(nk) average, + * and the position-order scans that build dst[] are unchanged, so the kept set is + * bit-identical. Mirrors the sampling-side fix in #335 (heap partial-select there). + * + * NOT a stable partition: callers must derive the threshold and then re-scan the + * ORIGINAL array (the DSA code does exactly this) rather than reading a[0..k). */ +static void partial_select_desc(float *a, int n, int k){ + if(k<=0) return; + if(k>=n) return; /* nothing to partition: all kept */ + int lo=0, hi=n-1; + while(lo>1); + if(a[mid]>a[lo]){ float t=a[lo]; a[lo]=a[mid]; a[mid]=t; } + if(a[hi]>a[lo]){ float t=a[lo]; a[lo]=a[hi]; a[hi]=t; } + if(a[mid]>a[hi]){ float t=a[hi]; a[hi]=a[mid]; a[mid]=t; } + float piv=a[hi]; + int i=lo, j=hi; + for(;;){ + while(a[i]>piv) i++; /* desc: large values go left */ + while(j>lo && a[j]=j) break; + float t=a[i]; a[i]=a[j]; a[j]=t; i++; if(i>j) break; j--; + } + /* partition point: a[lo..i) are all >= piv, a[i..hi] are all <= piv */ + if(k<=i-1) hi=i-1; /* the k-th largest is in the left partition */ + else lo=i; /* it's in the right partition */ + } +} + /* attenzione MLA con KV-cache compressa, su token nuovi x[S,hidden], pos_base = pos del primo */ /* kvs/pos describe a ragged decode batch: each row may belong to a different * sequence. NULL keeps the original contiguous, currently-bound KV path. */ @@ -2620,10 +2119,14 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p } isc[t]=a*wsc; } - /* top-keep: soglia via qsort desc, poi scan in ordine di posizione */ + /* top-keep: threshold via PARTIAL SELECT (#356), poi scan in ordine di posizione. + * Era un qsort completo su nk (O(nk log nk)); quickselect estrae solo il + * keep-esimo valore piu' grande in O(nk) medio. La soglia (= min del blocco + * dei keep maggiori) e' identica a tmp[keep-1] del vecchio qsort, quindi i + * due scan qui sotto costruiscono dst[] bit-identical. */ float *tmp=falloc(nk); memcpy(tmp,isc,nk*sizeof(float)); - qsort(tmp,nk,sizeof(float),cmp_fdesc); - float thr=tmp[keep-1]; + partial_select_desc(tmp,nk,keep); + float thr=tmp[0]; for(int t=1;tdsa_sel+(int64_t)s*dtopk, nd=0; for(int t=0;tthr) dst[nd++]=t; for(int t=0;tn_kv_b_shard>1){ + if(kvs&&g_cuda_enabled&&getenv("COLI_CUDA_ATTN")&&atoi(getenv("COLI_CUDA_ATTN"))&& + !dnsel&&l->kv_b.cuda_eligible&&l->o.cuda_eligible&& + qt_cuda_upload(&l->kv_b)&&qt_cuda_upload(&l->o)){ + const float **rl=malloc((size_t)S*sizeof(*rl)),**rr=malloc((size_t)S*sizeof(*rr)); + const void **rk=malloc((size_t)S*sizeof(*rk)); + int *rn=malloc((size_t)S*sizeof(*rn)); int mt=0; + if(rk&&rl&&rr&&rn){ + for(int s=0;skv_start[layer]; rn[s]=pos+1-st0; + rk[s]=kvs[s]; + rl[s]=coli_kv_row(kvs[s]->Lc[layer],st0,kvl); + rr[s]=coli_kv_row(kvs[s]->Rc[layer],st0,c->qk_rope); + if(rn[s]>mt)mt=rn[s]; + } + cuda_core=cuda_projected=coli_cuda_attention_project_ragged(l->kv_b.cuda,l->o.cuda, + out,Q,rk,rl,rr,rn,S,H,c->qk_nope,c->qk_rope,vh,kvl,mt,c->attn_scale); + } + free(rk);free(rl);free(rr);free(rn); + } else if(cuda_absorb&&l->n_kv_b_shard>1){ int n=l->n_kv_b_shard,st0=m->kv_start[layer],nt=pos_base+S-st0,ok=1; float *qs=falloc((int64_t)S*H*qh),*cs=falloc((int64_t)S*H*vh); for(int d=0;dc; int D=c->hidden, E=c->n_experts, K=c->topk, I=c->moe_inter; + /* DISK-CLASS: does THIS call need the pre-bump recency snapshot? Must agree with + * dc_needed() in expert_load_impl -- that's what reads what this writes. touched[] + * makes the write once-per-call: an expert routed by more than one position in a big + * batch (prefill's S) must snapshot the state from BEFORE this call started, not from + * an earlier position's bump within the SAME call (which would reintroduce the + * same-call contamination one position later). Unconditional VLA like the FASE B + * `seen[E]` below -- E is small, cost is noise. */ + int need_classify = dc_needed(); + unsigned char touched[E]; if(need_classify) memset(touched,0,(size_t)E); float *choice=falloc(E); int sI=c->moe_inter*c->n_shared; /* Rank buffer for CACHE_ROUTE max-rank selection (up to all E experts). */ @@ -2830,6 +2360,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int if(!rank_buf||!rank_w){ free(rank_buf); free(rank_w); rank_buf=NULL; rank_w=NULL; do_cache_route=0; } } /* ---- FASE A: routing di tutte le S posizioni ---- */ + double route_t0=g_prof?now_s():0; int *idxs=malloc((size_t)S*K*sizeof(int)); float *ws=malloc((size_t)S*K*sizeof(float)); int *keff=malloc(S*sizeof(int)); /* router in UN matmul batch: stessa matematica, via le S chiamate S=1 */ @@ -2845,7 +2376,21 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int for(int kk=0;kkeusage[layer][idxs[(int64_t)s*K+kk]]++; ehit_mark(m,layer,idxs[(int64_t)s*K+kk]); + if(need_classify){ /* DISK-CLASS private recency -- snapshot BEFORE this call's own bump, + * then tick. This path also bumps the REAL elast/eaccess_clock a few + * lines below (#417/cfcc742 fixed the once-missing bump on Metal + * decode) -- DISK-CLASS's own clock stays independent regardless of + * that fix, see elast_dc in Model. */ + int e=idxs[(int64_t)s*K+kk]; + if(!touched[e]){ m->elast_pre[layer][e]=m->elast_dc[layer][e]; touched[e]=1; } + m->elast_dc[layer][e]=++m->eaccess_clock_dc; + } if(m->eheat[layer][idxs[(int64_t)s*K+kk]]eheat[layer][idxs[(int64_t)s*K+kk]]++; + /* #417: la scorciatoia GPU-prerouted deve far avanzare l'orologio di recency + * come il percorso router completo (riga ~3055), altrimenti elast/eaccess_clock + * si congelano a fine prefill e il tie-breaker LFRU di REPIN gira su punteggi + * stantii durante il decode su Metal. */ + m->elast[layer][idxs[(int64_t)s*K+kk]]=++m->eaccess_clock; } for(int d=0;deusage[layer][idx[kk]]++; ehit_mark(m,layer,idx[kk]); + if(need_classify){ /* DISK-CLASS private recency -- snapshot BEFORE this call's own bump, + * then tick (same rate as the real clock below: one per (s,kk)) */ + if(!touched[idx[kk]]){ m->elast_pre[layer][idx[kk]]=m->elast_dc[layer][idx[kk]]; touched[idx[kk]]=1; } + m->elast_dc[layer][idx[kk]]=++m->eaccess_clock_dc; + } if(m->eheat[layer][idx[kk]]eheat[layer][idx[kk]]++; m->elast[layer][idx[kk]]=++m->eaccess_clock; } @@ -2979,6 +2529,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int for(int d=0;dt_route+=now_s()-route_t0; if(g_route_fp) g_route_call++; if(g_couple && cp_pred && S<=8) for(int s2=0;s2pin[layer]; - for(int z=0;znpin[layer];z++) if(P[z].eid==eid){ m->hits++; use[j]=&P[z]; break; } + for(int z=0;znpin[layer];z++) if(P[z].eid==eid){ m->hits++; m->hit_pin++; use[j]=&P[z]; break; } if(!use[j]){ ESlot *Sl=m->ecache[layer]; int nn=m->ecn[layer]; - for(int z=0;zhits++; Sl[z].used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); use[j]=&Sl[z]; break; } } + for(int z=0;zhits++; m->hit_ecache++; Sl[z].used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); use[j]=&Sl[z]; break; } } if(!use[j]){ qof[j]=nmiss; use[j]=&m->ws[nmiss]; missk[nmiss++]=j; m->miss++; if(g_disk_split){ if(m->ld_ctx==1) m->miss_draft++; else if(m->ld_ctx==2) m->miss_absorb++; } } } @@ -3165,7 +2716,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int * are timed as service inside expert_load */ } else { double t0=now_s(); /* ORIGINALE: blocking parallel load */ #pragma omp parallel for schedule(dynamic,1) - for(int q=0;qws[q],1); + for(int q=0;qws[q],1,1); /* demand=1: this IS the miss path */ m->t_ewait += now_s()-t0; } /* compute thread blocked for the whole load */ } /* I/O ASINCRONO: readahead (WILLNEED) del blocco SUCCESSIVO mentre calcoliamo @@ -3245,7 +2796,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int coli_cuda_expert_mlp(e->g.cuda,e->u.cuda,e->d.cuda,hh,xg,nr)){ for(int r=0;rt_emm+=now_s()-t0; continue; + double dt=now_s()-t0;m->t_emm+=dt;if(g_prof)m->t_egpu+=dt;continue; } if(!e->slab) expert_host_ensure(m,layer,e); #endif @@ -3254,7 +2805,9 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int matmul_qt(hh, gg, &e->d, nr); for(int r=0;rt_emm += now_s()-t0; + double dt=now_s()-t0;m->t_emm+=dt;if(g_prof){m->t_ecpu+=dt; + m->cpu_expert_bytes+=qt_bytes(&e->g)+qt_bytes(&e->u)+qt_bytes(&e->d); + m->cpu_expert_rows+=(uint64_t)nr;} } #ifdef COLI_CUDA ColiCudaTensor *dev_g[COLI_CUDA_MAX_DEVICES][64],*dev_u[COLI_CUDA_MAX_DEVICES][64]; @@ -3262,6 +2815,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int int dev_rows[COLI_CUDA_MAX_DEVICES][64],dev_which[COLI_CUDA_MAX_DEVICES][64]; int dev_nc[COLI_CUDA_MAX_DEVICES]={0},dev_total[COLI_CUDA_MAX_DEVICES]={0}; int dev_off[COLI_CUDA_MAX_DEVICES]={0},dev_ok[COLI_CUDA_MAX_DEVICES]={0}; + double dev_time[COLI_CUDA_MAX_DEVICES]={0}; for(int di=0;dig.cuda_device==g_cuda_devices[di]) dev_total[di]+=group_n[q]; for(int di=1;di1) schedule(static) - for(int di=0;dig.cuda,e->u.cuda,e->d.cuda,hh,xg,nr)){ expert_host_ensure(m,layer,e); expert_gate_up(gg,uu,xg,&e->g,&e->u,nr); for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z]; matmul_qt(hh,gg,&e->d,nr); + if(g_prof){m->cpu_expert_bytes+=qt_bytes(&e->g)+qt_bytes(&e->u)+qt_bytes(&e->d); + m->cpu_expert_rows+=(uint64_t)nr;} } + if(g_prof)m->t_ecpu+=now_s()-tc; } float *src=dev_ok[di]?group_y+(int64_t)off*D:hh; for(int r=0;rmx)mx=dev_time[di];m->t_egpu+=mx;} m->t_emm+=now_s()-tg; #endif /* No drain barrier: the per-expert pipe_wait(qof[j]) above (issued for every @@ -3460,7 +3022,7 @@ static void pilot_realload(Model *m, int layer, int eid){ g_pilot_inflight[layer]++; pthread_mutex_unlock(&g_pilot_mx); - int rc=expert_load(m,layer,eid,dst,0); /* pread VERO — fuori dal lock, sovrapposto al compute; fatal=0: un errore su una speculazione NON deve uccidere il server */ + int rc=expert_load(m,layer,eid,dst,0,0); /* pread VERO — fuori dal lock, sovrapposto al compute; fatal=0: un errore su una speculazione NON deve uccidere il server; demand=0: speculative, never classified */ pthread_mutex_lock(&g_pilot_mx); if(rc==0){ @@ -3907,7 +3469,11 @@ static void layers_forward_rows(Model *m, float *x, int S, int pos_base, float *dst=coli_cuda_pipe_scratch(dev,15,xb); if(dst){ if(x_dev_on<0) ok=coli_cuda_pipe_upload(dev,dst,x,xb); - else if(x_dev_on!=dev){ ok=coli_cuda_pipe_peer_copy(dev,dst,x_dev_on,x_dev,xb); } + else if(x_dev_on!=dev){ + double tp=g_prof?now_s():0; + ok=coli_cuda_pipe_peer_copy(dev,dst,x_dev_on,x_dev,xb); + if(g_prof){m->t_p2p+=now_s()-tp;m->n_p2p++;} + } else dst=x_dev; if(ok){ x_dev=dst; x_dev_on=dev; @@ -4003,7 +3569,7 @@ static float *step_all(Model *m, const int *ids, int S, int pos_base){ float *x=falloc((int64_t)S*D); for(int s=0;sh_all) memcpy(m->h_all, x, (int64_t)S*D*sizeof(float)); /* hidden di TUTTE le pos (S<=64) */ + if(m->h_all) memcpy(m->h_all, x, (int64_t)S*D*sizeof(float)); /* hidden di TUTTE le pos (S<=512) */ if(m->hlast) memcpy(m->hlast, x+(int64_t)(S-1)*D, D*sizeof(float)); float *lo=falloc((int64_t)S*c->vocab), *row=falloc(D); for(int s=0;sfinal_norm, D, c->eps); @@ -4016,8 +3582,8 @@ static float *step_all(Model *m, const int *ids, int S, int pos_base){ static float *step_decode_batch(Model *m, const DecodeRow *rows, int S){ Cfg *c=&m->c; int D=c->hidden; /* Ragged KV currently uses MLA absorption; the stack kernel is sized to 512. */ - if(!rows || S<1 || S>64 || c->kv_lora>512) return NULL; - KVState *kvs[64]; int positions[64]; + if(!rows || S<1 || S>512 || c->kv_lora>512) return NULL; + KVState *kvs[512]; int positions[512]; float *x=falloc((int64_t)S*D); for(int s=0;sLc || !rows[s].kv->Rc || !rows[s].kv->kv_start || @@ -4132,9 +3698,9 @@ static void mtp_absorb(Model *m, const int *next_ids, const float *x, int S, int free(hx); free(cat); free(e); free(hn); free(hf); free(nrm); free(tmp); } -static inline int argmax_v(const float *lo, int V){ - int b=0; float bv=lo[0]; for(int i=1;ibv){bv=lo[i];b=i;} return b; -} + +static void repin_pass_limit(Model *m,int limit); +static void repin_pass(Model *m){ repin_pass_limit(m,16); } /* ---- METODO F: draft grammaticale (#48) ---- * gr_feed consuma i byte di ogni token EMESSO e tiene il walker in sync con l'output; @@ -4216,76 +3782,6 @@ static int grammar_draft(int *draft, int cap){ * Rejection sampling di Leviathan: accetta il draft x_d con prob p(x_d); al rifiuto * ricampiona da p con x_d azzerato e rinormalizzato. La distribuzione risultante e' * ESATTAMENTE p: la speculazione resta invisibile all'output anche col sampling. */ -static uint64_t g_rng=0x9E3779B97F4A7C15ULL; -static inline double rndu(void){ g_rng^=g_rng<<13; g_rng^=g_rng>>7; g_rng^=g_rng<<17; - return (double)(g_rng>>11)*(1.0/9007199254740992.0); } -static float *g_pbuf=NULL; static int *g_pidx=NULL; /* buffer riusati (decode single-thread) */ -static int cmp_pdesc(const void *a,const void *b){ - float pa=g_pbuf[*(const int*)a], pb=g_pbuf[*(const int*)b]; - return papb ? -1 : 0; } -/* costruisce in g_pbuf la distribuzione target: softmax(lo/temp) troncata a top-p g_nuc */ -static void dist_build(const float *lo, int V){ - if(!g_pbuf){ g_pbuf=falloc(V); g_pidx=malloc(V*sizeof(int)); } - float mx=lo[0]; for(int i=1;imx) mx=lo[i]; - double s=0; float invt=1.f/(g_temp>1e-4f?g_temp:1e-4f); - for(int i=0;i0 && g_nuc<1.f){ - for(int i=0;i=g_nuc){ keep=i+1; break; } } - double s2=0; for(int i=keep;i=0 -> quel token e' escluso (rinormalizzando al volo) */ -static int dist_sample(int V, int ban){ - double z = 1.0 - (ban>=0 ? g_pbuf[ban] : 0.0); if(z<=1e-12) z=1e-12; - double u = rndu()*z, cum=0; - for(int i=0;i=u) return i; } - for(int i=V-1;i>=0;i--) if(i!=ban && g_pbuf[i]>0) return i; - return 0; -} -/* prossimo token dai logits: greedy se g_temp<=0, altrimenti sampling. - * ban = token escluso perche' rifiutato dalla verifica speculativa precedente. */ -static int pick_tok(const float *lo, int V, int ban){ - if(g_temp<=0) return argmax_v(lo,V); - dist_build(lo,V); - return dist_sample(V,ban); -} - -/* stop-set attivo (popolato da run_text/run_serve dal config; vuoto in validazione, - * dove si genera un numero fisso di token da confrontare con l'oracolo) */ -static int g_stop[64], g_nstop=0; /* config eos + ogni added-token "special" del tokenizer */ -static void repin_pass_limit(Model *m,int limit); -static void repin_pass(Model *m){ repin_pass_limit(m,16); } -static inline int is_stop(int t){ for(int i=0;i solo gli stop del config (validazione/oracolo, dove il tokenizer non serve). */ -static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){ - g_nstop=0; - for(int i=0;in_stop && g_nstop<64;i++) g_stop[g_nstop++]=c->stop_ids[i]; - if(tok_eos>=0 && !is_stop(tok_eos) && g_nstop<64) g_stop[g_nstop++]=tok_eos; - int nsp=0; - /* DIFESA IN PROFONDITA' (woolcoxm, #298): il tokenizer marca "special":true i token di - * CONTROLLO -- <|user|>, <|assistant|>, <|observation|>, , [gMASK], i marker - * image/video/audio. Nessuno di questi e' contenuto legittimo di una risposta: se il - * modello ne emette uno, il turno e' finito (infatti GLM ne elenca tre fra gli eos - * ufficiali). Senza questo, uno di quei token non elencato nel config veniva - * DETOKENIZZATO E STAMPATO IN CHAT come testo, e la generazione proseguiva oltre la - * fine reale -- l'"added stuff on the end" riportato su un checkpoint convertito. - * Fidarsi del config di pesi convertiti da terzi e' precisamente cio' che non - * possiamo controllare; il flag del tokenizer lo possiamo leggere. - * NB: // hanno "special":false e restano contenuto vero. */ - if(T) for(int id=0; idn_ids && g_nstop<64; id++) - if(T->id_special[id] && !is_stop(id)){ g_stop[g_nstop++]=id; nsp++; } - fprintf(stderr,"[stop] %d stop tokens:",g_nstop); - for(int i=0;imx){mx=lo[i];best=i;} } - double se=0; for(int i=0;i .. " (T=ctxlen+contlen) * output: riga " " per richiesta. @@ -4522,6 +4006,11 @@ static void profile_print(Model *m, double elapsed){ edisk_s(),m->t_ewait,m->t_emm,m->t_attn,m->t_kvb,m->t_head,elapsed-accounted); printf("ATTENTION: projection/RoPE %.3fs | score-softmax-value %.3fs | output projection %.3fs\n", m->t_aproj,m->t_acore,m->t_aout); + if(g_prof)printf("P0-EXEC: routed CPU %.3fs / %.2f GB/s (%llu row) | routed GPU critical %.3fs | router %.3fs | residual P2P %.3fs / %llu hop | orchestration %.3fs\n", + m->t_ecpu,m->t_ecpu>0?m->cpu_expert_bytes/1e9/m->t_ecpu:0.0, + (unsigned long long)m->cpu_expert_rows,m->t_egpu,m->t_route,m->t_p2p,(unsigned long long)m->n_p2p, + elapsed-m->t_ewait-m->t_emm-m->t_attn-m->t_head-m->t_route-m->t_p2p>0? + elapsed-m->t_ewait-m->t_emm-m->t_attn-m->t_head-m->t_route-m->t_p2p:0); #ifdef COLI_METAL if(g_metal_enabled){ uint64_t ok=0,fb=0,ex=0; double su=0,gp=0,sc=0; coli_metal_moe_counts(&ok,&fb,&ex); coli_metal_moe_times(&su,&gp,&sc); @@ -4535,6 +4024,8 @@ static void profile_print(Model *m, double elapsed){ static void profile_reset(Model *m){ m->t_ewait=m->t_emm=m->t_attn=m->t_kvb=m->t_head=0; + m->t_ecpu=m->t_egpu=m->t_route=m->t_p2p=0;m->n_p2p=0; + m->cpu_expert_bytes=0;m->cpu_expert_rows=0; m->t_aproj=m->t_acore=m->t_aout=0; atomic_store_explicit(&g_edisk_ns,0,memory_order_relaxed); } @@ -4571,19 +4062,67 @@ static void prof_report(Model *m, const ProfBase *b, double elapsed, int tokens, for(int i=0;i<=c->n_layers;i++){ if(m->npin)pinned+=m->npin[i]; if(m->ecn)lru+=m->ecn[i]; } double io_w=m->t_ewait-b->ewait; /* stall the compute thread felt */ double io_svc=edisk_s()-b->edisk; /* read service on the loading threads (overlaps compute) */ + uint64_t dhp=m->hit_pin-b->hit_pin, dhe=m->hit_ecache-b->hit_ecache; /* split #336 */ fprintf(f,"[PROF] expert I/O: %.3f GB fetched (%.1f MB/token, %.2f GB/s over the run%s) | " - "hit %.1f%% (%llu hit / %llu load) | %.1f loads/token | %.1fs read service / %.1fs felt wait\n", + "hit %.1f%% (%llu pin + %llu lru / %llu load) | %.1f loads/token | %.1fs read service / %.1fs felt wait\n", io/1e9, tokens>0?io/1e6/tokens:0.0, io/1e9/elapsed, g_mmap?"; COLI_MMAP=1: page cache may serve part":"", - hitp,(unsigned long long)dh,(unsigned long long)dm, tokens>0?(double)dq/tokens:0.0, + hitp,(unsigned long long)dhp,(unsigned long long)dhe,(unsigned long long)dm, tokens>0?(double)dq/tokens:0.0, io_svc,io_w); + /* DISK-CLASS: per-load cold/warm classification vs. which fd ACTUALLY served it. + * Three per-class rates, labeled to keep the units unambiguous (ambiguous units + * mislead -- measured lesson): GB/s-thread = bytes / thread-seconds (per-read + * service rate, same convention as the read-service line above); GB/s-wall = + * bytes / busy-wall (aggregate rate the disk actually delivered while >=1 load of + * the class was in flight); avg-conc = thread-seconds / busy-wall (mean overlap + * depth). disk-busy = combined busy-wall (either class in flight) as a share of + * the profile window. */ + { + uint64_t dcn[2], dcdn[2]; int64_t dcbytes[2], dcns[2], dcwall[2], wnow[2], dcwall_all, wall_now; + dc_wall_read(wnow,&wall_now); + for(int i=0;i<2;i++){ + dcn[i]=atomic_load_explicit(&g_dc_n[i],memory_order_relaxed)-b->dc_n[i]; + dcbytes[i]=atomic_load_explicit(&g_dc_bytes[i],memory_order_relaxed)-b->dc_bytes[i]; + dcns[i]=atomic_load_explicit(&g_dc_ns[i],memory_order_relaxed)-b->dc_ns[i]; + dcdn[i]=atomic_load_explicit(&g_dc_direct_n[i],memory_order_relaxed)-b->dc_direct_n[i]; + dcwall[i]=wnow[i]-b->dc_wall_ns[i]; + } + dcwall_all=wall_now-b->dc_wall_all_ns; + if(dcn[DC_COLD]+dcn[DC_WARM]){ + double ct=dcns[DC_COLD]>0?(double)dcbytes[DC_COLD]/dcns[DC_COLD]:0.0; /* bytes/ns = GB/s */ + double wt=dcns[DC_WARM]>0?(double)dcbytes[DC_WARM]/dcns[DC_WARM]:0.0; + double cw=dcwall[DC_COLD]>0?(double)dcbytes[DC_COLD]/dcwall[DC_COLD]:0.0; + double ww=dcwall[DC_WARM]>0?(double)dcbytes[DC_WARM]/dcwall[DC_WARM]:0.0; + double cc=dcwall[DC_COLD]>0?(double)dcns[DC_COLD]/dcwall[DC_COLD]:0.0; + double wc=dcwall[DC_WARM]>0?(double)dcns[DC_WARM]/dcwall[DC_WARM]:0.0; + fprintf(f,"[PROF] DISK-CLASS (recency split): cold %llu x %.2f GB @ %.2f GB/s-thread, wall %.1fs @ %.2f GB/s-wall, avg-conc %.1f (direct %llu/%llu) | " + "warm %llu x %.2f GB @ %.2f GB/s-thread, wall %.1fs @ %.2f GB/s-wall, avg-conc %.1f (direct %llu/%llu) | " + "disk-busy %.1fs (%.0f%% of window)\n", + (unsigned long long)dcn[DC_COLD],dcbytes[DC_COLD]/1e9,ct,dcwall[DC_COLD]/1e9,cw,cc, + (unsigned long long)dcdn[DC_COLD],(unsigned long long)dcn[DC_COLD], + (unsigned long long)dcn[DC_WARM],dcbytes[DC_WARM]/1e9,wt,dcwall[DC_WARM]/1e9,ww,wc, + (unsigned long long)dcdn[DC_WARM],(unsigned long long)dcn[DC_WARM], + dcwall_all/1e9,100.0*(dcwall_all/1e9)/elapsed); + } + } fprintf(f,"[PROF] resident experts: %d pinned (%.1f GB) + %d in LRU (%.1f GB, cap %d/layer)\n", pinned,pinned*eb/1e9,lru,lru*eb/1e9,m->ecap); - double emm=m->t_emm-b->emm, attn=m->t_attn-b->attn, head=m->t_head-b->head; - double other=elapsed-io_w-emm-attn-head; if(other<0) other=0; + double emm=m->t_emm-b->emm, ecpu=m->t_ecpu-b->ecpu, egpu=m->t_egpu-b->egpu; + double route=m->t_route-b->route,p2p=m->t_p2p-b->p2p; + uint64_t np2p=m->n_p2p-b->n_p2p; + int64_t cpu_bytes=m->cpu_expert_bytes-b->cpu_bytes; + uint64_t cpu_rows=m->cpu_expert_rows-b->cpu_rows; + double attn=m->t_attn-b->attn, head=m->t_head-b->head; + double other=elapsed-io_w-emm-attn-head-route-p2p; if(other<0) other=0; double f_io=io_w/elapsed, f_emm=emm/elapsed, f_attn=attn/elapsed; fprintf(f,"[PROF] time shares: expert-I/O %.0f%% | expert-matmul %.0f%% | attention %.0f%% | lm_head %.0f%% | other %.0f%%\n", 100*f_io,100*f_emm,100*f_attn,100*head/elapsed,100*other/elapsed); + double slow=ecpu>egpu?ecpu:egpu,fast=ecpu0?cpu_bytes/1e9/ecpu:0.0,(unsigned long long)cpu_rows, + egpu,fast>1e-9?slow/fast:0.0,route,p2p,(unsigned long long)np2p, + np2p?p2p*1e3/np2p:0.0,other); if(f_io>=0.30){ fprintf(f,"[PROF] verdict: I/O-bound — %.0f%% of the time waits on expert reads (hit %.0f%%).",100*f_io,hitp); if(hitp<90) fprintf(f," More cache is the lever: raise RAM_GB (or add RAM)."); @@ -4610,7 +4149,7 @@ static void run_replay(Model *m, const int *full, int nfull, int np){ if(np<2||nfull<=np){ fprintf(stderr,"REPLAY requires a non-empty prompt and continuation\n"); return; } kv_alloc(m,nfull+2); float *logit=step(m,full,np-1,0); free(logit); - m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; + m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; m->hit_pin=m->hit_ecache=0; profile_reset(m); ProfBase pb; prof_base(m,&pb); double t0=now_s(); int steps=0; @@ -4660,7 +4199,7 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ } prefill_t=now_s()-prefill_t; printf("PROFILO PREFILL (%.2fs):\n",prefill_t); profile_print(m,prefill_t); - m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; + m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; m->hit_pin=m->hit_ecache=0; m->n_emit=m->n_fw=0; g_last_repin=0; profile_reset(m); @@ -4673,8 +4212,9 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ double tot=m->hits+m->miss; int nsp=0; for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; printf("\n---\nprefill %d tokens in %.2fs | decode %d tokens in %.2fs (%.2f tok/s) | " - "expert hit rate %.1f%% | RSS %.2f GB", - np,prefill_t,produced,dt,produced/dt,tot?100.0*m->hits/tot:0.0,rss_gb()); + "expert hit rate %.1f%% (pin %.1f%% + lru %.1f%%) | RSS %.2f GB", /* split #336: quale tier serve gli hit */ + np,prefill_t,produced,dt,produced/dt,tot?100.0*m->hits/tot:0.0, + tot?100.0*m->hit_pin/tot:0.0, tot?100.0*m->hit_ecache/tot:0.0, rss_gb()); if(g_cache_route && m->route_slots) printf(" | swap %.1f%% (%llu/%llu)", 100.0*m->route_swaps/m->route_slots, @@ -4783,7 +4323,73 @@ static int repin_pick(Model *m, RepinCand *out, int maxc){ } return nb; } +/* ---- RSS GUARD (#403) ----------------------------------------------------- + * La proiezione di cap_for_ram e' una STIMA: sul GB10 (#403) le generazioni + * lunghe l'hanno sforata di ~40 GB (proiettato 74.4, reale 115.6 -> 3 kill del + * kernel). La run D dell'issue prova che un cap piu' basso CONTIENE la crescita: + * questa guardia lo fa da sola, sull'RSS MISURATO invece che sul proiettato. + * Al safe point (stessa sede di repin: nessun moe in volo), ogni ~16 token + * emessi: se l'RSS supera il budget, svuota gli slot LRU meno usati e abbassa + * ecap perche' non ricrescano. Gli slab sono >128KB (mmap'd da glibc): la free + * restituisce le pagine al kernel subito, quindi l'RSS scende davvero. + * Lo slot NON viene compattato via: resta al suo posto con eid=-1/used=0 (primo + * candidato al riuso), perche' con PILOT_REAL il worker tiene puntatori dentro + * ecache[] durante i suoi pread e uno spostamento li invaliderebbe; per lo + * stesso motivo gli slot eid<0 (riservati/in caricamento) non si toccano e la + * selezione avviene sotto g_pilot_mx. resident_bytes resta invariato: gli slot + * LRU non sono mai contati li' (solo pin e densa). + * EN: evict = free the slab in place (eid=-1, used=0, never compact: PILOT_REAL + * EN: holds pointers into ecache[] across its preads), skip eid<0 reservations, + * EN: select under g_pilot_mx. RSS_GUARD_GB= forces an explicit ceiling. */ +static double g_ram_budget_gb=0; /* budget risolto, scritto da cap_for_ram */ +static uint64_t g_rssg_last=0; +static void rss_guard(Model *m){ + double lim = getenv("RSS_GUARD_GB") ? atof(getenv("RSS_GUARD_GB")) : g_ram_budget_gb; + if(lim<=0) return; + if(m->n_emit - g_rssg_last < 16) return; + g_rssg_last = m->n_emit; + double rss=rss_gb(); + if(rss <= lim*1.02+0.3) return; /* tolleranza: 2% + 300MB */ + Cfg *c=&m->c; + int64_t need=(int64_t)((rss-lim)*1e9), freed=0; int dropped=0; + for(int pass=0; pass<8 && freedn_layers && freedecache || !m->ecache[l]) continue; + pthread_mutex_lock(&g_pilot_mx); + int nn=m->ecn[l], lru=-1; + for(int z=0;zecache[l][z]; + if(cand->eid<0 || !cand->slab) continue; + if(lru<0 || cand->usedecache[l][lru].used) lru=z; + } + if(lru<0){ pthread_mutex_unlock(&g_pilot_mx); continue; } + ESlot *s=&m->ecache[l][lru]; + s->eid=-1; /* nascosto: nessun hit/evict altrui */ + int64_t sb=s->slab_cap + s->fslab_cap*4; +#ifdef COLI_METAL + if(s->slab && g_metal_enabled) coli_metal_unregister(s->slab); +#endif + /* La free resta SOTTO g_pilot_mx. Sbloccando prima, lo slot e' visibile come + * {eid=-1, slab ancora valido}: il pilota (pilot_realload) riusa per primo gli + * slot eid==-1, quindi puo' prenderlo e fare pread dentro lo slab MENTRE lo + * liberiamo -> use-after-free / double-free. Slab valido e slot riusabile + * devono restare mutuamente esclusivi finche' il puntatore non e' NULL. */ + compat_aligned_free(s->slab); free(s->fslab); + s->slab=NULL; s->fslab=NULL; s->slab_cap=s->fslab_cap=0; + QT *q[3]={&s->g,&s->u,&s->d}; + for(int k=0;k<3;k++){ q[k]->qf=NULL; q[k]->q8=NULL; q[k]->q4=NULL; q[k]->s=NULL; } + s->used=0; /* primo candidato al riuso */ + pthread_mutex_unlock(&g_pilot_mx); + freed += sb; dropped++; + } + if(m->ecap>2) m->ecap--; /* il tetto scende: niente ricrescita */ + } + if(dropped) + fprintf(stderr,"[RAM-GUARD] RSS %.1f GB over the %.1f GB budget (#403): " + "dropped %d cached experts, cap -> %d\n", rss, lim, dropped, m->ecap); +} static void repin_pass_limit(Model *m,int limit){ + rss_guard(m); /* #403: il budget si fa rispettare sull'RSS MISURATO */ if(g_repin<=0) return; if(m->n_emit - g_last_repin < (uint64_t)g_repin) return; g_last_repin = m->n_emit; @@ -4841,7 +4447,7 @@ static void repin_pass_limit(Model *m,int limit){ +(int64_t)coli_cuda_tensor_bytes(s->d.cuda) : 0; #endif double t0=now_s(); - expert_load(m,cd[b].l,cd[b].eid,s,1); /* disk -> RAM, same resident slot */ + expert_load(m,cd[b].l,cd[b].eid,s,1,0); /* disk -> RAM, same resident slot; demand=0: repin, never classified */ const char *tier="RAM"; #ifdef COLI_CUDA if(gpu){ /* refresh the same VRAM slot now, not lazily */ @@ -4873,117 +4479,6 @@ static void repin_pass_limit(Model *m,int limit){ * si appendono SOLO le posizioni nuove e si riscrive nrec per ultimo: un crash a meta' * append lascia nrec vecchio = file coerente. La riga KV del layer MTP non si salva: * al resume kv_start=-1 e la finestra di draft riparte da sola. */ -static int g_kvsave=1; -#define KV_MAGIC "COLIKV1\0" -static void kv_hdr(Model *m, int32_t *h, int nrec){ - Cfg *c=&m->c; int nic=0; - for(int i=0;in_layers;i++) if(m->Ic && m->Ic[i]) nic++; - h[0]=c->n_layers; h[1]=c->kv_lora; h[2]=c->qk_rope; - h[3]=m->has_dsa?c->index_hd:0; h[4]=nic; h[5]=c->vocab; h[6]=nrec; h[7]=0; -} -/* Bytes of one on-disk record: [tok i32][Lc+Rc per layer][Ic per DSA layer]. - * Layout matches what kv_disk_append writes and kv_disk_load reads. */ -static int64_t kv_rec_bytes(Model *m){ - Cfg *c=&m->c; - int64_t rec = 4 + (int64_t)c->n_layers*(c->kv_lora+c->qk_rope)*4; - if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) rec+=(int64_t)c->index_hd*4; - return rec; -} -/* Open the persistent handle lazily; write the header if the file is new. After - * this returns successfully, k->disk_fp is valid for the engine's lifetime and - * positioned at end-of-header (nrec==0 case) or wherever the caller seeks. */ -static int kv_disk_open(Model *m){ - KVState *k=m->kv; - if(k->disk_fp) return 1; - k->disk_fp=fopen(k->disk_path,"r+b"); - if(!k->disk_fp){ /* not there yet -> create + header */ - k->disk_fp=fopen(k->disk_path,"wb"); - if(!k->disk_fp) return 0; - int32_t h[8]; kv_hdr(m,h,0); - fwrite(KV_MAGIC,1,8,k->disk_fp); fwrite(h,4,8,k->disk_fp); - fflush(k->disk_fp); - fclose(k->disk_fp); - k->disk_fp=fopen(k->disk_path,"r+b"); /* reopen r+b for append */ - if(!k->disk_fp) return 0; - } - return 1; -} -static void kv_disk_truncate(Model *m, int nrec){ - if(!g_kvsave) return; - KVState *k=m->kv; - if(k->disk_fp){ fclose(k->disk_fp); k->disk_fp=NULL; } /* drop to shrink on disc */ - FILE *f=fopen(k->disk_path,"r+b"); - if(!f){ k->disk_nrec=0; return; } - k->disk_nrec=nrec; - int32_t nr=nrec; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); - fflush(f); fclose(f); -} -static void kv_disk_reset(Model *m){ kv_disk_truncate(m,0); } -static void kv_disk_append(Model *m, const int *hist, int len){ - KVState *k=m->kv; - if(!g_kvsave || len<=k->disk_nrec) return; - Cfg *c=&m->c; - if(!kv_disk_open(m)) return; - FILE *f=k->disk_fp; - int64_t rec = kv_rec_bytes(m); - /* grow the contiguous staging buffer if the record is larger (#1 batching) */ - if(rec > k->disk_buf_cap){ - uint8_t *nb=realloc(k->disk_buf, rec); - if(!nb) return; /* OOM: skip this turn, retry next */ - k->disk_buf=nb; k->disk_buf_cap=rec; - } - fseek(f, 8+8*4 + (int64_t)k->disk_nrec*rec, SEEK_SET); - for(int p=k->disk_nrec;pdisk_buf; /* pack token + every layer into one record */ - *(int32_t*)b = hist[p]; b+=4; - for(int i=0;in_layers;i++){ - memcpy(b, m->Lc[i]+(int64_t)p*c->kv_lora, (size_t)c->kv_lora*4); b+=c->kv_lora*4; - memcpy(b, m->Rc[i]+(int64_t)p*c->qk_rope,(size_t)c->qk_rope*4); b+=c->qk_rope*4; - } - if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]){ - memcpy(b, m->Ic[i]+(int64_t)p*c->index_hd, (size_t)c->index_hd*4); b+=c->index_hd*4; - } - fwrite(k->disk_buf, 1, (size_t)rec, f); /* one fwrite per position (was ~157) */ - } - fflush(f); /* dati prima, contatore poi */ - int32_t nr=len; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); - fflush(f); /* persist the counter too */ - k->disk_nrec=len; -} -static int kv_disk_load(Model *m, int *hist, int maxctx){ - if(!g_kvsave) return 0; - KVState *k=m->kv; - Cfg *c=&m->c; - FILE *f=fopen(k->disk_path,"rb"); if(!f) return 0; - char mg[8]; int32_t h[8], w[8]; kv_hdr(m,w,0); - if(fread(mg,1,8,f)!=8 || memcmp(mg,KV_MAGIC,8) || fread(h,4,8,f)!=8 || - h[0]!=w[0]||h[1]!=w[1]||h[2]!=w[2]||h[3]!=w[3]||h[4]!=w[4]||h[5]!=w[5]){ - fprintf(stderr,"[KV] ignoring .coli_kv from a different model or version\n"); fclose(f); return 0; } - int nrec=h[6]; - if(nrec<1){ fclose(f); return 0; } - if(nrec>=maxctx-8-g_draft){ - fprintf(stderr,"[KV] saved conversation (%d tokens) exceeds the context: starting over\n",nrec); - fclose(f); return 0; } - double t0=now_s(); - for(int p=0;pn_layers;i++){ - if(fread(m->Lc[i]+(int64_t)p*c->kv_lora, 4, c->kv_lora, f)!=(size_t)c->kv_lora || - fread(m->Rc[i]+(int64_t)p*c->qk_rope, 4, c->qk_rope, f)!=(size_t)c->qk_rope){ nrec=p; goto out; } - } - if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) - if(fread(m->Ic[i]+(int64_t)p*c->index_hd, 4, c->index_hd, f)!=(size_t)c->index_hd){ nrec=p; goto out; } - } -out: - fclose(f); - if(nrec>0){ - if(m->has_mtp) m->kv_start[c->n_layers]=-1; /* la finestra MTP riparte da sola */ - fprintf(stderr,"[KV] resumed conversation from disk: %d tokens in %.1fs (no re-prefill)\n", - nrec, now_s()-t0); - } - k->disk_nrec=nrec; - return nrec; -} typedef struct { KVState kv; int *hist, len, first; } ServeCtx; static double kv_pool_bytes(Model *m, int max_ctx); @@ -5132,7 +4627,7 @@ static void run_serve_mux(Model *m, const char *snap){ g_draft=0; /* one scheduler owns every forward; MTP/speculation is not ragged-safe */ int maxctx=getenv("CTX")?atoi(getenv("CTX")):4096; int nctx=getenv("KV_SLOTS")?atoi(getenv("KV_SLOTS")):1; - if(nctx<1||nctx>16){fprintf(stderr,"KV_SLOTS deve essere tra 1 e 16\n");exit(2);} + if(nctx<1||nctx>512){fprintf(stderr,"KV_SLOTS must be between 1 and 512\n");exit(2);} g_kvsave=getenv("KVSAVE")?atoi(getenv("KVSAVE")):1; KVState *initial=m->kv; free(initial->kv_start); free(initial); ServeCtx *ctx=calloc(nctx,sizeof(*ctx)); ServeReq *req=calloc(nctx,sizeof(*req)); @@ -5188,7 +4683,7 @@ static void run_serve_mux(Model *m, const char *snap){ } active=0; for(int i=0;ilen;i++) r[i]=(int)a->kids[i]->num; *n=a->len; return r; } -/* byte residenti di un tensore [O,I] al numero di bit dato (specchio di qt_bytes) */ -static int64_t tbytes(int O,int I,int bits){ - if(bits>=16) return (int64_t)O*I*4; - if(bits>=5) return (int64_t)O*I + (int64_t)O*4; - return (int64_t)O*((I+1)/2) + (int64_t)O*4; -} -/* byte VERI di un expert: dal container se pre-quantizzato, altrimenti stima da ebits */ -static int64_t expert_bytes_probe(Model *m, int ebits){ - Cfg *c=&m->c; int64_t eb=0; char nm[256]; - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.gate_proj.weight",c->first_dense); - if(st_nbytes(&m->S,nm)>0){ - const char *suf[3]={"gate_proj","up_proj","down_proj"}; - for(int k=0;k<3;k++){ - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight",c->first_dense,suf[k]); - eb+=st_nbytes(&m->S,nm); - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight.qs",c->first_dense,suf[k]); - int64_t q=st_nbytes(&m->S,nm); if(q>0) eb+=q; - } - } - if(eb<=0) eb = tbytes(c->moe_inter,c->hidden,ebits)*2 + tbytes(c->hidden,c->moe_inter,ebits); - return eb; -} - -/* TIERS: fotografia della piramide expert per la dashboard web — - * "TIERS " sul canale di protocollo. - * ram = pinnati non-VRAM + LRU corrente; disk = tutto il resto. */ -/* BRAIN MAP (dashboard): per-turn expert hit bitmap + full residency/heat map. - * g_ehit[layer][eid]=1 quando l'expert viene instradato in questo turno; - * hits_emit lo serializza e lo azzera. emap_emit fotografa tier+heat di TUTTI. */ -static uint8_t **g_ehit; -static void ehit_mark(Model *m, int layer, int eid){ - if(!g_ehit){ Cfg *c=&m->c; - g_ehit=calloc(c->n_layers+1,sizeof(uint8_t*)); - for(int i=0;i<=c->n_layers;i++) g_ehit[i]=calloc(c->n_experts,1); - } - g_ehit[layer][eid]=1; -} - -/* HWINFO: hardware snapshot for the web dashboard — emitted once at READY. */ -/* CPU model + cores + RAM (GB); empty/zero where unavailable. - * Shared by the dashboard HWINFO line and the PROF=1 header. */ -static void hw_probe(char *cpu, size_t cn, int *cores, double *ram_total, double *ram_avail){ - cpu[0]=0; -#ifdef _WIN32 - /* niente /proc su Windows: brand string via CPUID (0x80000002..4), zero - * dipendenze extra. La dashboard mostrava "0 GB RAM / 0 cores" perche' - * tutto questo blocco era solo-Linux mentre il ramo CUDA funzionava. */ -#if defined(__x86_64__) || defined(__i386__) - { unsigned int r[12]={0}; unsigned int *w=r; - for(unsigned int f=0x80000002u; f<=0x80000004u; f++,w+=4) - __get_cpuid(f,&w[0],&w[1],&w[2],&w[3]); - char *b=(char*)r; b[47]=0; while(*b==' ')b++; - snprintf(cpu,cn,"%s",b); } -#endif -#else - FILE *ci=fopen("/proc/cpuinfo","r"); - if(ci){ char ln[256]; - while(fgets(ln,sizeof(ln),ci)) if(!strncmp(ln,"model name",10)){ - char *p=strchr(ln,':'); if(p){ p++; while(*p==' ')p++; - int n=(int)strlen(p); if(n>0&&p[n-1]=='\n')p[--n]=0; - snprintf(cpu,cn,"%s",p); } break; } - fclose(ci); } -#endif - *cores=0; -#ifdef _WIN32 - { SYSTEM_INFO si; GetSystemInfo(&si); *cores=(int)si.dwNumberOfProcessors; } -#elif defined(_SC_NPROCESSORS_ONLN) - *cores=(int)sysconf(_SC_NPROCESSORS_ONLN); -#endif - *ram_total=*ram_avail=0; -#ifdef _WIN32 - compat_meminfo(ram_total,ram_avail); /* GlobalMemoryStatusEx, gia' in compat.h */ -#else - FILE *mi=fopen("/proc/meminfo","r"); - if(mi){ char ln[256]; double mt=0,ma=0; - while(fgets(ln,sizeof(ln),mi)){ - if(sscanf(ln,"MemTotal: %lf",&mt)==1) *ram_total=mt/1e6; - if(sscanf(ln,"MemAvailable: %lf",&ma)==1) *ram_avail=ma/1e6; - } fclose(mi); } -#endif -} - -static void hwinfo_emit(Model *m){ - Cfg *c=&m->c; (void)c; /* silence -Wunused on builds without /proc (#148 report) */ - char cpu[256]; int cores; double ram_total,ram_avail; - hw_probe(cpu,sizeof(cpu),&cores,&ram_total,&ram_avail); - /* GPU */ - int ngpu=0; double vram_total=0; - char gpu_name[128]=""; -#ifdef COLI_CUDA - ngpu=g_cuda_ndev; vram_total=m->gpu_expert_bytes/1e9; - for(int i=0;i0){ - /* We don't have a device-name API; parse from the init log line stored in stderr. - * Simpler: just read the nvidia driver sysfs or use a fixed label. */ - snprintf(gpu_name,sizeof(gpu_name),"CUDA device x%d",g_cuda_ndev); - } -#endif - printf("HWINFO %d %.1f %.1f %d %.1f %s|%s\n", - cores,ram_total,ram_avail,ngpu,vram_total,cpu,gpu_name); - fflush(stdout); -} - -static void tiers_emit(Model *m){ - Cfg *c=&m->c; int nsp=0; - for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; - int total=(nsp+(m->has_mtp?1:0))*c->n_experts; - int pinned=0,lru=0; - for(int i=0;i<=c->n_layers;i++){ pinned+=m->npin?m->npin[i]:0; lru+=m->ecn?m->ecn[i]:0; } - int vram=0; double vram_gb=0; -#ifdef COLI_CUDA - vram=m->gpu_expert_count; vram_gb=m->gpu_expert_bytes/1e9; -#endif - int ram=pinned-vram+lru; if(ram<0) ram=0; - int disk=total-vram-ram; if(disk<0) disk=0; - double eb=(double)expert_bytes_probe(m,m->ebits); - printf("TIERS %d %d %d %.2f %.2f\n",vram,ram,disk,vram_gb,ram*eb/1e9); - fflush(stdout); -} - -/* EMAP: 1 byte/expert (2bit tier: 0=disk 1=RAM 2=VRAM | 6bit heat log2-bucket), - * righe = layer sparsi in ordine (+MTP se presente), colonne = n_experts. Hex. */ -static void emap_emit(Model *m){ - Cfg *c=&m->c; - int rows=0; - for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; - int has_mtp = m->has_mtp && m->eusage[c->n_layers]; - if(has_mtp) rows++; - int cols=c->n_experts; - char *hex=malloc((size_t)rows*cols*2+1); int w=0; - for(int i=0;i<=c->n_layers;i++){ - int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); - if(!is_row) continue; - for(int e=0;epin[i]; - for(int z=0;znpin[i];z++) if(P[z].eid==e){ -#ifdef COLI_CUDA - tier = P[z].g.cuda?2:1; -#else - tier = 1; -#endif - break; } - if(!tier && m->ecache && m->ecache[i]) - for(int z=0;zecn[i];z++) if(m->ecache[i][z].eid==e){ tier=1; break; } - uint32_t u = m->eusage[i]?m->eusage[i][e]:0; - int heat=0; while(u){ heat++; u>>=1; } if(heat>63) heat=63; - int b=(tier<<6)|heat; - hex[w++]="0123456789abcdef"[b>>4]; hex[w++]="0123456789abcdef"[b&15]; - } - } - hex[w]=0; - printf("EMAP %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); -} - -/* HITS: bitmap 1bit/expert (stesso ordine di EMAP), poi azzera per il turno dopo. */ -static void hits_emit(Model *m){ - Cfg *c=&m->c; if(!g_ehit) return; - int rows=0; - for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; - int has_mtp = m->has_mtp && m->eusage[c->n_layers]; - if(has_mtp) rows++; - int cols=c->n_experts, nb=(rows*cols+7)/8; - uint8_t *bm=calloc(nb,1); int bit=0; - for(int i=0;i<=c->n_layers;i++){ - int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); - if(!is_row) continue; - for(int e=0;e>3]|=1<<(bit&7); g_ehit[i][e]=0; } - } - char *hex=malloc((size_t)nb*2+1); int w=0; - for(int b=0;b>4]; hex[w++]="0123456789abcdef"[bm[b]&15]; } - hex[w]=0; - printf("HITS %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); free(bm); -} - -/* scarica su file l'istogramma d'uso degli expert: righe "layer eid count" (per PIN). - * Include la riga MTP (layer n_layers). Scrittura atomica (tmp+rename): viene chiamata - * anche a ogni turno di serve e il processo puo' morire in qualsiasi momento. */ -static void stats_dump_q(Model *m, const char *path, int quiet){ - char tmp[2100]; snprintf(tmp,sizeof(tmp),"%s.tmp",path); - FILE *f=fopen(tmp,"w"); if(!f){ if(!quiet) perror(tmp); return; } - Cfg *c=&m->c; int64_t tot=0, nz=0; - for(int i=0;i<=c->n_layers;i++){ if(!m->eusage[i]) continue; - for(int e=0;en_experts;e++) if(m->eusage[i][e]){ fprintf(f,"%d %d %u\n",i,e,m->eusage[i][e]); tot+=m->eusage[i][e]; nz++; } } - fclose(f); rename(tmp,path); - if(!quiet) fprintf(stderr,"[STATS] %lld selections across %lld distinct experts -> %s\n",(long long)tot,(long long)nz,path); -} -static void stats_dump(Model *m, const char *path){ stats_dump_q(m,path,0); } - -/* CACHE CHE IMPARA: istogramma d'uso PERSISTENTE in /.coli_usage. - * Caricato all'avvio (i contatori ripartono dalla storia), salvato a ogni turno: - * piu' usi colibri', meglio l'auto-pin conosce i TUOI expert caldi. */ -static char g_usage_path[2100]=""; -static int64_t usage_load(Model *m, const char *path){ - FILE *f=fopen(path,"r"); if(!f) return 0; - Cfg *c=&m->c; int l,e; uint32_t cnt; int64_t tot=0; - while(fscanf(f,"%d %d %u",&l,&e,&cnt)==3) - if(l>=0&&l<=c->n_layers&&e>=0&&en_experts&&m->eusage[l]){ m->eusage[l][e]+=cnt; tot+=cnt; } - fclose(f); return tot; -} -static void usage_save(Model *m){ if(g_usage_path[0]) stats_dump_q(m,g_usage_path,1); } +/* telemetry, stats, usage persistence — moved to telemetry.h */ /* HOT-STORE ("il redis del colibri'"): carica in RAM, UNA VOLTA e per sempre, i top expert * per frequenza d'uso misurata (file STATS di un run precedente), entro un budget in GB. @@ -5742,6 +5032,29 @@ static void pin_load(Model *m, const char *statspath, double gb){ double avail=expert_avail(m,ram_env,m->ebits,est_ctx); npin=avail>0?(int)(avail/eb):0; } else npin=(int)(gb*1e9/eb); +#ifdef COLI_CUDA + /* The VRAM budget must be known BEFORE npin is finalized: with + * CUDA_RELEASE_HOST the VRAM-ranked prefix's host slabs are freed right + * after upload, so those slots must NOT consume the RAM pin budget. + * Before this, a 6-GPU host lost its top ~9k ranked experts from the RAM + * count and pinned only the leftovers (measured: 9,280 VRAM + 1,721 RAM + * on a box whose RAM fits ~11k — the cold tail then paid disk forever). */ + double remaining[COLI_CUDA_MAX_DEVICES]={0}, placed_b[COLI_CUDA_MAX_DEVICES]={0}; + int placed_n[COLI_CUDA_MAX_DEVICES]={0}, gpu_prefix=0, prefix_est=0; + double budget=g_cuda_expert_gb*1e9, safe_total=0; + if(g_cuda_enabled&&(g_cuda_expert_gb>0||g_cuda_expert_auto)) for(int i=0;isafe_total) budget=safe_total; + if(g_cuda_enabled&&g_cuda_release_host&&budget>0){ + prefix_est=(int)(budget/eb)+g_cuda_ndev; + npin+=prefix_est; /* additive: prefix RAM is returned after upload */ + } +#endif if(npin>n) npin=n; if(npin<1){ free(r); return; } int *cnt_l=calloc(c->n_layers+1,sizeof(int)); /* +1: riga MTP */ @@ -5752,18 +5065,7 @@ static void pin_load(Model *m, const char *statspath, double gb){ for(int i=0;i<=c->n_layers;i++) m->npin[i]=cnt_l[i]; double t0=now_s(); #ifdef COLI_CUDA - double remaining[COLI_CUDA_MAX_DEVICES]={0}, placed_b[COLI_CUDA_MAX_DEVICES]={0}; - int placed_n[COLI_CUDA_MAX_DEVICES]={0}, gpu_prefix=0; - double budget=g_cuda_expert_gb*1e9, safe_total=0; - if(g_cuda_enabled&&(g_cuda_expert_gb>0||g_cuda_expert_auto)) for(int i=0;isafe_total) budget=safe_total; - if(g_cuda_enabled&&g_cuda_release_host&&budget>0){ gpu_prefix=(int)(budget/eb)+g_cuda_ndev; if(gpu_prefix>npin)gpu_prefix=npin; } + if(prefix_est>0){ gpu_prefix=prefix_est; if(gpu_prefix>npin) gpu_prefix=npin; } #else int gpu_prefix=0; #endif @@ -5771,7 +5073,7 @@ static void pin_load(Model *m, const char *statspath, double gb){ * released before the disjoint RAM-ranked suffix is allocated. */ #pragma omp parallel for schedule(dynamic,1) for(int a=0;a<(gpu_prefix?gpu_prefix:npin);a++) - expert_load(m,r[a].l,r[a].e,&m->pin[r[a].l][slot_of[a]],1); + expert_load(m,r[a].l,r[a].e,&m->pin[r[a].l][slot_of[a]],1,0); /* startup pin load; demand=0, never classified */ m->resident_bytes+=(int64_t)(gpu_prefix?gpu_prefix:npin)*eb; #ifdef COLI_CUDA if(g_cuda_enabled && budget>0){ @@ -5815,7 +5117,7 @@ static void pin_load(Model *m, const char *statspath, double gb){ if(gpu_prefix>0&&gpu_prefixpin[r[a].l][slot_of[a]],1); + expert_load(m,r[a].l,r[a].e,&m->pin[r[a].l][slot_of[a]],1,0); /* startup pin load; demand=0, never classified */ m->resident_bytes+=(int64_t)(npin-gpu_prefix)*eb; } fprintf(stderr,"[PIN] placement: %d VRAM + %d RAM expert (%.1f GB warm) in %.0fs da %s\n", @@ -5883,6 +5185,7 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){ if(auto_b){ ram_gb = g_mem_avail_boot*0.88; /* misurata PRIMA del load: il residente gia' * allocato viene sottratto sotto, non due volte */ if(ram_gb<4){ fprintf(stderr,"[RAM] MemAvailable is unreadable or too low; assuming 8 GB\n"); ram_gb=8; } } + g_ram_budget_gb = ram_gb; /* #403: la RSS-guard usa il budget RISOLTO */ /* slack ONESTO, non forfettario (l'OOM del 2026-07-04 veniva da qui): * ws[64] slab del working-set (si materializzano TUTTI nel prefill batch-union), * KV cache a max_ctx, kvb_all della ricostruzione k/v in attention, @@ -5972,9 +5275,9 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){ * self-test, and would "generate" from "$P$G". So on Windows a PROMPT carrying * cmd's $-metacodes is ignored; set COLI_PROMPT to pass a real prompt from cmd. */ static const char *coli_user_prompt(void){ - const char *p = getenv("COLI_PROMPT"); + const char *p = getenv_utf8("COLI_PROMPT"); if(p) return p; - p = getenv("PROMPT"); + p = getenv_utf8("PROMPT"); #ifdef _WIN32 if(p) for(const char *q=p; q[0]; q++) if(q[0]=='$' && q[1] && strchr("ABCDEFGHLNPQSTV_+|$", q[1]&~0x20)){ p=NULL; break; } @@ -6034,6 +5337,14 @@ int main(int argc, char **argv){ !getenv("COLI_CUDA") && !getenv("COLI_METAL")){ setenv("OMP_WAIT_POLICY","active",0); /* keep the team hot across the tiny per-expert matmul regions */ setenv("GOMP_SPINCOUNT","200000",0); /* spin briefly, then yield so long disk waits don't burn a core */ + /* LLVM libomp (clang builds: FreeBSD cc, macOS, some Linux setups) does not + * read GOMP_*: with OMP_WAIT_POLICY=active it sets KMP_BLOCKTIME=infinite, + * so the idle team SPINS FOREVER once generation ends — a serve-mode engine + * parked on stdin burns ~100% x nthreads (#341, measured 3000% on FreeBSD). + * 200 ms of blocktime keeps the team hot across back-to-back expert matmuls + * and lets it sleep at the prompt. libgomp ignores KMP_*; overwrite=0 keeps + * the user's own setting authoritative. */ + setenv("KMP_BLOCKTIME","200",0); setenv("OMP_PROC_BIND","close",0); /* pack the team onto adjacent cores for cache locality */ setenv("OMP_DYNAMIC","FALSE",0); /* fixed team size: no per-region thread-count churn */ setenv("COLI_OMP_TUNED","1",1); @@ -6150,6 +5461,8 @@ int main(int argc, char **argv){ if(g_pipe_nw<1) g_pipe_nw=1; g_pipe_block = getenv("COLI_PIPE_BLOCK")?atoi(getenv("COLI_PIPE_BLOCK")):0; /* blocking pipe_wait (default: spin) */ g_direct = getenv("DIRECT")?atoi(getenv("DIRECT")):0; + { const char *dh=getenv("COLI_DISKCLASS_WINDOW"); /* DISK-CLASS recency window, see its declaration */ + if(dh){ g_direct_heat_ticks=(uint32_t)strtoul(dh,NULL,10); g_direct_heat_explicit=1; } } g_uring = getenv("URING")?atoi(getenv("URING")):0; if(g_uring){ #ifdef __linux__ @@ -6192,8 +5505,9 @@ int main(int argc, char **argv){ int cap = argc>1?atoi(argv[1]):64; int ebits= argc>2?atoi(argv[2]):8; int dbits= argc>3?atoi(argv[3]):ebits; - if(getenv("SERVE") && (kv_slot_count()<1 || kv_slot_count()>16)){ - fprintf(stderr,"KV_SLOTS must be between 1 and 16\n"); return 2; + int kv_limit=(getenv("SERVE_BATCH")&&atoi(getenv("SERVE_BATCH")))?512:16; + if(getenv("SERVE") && (kv_slot_count()<1 || kv_slot_count()>kv_limit)){ + fprintf(stderr,"KV_SLOTS must be between 1 and %d\n",kv_limit); return 2; } #ifdef COLI_CUDA if(getenv("COLI_CUDA") && atoi(getenv("COLI_CUDA"))){ @@ -6247,6 +5561,19 @@ int main(int argc, char **argv){ printf("== GLM C engine (glm_moe_dsa), cache=%d experts/layer | experts@%d-bit dense@%d-bit | idot: " IDOT_KERNEL " ==\n", cap, ebits, dbits); g_mem_avail_boot = mem_available_gb(); Model m; double t0=now_s(); model_init(&m,snap,cap,ebits,dbits); + if(!g_direct_heat_explicit){ /* COLI_DISKCLASS_WINDOW default, needs m.c (topk/n_layers) */ + /* CURRENT-STATE CALIBRATION: the "8" multiplier (recency window ~= the last 8 + * tokens' worth of routing) is a measured-config constant, not a derived truth. + * Coordinates: measured 2026-07, macOS 26.5, M5 Max 128 GB, base caa49f7, + * GLM-5.2 int4 (topk=8, n_layers=78). On that setup it splits the classes + * cleanly (decode cold share ~71% of classified bytes, 1061.85/1491.49 GB); + * other models, page-cache pressures, or disks may want a different window -- + * COLI_DISKCLASS_WINDOW overrides, and the DISK-CLASS line itself is the + * tuning instrument. */ + uint32_t nl=(uint32_t)(m.c.n_layers>0?m.c.n_layers:1), k=(uint32_t)(m.c.topk>0?m.c.topk:1); + g_direct_heat_ticks = k*nl*8u; /* ~last 8 tokens' worth of routing, see the declaration */ + if(!g_direct_heat_ticks) g_direct_heat_ticks=1; + } if(g_draft<0){ #ifdef COLI_CUDA /* MTP is disabled under CUDA by default: cold (streaming) experts still @@ -6264,14 +5591,32 @@ int main(int argc, char **argv){ #endif } if(getenv("DSA_TOPK")) m.c.index_topk=atoi(getenv("DSA_TOPK")); /* override per test */ + /* Il path MUX (SERVE_BATCH=1, cioe' `coli serve`) forza g_draft=0 sotto — + * la speculazione non e' ragged-safe nel batch multi-slot. Segnalarlo QUI, + * altrimenti "MTP active (draft=8)" mentirebbe: il messaggio e' stampato + * prima della scelta del path (run_serve_mux, sotto), e con DRAFT=8 diceva + * "active" per poi disabilitarlo in silenzio (#358, LordMZTE). */ + int mux_will_disable_mtp = getenv("SERVE") && getenv("SERVE_BATCH") && atoi(getenv("SERVE_BATCH")); + int eff_draft = mux_will_disable_mtp ? 0 : g_draft; printf("loaded in %.2fs | resident dense: %.2f MB | layers=%d experts=%d | MTP %s (draft=%d)\n", now_s()-t0, m.resident_bytes/(1024.0*1024.0), m.c.n_layers, m.c.n_experts, - m.has_mtp?"ACTIVE":"absent", g_draft); + m.has_mtp?(mux_will_disable_mtp?"DISABLED (multiplexed serve)":"ACTIVE"):"absent", eff_draft); /* anche su stderr: e' il canale che le UI (coli) mostrano all'utente */ - fprintf(stderr,"[MTP] %s (draft=%d)\n", m.has_mtp?"active: native speculative decoding":"absent", g_draft); - if(!strncmp(snap,"/mnt/",5)) - fprintf(stderr,"WARNING: the model is on %s (slow 9p/Windows filesystem; fadvise is ineffective).\n" - " Keep it on ext4 (for example, /home/...) for memory efficiency and speed.\n", snap); + if(mux_will_disable_mtp && m.has_mtp) + fprintf(stderr,"[MTP] disabled in multiplexed serve (SERVE_BATCH=1): speculation is not " + "ragged-safe across KV slots. Single-client interactive use (`coli chat`) keeps MTP.\n"); + else + fprintf(stderr,"[MTP] %s (draft=%d)\n", m.has_mtp?"active: native speculative decoding":"absent", eff_draft); +#ifdef __linux__ + { /* Only warn for a GENUINE 9p mount (WSL Windows drives, magic 0x01021997), where + * fadvise is a no-op. The old check was `snap` starting with "/mnt/", which + * false-positives on native-Linux ZFS/ext4/xfs/NFS mounts that also live under /mnt. */ + struct statfs sfb; + if(statfs(snap,&sfb)==0 && (unsigned long)sfb.f_type==0x01021997UL) + fprintf(stderr,"WARNING: the model is on %s (9p/Windows filesystem; fadvise is ineffective).\n" + " Keep it on a native Linux fs (ext4/xfs/zfs) for memory efficiency and speed.\n", snap); + } +#endif /* HOT-STORE: PIN= [PIN_GB=g] -> top expert per frequenza fissi in RAM. * Va PRIMA di cap_for_ram: i pinnati contano nel residente. */ if(getenv("PIN")){ @@ -6411,8 +5756,9 @@ int main(int argc, char **argv){ double tot=m.hits+m.miss; printf("N-gram speculation (DRAFT=%d): %.2f tokens/forward (%llu forwards per %llu tokens)\n", g_draft, m.n_fw?(double)m.n_emit/m.n_fw:1.0, (unsigned long long)m.n_fw, (unsigned long long)m.n_emit); - printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu) | RSS: %.2f GB | %.1f tok/s\n", - tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hits, (unsigned long long)m.miss, rss_gb(), n_new/dt); + printf("Expert cache hit rate: %.1f%% (%llu pin + %llu lru / %llu miss) | RSS: %.2f GB | %.1f tok/s\n", + tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hit_pin, (unsigned long long)m.hit_ecache, + (unsigned long long)m.miss, rss_gb(), n_new/dt); profile_print(&m,dt); if(g_prof) prof_report(&m,&pb,dt,n_new,stdout); #ifdef COLI_CUDA diff --git a/c/compat.h b/c/compat.h index 82de8b6..1b56f5f 100644 --- a/c/compat.h +++ b/c/compat.h @@ -80,6 +80,7 @@ static inline int compat_open_direct(const char *path){ #endif #include #include +#include /* _mkdir (for the mkdtemp shim below) */ #include #include #include @@ -143,6 +144,12 @@ static inline int compat_fadvise(int fd, off_t off, off_t len, int advice){ * Thread-safe (no shared seek position). Gestisce offset >4 GB e chunking * per letture >2 GB (anche se i tensori individuali sono nell'ordine dei * MB-centinaia di MB, il wrapper e' robusto per ogni taglia). */ +/* Ultimo GetLastError() di una ReadFile fallita, per thread: il chiamante + * (pread_full in glm.c) lo stampa accanto a strerror. Senza questo, OGNI + * fallimento Windows collassa in "EIO -> Input/output error" e la diagnosi + * dal campo diventa un tirare a indovinare (#307: tre giri di ipotesi tra + * tre persone perche' il codice vero non compariva da nessuna parte). */ +static __thread DWORD compat_pread_lasterr __attribute__((unused)); static inline ssize_t compat_pread(int fd, void *buf, size_t n, off_t off){ intptr_t osfh = _get_osfhandle(fd); if(osfh == -1 || osfh == -2){ errno = EBADF; return -1; } @@ -158,6 +165,7 @@ static inline ssize_t compat_pread(int fd, void *buf, size_t n, off_t off){ if(!ReadFile(h, (char*)buf + total, chunk32, &rd, &ov)){ DWORD err = GetLastError(); if(err == ERROR_HANDLE_EOF) break; /* past EOF → return bytes read (0 if none, matching POSIX pread) */ + compat_pread_lasterr = err; /* preserva il codice VERO per il report (#307) */ if(err == ERROR_INVALID_HANDLE || err == ERROR_INVALID_FUNCTION) errno = EBADF; else errno = EIO; return -1; @@ -237,7 +245,9 @@ static inline int compat_rename(const char *old, const char *new){ /* --- rss_gb: getrusage -> GetProcessMemoryInfo --- * ru_maxrss in KB (come Linux): rss_gb() divide per 1e6 → GB corretti. */ #include -#pragma comment(lib, "psapi.lib") +#ifdef _MSC_VER +#pragma comment(lib, "psapi.lib") /* MSVC: link psapi; MinGW/GCC uses -lpsapi */ +#endif struct rusage { long ru_maxrss; }; #define RUSAGE_SELF 0 static inline int getrusage(int who, struct rusage *r){ @@ -304,8 +314,52 @@ static inline int compat_setenv(const char *name, const char *value, int overwri } #define setenv(name,value,overwrite) compat_setenv(name,value,overwrite) +/* --- getenv_utf8: read an env var as UTF-8, not through the ANSI codepage --- + * Plain getenv()/_environ are populated by the CRT from the ANSI-codepage view + * of the process environment block, not UTF-8. A parent that hands the child a + * Unicode value via CreateProcessW's wide env block (e.g. Python's subprocess + * module, which coli uses to pass the chat prompt) round-trips correctly only + * through GetEnvironmentVariableW; going through narrow getenv() re-encodes it + * via CP_ACP first, so any non-ASCII prompt text (Cyrillic, CJK, ...) comes out + * corrupted before the byte-level tokenizer ever sees it. Read the wide value + * directly and convert straight to UTF-8, bypassing the ANSI codepage entirely. + * Returned buffer is intentionally leaked: called a handful of times at + * startup, lives for the process. */ +static inline const char *compat_getenv_utf8(const char *name){ + wchar_t wname[64]; + if(MultiByteToWideChar(CP_UTF8, 0, name, -1, wname, 64) <= 0) return getenv(name); + DWORD need = GetEnvironmentVariableW(wname, NULL, 0); + if(!need) return NULL; + wchar_t *wval = (wchar_t*)malloc(need * sizeof(wchar_t)); + if(!wval) return NULL; + GetEnvironmentVariableW(wname, wval, need); + int blen = WideCharToMultiByte(CP_UTF8, 0, wval, -1, NULL, 0, NULL, NULL); + char *val = blen>0 ? (char*)malloc((size_t)blen) : NULL; + if(val) WideCharToMultiByte(CP_UTF8, 0, wval, -1, val, blen, NULL, NULL); + free(wval); + return val; +} +#define getenv_utf8(name) compat_getenv_utf8(name) + +/* --- mkdtemp -> _mktemp + _mkdir (POSIX mkdtemp assente su Windows) --- + * Test binaries (test_stops.c) create a scratch dir in the CWD via a + * "name_XXXXXX" template; POSIX mkdtemp fills the X's and mkdirs 0700. The + * Windows CRT has _mktemp (in-place, same XXXXXX contract) so we compose it. + * Returns the template pointer on success, NULL on failure — matching POSIX. */ +static inline char *compat_mkdtemp(char *tmpl){ + if(!tmpl) return NULL; + if(!_mktemp(tmpl)) return NULL; /* fills the trailing X's in place */ + if(_mkdir(tmpl) != 0) return NULL; /* EEXIST is impossible post-_mktemp */ + return tmpl; +} +#define mkdtemp(tmpl) compat_mkdtemp(tmpl) + #endif /* _WIN32 */ +#ifndef getenv_utf8 +#define getenv_utf8(name) getenv(name) +#endif + /* --- compat_aligned_free su piattaforme diverse da Windows --- * Su Linux/macOS, posix_memalign usa free() normale. */ #ifndef compat_aligned_free diff --git a/c/doctor.py b/c/doctor.py index 4cb5a42..cbe9c87 100644 --- a/c/doctor.py +++ b/c/doctor.py @@ -19,16 +19,33 @@ def _check(identifier, status, summary, **details): def cuda_linkage(engine_path): """Return CUDA linkage state without loading the executable or CUDA runtime.""" - if not Path(engine_path).is_file() or os.name != "posix": + engine = Path(engine_path) + if not engine.is_file(): return {"linked": False, "missing": False} - try: - result = subprocess.run(["ldd", str(engine_path)], capture_output=True, text=True, - timeout=3, check=False) - except (OSError, subprocess.SubprocessError): - return {"linked": False, "missing": False} - lines = [line for line in result.stdout.splitlines() if "libcudart" in line] - return {"linked": any("not found" not in line for line in lines), - "missing": any("not found" in line for line in lines)} + if os.name == "posix": + try: + result = subprocess.run(["ldd", str(engine)], capture_output=True, text=True, + timeout=3, check=False) + except (OSError, subprocess.SubprocessError): + return {"linked": False, "missing": False} + lines = [line for line in result.stdout.splitlines() if "libcudart" in line] + return {"linked": any("not found" not in line for line in lines), + "missing": any("not found" in line for line in lines)} + if sys.platform == "win32": + # Windows CUDA_DLL=1 builds never link libcudart directly: glm.exe loads + # coli_cuda.dll at runtime via LoadLibrary (backend_loader.c), so there's no + # import-table entry for ldd/dumpbin to see. Detect the COLI_CUDA build via a + # marker string baked into glm.c's #ifdef COLI_CUDA block instead, and require + # coli_cuda.dll to actually sit next to glm.exe (else CUDA init fails at startup). + try: + built = b"[CUDA] mode: routed experts" in engine.read_bytes() + except OSError: + return {"linked": False, "missing": False} + if not built: + return {"linked": False, "missing": False} + dll_present = (engine.parent / "coli_cuda.dll").is_file() + return {"linked": dll_present, "missing": not dll_present} + return {"linked": False, "missing": False} def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *, diff --git a/c/kv_persist.h b/c/kv_persist.h new file mode 100644 index 0000000..51289a8 --- /dev/null +++ b/c/kv_persist.h @@ -0,0 +1,121 @@ +/* kv_persist.h — .coli_kv on-disk KV cache persistence. + * Conversations reopen warm across engine restarts: the compressed MLA KV-cache + * is appended incrementally after every turn, crash-safe (nrec written last). + * Include after Model/KVState/Cfg are defined; requires now_s() and g_draft. */ +#ifndef KV_PERSIST_H +#define KV_PERSIST_H + +static int g_kvsave=1; +#define KV_MAGIC "COLIKV1\0" + +static void kv_hdr(Model *m, int32_t *h, int nrec){ + Cfg *c=&m->c; int nic=0; + for(int i=0;in_layers;i++) if(m->Ic && m->Ic[i]) nic++; + h[0]=c->n_layers; h[1]=c->kv_lora; h[2]=c->qk_rope; + h[3]=m->has_dsa?c->index_hd:0; h[4]=nic; h[5]=c->vocab; h[6]=nrec; h[7]=0; +} + +static int64_t kv_rec_bytes(Model *m){ + Cfg *c=&m->c; + int64_t rec = 4 + (int64_t)c->n_layers*(c->kv_lora+c->qk_rope)*4; + if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) rec+=(int64_t)c->index_hd*4; + return rec; +} + +static int kv_disk_open(Model *m){ + KVState *k=m->kv; + if(k->disk_fp) return 1; + k->disk_fp=fopen(k->disk_path,"r+b"); + if(!k->disk_fp){ + k->disk_fp=fopen(k->disk_path,"wb"); + if(!k->disk_fp) return 0; + int32_t h[8]; kv_hdr(m,h,0); + fwrite(KV_MAGIC,1,8,k->disk_fp); fwrite(h,4,8,k->disk_fp); + fflush(k->disk_fp); + fclose(k->disk_fp); + k->disk_fp=fopen(k->disk_path,"r+b"); + if(!k->disk_fp) return 0; + } + return 1; +} + +static void kv_disk_truncate(Model *m, int nrec){ + if(!g_kvsave) return; + KVState *k=m->kv; + if(k->disk_fp){ fclose(k->disk_fp); k->disk_fp=NULL; } + FILE *f=fopen(k->disk_path,"r+b"); + if(!f){ k->disk_nrec=0; return; } + k->disk_nrec=nrec; + int32_t nr=nrec; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); + fflush(f); fclose(f); +} + +static void kv_disk_reset(Model *m){ kv_disk_truncate(m,0); } + +static void kv_disk_append(Model *m, const int *hist, int len){ + KVState *k=m->kv; + if(!g_kvsave || len<=k->disk_nrec) return; + Cfg *c=&m->c; + if(!kv_disk_open(m)) return; + FILE *f=k->disk_fp; + int64_t rec = kv_rec_bytes(m); + if(rec > k->disk_buf_cap){ + uint8_t *nb=realloc(k->disk_buf, rec); + if(!nb) return; + k->disk_buf=nb; k->disk_buf_cap=rec; + } + fseek(f, 8+8*4 + (int64_t)k->disk_nrec*rec, SEEK_SET); + for(int p=k->disk_nrec;pdisk_buf; + *(int32_t*)b = hist[p]; b+=4; + for(int i=0;in_layers;i++){ + memcpy(b, m->Lc[i]+(int64_t)p*c->kv_lora, (size_t)c->kv_lora*4); b+=c->kv_lora*4; + memcpy(b, m->Rc[i]+(int64_t)p*c->qk_rope,(size_t)c->qk_rope*4); b+=c->qk_rope*4; + } + if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]){ + memcpy(b, m->Ic[i]+(int64_t)p*c->index_hd, (size_t)c->index_hd*4); b+=c->index_hd*4; + } + fwrite(k->disk_buf, 1, (size_t)rec, f); + } + fflush(f); + int32_t nr=len; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); + fflush(f); + k->disk_nrec=len; +} + +static int kv_disk_load(Model *m, int *hist, int maxctx){ + if(!g_kvsave) return 0; + KVState *k=m->kv; + Cfg *c=&m->c; + FILE *f=fopen(k->disk_path,"rb"); if(!f) return 0; + char mg[8]; int32_t h[8], w[8]; kv_hdr(m,w,0); + if(fread(mg,1,8,f)!=8 || memcmp(mg,KV_MAGIC,8) || fread(h,4,8,f)!=8 || + h[0]!=w[0]||h[1]!=w[1]||h[2]!=w[2]||h[3]!=w[3]||h[4]!=w[4]||h[5]!=w[5]){ + fprintf(stderr,"[KV] ignoring .coli_kv from a different model or version\n"); fclose(f); return 0; } + int nrec=h[6]; + if(nrec<1){ fclose(f); return 0; } + if(nrec>=maxctx-8-g_draft){ + fprintf(stderr,"[KV] saved conversation (%d tokens) exceeds the context: starting over\n",nrec); + fclose(f); return 0; } + double t0=now_s(); + for(int p=0;pn_layers;i++){ + if(fread(m->Lc[i]+(int64_t)p*c->kv_lora, 4, c->kv_lora, f)!=(size_t)c->kv_lora || + fread(m->Rc[i]+(int64_t)p*c->qk_rope, 4, c->qk_rope, f)!=(size_t)c->qk_rope){ nrec=p; goto out; } + } + if(m->has_dsa) for(int i=0;in_layers;i++) if(m->Ic[i]) + if(fread(m->Ic[i]+(int64_t)p*c->index_hd, 4, c->index_hd, f)!=(size_t)c->index_hd){ nrec=p; goto out; } + } +out: + fclose(f); + if(nrec>0){ + if(m->has_mtp) m->kv_start[c->n_layers]=-1; + fprintf(stderr,"[KV] resumed conversation from disk: %d tokens in %.1fs (no re-prefill)\n", + nrec, now_s()-t0); + } + k->disk_nrec=nrec; + return nrec; +} + +#endif /* KV_PERSIST_H */ diff --git a/c/olmoe.c b/c/olmoe.c index cda5913..840b59a 100644 --- a/c/olmoe.c +++ b/c/olmoe.c @@ -5,6 +5,15 @@ * Densa (embed, attn, router, norme, lm_head) residente in RAM (float32). * Expert letti dal disco on-demand via pread+fadvise(DONTNEED), cache LRU per-layer. * Matmul multi-thread con OpenMP (niente BLAS). + * + * ENV VARS: + * PILOT=0/1/2/3 : 0=no prefetch, 1=1-layer lookahead, 2=2-layer, 3=3-layer lookahead + * HOT=N : pin top-N hot experts per layer permanently (never evict) + * WARMUP=N : tokens before hot pinning activates (default 5) + * WIDE=N : prefetch top-K*N candidates (default 1, try 2 or 3) + * SMOOTH=F : EMA coefficient for routing momentum (default 0.3, range 0.0-0.95) + * CONF_LIMIT=F : cumulative gate probability threshold for prefetch cutoff (default 0.92) + * (expert queue is sorted by eid for SSD read locality) */ #define _GNU_SOURCE #include @@ -12,11 +21,22 @@ #include #include #include +#include #if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) #include +#include #endif #include "st.h" +#ifdef _WIN32 +#include +#define sleep_ms(ms) Sleep(ms) +#else +#define sleep_ms(ms) usleep((ms) * 1000) +#endif + + + /* ---------- config ---------- */ typedef struct { int hidden, n_layers, n_heads, n_kv_heads, head_dim; @@ -33,22 +53,61 @@ typedef struct { * Ogni weight [out,in] tenuto come int8 (per-riga) + scala float per riga. * Cosi' la RAM-cache scende da 4 byte/param (f32) a 1 byte/param: e' il * meccanismo che fa stare GLM-5.2 nei 15 GB. dequant-on-use nel matmul. */ -typedef struct { int eid; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot; +/* pinned=1 means this slot is strongly preferred to keep (hot expert); it will + * not be evicted during normal LRU eviction, but may be displaced under extreme + * cache pressure when all slots are pinned or in-flight. */ +typedef struct { int eid; int pinned; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot; typedef struct { Slot *slots; int n, cap; } LCache; typedef struct { Cfg c; shards S; - int quant_bits; /* bit di quantizzazione degli expert (2..8); storage int8, niente f32 (#134) */ + int quant_bits; float *embed, *lm_head, *final_norm; Layer *L; LCache *cache; /* [n_layers] */ uint64_t clock, hits, miss; - /* kv-cache per-layer: K,V come [H * maxT * head_dim] */ float **K, **V; int kv_len, max_t; double dense_load_s; + /* IMPROVEMENT 2: expert frequency heatmap */ + uint32_t *freq; + int freq_token_count, hot_pinned, hot_n, warmup_tokens; + int token_count; + /* PREDICTION IMPROVEMENT A: per-layer EMA of gate logits across tokens. + * momentum_logits[l*E .. (l+1)*E-1] = EMA of gate outputs for layer l. + * Used exclusively by the PILOT prefetcher to stabilise routing predictions + * across tokens; does NOT affect actual MoE routing (pr is unchanged). */ + float *momentum_logits; /* [n_layers * n_experts], EMA of gate logits */ + float pilot_smooth; /* SMOOTH env: EMA coefficient 0.0-0.9 (default 0.3) */ + uint8_t *is_pinned; /* [n_layers * n_experts], 1 if expert is globally pinned */ + uint8_t *is_queued; /* [n_layers * n_experts], 1 if expert is currently in the prefetch queue */ + float pilot_conf_limit; /* CONF_LIMIT env: cumulative gate probability threshold (e.g. 0.92) */ } Model; +static pthread_mutex_t g_pilot_mx = PTHREAD_MUTEX_INITIALIZER; +static struct { int l, e; } pilot_q[4096]; +static volatile unsigned pilot_r = 0, pilot_w = 0; +static Model *pilot_m = NULL; +static int g_pilot = 0; +static int g_wide = 1; /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */ + +static void pilot_prefetch(Model *m, int lnext, const float *x, int S); +static void *pilot_worker(void *arg); +static void ensure_pilot_worker_started(Model *m); +static void slot_ensure_allocated(Model *m, Slot *s); + +static void ensure_pilot_worker_started(Model *m) { + if (!pilot_m) { + pilot_m = m; + pthread_t t; + if (pthread_create(&t, NULL, pilot_worker, NULL) != 0) { + fprintf(stderr, "Error: Failed to create pilot prefetch worker thread\n"); + exit(1); + } + pthread_detach(t); + } +} + /* ---------- utility ---------- */ static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec*1e-9; } #if defined(__APPLE__) @@ -210,51 +269,224 @@ static void model_init(Model *m, const char *snap, int cap, int bits) { #undef LD } m->cache = calloc(c->n_layers, sizeof(LCache)); - for (int i = 0; i < c->n_layers; i++) { m->cache[i].cap = cap; m->cache[i].slots = calloc(cap, sizeof(Slot)); } + for (int i = 0; i < c->n_layers; i++) { + m->cache[i].cap = cap; + m->cache[i].slots = calloc(cap, sizeof(Slot)); + } + /* IMPROVEMENT 2: frequency heatmap for hot expert pinning */ + m->freq = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint32_t)); + m->hot_pinned = 0; m->freq_token_count = 0; + m->hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0; + m->warmup_tokens = getenv("WARMUP") ? atoi(getenv("WARMUP")) : 5; + m->token_count = 0; + /* PREDICTION A: routing momentum — EMA of gate logits across tokens. + * Initialized to zero; first token sets EMA = fresh logits. */ + m->momentum_logits = calloc((size_t)c->n_layers * c->n_experts, sizeof(float)); + float sv = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f; + if (sv < 0.f) sv = 0.f; if (sv > 0.95f) sv = 0.95f; + m->pilot_smooth = sv; + m->is_pinned = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t)); + m->is_queued = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t)); + float cl = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f; + if (cl < 0.1f) cl = 0.1f; if (cl > 1.0f) cl = 1.0f; + m->pilot_conf_limit = cl; m->dense_load_s = now_s() - t0; + + // Persistent Hot Pinning: try to load hot_pinned.bin + char pinpath[512]; + snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap); + FILE *pinf = fopen(pinpath, "rb"); + if (pinf) { + size_t expected_size = (size_t)c->n_layers * c->n_experts; + if (fread(m->is_pinned, 1, expected_size, pinf) == expected_size) { + m->hot_pinned = 1; + printf("[HOT] Loaded persistent pinning from %s\n", pinpath); + + if (g_pilot) { + ensure_pilot_worker_started(m); + for (int l = 0; l < c->n_layers; l++) { + for (int e = 0; e < c->n_experts; e++) { + if (m->is_pinned[l * c->n_experts + e]) { + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w - r < 4096) { + pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = e; + pthread_mutex_lock(&g_pilot_mx); + m->is_queued[l * c->n_experts + e] = 1; + pthread_mutex_unlock(&g_pilot_mx); + __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE); + } + } + } + } + printf("[HOT] Pre-loading pinned experts into cache...\n"); + double t_wait = now_s(); + while (1) { + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE); + if (r == w) break; + sleep_ms(2); + } + printf("[HOT] Pre-loaded in %.1fs!\n", now_s() - t_wait); + } + } + fclose(pinf); + } } -/* legge un weight dal disco (streaming) e lo quantizza in q[O,I]+scale[O]. - * Container pre-quantizzato (convert_olmoe.py: int8 + scale f32 in "name.qs"): - * lettura raw diretta — meta' I/O e zero quantize_rows a runtime. Prima di - * questa patch il container int8 causava SIGBUS (st_read_f32 su tensori I8). */ -static void load_expert_w(Model *m, const char *name, int8_t *q, float *scale, int O, int I, float *tmp) { - st_tensor *t = st_find(&m->S, name); - if (t && t->dtype == 3) { /* I8/U8: container colibri */ - char qs[300]; snprintf(qs, sizeof(qs), "%s.qs", name); - st_read_raw(&m->S, name, q, 1); - st_read_f32(&m->S, qs, scale, 1); - return; +static void slot_ensure_allocated(Model *m, Slot *s) { + if (s->g) return; + Cfg *c = &m->c; + int64_t ng = (int64_t)c->inter * c->hidden; + int64_t nd = (int64_t)c->hidden * c->inter; + int8_t *w_block = malloc(ng + ng + nd); + if (!w_block) { + fprintf(stderr, "Error: Out of memory allocating slot weights block\n"); + exit(1); } - st_read_f32(&m->S, name, tmp, 1); /* pread + fadvise DONTNEED */ - quantize_rows(tmp, q, scale, O, I, m->quant_bits); + s->g = w_block; + s->u = w_block + ng; + s->d = w_block + ng + ng; + float *s_block = falloc(c->inter + c->inter + c->hidden); + s->gs = s_block; + s->us = s_block + c->inter; + s->ds = s_block + c->inter + c->inter; + s->pinned = 0; +} + +static void load_expert_merged(Model *m, int layer, int eid, Slot *s) { + char nm[256], qsnm[256]; + snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.merged_weight", layer, eid); + snprintf(qsnm, sizeof(qsnm), "model.layers.%d.mlp.experts.%d.qs", layer, eid); + st_read_raw(&m->S, nm, s->g, 1); + st_read_f32(&m->S, qsnm, s->gs, 0); /* scales are F32; use typed reader for dtype safety */ } /* ---------- cache expert: ritorna i pesi quantizzati (q+scale) da cache o disco ---------- */ static void expert_get(Model *m, int layer, int eid, Slot **out) { LCache *lc = &m->cache[layer]; + pthread_mutex_lock(&g_pilot_mx); for (int i = 0; i < lc->n; i++) if (lc->slots[i].eid == eid) { - m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; return; + m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; + pthread_mutex_unlock(&g_pilot_mx); + return; } m->miss++; Cfg *c = &m->c; - int64_t ng = (int64_t)c->inter * c->hidden, nd = (int64_t)c->hidden * c->inter; Slot *s; if (lc->n < lc->cap) { s = &lc->slots[lc->n++]; - s->g = malloc(ng); s->u = malloc(ng); s->d = malloc(nd); - s->gs = falloc(c->inter); s->us = falloc(c->inter); s->ds = falloc(c->hidden); - } else { int lru = 0; for (int i = 1; i < lc->n; i++) if (lc->slots[i].used < lc->slots[lru].used) lru = i; s = &lc->slots[lru]; } - float *tmp = falloc(ng > nd ? ng : nd); - char nm[256]; - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.gate_proj.weight",layer,eid); load_expert_w(m,nm,s->g,s->gs,c->inter,c->hidden,tmp); - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.up_proj.weight", layer,eid); load_expert_w(m,nm,s->u,s->us,c->inter,c->hidden,tmp); - snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.down_proj.weight",layer,eid); load_expert_w(m,nm,s->d,s->ds,c->hidden,c->inter,tmp); - free(tmp); - s->eid = eid; s->used = ++m->clock; + slot_ensure_allocated(m, s); + } else { + /* LRU eviction — skip pinned and in-flight (eid==-1) slots */ + int lru = -1; + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue; + if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; + } + if (lru < 0) { + /* All slots are pinned or in-flight; find oldest non-in-flight slot + * (may be pinned, but never select one currently being loaded). */ + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].eid < 0) continue; /* never evict in-flight */ + if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; + } + } + if (lru < 0) lru = 0; /* absolute last resort: all in-flight, evict slot 0 */ + s = &lc->slots[lru]; + s->pinned = 0; + } + s->eid = -1; + s->used = ++m->clock; + pthread_mutex_unlock(&g_pilot_mx); + + load_expert_merged(m, layer, eid, s); + + pthread_mutex_lock(&g_pilot_mx); + s->eid = eid; + s->pinned = m->is_pinned[layer * c->n_experts + eid]; + s->used = ++m->clock; *out = s; + pthread_mutex_unlock(&g_pilot_mx); } +/* ---------- IMPROVEMENT 2: pin top-N hot experts per layer ---------- */ +static void pin_hot_experts(Model *m) { + Cfg *c = &m->c; + if (m->hot_n <= 0 || m->hot_pinned) return; + m->hot_pinned = 1; + + int is_dynamic = (m->hot_n >= 100); + double thresh = is_dynamic ? (double)m->hot_n / 1000.0 : 0.0; + + int pinned_total = 0; + for (int l = 0; l < c->n_layers; l++) { + uint32_t *freq_l = m->freq + (int64_t)l * c->n_experts; + + uint64_t layer_total = 0; + for (int e = 0; e < c->n_experts; e++) layer_total += freq_l[e]; + if (layer_total == 0) continue; + + int max_pin = m->cache[l].cap - 8; + if (max_pin < 4) max_pin = 4; + + int hn = is_dynamic ? max_pin : (m->hot_n < c->n_experts ? m->hot_n : c->n_experts); + if (hn > 256) hn = 256; + int hot_eids[256]; + int actual_hn = 0; + + for (int k = 0; k < hn; k++) { + int best = -1; uint32_t bv = 0; + for (int e = 0; e < c->n_experts; e++) { + int already = 0; + for (int j = 0; j < k; j++) if (hot_eids[j] == e) { already = 1; break; } + if (!already && freq_l[e] > bv) { bv = freq_l[e]; best = e; } + } + if (best < 0 || bv == 0) break; + if (is_dynamic && bv < thresh * layer_total) break; + hot_eids[k] = best; + actual_hn++; + } + + for (int k = 0; k < actual_hn; k++) { + int eid = hot_eids[k]; + m->is_pinned[l * c->n_experts + eid] = 1; + + LCache *lc = &m->cache[l]; + int found = 0; + pthread_mutex_lock(&g_pilot_mx); + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].eid == eid) { lc->slots[i].pinned = 1; found = 1; break; } + } + pthread_mutex_unlock(&g_pilot_mx); + if (!found && g_pilot > 0) { + /* Only enqueue when the prefetch worker is active (PILOT>0). */ + ensure_pilot_worker_started(m); + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + int gidx = l * c->n_experts + eid; + pthread_mutex_lock(&g_pilot_mx); + int already = m->is_queued[gidx]; + if (!already && w - r < 4096) { + pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = eid; + m->is_queued[gidx] = 1; + __atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE); + } + pthread_mutex_unlock(&g_pilot_mx); + } + pinned_total++; + } + } + if (is_dynamic) { + printf("[HOT] Dynamic Pinned %d experts total (thresh=%.1f%%) after %d warmup tokens\n", + pinned_total, thresh * 100.0, m->freq_token_count); + } else { + printf("[HOT] Pinned %d experts (top-%d/layer) after %d warmup tokens\n", + pinned_total, m->hot_n, m->freq_token_count); + } +} + + /* ---------- RoPE su un vettore di una testa (head_dim) a posizione assoluta pos ---------- */ static void rope_head(float *x, int pos, const Cfg *c) { int h = c->head_dim / 2; @@ -325,6 +557,19 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { float *g = falloc(I), *u = falloc(I), *hh = falloc(D); for (int s = 0; s < S; s++) { float *pr = logits + (int64_t)s*E; + if (m->momentum_logits && m->pilot_smooth > 0.f) { + float *ema = m->momentum_logits + (int64_t)layer * E; + int is_zero = 1; + for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } + if (is_zero) { + for (int e = 0; e < E; e++) ema[e] = pr[e]; + } else { + for (int e = 0; e < E; e++) { + ema[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e]; + } + } + } + softmax_row(pr, E); /* top-K indici (selezione parziale) */ int idx[64]; float val[64]; @@ -337,6 +582,11 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { idx[kk] = best; val[kk] = bv; } if (c->norm_topk) { float sm=0; for(int kk=0;kkhot_pinned && m->freq) { + uint32_t *freq_l = m->freq + (int64_t)layer * E; + for (int kk = 0; kk < K; kk++) if (idx[kk] >= 0) freq_l[idx[kk]]++; + } const float *xs = x + (int64_t)s*D; for (int kk = 0; kk < K; kk++) { Slot *e; expert_get(m, layer, idx[kk], &e); @@ -352,9 +602,20 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) { free(logits); free(g); free(u); free(hh); } -/* un passo: token nuovi ids[S] a posizione pos_base. Ritorna logits dell'ultimo token (malloc'd). */ static float *step(Model *m, const int *ids, int S, int pos_base) { Cfg *c = &m->c; int D = c->hidden; + if (g_pilot && m->token_count > 0) { + /* Flush stale prefetch requests: clear is_queued so pilot_realload + * will skip any entries still sitting in pilot_q for the previous + * token. We deliberately do NOT move pilot_w backwards; that would + * break the ring-buffer invariant (pilot_r could exceed pilot_w if + * the worker consumed an entry concurrently). The worker will drain + * the stale slots harmlessly because pilot_realload already exits + * early when the expert is already cached or is_queued is clear. */ + pthread_mutex_lock(&g_pilot_mx); + memset(m->is_queued, 0, (size_t)c->n_layers * c->n_experts); + pthread_mutex_unlock(&g_pilot_mx); + } float *x = falloc((int64_t)S*D); for (int s = 0; s < S; s++) memcpy(x + (int64_t)s*D, m->embed + (int64_t)ids[s]*D, D*sizeof(float)); float *nrm = falloc((int64_t)S*D), *tmp = falloc((int64_t)S*D); @@ -363,12 +624,26 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->in_ln, D, c->eps); attention(m, l, i, nrm, S, pos_base, tmp); for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j]; + /* IMPROVEMENT 1: PILOT=1 -> 1-layer lookahead */ + if (g_pilot >= 1 && S <= 8 && i + 1 < c->n_layers) + pilot_prefetch(m, i + 1, x, S); for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->post_ln, D, c->eps); moe(m, l, i, nrm, S, tmp); for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j]; + + /* PREDICTION IMPROVEMENT C (Residual gate trick): + * PILOT=2 -> prefetch layer i+2 using completed state x (containing MoE residual). */ + if (g_pilot >= 2 && S <= 8 && i + 2 < c->n_layers) + pilot_prefetch(m, i + 2, x, S); + if (g_pilot >= 3 && S <= 8 && i + 3 < c->n_layers) + pilot_prefetch(m, i + 3, x, S); + } + /* count actual tokens processed (S>1 during prefill) */ + m->token_count += S; m->freq_token_count += S; + if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens) + pin_hot_experts(m); m->kv_len = pos_base + S; - /* solo l'ultimo token -> logits */ float *last = falloc(D); rmsnorm_row(last, x + (int64_t)(S-1)*D, m->final_norm, D, c->eps); float *logit = falloc(c->vocab); @@ -377,6 +652,192 @@ static float *step(Model *m, const int *ids, int S, int pos_base) { return logit; } +static void pilot_realload(Model *m, int layer, int eid) { + LCache *lc = &m->cache[layer]; + Cfg *c = &m->c; + + pthread_mutex_lock(&g_pilot_mx); + /* Early-exit if entry was flushed (is_queued cleared) while waiting. */ + if (!m->is_queued[layer * c->n_experts + eid]) { + pthread_mutex_unlock(&g_pilot_mx); + return; + } + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].eid == eid) { + m->is_queued[layer * c->n_experts + eid] = 0; + pthread_mutex_unlock(&g_pilot_mx); + return; + } + } + Slot *s; + if (lc->n < lc->cap) { + s = &lc->slots[lc->n++]; + slot_ensure_allocated(m, s); + } else { + /* LRU eviction — skip pinned and in-flight (eid==-1) slots */ + int lru = -1; + for (int i = 0; i < lc->n; i++) { + if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue; + if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i; + } + if (lru < 0) { + m->is_queued[layer * c->n_experts + eid] = 0; + pthread_mutex_unlock(&g_pilot_mx); + return; /* all pinned/in-flight, skip */ + } + s = &lc->slots[lru]; s->pinned = 0; + } + s->eid = -1; s->used = ++m->clock; + pthread_mutex_unlock(&g_pilot_mx); + + load_expert_merged(m, layer, eid, s); + + pthread_mutex_lock(&g_pilot_mx); + s->eid = eid; + s->pinned = m->is_pinned[layer * c->n_experts + eid]; + s->used = ++m->clock; + m->is_queued[layer * c->n_experts + eid] = 0; + pthread_mutex_unlock(&g_pilot_mx); +} + +static void *pilot_worker(void *arg) { + (void)arg; + while (1) { + unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE); + if (r == w) { + sleep_ms(1); + continue; + } + int layer = pilot_q[r & 4095].l; + int eid = pilot_q[r & 4095].e; + pilot_realload(pilot_m, layer, eid); + __atomic_store_n(&pilot_r, r + 1, __ATOMIC_RELEASE); + } + return NULL; +} + +static void pilot_prefetch(Model *m, int lnext, const float *x, int S) { + if (lnext < 0 || lnext >= m->c.n_layers) return; + Cfg *c = &m->c; int D = c->hidden, E = c->n_experts; + ensure_pilot_worker_started(m); + float *logits = falloc((int64_t)S * E); + Layer *l = &m->L[lnext]; + + // PREDICTION IMPROVEMENT B: Apply RMSNorm to x using destination layer's post_ln + // This scales inputs to the distribution expected by l->gate. + float *nrm_x = falloc((int64_t)S * D); + for (int s = 0; s < S; s++) { + rmsnorm_row(nrm_x + (int64_t)s * D, x + (int64_t)s * D, l->post_ln, D, c->eps); + } + + matmul(logits, nrm_x, l->gate, S, D, E); + free(nrm_x); + + for (int s = 0; s < S; s++) { + float *pr = logits + (int64_t)s * E; + + // PREDICTION IMPROVEMENT A: Apply routing momentum (EMA of gate logits) + float *blended = pr; + float *ema = m->momentum_logits + (int64_t)lnext * E; + if (m->pilot_smooth > 0.f) { + blended = falloc(E); + int is_zero = 1; + for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } } + if (is_zero) { + for (int e = 0; e < E; e++) { + ema[e] = pr[e]; + blended[e] = pr[e]; + } + } else { + for (int e = 0; e < E; e++) { + blended[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e]; + ema[e] = blended[e]; // update EMA + } + } + } + + int cand = 0; + int idx[128]; + + float max_logit = -1e30f; + for (int e = 0; e < E; e++) { if (blended[e] > max_logit) max_logit = blended[e]; } + float *exps = falloc(E); + float sum_exps = 0.f; + for (int e = 0; e < E; e++) { + exps[e] = expf(blended[e] - max_logit); + sum_exps += exps[e]; + } + + float cum_sum = 0.f; + int min_cand = c->topk; + int max_cand = c->topk * g_wide; + if (max_cand < min_cand) max_cand = min_cand; + if (max_cand > 128) max_cand = 128; /* idx[] buffer bound */ + if (max_cand > E) max_cand = E; + + for (int kk = 0; kk < max_cand; kk++) { + int best = -1; float bv = -1.f; + for (int e = 0; e < E; e++) { + int taken = 0; for (int j = 0; j < kk; j++) if (idx[j] == e) { taken=1; break; } + if (!taken && exps[e] > bv) { bv = exps[e]; best = e; } + } + if (best < 0) break; + idx[kk] = best; + cum_sum += bv; + cand++; + if (cum_sum >= m->pilot_conf_limit * sum_exps && cand >= min_cand) { + break; + } + } + free(exps); + + if (blended != pr) free(blended); + + /* IMPROVEMENT 5: sort candidates by eid for sequential SSD read locality */ + for (int a = 0; a < cand-1; a++) + for (int b = a+1; b < cand; b++) + if (idx[b] >= 0 && (idx[a] < 0 || idx[a] > idx[b])) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; } + + for (int kk = 0; kk < cand; kk++) { + int eid = idx[kk]; + if (eid < 0) continue; + int found = 0; + pthread_mutex_lock(&g_pilot_mx); + LCache *lc = &m->cache[lnext]; + for (int z = 0; z < lc->n; z++) { + if (lc->slots[z].eid == eid) { found = 1; break; } + } + pthread_mutex_unlock(&g_pilot_mx); + if (!found) { + int gidx = lnext * E + eid; + pthread_mutex_lock(&g_pilot_mx); + int already_queued = m->is_queued[gidx]; + if (!already_queued) { + m->is_queued[gidx] = 1; + } + pthread_mutex_unlock(&g_pilot_mx); + + if (!already_queued) { + unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED); + unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE); + if (w2 - r2 < 4096) { + pilot_q[w2 & 4095].l = lnext; + pilot_q[w2 & 4095].e = eid; + __atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE); + } else { + pthread_mutex_lock(&g_pilot_mx); + m->is_queued[gidx] = 0; + pthread_mutex_unlock(&g_pilot_mx); + } + } + } + } + } + free(logits); +} + + /* generazione greedy. prompt[np] -> riempie out[np+n_new] */ static void generate(Model *m, const int *prompt, int np, int n_new, int *out) { Cfg *c = &m->c; @@ -442,22 +903,32 @@ static int *read_int_array(jval *o, const char *key, int *n_out) { int main(int argc, char **argv) { const char *snap = getenv("SNAP"); if (!snap) { fprintf(stderr, "set SNAP=\n"); return 1; } - int cap = argc > 1 ? atoi(argv[1]) : 16; - int bits = argc > 2 ? atoi(argv[2]) : 8; - if (bits < 2 || bits > 8) { /* expert storage is int8_t: bits>8 truncates in quantize_rows (#134). f32 mode is not implemented here — int8 is already token-exact vs the oracle. */ - fprintf(stderr, "quant_bits must be 2..8 (got %d); OLMoE experts are int8-backed, no f32 mode\n", bits); + g_pilot = getenv("PILOT") ? atoi(getenv("PILOT")) : 0; + g_wide = getenv("WIDE") ? atoi(getenv("WIDE")) : 1; + if (g_wide < 1) g_wide = 1; + if (g_wide > 4) g_wide = 4; + int hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0; + int cap = argc > 1 ? atoi(argv[1]) : 16; + int bits = argc > 2 ? atoi(argv[2]) : 8; + if (bits < 2 || bits > 8) { + fprintf(stderr, "quant_bits must be 2..8 (got %d)\n", bits); return 1; } const char *refpath = argc > 3 ? argv[3] : "ref.json"; - FILE *f = fopen(refpath, "rb"); if(!f){perror(refpath);return 1;} + float smooth = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f; + float conf = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f; + + printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d smooth=%.2f conf=%.2f ==\n", + cap, bits, g_pilot, g_wide, hot_n, smooth, conf); + + FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; } fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); - char *buf=malloc(n+1); if(fread(buf,1,n,f)!=(size_t)n){} buf[n]=0; fclose(f); + char *buf=malloc(n+1); if (fread(buf,1,n,f)!=(size_t)n) {} buf[n]=0; fclose(f); char *arena=NULL; jval *ref = json_parse(buf, &arena); int np, nfull; int *prompt = read_int_array(ref,"prompt_ids",&np); int *full = read_int_array(ref,"full_ids",&nfull); int n_new = nfull - np; - printf("== Streaming C engine, cache = %d experts/layer, experts @ %d-bit ==\n", cap, bits); Model m; model_init(&m, snap, cap, bits); printf("resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb()); @@ -487,6 +958,26 @@ int main(int argc, char **argv) { printf("\nPEAK RSS: %.2f GB\n", rss_gb()); printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hits, (unsigned long long)m.miss); + + + // Persistent Hot Pinning: save dynamic pinning if newly created + if (m.hot_pinned) { + char pinpath[512]; + snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap); + FILE *pinf_chk = fopen(pinpath, "rb"); + if (!pinf_chk) { + FILE *pinf_save = fopen(pinpath, "wb"); + if (pinf_save) { + size_t expected_size = (size_t)m.c.n_layers * m.c.n_experts; + fwrite(m.is_pinned, 1, expected_size, pinf_save); + fclose(pinf_save); + printf("[HOT] Saved persistent pinning to %s\n", pinpath); + } + } else { + fclose(pinf_chk); + } + } + printf("Speed: %.2f tok/s (%.1fs for %d tokens)\n", n_new/dt, dt, n_new); free(buf); free(arena); return 0; diff --git a/c/openai_server.py b/c/openai_server.py index d55fbc8..be21d91 100644 --- a/c/openai_server.py +++ b/c/openai_server.py @@ -271,6 +271,14 @@ def parse_tool_calls(reply, tools=None): salvaged.append(name) calls.append({"id": "call_" + uuid.uuid4().hex[:24], "type": "function", "function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)}}) + if tools and not calls and re.search(r"||", reply): + # Diagnosi per la #401: il client ha dichiarato i tools e il modello ha PROVATO la + # sintassi, ma il parse rigoroso non ha agganciato nulla (tipico output int4 storpiato). + # EN: #401 field diagnosis: tools were declared and the model attempted the syntax, + # EN: but the strict parse matched nothing (typically quantization-mangled output). + sys.stderr.write("[api] tools declared and tool-call markers present, but no call " + "parsed -- output may be quantization-mangled; try COLI_TOOL_SALVAGE=1\n") + sys.stderr.flush() text = _BOX_RE.sub("", reply) if THINK_CLOSE in text: text = text.split(THINK_CLOSE, 1)[1] @@ -366,6 +374,27 @@ def generation_options(body, limit): if body.get("n", 1) != 1: raise APIError(400, "Colibri currently supports `n=1` only.", "n", "unsupported_value") # `tools`/`functions` are handled by render_chat (declaration) + parse_tool_calls (output). + # Validate tools/functions structure early so malformed input fails with a clear error. + tools_raw = body.get("tools") or body.get("functions") + if tools_raw is not None: + if not isinstance(tools_raw, list): + raise APIError(400, "`tools` must be a non-empty array.", "tools", "invalid_value") + if not tools_raw: + raise APIError(400, "`tools` must be a non-empty array.", "tools", "invalid_value") + for idx, tool in enumerate(tools_raw): + if not isinstance(tool, dict): + raise APIError(400, f"Each tool must be an object, got {type(tool).__name__} at index {idx}.", + f"tools.{idx}", "invalid_value") + fn = tool.get("function", tool) if isinstance(tool, dict) else {} + if not isinstance(fn, dict): + raise APIError(400, f"Tool function must be an object at index {idx}.", + f"tools.{idx}.function", "invalid_value") + if not fn.get("name"): + raise APIError(400, f"Each tool must have a `name` at index {idx}.", + f"tools.{idx}.function.name", "invalid_value") + if not isinstance(fn["name"], str): + raise APIError(400, f"Tool `name` must be a string at index {idx}.", + f"tools.{idx}.function.name", "invalid_value") choice = body.get("tool_choice") if choice is not None: if isinstance(choice, str): @@ -406,7 +435,10 @@ def generation_options(body, limit): maximum = body.get("max_tokens") maximum_param = "max_tokens" if maximum is None: - maximum = min(256, limit) + # Client omitted max_tokens: honor the operator's configured budget (--max-tokens / + # --ngen), not an arbitrary 256 — `coli serve --ngen 32768` must mean 32768 (#382). + # Generation still ends at EOS, so this is a cap, not a target. + maximum = limit temperature = body.get("temperature") top_p = body.get("top_p") temperature = 0.7 if temperature is None else temperature @@ -852,7 +884,7 @@ class APIHandler(BaseHTTPRequestHandler): except OSError: pass - def generation(self, body, prompt, request_id, chat): + def generation(self, body, prompt, request_id, chat, tools=None, tool_choice=None): # COLI_DEBUG tees the engine transaction to stderr: 1 = decoded output stream only, # 2 = both sides (rendered prompt + output). render_chat already folds prior turns and # tool results into `prompt`, so level 2 is the full conversation the engine saw. @@ -864,8 +896,8 @@ class APIHandler(BaseHTTPRequestHandler): sys.stderr.write(f"\n===== PROMPT [{request_id}] =====\n{prompt}\n===== OUTPUT [{request_id}] =====\n") sys.stderr.flush() maximum, temperature, top_p = generation_options(body, self.server.max_tokens) - tools = (body.get("tools") or body.get("functions") or None) if chat else None - if body.get("tool_choice") == "none": + # tools and tool_choice come from chat_completion() already processed/filtered + if chat and tool_choice == "none": tools = None # client forbade tools: never surface tool_calls cache_slot = body.get("cache_slot") if (cache_slot is not None and @@ -1067,9 +1099,10 @@ class APIHandler(BaseHTTPRequestHandler): if not isinstance(enable_thinking, bool): raise APIError(400, "`enable_thinking` must be a boolean.", "enable_thinking") tools = body.get("tools") or body.get("functions") or None + tool_choice = body.get("tool_choice") prompt = render_chat(body.get("messages"), enable_thinking, reasoning_effort, tools, - body.get("tool_choice")) - self.generation(body, prompt, request_id, True) + tool_choice) + self.generation(body, prompt, request_id, True, tools, tool_choice) def completion(self, body, request_id): prompt = body.get("prompt") diff --git a/c/quant.h b/c/quant.h new file mode 100644 index 0000000..928de96 --- /dev/null +++ b/c/quant.h @@ -0,0 +1,672 @@ +/* quant.h — quantized matmul kernels (header-only, all functions static). + * Multi-architecture SIMD: AVX2 / AVX-512 / AVX-VNNI / ARM NEON / NEON-SDOT / + * NEON-i8mm / POWER VSX. Pure compute — no Model or QT dependency. */ +#ifndef COLI_QUANT_H +#define COLI_QUANT_H + +#include +#include +#include +#include +#include + +#ifdef _OPENMP +#include +#endif + +/* ---- SIMD includes -------------------------------------------------------- */ +#ifdef __AVX2__ +#include +static inline float hsum256(__m256 v){ + __m128 lo=_mm256_castps256_ps128(v), hi=_mm256_extractf128_ps(v,1); + lo=_mm_add_ps(lo,hi); __m128 sh=_mm_movehl_ps(lo,lo); lo=_mm_add_ps(lo,sh); + sh=_mm_shuffle_ps(lo,lo,1); lo=_mm_add_ss(lo,sh); return _mm_cvtss_f32(lo); +} +static inline int hsum256_i32(__m256i v){ + __m128i lo=_mm256_castsi256_si128(v), hi=_mm256_extracti128_si256(v,1); + lo=_mm_add_epi32(lo,hi); lo=_mm_hadd_epi32(lo,lo); lo=_mm_hadd_epi32(lo,lo); + return _mm_cvtsi128_si32(lo); +} +#endif +#if defined(__AVXVNNI__) && defined(__AVX2__) +static inline int hsum128_i32(__m128i v){ + v=_mm_hadd_epi32(v,v); v=_mm_hadd_epi32(v,v); return _mm_cvtsi128_si32(v); +} +#endif +#ifdef __ARM_NEON +#include +#endif +#ifdef __VSX__ +#include +#undef vector +#undef pixel +#undef bool +#endif + +/* ---- AVX-512 int4->float accumulator -------------------------------------- */ +#if defined(__AVX512F__) && defined(__AVX512BW__) +static int g_i4_acc512=1; +static inline float dot_i4f_avx512(const uint8_t *w,const float *x,int I){ + const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8); + __m512 acc0=_mm512_setzero_ps(),acc1=_mm512_setzero_ps(); int i=0; + for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi); + __m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8)); + __m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8)); + acc0=_mm512_fmadd_ps(_mm512_loadu_ps(x+i),w0,acc0); + acc1=_mm512_fmadd_ps(_mm512_loadu_ps(x+i+16),w1,acc1); + } + return _mm512_reduce_add_ps(_mm512_add_ps(acc0,acc1)); +} +static int i4_acc512_selftest(void){ + enum { N=224 }; uint8_t w[(N+1)/2]; float x[N]; + for(int i=0;i>1]=(uint8_t)(q+8); + else w[i>>1]|=(uint8_t)((q+8)<<4); + x[i]=(float)(((i*29+7)%101)-50)/37.f; + } + for(int n=32;n<=N;n+=32){ + float ref=0; for(int i=0;i>1]>>((i&1)*4))&15)-8); + float got=dot_i4f_avx512(w,x,n),tol=2e-5f*(1.f+fabsf(ref)); + if(fabsf(got-ref)>tol){ fprintf(stderr,"AVX512 i4 selftest n=%d: %.9g != %.9g\n",n,got,ref); return 0; } + } + return 1; +} +#endif + +/* ---- y[S,O] = x[S,I] @ W^T, W[O,I] f32 ---------------------------------- */ +static void matmul(float *y, const float *x, const float *W, int S, int I, int O){ + #pragma omp parallel for schedule(static) + for (int o=0;o>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i nib=_mm_unpacklo_epi8(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } + a=hsum256(acc); +#elif defined(__ARM_NEON) + const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); + float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); + for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); + uint8x8x2_t z=vzip_u8(vand_u8(by,m4), vshr_n_u8(by,4)); + int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[0]),b8)); + int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[1]),b8)); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); +#endif +#if defined(__AVX512F__) && defined(__AVX512BW__) + } +#endif + for(;i+1>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8; + a += xs[i]*(float)lo + xs[i+1]*(float)hi; } + if(i>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; } + y[(int64_t)s*O+o]=a*sc; } } +} + +/* ---- y[S,O] = x[S,I] @ W^T, W int4 packed + per-GROUP scales (fmt=4) ----- */ +static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const float *scale, + int S, int I, int O, int gs){ + int rb=(I+1)/2; int ng=(I+gs-1)/gs; + #pragma omp parallel for schedule(static) + for(int o=0;oI) glen=I-base; + float sc=scl[g]; + int i=base; +#ifdef __AVX2__ + const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8); + __m256 acc=_mm256_setzero_ps(); + for(; i+16<=base+glen; i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i nib=_mm_unpacklo_epi8(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } + a+=hsum256(acc)*sc; +#endif + for(; i>1]; + a+=(xs[i]*(float)((int)(byte&0xF)-8)+xs[i+1]*(float)((int)(byte>>4)-8))*sc; } + else { uint8_t byte=w[i>>1]; a+=xs[i]*(float)((int)(byte&0xF)-8)*sc; } + } + } + y[(int64_t)s*O+o]=a; + } + } +} + +/* ---- fused gate+up: one OMP dispatch for both matrices -------------------- */ +static void matmul_i4_pair(float *yg, float *yu, const float *x, + const uint8_t *qg, const float *sg, + const uint8_t *qu, const float *su, int I, int O){ + int rb=(I+1)/2; + #pragma omp parallel for schedule(static) + for(int z=0;z<2*O;z++){ + int o=z>1))); + __m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i nib=_mm_unpacklo_epi8(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); } + a=hsum256(acc); +#elif defined(__ARM_NEON) + const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8); + float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0); + for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); + uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4)); + int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8)); + int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8)); + ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); +#endif +#if defined(__AVX512F__) && defined(__AVX512BW__) + } +#endif + for(;i+1>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); } + if(i>1]&15)-8); + (z>2))); + __m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2); + __m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2); + __m128i lo=_mm_unpacklo_epi8(p0,p1), hi=_mm_unpacklo_epi8(p2,p3); + __m128i nib=_mm_unpacklo_epi16(lo,hi); + __m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b2)); + __m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b2)); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc); + acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); } + a=hsum256(acc); +#elif defined(__ARM_NEON) + const uint8x8_t m2v=vdup_n_u8(3); const int8x8_t b2v=vdup_n_s8(2); + float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0); + for(;i+16<=I;i+=16){ uint32_t wd; memcpy(&wd, w+(i>>2), 4); + uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd)); + uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v)); + uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6)); + uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0])); + int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[0]),b2v)); + int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[1]),b2v)); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0)))); + ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1)))); + ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); } + a=vaddvq_f32(vaddq_f32(ac0,ac1)); +#endif + for(;i>2]; int sh=(i&3)*2; a += xs[i]*(float)((int)((byte>>sh)&3)-2); } + y[(int64_t)s*O+o]=a*sc; } } +} + +/* ---- IDOT: integer dot kernels (int8-quantized activations) --------------- */ +#if defined(__AVX512VNNI__) && defined(__AVX512BW__) +#define IDOT_KERNEL "avx512-vnni" +#elif defined(__AVXVNNI__) && defined(__AVX2__) +#define IDOT_KERNEL "avx-vnni" +#elif defined(__AVX2__) +#define IDOT_KERNEL "avx2" +#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) +#define IDOT_KERNEL "neon-i8mm" +#elif defined(__ARM_NEON) +#define IDOT_KERNEL "neon" +#elif defined(__VSX__) +#define IDOT_KERNEL "vsx" +#else +#define IDOT_KERNEL "scalar" +#endif +static int g_idot=1; +#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD) +static int g_i4s=1; +#elif defined(__VSX__) +static int g_i4s=1; +#else +static int g_i4s=2; +#endif + +static inline float qrow_i8(const float *x, int8_t *q, int I){ + float amax=0; for(int i=0;iamax)amax=a; } + float s=amax/127.f; if(s<1e-12f) s=1e-12f; float inv=1.f/s; + for(int i=0;i>1))); + __m256i lo=_mm256_and_si256(by,m4v), hi=_mm256_and_si256(_mm256_srli_epi16(by,4),m4v); + __m256i z0=_mm256_unpacklo_epi8(lo,hi), z1=_mm256_unpackhi_epi8(lo,hi); + __m512i wv=_mm512_sub_epi8(_mm512_inserti64x4(_mm512_castsi256_si512(z0),z1,1),b8v); + __m512i xv=_mm512_permutexvar_epi64(xidx,_mm512_loadu_si512((const void*)(x+i))); + __mmask64 neg=_mm512_movepi8_mask(wv); + __m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv); + acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs); + } + sum=_mm512_reduce_add_epi32(acc); +#elif defined(__AVXVNNI__) && defined(__AVX2__) + const __m128i m4=_mm_set1_epi8(0x0F); const __m128i b8=_mm_set1_epi8(8); + __m128i acc=_mm_setzero_si128(); + for(;i+32<=I;i+=32){ + __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); + __m128i w0=_mm_sub_epi8(n0,b8), w1=_mm_sub_epi8(n1,b8); + __m128i x0=_mm_loadu_si128((const __m128i*)(x+i)); + __m128i x1=_mm_loadu_si128((const __m128i*)(x+i+16)); + acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0)); + acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1)); + } + sum=hsum128_i32(acc); +#elif defined(__AVX2__) + const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi8(8); + const __m256i ones=_mm256_set1_epi16(1); + __m256i acc=_mm256_setzero_si256(); + for(;i+32<=I;i+=32){ + __m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); + __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4); + __m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); + __m256i wv=_mm256_sub_epi8(_mm256_set_m128i(n1,n0),b8); + __m256i xv=_mm256_loadu_si256((const __m256i*)(x+i)); + __m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv)); + acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones)); + } + sum=hsum256_i32(acc); +#elif defined(__ARM_NEON) + const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8); +#if defined(__ARM_FEATURE_DOTPROD) + int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); + for(;i+64<=I;i+=64){ + uint8x16_t byA=vld1q_u8(w4+(i>>1)), byB=vld1q_u8(w4+(i>>1)+16); + uint8x16x2_t zA=vzipq_u8(vandq_u8(byA,m4q), vshrq_n_u8(byA,4)); + uint8x16x2_t zB=vzipq_u8(vandq_u8(byB,m4q), vshrq_n_u8(byB,4)); + a0=vdotq_s32(a0,vsubq_s8(vreinterpretq_s8_u8(zA.val[0]),b8q),vld1q_s8(x+i)); + a1=vdotq_s32(a1,vsubq_s8(vreinterpretq_s8_u8(zA.val[1]),b8q),vld1q_s8(x+i+16)); + a2=vdotq_s32(a2,vsubq_s8(vreinterpretq_s8_u8(zB.val[0]),b8q),vld1q_s8(x+i+32)); + a3=vdotq_s32(a3,vsubq_s8(vreinterpretq_s8_u8(zB.val[1]),b8q),vld1q_s8(x+i+48)); + } + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + for(;i+32<=I;i+=32){ + uint8x16_t by=vld1q_u8(w4+(i>>1)); + uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); + acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q),vld1q_s8(x+i)); + acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q),vld1q_s8(x+i+16)); + } + sum=vaddvq_s32(acc); +#else + int32x4_t acc=vdupq_n_s32(0); + for(;i+32<=I;i+=32){ + uint8x16_t by=vld1q_u8(w4+(i>>1)); + uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); + int8x16_t w0=vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q); + int8x16_t w1=vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q); + int8x16_t x0=vld1q_s8(x+i), x1=vld1q_s8(x+i+16); + int16x8_t p=vmull_s8(vget_low_s8(w0),vget_low_s8(x0)); + p=vmlal_s8(p,vget_high_s8(w0),vget_high_s8(x0)); + acc=vpadalq_s16(acc,p); + p=vmull_s8(vget_low_s8(w1),vget_low_s8(x1)); + p=vmlal_s8(p,vget_high_s8(w1),vget_high_s8(x1)); + acc=vpadalq_s16(acc,p); + } + sum=vaddvq_s32(acc); +#endif +#elif defined(__VSX__) + const __vector unsigned char m4v=vec_splats((unsigned char)0x0F); + const __vector unsigned char sh4=vec_splats((unsigned char)4); + const __vector signed char b8v=vec_splats((signed char)8); + const __vector signed char vz=vec_splats((signed char)0); + __vector signed int acc=vec_splats(0); + for(;i+32<=I;i+=32){ + __vector unsigned char by=vec_xl(0,w4+(i>>1)); + __vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4); + __vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v); + __vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v); + __vector signed char x0=vec_xl(0,(const signed char*)(x+i)); + __vector signed char x1=vec_xl(0,(const signed char*)(x+i+16)); + __vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz); + acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0), + (__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc); + acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1), + (__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc); + } + sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); +#endif + for(;i+1>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } + if(i>1]; sum+=((int)(b&0xF)-8)*x[i]; } + return sum; +} + +/* ---- ARM i8mm SMMLA tiled kernels ---------------------------------------- */ +#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) +static inline int32x4_t mm_tile16(int32x4_t acc, int8x16_t wo, int8x16_t wo1, + int8x16_t xs, int8x16_t xs1){ + acc=vmmlaq_s32(acc, vcombine_s8(vget_low_s8(wo), vget_low_s8(wo1)), + vcombine_s8(vget_low_s8(xs), vget_low_s8(xs1))); + return vmmlaq_s32(acc, vcombine_s8(vget_high_s8(wo), vget_high_s8(wo1)), + vcombine_s8(vget_high_s8(xs), vget_high_s8(xs1))); +} +static void matmul_q_idot_mm(float *y, const int8_t *xq, const float *sx, const int8_t *q, + const float *scale, int S, int I, int O){ + #pragma omp parallel for schedule(static) + for(int o=0;o<(O&~1);o+=2){ + const int8_t *wo=q+(int64_t)o*I, *wo1=q+(int64_t)(o+1)*I; + float sc0=scale[o], sc1=scale[o+1]; + for(int s=0;s<(S&~1);s+=2){ + const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I; + int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0; + for(;i+64<=I;i+=64){ + a0=mm_tile16(a0,vld1q_s8(wo+i), vld1q_s8(wo1+i), vld1q_s8(xs+i), vld1q_s8(xs1+i)); + a1=mm_tile16(a1,vld1q_s8(wo+i+16),vld1q_s8(wo1+i+16),vld1q_s8(xs+i+16),vld1q_s8(xs1+i+16)); + a2=mm_tile16(a2,vld1q_s8(wo+i+32),vld1q_s8(wo1+i+32),vld1q_s8(xs+i+32),vld1q_s8(xs1+i+32)); + a3=mm_tile16(a3,vld1q_s8(wo+i+48),vld1q_s8(wo1+i+48),vld1q_s8(xs+i+48),vld1q_s8(xs1+i+48)); + } + for(;i+16<=I;i+=16) + a0=mm_tile16(a0,vld1q_s8(wo+i),vld1q_s8(wo1+i),vld1q_s8(xs+i),vld1q_s8(xs1+i)); + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); + int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); + for(;i>1)), byo1=vld1q_u8(wo1+(i>>1)); + uint8x16_t cyo=vld1q_u8(wo+(i>>1)+16), cyo1=vld1q_u8(wo1+(i>>1)+16); + uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); + uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); + uint8x16x2_t ko =vzipq_u8(vandq_u8(cyo, m4q), vshrq_n_u8(cyo, 4)); + uint8x16x2_t ko1=vzipq_u8(vandq_u8(cyo1,m4q), vshrq_n_u8(cyo1,4)); + a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), + vld1q_s8(xs+i), vld1q_s8(xs1+i)); + a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), + vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); + a2=mm_tile16(a2, vsubq_s8(vreinterpretq_s8_u8(ko.val[0]),b8q), + vsubq_s8(vreinterpretq_s8_u8(ko1.val[0]),b8q), + vld1q_s8(xs+i+32), vld1q_s8(xs1+i+32)); + a3=mm_tile16(a3, vsubq_s8(vreinterpretq_s8_u8(ko.val[1]),b8q), + vsubq_s8(vreinterpretq_s8_u8(ko1.val[1]),b8q), + vld1q_s8(xs+i+48), vld1q_s8(xs1+i+48)); + } + for(;i+32<=I;i+=32){ + uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1)); + uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4)); + uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4)); + a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q), + vld1q_s8(xs+i), vld1q_s8(xs1+i)); + a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q), + vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q), + vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16)); + } + int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3)); + int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1); + int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3); + for(;i+1>1], bo1=wo1[i>>1]; + int a0=(int)(bo&0xF)-8, a1=(int)(bo>>4)-8, b0=(int)(bo1&0xF)-8, b1=(int)(bo1>>4)-8; + int u0=xs[i],u1=xs[i+1],v0=xs1[i],v1=xs1[i+1]; + d00+=a0*u0+a1*u1; d01+=a0*v0+a1*v1; d10+=b0*u0+b1*u1; d11+=b0*v0+b1*v1; } + if(i>1], bo1=wo1[i>>1]; + int a0=(int)(bo&0xF)-8, b0=(int)(bo1&0xF)-8; + d00+=a0*xs[i]; d01+=a0*xs1[i]; d10+=b0*xs[i]; d11+=b0*xs1[i]; } + y[(int64_t)s*O+o] =(float)d00*sc0*sx[s]; + y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s]; + y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1]; + y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1]; + } + if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I; + y[(int64_t)s*O+o] =(float)dot_i4i8(wo, xs,I)*sc0*sx[s]; + y[(int64_t)s*O+(o+1)]=(float)dot_i4i8(wo1,xs,I)*sc1*sx[s]; } + } + if(O&1){ int o=O-1; const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o]; + #pragma omp parallel for schedule(static) + for(int s=0;s=2){ matmul_q_idot_mm(y,xq,sx,q,scale,S,I,O); return; } +#endif + #pragma omp parallel for schedule(static) + for(int o=0;o=2){ matmul_i4_idot_mm(y,xq,sx,q4,scale,S,I,O); return; } +#endif + #pragma omp parallel for schedule(static) + for(int o=0;og_qscratch.xq_cap){ + int8_t *p=realloc(g_qscratch.xq,xn); + if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); } + g_qscratch.xq=p; g_qscratch.xq_cap=xn; + } + if(sn>g_qscratch.sx_cap){ + float *p=realloc(g_qscratch.sx,sn*sizeof(float)); + if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); } + g_qscratch.sx=p; g_qscratch.sx_cap=sn; + } + *xq=g_qscratch.xq; *sx=g_qscratch.sx; +} + +/* ---- f32 -> quantized packing --------------------------------------------- */ +static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits){ + int qmax=(1<<(bits-1))-1; + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; + int8_t *qr=q+(int64_t)o*I; + for(int i=0;iqmax)v=qmax; if(v<-qmax-1)v=-qmax-1; qr[i]=(int8_t)v; } + } +} +static void pack_int4(const float *w, uint8_t *q4, float *scale, int O, int I, int bits){ + int qmax=(1<<(bits-1))-1, rb=(I+1)/2; + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; + uint8_t *qr=q4+(int64_t)o*rb; + for(int i=0;iqmax)v0=qmax; if(v0<-8)v0=-8; + int v1=0; if(i+1qmax)v1=qmax; if(v1<-8)v1=-8; } + qr[i>>1] = (uint8_t)((v0+8) | ((v1+8)<<4)); + } + } +} +static void pack_int2(const float *w, uint8_t *q2, float *scale, int O, int I, int bits){ + int qmax=(1<<(bits-1))-1, rb=(I+3)/4; + #pragma omp parallel for schedule(static) + for(int o=0;oamax)amax=a; } + float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s; + uint8_t *qr=q2+(int64_t)o*rb; + for(int i=0;iqmax)v=qmax; if(v<-2)v=-2; byte|=(uint8_t)((v+2)<<(k*2)); } + qr[i>>2]=byte; + } + } +} + +#endif /* COLI_QUANT_H */ diff --git a/c/ref_olmoe_real.json b/c/ref_olmoe_real.json new file mode 100644 index 0000000..b271e07 --- /dev/null +++ b/c/ref_olmoe_real.json @@ -0,0 +1,28 @@ +{ + "prompt_ids": [ + 510, + 5347, + 273, + 6181, + 310 + ], + "full_ids": [ + 510, + 5347, + 273, + 6181, + 310, + 7785, + 15, + 187, + 187, + 510, + 3565, + 3448, + 273, + 6181, + 310, + 5112, + 15 + ] +} \ No newline at end of file diff --git a/c/resource_plan.py b/c/resource_plan.py index 6b3f2fe..1374a3b 100644 --- a/c/resource_plan.py +++ b/c/resource_plan.py @@ -166,14 +166,33 @@ def discover_gpus(): return devices +def _physical_cores_warn(message): + """Visibility for a mis-detected core count: a silent "1" here becomes + OMP_NUM_THREADS=1 and pins the whole run to a single core (#325). Emit on + stderr so it surfaces in the [PLAN]/[OMP] stream without being swallowed.""" + print(f"[plan] warning: {message}", file=sys.stderr) + + def physical_cpu_count(): + """Number of physical CPU cores (not SMT siblings). + + Per-expert matmul regions are tiny and back-to-back; two SMT siblings share + one AVX-512 unit and contend, so logical (SMT) counts over-subscribe and + hurt throughput. We want true physical cores. A silent 1 here propagates to + OMP_NUM_THREADS=1 and pins the run to one core (#325), so every fallback + must be visible, never just ``or 1``. + """ if sys.platform == "win32": - # os.cpu_count() conta i processori logici (SMT): 2 thread/core saturano - # le unita' AVX-512 e peggiorano il matmul. Contiamo i core fisici veri - # con GetLogicalProcessorInformationEx(RelationProcessorCore). + # Contiamo i core fisici veri con GetLogicalProcessorInformationEx + # (RelationProcessorCore). Le firme vanno dichiarate: su Python a 64 bit + # una WinAPI non dichiarata ritorna c_int (32 bit) e riceve i puntatori + # come c_int di default, quindi il probe puo' fallire silenziosamente. try: import ctypes k32 = ctypes.windll.kernel32 + k32.GetLogicalProcessorInformationEx.argtypes = [ + ctypes.c_uint, ctypes.c_void_p, ctypes.POINTER(ctypes.c_ulong)] + k32.GetLogicalProcessorInformationEx.restype = ctypes.c_int need = ctypes.c_ulong(0) k32.GetLogicalProcessorInformationEx(0, None, ctypes.byref(need)) buf = (ctypes.c_char * need.value)() @@ -189,18 +208,133 @@ def physical_cpu_count(): off += size if cores: return cores - except (OSError, ValueError, AttributeError): - pass + _physical_cores_warn("GetLogicalProcessorInformationEx returned no cores") + except (OSError, ValueError, AttributeError) as error: + _physical_cores_warn(f"Windows core probe failed: {error}") try: + # Ask lscpu for exactly core,socket and dedupe on (core, socket). + # Counting un-deduplicated rows would return logical threads (SMT), + # which was the original over-subscription bug. Empty fields ("-") + # mark an offline core/socket and fail int() -> skipped. + # + # Column layout robustness: `lscpu -p=` emits *exactly* the + # requested columns (no CPU prefix), while bare `lscpu -p` prepends + # CPU. We requested two columns, but take the LAST TWO fields so the + # parser stays correct whether or not a CPU column is present + # (JustVugg review: the previous fields[1]/fields[2] indexing assumed + # a 3-column layout and regressed 2-column output to the logical + # count -- the opposite of the fix). result = subprocess.run(["lscpu", "-p=core,socket"], text=True, capture_output=True, check=True, timeout=5) - cores = {tuple(map(int, line.split(","))) for line in result.stdout.splitlines() - if line and not line.startswith("#")} + cores = set() + for line in result.stdout.splitlines(): + if not line or line.startswith("#"): + continue + fields = line.split(",") + if len(fields) < 2: + continue + try: + core, socket = int(fields[-2]), int(fields[-1]) + except ValueError: + continue # "-" for an offline core/socket + cores.add((core, socket)) if cores: return len(cores) + except (OSError, ValueError, subprocess.SubprocessError) as error: + _physical_cores_warn(f"lscpu core probe failed: {error}") + logical = os.cpu_count() + if not logical: + _physical_cores_warn( + "could not detect any CPU cores; falling back to 1. " + "Set OMP_NUM_THREADS manually to fix single-core decode (#325).") + return 1 + _physical_cores_warn( + f"physical-core probes unavailable; using {logical} logical CPUs " + f"(SMT may over-subscribe). Set OMP_NUM_THREADS to physical cores if slow.") + return logical + + +def _resolve_physical_cores(physical_cpus): + """Coerce the build_plan() physical-core argument to a sane positive int. + + A None/0/None-ish value reaching here means physical_cpu_count() already + warned; clamp to 1 (so the engine always gets a positive team size) but keep + that clamp visible rather than silently masking it as the old ``max(1, int())`` + did (#325).""" + try: + count = int(physical_cpus or 0) + except (TypeError, ValueError): + count = 0 + if count < 1: + _physical_cores_warn( + "physical core count resolved to 0; defaulting to 1. " + "Set OMP_NUM_THREADS to fix single-core decode (#325).") + return 1 + return count + + +def cpu_socket_count(): + """Return the number of physical CPU sockets visible to this process.""" + if not sys.platform.startswith("linux"): + return 1 + try: + result = subprocess.run(["lscpu", "-p=socket"], text=True, + capture_output=True, check=True, timeout=5) + sockets = {int(line) for line in result.stdout.splitlines() + if line and not line.startswith("#")} + if sockets: + return len(sockets) except (OSError, ValueError, subprocess.SubprocessError): pass - return os.cpu_count() or 1 + return 1 + + +def _auto_tune(bottleneck_class, projected_hit, gpus, cpu_sockets, plan_has_metal): + """Derive tuning knobs from the bottleneck classification.""" + tune = {} + has_gpu = bool(gpus) + n_gpu = len(gpus) + + # MTP: costs more than it saves when compute-bound (#389 measured 42% loss) + if bottleneck_class == "compute": + tune["DRAFT"] = {"value": "0", + "reason": "compute-bound: MTP batch overhead exceeds yield"} + elif bottleneck_class == "disk" and projected_hit < 0.90: + tune["DRAFT"] = {"value": "0", + "reason": "low hit rate: MTP widens expert union, adds disk reads"} + # otherwise leave DRAFT unset (engine default: auto) + + # PIPE: resident pipeline mode depends on GPU count + if has_gpu and n_gpu == 1: + tune["COLI_CUDA_PIPE"] = {"value": "1", + "reason": "single GPU: S=1 pipeline gate"} + elif has_gpu and n_gpu > 1: + tune["COLI_CUDA_PIPE"] = {"value": "2", + "reason": "multi-GPU: residual stays on-device across layers"} + elif not has_gpu and bottleneck_class == "disk": + tune["PIPE"] = {"value": "1", + "reason": "overlap disk reads with resident expert compute"} + + # NUMA: selective interleave for GPU hosts, blanket hint for CPU-only + if cpu_sockets > 1 and has_gpu: + tune["COLI_NUMA"] = {"value": "1", + "reason": "multi-socket + GPU: interleave expert slabs, protect DMA buffers"} + elif cpu_sockets > 1 and not has_gpu: + tune["COLI_NUMA"] = {"value": "1", + "reason": "multi-socket CPU-only: interleave expert slabs across nodes"} + tune["_numa_hint"] = "numactl --interleave=all may perform better on CPU-only hosts" + + # OMP: kill hot-thread spin when GPU/Metal owns the power budget + if plan_has_metal: + tune["COLI_NO_OMP_TUNE"] = {"value": "1", + "reason": "Metal: OMP spin-wait steals GPU power budget"} + + # PIN: fully resident if RAM allows and no GPU tier competes + if projected_hit >= 0.99 and not has_gpu: + tune["PIN_GB"] = {"value": "all", + "reason": "enough RAM for full expert residency"} + + return tune POLICIES = { @@ -212,11 +346,12 @@ POLICIES = { def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, available_memory=None, available_disk=None, gpus=None, - policy="quality", physical_cpus=None): + policy="quality", physical_cpus=None, cpu_sockets=None): if policy not in POLICIES: raise ValueError(f"unknown policy: {policy}") info = analyze_model(model) physical_cpus = physical_cpu_count() if physical_cpus is None else physical_cpus + cpu_sockets = cpu_socket_count() if cpu_sockets is None else cpu_sockets cfg = info["config"] available_memory = memory_available() if available_memory is None else available_memory if available_disk is None: @@ -273,19 +408,36 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, if cold_bytes: warnings.append("cold expert misses may reach disk; normal decode speed depends on hit rate") + total_expert = info["expert_bytes"] + resident_expert = hot_bytes + warm_bytes + projected_hit = resident_expert / total_expert if total_expert else 1.0 + if cold_bytes: bottleneck = "disk expert misses" - elif warm_bytes: - bottleneck = "CPU expert compute and RAM bandwidth" + bottleneck_class = "disk" + elif warm_bytes and gpus: + bottleneck = "CPU expert tail and GPU compute" + bottleneck_class = "mixed" + elif projected_hit >= 0.99: + if gpus: + bottleneck = "GPU compute and interconnect" + else: + bottleneck = "CPU expert compute (fully resident)" + bottleneck_class = "compute" else: - bottleneck = "GPU compute and interconnect" + bottleneck = "CPU expert compute and RAM bandwidth" + bottleneck_class = "memory" + + tune = _auto_tune(bottleneck_class, projected_hit, gpus, cpu_sockets, + plan_has_metal=False) return { "version": 2, "policy": {"name": policy, **POLICIES[policy], "quality_preserving": policy != "experimental-fast"}, "model": {key: value for key, value in info.items() if key != "config"}, - "cpu": {"physical_cores": max(1, int(physical_cpus)), + "cpu": {"physical_cores": _resolve_physical_cores(physical_cpus), + "sockets": max(1, int(cpu_sockets)), "thread_policy": "physical-cores"}, "tiers": { "disk": {"role": "cold-backing", "model_bytes": info["model_bytes"], @@ -299,6 +451,9 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, "expert_capacity": vram_experts, "requires_host_backing": False}, }, "expected_bottleneck": bottleneck, + "bottleneck_class": bottleneck_class, + "projected_hit_rate": round(projected_hit, 4), + "tune": tune, "decisions": [ {"target": "VRAM", "reason": "profile-ranked hot experts"}, {"target": "RAM", "reason": "warm experts execute on CPU without quality loss"}, @@ -313,11 +468,23 @@ def environment_for_plan(plan, env=None, cuda_enabled=True): result = dict(env or {}) result.setdefault("COLI_POLICY", plan["policy"]["name"]) result.setdefault("OMP_NUM_THREADS", str(plan["cpu"]["physical_cores"])) - if sys.platform != "win32": - # la libgomp di MinGW non supporta l'affinity su Windows - # ("Affinity not supported on this configuration"): non impostarle li'. - result.setdefault("OMP_PROC_BIND", "spread") - result.setdefault("OMP_PLACES", "cores") + # NOTE: we intentionally do NOT set OMP_PROC_BIND / OMP_PLACES here. + # The engine's own hot-thread tuning (glm.c main(), the COLI_OMP_TUNED + # self-exec) sets OMP_PROC_BIND=close with overwrite=0 -- it prefers + # packing the team onto adjacent cores for the tiny back-to-back per-expert + # matmuls. Pre-setting OMP_PROC_BIND=spread here ran first and won (the + # engine's overwrite=0 setenv could not override an already-set var), and + # spread + OMP_PLACES=cores collapsed the team to one CPU on some libgomp / + # multi-socket topologies (#325: --auto-tier pinned decode to 1 core on a + # 64-core box even with OMP_NUM_THREADS=64). Leaving affinity to the engine + # makes --auto-tier match the plain (working) path. A user who wants a + # specific policy can still set OMP_PROC_BIND/OMP_PLACES in the environment + # themselves -- setdefault above only covers OMP_NUM_THREADS. + tune = plan.get("tune", {}) + for key, entry in tune.items(): + if key.startswith("_"): + continue + result.setdefault(key, entry["value"]) if plan["policy"]["name"] == "balanced": result.setdefault("REPIN", "64") ram = plan["tiers"]["ram"] @@ -364,5 +531,18 @@ def format_plan(plan): else: lines.append("VRAM no NVIDIA device detected · CPU path") lines.append(f"limit {plan['expected_bottleneck']}") + hit = plan.get("projected_hit_rate", 0) + lines.append(f"hit {hit:.0%} projected expert residency") + tune = plan.get("tune", {}) + if tune: + lines.append("") + lines.append("auto-tune:") + for key, entry in tune.items(): + if key.startswith("_"): + continue + lines.append(f" {key}={entry['value']:12s} {entry['reason']}") + hint = tune.get("_numa_hint") + if hint: + lines.append(f" hint: {hint}") lines.extend(f"warn {warning}" for warning in plan["warnings"]) return "\n".join(lines) diff --git a/c/sample.h b/c/sample.h new file mode 100644 index 0000000..414c621 --- /dev/null +++ b/c/sample.h @@ -0,0 +1,153 @@ +/* sample.h — sampling (temperature + nucleus) and stop-set management. + * Header-only: all functions are static — include from the main engine file. */ +#ifndef SAMPLE_H +#define SAMPLE_H + +#include +#include +#include +#include "tok.h" + +/* ---- RNG (xorshift64*) -------------------------------------------------- */ +static uint64_t g_rng = 0x9E3779B97F4A7C15ULL; +static inline double rndu(void){ + g_rng ^= g_rng << 13; g_rng ^= g_rng >> 7; g_rng ^= g_rng << 17; + return (double)(g_rng >> 11) * (1.0 / 9007199254740992.0); +} + +/* ---- argmax over a float vector ----------------------------------------- */ +static inline int argmax_v(const float *lo, int V){ + int b=-1; float bv=-INFINITY; + for(int i=0;ibv){ bv=x; b=i; } } + return b<0?0:b; +} + +/* ---- distribution buffers (reused, single-threaded decode) --------------- */ +static float *g_pbuf = NULL; +static int *g_pidx = NULL; + +/* sift-down on max-heap in h[0..n), key = g_pbuf[h[i]] (#335: partial top-p). + * "hole" variant: carries the root value and deposits only at the end, so + * heapify is O(V) and each pop is O(log n) without qsort on the full vocab. */ +static void topp_siftdown(int *h, int n, int i){ + int iv = h[i]; float kv = g_pbuf[iv]; + for (;;) { + int l = 2*i + 1; + if (l >= n) break; + int b = l; if (l+1 < n && g_pbuf[h[l+1]] > g_pbuf[h[l]]) b = l+1; + if (g_pbuf[h[b]] <= kv) break; + h[i] = h[b]; i = b; + } + h[i] = iv; +} + +/* build the target distribution in g_pbuf: softmax(lo/temp) truncated to + * top-p g_nuc. Invariant: g_pbuf stays indexed by token-id (never reordered); + * the truncated tail is zeroed (dist_sample reads by id directly). + * Requires: g_temp, g_nuc, falloc() — declared in the main engine file. */ +static void dist_build(const float *lo, int V){ + if (!g_pbuf) { g_pbuf = falloc(V); g_pidx = malloc(V * sizeof(int)); } + int mxi = -1; float mx = 0; + for (int i = 0; i < V; i++) + if (isfinite(lo[i]) && (mxi < 0 || lo[i] > mx)) { mx = lo[i]; mxi = i; } + double s = 0; float invt = 1.f / (g_temp > 1e-4f ? g_temp : 1e-4f); + if (mxi >= 0) { + for (int i = 0; i < V; i++) { + g_pbuf[i] = isfinite(lo[i]) ? expf((lo[i] - mx) * invt) : 0.f; + s += g_pbuf[i]; + } + } + if (mxi < 0 || !isfinite(s) || s <= 0.0) { + static int warned = 0; + if (!warned) { warned = 1; fprintf(stderr, + "[SAMPLE] warning: non-finite logits (NaN/Inf) — falling back to argmax; " + "output may be degraded. This usually means a numerical blow-up upstream.\n"); } + int a = (mxi >= 0) ? mxi : 0; + for (int i = 0; i < V; i++) g_pbuf[i] = 0.f; + g_pbuf[a] = 1.f; + return; + } + for (int i = 0; i < V; i++) g_pbuf[i] /= (float)s; + if (g_nuc > 0 && g_nuc < 1.f) { + for (int i = 0; i < V; i++) g_pidx[i] = i; + for (int i = V/2-1; i >= 0; i--) topp_siftdown(g_pidx, V, i); + double s2 = 0, cum = 0; int out = V; + do { + int root = g_pidx[0]; + g_pidx[0] = g_pidx[--out]; g_pidx[out] = root; + s2 += g_pbuf[root]; cum += g_pbuf[root]; + if (out > 0) topp_siftdown(g_pidx, out, 0); + } while (cum < g_nuc && out > 0); + for (int i = 0; i < out; i++) g_pbuf[g_pidx[i]] = 0; + float s2f = (float)s2; + for (int i = out; i < V; i++) g_pbuf[g_pidx[i]] /= s2f; + } +} + +/* sample from g_pbuf; ban>=0 excludes that token (renormalizing on the fly) */ +static int dist_sample(int V, int ban){ + double z = 1.0 - (ban >= 0 ? g_pbuf[ban] : 0.0); + if (z <= 1e-12) z = 1e-12; + double u = rndu() * z, cum = 0; + for (int i = 0; i < V; i++) { if (i == ban) continue; cum += g_pbuf[i]; if (cum >= u) return i; } + for (int i = V-1; i >= 0; i--) if (i != ban && g_pbuf[i] > 0) return i; + return 0; +} + +/* next token from logits: greedy if g_temp<=0, sampling otherwise. + * ban = token excluded because it was rejected by speculative verification. */ +static int pick_tok(const float *lo, int V, int ban){ + if (g_temp <= 0) return argmax_v(lo, V); + dist_build(lo, V); + return dist_sample(V, ban); +} + +/* ---- stop set ----------------------------------------------------------- */ +static int g_stop[64], g_nstop = 0; +static inline int is_stop(int t){ + for (int i = 0; i < g_nstop; i++) if (t == g_stop[i]) return 1; + return 0; +} +/* T=NULL -> config stops only (validation/oracle, where the tokenizer is not needed). */ +static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){ + g_nstop = 0; + for (int i = 0; i < c->n_stop && g_nstop < 64; i++) g_stop[g_nstop++] = c->stop_ids[i]; + if (tok_eos >= 0 && !is_stop(tok_eos) && g_nstop < 64) g_stop[g_nstop++] = tok_eos; + int nsp = 0; + if (T) for (int id = 0; id < T->n_ids && g_nstop < 64; id++) + if (T->id_special[id] && !is_stop(id)) { g_stop[g_nstop++] = id; nsp++; } + /* #401: in serve mode keep ONLY <|endoftext|>. Role markers <|user|>/<|observation|> + * (config stops + tokenizer special set) are boundaries the Python server owns; as + * hard stops they cut generation the moment the model opens a block, + * because int4 argmax noise picks a stop-token ID over the correct '<' token. */ + if (getenv("SERVE") && tok_eos >= 0) { + int kept = 0; + for (int i = 0; i < g_nstop; i++) if (g_stop[i] == tok_eos) g_stop[kept++] = g_stop[i]; + if (kept < g_nstop) fprintf(stderr, "[stop] serve mode: filtered %d non-EOS stop tokens (tool-call safety, #401)\n", g_nstop - kept); + g_nstop = kept; nsp = 0; + } + fprintf(stderr, "[stop] %d stop tokens:", g_nstop); + for (int i = 0; i < g_nstop; i++) fprintf(stderr, " %d", g_stop[i]); + if (nsp) fprintf(stderr, " (%d from the tokenizer's special set)", nsp); + fprintf(stderr, "\n"); +} +static void stops_arm(const Cfg *c, int tok_eos){ stops_arm_tok(c, tok_eos, NULL); } + +/* ---- log-prob of a target token given the logit vector ------------------- */ +static double logprob_target(const float *lo, int V, int target, int *am){ + float mx = lo[0]; int best = 0; + for (int i = 1; i < V; i++) if (lo[i] > mx) { mx = lo[i]; best = i; } + double se = 0; + for (int i = 0; i < V; i++) se += exp((double)lo[i] - mx); + if (am) *am = (best == target); + return (double)(lo[target] - mx) - log(se); +} + +/* "glm" in model_type, case-insensitive */ +static int mt_is_glm(const char *s){ + if (s) for (; *s; s++) + if ((s[0]|32) == 'g' && (s[1]|32) == 'l' && (s[2]|32) == 'm') return 1; + return 0; +} + +#endif /* SAMPLE_H */ diff --git a/c/setup.sh b/c/setup.sh index e31e60a..dcb851d 100755 --- a/c/setup.sh +++ b/c/setup.sh @@ -32,11 +32,11 @@ esac # 2) build: nativa (veloce, per QUESTA macchina). Per un binario da distribuire: make portable echo " building (ARCH=${ARCH:-native})…" -make -s glm ARCH="${ARCH:-native}" +make -s colibri ARCH="${ARCH:-native}" # 3) self-test sull'oracolo tiny, se presente if [ -d glm_tiny ] && [ -f ref_glm.json ]; then - r=$(SNAP=./glm_tiny TF=1 ./glm 64 16 16 2>/dev/null | grep -oE "[0-9]+/[0-9]+ positions" || true) + r=$(SNAP=./glm_tiny TF=1 ./colibri 64 16 16 2>/dev/null | grep -oE "[0-9]+/[0-9]+ positions" || true) echo " engine self-test: ${r:-?} (expected 32/32)" fi diff --git a/c/st.h b/c/st.h index 6b4a710..a2c2c3f 100644 --- a/c/st.h +++ b/c/st.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -104,6 +105,38 @@ static int st_direct_fd(shards *S, int fd) { } /* indicizza tutti i model-*.safetensors in snap_dir */ +/* pread completo: chunk-loop (una singola pread si ferma a ~2^31 byte su Linux + * — i tensori bf16 grandi la superano), riprova su EINTR e riporta un errore + * ONESTO: perror stampava "Success" su una short-read (errno resta 0), lo + * stesso sintomo corretto in glm.c per #236. ST_PREAD_CHUNK e' sovrascrivibile + * per i test. EN: full pread — chunk loop (one pread caps at ~2^31 bytes and + * big bf16 tensors exceed it), EINTR retry, honest short-read errors. + * Exits on failure, like every st.h reader. */ +#ifndef ST_PREAD_CHUNK +#define ST_PREAD_CHUNK (1u << 30) +#endif +static void st_pread_full(int fd, void *buf, int64_t n, int64_t off, const char *tag) { + char *p = (char *)buf; + int64_t got = 0; + while (got < n) { + int64_t want = n - got; + if (want > (int64_t)ST_PREAD_CHUNK) want = ST_PREAD_CHUNK; + ssize_t r = pread(fd, p + got, (size_t)want, off + got); + if (r < 0) { + if (errno == EINTR) continue; + fprintf(stderr, "%s: %s (off %lld, %lld/%lld bytes)\n", tag, strerror(errno), + (long long)off, (long long)got, (long long)n); + exit(1); + } + if (r == 0) { + fprintf(stderr, "%s: short read at EOF (off %lld, %lld/%lld bytes) — truncated file?\n", + tag, (long long)off, (long long)got, (long long)n); + exit(1); + } + got += r; + } +} + static void st_init(shards *S, const char *snap_dir) { memset(S, 0, sizeof(*S)); S->cap = 4096; S->t = calloc(S->cap, sizeof(st_tensor)); @@ -128,7 +161,7 @@ static void st_init(shards *S, const char *snap_dir) { if (fstat(fd, &sst) != 0) { perror("fstat shard"); exit(1); } int64_t fsz = (int64_t)sst.st_size; uint64_t hlen; - if (pread(fd, &hlen, 8, 0) != 8) { perror("pread hlen"); exit(1); } + st_pread_full(fd, &hlen, 8, 0, "pread hlen"); /* file malevolo/troncato: hlen deve stare nel file dopo gli 8 byte di * prefisso e sotto il tetto. Senza questo bound hlen+1 puo' andare in * overflow (malloc(0) e poi hdr[hlen]=0 fuori limiti) o forzare una @@ -138,7 +171,7 @@ static void st_init(shards *S, const char *snap_dir) { files[fi], (unsigned long long)hlen, (long long)fsz); exit(1); } char *hdr = malloc(hlen + 1); if (!hdr) { perror("malloc safetensors header"); exit(1); } - if (pread(fd, hdr, hlen, 8) != (ssize_t)hlen) { perror("pread hdr"); exit(1); } + st_pread_full(fd, hdr, (int64_t)hlen, 8, "pread hdr"); hdr[hlen] = 0; int64_t data_start = 8 + (int64_t)hlen; char *arena = NULL; @@ -167,7 +200,19 @@ static void st_init(shards *S, const char *snap_dir) { if (a0 < 0 || b0 < a0 || data_start + b0 > fsz) { fprintf(stderr, "%s: tensor '%s' data_offsets [%lld,%lld] out of file bounds (%lld)\n", files[fi], name, (long long)a0, (long long)b0, (long long)fsz); exit(1); } - int64_t numel = 1; for (int k = 0; k < shp->len; k++) numel *= (int64_t)shp->kids[k]->num; + /* SEC: lo shape viene da un file non fidato (mirror). Senza il guard + * di overflow, uno shape tipo [65535,65535,65535,...] fa avvolgere + * numel a un valore piccolo/negativo che poi passerebbe il cross-check + * numel*esz==nbytes in st_read_f32, riaprendo l'OOB. */ + int64_t numel = 1; int bad_shape = 0; + for (int k = 0; k < shp->len; k++) { + int64_t d = (int64_t)shp->kids[k]->num; + if (d < 0 || (d != 0 && numel > INT64_MAX / d)) { bad_shape = 1; break; } + numel *= d; + } + if (bad_shape) { + fprintf(stderr, "%s: tensor '%s' shape overflows int64 — refusing (hostile or corrupt file)\n", + files[fi], name); exit(1); } if (S->n == S->cap) { S->cap *= 2; S->t = realloc(S->t, S->cap*sizeof(st_tensor)); } st_tensor *t = &S->t[S->n++]; t->name = strdup(name); t->fd = fd; t->off = data_start + a0; @@ -216,9 +261,18 @@ static void st_prefetch(shards *S, const char *name) { static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) { st_tensor *t = st_find(S, name); if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); } + /* SEC: numel viene dallo shape, nbytes dagli offset — due campi indipendenti + * del file. Se non concordano, la memcpy F32 (nbytes) o i loop BF16/F16 + * (numel elementi da un raw di soli nbytes) sforano il buffer del chiamante, + * che e' dimensionato sul config, non sul file. Il chiamante che alloca su + * st_numel resta coerente; questo blocca l'ingresso ostile a monte. */ + int esz = (t->dtype == 2) ? 4 : 2; + if (t->numel < 0 || t->numel > t->nbytes / esz || t->numel * (int64_t)esz != t->nbytes) { + fprintf(stderr, "%s: tensor '%s' shape/bytes mismatch (numel %lld, %lld bytes, dtype %d) — refusing (hostile or corrupt file)\n", + name, name, (long long)t->numel, (long long)t->nbytes, t->dtype); exit(1); } void *raw = malloc(t->nbytes); if (!raw) { fprintf(stderr, "malloc %lld bytes for tensor %s failed\n", (long long)t->nbytes, name); exit(1); } - if (pread(t->fd, raw, t->nbytes, t->off) != t->nbytes) { perror("pread data"); exit(1); } + st_pread_full(t->fd, raw, t->nbytes, t->off, "pread data"); if (t->dtype == 2) { memcpy(out, raw, t->nbytes); } else if (t->dtype == 0) { @@ -243,7 +297,7 @@ static int64_t st_nbytes(shards *S, const char *name) { static void st_read_raw(shards *S, const char *name, void *out, int drop) { st_tensor *t = st_find(S, name); if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); } - if (pread(t->fd, out, t->nbytes, t->off) != t->nbytes) { perror("pread raw"); exit(1); } + st_pread_full(t->fd, out, t->nbytes, t->off, "pread raw"); if (drop) posix_fadvise(t->fd, t->off, t->nbytes, POSIX_FADV_DONTNEED); } @@ -256,7 +310,7 @@ static void st_read_slice_f32(shards *S, const char *name, int64_t elem_off, int int esz = (t->dtype == 2) ? 4 : 2; int64_t boff = t->off + elem_off * esz, nb = n_elems * esz; void *raw = malloc(nb); - if (pread(t->fd, raw, nb, boff) != nb) { perror("pread slice"); exit(1); } + st_pread_full(t->fd, raw, nb, boff, "pread slice"); if (t->dtype == 2) memcpy(out, raw, nb); else if (t->dtype == 0) { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = bf16_to_f32(p[i]); } else { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = f16_to_f32(p[i]); } diff --git a/c/telemetry.h b/c/telemetry.h new file mode 100644 index 0000000..3c3fe85 --- /dev/null +++ b/c/telemetry.h @@ -0,0 +1,189 @@ +/* telemetry.h — dashboard protocol lines, stats/usage persistence, hardware probe. + * Include after Model/Cfg/QT/ESlot/shards and st.h are defined; requires + * qt_bytes(), now_s(), rss_gb(), edisk_s(), and the g_cuda_* globals (ifdef). */ +#ifndef TELEMETRY_H +#define TELEMETRY_H + +static int64_t tbytes(int O,int I,int bits){ + if(bits>=16) return (int64_t)O*I*4; + if(bits>=5) return (int64_t)O*I + (int64_t)O*4; + return (int64_t)O*((I+1)/2) + (int64_t)O*4; +} + +static int64_t expert_bytes_probe(Model *m, int ebits){ + Cfg *c=&m->c; int64_t eb=0; char nm[256]; + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.gate_proj.weight",c->first_dense); + if(st_nbytes(&m->S,nm)>0){ + const char *suf[3]={"gate_proj","up_proj","down_proj"}; + for(int k=0;k<3;k++){ + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight",c->first_dense,suf[k]); + eb+=st_nbytes(&m->S,nm); + snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight.qs",c->first_dense,suf[k]); + int64_t q=st_nbytes(&m->S,nm); if(q>0) eb+=q; + } + } + if(eb<=0) eb = tbytes(c->moe_inter,c->hidden,ebits)*2 + tbytes(c->hidden,c->moe_inter,ebits); + return eb; +} + +/* BRAIN MAP: per-turn expert hit bitmap for the dashboard. */ +static uint8_t **g_ehit; +static void ehit_mark(Model *m, int layer, int eid){ + if(!g_ehit){ Cfg *c=&m->c; + g_ehit=calloc(c->n_layers+1,sizeof(uint8_t*)); + for(int i=0;i<=c->n_layers;i++) g_ehit[i]=calloc(c->n_experts,1); + } + g_ehit[layer][eid]=1; +} + +/* CPU model + cores + RAM (GB); empty/zero where unavailable. */ +static void hw_probe(char *cpu, size_t cn, int *cores, double *ram_total, double *ram_avail){ + cpu[0]=0; +#ifdef _WIN32 +#if defined(__x86_64__) || defined(__i386__) + { unsigned int r[12]={0}; unsigned int *w=r; + for(unsigned int f=0x80000002u; f<=0x80000004u; f++,w+=4) + __get_cpuid(f,&w[0],&w[1],&w[2],&w[3]); + char *b=(char*)r; b[47]=0; while(*b==' ')b++; + snprintf(cpu,cn,"%s",b); } +#endif +#else + FILE *ci=fopen("/proc/cpuinfo","r"); + if(ci){ char ln[256]; + while(fgets(ln,sizeof(ln),ci)) if(!strncmp(ln,"model name",10)){ + char *p=strchr(ln,':'); if(p){ p++; while(*p==' ')p++; + int n=(int)strlen(p); if(n>0&&p[n-1]=='\n')p[--n]=0; + snprintf(cpu,cn,"%s",p); } break; } + fclose(ci); } +#endif + *cores=0; +#ifdef _WIN32 + { SYSTEM_INFO si; GetSystemInfo(&si); *cores=(int)si.dwNumberOfProcessors; } +#elif defined(_SC_NPROCESSORS_ONLN) + *cores=(int)sysconf(_SC_NPROCESSORS_ONLN); +#endif + *ram_total=*ram_avail=0; +#ifdef _WIN32 + compat_meminfo(ram_total,ram_avail); +#else + FILE *mi=fopen("/proc/meminfo","r"); + if(mi){ char ln[256]; double mt=0,ma=0; + while(fgets(ln,sizeof(ln),mi)){ + if(sscanf(ln,"MemTotal: %lf",&mt)==1) *ram_total=mt/1e6; + if(sscanf(ln,"MemAvailable: %lf",&ma)==1) *ram_avail=ma/1e6; + } fclose(mi); } +#endif +} + +static void hwinfo_emit(Model *m){ + Cfg *c=&m->c; (void)c; + char cpu[256]; int cores; double ram_total,ram_avail; + hw_probe(cpu,sizeof(cpu),&cores,&ram_total,&ram_avail); + int ngpu=0; double vram_total=0; + char gpu_name[128]=""; +#ifdef COLI_CUDA + ngpu=g_cuda_ndev; vram_total=m->gpu_expert_bytes/1e9; + for(int i=0;i0) + snprintf(gpu_name,sizeof(gpu_name),"CUDA device x%d",g_cuda_ndev); +#endif + printf("HWINFO %d %.1f %.1f %d %.1f %s|%s\n", + cores,ram_total,ram_avail,ngpu,vram_total,cpu,gpu_name); + fflush(stdout); +} + +static void tiers_emit(Model *m){ + Cfg *c=&m->c; int nsp=0; + for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; + int total=(nsp+(m->has_mtp?1:0))*c->n_experts; + int pinned=0,lru=0; + for(int i=0;i<=c->n_layers;i++){ pinned+=m->npin?m->npin[i]:0; lru+=m->ecn?m->ecn[i]:0; } + int vram=0; double vram_gb=0; +#ifdef COLI_CUDA + vram=m->gpu_expert_count; vram_gb=m->gpu_expert_bytes/1e9; +#endif + int ram=pinned-vram+lru; if(ram<0) ram=0; + int disk=total-vram-ram; if(disk<0) disk=0; + double eb=(double)expert_bytes_probe(m,m->ebits); + printf("TIERS %d %d %d %.2f %.2f\n",vram,ram,disk,vram_gb,ram*eb/1e9); + fflush(stdout); +} + +static void emap_emit(Model *m){ + Cfg *c=&m->c; + int rows=0; + for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; + int has_mtp = m->has_mtp && m->eusage[c->n_layers]; + if(has_mtp) rows++; + int cols=c->n_experts; + char *hex=malloc((size_t)rows*cols*2+1); int w=0; + for(int i=0;i<=c->n_layers;i++){ + int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); + if(!is_row) continue; + for(int e=0;epin[i]; + for(int z=0;znpin[i];z++) if(P[z].eid==e){ +#ifdef COLI_CUDA + tier = P[z].g.cuda?2:1; +#else + tier = 1; +#endif + break; } + if(!tier && m->ecache && m->ecache[i]) + for(int z=0;zecn[i];z++) if(m->ecache[i][z].eid==e){ tier=1; break; } + uint32_t u = m->eusage[i]?m->eusage[i][e]:0; + int heat=0; while(u){ heat++; u>>=1; } if(heat>63) heat=63; + int b=(tier<<6)|heat; + hex[w++]="0123456789abcdef"[b>>4]; hex[w++]="0123456789abcdef"[b&15]; + } + } + hex[w]=0; + printf("EMAP %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); +} + +static void hits_emit(Model *m){ + Cfg *c=&m->c; if(!g_ehit) return; + int rows=0; + for(int i=0;in_layers;i++) if(m->L[i].sparse) rows++; + int has_mtp = m->has_mtp && m->eusage[c->n_layers]; + if(has_mtp) rows++; + int cols=c->n_experts, nb=(rows*cols+7)/8; + uint8_t *bm=calloc(nb,1); int bit=0; + for(int i=0;i<=c->n_layers;i++){ + int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); + if(!is_row) continue; + for(int e=0;e>3]|=1<<(bit&7); g_ehit[i][e]=0; } + } + char *hex=malloc((size_t)nb*2+1); int w=0; + for(int b=0;b>4]; hex[w++]="0123456789abcdef"[bm[b]&15]; } + hex[w]=0; + printf("HITS %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); free(bm); +} + +static void stats_dump_q(Model *m, const char *path, int quiet){ + char tmp[2100]; snprintf(tmp,sizeof(tmp),"%s.tmp",path); + FILE *f=fopen(tmp,"w"); if(!f){ if(!quiet) perror(tmp); return; } + Cfg *c=&m->c; int64_t tot=0, nz=0; + for(int i=0;i<=c->n_layers;i++){ if(!m->eusage[i]) continue; + for(int e=0;en_experts;e++) if(m->eusage[i][e]){ fprintf(f,"%d %d %u\n",i,e,m->eusage[i][e]); tot+=m->eusage[i][e]; nz++; } } + fclose(f); rename(tmp,path); + if(!quiet) fprintf(stderr,"[STATS] %lld selections across %lld distinct experts -> %s\n",(long long)tot,(long long)nz,path); +} +static void stats_dump(Model *m, const char *path){ stats_dump_q(m,path,0); } + +static char g_usage_path[2100]=""; +static int64_t usage_load(Model *m, const char *path){ + FILE *f=fopen(path,"r"); if(!f) return 0; + Cfg *c=&m->c; int l,e; uint32_t cnt; int64_t tot=0; + while(fscanf(f,"%d %d %u",&l,&e,&cnt)==3) + if(l>=0&&l<=c->n_layers&&e>=0&&en_experts&&m->eusage[l]){ m->eusage[l][e]+=cnt; tot+=cnt; } + fclose(f); return tot; +} +static void usage_save(Model *m){ if(g_usage_path[0]) stats_dump_q(m,g_usage_path,1); } + +#endif /* TELEMETRY_H */ diff --git a/c/tests/README_efficiency.md b/c/tests/README_efficiency.md new file mode 100644 index 0000000..1f8d339 --- /dev/null +++ b/c/tests/README_efficiency.md @@ -0,0 +1,98 @@ +# Efficiency suite — regression tests + optimization dossier + +Two layers: + +1. **`test_inefficiency.py`** — tiny-model *asserted* regression tests. Fast + (~0.15s/run), gate CI, catch breakage. Run as part of `make test`. +2. **`test_efficiency_report.py`** — an *opt-in optimization dossier* for a real + model. Runs every instrumentation flag, prints a 9-section report answering + *what is doing what, when, with what, is it inefficient, how to improve*. + Never fails CI (it's a report, not a gate). + +## The dossier (what you run when optimizing) + +```bash +# CPU-only (safe, fast to validate): +COLI_EFFICIENCY_MODEL=../glm52_i4_g64 make efficiency-report + +# CUDA (dense + expert tiers — needs a CUDA build, see below): +COLI_EFFICIENCY_MODEL=../glm52_i4_g64 COLI_EFFICIENCY_CUDA=1 make efficiency-report +``` + +It turns ON every observability flag the engine supports — `PROF=1`, +`COLI_CUDA_PROFILE=1`, `CACHE_ROUTE=1` (auto-unlocks `route_agree`/`route_kl`), +`DISK_SPLIT=1`, `LOOKA=1` — so nothing the engine can tell you is left dark. +None of these change the computed output; they only add telemetry. + +The 9 sections, and the question each answers: + +| § | section | answers | +|---|---|---| +| 1 | PROVENANCE | what is running, on what CPU/backend, with what effective config | +| 2 | THROUGHPUT | tok/s + forward-latency p50/p90/p99/max (is the tail healthy?) | +| 3 | WHERE TIME GOES | the 5 PROFILE phases as % of decode + absolute seconds + verdict | +| 3a | ATTENTION BREAKDOWN | attention split into projection/RoPE, score-softmax-value, output | +| 4 | EXPERT CACHE | hit %, experts-loaded/token vs baseline topk | +| 5 | DISK I/O | GB fetched, MB/token, GB/s, read-service vs felt-wait, phase split | +| 5a | DISK-LOAD SPLIT | loads by decode phase (draft/absorb/verify) + MTP-vs-main bytes | +| 6 | ROUTING QUALITY | route_agree %, route_kl, cache swaps | +| 6a | ROUTING PREDICTABILITY | LOOKAHEAD recall per predictor (which prefetch wins) | +| 7 | SPECULATION | tokens/forward, MTP acceptance % | +| 8 | GPU TIERS | resident tensors, expert tier (count/GB/calls), H2D/kernel/D2H ms | + +Every line that crosses an advisory threshold is marked `[FLAG]` with the +concrete lever to pull (raise RAM_GB, add PIN_GB, try DIRECT=1, lower CTX, …), +and all flags repeat in a summary at the end. + +## Tunable thresholds + +The `IS IT INEFFICIENT?` lines are advisory constants at the top of +`test_efficiency_report.py`: + +| constant | default | meaning | +|---|---|---| +| `DISK_WAIT_DOMINANT` | 0.40 | >40% decode waiting on expert reads → I/O-bound | +| `LOW_HIT_RATE` | 0.30 | <30% cache hit → thrashing | +| `LOW_ROUTE_AGREE` | 0.80 | <80% routing overlap → prefetch guessing wrong | +| `HIGH_TAIL_RATIO` | 3.0 | p99 > 3× p50 → decode stalls | +| `LOW_MTP_ACCEPT` | 0.20 | <20% MTP acceptance → draft decoder is dead weight | + +The tiny-model asserted floors live in `tools/efficiency.py` (`TINY_TOK_S_FLOOR`, +`MAX_DISK_WAIT_SHARE`, `MIN_CPU_CUDA_AGREEMENT`). + +## The regression tests (what gates CI) + +`test_inefficiency.py` runs on the bundled `glm_tiny` model and asserts: + +- telemetry parses (no format drift) +- tiny tok/s ≥ floor (throughput regression) +- PROFILE phases present and non-negative (accounting sanity) +- disk-wait not dominant on a resident model (I/O-path regression) +- CPU determinism (two greedy runs agree) +- **CUDA** (skip unless CUDA built): init path, dense uploads VRAM, CPU-vs-CUDA + argmax agreement ≥ 70% (kernel-correctness guard) + +```bash +make efficiency # tiny CPU tests +make efficiency-cuda # tiny CUDA tests (needs CUDA build) +``` + +## CUDA build prerequisite + +The default `make glm.exe` builds **without** CUDA. The CUDA tests and the CUDA +dossier need a host built with `-DCOLI_CUDA` plus the runtime DLL: + +```bash +make clean && make glm.exe CUDA_DLL=1 && make cuda-dll +``` + +`make efficiency-cuda` auto-skips with a clear message if the host is CPU-only +(it scans the binary for the "CPU-only" marker the engine embeds). + +## Files + +- `tools/efficiency.py` — shared harness: `parse_run()` (captures every + telemetry signal), `run_engine()`, thresholds. Reuses `PROFILE_RE`/`SPEED_RE` + from `tools/benchmark_cuda_fixture.py`. +- `tests/test_inefficiency.py` — tiny-model asserted tests (CPU + CUDA). +- `tests/test_efficiency_report.py` — the opt-in optimization dossier. diff --git a/c/tests/bench_dsa_select.c b/c/tests/bench_dsa_select.c new file mode 100644 index 0000000..676e4c2 --- /dev/null +++ b/c/tests/bench_dsa_select.c @@ -0,0 +1,165 @@ +/* Microbenchmark: old (full-qsort) vs new (quickselect partial-select) DSA top-keep. + * + * This is NOT a unit test -- test_dsa_select.c proves correctness. This measures the + * headline claim of #356: that replacing the O(nk log nk) qsort over all nk context + * scores with an O(nk) partial_select_desc is materially faster per call, which is + * the win the issue was opened for -- and that the win GROWS with context length + * (because quickselect is linear average, qsort is n-log-n). + * + * It re-implements the OLD top-keep inline (qsort + threshold + scans) on a private + * buffer so the A/B runs in one process, same inputs, same warm caches -- a controlled + * comparison. It calls the REAL (new) partial_select_desc via the include-glm.c + * pattern, replicating the production threshold derivation + scans. + * + * Methodology (chosen to be honest, not to flatter the change): + * - keep = 2048 (the real GLM-5.2 index_topk), nk swept across context lengths from + * the 2049 activation boundary up to 65536 (a long conversation). + * - Three score shapes: (a) realistic peaked -- a few hot keys, long tail, the shape + * real DSA attention scores take; (b) uniform random -- no structure; (c) a plateau + * of ties, to exercise the boundary-membership path. + * - Each (shape, nk) is timed over N_REPEAT=2000 iterations, with the scores frozen + * so both algorithms do IDENTICAL work. We report median ns/call and the new/old + * ratio. A warmup pass primes caches before timing. + * + * Run: make tests/bench_dsa_select && ./tests/bench_dsa_select (not in TEST_BINS) + */ +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main + +#include +#include + +/* ---- the OLD algorithm, verbatim from dev before #356, on a private buffer ---- */ +static int cmp_pdesc_old(const void *a, const void *b){ + float x=*(const float*)a, y=*(const float*)b; return xy?-1:0; } +static void keep_old(const float *isc, int nk, int keep, int *dst, int *nd_out){ + float *tmp=malloc((size_t)nk*sizeof(float)); + memcpy(tmp,isc,(size_t)nk*sizeof(float)); + qsort(tmp,(size_t)nk,sizeof(float),cmp_pdesc_old); + float thr=tmp[keep-1]; int nd=0; + for(int t=0;tthr) dst[nd++]=t; + for(int t=0;t=0 && ts[b]>k){ ts[b+1]=ts[b]; b--; } ts[b+1]=k; } + free(dst); + return ts[N_REPEAT/2]; +} + +/* Sort an array of doubles ascending (median-of-medians aggregation below). */ +static void dsort(double *a, int n){ + for(int s=1;s=0 && a[b]>k){ a[b+1]=a[b]; b--; } a[b+1]=k; } +} + +/* the NEW algorithm calls the real partial_select_desc + replicates the production + * threshold derivation and position scans. */ +static void keep_new(const float *isc, int nk, int keep, int *dst, int *nd_out){ + float *tmp=malloc((size_t)nk*sizeof(float)); + memcpy(tmp,isc,(size_t)nk*sizeof(float)); + partial_select_desc(tmp,nk,keep); + float thr=tmp[0]; for(int t=1;tthr) dst[nd++]=t; + for(int t=0;t> 17; brng ^= brng << 5; + return (double)(brng >> 8) * (1.0 / 16777216.0); } +static void fill_realistic(float *isc, int nk){ /* few hot, long distinct tail */ + for(int i=0;i boundary path */ + for(int i=0;i +#include + +/* ---- the OLD algorithm, verbatim from dev before #354, on a private buffer ---- */ +static float *s_pbuf; static int *s_pidx; static double *s_ref; +static int cmp_pdesc_old(const void *a, const void *b){ + double pa = s_ref[*(const int*)a], pb = s_ref[*(const int*)b]; + return pa < pb ? 1 : pa > pb ? -1 : 0; } +static void dist_build_old(const float *lo, int V, double temp, double nuc){ + double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i]; + double s = 0, invt = 1.0 / (temp > 1e-4 ? temp : 1e-4); + for (int i = 0; i < V; i++){ s_ref[i] = exp((lo[i]-mx)*invt); s += s_ref[i]; } + for (int i = 0; i < V; i++) s_ref[i] /= s; + if (nuc > 0 && nuc < 1.0){ + for (int i = 0; i < V; i++) s_pidx[i] = i; + qsort(s_pidx, V, sizeof(int), cmp_pdesc_old); + double cum = 0; int keep = V; + for (int i = 0; i < V; i++){ cum += s_ref[s_pidx[i]]; if (cum >= nuc){ keep = i+1; break; } } + double s2 = 0; + for (int i = keep; i < V; i++) s_ref[s_pidx[i]] = 0; + for (int i = 0; i < keep; i++) s2 += s_ref[s_pidx[i]]; + for (int i = 0; i < keep; i++) s_ref[s_pidx[i]] /= s2; + } + (void)s_pbuf; +} + +/* ---- timing: median of N_REPEAT runs in ns/call, sorted ascending ---- */ +#define N_REPEAT 2000 +static double bench_ns(void (*fn)(const float*,int,double,double), + const float *lo, int V, double temp, double nuc){ + static double ts[N_REPEAT]; + for (int r = 0; r < N_REPEAT; r++){ + double t0 = now_s(); + fn(lo, V, temp, nuc); + ts[r] = (now_s() - t0) * 1e9; + } + /* insertion sort the N_REPEAT samples (small), take median */ + for (int a = 1; a < N_REPEAT; a++){ double k = ts[a]; int b = a-1; + while (b >= 0 && ts[b] > k){ ts[b+1] = ts[b]; b--; } ts[b+1] = k; } + return ts[N_REPEAT/2]; +} + +/* the NEW algorithm is the real dist_build, but it writes g_pbuf (not a private buf). + * Wrap it so the bench signature matches, and set the globals it reads. */ +static void dist_build_new(const float *lo, int V, double temp, double nuc){ + g_temp = (float)temp; g_nuc = (float)nuc; + dist_build(lo, V); +} + +/* deterministic logit fill for three shapes */ +static uint32_t brng = 0xA5A5A5A5u; +static double brand(void){ brng ^= brng << 13; brng ^= brng >> 17; brng ^= brng << 5; + return (double)(brng >> 8) * (1.0 / 16777216.0); } +static void fill_realistic(float *lo, int V){ /* one hot, exponential tail -- like real logits */ + for (int i = 0; i < V; i++) lo[i] = (float)(-4.0 * brand() - (double)i * 0.0001); + lo[0] = 6.f; lo[V/50] = 4.f; lo[V/200] = 3.f; +} +static void fill_uniform(float *lo, int V){ /* worst case for the heap: max pop count */ + for (int i = 0; i < V; i++) lo[i] = 0.f; +} +static void fill_plateau(float *lo, int V){ /* ties: blocks of equal value */ + for (int i = 0; i < V; i++) lo[i] = (float)(-(double)(i / 50)); +} + +int main(void){ + int V = 151936; + float *lo = malloc((size_t)V * sizeof(float)); + s_ref = malloc((size_t)V * sizeof(double)); + s_pidx = malloc((size_t)V * sizeof(int)); + /* force the new dist_build to allocate g_pbuf/g_pidx at full V once */ + g_temp = 0.7f; g_nuc = 0.9f; dist_build(lo, V); + + double temp = 0.7; + struct { const char *name; void (*fill)(float*,int); } shapes[] = { + { "realistic", fill_realistic }, + { "uniform", fill_uniform }, + { "plateau", fill_plateau }, + }; + double nucs[] = { 0.5, 0.9, 0.95, 0.99 }; + + printf("bench_topp: top-p truncation, old (qsort) vs new (heap) V=%d temp=%.2f\n", V, temp); + printf("%-12s %6s %14s %14s %9s %9s\n", "shape", "nuc", "old ns/call", "new ns/call", "speedup", "keep"); + printf("-----------------------------------------------------------------------------\n"); + + for (size_t sh = 0; sh < sizeof(shapes)/sizeof(shapes[0]); sh++){ + shapes[sh].fill(lo, V); + for (size_t ni = 0; ni < sizeof(nucs)/sizeof(nucs[0]); ni++){ + double nuc = nucs[ni]; + /* warmup both paths so caches/branch predictors are primed */ + for (int w = 0; w < 50; w++){ dist_build_old(lo, V, temp, nuc); dist_build_new(lo, V, temp, nuc); } + double t_old = bench_ns(dist_build_old, lo, V, temp, nuc); + double t_new = bench_ns(dist_build_new, lo, V, temp, nuc); + /* keep count = non-zero entries the new path leaves (== old's keep) */ + int keep = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) keep++; + printf("%-12s %6.2f %14.0f %14.0f %8.2fx %9d\n", + shapes[sh].name, nuc, t_old, t_new, t_old / t_new, keep); + } + printf("\n"); + } + + printf("bench_topp: done (lower ns is better; speedup = old/new)\n"); + free(lo); free(s_ref); free(s_pidx); + return 0; +} diff --git a/c/tests/test_benchmark_cuda_fixture.py b/c/tests/test_benchmark_cuda_fixture.py index a704052..3feaf11 100644 --- a/c/tests/test_benchmark_cuda_fixture.py +++ b/c/tests/test_benchmark_cuda_fixture.py @@ -1,11 +1,12 @@ import unittest -from tools.benchmark_cuda_fixture import parse_output +from tools.benchmark_cuda_fixture import parse_output, parse_p0 SAMPLE = """ REPLAY decode: 4 tokens | 12.34 tok/s PROFILE: expert-disk 1.25s | expert-matmul 2.50s | attention 0.75s | lm_head 0.10s | other -0.05s +P0-EXEC: routed CPU 1.200s / 123.40 GB/s (456 row) | routed GPU critical 0.150s | router 0.200s | residual P2P 0.030s / 75 hop | orchestration 0.100s """ @@ -19,6 +20,9 @@ class ParseOutputTest(unittest.TestCase): with self.assertRaisesRegex(RuntimeError, "benchmark output missing"): parse_output("REPLAY decode: 4 tokens | 12.34 tok/s", "engine failed") + def test_extracts_p0_profile(self): + self.assertEqual(parse_p0(SAMPLE), [1.2, 123.4, 456.0, 0.15, 0.2, 0.03, 75.0, 0.1]) + if __name__ == "__main__": unittest.main() diff --git a/c/tests/test_dsa_select.c b/c/tests/test_dsa_select.c new file mode 100644 index 0000000..ef3fd7b --- /dev/null +++ b/c/tests/test_dsa_select.c @@ -0,0 +1,223 @@ +/* DSA top-keep partial-select: the quickselect rewrite (#356) must produce a + * BIT-IDENTICAL kept-position set to the old full-vocab qsort, for every score + * shape the attention indexer can see. + * + * Why this test exists (#356): attention_rows() selects the top-`keep` context + * keys (index_topk=2048 on GLM-5.2) to attend to. It previously did this by + * full-qsorting all `nk` scores (O(nk log nk)) and reading tmp[keep-1] as the + * threshold. It now does a partial_select_desc (quickselect, O(nk) average) and + * takes the threshold as the min of the selected top-keep block. The contract is + * subtle but STRONGER than test_topp's: + * + * The two position-order scans that build dst[] -- + * for t: if isc[t] > thr -> keep (strictly above threshold) + * for t: if isc[t] == thr -> keep (ties, in position order) + * -- are UNCHANGED by the rewrite. So if the threshold value is identical, + * the kept-position set is identical element-by-element (not just as a + * multiset, which is all the unstable sampling heap in #335 could promise). + * + * Strategy: drive the REAL partial_select_desc (via the include-glm.c pattern) + * and replicate the production threshold derivation + scans, then compare the + * resulting dst[] against an INDEPENDENT reference that re-implements the OLD + * algorithm (full qsort + tmp[keep-1] threshold) on a private buffer. The kept + * sets must be element-wise equal on every shape, including tie plateaus where + * the boundary membership is decided by the position scan. + * + * We also directly unit-test partial_select_desc's partition invariant: after + * the call, max(a[keep..n)) <= min(a[0..keep)) -- i.e. the keep largest really + * did land in the prefix. This catches a broken quickselect even before the + * end-to-end comparison. + * + * In-memory only (no scratch files), so it builds clean on the Windows MinGW CI + * job without the unmerged compat shim. */ +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main + +#include + +static int g_nfails = 0; + +#define FAIL(fmt, ...) do { \ + fprintf(stderr, " FAIL [%s nk=%d keep=%d shape=%s]: " fmt "\n", \ + label, nk, keep, shape_name, ##__VA_ARGS__); \ + g_nfails++; \ + return; \ +} while (0) + +/* ---- independent reference: the OLD algorithm (full qsort + tmp[keep-1]) ---- */ +/* qsort comparator matching the production cmp_fdesc exactly (desc, unstable). */ +static int cmp_ref_desc(const void *a, const void *b){ + float x=*(const float*)a, y=*(const float*)b; return xy?-1:0; } + +/* Reproduce the OLD glm.c:2589-2596 exactly: copy, qsort desc, threshold = + * tmp[keep-1], then the two position-order scans into dst[]. Returns nd. */ +static int keep_old(const float *isc, int nk, int keep, int *dst){ + float *tmp=malloc((size_t)nk*sizeof(float)); + memcpy(tmp,isc,(size_t)nk*sizeof(float)); + qsort(tmp,(size_t)nk,sizeof(float),cmp_ref_desc); + float thr=tmp[keep-1]; + int nd=0; + for(int t=0;tthr) dst[nd++]=t; + for(int t=0;tthr) dst[nd++]=t; + for(int t=0;t= every element of tmp[keep..n). + * (>=, not >: equal values may sit on either side of the partition boundary, + * which is fine -- the threshold is the MIN of the prefix, and the position + * scan handles ties.) */ + float top_min=INFINITY, tail_max=-INFINITY; + for(int i=0;itail_max) tail_max=tmp[i]; + if(!(top_min >= tail_max)) + FAIL("partition invariant violated: top_min=%.9g < tail_max=%.9g", top_min, tail_max); + free(tmp); +} + +/* ---- end-to-end: old vs new kept-set must be element-wise identical ---- */ +static void check_case(const char *label, int nk, int keep, const char *shape_name, + const float *isc){ + int *da=malloc((size_t)nk*sizeof(int)); + int *db=malloc((size_t)nk*sizeof(int)); + int na=keep_old(isc,nk,keep,da); + int nb=keep_new(isc,nk,keep,db); + + /* 1. both keep exactly `keep` positions (the contract: keep the top-keep by + * count). A count mismatch is a real bug, not a tie artifact. */ + if(na!=keep) FAIL("old kept %d, expected %d (old path is the reference)", na, keep); + if(nb!=keep) FAIL("new kept %d, expected %d", nb, keep); + if(na!=nb) FAIL("keep-count mismatch: old=%d new=%d", na, nb); + + /* 2. element-wise identical dst[]. This is the strong contract: because the + * threshold is derived identically and the position-order scans are byte- + * for-byte the same, the kept SET and its ORDER must match exactly. (This + * is what makes #356 cleaner than #335, which was multiset-only.) */ + int first_diff=-1; + for(int i=0;i=0) + FAIL("kept-set differs at index %d: old dst[%d]=%d new dst[%d]=%d", + first_diff, first_diff, da[first_diff], first_diff, db[first_diff]); + + /* 3. also check the partition invariant directly (catches a subtly broken + * quickselect even if the threshold happened to come out right). */ + check_partition(label,isc,nk,keep,shape_name); + + free(da); free(db); + printf(" ok [nk=%d keep=%d shape=%s]\n", nk, keep, shape_name); +} +#undef FAIL + +/* deterministic xorshift32 RNG (matches the test_i4_grouped.c / test_topp.c convention) */ +static uint32_t rng_state = 0x12345678u; +static uint32_t xr(void){ rng_state ^= rng_state << 13; rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; return rng_state; } +static double frand(void){ return (xr() >> 8) * (1.0 / 16777216.0); } /* [0,1) */ + +/* fill scores for a given shape. Shapes stress the threshold boundary and the + * quickselect's median-of-three pivot differently. */ +static void fill_shape(float *isc, int nk, int shape){ + switch(shape){ + case 0: /* uniform random distinct (no ties): the clean contract case */ + for(int i=0;i3) isc[nk/3]=1.f; if(nk>2) isc[nk/2]=0.5f; break; + case 2: /* strictly decreasing geometric (no ties): sorted input -- worst case + * for a naive quickselect; median-of-three must handle it */ + for(int i=0;i boundary membership decided + * entirely by the position scan (exercises the ==thr path) */ + for(int i=0;i degenerate threshold, all kept + * via the ==thr scan; quickselect must not infinite-loop or corrupt */ + for(int i=0;i=n / k==1 / k==n edges. */ + int nks[] = {1, 2, 8, 64, 2049, 4097, 8193}; + int keeps[] = {1, 8, 256, 1024, 2048}; + int n_shapes = 6; + + int cases = 0; + for(size_t ni=0; nin, but skip the plateau/geometric edge if nk<7) */ + fill_shape(isc,nk,shape); + for(size_t ki=0; kink) continue; /* keep<=nk invariant of the production code */ + if(keep<=0) continue; + char label[40]; snprintf(label,sizeof(label),"nk[%zu]/keep[%zu]/shape[%d]",ni,ki,shape); + const char *sn=(const char*[]){"random","peaked","decreasing","increasing","plateau","all-equal"}[shape]; + check_case(label,nk,keep,sn,isc); + cases++; + } + } + free(isc); + } + + /* edge: keep == nk (nothing to partition; both paths keep everything) */ + { + int nk=100, keep=100; float isc[100]; + for(int i=0;i every kept slot is a tie; + * the position scan must pick positions 0..keep-1 deterministically */ + { + int nk=1000, keep=500; float isc[1000]; + for(int i=0;i positions 0..%d]\n",keep,keep-1); cases++; + free(db); + } + + printf("\ntest_dsa_select: %d cases run, %d failure(s)\n", cases, g_nfails); + if(g_nfails){ printf("test_dsa_select: FAIL\n"); return 1; } + printf("test_dsa_select: ok\n"); + return 0; +} diff --git a/c/tests/test_efficiency_report.py b/c/tests/test_efficiency_report.py new file mode 100644 index 0000000..ba01b13 --- /dev/null +++ b/c/tests/test_efficiency_report.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +"""Exhaustive optimization dossier for a colibri engine run. + +This is NOT a pass/fail test. It runs the engine with every instrumentation flag +on (PROF, COLI_CUDA_PROFILE, CACHE_ROUTE, DISK_SPLIT, LOOKA) and prints a section- +by-section report answering, for each subsystem: + + WHAT is doing it — which phase/kernel/tier + WHEN it is doing it — how much of decode wall-time it owns + WITH WHAT — the config/weights/tier it used + IS IT INEFFICIENT? — a verdict, with the threshold + HOW TO IMPROVE — the concrete knob, named + +Activation (opt-in only — NOT in `make test`): + COLI_EFFICIENCY_MODEL= python tests/test_efficiency_report.py + +Optional env: + COLI_EFFICIENCY_CUDA=1 also exercise the CUDA dense/expert tiers + COLI_EFFICIENCY_NGEN=N decode tokens (default 24) + COLI_EFFICIENCY_RAM_GB=N RAM budget (default 28) + COLI_EFFICIENCY_VRAM_GB=N CUDA expert-tier budget GB (default 4) + COLI_EFFICIENCY_PROMPT=... prompt (default: a code-gen prompt) + +Exit code is always 0 (it's a dossier, not a gate). Lines marked FLAG point at +the most likely lever to move tok/s for the observed bottleneck. +""" +import os +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from tools.efficiency import run_engine, disk_wait_share # noqa: E402 + + +# --- advisory thresholds (the "IS IT INEFFICIENT?" lines) --- +DISK_WAIT_DOMINANT = 0.40 # >40% decode waiting on expert reads -> I/O-bound +LOW_HIT_RATE = 0.30 # <30% cache hit -> thrashing (cap too small) +LOW_ROUTE_AGREE = 0.80 # <80% routing overlap -> prefetch is guessing wrong +HIGH_TAIL_RATIO = 3.0 # p99 > 3x p50 -> decode stalls (I/O hiccups / KV grow) +LOW_MTP_ACCEPT = 0.20 # <20% MTP acceptance -> draft decoder is dead weight +VRAM_WASTE_CALLS = 0 # experts pinned in VRAM but 0 calls served + + +def _flag(ok): return "OK " if ok else "FLAG" + + +def _bar(frac, width=24): + """A simple ASCII bar for share visualization.""" + n = max(0, min(width, round(frac * width))) + return "#" * n + "." * (width - n) + + +def _line(label, value, flag=None, note=""): + tag = f" [{flag}]" if flag else "" + print(f" {label:<22} {value}{tag} {note}" if note else f" {label:<22} {value}{tag}") + + +def main() -> int: + model = os.environ.get("COLI_EFFICIENCY_MODEL") + if not model: + print(__doc__) + print("\nNot activated: set COLI_EFFICIENCY_MODEL= to run.") + return 0 + model = str(Path(model).resolve()) + if not Path(model).is_dir(): + print(f"ERROR: {model} is not a directory", file=sys.stderr) + return 0 + + ngen = int(os.environ.get("COLI_EFFICIENCY_NGEN", "24")) + ram_gb = os.environ.get("COLI_EFFICIENCY_RAM_GB", "28") + vram_gb = os.environ.get("COLI_EFFICIENCY_VRAM_GB", "4") + prompt = os.environ.get( + "COLI_EFFICIENCY_PROMPT", + "Write a Python function that computes the factorial of a number. " + "Include error handling and a docstring.") + use_cuda = os.environ.get("COLI_EFFICIENCY_CUDA") == "1" + + # Turn ON every instrumentation flag so the dossier has maximum detail. + # These are all observability toggles (PROF/COLI_CUDA_PROFILE/CACHE_ROUTE/ + # DISK_SPLIT/LOOKA); none change the computed output. + overlay = dict( + NGEN=str(ngen), TEMP="0", RAM_GB=ram_gb, PROMPT=prompt, + PROF="1", CACHE_ROUTE="1", DISK_SPLIT="1", LOOKA="1", ROUTE_AGREE="1", + ) + if use_cuda: + overlay.update(COLI_CUDA="1", COLI_GPU="0", CUDA_DENSE="1", + COLI_CUDA_PROFILE="1", CUDA_EXPERT_GB=vram_gb) + + print("=" * 78) + print(f"OPTIMIZATION DOSSIER — {Path(model).name}") + print(f" mode : {'CUDA (dense+expert tiers)' if use_cuda else 'CPU-only'} " + f"ngen : {ngen} ram : {ram_gb} GB" + + (f" vram : {vram_gb} GB" if use_cuda else "")) + print("=" * 78) + + t0 = time.time() + t, proc = run_engine(overlay, snap=model, timeout=3600.0) + wall = time.time() - t0 + flags = [] # collected FLAG lines for the summary + + print(f"\n[0] RUN") + _line("wall clock", f"{wall:.0f}s") + _line("exit code", proc.returncode, + None if proc.returncode == 0 else "FLAG", + "" if proc.returncode == 0 else "non-zero exit") + if proc.returncode != 0: + print(" stderr tail:") + for ln in proc.stderr.strip().splitlines()[-8:]: + print(f" {ln}") + return 0 + + # ---------------------------------------------------------------- [1] WHO ---- + print(f"\n[1] PROVENANCE — what is running, on what, with what config") + if t.get("machine"): + m = t["machine"] + _line("CPU", m["cpu"]) + _line("cores / omp", f"{m['cores']} cores") + _line("backend", m["backend"]) + if t.get("load"): + ld = t["load"] + _line("model load time", f"{ld['load_s']:.2f}s") + _line("resident dense", f"{ld['resident_dense_mb']:.1f} MB") + _line("layers / experts", f"{ld['layers']} layers, {ld['experts']} experts") + _line("MTP", f"{ld['mtp_status']} (draft={ld['draft']})") + if t.get("config_str"): + _line("resolved config", t["config_str"]) + print(" (this is the EFFECTIVE config after auto-budgeting — not your env verbatim)") + + # ---------------------------------------------------------------- [2] SPEED -- + print(f"\n[2] THROUGHPUT — is it fast, is the tail healthy") + if t.get("tok_s") is not None: + _line("tok/s", f"{t['tok_s']:.3f}") + else: + flags.append("throughput line missing — engine output format may have changed") + if t.get("latency"): + la = t["latency"] + _line("decode forwards", f"{int(la['forwards'])}") + _line("p50 / p90", f"{la['p50_ms']:.1f} / {la['p90_ms']:.1f} ms") + _line("p99 / max", f"{la['p99_ms']:.1f} / {la['max_ms']:.1f} ms") + tail_ok = la["p99_ms"] <= HIGH_TAIL_RATIO * la["p50_ms"] + _line("tail ratio (p99/p50)", f"{la['p99_ms']/max(la['p50_ms'],1e-9):.2f}x", + _flag(tail_ok), + "high tail = decode stalls (I/O hiccups, KV growth, re-pin)") + if not tail_ok: + flags.append(f"tail latency p99={la['p99_ms']:.1f}ms >> p50={la['p50_ms']:.1f}ms " + "(look for REPIN swaps or disk stalls)") + + # ---------------------------------------------------------------- [3] TIME --- + print(f"\n[3] WHERE TIME GOES — what is doing it, when (share of decode)") + ts = t.get("time_shares") + prof = t.get("profile") + if ts: + order = [("io", "expert-disk I/O", DISK_WAIT_DOMINANT, "the cache is too small / disk is slow"), + ("matmul", "expert matmul", 0.40, "compute-bound; more cores or a GPU expert tier"), + ("attention", "attention", 0.35, "context length is the cost; lower CTX"), + ("head", "lm_head", 0.10, "vocab projection; unusual to dominate"), + ("other", "other", 0.30, "scheduling / KV bookkeeping overhead")] + for key, name, thresh, lever in order: + f = ts[key] + ok = f < thresh + _line(name, f"{f:5.0%} {_bar(f)}", _flag(ok), + "" if ok else f"->{lever}") + if not ok: + flags.append(f"{name} dominates ({f:.0%}) -> {lever}") + if t.get("verdict"): + print(f" engine verdict : {t['verdict']}") + elif prof: + print(" (no [PROF] time shares — set PROF=1 for phase percentages)") + if prof: + print(" absolute seconds :") + for k in ("disk", "expert_matmul", "attention", "lm_head", "other"): + _line(k, f"{prof[k]:.3f}s") + + # attention sub-breakdown: how is attention being read + ab = t.get("attn_breakdown") + if ab: + print(f"\n[3a] ATTENTION BREAKDOWN — how the attention phase is spent") + atot = sum(ab.values()) or 1.0 + for k, label in (("proj_rope", "projection + RoPE"), + ("score_sm_value", "score-softmax-value"), + ("out_proj", "output projection")): + _line(label, f"{ab[k]:.3f}s ({ab[k]/atot:.0%} of attn)") + + # ---------------------------------------------------------------- [4] CACHE -- + print(f"\n[4] EXPERT CACHE — is the cache efficient") + hit = t.get("hit_pct") + if hit is not None: + ok = hit >= LOW_HIT_RATE * 100 + _line("hit rate", f"{hit:.1f}%", _flag(ok), + "" if ok else "<30% = thrashing; raise RAM_GB or cap") + if not ok: + flags.append(f"cache hit {hit:.1f}% is low -> raise RAM_GB (or cap), add PIN_GB") + el = t.get("experts_loaded") + if el: + per_tok = el["per_tok"] + _line("experts loaded/token", f"{per_tok:.1f}") + _line(" per-layer", f"{el['per_layer']:.2f} across {el['n_sparse_layers']} sparse layers") + _line(" baseline", f"topk={el['baseline_topk']} active experts/token") + base_topk = el["baseline_topk"] + if base_topk > 0 and per_tok > 2 * base_topk: + flags.append(f"loading {per_tok:.0f} experts/token vs topk={base_topk} " + "-> redundant I/O; cache is re-fetching evicted experts") + + # ---------------------------------------------------------------- [5] DISK --- + print(f"\n[5] DISK I/O — is I/O the bottleneck, and where") + eio = t.get("expert_io") + if eio: + _line("total fetched", f"{eio['gb_fetched']:.3f} GB") + _line("per token", f"{eio['mb_per_tok']:.1f} MB/token") + _line("disk throughput", f"{eio['gb_per_s']:.2f} GB/s over the run") + _line("read service", f"{eio['read_service_s']:.2f}s (on I/O threads)") + _line("felt wait", f"{eio['felt_wait_s']:.2f}s (stall compute felt)") + if eio["felt_wait_s"] > eio["read_service_s"] * 0.5 and eio["read_service_s"] > 0: + flags.append("felt wait is a large fraction of read service -> PIPE=1 may not be " + "overlapping fully, or DIRECT=1 on NVMe") + ds = t.get("disk_split") + if ds: + print(f"\n[5a] DISK-LOAD SPLIT — which decode phase reads the bytes") + _line("draft phase", f"{ds['draft']} loads") + _line("absorb phase", f"{ds['absorb']} loads") + _line("verify/main", f"{ds['verify_main']} loads") + _line("MTP-layer bytes", f"{ds['mtp_loads']} loads, {ds['mtp_gb']:.2f} GB") + _line("main-layer bytes", f"{ds['main_loads']} loads, {ds['main_gb']:.2f} GB") + if ds.get("mtp_bytes_pct") is not None: + _line("MTP share of bytes", f"{ds['mtp_bytes_pct']:.1f}%") + share = disk_wait_share(t) + if share is not None: + ok = share < DISK_WAIT_DOMINANT + _line("disk-wait share", f"{share:.0%}", _flag(ok), + "" if ok else "I/O-bound (see levers in [3])") + + # ---------------------------------------------------------------- [6] ROUTE -- + print(f"\n[6] ROUTING QUALITY — is the router / prefetch accurate") + ra = t.get("route_agree") + if ra: + ok = ra["agree_pct"] >= LOW_ROUTE_AGREE * 100 + _line("route_agree", f"{ra['agree_pct']:.1f}% overlap with true top-K", + _flag(ok), + "" if ok else "prefetch is guessing wrong; CACHE_ROUTE params may need tuning") + _line("route_kl", f"{ra['kl']:.4f} mean KL (lower = closer to true routing)") + if not ok: + flags.append(f"route_agree {ra['agree_pct']:.1f}% low -> tune ROUTE_J/M/P, " + "or prefetch is hurting more than helping") + sw = t.get("swap") + if sw: + _line("cache swaps", f"{sw['swaps']}/{sw['slots']} ({sw['pct']:.1f}%)", + None, "high swap = churn between turns") + la = t.get("lookahead") + if la: + print(f"\n[6a] ROUTING PREDICTABILITY — recall of true experts in predicted top-8") + print(" (which predictor should drive prefetch? highest recall wins)") + for row in la: + _line(row["predictor"][:34], f"{row['pct']:5.1f}% ({row['hit']}/{row['tot']})") + + # ---------------------------------------------------------------- [7] SPEC --- + print(f"\n[7] SPECULATION — is the draft decoder pulling weight") + sp = t.get("speculation") + if sp: + _line("tokens/forward", f"{sp['tok_per_fw']:.2f} (>1.0 means speculation helps)") + _line("forwards/tokens", f"{sp['forwards']} forwards for {sp['tokens']} tokens") + # Speculation helps only if acceptance is high enough that tok/forward > 1. + # tok_per_fw already == 1.0 when nothing verifies, so judge by acceptance. + ok = sp["mtp_accept_pct"] >= LOW_MTP_ACCEPT * 100 + _line("MTP acceptance", f"{sp['mtp_accept_pct']:.0f}%", _flag(ok), + "" if ok else "<20% -> drafts rarely verify; DRAFT=0 may be faster") + if not ok: + flags.append(f"MTP acceptance {sp['mtp_accept_pct']:.0f}% low -> " + "drafts cost more I/O than they save; try DRAFT=0") + + # ---------------------------------------------------------------- [8] GPU ---- + print(f"\n[8] GPU TIERS — is the GPU actually used") + cuda = t.get("cuda") or {} + if not cuda.get("enabled"): + print(" (CUDA not enabled — CPU-only run)") + else: + if cuda.get("resident_tensors") is not None: + _line("resident dense tensors", f"{cuda['resident_tensors']} tensors, " + f"{cuda['resident_gb']:.2f} GB") + if cuda.get("expert_count") is not None: + waste = cuda["calls_served"] <= VRAM_WASTE_CALLS + _line("expert tier", f"{cuda['expert_count']} experts pinned " + f"({cuda['expert_gb']:.2f} GB)", _flag(not waste)) + _line(" calls served", f"{cuda['calls_served']} from VRAM", + _flag(not waste), + "" if not waste else "WASTE: pinned but never routed -> lower CUDA_EXPERT_GB") + if waste: + flags.append("VRAM expert tier has 0 calls served -> experts pinned but unused; " + "PIN stats may not match this workload") + if cuda.get("groups"): + g = cuda["groups"] + _line("expert groups", f"{g['calls']} calls, {g['experts']} experts, " + f"{g['rows']} rows ({g['experts_per_call']:.1f} experts/call)") + if cuda.get("groups_timing"): + gt = cuda["groups_timing"] + _line("GPU timing", f"H2D {gt['h2d_ms']:.1f} ms | kernel {gt['kernel_ms']:.1f} ms | " + f"D2H {gt['d2h_ms']:.1f} ms") + if gt["h2d_ms"] + gt["d2h_ms"] > gt["kernel_ms"]: + flags.append("CUDA H2D+D2H > kernel time -> transfer-bound; " + "consider larger expert tier to keep weights resident") + + # ---------------------------------------------------------------- summary ---- + print("\n" + "=" * 78) + if flags: + print(f" {len(flags)} FLAG(s) — the most likely levers to move tok/s:") + for f in flags: + print(f" - {f}") + else: + print(" no flags — every measured subsystem is within advisory thresholds.") + print("=" * 78) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/c/tests/test_env_defaults.py b/c/tests/test_env_defaults.py index 12a3c99..3bb08fc 100644 --- a/c/tests/test_env_defaults.py +++ b/c/tests/test_env_defaults.py @@ -32,9 +32,16 @@ def args(**over): class EnvDefaultsTest(unittest.TestCase): - def env_for_with(self, environ, platform): + def env_for_with(self, environ, platform, cuda=False): + """Run env_for on a bare-chat args() under a faked env + platform. + + cuda=False by default so the existing default-I/O tests stay + deterministic: the Windows auto-enable branch calls cuda_binary() and + (if True) discover_gpus(), both of which reach the real machine — faking + False keeps these tests independent of the host's GPU.""" with mock.patch.dict(os.environ, environ, clear=True), \ - mock.patch.object(sys, "platform", platform): + mock.patch.object(sys, "platform", platform), \ + mock.patch.object(coli, "cuda_binary", return_value=cuda): return coli.env_for(args()) def test_win32_sets_measured_defaults(self): @@ -64,5 +71,80 @@ class EnvDefaultsTest(unittest.TestCase): self.assertNotIn(k, e) +class CudaAutoEnableTest(unittest.TestCase): + """Windows bare `coli chat` (no --gpu/--vram/--auto-tier) used to ALWAYS run + CPU-only even on a CUDA build with a GPU present. env_for now auto-enables + CUDA on win32 when cuda_binary() is True and a GPU is discoverable; falls + back to CPU with a warning if nvidia-smi (discover_gpus) is missing; stays + silent on a CPU build; and never touches the Linux path.""" + + def _env_for(self, platform, cuda, gpus, plan=None): + # Patch discover_gpus / build_plan / environment_for_plan at the + # resource_plan module (env_for imports them lazily on each call, so the + # patches are live when those imports run). Stubbing the planner keeps + # the test independent of a real model dir (args().model == "X"). + import resource_plan + a = args() + GPB = 1024 ** 3 + if plan is None: + plan = {"tiers": {"ram": {"budget_bytes": 16 * GPB, "cache_slots_per_layer": 4}, + "vram": {"budget_bytes": int(8.0 * GPB), "devices": gpus}}} + + def fake_environment_for_plan(p, env, cuda_enabled=True): + # Mirror the real contract: size CUDA_EXPERT_GB from the plan's VRAM + # budget (this is the value env_for propagates into the engine env). + r = dict(env) + if cuda_enabled and p["tiers"]["vram"]["devices"] and p["tiers"]["vram"]["budget_bytes"] > 0: + r["CUDA_EXPERT_GB"] = f"{p['tiers']['vram']['budget_bytes'] / GPB:.3f}" + return r + + with mock.patch.dict(os.environ, {}, clear=True), \ + mock.patch.object(sys, "platform", platform), \ + mock.patch.object(coli, "cuda_binary", return_value=cuda), \ + mock.patch.object(resource_plan, "discover_gpus", return_value=gpus), \ + mock.patch.object(resource_plan, "build_plan", return_value=plan), \ + mock.patch.object(resource_plan, "environment_for_plan", + side_effect=fake_environment_for_plan): + return coli.env_for(a) + + def _fake_gpu(self, index=0, name="NVIDIA GeForce RTX 5070 Ti", + total_mib=16384, free_mib=15000): + return {"index": index, "name": name, + "total_bytes": total_mib * 1024 * 1024, + "free_bytes": free_mib * 1024 * 1024} + + def test_win32_auto_enables_cuda_when_gpu_present(self): + e = self._env_for("win32", cuda=True, gpus=[self._fake_gpu()]) + self.assertEqual(e["COLI_CUDA"], "1") + self.assertEqual(e["COLI_GPUS"], "0") + # VRAM budget is sized from free VRAM by build_plan (real minus reserve), + # so it must be present and positive — never a guess or zero. + self.assertIn("CUDA_EXPERT_GB", e) + self.assertGreater(float(e["CUDA_EXPERT_GB"]), 0.0) + # Dense offload is an explicit opt-in (matches --auto-tier): not set here. + self.assertNotIn("CUDA_DENSE", e) + + def test_win32_falls_back_to_cpu_when_nvidia_smi_missing(self): + # coli_cuda.dll present (cuda=True) but nvidia-smi absent (no GPUs found) + # -> warn + CPU-only, never crash, never set COLI_CUDA. + e = self._env_for("win32", cuda=True, gpus=[]) + self.assertNotIn("COLI_CUDA", e) + self.assertNotIn("COLI_GPUS", e) + self.assertNotIn("CUDA_EXPERT_GB", e) + + def test_win32_cpu_build_stays_silent(self): + # No coli_cuda.dll (cuda=False) -> CPU build, nothing GPU-related emitted. + e = self._env_for("win32", cuda=False, gpus=[self._fake_gpu()]) + self.assertNotIn("COLI_CUDA", e) + self.assertNotIn("COLI_GPUS", e) + + def test_linux_bare_chat_not_auto_enabled(self): + # The auto-enable is scoped to win32: a Linux bare chat with a GPU + # present must NOT turn CUDA on (Linux keeps the explicit-flag UX). + e = self._env_for("linux", cuda=True, gpus=[self._fake_gpu()]) + self.assertNotIn("COLI_CUDA", e) + self.assertNotIn("CUDA_EXPERT_GB", e) + + if __name__ == "__main__": unittest.main() diff --git a/c/tests/test_i4_grouped.c b/c/tests/test_i4_grouped.c index 2a9521e..04e8d88 100644 --- a/c/tests/test_i4_grouped.c +++ b/c/tests/test_i4_grouped.c @@ -18,7 +18,7 @@ * index, a wrong group boundary or a swapped nibble cannot hide under it — * those are O(1) relative errors, not O(1e-6). */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static uint32_t rng_state=0xC0FFEEu; diff --git a/c/tests/test_idot.c b/c/tests/test_idot.c index b7109d4..343535c 100644 --- a/c/tests/test_idot.c +++ b/c/tests/test_idot.c @@ -7,7 +7,7 @@ * (sign-trick kernels must treat |−128| as 128 unsigned, not saturate to 127), * and random data at qrow_i8's contract (|x| <= 127, w full int8 range). */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static uint32_t rng_state=0x12345678u; diff --git a/c/tests/test_inefficiency.py b/c/tests/test_inefficiency.py new file mode 100644 index 0000000..00d8b58 --- /dev/null +++ b/c/tests/test_inefficiency.py @@ -0,0 +1,257 @@ +"""Inefficiency / regression tests for the colibri engine (tiny model, asserted). + +These run against the bundled glm_tiny model (~0.6 MB resident, ~0.1s/run) and +gate CI: a regression here means something broke. They run on a plain CPU-only +`glm.exe` build; the CUDA_* tests auto-skip if the engine wasn't built with +CUDA_DLL=1 (see tests/README_efficiency.md for the build command). + +The signals under test, and the inefficiency each catches: + - tok/s floor : a throughput regression (broken build / bad config) + - profile phases sum : telemetry accounting bug (other balloons) + - disk-wait not dominant: a tiny resident model should never be I/O-bound + - CPU determinism : greedy decode is reproducible (no stray RNG/threading) + - CUDA init path : COLI_CUDA=1 initializes and does not silently exit 2 + - CUDA dense uses VRAM : CUDA_DENSE=1 actually uploads tensors (no silent CPU fallback) + - CPU vs CUDA TF-match : identical weights+inputs → identical argmax (kernel bug guard) +""" +import os +import shutil +import unittest +from pathlib import Path + +from tools.efficiency import ( + parse_run, run_engine, disk_wait_share, tf_agreement, + TINY_TOK_S_FLOOR, MAX_DISK_WAIT_SHARE, MIN_CPU_CUDA_AGREEMENT, +) + +HERE = Path(__file__).resolve().parent +C_DIR = HERE.parent +ENGINE = C_DIR / "glm.exe" +TINY = C_DIR / "glm_tiny" + + +def _engine_present() -> bool: + """True iff BOTH the built engine AND the tiny fixture are available. + + These tests need glm.exe (a build artifact) AND glm_tiny/ (a generated + fixture, gitignored). CI runs `make check` = "dependency-free tests, no + model downloads" (workflow .github/workflows/check.yml, by design #140), so + neither is present there and these tests must SKIP rather than fail. They + run locally after `make glm.exe` (glm_tiny ships alongside the source, or + is regenerated by tools/make_glm_oracle.py). + """ + return ENGINE.exists() and (TINY / "config.json").exists() + + +def _skip_reason() -> str: + """Name exactly which prerequisite is missing, so the skip is actionable.""" + if not ENGINE.exists(): + return "glm.exe not built (run: make glm.exe)" + if not (TINY / "config.json").exists(): + return "glm_tiny fixture absent (gitignored; ship it locally or run tools/make_glm_oracle.py)" + return "" + + +def _cuda_available() -> bool: + """True iff the engine binary has the CUDA loader compiled in AND the DLL is present. + + The host is built with -DCOLI_CUDA only when CUDA_DLL=1 (Makefile). A binary + built without it embeds the string "this binary is CPU-only; rebuild" and + exits 2 on any CUDA env var — so we detect the CPU-only build by scanning + the binary for that marker (avoids a slow ldd/strings on every import; we + only read enough to find it). On Windows the DLL is also required + (backend_loader.c loads it at runtime); on Linux it's direct-linked. + """ + if not _engine_present(): + return False + try: + # Read once; the marker is near the read-only string table. 256 KB is + # plenty for this string and avoids loading a 1 MB binary into memory. + with open(ENGINE, "rb") as f: + blob = f.read(2 * 1024 * 1024) + if b"this binary is CPU-only" in blob: + return False # CPU-only build: COLI_CUDA=1 would exit 2. + except OSError: + pass + # Windows: DLL required at runtime. Linux: direct-linked (no DLL). + if os.name == "nt": + return (C_DIR / "coli_cuda.dll").exists() + import subprocess + try: + out = subprocess.run(["ldd", str(ENGINE)], capture_output=True, text=True) + return "libcudart" in out.stdout + except (FileNotFoundError, OSError): + return False + + +@unittest.skipUnless(_engine_present(), _skip_reason() or "glm.exe + glm_tiny required") +class TinyEfficiencyTest(unittest.TestCase): + """Asserted regression tests on the resident tiny model. Gates CI.""" + + def _run(self, **overlay): + return run_engine(overlay, engine=str(ENGINE), snap=str(TINY))[0] + + # -- telemetry contract --------------------------------------------------- + + def test_telemetry_parses(self): + """A REPLAY run must emit the throughput + PROFILE lines the suite keys on. + + If this fails, either the engine changed its output format (update the + parsers in tools/efficiency.py) or the run crashed early.""" + t = self._run(REPLAY="1", TEMP="0", NGEN="4") + self.assertEqual(t["returncode"], 0, f"engine exited non-zero:\n{t['stderr']}") + self.assertIn("tok_s", t["parsed"], f"missing tok/s line:\n{t['stderr']}") + self.assertIn("profile", t["parsed"], f"missing PROFILE line:\n{t['stderr']}") + self.assertIn("hit_pct", t["parsed"], f"missing expert hit line:\n{t['stderr']}") + self.assertIsNotNone(t["tok_s"]) + self.assertIsNotNone(t["profile"]) + + # -- throughput floor ----------------------------------------------------- + + def test_tiny_tok_s_floor(self): + """Tiny decode must beat TINY_TOK_S_FLOOR. + + The tiny model is fully resident and runs ~200 tok/s; the 20 tok/s + default floor is a 10x margin that catches broken builds or a pathological + config cascade (the cap=1 trap from ISSUE_new_model_resource_regression.md) + without flapping on machine noise.""" + t = self._run(REPLAY="1", TEMP="0", NGEN="8") + self.assertEqual(t["returncode"], 0, f"engine exited non-zero:\n{t['stderr']}") + self.assertGreaterEqual( + t["tok_s"], TINY_TOK_S_FLOOR, + f"tok/s {t['tok_s']:.1f} below floor {TINY_TOK_S_FLOOR} " + f"(regression, or a config cascade starving the cache)", + ) + + # -- accounting sanity ---------------------------------------------------- + + def test_profile_phases_present_and_nonneg(self): + """Every PROFILE phase must be present and non-negative. + + `other` can go slightly negative from timer overhead (the engine allows + it), but a large negative means the timers are double-counting.""" + t = self._run(REPLAY="1", TEMP="0", NGEN="4") + p = t["profile"] + for phase in ("disk", "expert_matmul", "attention", "lm_head"): + self.assertGreaterEqual(p[phase], -0.01, f"{phase} went negative: {p}") + # 'other' is a residual; allow a small negative from timer overlap. + self.assertGreaterEqual(p["other"], -0.05, f"other too negative (double-count): {p}") + + # -- disk-wait not dominant on a resident model --------------------------- + + def test_disk_wait_not_dominant(self): + """A fully-resident tiny model must NOT be I/O-bound. + + Everything fits in RAM; the expert-disk wait share should be ~0. If it + exceeds MAX_DISK_WAIT_SHARE, the cache/I/O path regressed — on a real + model this same regression would make decode I/O-bound (the exact + failure mode the [PROF] verdict flags).""" + t = self._run(REPLAY="1", TEMP="0", NGEN="8", PROF="1") + self.assertIn("time_shares", t["parsed"], f"missing [PROF] time shares:\n{t['stderr']}") + share = disk_wait_share(t) + self.assertIsNotNone(share) + self.assertLess( + share, MAX_DISK_WAIT_SHARE, + f"expert-I/O share {share:.0%} on a resident model — I/O path regressed", + ) + + # -- determinism ---------------------------------------------------------- + + def test_cpu_vs_cpu_determinism(self): + """Two greedy REPLAY runs with the same seed produce identical telemetry. + + TEMP=0 = greedy (no sampling), so tok/s and hit-rate must be reproducible. + A drift here means non-determinism crept into the decode path (stray + threading, uninitialized state) — which on a real model would make A/B + comparisons meaningless.""" + a = self._run(REPLAY="1", TEMP="0", NGEN="8", SEED="1") + b = self._run(REPLAY="1", TEMP="0", NGEN="8", SEED="1") + self.assertEqual(a["returncode"], 0) + self.assertEqual(b["returncode"], 0) + self.assertEqual(a["hit_pct"], b["hit_pct"], "greedy hit-rate drifted between runs") + # tok/s within 25% — exact equality is too strict across scheduler noise. + self.assertLess(abs(a["tok_s"] - b["tok_s"]) / max(a["tok_s"], b["tok_s"]), 0.25) + + +@unittest.skipUnless(_cuda_available(), + _skip_reason() or "CUDA build not present (run: make clean && make glm.exe CUDA_DLL=1 && make cuda-dll)") +class TinyCudaEfficiencyTest(unittest.TestCase): + """CUDA-path regression tests on the tiny model. Skip unless CUDA built. + + glm_tiny is small and fully resident, so CUDA here is fast and exercises the + real GPU code path (init, dense upload, kernel correctness) without the + long load time or memory pressure of the full model. These guard the + silent-failure modes that are otherwise invisible: + - COLI_CUDA=1 silently falling back to CPU (loader/DLL missing) + - CUDA_DENSE=1 uploading nothing + - a CUDA kernel producing different argmax than CPU on identical inputs + """ + + def _run(self, **overlay): + return run_engine(overlay, engine=str(ENGINE), snap=str(TINY))[0] + + def test_cuda_init_path(self): + """COLI_CUDA=1 must initialize the device and NOT exit 2. + + Exit 2 is the engine's "requested backend is unavailable" path + (glm.c: g_cuda_enabled check). A clean init prints the [CUDA] device + banner to stderr. If this fails, the DLL is broken or the loader can't + resolve symbols (ABI drift between backend_cuda.h and the dll).""" + t = self._run(COLI_CUDA="1", COLI_GPU="0", REPLAY="1", TEMP="0", NGEN="4") + self.assertNotEqual(t["returncode"], 2, + f"engine refused CUDA backend:\n{t['stderr']}") + self.assertTrue(t["cuda"]["enabled"], + f"no [CUDA] device banner on stderr:\n{t['stderr']}") + + def test_cuda_dense_uses_vram(self): + """CUDA_DENSE=1 must actually upload dense tensors to VRAM. + + The minimal GPU-exercising config (per backend_loader.c analysis): + COLI_CUDA=1 + CUDA_DENSE=1. Without CUDA_DENSE the dense path stays on + CPU and [CUDA] resident set reports 0 tensors — a silent no-op. This + catches that regression: after the run, resident_tensors > 0.""" + t = self._run(COLI_CUDA="1", COLI_GPU="0", CUDA_DENSE="1", + REPLAY="1", TEMP="0", NGEN="4") + self.assertEqual(t["returncode"], 0, f"engine exited non-zero:\n{t['stderr']}") + rt = t["cuda"]["resident_tensors"] + self.assertIsNotNone(rt, f"no [CUDA] resident set line:\n{t['stderr']}") + self.assertGreater(rt, 0, + f"CUDA_DENSE=1 but {rt} tensors resident — silent CPU fallback") + + def test_cpu_vs_cuda_tf_match(self): + """CPU and CUDA teacher-forcing must agree on most positions (DIRECTLY). + + Both paths prefill the SAME oracle sequence on the SAME weights, so their + argmaxes should match position-for-position — but not exactly: the two + backends accumulate dot-products in different orders (x86 SIMD vs CUDA + kernel), so a few near-tied logits flip. That divergence is expected + numeric behavior, not a kernel bug. A *catastrophic* kernel regression + (wrong GEMM, wrong scale, wrong fmt) would collapse agreement toward + random (~1/vocab = ~4%); the MIN_CPU_CUDA_AGREEMENT floor (default 70%) + catches that while tolerating harmless drift. + + We compare CPU-vs-CUDA *directly* (not via the oracle match-counts), + because both backends differ from the oracle at different positions and + the summary line can't tell "CPU≠CUDA" from "CPU≠oracle".""" + import json + ref = json.loads((C_DIR / "ref_glm.json").read_text()) + oracle = ref["tf_pred"] + + cpu = self._run(TF="1", TEMP="0") + cuda = self._run(COLI_CUDA="1", COLI_GPU="0", CUDA_DENSE="1", TF="1", TEMP="0") + self.assertEqual(cpu["returncode"], 0, f"CPU TF run failed:\n{cpu['stderr']}") + self.assertEqual(cuda["returncode"], 0, f"CUDA TF run failed:\n{cuda['stderr']}") + self.assertIn("tf_mismatches", cpu["parsed"], "CPU run missing per-position mismatches") + self.assertIn("tf_mismatches", cuda["parsed"], "CUDA run missing per-position mismatches") + + agree, diff = tf_agreement(cpu, cuda, oracle) + self.assertGreaterEqual( + agree, MIN_CPU_CUDA_AGREEMENT, + f"CPU-vs-CUDA argmax agreement {agree:.0%} below floor " + f"{MIN_CPU_CUDA_AGREEMENT:.0%} — CUDA kernel likely regressed. " + f"Differing positions: {diff[:10]}{'...' if len(diff)>10 else ''}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_kv_alloc.c b/c/tests/test_kv_alloc.c index 40ec00d..d080c08 100644 --- a/c/tests/test_kv_alloc.c +++ b/c/tests/test_kv_alloc.c @@ -4,7 +4,7 @@ * arrays twice on the second call -> allocator abort. No model file needed: * the CPU path of kv_alloc only reads c->n_layers/kv_lora/qk_rope. */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main int main(void){ diff --git a/c/tests/test_logit_nan.c b/c/tests/test_logit_nan.c new file mode 100644 index 0000000..de6dc2d --- /dev/null +++ b/c/tests/test_logit_nan.c @@ -0,0 +1,70 @@ +/* Regression for non-finite-logit poisoning of sampling. + * + * A single NaN or +Inf in the logits (a bad streamed expert tile, or an fp + * overflow in the matmul at a low-RAM eviction boundary) used to make softmax + * produce an all-NaN g_pbuf; dist_sample then never satisfied cum>=u and fell + * through to return token 0 — so the engine silently emitted an unbroken run of + * token 0 with no error, on the DEFAULT serve path (TEMP>0, 0 +#include +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main + +static int approx1(double x){ return x > 0.999 && x < 1.001; } + +int main(void){ + /* --- argmax_v must skip NaN (greedy decode + speculative-verify paths) --- */ + { float lo[8]={NAN,1.f,5.f,2.f,NAN,-3.f,4.f,0.f}; + assert(argmax_v(lo,8)==2 && "pick max finite (idx2=5.0), not NaN-pinned idx0"); } + { float lo[8]={3.f,INFINITY,1.f,2.f,0.f,-1.f,2.5f,1.5f}; + assert(argmax_v(lo,8)==1 && "pick the +Inf position"); } + { float lo[8]; for(int i=0;i<8;i++) lo[i]=NAN; + assert(argmax_v(lo,8)==0 && "all-NaN: no crash, defined fallback"); } + + g_temp=0.7f; g_nuc=0.9f; /* the default serve/chat sampling path */ + + /* --- dist_build: a NaN logit must yield a finite one-hot, not all-NaN --- */ + { float lo[8]={0.5f,1.f,NAN,8.f,0.2f,-1.f,0.f,0.3f}; /* max finite = idx3 (8.0) */ + dist_build(lo,8); + double sum=0; int nan=0; + for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; } + assert(!nan && "g_pbuf must be finite after a NaN logit"); + assert(approx1(sum) && "g_pbuf must normalize to 1"); + assert(approx1(g_pbuf[3]) && "mass must land on the max finite logit (idx3)"); + assert(dist_sample(8,-1)==3 && "sampler emits the finite argmax, not token 0"); } + + /* --- NaN at index 0: poisons the max scan itself (the old mx=lo[0] seed), + * the failure mode that starts before the sum (review note on #369) --- */ + { float lo[8]={NAN,1.f,0.5f,6.f,0.2f,-1.f,0.f,0.3f}; /* max finite = idx3 (6.0) */ + dist_build(lo,8); + double sum=0; int nan=0; + for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; } + assert(!nan && "NaN at lo[0] must not poison via the mx seed"); + assert(approx1(sum) && "still normalizes to 1"); + assert(dist_sample(8,-1)==3 && "emits the max finite logit, not token 0"); } + + /* --- all-NaN logits: worst case — must stay finite, no crash --- */ + { float lo[8]; for(int i=0;i<8;i++) lo[i]=NAN; + dist_build(lo,8); + for(int i=0;i<8;i++) assert(g_pbuf[i]==g_pbuf[i] && "all-NaN: g_pbuf stays finite"); } + + /* --- regression: clean logits still produce a valid distribution --- */ + { float lo[8]={0.1f,0.2f,3.0f,0.4f,0.5f,0.6f,0.7f,0.8f}; /* peak = idx2 */ + dist_build(lo,8); + double sum=0; int nan=0; + for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; } + assert(!nan && "clean softmax stays finite"); + assert(approx1(sum) && "clean softmax must sum to 1"); + assert(g_pbuf[2]>=g_pbuf[0] && "peak token keeps the most mass"); } + + printf("OK test_logit_nan: argmax_v NaN-skip + dist_build finite-collapse\n"); + return 0; +} diff --git a/c/tests/test_makefile_platform.py b/c/tests/test_makefile_platform.py index 3c72880..50b4c7a 100644 --- a/c/tests/test_makefile_platform.py +++ b/c/tests/test_makefile_platform.py @@ -35,7 +35,7 @@ class MakefilePlatformTests(unittest.TestCase): env["PATH"] = "" result = subprocess.run( - [MAKE, "--no-print-directory", "-B", "-n", "glm"], + [MAKE, "--no-print-directory", "-B", "-n", "colibri"], cwd=C_DIR, env=env, text=True, @@ -43,7 +43,7 @@ class MakefilePlatformTests(unittest.TestCase): check=True, ) - self.assertIn("-o glm.exe", result.stdout) + self.assertIn("-o colibri.exe", result.stdout) self.assertIn("-fopenmp", result.stdout) self.assertIn("-static", result.stdout) diff --git a/c/tests/test_openai_tools_e2e.py b/c/tests/test_openai_tools_e2e.py new file mode 100644 index 0000000..d921843 --- /dev/null +++ b/c/tests/test_openai_tools_e2e.py @@ -0,0 +1,182 @@ +"""End-to-end tool-calling test for the OpenAI gateway (#401). + +Unlike the unit tests in test_openai_server.py (which call parse_tool_calls / +render_chat directly), this suite runs openai_server.py as a real subprocess +against a mock engine that speaks the actual SERVE wire protocol +(READY / SUBMIT / DATA / DONE), then talks to it over real HTTP. It pins down +the full path a coding client exercises: tool declaration rendering, marker +suppression in streamed deltas (across chunk boundaries), tool_calls in both +response shapes, and the <|observation|> round trip. +""" +import json +import os +import socket +import subprocess +import sys +import tempfile +import unittest +import urllib.request +from pathlib import Path + +SERVER = Path(__file__).resolve().parent.parent / "openai_server.py" +MODEL_ID = "glm-5.2-colibri" + +# Mock engine: replies are keyed on the prompt so one process covers every case. +# Prompts received are appended to MOCK_LOG for assertions on the rendering. +MOCK_ENGINE = r'''#!/usr/bin/env python3 +import sys, os +out, inp = sys.stdout.buffer, sys.stdin.buffer +out.write(b"\x01\x01READY\x01\x01\n" + b"STAT 0 0 0 0 0\n"); out.flush() + +def reply(rid, text, chunks=1): + data = text.encode("utf-8") + n = max(1, len(data) // chunks) + for i in range(0, len(data), n): + part = data[i:i+n] + out.write(("DATA %s %d\n" % (rid, len(part))).encode() + part + b"\n"); out.flush() + out.write(("DONE %s STAT %d 1.0 50.0 10.0 42 0\n" % (rid, len(text.split()))).encode()) + out.flush() + +while True: + line = inp.readline() + if not line: break + f = line.decode().strip().split() + if not f or f[0] != "SUBMIT": continue + rid, plen = f[1], int(f[3]) + prompt = inp.read(plen).decode("utf-8", "replace"); inp.read(1) + with open(os.environ["MOCK_LOG"], "a") as log: + log.write(prompt + "\n\x00\n") + if "" in prompt: + reply(rid, "25 degrees and sunny in Rome.") + elif "weather in Rome" in prompt: + reply(rid, "get_weathercity" + "Rome") + elif "weather in Milan" in prompt: + # split across many tiny DATA chunks: streamed marker suppression must + # hold even when a marker straddles a chunk boundary + reply(rid, "Checking. get_weathercity" + "Milan", chunks=20) + else: + reply(rid, "Hello from the mock engine.") +''' + +TOOLS = [{"type": "function", "function": { + "name": "get_weather", + "description": "Current weather for a city", + "parameters": {"type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"]}}}] + + +@unittest.skipUnless(os.name == "posix", + "the mock engine is a shebang script the gateway execs directly; " + "Windows CreateProcess cannot run it. The gateway logic under test " + "is platform-independent and covered by the POSIX CI jobs.") +class ToolCallingE2E(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + mock = Path(cls.tmp.name) / "mock_engine.py" + mock.write_text(MOCK_ENGINE) + mock.chmod(0o755) + cls.mock_log = Path(cls.tmp.name) / "prompts.log" + cls.mock_log.touch() + with socket.socket() as probe: # free port, then hand it to the server + probe.bind(("127.0.0.1", 0)) + cls.port = probe.getsockname()[1] + env = dict(os.environ, MOCK_LOG=str(cls.mock_log)) + cls.server = subprocess.Popen( + [sys.executable, str(SERVER), "--model", cls.tmp.name, + "--engine", str(mock), "--port", str(cls.port)], + env=env, stderr=subprocess.DEVNULL) + cls.base = f"http://127.0.0.1:{cls.port}/v1" + for _ in range(100): + try: + urllib.request.urlopen(cls.base + "/models", timeout=2) + return + except OSError: + if cls.server.poll() is not None: + raise RuntimeError("gateway exited during startup") + import time + time.sleep(0.1) + raise RuntimeError("gateway did not come up") + + @classmethod + def tearDownClass(cls): + cls.server.terminate() + cls.server.wait(timeout=5) + cls.tmp.cleanup() + + def post(self, body, stream=False): + req = urllib.request.Request( + self.base + "/chat/completions", json.dumps(body).encode(), + {"Content-Type": "application/json"}) + resp = urllib.request.urlopen(req, timeout=30) + if not stream: + return json.loads(resp.read()) + events = [] + for raw in resp: + line = raw.decode().strip() + if line.startswith("data: ") and line != "data: [DONE]": + events.append(json.loads(line[6:])) + return events + + def test_tool_call_non_stream(self): + r = self.post({"model": MODEL_ID, "tools": TOOLS, + "messages": [{"role": "user", + "content": "What is the weather in Rome?"}]}) + choice = r["choices"][0] + self.assertEqual(choice["finish_reason"], "tool_calls") + calls = choice["message"]["tool_calls"] + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0]["function"]["name"], "get_weather") + self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"city": "Rome"}) + self.assertNotIn("", choice["message"].get("content") or "") + + def test_tool_call_streamed_markers_suppressed(self): + events = self.post({"model": MODEL_ID, "tools": TOOLS, "stream": True, + "messages": [{"role": "user", + "content": "What is the weather in Milan?"}]}, + stream=True) + deltas = [e["choices"][0]["delta"] for e in events if e["choices"]] + text = "".join(d.get("content") or "" for d in deltas) + self.assertNotIn("", text) + self.assertNotIn("", text) + calls = [d["tool_calls"] for d in deltas if d.get("tool_calls")] + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0][0]["function"]["name"], "get_weather") + self.assertEqual(json.loads(calls[0][0]["function"]["arguments"]), + {"city": "Milan"}) + finish = [e["choices"][0]["finish_reason"] for e in events + if e["choices"] and e["choices"][0].get("finish_reason")] + self.assertEqual(finish, ["tool_calls"]) + + def test_tool_result_round_trip(self): + r = self.post({"model": MODEL_ID, "tools": TOOLS, "messages": [ + {"role": "user", "content": "What is the weather in Rome?"}, + {"role": "assistant", "content": None, "tool_calls": [ + {"id": "call_x", "type": "function", + "function": {"name": "get_weather", + "arguments": "{\"city\": \"Rome\"}"}}]}, + {"role": "tool", "tool_call_id": "call_x", + "content": "25 degrees, sunny"}]}) + choice = r["choices"][0] + self.assertEqual(choice["finish_reason"], "stop") + self.assertFalse(choice["message"].get("tool_calls")) + self.assertIn("25 degrees", choice["message"]["content"]) + rendered = self.mock_log.read_text().split("\x00")[-2] + self.assertIn("<|observation|>25 degrees, sunny", + rendered) + self.assertIn("# Tools", rendered) + self.assertIn('"get_weather"', rendered) + + def test_no_tools_plain_text(self): + r = self.post({"model": MODEL_ID, + "messages": [{"role": "user", "content": "Hi!"}]}) + choice = r["choices"][0] + self.assertEqual(choice["finish_reason"], "stop") + self.assertIn("mock engine", choice["message"]["content"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_ragged_attention.cu b/c/tests/test_ragged_attention.cu new file mode 100644 index 0000000..dd1e2f8 --- /dev/null +++ b/c/tests/test_ragged_attention.cu @@ -0,0 +1,39 @@ +#include "../backend_cuda.h" + +#include +#include +#include + +int main(){ + int dev=0;if(!coli_cuda_init(&dev,1))return 77; + constexpr int S=3,H=2,Q=2,R=1,V=2,K=3,D=H*V,O=3,T=3; + std::vector w(H*(Q+V)*K),p(O*D),q(S*H*(Q+R)); + for(size_t i=0;i> l(S),r(S); + const float *lp[S],*rp[S]; + const void *keys[S]; + for(int s=0;s 24 threads, 12 physical cores. + + # The parser must return 12 physical cores under BOTH lscpu layouts: + # - 2-col: `lscpu -p=core,socket` emits exactly [core,socket] (this is + # what the probe actually requests; the previous fields[1]/[2] + # indexing skipped every line here and fell through to the + # logical count -> the regression JustVugg caught). + # - 3-col: bare `lscpu -p` prepends a CPU column -> [cpu,core,socket]. + # Taking the last two fields is correct in both cases. + layouts = { + "2-col (-p=core,socket)": ( + "# core,socket\n" + + "\n".join(f"{core},0" for core in range(12) for _ in range(2))), + "3-col (bare -p, CPU prefix)": ( + "# CPU,Core,Socket\n" + + "\n".join(f"{cpu},{core},0" for core in range(12) for cpu in range(2))), + } + for label, blob in layouts.items(): + with mock.patch("resource_plan.subprocess.run", + return_value=lscpu(blob)), \ + mock.patch.object(sys, "platform", "linux"): + plan = build_plan(self.model, available_memory=16 * GB, + available_disk=1, gpus=[]) + env = environment_for_plan(plan) + self.assertEqual(plan["cpu"]["physical_cores"], 12, label) + self.assertEqual(env["OMP_NUM_THREADS"], "12", label) + + def test_plan_does_not_set_omp_affinity_vars(self): + # The real #325 regression: --auto-tier set OMP_PROC_BIND=spread + + # OMP_PLACES=cores, which ran before the engine's overwrite=0 setenv and + # so won, collapsing the OpenMP team to one CPU on the reporter's 64-core + # Linux box even though OMP_NUM_THREADS was correct. The plan must leave + # affinity to the engine's own hot-thread tuning (which prefers 'close'). + plan = build_plan(self.model, available_memory=16 * GB, + available_disk=1, gpus=[], physical_cpus=64) + env = environment_for_plan(plan) + self.assertEqual(env["OMP_NUM_THREADS"], "64") + self.assertNotIn("OMP_PROC_BIND", env) + self.assertNotIn("OMP_PLACES", env) + + def test_plan_conserves_budget_and_experts_above_256gb(self): + # Regression for #325's reporter: a 512 GB machine loading the whole + # model into RAM. Verify the budget math stays exact at large RAM sizes + # (no integer truncation, no over-allocation, no experts lost between + # tiers). Checked at 256/512/800 GB to bracket the reporter's box. + for ram_gb in (256, 512, 800): + plan = build_plan(self.model, ram_gb=ram_gb, available_disk=1, + gpus=[], physical_cpus=64) + ram = plan["tiers"]["ram"] + # RAM budget never over-allocated: dense + runtime + cache <= budget. + allocated = (ram["dense_bytes"] + ram["runtime_bytes"] + + ram["expert_cache_bytes"]) + self.assertLessEqual(allocated, ram["budget_bytes"], + f"over-allocated RAM at {ram_gb} GB") + # Every expert byte is accounted for exactly once across the tiers. + tiers = plan["tiers"] + tiered = (tiers["vram"]["hot_expert_bytes"] + + ram["warm_expert_bytes"] + + tiers["disk"]["cold_expert_bytes"]) + self.assertEqual(tiered, plan["model"]["expert_bytes"], + f"expert bytes lost/duplicated at {ram_gb} GB") + # A positive RAM budget yields a non-negative cache and a sensible cap. + self.assertGreaterEqual(ram["expert_cache_bytes"], 0) + self.assertGreaterEqual(ram["cache_slots_per_layer"], 0) + def test_filters_requested_devices(self): gpus = [{"index": 0, "name": "a", "total_bytes": 8 * GB, "free_bytes": 8 * GB}] plan = build_plan(self.model, available_memory=16 * GB, available_disk=1, @@ -105,26 +185,111 @@ class ResourcePlanTest(unittest.TestCase): {"index": 1, "name": "b", "total_bytes": 12 * GB, "free_bytes": 10 * GB}, ] plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB, - available_disk=1, gpus=gpus) + available_disk=1, gpus=gpus, cpu_sockets=2) env = environment_for_plan(plan, {"RAM_GB": "12", "PIN": "stats.txt", "COLI_GPUS": "1"}) self.assertEqual(env["RAM_GB"], "12") self.assertEqual(env["COLI_CUDA"], "1") self.assertEqual(env["COLI_GPUS"], "1") self.assertEqual(env["OMP_NUM_THREADS"], str(plan["cpu"]["physical_cores"])) - if sys.platform == "win32": - # MinGW libgomp: niente affinity su Windows, le chiavi non vanno emesse - self.assertNotIn("OMP_PROC_BIND", env) - self.assertNotIn("OMP_PLACES", env) - else: - self.assertEqual(env["OMP_PROC_BIND"], "spread") - self.assertEqual(env["OMP_PLACES"], "cores") + # The plan must NOT set OMP_PROC_BIND / OMP_PLACES on any platform: + # the engine's own hot-thread tuning owns affinity (it prefers + # OMP_PROC_BIND=close for the back-to-back per-expert matmuls). Setting + # spread + cores here ran before the engine's overwrite=0 setenv and so + # won, collapsing the team to one CPU on some libgomp topologies (#325). + self.assertNotIn("OMP_PROC_BIND", env) + self.assertNotIn("OMP_PLACES", env) self.assertEqual(env["PIN_GB"], env["CUDA_EXPERT_GB"]) explicit_threads = environment_for_plan(plan, {"OMP_NUM_THREADS": "7", "OMP_PROC_BIND": "close"}) self.assertEqual(explicit_threads["OMP_NUM_THREADS"], "7") self.assertEqual(explicit_threads["OMP_PROC_BIND"], "close") + + if sys.platform.startswith("linux"): + self.assertEqual(env["COLI_NUMA"], "1") + explicit_numa = environment_for_plan(plan, {"COLI_NUMA": "0"}) + self.assertEqual(explicit_numa["COLI_NUMA"], "0") + + def test_single_socket_plan_does_not_enable_numa(self): + plan = build_plan(self.model, available_memory=16 * GB, available_disk=1, + gpus=[], physical_cpus=8, cpu_sockets=1) + self.assertNotIn("COLI_NUMA", environment_for_plan(plan)) + + def test_auto_tune_mtp_off_when_compute_bound(self): + # Tiny model with 64 GB RAM and no GPU: all experts fit in RAM with no + # warm tier, so the plan classifies as compute-bound. + plan = build_plan(self.model, ram_gb=64, available_memory=64 * GB, + available_disk=100 * GB, gpus=[], physical_cpus=24, + cpu_sockets=2) + # With such a small model fully in RAM and no GPU, bottleneck is compute + self.assertEqual(plan["bottleneck_class"], "compute") + self.assertIn("DRAFT", plan["tune"]) + self.assertEqual(plan["tune"]["DRAFT"]["value"], "0") + env = environment_for_plan(plan) + self.assertEqual(env["DRAFT"], "0") + explicit = environment_for_plan(plan, {"DRAFT": "3"}) + self.assertEqual(explicit["DRAFT"], "3") + + def test_auto_tune_mtp_off_when_disk_low_hit(self): + # Use a model large enough that 8 GB RAM can't hold all experts. + big = tempfile.TemporaryDirectory() + bigmodel = Path(big.name) + (bigmodel / "config.json").write_text(json.dumps({ + "num_hidden_layers": 2, "n_routed_experts": 4, + "kv_lora_rank": 4, "qk_rope_head_dim": 2, + "qk_nope_head_dim": 3, "v_head_dim": 5, "num_attention_heads": 2, + })) + expert_size = 3 * GB # each expert 3 GB → 12 GB total, won't fit in 8 GB budget + write_shard(bigmodel / "out-00000.safetensors", [ + ("model.embed_tokens.weight", 100), + ("model.layers.0.self_attn.q_a_proj.weight", 200), + ]) + for i in range(4): + write_shard(bigmodel / f"out-{i+1:05d}.safetensors", [ + (f"model.layers.1.mlp.experts.{i}.gate_proj.weight", expert_size), + ]) + plan = build_plan(bigmodel, ram_gb=0, available_memory=4 * GB, + available_disk=100 * GB, gpus=[], physical_cpus=8, + cpu_sockets=1) + big.cleanup() + self.assertEqual(plan["bottleneck_class"], "disk") + self.assertLess(plan["projected_hit_rate"], 0.90) + self.assertEqual(plan["tune"]["DRAFT"]["value"], "0") + + def test_auto_tune_pipe_multi_gpu(self): + gpus = [ + {"index": 0, "name": "a", "total_bytes": 32 * GB, "free_bytes": 30 * GB}, + {"index": 1, "name": "b", "total_bytes": 32 * GB, "free_bytes": 30 * GB}, + ] + plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB, + available_disk=1, gpus=gpus, cpu_sockets=2) + self.assertEqual(plan["tune"]["COLI_CUDA_PIPE"]["value"], "2") + env = environment_for_plan(plan) + self.assertEqual(env["COLI_CUDA_PIPE"], "2") + + def test_auto_tune_pipe_single_gpu(self): + gpus = [{"index": 0, "name": "a", "total_bytes": 12 * GB, "free_bytes": 10 * GB}] + plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB, + available_disk=1, gpus=gpus, cpu_sockets=1) + self.assertEqual(plan["tune"]["COLI_CUDA_PIPE"]["value"], "1") + + def test_auto_tune_numa_hint_for_cpu_only(self): + plan = build_plan(self.model, ram_gb=64, available_memory=64 * GB, + available_disk=1, gpus=[], physical_cpus=64, cpu_sockets=2) + self.assertIn("_numa_hint", plan["tune"]) + self.assertIn("numactl", plan["tune"]["_numa_hint"]) + self.assertIn("auto-tune", format_plan(plan)) + + def test_format_plan_shows_tune_and_hit_rate(self): + plan = build_plan(self.model, ram_gb=64, available_memory=64 * GB, + available_disk=100 * GB, gpus=[], physical_cpus=24, + cpu_sockets=1) + text = format_plan(plan) + self.assertIn("hit", text) + self.assertIn("auto-tune", text) + self.assertIn("DRAFT", text) + def test_cpu_binary_does_not_apply_gpu_tier(self): plan = build_plan(self.model, available_memory=16 * GB, available_disk=1, gpus=[{"index": 0, "name": "a", "total_bytes": 8 * GB, @@ -163,5 +328,73 @@ class ResourcePlanTest(unittest.TestCase): self.assertIn("expected_bottleneck", plan) +class PhysicalCpuCountTest(unittest.TestCase): + """Regression for #325: --auto-tier pinned decode to one core because + physical_cpu_count() silently returned 1. + + Two root causes this locks down: + 1. lscpu -p prepends a CPU column, so `-p=core,socket` emits + CPU,Core,Socket; counting rows counted logical SMT siblings. + 2. any probe failure fell through to ``os.cpu_count() or 1`` and the + ``or 1`` could pin a constrained/cgroup'd box to a single core. + """ + + def _lscpu(self, stdout): + return subprocess.CompletedProcess(args=[], returncode=0, + stdout=stdout, stderr="") + + def _lscpu_topology(self, sockets, cores_per_socket, threads_per_core): + # Real lscpu shape: socket-local core IDs repeat across sockets; the + # CPU column (always prepended) is a unique logical-CPU index. + rows, cpu = [], 0 + for sock in range(sockets): + for core in range(cores_per_socket): + for _ in range(threads_per_core): + rows.append(f"{cpu},{core},{sock}") + cpu += 1 + return "# CPU,Core,Socket\n" + "\n".join(rows) + + def test_counts_physical_cores_not_smt_threads(self): + blob = self._lscpu_topology(sockets=2, cores_per_socket=16, threads_per_core=2) + with mock.patch("resource_plan.subprocess.run", return_value=self._lscpu(blob)), \ + mock.patch.object(sys, "platform", "linux"): + self.assertEqual(physical_cpu_count(), 32) + + def test_single_socket_no_smt(self): + blob = self._lscpu_topology(sockets=1, cores_per_socket=8, threads_per_core=1) + with mock.patch("resource_plan.subprocess.run", return_value=self._lscpu(blob)), \ + mock.patch.object(sys, "platform", "linux"): + self.assertEqual(physical_cpu_count(), 8) + + def test_skips_offline_core_socket_fields(self): + # VMs / large NUMA boxes emit "-" for offline core or socket IDs; that + # used to raise ValueError, discard the whole parse, and fall through + # to the single-core fallback. + blob = "# CPU,Core,Socket\n0,0,0\n1,-,0\n2,1,0\n3,1,0\n" + with mock.patch("resource_plan.subprocess.run", return_value=self._lscpu(blob)), \ + mock.patch.object(sys, "platform", "linux"): + self.assertEqual(physical_cpu_count(), 2) + + def test_lscpu_missing_falls_back_to_logical_not_silent_one(self): + # The bug: lscpu absent -> os.cpu_count() or 1. On a constrained box + # os.cpu_count() can be 1. We still must never silently pick 1 without + # a warning, and when logical cores exist they must be used. + import os + with mock.patch("resource_plan.subprocess.run", side_effect=FileNotFoundError), \ + mock.patch.object(sys, "platform", "linux"), \ + mock.patch("resource_plan.os.cpu_count", return_value=16), \ + mock.patch("sys.stderr"): + self.assertEqual(physical_cpu_count(), 16) + + def test_zero_logical_cores_warns_and_returns_one(self): + # The genuine degenerate case: no probe works and os.cpu_count() is + # None/1. Must return 1 (engine needs a positive team size) but warn. + with mock.patch("resource_plan.subprocess.run", side_effect=FileNotFoundError), \ + mock.patch.object(sys, "platform", "linux"), \ + mock.patch("resource_plan.os.cpu_count", return_value=None), \ + mock.patch("sys.stderr"): + self.assertEqual(physical_cpu_count(), 1) + + if __name__ == "__main__": unittest.main() diff --git a/c/tests/test_sample_nan.c b/c/tests/test_sample_nan.c new file mode 100644 index 0000000..4f31def --- /dev/null +++ b/c/tests/test_sample_nan.c @@ -0,0 +1,62 @@ +/* Regressione #369: un solo logit NaN o +Inf faceva emettere a dist_build/dist_sample + * il token 0 PER SEMPRE, in silenzio (il fallback `g_pbuf[i]>0` e' falso su NaN ovunque). + * Ora la distribuzione degenere ripiega sull'argmax dei logit FINITI e avvisa una volta. + * + * Il test verifica: (a) con logit sani il campionamento resta corretto; (b) con NaN/+Inf + * iniettato il token scelto e' l'argmax dei FINITI (mai 0 per default), su ogni posizione + * del NaN inclusa lo[0]; (c) nessun NaN sopravvive in g_pbuf. */ +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main +#include + +static uint32_t rs=0x1234abcd; static uint32_t xr(){rs^=rs<<13;rs^=rs>>17;rs^=rs<<5;return rs;} + +static int pbuf_has_nan(int V){ for(int i=0;i 0 e nessun NaN nel buffer */ + for(int i=0;ipv){pv=g_pbuf[i];picked=i;} + if(b<2 && picked!=amax){ /* NaN e +Inf: delta esatto su amax */ + printf(" FAIL: %s @ %d -> picked %d, atteso argmax finito %d\n",bn[b],at,picked,amax); fail=1; + } + } + } + if(!fail) printf(" NaN/+Inf iniettato: argmax dei finiti vince, mai 0/NaN ok\n"); + + /* (c) caso estremo: TUTTI non finiti -> non deve crashare, buffer valido */ + for(int i=0;i +#include +#include +#ifndef _WIN32 +#include +#include +#endif + +#include "../st.h" + +#define CHECK(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "%s:%d: check failed: %s\n", __FILE__, __LINE__, #condition); \ + return 1; \ + } \ +} while (0) + +static void write_snap(const char *dir, int truncate_bytes) { + char path[512]; + snprintf(path, sizeof(path), "%s/model.safetensors", dir); + unsigned char data[96]; + for (int i = 0; i < 96; i++) data[i] = (unsigned char)(i * 7 + 3); + const char *hdr = "{\"t\":{\"dtype\":\"U8\",\"shape\":[96],\"data_offsets\":[0,96]}}"; + uint64_t hlen = strlen(hdr); + FILE *f = fopen(path, "wb"); + fwrite(&hlen, 8, 1, f); + fwrite(hdr, 1, hlen, f); + fwrite(data, 1, (size_t)(96 - truncate_bytes), f); + fclose(f); +} + +int main(void) { + /* relative to the CWD, per test_stops: MinGW .exe files resolve Windows + * paths and "/tmp" is not one */ + char dir[] = "test_st_pread_XXXXXX"; + if (!mkdtemp(dir)) { perror("mkdtemp"); return 1; } + + /* 1) chunk loop: 96-byte tensor read 7 bytes at a time, content exact */ + write_snap(dir, 0); + shards S; st_init(&S, dir); + unsigned char out[96] = {0}; + st_read_raw(&S, "t", out, 0); + for (int i = 0; i < 96; i++) CHECK(out[i] == (unsigned char)(i * 7 + 3)); + +#ifndef _WIN32 + /* 2) shard truncated AFTER st_init (init validates static bounds, so the + * pread path only fires when the file shrinks underneath a live handle): + * child must exit(1) with an honest message, not perror's "Success" */ + char shard[512]; snprintf(shard, sizeof(shard), "%s/model.safetensors", dir); + struct stat sb; CHECK(stat(shard, &sb) == 0); + CHECK(truncate(shard, sb.st_size - 40) == 0); + int pipefd[2]; CHECK(pipe(pipefd) == 0); + pid_t pid = fork(); CHECK(pid >= 0); + if (pid == 0) { + dup2(pipefd[1], 2); close(pipefd[0]); close(pipefd[1]); + unsigned char buf[96]; + st_read_raw(&S, "t", buf, 0); /* inherited handles; must exit(1) inside */ + _exit(42); /* reaching here = bug */ + } + close(pipefd[1]); + char err[512] = {0}; + ssize_t n = read(pipefd[0], err, sizeof(err)-1); (void)n; + close(pipefd[0]); + int status = 0; waitpid(pid, &status, 0); + CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 1); + CHECK(strstr(err, "short read") != NULL); + CHECK(strstr(err, "Success") == NULL); +#else + /* fork/pipe/truncate are POSIX; Windows still runs the chunk-loop check */ + printf("test_st_pread: truncation subtest skipped on Windows\n"); +#endif + + char cmd[600]; +#ifdef _WIN32 + snprintf(cmd, sizeof(cmd), "rmdir /s /q %s", dir); +#else + snprintf(cmd, sizeof(cmd), "rm -rf %s", dir); +#endif + if (system(cmd)) {} + printf("test_st_pread: chunk loop + honest truncation error: ok\n"); + return 0; +} diff --git a/c/tests/test_stops.c b/c/tests/test_stops.c index 00f922b..5e7e9d2 100644 --- a/c/tests/test_stops.c +++ b/c/tests/test_stops.c @@ -20,7 +20,7 @@ * Defense 2 is what makes this robust against checkpoints we don't control: * even with BOTH configs mutilated, a control token cannot leak into a reply. */ #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static const char *TOKJSON = diff --git a/c/tests/test_tok_o200k.c b/c/tests/test_tok_o200k.c new file mode 100644 index 0000000..fe62890 --- /dev/null +++ b/c/tests/test_tok_o200k.c @@ -0,0 +1,64 @@ +/* o200k pre-tokenizer validation against HF-tokenizers-generated expectations. + * Self-contained for the test-c harness: loads tests/tok_o200k_tiny.json (a + * synthetic byte-level BPE whose Split regex is the o200k pattern — a few KB, + * no model download) and scores tests/tok_o200k_cases.txt, whose expected ids + * were produced by HF `tokenizers` on the same file. Guards the case-aware + * letter matcher, contractions, digit groups, the [\r\n/]* punctuation tail, + * whitespace branches, and added-token atomicity; round-trips every case. + * The cl100k path is untouched by construction (dispatch requires \p{Lu} in + * the tokenizer's own Split pattern) and stays covered by the GLM oracle. */ +#define _GNU_SOURCE +#include "../tok.h" + +int main(void) { + Tok T; + tok_load(&T, "tests/tok_o200k_tiny.json"); + if (!T.o200k) { fprintf(stderr, "test_tok_o200k: o200k pattern not detected\n"); return 1; } + FILE *f = fopen("tests/tok_o200k_cases.txt", "rb"); + if (!f) { perror("tests/tok_o200k_cases.txt"); return 1; } + /* fgets, not getline: MinGW's UCRT lacks getline and this must run on + * the windows job. Case lines are short; 8 KB is generous. */ + char line[8192]; + int pass = 0, tot = 0, dpass = 0; + while (fgets(line, sizeof(line), f)) { + size_t nr = strlen(line); + while (nr > 0 && (line[nr-1] == '\n' || line[nr-1] == '\r')) line[--nr] = 0; + if (nr == 0) continue; + char *tab = strchr(line, '\t'); if (!tab) continue; + *tab = 0; + const char *text = line, *idstr = tab + 1; + char tbuf[4096]; int tn = 0; + for (const char *q = text; *q && tn < 4095; q++) { + if (q[0]=='\\' && q[1]=='n') { tbuf[tn++]='\n'; q++; } + else if (q[0]=='\\' && q[1]=='t') { tbuf[tn++]='\t'; q++; } + else if (q[0]=='\\' && q[1]=='r') { tbuf[tn++]='\r'; q++; } + else if (q[0]=='\\' && q[1]=='\\') { tbuf[tn++]='\\'; q++; } + else tbuf[tn++] = *q; + } + tbuf[tn] = 0; + int exp[512], ne = 0; + for (const char *q = idstr; *q; ) { + while (*q == ',' || *q == ' ') q++; + if (!*q) break; + exp[ne++] = atoi(q); + while (*q && *q != ',') q++; + } + int got[512]; int ng = tok_encode(&T, tbuf, tn, got, 512); + int ok = (ng == ne); + for (int i = 0; i < ng && ok; i++) ok = (got[i] == exp[i]); + tot++; if (ok) pass++; + char dec[8192]; int dn = tok_decode(&T, got, ng, dec, 8191); + int drt = (dn == tn) && !memcmp(dec, tbuf, tn); + if (drt) dpass++; + if (!ok || !drt) { + fprintf(stderr, "MISMATCH text=%s\n exp(%d):", text, ne); + for (int i = 0; i < ne; i++) fprintf(stderr, " %d", exp[i]); + fprintf(stderr, "\n got(%d):", ng); + for (int i = 0; i < ng; i++) fprintf(stderr, " %d", got[i]); + fprintf(stderr, "\n decode_ok=%d\n", drt); + } + } + fclose(f); + printf("test_tok_o200k: ENCODE %d/%d DECODE %d/%d\n", pass, tot, dpass, tot); + return (pass == tot && dpass == tot) ? 0 : 2; +} diff --git a/c/tests/test_topp.c b/c/tests/test_topp.c new file mode 100644 index 0000000..49f45d6 --- /dev/null +++ b/c/tests/test_topp.c @@ -0,0 +1,280 @@ +/* Top-p (nucleus) truncation in dist_build: the partial-select rewrite (#335) must be + * indistinguishable from the old full-vocab qsort for every shape dist_sample can see. + * + * Why this test exists (#335): dist_build() previously qsort-ed the entire 151936-entry + * vocab on every sampled token to find the few-hundred-token head whose cumulative mass + * reaches g_nuc. It now heapifies (O(V)) and pops only the head (k * O(log V)). The win + * is structural; the risk is a silent sampling-distribution change, because the contract + * is subtle: + * + * dist_sample() iterates g_pbuf[0..V-1] BY TOKEN ID and sums probabilities directly. + * So dist_build MUST leave g_pbuf indexed by id (never reordered) AND must zero every + * truncated tail entry -- merely excluding the tail from the head would leave mass on + * it and the sampled distribution would drift with no crash and no error. + * + * Strategy: drive the REAL dist_build (via the test_stops.c include-glm.c pattern) on a + * sweep of distributions and g_nuc values, and compare against an INDEPENDENT reference + * that re-implements the OLD algorithm (full qsort + zero-tail + renorm) in double on a + * private buffer. On shapes with no ties the renormalized head must be BIT-IDENTICAL to + * the reference (the issue's stated invariant: s2 accumulates in the same descending + * order). On tie shapes, where the unstable qsort already left ordering unspecified, we + * check multiset equality instead. Every shape also checks: exact-zero tails, head sums + * to 1.0, and a sane keep-count. + * + * No scratch files: the test runs entirely in memory (no mkdtemp), so it builds clean on + * the Windows MinGW CI job without the unmerged compat shim (#352). */ +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main + +#include + +static int g_nfails = 0; + +/* pointer set by ref_build so cmp_ref_desc can read the current reference buffer + * (the qsort comparator gets no user-data argument in C). */ +static const double *g_ref_p = NULL; + +#define FAIL(fmt, ...) do { \ + fprintf(stderr, " FAIL [%s V=%d nuc=%.3f shape=%s]: " fmt "\n", \ + label, V, nuc, shape_name, ##__VA_ARGS__); \ + g_nfails++; \ + return; \ +} while (0) + +/* ---- independent reference: the OLD algorithm, in double, on a private buffer ------- */ +/* Stable qsort by descending probability (ties broken by ascending index, which makes + * the reference deterministic regardless of the production comparator). */ +static int cmp_ref_desc(const void *a, const void *b){ + double pa = ((const double *)g_ref_p)[*(const int*)a]; + double pb = ((const double *)g_ref_p)[*(const int*)b]; + if (pa < pb) return 1; + if (pa > pb) return -1; + /* tie -> lower index first (stable, unlike the production comparator) */ + return *(const int*)a - *(const int*)b; +} + +/* Build the reference distribution into out[0..V-1] (indexed by token id), mirroring the + * old dist_build: softmax(lo/temp) truncated to top-p nuc, tail zeroed, head renormalized. + * Returns the keep-count through *keep_out. */ +static void ref_build(const float *lo, int V, double temp, double nuc, + double *out, int *pidx, int *keep_out){ + double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i]; + double s = 0, invt = 1.0 / (temp > 1e-4 ? temp : 1e-4); + for (int i = 0; i < V; i++){ out[i] = exp((lo[i]-mx)*invt); s += out[i]; } + for (int i = 0; i < V; i++) out[i] /= s; + + if (nuc > 0 && nuc < 1.0){ + for (int i = 0; i < V; i++) pidx[i] = i; + qsort(pidx, V, sizeof(int), cmp_ref_desc); + double cum = 0; int keep = V; + for (int i = 0; i < V; i++){ cum += out[pidx[i]]; if (cum >= nuc){ keep = i+1; break; } } + double s2 = 0; + for (int i = keep; i < V; i++) out[pidx[i]] = 0; + for (int i = 0; i < keep; i++) s2 += out[pidx[i]]; + for (int i = 0; i < keep; i++) out[pidx[i]] /= s2; + *keep_out = keep; + } else { + *keep_out = V; + } +} + +/* count how many production g_pbuf entries are non-zero == the head size */ +static int head_count(int V){ + int n = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) n++; return n; +} + +/* Run one case: load logits into g_pbuf via the real dist_build, compare to reference. + * shape_name is for diagnostics only. */ +static void check_case(const char *label, int V, double nuc, const char *shape_name, + const float *lo){ + /* reference on a private buffer */ + double *ref = malloc((size_t)V * sizeof(double)); + int *ridx = malloc((size_t)V * sizeof(int)); + int ref_keep = 0; + g_ref_p = ref; /* cmp_ref_desc reads this */ + ref_build(lo, V, g_temp, nuc, ref, ridx, &ref_keep); + + /* production: drive the real dist_build (writes the global g_pbuf) */ + g_nuc = (float)nuc; + dist_build(lo, V); + + int got_keep = head_count(V); + + /* 1. keep-count must match the reference exactly. The partial select and the old + * qsort keep the same NUMBER of tokens by construction (same cumulative-mass rule); + * a count divergence is a real bug, not a tie artifact. */ + if (got_keep != ref_keep) + FAIL("keep-count mismatch: got %d, ref %d", got_keep, ref_keep); + + /* 2. Detect ties across the WHOLE pre-truncation distribution, not just the kept set. + * A tie at the head/tail boundary makes which-side-a-token-lands-on interchangeable: + * both algorithms keep the right count but may keep different MEMBERS. So any input + * with a duplicated softmax value needs the relaxed multiset comparison below. We + * detect this on the reference softmax (pre-truncation) by sorting all V values. */ + int has_ties = 0; + { + double *all = malloc((size_t)V * sizeof(double)); + /* reconstruct the pre-truncation softmax the same way ref_build does */ + double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i]; + double s = 0, invt = 1.0 / (g_temp > 1e-4 ? g_temp : 1e-4); + for (int i = 0; i < V; i++){ all[i] = exp((lo[i]-mx)*invt); s += all[i]; } + for (int i = 0; i < V; i++) all[i] /= s; + for (int a = 1; a < V; a++){ double k = all[a]; int b = a-1; + while (b >= 0 && all[b] > k){ all[b+1] = all[b]; b--; } all[b+1] = k; } + for (int a = 1; a < V; a++) if (all[a] == all[a-1]){ has_ties = 1; break; } + free(all); + } + + if (has_ties){ + /* Multiset equality of the non-zero (head) values. Ties make membership + * interchangeable, so we compare sorted value-multisets, not id-aligned values. + * Tolerance is 1e-6 relative -- the engine uses float arithmetic, the reference + * double, so sub-ULP noise is expected (matches test_i4_grouped.c's convention). */ + double *got = malloc((size_t)ref_keep * sizeof(double)); + int gm = 0; + for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) got[gm++] = (double)g_pbuf[i]; + if (gm != ref_keep) + FAIL("tie-shape head size mismatch: got %d non-zero, ref %d", gm, ref_keep); + for (int a = 1; a < gm; a++){ double k = got[a]; int b = a-1; + while (b >= 0 && got[b] > k){ got[b+1] = got[b]; b--; } got[b+1] = k; } + double *rsort = malloc((size_t)ref_keep * sizeof(double)); + int rm = 0; + for (int i = 0; i < V; i++) if (ref[i] != 0.0) rsort[rm++] = ref[i]; + for (int a = 1; a < rm; a++){ double k = rsort[a]; int b = a-1; + while (b >= 0 && rsort[b] > k){ rsort[b+1] = rsort[b]; b--; } rsort[b+1] = k; } + int mm = 0; double worst = 0; + for (int i = 0; i < gm; i++){ + double d = fabs(got[i] - rsort[i]); + double rel = rsort[i] > 1e-30 ? d / rsort[i] : d; + if (rel > worst) worst = rel; + if (rel > 1e-6) mm++; + } + free(got); free(rsort); + if (mm) FAIL("tie-shape multiset mismatch: %d/%d head values differ beyond 1e-6 rel (worst %.3g)", + mm, ref_keep, worst); + } else { + /* No ties anywhere: membership is forced, so compare id-aligned head values. The + * engine computes in float (g_pbuf /= (float)s2) while the reference uses double, + * so the comparison is relative-tolerance (1e-6), not bit-exact -- the partial + * select and qsort accumulate s2 in the same descending order, so any difference + * is pure float-rounding noise, not an ordering bug. */ + int bad = 0; int first_id = -1; float gv = 0, rv = 0; double worst = 0; + for (int i = 0; i < V; i++){ + if (ref[i] == 0.0) continue; /* tail */ + float want = (float)ref[i]; + double d = fabs((double)g_pbuf[i] - (double)want); + double rel = fabs((double)want) > 1e-30 ? d / fabs((double)want) : d; + if (rel > worst) worst = rel; + if (rel > 1e-6){ + bad++; if (first_id < 0){ first_id = i; gv = g_pbuf[i]; rv = want; } + if (bad > 3) break; + } + } + if (bad) + FAIL("head not within 1e-6 rel of reference: %d entries differ (first id %d: got %.9g want %.9g, worst %.3g)", + bad, first_id, (double)gv, (double)rv, worst); + } + + /* 3. head must renormalize to 1.0 (within float epsilon) */ + double sum = 0; for (int i = 0; i < V; i++) sum += g_pbuf[i]; + if (fabs(sum - 1.0) > 1e-5) + FAIL("head does not sum to 1.0: sum=%.12g (keep=%d)", sum, got_keep); + + free(ref); free(ridx); + printf(" ok [V=%d nuc=%.3f shape=%s keep=%d%s sum=%.10f]\n", + V, nuc, shape_name, got_keep, has_ties ? " (ties)" : "", sum); +} +#undef FAIL + +/* deterministic xorshift32 RNG (matches the test_i4_grouped.c convention) */ +static uint32_t rng_state = 0x12345678u; +static uint32_t xr(void){ rng_state ^= rng_state << 13; rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; return rng_state; } +static double frand(void){ return (xr() >> 8) * (1.0 / 16777216.0); } /* [0,1) */ + +/* fill logits for a given shape. Shapes chosen to stress the comparator and the head/tail + * boundary differently. */ +static void fill_shape(float *lo, int V, int shape){ + switch (shape){ + case 0: /* uniform -> every token equal probability -> massive tie plateau */ + for (int i = 0; i < V; i++) lo[i] = 0.f; break; + case 1: /* peaked: one dominant token, rest small and distinct (no ties). + * The fixed hot-spots are clamped to V-1 so small V (incl. V=1) doesn't + * write out of bounds and corrupt heap metadata on the later free(lo). */ + for (int i = 0; i < V; i++) lo[i] = (float)(-1.0 - frand()*4.0); + lo[0] = 3.f; lo[V/3 comparator tie handling */ + for (int i = 0; i < V; i++) lo[i] = (float)(-(double)(i / 7)); /* 7-wide plateaus */ + break; + case 4: /* sharp-tail: a few hot, then a long flat floor (small tie at the floor). + * Hot count is min(12,V) so V<12 (incl. V=1) stays in bounds. */ + for (int i = 0; i < V; i++) lo[i] = -8.f; + { int hot = V<12 ? V : 12; for (int i = 0; i < hot; i++) lo[i] = (float)(2.0 - frand()); } break; + } +} + +int main(void){ + /* sizes: small for exhaustive tie detection up to near-production scale */ + int sizes[] = {1, 2, 8, 64, 257, 1519}; /* 1519 ~= V/100 of GLM-5.2 */ + double nucs[] = {0.001, 0.5, 0.9, 0.999}; /* tight -> almost-everything */ + int n_shapes = 5; + + /* temperature used by dist_build: pick a normal serving value */ + g_temp = 0.7f; + + int cases = 0; + for (size_t si = 0; si < sizeof(sizes)/sizeof(sizes[0]); si++){ + int V = sizes[si]; + /* dist_build allocates g_pbuf/g_pidx ONCE and reuses them (single-V invariant in + * real serving, where V is the constant model vocab). This sweep varies V, so free + * and force a reallocation per size -- otherwise a later, larger V would overflow + * the buffer sized for the first (smallest) V. */ + free(g_pbuf); g_pbuf = NULL; free(g_pidx); g_pidx = NULL; + float *lo = malloc((size_t)V * sizeof(float)); + for (int shape = 0; shape < n_shapes; shape++){ + fill_shape(lo, V, shape); + for (size_t ni = 0; ni < sizeof(nucs)/sizeof(nucs[0]); ni++){ + char label[32]; snprintf(label, sizeof(label), "size[%zu]/shape[%d]", si, shape); + const char *sn = (const char*[]){"uniform","peaked","geometric","plateau","sharptail"}[shape]; + check_case(label, V, nucs[ni], sn, lo); + cases++; + } + } + free(lo); + } + + /* guard-off path: g_nuc >= 1 must skip truncation entirely (full softmax kept) */ + { + int V = 256; float lo[256]; + for (int i = 0; i < V; i++) lo[i] = (float)(frand()*4 - 2); + g_nuc = 1.0f; dist_build(lo, V); + int nz = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) nz++; + if (nz != V){ fprintf(stderr, " FAIL [guard-off nuc=1.0]: %d/%d entries kept, expected all\n", nz, V); g_nfails++; } + else printf(" ok [guard-off nuc=1.0 keep=%d]\n", nz); + cases++; + + g_nuc = 0.0f; dist_build(lo, V); + nz = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) nz++; + if (nz != V){ fprintf(stderr, " FAIL [guard-off nuc=0.0]: %d/%d entries kept, expected all\n", nz, V); g_nfails++; } + else printf(" ok [guard-off nuc=0.0 keep=%d]\n", nz); + cases++; + } + + /* extreme tie edge case: V=1, single token -> keep=1 regardless of nuc */ + { + float lo[1] = {5.f}; + g_nuc = 0.5f; dist_build(lo, 1); + if (g_pbuf[0] == 0.f || !(fabs((double)g_pbuf[0] - 1.0) < 1e-6)){ + fprintf(stderr, " FAIL [V=1]: g_pbuf[0]=%.9g, expected 1.0\n", (double)g_pbuf[0]); g_nfails++; + } else printf(" ok [V=1 keep=1]\n"); + cases++; + } + + printf("\ntest_topp: %d cases run, %d failure(s)\n", cases, g_nfails); + if (g_nfails){ printf("test_topp: FAIL\n"); return 1; } + printf("test_topp: ok\n"); + return 0; +} diff --git a/c/tests/test_uring.c b/c/tests/test_uring.c index e64b9f2..625c79b 100644 --- a/c/tests/test_uring.c +++ b/c/tests/test_uring.c @@ -6,7 +6,7 @@ #include #include #define main coli_glm_main_unused -#include "../glm.c" +#include "../colibri.c" #undef main static int fail(const char *s){ fprintf(stderr,"FAIL: %s\n",s); return 1; } diff --git a/c/tests/tok_o200k_cases.txt b/c/tests/tok_o200k_cases.txt new file mode 100644 index 0000000..38e37fa --- /dev/null +++ b/c/tests/tok_o200k_cases.txt @@ -0,0 +1,40 @@ +hello world 259,32,119,111,114,263 +HelloWorld 72,101,257,111,262 +XMLHttpRequest 88,77,76,72,116,116,112,82,101,113,117,101,115,116 +helloWORLDhello 259,87,79,82,76,68,259 +dog's 100,111,103,270 +DON'T 68,79,78,39,84 +don't 100,111,110,39,116 +I'll've 73,39,257,39,118,101 +O'Brien 79,39,66,114,105,101,110 +the theatre 265,32,116,256,97,116,114,101 +12345 268,52,53 +a1b22c333d4444 97,49,98,50,50,99,51,51,51,100,52,52,52,52 +3.14 51,46,49,52 +http://x.com/a/b 104,116,116,112,58,47,47,120,46,99,111,109,47,97,47,98 +path/to/file 112,97,264,47,116,111,47,102,105,108,101 +a//b///c 97,47,47,98,47,47,47,99 +!!\r\n//x 33,33,13,10,47,47,120 +one\ntwo\r\nthree 111,110,101,10,116,119,111,13,10,264,114,101,101 + \n x 32,32,10,32,32,120 + 32,32,32 +a b c 97,32,32,98,32,32,32,99 +tab\there 116,269,9,256,114,101 +Café 67,97,102,101,204,129 +naiveBayes 110,97,105,118,101,66,97,121,101,115 +Éclair 195,137,99,108,97,105,114 +北京大学 229,140,151,228,186,172,229,164,167,229,173,166 +ΑΒαβ 206,145,206,146,206,177,206,178 +Иван 208,152,208,178,208,176,208,189 +ẞßscharf 225,186,158,195,159,115,99,104,97,114,102 +i̇stanbul 105,204,135,115,116,97,110,98,117,108 +hello<|endoftext|>world 259,274,119,111,114,263 +<|message_user|>hi 275,104,105 +mixedCASEand123 109,105,120,101,100,67,65,83,69,97,110,100,268 +'s 270 + 's 32,39,115 +A 65 +aB 97,66 +Ab 65,98 +AB 65,66 +ab 269 diff --git a/c/tests/tok_o200k_tiny.json b/c/tests/tok_o200k_tiny.json new file mode 100644 index 0000000..829301e --- /dev/null +++ b/c/tests/tok_o200k_tiny.json @@ -0,0 +1 @@ +{"version": "1.0", "truncation": null, "padding": null, "added_tokens": [{"id": 274, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 275, "content": "<|message_user|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}], "normalizer": null, "pre_tokenizer": {"type": "Sequence", "pretokenizers": [{"type": "Split", "pattern": {"Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, "behavior": "Isolated", "invert": false}, {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false}]}, "post_processor": null, "decoder": {"type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true}, "model": {"type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": true, "vocab": {"Ā": 0, "ā": 1, "Ă": 2, "ă": 3, "Ą": 4, "ą": 5, "Ć": 6, "ć": 7, "Ĉ": 8, "ĉ": 9, "Ċ": 10, "ċ": 11, "Č": 12, "č": 13, "Ď": 14, "ď": 15, "Đ": 16, "đ": 17, "Ē": 18, "ē": 19, "Ĕ": 20, "ĕ": 21, "Ė": 22, "ė": 23, "Ę": 24, "ę": 25, "Ě": 26, "ě": 27, "Ĝ": 28, "ĝ": 29, "Ğ": 30, "ğ": 31, "Ġ": 32, "!": 33, "\"": 34, "#": 35, "$": 36, "%": 37, "&": 38, "'": 39, "(": 40, ")": 41, "*": 42, "+": 43, ",": 44, "-": 45, ".": 46, "/": 47, "0": 48, "1": 49, "2": 50, "3": 51, "4": 52, "5": 53, "6": 54, "7": 55, "8": 56, "9": 57, ":": 58, ";": 59, "<": 60, "=": 61, ">": 62, "?": 63, "@": 64, "A": 65, "B": 66, "C": 67, "D": 68, "E": 69, "F": 70, "G": 71, "H": 72, "I": 73, "J": 74, "K": 75, "L": 76, "M": 77, "N": 78, "O": 79, "P": 80, "Q": 81, "R": 82, "S": 83, "T": 84, "U": 85, "V": 86, "W": 87, "X": 88, "Y": 89, "Z": 90, "[": 91, "\\": 92, "]": 93, "^": 94, "_": 95, "`": 96, "a": 97, "b": 98, "c": 99, "d": 100, "e": 101, "f": 102, "g": 103, "h": 104, "i": 105, "j": 106, "k": 107, "l": 108, "m": 109, "n": 110, "o": 111, "p": 112, "q": 113, "r": 114, "s": 115, "t": 116, "u": 117, "v": 118, "w": 119, "x": 120, "y": 121, "z": 122, "{": 123, "|": 124, "}": 125, "~": 126, "ġ": 127, "Ģ": 128, "ģ": 129, "Ĥ": 130, "ĥ": 131, "Ħ": 132, "ħ": 133, "Ĩ": 134, "ĩ": 135, "Ī": 136, "ī": 137, "Ĭ": 138, "ĭ": 139, "Į": 140, "į": 141, "İ": 142, "ı": 143, "IJ": 144, "ij": 145, "Ĵ": 146, "ĵ": 147, "Ķ": 148, "ķ": 149, "ĸ": 150, "Ĺ": 151, "ĺ": 152, "Ļ": 153, "ļ": 154, "Ľ": 155, "ľ": 156, "Ŀ": 157, "ŀ": 158, "Ł": 159, "ł": 160, "¡": 161, "¢": 162, "£": 163, "¤": 164, "¥": 165, "¦": 166, "§": 167, "¨": 168, "©": 169, "ª": 170, "«": 171, "¬": 172, "Ń": 173, "®": 174, "¯": 175, "°": 176, "±": 177, "²": 178, "³": 179, "´": 180, "µ": 181, "¶": 182, "·": 183, "¸": 184, "¹": 185, "º": 186, "»": 187, "¼": 188, "½": 189, "¾": 190, "¿": 191, "À": 192, "Á": 193, "Â": 194, "Ã": 195, "Ä": 196, "Å": 197, "Æ": 198, "Ç": 199, "È": 200, "É": 201, "Ê": 202, "Ë": 203, "Ì": 204, "Í": 205, "Î": 206, "Ï": 207, "Ð": 208, "Ñ": 209, "Ò": 210, "Ó": 211, "Ô": 212, "Õ": 213, "Ö": 214, "×": 215, "Ø": 216, "Ù": 217, "Ú": 218, "Û": 219, "Ü": 220, "Ý": 221, "Þ": 222, "ß": 223, "à": 224, "á": 225, "â": 226, "ã": 227, "ä": 228, "å": 229, "æ": 230, "ç": 231, "è": 232, "é": 233, "ê": 234, "ë": 235, "ì": 236, "í": 237, "î": 238, "ï": 239, "ð": 240, "ñ": 241, "ò": 242, "ó": 243, "ô": 244, "õ": 245, "ö": 246, "÷": 247, "ø": 248, "ù": 249, "ú": 250, "û": 251, "ü": 252, "ý": 253, "þ": 254, "ÿ": 255, "he": 256, "ll": 257, "hell": 258, "hello": 259, "Wo": 260, "Wor": 261, "World": 262, "ld": 263, "th": 264, "the": 265, "Ġthe": 266, "12": 267, "123": 268, "ab": 269, "'s": 270, "Ġa": 271, "./": 272, "ĊĊ": 273}, "merges": [["h", "e"], ["l", "l"], ["he", "ll"], ["hell", "o"], ["W", "o"], ["Wo", "r"], ["Wor", "ld"], ["l", "d"], ["t", "h"], ["th", "e"], ["Ġ", "the"], ["1", "2"], ["12", "3"], ["a", "b"], ["'", "s"], ["Ġ", "a"], [".", "/"], ["Ċ", "Ċ"]]}} \ No newline at end of file diff --git a/c/tok.h b/c/tok.h index 7d1140b..6b957df 100644 --- a/c/tok.h +++ b/c/tok.h @@ -19,6 +19,7 @@ #include #include "json.h" #include "tok_unicode.h" +#include "tok_unicode_o200k.h" /* ---------- hash map (chiavi binarie con lunghezza) ---------- */ typedef struct { const char *k; int klen; int v; int used; } ment; @@ -50,6 +51,7 @@ typedef struct { Special *sp; int nsp; /* added tokens, ordinati per lunghezza decrescente */ uint32_t byte2cp[256]; int byte2cp_len[256]; char byte2str[256][3]; int16_t cp2byte[1024]; + int o200k; /* pre_tokenizer regex family: 0 = cl100k (GLM), 1 = o200k (Inkling) */ } Tok; /* ---------- UTF-8 ---------- */ @@ -144,6 +146,17 @@ static void tok_load(Tok *T, const char *path){ } qsort(T->sp,T->nsp,sizeof(Special),cmp_sp_len); /* match piu' lungo per primo */ } + /* pre_tokenizer family: the o200k Split regex is recognizable by its + * case-category classes (\p{Lu}...) which cl100k does not use */ + jval *pt=json_get(root,"pre_tokenizer"); + if(pt){ + jval *ps=json_get(pt,"pretokenizers"); + if(ps&&ps->t==J_ARR) for(int i=0;ilen;i++){ + jval *pat=json_get(ps->kids[i],"pattern"); + jval *rx=pat?json_get(pat,"Regex"):NULL; + if(rx&&rx->t==J_STR&&strstr(rx->str,"\\p{Lu}")) T->o200k=1; + } + } /* arena/buf restano allocati: le stringhe (j_dup) sono malloc indipendenti e ci servono vive */ (void)arena; } @@ -241,6 +254,104 @@ static void pretok_chunk(Tok *T, const unsigned char *p, int a, int b, int *out, free(cp); free(off); } +/* ---------- pre-tokenizer o200k (Inkling / GPT-4o family) ---------- + * Split regex: + * A: [^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)? + * B: [^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)? + * C: \p{N}{1,3} D: ' ?[^\s\p{L}\p{N}]+[\r\n/]*' E: \s*[\r\n]+ F: \s+(?!\S) G: \s+ + * S1 = Lu|Lt|Lm|Lo|M, S2 = Ll|Lm|Lo|M. The letter matcher below replays the + * regex engine's backtracking order exactly: A with greedy optional prefix and + * maximally-greedy S1* given back until S2+ can take >=1 char, then B. */ +#define O2_S1(c) (is_U(c)||is_X(c)) +#define O2_S2(c) (is_X(c)||(is_L(c)&&!is_U(c))) +static uint32_t o2_low(uint32_t c){ return (c>='A'&&c<='Z')?c+32:c; } +static int o2_contraction(const uint32_t *cp, int n, int k){ + if(k=0; pfx--){ + int j0=i; + if(pfx){ + uint32_t c=cp[i]; + if(c=='\r'||c=='\n'||is_L(c)||is_N(c)||i+1>=n) continue; + j0=i+1; + } + int m1=j0; while(m1=j0; s--){ + if(s=0; pfx--){ + int j0=i; + if(pfx){ + uint32_t c=cp[i]; + if(c=='\r'||c=='\n'||is_L(c)||is_N(c)||i+1>=n) continue; + j0=i+1; + } + int m1=j0; while(m1j0){ + int k=m1; while(ki){ i=e; bpe_piece(T,p,off[start],off[i],out,no,max); continue; } + } + /* C: \p{N}{1,3} */ + if(is_N(c)){ int j=i,k=0; while(ji){ int last=-1; for(int j=i;j=0){ i=last+1; bpe_piece(T,p,off[start],off[i],out,no,max); continue; } + int end = (r id (split sugli added token, poi pretok+BPE) ---------- */ static int tok_encode(Tok *T, const char *text, int len, int *out, int max){ const unsigned char *p=(const unsigned char*)text; int no=0; int i=0; @@ -254,7 +365,10 @@ static int tok_encode(Tok *T, const char *text, int len, int *out, int max){ } } int chunk_end = (hitpos<0) ? len : hitpos; - if(chunk_end>i) pretok_chunk(T,p,i,chunk_end,out,&no,max); + if(chunk_end>i){ + if(T->o200k) pretok_chunk_o200k(T,p,i,chunk_end,out,&no,max); + else pretok_chunk(T,p,i,chunk_end,out,&no,max); + } if(hitpos<0) break; if(no + +static const uint32_t uni_U[][2] = { + {0x41,0x5A},{0xC0,0xD6},{0xD8,0xDE},{0x100,0x100},{0x102,0x102},{0x104,0x104}, + {0x106,0x106},{0x108,0x108},{0x10A,0x10A},{0x10C,0x10C},{0x10E,0x10E},{0x110,0x110}, + {0x112,0x112},{0x114,0x114},{0x116,0x116},{0x118,0x118},{0x11A,0x11A},{0x11C,0x11C}, + {0x11E,0x11E},{0x120,0x120},{0x122,0x122},{0x124,0x124},{0x126,0x126},{0x128,0x128}, + {0x12A,0x12A},{0x12C,0x12C},{0x12E,0x12E},{0x130,0x130},{0x132,0x132},{0x134,0x134}, + {0x136,0x136},{0x139,0x139},{0x13B,0x13B},{0x13D,0x13D},{0x13F,0x13F},{0x141,0x141}, + {0x143,0x143},{0x145,0x145},{0x147,0x147},{0x14A,0x14A},{0x14C,0x14C},{0x14E,0x14E}, + {0x150,0x150},{0x152,0x152},{0x154,0x154},{0x156,0x156},{0x158,0x158},{0x15A,0x15A}, + {0x15C,0x15C},{0x15E,0x15E},{0x160,0x160},{0x162,0x162},{0x164,0x164},{0x166,0x166}, + {0x168,0x168},{0x16A,0x16A},{0x16C,0x16C},{0x16E,0x16E},{0x170,0x170},{0x172,0x172}, + {0x174,0x174},{0x176,0x176},{0x178,0x179},{0x17B,0x17B},{0x17D,0x17D},{0x181,0x182}, + {0x184,0x184},{0x186,0x187},{0x189,0x18B},{0x18E,0x191},{0x193,0x194},{0x196,0x198}, + {0x19C,0x19D},{0x19F,0x1A0},{0x1A2,0x1A2},{0x1A4,0x1A4},{0x1A6,0x1A7},{0x1A9,0x1A9}, + {0x1AC,0x1AC},{0x1AE,0x1AF},{0x1B1,0x1B3},{0x1B5,0x1B5},{0x1B7,0x1B8},{0x1BC,0x1BC}, + {0x1C4,0x1C5},{0x1C7,0x1C8},{0x1CA,0x1CB},{0x1CD,0x1CD},{0x1CF,0x1CF},{0x1D1,0x1D1}, + {0x1D3,0x1D3},{0x1D5,0x1D5},{0x1D7,0x1D7},{0x1D9,0x1D9},{0x1DB,0x1DB},{0x1DE,0x1DE}, + {0x1E0,0x1E0},{0x1E2,0x1E2},{0x1E4,0x1E4},{0x1E6,0x1E6},{0x1E8,0x1E8},{0x1EA,0x1EA}, + {0x1EC,0x1EC},{0x1EE,0x1EE},{0x1F1,0x1F2},{0x1F4,0x1F4},{0x1F6,0x1F8},{0x1FA,0x1FA}, + {0x1FC,0x1FC},{0x1FE,0x1FE},{0x200,0x200},{0x202,0x202},{0x204,0x204},{0x206,0x206}, + {0x208,0x208},{0x20A,0x20A},{0x20C,0x20C},{0x20E,0x20E},{0x210,0x210},{0x212,0x212}, + {0x214,0x214},{0x216,0x216},{0x218,0x218},{0x21A,0x21A},{0x21C,0x21C},{0x21E,0x21E}, + {0x220,0x220},{0x222,0x222},{0x224,0x224},{0x226,0x226},{0x228,0x228},{0x22A,0x22A}, + {0x22C,0x22C},{0x22E,0x22E},{0x230,0x230},{0x232,0x232},{0x23A,0x23B},{0x23D,0x23E}, + {0x241,0x241},{0x243,0x246},{0x248,0x248},{0x24A,0x24A},{0x24C,0x24C},{0x24E,0x24E}, + {0x370,0x370},{0x372,0x372},{0x376,0x376},{0x37F,0x37F},{0x386,0x386},{0x388,0x38A}, + {0x38C,0x38C},{0x38E,0x38F},{0x391,0x3A1},{0x3A3,0x3AB},{0x3CF,0x3CF},{0x3D2,0x3D4}, + {0x3D8,0x3D8},{0x3DA,0x3DA},{0x3DC,0x3DC},{0x3DE,0x3DE},{0x3E0,0x3E0},{0x3E2,0x3E2}, + {0x3E4,0x3E4},{0x3E6,0x3E6},{0x3E8,0x3E8},{0x3EA,0x3EA},{0x3EC,0x3EC},{0x3EE,0x3EE}, + {0x3F4,0x3F4},{0x3F7,0x3F7},{0x3F9,0x3FA},{0x3FD,0x42F},{0x460,0x460},{0x462,0x462}, + {0x464,0x464},{0x466,0x466},{0x468,0x468},{0x46A,0x46A},{0x46C,0x46C},{0x46E,0x46E}, + {0x470,0x470},{0x472,0x472},{0x474,0x474},{0x476,0x476},{0x478,0x478},{0x47A,0x47A}, + {0x47C,0x47C},{0x47E,0x47E},{0x480,0x480},{0x48A,0x48A},{0x48C,0x48C},{0x48E,0x48E}, + {0x490,0x490},{0x492,0x492},{0x494,0x494},{0x496,0x496},{0x498,0x498},{0x49A,0x49A}, + {0x49C,0x49C},{0x49E,0x49E},{0x4A0,0x4A0},{0x4A2,0x4A2},{0x4A4,0x4A4},{0x4A6,0x4A6}, + {0x4A8,0x4A8},{0x4AA,0x4AA},{0x4AC,0x4AC},{0x4AE,0x4AE},{0x4B0,0x4B0},{0x4B2,0x4B2}, + {0x4B4,0x4B4},{0x4B6,0x4B6},{0x4B8,0x4B8},{0x4BA,0x4BA},{0x4BC,0x4BC},{0x4BE,0x4BE}, + {0x4C0,0x4C1},{0x4C3,0x4C3},{0x4C5,0x4C5},{0x4C7,0x4C7},{0x4C9,0x4C9},{0x4CB,0x4CB}, + {0x4CD,0x4CD},{0x4D0,0x4D0},{0x4D2,0x4D2},{0x4D4,0x4D4},{0x4D6,0x4D6},{0x4D8,0x4D8}, + {0x4DA,0x4DA},{0x4DC,0x4DC},{0x4DE,0x4DE},{0x4E0,0x4E0},{0x4E2,0x4E2},{0x4E4,0x4E4}, + {0x4E6,0x4E6},{0x4E8,0x4E8},{0x4EA,0x4EA},{0x4EC,0x4EC},{0x4EE,0x4EE},{0x4F0,0x4F0}, + {0x4F2,0x4F2},{0x4F4,0x4F4},{0x4F6,0x4F6},{0x4F8,0x4F8},{0x4FA,0x4FA},{0x4FC,0x4FC}, + {0x4FE,0x4FE},{0x500,0x500},{0x502,0x502},{0x504,0x504},{0x506,0x506},{0x508,0x508}, + {0x50A,0x50A},{0x50C,0x50C},{0x50E,0x50E},{0x510,0x510},{0x512,0x512},{0x514,0x514}, + {0x516,0x516},{0x518,0x518},{0x51A,0x51A},{0x51C,0x51C},{0x51E,0x51E},{0x520,0x520}, + {0x522,0x522},{0x524,0x524},{0x526,0x526},{0x528,0x528},{0x52A,0x52A},{0x52C,0x52C}, + {0x52E,0x52E},{0x531,0x556},{0x10A0,0x10C5},{0x10C7,0x10C7},{0x10CD,0x10CD},{0x13A0,0x13F5}, + {0x1C90,0x1CBA},{0x1CBD,0x1CBF},{0x1E00,0x1E00},{0x1E02,0x1E02},{0x1E04,0x1E04},{0x1E06,0x1E06}, + {0x1E08,0x1E08},{0x1E0A,0x1E0A},{0x1E0C,0x1E0C},{0x1E0E,0x1E0E},{0x1E10,0x1E10},{0x1E12,0x1E12}, + {0x1E14,0x1E14},{0x1E16,0x1E16},{0x1E18,0x1E18},{0x1E1A,0x1E1A},{0x1E1C,0x1E1C},{0x1E1E,0x1E1E}, + {0x1E20,0x1E20},{0x1E22,0x1E22},{0x1E24,0x1E24},{0x1E26,0x1E26},{0x1E28,0x1E28},{0x1E2A,0x1E2A}, + {0x1E2C,0x1E2C},{0x1E2E,0x1E2E},{0x1E30,0x1E30},{0x1E32,0x1E32},{0x1E34,0x1E34},{0x1E36,0x1E36}, + {0x1E38,0x1E38},{0x1E3A,0x1E3A},{0x1E3C,0x1E3C},{0x1E3E,0x1E3E},{0x1E40,0x1E40},{0x1E42,0x1E42}, + {0x1E44,0x1E44},{0x1E46,0x1E46},{0x1E48,0x1E48},{0x1E4A,0x1E4A},{0x1E4C,0x1E4C},{0x1E4E,0x1E4E}, + {0x1E50,0x1E50},{0x1E52,0x1E52},{0x1E54,0x1E54},{0x1E56,0x1E56},{0x1E58,0x1E58},{0x1E5A,0x1E5A}, + {0x1E5C,0x1E5C},{0x1E5E,0x1E5E},{0x1E60,0x1E60},{0x1E62,0x1E62},{0x1E64,0x1E64},{0x1E66,0x1E66}, + {0x1E68,0x1E68},{0x1E6A,0x1E6A},{0x1E6C,0x1E6C},{0x1E6E,0x1E6E},{0x1E70,0x1E70},{0x1E72,0x1E72}, + {0x1E74,0x1E74},{0x1E76,0x1E76},{0x1E78,0x1E78},{0x1E7A,0x1E7A},{0x1E7C,0x1E7C},{0x1E7E,0x1E7E}, + {0x1E80,0x1E80},{0x1E82,0x1E82},{0x1E84,0x1E84},{0x1E86,0x1E86},{0x1E88,0x1E88},{0x1E8A,0x1E8A}, + {0x1E8C,0x1E8C},{0x1E8E,0x1E8E},{0x1E90,0x1E90},{0x1E92,0x1E92},{0x1E94,0x1E94},{0x1E9E,0x1E9E}, + {0x1EA0,0x1EA0},{0x1EA2,0x1EA2},{0x1EA4,0x1EA4},{0x1EA6,0x1EA6},{0x1EA8,0x1EA8},{0x1EAA,0x1EAA}, + {0x1EAC,0x1EAC},{0x1EAE,0x1EAE},{0x1EB0,0x1EB0},{0x1EB2,0x1EB2},{0x1EB4,0x1EB4},{0x1EB6,0x1EB6}, + {0x1EB8,0x1EB8},{0x1EBA,0x1EBA},{0x1EBC,0x1EBC},{0x1EBE,0x1EBE},{0x1EC0,0x1EC0},{0x1EC2,0x1EC2}, + {0x1EC4,0x1EC4},{0x1EC6,0x1EC6},{0x1EC8,0x1EC8},{0x1ECA,0x1ECA},{0x1ECC,0x1ECC},{0x1ECE,0x1ECE}, + {0x1ED0,0x1ED0},{0x1ED2,0x1ED2},{0x1ED4,0x1ED4},{0x1ED6,0x1ED6},{0x1ED8,0x1ED8},{0x1EDA,0x1EDA}, + {0x1EDC,0x1EDC},{0x1EDE,0x1EDE},{0x1EE0,0x1EE0},{0x1EE2,0x1EE2},{0x1EE4,0x1EE4},{0x1EE6,0x1EE6}, + {0x1EE8,0x1EE8},{0x1EEA,0x1EEA},{0x1EEC,0x1EEC},{0x1EEE,0x1EEE},{0x1EF0,0x1EF0},{0x1EF2,0x1EF2}, + {0x1EF4,0x1EF4},{0x1EF6,0x1EF6},{0x1EF8,0x1EF8},{0x1EFA,0x1EFA},{0x1EFC,0x1EFC},{0x1EFE,0x1EFE}, + {0x1F08,0x1F0F},{0x1F18,0x1F1D},{0x1F28,0x1F2F},{0x1F38,0x1F3F},{0x1F48,0x1F4D},{0x1F59,0x1F59}, + {0x1F5B,0x1F5B},{0x1F5D,0x1F5D},{0x1F5F,0x1F5F},{0x1F68,0x1F6F},{0x1F88,0x1F8F},{0x1F98,0x1F9F}, + {0x1FA8,0x1FAF},{0x1FB8,0x1FBC},{0x1FC8,0x1FCC},{0x1FD8,0x1FDB},{0x1FE8,0x1FEC},{0x1FF8,0x1FFC}, + {0x2102,0x2102},{0x2107,0x2107},{0x210B,0x210D},{0x2110,0x2112},{0x2115,0x2115},{0x2119,0x211D}, + {0x2124,0x2124},{0x2126,0x2126},{0x2128,0x2128},{0x212A,0x212D},{0x2130,0x2133},{0x213E,0x213F}, + {0x2145,0x2145},{0x2183,0x2183},{0x2C00,0x2C2E},{0x2C60,0x2C60},{0x2C62,0x2C64},{0x2C67,0x2C67}, + {0x2C69,0x2C69},{0x2C6B,0x2C6B},{0x2C6D,0x2C70},{0x2C72,0x2C72},{0x2C75,0x2C75},{0x2C7E,0x2C80}, + {0x2C82,0x2C82},{0x2C84,0x2C84},{0x2C86,0x2C86},{0x2C88,0x2C88},{0x2C8A,0x2C8A},{0x2C8C,0x2C8C}, + {0x2C8E,0x2C8E},{0x2C90,0x2C90},{0x2C92,0x2C92},{0x2C94,0x2C94},{0x2C96,0x2C96},{0x2C98,0x2C98}, + {0x2C9A,0x2C9A},{0x2C9C,0x2C9C},{0x2C9E,0x2C9E},{0x2CA0,0x2CA0},{0x2CA2,0x2CA2},{0x2CA4,0x2CA4}, + {0x2CA6,0x2CA6},{0x2CA8,0x2CA8},{0x2CAA,0x2CAA},{0x2CAC,0x2CAC},{0x2CAE,0x2CAE},{0x2CB0,0x2CB0}, + {0x2CB2,0x2CB2},{0x2CB4,0x2CB4},{0x2CB6,0x2CB6},{0x2CB8,0x2CB8},{0x2CBA,0x2CBA},{0x2CBC,0x2CBC}, + {0x2CBE,0x2CBE},{0x2CC0,0x2CC0},{0x2CC2,0x2CC2},{0x2CC4,0x2CC4},{0x2CC6,0x2CC6},{0x2CC8,0x2CC8}, + {0x2CCA,0x2CCA},{0x2CCC,0x2CCC},{0x2CCE,0x2CCE},{0x2CD0,0x2CD0},{0x2CD2,0x2CD2},{0x2CD4,0x2CD4}, + {0x2CD6,0x2CD6},{0x2CD8,0x2CD8},{0x2CDA,0x2CDA},{0x2CDC,0x2CDC},{0x2CDE,0x2CDE},{0x2CE0,0x2CE0}, + {0x2CE2,0x2CE2},{0x2CEB,0x2CEB},{0x2CED,0x2CED},{0x2CF2,0x2CF2},{0xA640,0xA640},{0xA642,0xA642}, + {0xA644,0xA644},{0xA646,0xA646},{0xA648,0xA648},{0xA64A,0xA64A},{0xA64C,0xA64C},{0xA64E,0xA64E}, + {0xA650,0xA650},{0xA652,0xA652},{0xA654,0xA654},{0xA656,0xA656},{0xA658,0xA658},{0xA65A,0xA65A}, + {0xA65C,0xA65C},{0xA65E,0xA65E},{0xA660,0xA660},{0xA662,0xA662},{0xA664,0xA664},{0xA666,0xA666}, + {0xA668,0xA668},{0xA66A,0xA66A},{0xA66C,0xA66C},{0xA680,0xA680},{0xA682,0xA682},{0xA684,0xA684}, + {0xA686,0xA686},{0xA688,0xA688},{0xA68A,0xA68A},{0xA68C,0xA68C},{0xA68E,0xA68E},{0xA690,0xA690}, + {0xA692,0xA692},{0xA694,0xA694},{0xA696,0xA696},{0xA698,0xA698},{0xA69A,0xA69A},{0xA722,0xA722}, + {0xA724,0xA724},{0xA726,0xA726},{0xA728,0xA728},{0xA72A,0xA72A},{0xA72C,0xA72C},{0xA72E,0xA72E}, + {0xA732,0xA732},{0xA734,0xA734},{0xA736,0xA736},{0xA738,0xA738},{0xA73A,0xA73A},{0xA73C,0xA73C}, + {0xA73E,0xA73E},{0xA740,0xA740},{0xA742,0xA742},{0xA744,0xA744},{0xA746,0xA746},{0xA748,0xA748}, + {0xA74A,0xA74A},{0xA74C,0xA74C},{0xA74E,0xA74E},{0xA750,0xA750},{0xA752,0xA752},{0xA754,0xA754}, + {0xA756,0xA756},{0xA758,0xA758},{0xA75A,0xA75A},{0xA75C,0xA75C},{0xA75E,0xA75E},{0xA760,0xA760}, + {0xA762,0xA762},{0xA764,0xA764},{0xA766,0xA766},{0xA768,0xA768},{0xA76A,0xA76A},{0xA76C,0xA76C}, + {0xA76E,0xA76E},{0xA779,0xA779},{0xA77B,0xA77B},{0xA77D,0xA77E},{0xA780,0xA780},{0xA782,0xA782}, + {0xA784,0xA784},{0xA786,0xA786},{0xA78B,0xA78B},{0xA78D,0xA78D},{0xA790,0xA790},{0xA792,0xA792}, + {0xA796,0xA796},{0xA798,0xA798},{0xA79A,0xA79A},{0xA79C,0xA79C},{0xA79E,0xA79E},{0xA7A0,0xA7A0}, + {0xA7A2,0xA7A2},{0xA7A4,0xA7A4},{0xA7A6,0xA7A6},{0xA7A8,0xA7A8},{0xA7AA,0xA7AE},{0xA7B0,0xA7B4}, + {0xA7B6,0xA7B6},{0xA7B8,0xA7B8},{0xA7BA,0xA7BA},{0xA7BC,0xA7BC},{0xA7BE,0xA7BE},{0xA7C2,0xA7C2}, + {0xA7C4,0xA7C7},{0xA7C9,0xA7C9},{0xA7F5,0xA7F5},{0xFF21,0xFF3A},{0x10400,0x10427},{0x104B0,0x104D3}, + {0x10C80,0x10CB2},{0x118A0,0x118BF},{0x16E40,0x16E5F},{0x1D400,0x1D419},{0x1D434,0x1D44D},{0x1D468,0x1D481}, + {0x1D49C,0x1D49C},{0x1D49E,0x1D49F},{0x1D4A2,0x1D4A2},{0x1D4A5,0x1D4A6},{0x1D4A9,0x1D4AC},{0x1D4AE,0x1D4B5}, + {0x1D4D0,0x1D4E9},{0x1D504,0x1D505},{0x1D507,0x1D50A},{0x1D50D,0x1D514},{0x1D516,0x1D51C},{0x1D538,0x1D539}, + {0x1D53B,0x1D53E},{0x1D540,0x1D544},{0x1D546,0x1D546},{0x1D54A,0x1D550},{0x1D56C,0x1D585},{0x1D5A0,0x1D5B9}, + {0x1D5D4,0x1D5ED},{0x1D608,0x1D621},{0x1D63C,0x1D655},{0x1D670,0x1D689},{0x1D6A8,0x1D6C0},{0x1D6E2,0x1D6FA}, + {0x1D71C,0x1D734},{0x1D756,0x1D76E},{0x1D790,0x1D7A8},{0x1D7CA,0x1D7CA},{0x1E900,0x1E921}, +}; +static const int uni_U_n = 641; + +static const uint32_t uni_X[][2] = { + {0xAA,0xAA},{0xBA,0xBA},{0x1BB,0x1BB},{0x1C0,0x1C3},{0x294,0x294},{0x2B0,0x2C1}, + {0x2C6,0x2D1},{0x2E0,0x2E4},{0x2EC,0x2EC},{0x2EE,0x2EE},{0x300,0x36F},{0x374,0x374}, + {0x37A,0x37A},{0x483,0x489},{0x559,0x559},{0x591,0x5BD},{0x5BF,0x5BF},{0x5C1,0x5C2}, + {0x5C4,0x5C5},{0x5C7,0x5C7},{0x5D0,0x5EA},{0x5EF,0x5F2},{0x610,0x61A},{0x620,0x65F}, + {0x66E,0x6D3},{0x6D5,0x6DC},{0x6DF,0x6E8},{0x6EA,0x6EF},{0x6FA,0x6FC},{0x6FF,0x6FF}, + {0x710,0x74A},{0x74D,0x7B1},{0x7CA,0x7F5},{0x7FA,0x7FA},{0x7FD,0x7FD},{0x800,0x82D}, + {0x840,0x85B},{0x860,0x86A},{0x8A0,0x8B4},{0x8B6,0x8C7},{0x8D3,0x8E1},{0x8E3,0x963}, + {0x971,0x983},{0x985,0x98C},{0x98F,0x990},{0x993,0x9A8},{0x9AA,0x9B0},{0x9B2,0x9B2}, + {0x9B6,0x9B9},{0x9BC,0x9C4},{0x9C7,0x9C8},{0x9CB,0x9CE},{0x9D7,0x9D7},{0x9DC,0x9DD}, + {0x9DF,0x9E3},{0x9F0,0x9F1},{0x9FC,0x9FC},{0x9FE,0x9FE},{0xA01,0xA03},{0xA05,0xA0A}, + {0xA0F,0xA10},{0xA13,0xA28},{0xA2A,0xA30},{0xA32,0xA33},{0xA35,0xA36},{0xA38,0xA39}, + {0xA3C,0xA3C},{0xA3E,0xA42},{0xA47,0xA48},{0xA4B,0xA4D},{0xA51,0xA51},{0xA59,0xA5C}, + {0xA5E,0xA5E},{0xA70,0xA75},{0xA81,0xA83},{0xA85,0xA8D},{0xA8F,0xA91},{0xA93,0xAA8}, + {0xAAA,0xAB0},{0xAB2,0xAB3},{0xAB5,0xAB9},{0xABC,0xAC5},{0xAC7,0xAC9},{0xACB,0xACD}, + {0xAD0,0xAD0},{0xAE0,0xAE3},{0xAF9,0xAFF},{0xB01,0xB03},{0xB05,0xB0C},{0xB0F,0xB10}, + {0xB13,0xB28},{0xB2A,0xB30},{0xB32,0xB33},{0xB35,0xB39},{0xB3C,0xB44},{0xB47,0xB48}, + {0xB4B,0xB4D},{0xB55,0xB57},{0xB5C,0xB5D},{0xB5F,0xB63},{0xB71,0xB71},{0xB82,0xB83}, + {0xB85,0xB8A},{0xB8E,0xB90},{0xB92,0xB95},{0xB99,0xB9A},{0xB9C,0xB9C},{0xB9E,0xB9F}, + {0xBA3,0xBA4},{0xBA8,0xBAA},{0xBAE,0xBB9},{0xBBE,0xBC2},{0xBC6,0xBC8},{0xBCA,0xBCD}, + {0xBD0,0xBD0},{0xBD7,0xBD7},{0xC00,0xC0C},{0xC0E,0xC10},{0xC12,0xC28},{0xC2A,0xC39}, + {0xC3D,0xC44},{0xC46,0xC48},{0xC4A,0xC4D},{0xC55,0xC56},{0xC58,0xC5A},{0xC60,0xC63}, + {0xC80,0xC83},{0xC85,0xC8C},{0xC8E,0xC90},{0xC92,0xCA8},{0xCAA,0xCB3},{0xCB5,0xCB9}, + {0xCBC,0xCC4},{0xCC6,0xCC8},{0xCCA,0xCCD},{0xCD5,0xCD6},{0xCDE,0xCDE},{0xCE0,0xCE3}, + {0xCF1,0xCF2},{0xD00,0xD0C},{0xD0E,0xD10},{0xD12,0xD44},{0xD46,0xD48},{0xD4A,0xD4E}, + {0xD54,0xD57},{0xD5F,0xD63},{0xD7A,0xD7F},{0xD81,0xD83},{0xD85,0xD96},{0xD9A,0xDB1}, + {0xDB3,0xDBB},{0xDBD,0xDBD},{0xDC0,0xDC6},{0xDCA,0xDCA},{0xDCF,0xDD4},{0xDD6,0xDD6}, + {0xDD8,0xDDF},{0xDF2,0xDF3},{0xE01,0xE3A},{0xE40,0xE4E},{0xE81,0xE82},{0xE84,0xE84}, + {0xE86,0xE8A},{0xE8C,0xEA3},{0xEA5,0xEA5},{0xEA7,0xEBD},{0xEC0,0xEC4},{0xEC6,0xEC6}, + {0xEC8,0xECD},{0xEDC,0xEDF},{0xF00,0xF00},{0xF18,0xF19},{0xF35,0xF35},{0xF37,0xF37}, + {0xF39,0xF39},{0xF3E,0xF47},{0xF49,0xF6C},{0xF71,0xF84},{0xF86,0xF97},{0xF99,0xFBC}, + {0xFC6,0xFC6},{0x1000,0x103F},{0x1050,0x108F},{0x109A,0x109D},{0x10FC,0x10FC},{0x1100,0x1248}, + {0x124A,0x124D},{0x1250,0x1256},{0x1258,0x1258},{0x125A,0x125D},{0x1260,0x1288},{0x128A,0x128D}, + {0x1290,0x12B0},{0x12B2,0x12B5},{0x12B8,0x12BE},{0x12C0,0x12C0},{0x12C2,0x12C5},{0x12C8,0x12D6}, + {0x12D8,0x1310},{0x1312,0x1315},{0x1318,0x135A},{0x135D,0x135F},{0x1380,0x138F},{0x1401,0x166C}, + {0x166F,0x167F},{0x1681,0x169A},{0x16A0,0x16EA},{0x16F1,0x16F8},{0x1700,0x170C},{0x170E,0x1714}, + {0x1720,0x1734},{0x1740,0x1753},{0x1760,0x176C},{0x176E,0x1770},{0x1772,0x1773},{0x1780,0x17D3}, + {0x17D7,0x17D7},{0x17DC,0x17DD},{0x180B,0x180D},{0x1820,0x1878},{0x1880,0x18AA},{0x18B0,0x18F5}, + {0x1900,0x191E},{0x1920,0x192B},{0x1930,0x193B},{0x1950,0x196D},{0x1970,0x1974},{0x1980,0x19AB}, + {0x19B0,0x19C9},{0x1A00,0x1A1B},{0x1A20,0x1A5E},{0x1A60,0x1A7C},{0x1A7F,0x1A7F},{0x1AA7,0x1AA7}, + {0x1AB0,0x1AC0},{0x1B00,0x1B4B},{0x1B6B,0x1B73},{0x1B80,0x1BAF},{0x1BBA,0x1BF3},{0x1C00,0x1C37}, + {0x1C4D,0x1C4F},{0x1C5A,0x1C7D},{0x1CD0,0x1CD2},{0x1CD4,0x1CFA},{0x1D2C,0x1D6A},{0x1D78,0x1D78}, + {0x1D9B,0x1DF9},{0x1DFB,0x1DFF},{0x2071,0x2071},{0x207F,0x207F},{0x2090,0x209C},{0x20D0,0x20F0}, + {0x2135,0x2138},{0x2C7C,0x2C7D},{0x2CEF,0x2CF1},{0x2D30,0x2D67},{0x2D6F,0x2D6F},{0x2D7F,0x2D96}, + {0x2DA0,0x2DA6},{0x2DA8,0x2DAE},{0x2DB0,0x2DB6},{0x2DB8,0x2DBE},{0x2DC0,0x2DC6},{0x2DC8,0x2DCE}, + {0x2DD0,0x2DD6},{0x2DD8,0x2DDE},{0x2DE0,0x2DFF},{0x2E2F,0x2E2F},{0x3005,0x3006},{0x302A,0x302F}, + {0x3031,0x3035},{0x303B,0x303C},{0x3041,0x3096},{0x3099,0x309A},{0x309D,0x309F},{0x30A1,0x30FA}, + {0x30FC,0x30FF},{0x3105,0x312F},{0x3131,0x318E},{0x31A0,0x31BF},{0x31F0,0x31FF},{0x3400,0x4DBF}, + {0x4E00,0x9FFC},{0xA000,0xA48C},{0xA4D0,0xA4FD},{0xA500,0xA60C},{0xA610,0xA61F},{0xA62A,0xA62B}, + {0xA66E,0xA672},{0xA674,0xA67D},{0xA67F,0xA67F},{0xA69C,0xA6E5},{0xA6F0,0xA6F1},{0xA717,0xA71F}, + {0xA770,0xA770},{0xA788,0xA788},{0xA78F,0xA78F},{0xA7F7,0xA7F9},{0xA7FB,0xA827},{0xA82C,0xA82C}, + {0xA840,0xA873},{0xA880,0xA8C5},{0xA8E0,0xA8F7},{0xA8FB,0xA8FB},{0xA8FD,0xA8FF},{0xA90A,0xA92D}, + {0xA930,0xA953},{0xA960,0xA97C},{0xA980,0xA9C0},{0xA9CF,0xA9CF},{0xA9E0,0xA9EF},{0xA9FA,0xA9FE}, + {0xAA00,0xAA36},{0xAA40,0xAA4D},{0xAA60,0xAA76},{0xAA7A,0xAAC2},{0xAADB,0xAADD},{0xAAE0,0xAAEF}, + {0xAAF2,0xAAF6},{0xAB01,0xAB06},{0xAB09,0xAB0E},{0xAB11,0xAB16},{0xAB20,0xAB26},{0xAB28,0xAB2E}, + {0xAB5C,0xAB5F},{0xAB69,0xAB69},{0xABC0,0xABEA},{0xABEC,0xABED},{0xAC00,0xD7A3},{0xD7B0,0xD7C6}, + {0xD7CB,0xD7FB},{0xF900,0xFA6D},{0xFA70,0xFAD9},{0xFB1D,0xFB28},{0xFB2A,0xFB36},{0xFB38,0xFB3C}, + {0xFB3E,0xFB3E},{0xFB40,0xFB41},{0xFB43,0xFB44},{0xFB46,0xFBB1},{0xFBD3,0xFD3D},{0xFD50,0xFD8F}, + {0xFD92,0xFDC7},{0xFDF0,0xFDFB},{0xFE00,0xFE0F},{0xFE20,0xFE2F},{0xFE70,0xFE74},{0xFE76,0xFEFC}, + {0xFF66,0xFFBE},{0xFFC2,0xFFC7},{0xFFCA,0xFFCF},{0xFFD2,0xFFD7},{0xFFDA,0xFFDC},{0x10000,0x1000B}, + {0x1000D,0x10026},{0x10028,0x1003A},{0x1003C,0x1003D},{0x1003F,0x1004D},{0x10050,0x1005D},{0x10080,0x100FA}, + {0x101FD,0x101FD},{0x10280,0x1029C},{0x102A0,0x102D0},{0x102E0,0x102E0},{0x10300,0x1031F},{0x1032D,0x10340}, + {0x10342,0x10349},{0x10350,0x1037A},{0x10380,0x1039D},{0x103A0,0x103C3},{0x103C8,0x103CF},{0x10450,0x1049D}, + {0x10500,0x10527},{0x10530,0x10563},{0x10600,0x10736},{0x10740,0x10755},{0x10760,0x10767},{0x10800,0x10805}, + {0x10808,0x10808},{0x1080A,0x10835},{0x10837,0x10838},{0x1083C,0x1083C},{0x1083F,0x10855},{0x10860,0x10876}, + {0x10880,0x1089E},{0x108E0,0x108F2},{0x108F4,0x108F5},{0x10900,0x10915},{0x10920,0x10939},{0x10980,0x109B7}, + {0x109BE,0x109BF},{0x10A00,0x10A03},{0x10A05,0x10A06},{0x10A0C,0x10A13},{0x10A15,0x10A17},{0x10A19,0x10A35}, + {0x10A38,0x10A3A},{0x10A3F,0x10A3F},{0x10A60,0x10A7C},{0x10A80,0x10A9C},{0x10AC0,0x10AC7},{0x10AC9,0x10AE6}, + {0x10B00,0x10B35},{0x10B40,0x10B55},{0x10B60,0x10B72},{0x10B80,0x10B91},{0x10C00,0x10C48},{0x10D00,0x10D27}, + {0x10E80,0x10EA9},{0x10EAB,0x10EAC},{0x10EB0,0x10EB1},{0x10F00,0x10F1C},{0x10F27,0x10F27},{0x10F30,0x10F50}, + {0x10FB0,0x10FC4},{0x10FE0,0x10FF6},{0x11000,0x11046},{0x1107F,0x110BA},{0x110D0,0x110E8},{0x11100,0x11134}, + {0x11144,0x11147},{0x11150,0x11173},{0x11176,0x11176},{0x11180,0x111C4},{0x111C9,0x111CC},{0x111CE,0x111CF}, + {0x111DA,0x111DA},{0x111DC,0x111DC},{0x11200,0x11211},{0x11213,0x11237},{0x1123E,0x1123E},{0x11280,0x11286}, + {0x11288,0x11288},{0x1128A,0x1128D},{0x1128F,0x1129D},{0x1129F,0x112A8},{0x112B0,0x112EA},{0x11300,0x11303}, + {0x11305,0x1130C},{0x1130F,0x11310},{0x11313,0x11328},{0x1132A,0x11330},{0x11332,0x11333},{0x11335,0x11339}, + {0x1133B,0x11344},{0x11347,0x11348},{0x1134B,0x1134D},{0x11350,0x11350},{0x11357,0x11357},{0x1135D,0x11363}, + {0x11366,0x1136C},{0x11370,0x11374},{0x11400,0x1144A},{0x1145E,0x11461},{0x11480,0x114C5},{0x114C7,0x114C7}, + {0x11580,0x115B5},{0x115B8,0x115C0},{0x115D8,0x115DD},{0x11600,0x11640},{0x11644,0x11644},{0x11680,0x116B8}, + {0x11700,0x1171A},{0x1171D,0x1172B},{0x11800,0x1183A},{0x118FF,0x11906},{0x11909,0x11909},{0x1190C,0x11913}, + {0x11915,0x11916},{0x11918,0x11935},{0x11937,0x11938},{0x1193B,0x11943},{0x119A0,0x119A7},{0x119AA,0x119D7}, + {0x119DA,0x119E1},{0x119E3,0x119E4},{0x11A00,0x11A3E},{0x11A47,0x11A47},{0x11A50,0x11A99},{0x11A9D,0x11A9D}, + {0x11AC0,0x11AF8},{0x11C00,0x11C08},{0x11C0A,0x11C36},{0x11C38,0x11C40},{0x11C72,0x11C8F},{0x11C92,0x11CA7}, + {0x11CA9,0x11CB6},{0x11D00,0x11D06},{0x11D08,0x11D09},{0x11D0B,0x11D36},{0x11D3A,0x11D3A},{0x11D3C,0x11D3D}, + {0x11D3F,0x11D47},{0x11D60,0x11D65},{0x11D67,0x11D68},{0x11D6A,0x11D8E},{0x11D90,0x11D91},{0x11D93,0x11D98}, + {0x11EE0,0x11EF6},{0x11FB0,0x11FB0},{0x12000,0x12399},{0x12480,0x12543},{0x13000,0x1342E},{0x14400,0x14646}, + {0x16800,0x16A38},{0x16A40,0x16A5E},{0x16AD0,0x16AED},{0x16AF0,0x16AF4},{0x16B00,0x16B36},{0x16B40,0x16B43}, + {0x16B63,0x16B77},{0x16B7D,0x16B8F},{0x16F00,0x16F4A},{0x16F4F,0x16F87},{0x16F8F,0x16F9F},{0x16FE0,0x16FE1}, + {0x16FE3,0x16FE4},{0x16FF0,0x16FF1},{0x17000,0x187F7},{0x18800,0x18CD5},{0x18D00,0x18D08},{0x1B000,0x1B11E}, + {0x1B150,0x1B152},{0x1B164,0x1B167},{0x1B170,0x1B2FB},{0x1BC00,0x1BC6A},{0x1BC70,0x1BC7C},{0x1BC80,0x1BC88}, + {0x1BC90,0x1BC99},{0x1BC9D,0x1BC9E},{0x1D165,0x1D169},{0x1D16D,0x1D172},{0x1D17B,0x1D182},{0x1D185,0x1D18B}, + {0x1D1AA,0x1D1AD},{0x1D242,0x1D244},{0x1DA00,0x1DA36},{0x1DA3B,0x1DA6C},{0x1DA75,0x1DA75},{0x1DA84,0x1DA84}, + {0x1DA9B,0x1DA9F},{0x1DAA1,0x1DAAF},{0x1E000,0x1E006},{0x1E008,0x1E018},{0x1E01B,0x1E021},{0x1E023,0x1E024}, + {0x1E026,0x1E02A},{0x1E100,0x1E12C},{0x1E130,0x1E13D},{0x1E14E,0x1E14E},{0x1E2C0,0x1E2EF},{0x1E800,0x1E8C4}, + {0x1E8D0,0x1E8D6},{0x1E944,0x1E94B},{0x1EE00,0x1EE03},{0x1EE05,0x1EE1F},{0x1EE21,0x1EE22},{0x1EE24,0x1EE24}, + {0x1EE27,0x1EE27},{0x1EE29,0x1EE32},{0x1EE34,0x1EE37},{0x1EE39,0x1EE39},{0x1EE3B,0x1EE3B},{0x1EE42,0x1EE42}, + {0x1EE47,0x1EE47},{0x1EE49,0x1EE49},{0x1EE4B,0x1EE4B},{0x1EE4D,0x1EE4F},{0x1EE51,0x1EE52},{0x1EE54,0x1EE54}, + {0x1EE57,0x1EE57},{0x1EE59,0x1EE59},{0x1EE5B,0x1EE5B},{0x1EE5D,0x1EE5D},{0x1EE5F,0x1EE5F},{0x1EE61,0x1EE62}, + {0x1EE64,0x1EE64},{0x1EE67,0x1EE6A},{0x1EE6C,0x1EE72},{0x1EE74,0x1EE77},{0x1EE79,0x1EE7C},{0x1EE7E,0x1EE7E}, + {0x1EE80,0x1EE89},{0x1EE8B,0x1EE9B},{0x1EEA1,0x1EEA3},{0x1EEA5,0x1EEA9},{0x1EEAB,0x1EEBB},{0x20000,0x2A6DD}, + {0x2A700,0x2B734},{0x2B740,0x2B81D},{0x2B820,0x2CEA1},{0x2CEB0,0x2EBE0},{0x2F800,0x2FA1D},{0x30000,0x3134A}, + {0xE0100,0xE01EF}, +}; +static const int uni_X_n = 595; + +static inline int is_U(uint32_t c){ return uni_in(uni_U,uni_U_n,c); } +static inline int is_X(uint32_t c){ return uni_in(uni_X,uni_X_n,c); } +#endif diff --git a/c/tools/benchmark_cuda_fixture.py b/c/tools/benchmark_cuda_fixture.py index bced18e..26512dd 100644 --- a/c/tools/benchmark_cuda_fixture.py +++ b/c/tools/benchmark_cuda_fixture.py @@ -17,6 +17,12 @@ PROFILE_RE = re.compile( r"\| attention ([0-9.]+)s .* lm_head ([0-9.]+)s \| other ([0-9.-]+)s" ) PROFILE_KEYS = ("disk", "expert_matmul", "attention", "lm_head", "other") +P0_RE = re.compile( + r"P0-EXEC: routed CPU ([0-9.]+)s / ([0-9.]+) GB/s \(([0-9]+) row\) \| routed GPU critical ([0-9.]+)s \| " + r"router ([0-9.]+)s \| residual P2P ([0-9.]+)s / ([0-9]+) hop \| orchestration ([0-9.]+)s" +) +P0_KEYS = ("routed_cpu", "routed_cpu_gb_s", "routed_cpu_rows", "routed_gpu_critical", + "router", "p2p", "p2p_hops", "orchestration") def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]: @@ -30,11 +36,20 @@ def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]: return float(speed.group(1)), [disk] + [float(value) for value in rest] -def execute(engine: str, env: dict[str, str]) -> tuple[float, list[float]]: +def parse_p0(stdout: str) -> list[float]: + """Extract the optional PROF=1 execution-layer breakdown.""" + row = P0_RE.search(stdout) + if not row: + raise RuntimeError("benchmark output missing P0-EXEC profile") + return [float(value) for value in row.groups()] + + +def execute(engine: str, env: dict[str, str]) -> tuple[float, list[float], list[float]]: run = subprocess.run( [engine, "4", "4", "4"], env=env, text=True, capture_output=True, check=True ) - return parse_output(run.stdout, run.stderr) + speed, profile = parse_output(run.stdout, run.stderr) + return speed, profile, parse_p0(run.stdout) def main() -> None: @@ -63,6 +78,8 @@ def main() -> None: OMP_NUM_THREADS=str(args.threads), OMP_PROC_BIND="spread", OMP_PLACES="cores", + DRAFT="0", + PROF="1", ) execute(args.engine, base | {"STATS": str(stats)}) @@ -86,13 +103,15 @@ def main() -> None: execute(args.engine, base | extra) # warm-up speeds = {name: [] for name in modes} profiles = {name: [] for name in modes} + p0_profiles = {name: [] for name in modes} names = list(modes) for run_index in range(args.runs): order = names[run_index % len(names):] + names[:run_index % len(names)] for name in order: - speed, profile = execute(args.engine, base | modes[name]) + speed, profile, p0 = execute(args.engine, base | modes[name]) speeds[name].append(speed) profiles[name].append(profile) + p0_profiles[name].append(p0) result = {} for name in names: @@ -103,6 +122,10 @@ def main() -> None: key: statistics.median(row[index] for row in profiles[name]) for index, key in enumerate(PROFILE_KEYS) }, + "median_p0": { + key: statistics.median(row[index] for row in p0_profiles[name]) + for index, key in enumerate(P0_KEYS) + }, } print(json.dumps(result, indent=2)) diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py index 382125e..75f2164 100644 --- a/c/tools/convert_fp8_to_int4.py +++ b/c/tools/convert_fp8_to_int4.py @@ -247,6 +247,32 @@ def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits, def free_gb(p): return shutil.disk_usage(p).free / 1e9 +def check_or_record_params(outdir, prefix, params): + """#383-class guard, mirrored onto the --repo download loops from the --indir + path's resume manifest (below): a resumed run with DIFFERENT conversion + parameters (bits, group size, PROJ_BITS, ...) must not silently mix bit-widths + across shards in the same outdir -- the #355 failure mode (a second pass with + changed flags overwriting/interleaving with a finished container in silence). + Unlike the --indir manifest this doesn't need to track per-shard completion: + the --repo loops already do that via out-NNNNN.safetensors existence, since + shard index maps directly to output filename there. Only whether the params + used SO FAR match this run's needs checking. Returns False (caller should + abort) on a mismatch, True otherwise; records params on first use.""" + path = os.path.join(outdir, f".{prefix}params.json") + if os.path.exists(path): + try: prev = json.loads(open(path).read()) + except (OSError, ValueError): prev = None + if prev is not None and prev != params: + print(f"ERROR: {path} records a conversion with {prev};\n" + f" this run uses {params}. Refusing to mix conversions in the " + f"same outdir — use a fresh --outdir (or delete {path} and the " + f"{prefix}*.safetensors shards to redo).") + return False + tmp = path + ".tmp" + with open(tmp, "w") as f: json.dump(params, f, indent=1) # atomic write, same reasoning as the --indir manifest + os.replace(tmp, path) + return True + def main(): ap = argparse.ArgumentParser() ap.add_argument("--repo", default=None) @@ -286,6 +312,17 @@ def main(): # testa MTP a int4 = acceptance ~0-4% (misurato, issue #8): il draft sbaglia sempre # e la speculazione non parte mai. A int8: 39-59%, 2.2-2.8 token/forward. a.ebits = 8 if (a.mtp or a.indexer) else 4 + if a.mtp and a.ebits < 8 and a.group_size <= 0: + # Non solo lossy: eh_proj ha ~20-30x di asimmetria di scala fra le due meta' di + # colonna, quindi l'int4 per-riga (UNA scala per riga) arrotonda a ZERO l'intera + # meta' embedding -> il draft non vede il token -> acceptance ~0% (issue #8). + # EN: not merely lossy: eh_proj has ~20-30x column-scale asymmetry, so per-row + # EN: int4 rounds its ENTIRE embedding half to exact zeros -> the draft cannot + # EN: see the input token -> ~0% acceptance (issue #8). A container converted + # EN: this way is repairable in place with tools/repair_mtp_int8.py. + print(f"WARNING: --mtp with --ebits {a.ebits} and per-row scales ZEROES eh_proj's " + "embedding half -> MTP acceptance ~0% (issue #8). Use the default --ebits 8, " + "or add --group-size 128 for group-scaled int4.") if a.xbits is None: a.xbits = a.ebits # Build per-type bits map. If a type-specific arg is set, use it; otherwise the @@ -299,6 +336,16 @@ def main(): if bits_map: print(f"[MIXED] precision map: " + ", ".join(f"{k}={v}bit" for k,v in sorted(bits_map.items()))) + # Il PIANO risolto, PRIMA di toccare qualunque cosa (#383): --mtp/--indexer cambiano il + # default di ebits a 8 (testa int4 = acceptance ~0%, issue #8) e il ramo grouped e' + # gated su bits<=4 — combinazioni sorprendenti devono mostrarsi al secondo 1 di un job + # da ore, non nel size-check dopo. EN: print the RESOLVED plan before doing anything. + mode = "MTP head only" if a.mtp else "DSA indexer only" if a.indexer else "main model" + grp = f"grouped gs={a.group_size} (fmt=4)" if (a.group_size and a.ebits <= 4) else \ + (f"PER-ROW (grouped branch needs bits<=4; ebits={a.ebits} disables it)" if a.group_size else "per-row") + print(f"[PLAN] mode: {mode} | source: {'local ' + a.indir if a.indir else 'download ' + a.repo} | " + f"experts {a.ebits}-bit, embed/lm_head {a.io_bits}-bit, x {a.xbits}-bit | {grp}") + if a.selftest_nvfp4: import torch # 1) LUT e2m1: i 16 codici devono decodificare esattamente ai valori attesi. @@ -379,14 +426,100 @@ def main(): if a.indir: # conversione locale (test) shards = sorted(glob.glob(os.path.join(a.indir, "*.safetensors"))) from safetensors.numpy import save_file + # #383: se l'indice c'e', i passaggi --mtp/--indexer convertono SOLO gli shard + # che contengono i tensori richiesti (3 invece di scandire tutti i 141 — ogni + # scansione a vuoto apre comunque uno shard da 5 GB). Senza indice: scansione + # completa come prima. + # EN: #383: when the index is present, the --mtp/--indexer passes convert ONLY + # the shards that hold the requested tensors (3 instead of scanning all 141 — + # every empty scan still opens a 5 GB shard). Without the index: full scan as + # before. + if a.mtp or a.indexer: + idxp = os.path.join(a.indir, "model.safetensors.index.json") + if os.path.exists(idxp): + wmap = json.load(open(idxp))["weight_map"] + if a.mtp: + want = {v for k, v in wmap.items() if k.startswith(f"model.layers.{a.n_layers}.")} + else: + want = {v for k, v in wmap.items() if "indexer" in k and 0 <= layer_idx(k) < a.n_layers} + keep = [sp for sp in shards if os.path.basename(sp) in want] + print(f"[PLAN] index: {len(keep)}/{len(shards)} local shard(s) hold the requested tensors") + shards = keep + # BUG #355: questo ramo ignorava --mtp/--indexer. Con --mtp scriveva + # out-NNNNN (gli STESSI nomi di una conversione normale) in ebits=8 e + # keep_mtp=False -> il "secondo passaggio MTP" nella stessa outdir + # SOVRASCRIVEVA il container gia' finito con una riconversione int8 + # completa, in silenzio (137/141 shard distrutti prima di accorgersene). + # Ora il ramo locale rispecchia il download path: prefisso corretto, + # flag passate, shard vuoti saltati. + prefix = "out-mtp-" if a.mtp else "out-idx-" if a.indexer else "out-" + # RIPRESA (#383): i nomi out-NNNNN contano gli shard EMESSI, non l'indice di + # input (gli shard senza tensori rilevanti non producono file), quindi "il + # file esiste" non basta per saltare il lavoro gia' fatto. Un manifest + # sidecar ricorda input -> output (o "vuoto") e con quali parametri: la + # ripresa salta solo cio' che combacia, e parametri diversi sulla stessa + # outdir vengono rifiutati invece di mescolare container (il modo #355). + # EN: RESUME (#383): out-NNNNN names count EMITTED shards, not the input + # EN: index (shards with no relevant tensors emit no file), so "the file + # EN: exists" is not enough to skip completed work. A sidecar manifest + # EN: records input -> output (or "empty") plus the conversion parameters: + # EN: resume skips only what matches, and different parameters on the same + # EN: outdir are refused instead of mixing containers (the #355 failure mode). + params = {"ebits": a.ebits, "io_bits": a.io_bits, "xbits": a.xbits, + "group_size": a.group_size, "n_layers": a.n_layers, "bits_map": bits_map, + "proj_bits": dict(PROJ_BITS)} + prog_path = os.path.join(a.outdir, f".{prefix}progress.json") + prog = {} + if os.path.exists(prog_path): + try: prog = json.loads(open(prog_path).read()) + except (OSError, ValueError): prog = {} + if prog and prog.get("params") != params: + print(f"ERROR: {prog_path} records a conversion with {prog.get('params')};\n" + f" this run uses {params}. Refusing to mix conversions in the same " + f"outdir — use a fresh --outdir (or delete the manifest and the " + f"{prefix}*.safetensors shards to redo).") + return + done = prog.setdefault("shards", {}); prog["params"] = params + n = 0; fresh = 0; skipped = 0 for i, sp in enumerate(shards): - out = {}; convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits, group_size=a.group_size, bits_map=bits_map) - save_file(out, os.path.join(a.outdir, f"out-{i:05d}.safetensors")) - # copia config + tokenizer - for fn in ["config.json"]: - src = os.path.join(a.indir, fn) - if os.path.exists(src): shutil.copy(src, a.outdir) - print(f"converted {len(shards)} shards -> {a.outdir}") + key = os.path.basename(sp) + prev = done.get(key) # None = mai visto; "" = visto, vuoto; nome = emesso + if prev is not None and (prev == "" or os.path.exists(os.path.join(a.outdir, prev))): + if prev: n += 1 + skipped += 1 + continue + out = {} + convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits, + keep_mtp=a.mtp, keep_idx=a.indexer, + group_size=a.group_size, bits_map=bits_map) + if not out: # shard senza MTP/idx: niente file (come il download path) + done[key] = "" + else: + name = f"{prefix}{n:05d}.safetensors" + save_file(out, os.path.join(a.outdir, name)) + done[key] = name; n += 1; fresh += 1 + tmp_prog = prog_path + ".tmp" # scrittura atomica: una ripresa non vede mai un manifest mezzo scritto + with open(tmp_prog, "w") as f: json.dump(prog, f, indent=1) # EN: atomic write: a resume never sees a half-written manifest + os.replace(tmp_prog, prog_path) + if skipped: print(f"[RESUME] {skipped} shard(s) already done in {a.outdir}, skipped") + # metadati per la conversione principale: gli stessi quattro file del download + # path — senza tokenizer.json chat/serve non partono. I passaggi mtp/idx vanno + # nella stessa outdir di un container gia' completo di metadati. + # EN: metadata for the main pass: the same four files as the download path — + # EN: chat/serve won't start without tokenizer.json. The mtp/idx passes target + # EN: an outdir whose container already has its metadata. + if not a.mtp and not a.indexer: + copied, missing = [], [] + for fn in ["config.json", "tokenizer.json", "tokenizer_config.json", "generation_config.json"]: + src = os.path.join(a.indir, fn) + if os.path.exists(src): shutil.copy(src, a.outdir); copied.append(fn) + else: missing.append(fn) + print(f"[META] copied from {a.indir}: {', '.join(copied) if copied else 'nothing'}") + if missing: + print(f"[META] WARNING: not found in {a.indir}: {', '.join(missing)}" + + (" — chat/serve need tokenizer.json" if "tokenizer.json" in missing else "")) + tag = "MTP" if a.mtp else "indexer" if a.indexer else "main" + print(f"converted {fresh} {tag} shard(s), {n} in container -> {a.outdir} ({prefix}NNNNN)") return # reale: scarica shard per shard, converte, cancella @@ -612,6 +745,10 @@ def main(): except Exception: pass tmp = os.path.join(a.outdir, "_inflight"); os.makedirs(tmp, exist_ok=True) if a.mtp: + params = {"ebits": a.ebits, "io_bits": a.io_bits, "xbits": a.xbits, + "group_size": a.group_size, "n_layers": a.n_layers, "bits_map": bits_map, + "proj_bits": dict(PROJ_BITS)} + if not check_or_record_params(a.outdir, "out-mtp-", params): return import urllib.request idx = json.loads(urllib.request.urlopen( f"https://huggingface.co/{a.repo}/resolve/main/model.safetensors.index.json", timeout=30).read())["weight_map"] @@ -631,6 +768,10 @@ def main(): print(f" -> {os.path.basename(outp)} ({os.path.getsize(outp)/1e9:.2f} GB, {len(out)} tensors)", flush=True) shutil.rmtree(tmp, ignore_errors=True); print("[MTP] DONE."); return if a.indexer: + params = {"ebits": a.ebits, "io_bits": a.io_bits, "xbits": a.xbits, + "group_size": a.group_size, "n_layers": a.n_layers, "bits_map": bits_map, + "proj_bits": dict(PROJ_BITS)} + if not check_or_record_params(a.outdir, "out-idx-", params): return import urllib.request idx = json.loads(urllib.request.urlopen( f"https://huggingface.co/{a.repo}/resolve/main/model.safetensors.index.json", timeout=30).read())["weight_map"] @@ -650,6 +791,10 @@ def main(): if os.path.isfile(blob): os.remove(blob) print(f" -> {os.path.basename(outp)} ({len(out)} tensors)", flush=True) shutil.rmtree(tmp, ignore_errors=True); print("[IDX] DONE."); return + params = {"ebits": a.ebits, "io_bits": a.io_bits, "xbits": a.xbits, + "group_size": a.group_size, "n_layers": a.n_layers, "bits_map": bits_map, + "proj_bits": dict(PROJ_BITS)} + if not check_or_record_params(a.outdir, "out-", params): return for i, sh in enumerate(shards): if free_gb(a.outdir) < a.min_free_gb: print(f"STOP: free space is below {a.min_free_gb} GB. Free space and rerun to resume."); break diff --git a/c/tools/convert_olmoe.py b/c/tools/convert_olmoe.py index dd45806..dc25f6a 100644 --- a/c/tools/convert_olmoe.py +++ b/c/tools/convert_olmoe.py @@ -3,7 +3,10 @@ Downloads or converts a local OLMoE checkpoint (e.g., allenai/OLMoE-1B-7B-0125-Instruct). Dense weights stay as-is (engine reads BF16/F16 → F32 on load). -Expert weights get row-wise int8 quantization with float32 scales. +Expert weights get row-wise symmetric quantization to --ebits bits (default 4) +with float32 scales. Storage stays one value per int8 byte regardless of bits, +matching the engine's expert layout (olmoe.c quantize_rows) — for 4 bits the +values are simply confined to [-8, 7] with scales computed against qmax=7. Usage: python tools/convert_olmoe.py --repo allenai/OLMoE-1B-7B-0125-Instruct --out ./olmoe_i4 @@ -29,12 +32,21 @@ except ImportError as exc: EXPERT_KEY_RE = r"model\.layers\.\d+\.mlp\.experts\.\d+\.(gate_proj|up_proj|down_proj)\.weight" -def quantize_row(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: - """Row-wise int8 quantization. Returns (int8_weights, float32_scales).""" +def quantize_row(w: torch.Tensor, bits: int = 8) -> tuple[torch.Tensor, torch.Tensor]: + """Row-wise symmetric quantization to `bits` (2..8). + + Returns (int8_weights, float32_scales). Storage is one value per int8 byte + for every bit width — the engine dequantizes as q*scale and never assumes + the full int8 range — mirroring olmoe.c quantize_rows(): + qmax = 2**(bits-1) - 1 (8 -> 127, 4 -> 7, 2 -> 1) + scale = amax(|w|, row) / qmax + q = clamp(round(w / scale), -qmax-1, qmax) + """ + qmax = (1 << (bits - 1)) - 1 w_f32 = w.float() row_max = w_f32.abs().amax(dim=1, keepdim=True).clamp(min=1e-12) - scales = row_max / 127.0 - q = (w_f32 / scales).round().clamp(-128, 127).to(torch.int8) + scales = row_max / qmax + q = (w_f32 / scales).round().clamp(-qmax - 1, qmax).to(torch.int8) return q, scales.squeeze(1) @@ -50,9 +62,12 @@ def main(): src.add_argument("--model", help="Local HF checkpoint directory") ap.add_argument("--out", required=True, help="Output directory for int4 model") ap.add_argument("--ebits", type=int, default=4, - help="Expert quant bits (4 or 8, default 4)") + help="Expert quant bits (2..8, default 4)") args = ap.parse_args() + if not 2 <= args.ebits <= 8: # storage is int8_t; engine rejects the same range (olmoe.c) + sys.exit(f"--ebits must be 2..8 (got {args.ebits})") + if args.repo: from huggingface_hub import snapshot_download from huggingface_hub.errors import LocalEntryNotFoundError @@ -96,7 +111,7 @@ def main(): for name, tensor in tensors.items(): if is_expert_weight(name): expert_count += 1 - q, scales = quantize_row(tensor) + q, scales = quantize_row(tensor, args.ebits) total_expert_f32 += tensor.numel() * tensor.element_size() total_expert_q += q.numel() * 1 + scales.numel() * 4 out_tensors[name] = q @@ -109,7 +124,7 @@ def main(): ratio = total_expert_q / max(total_expert_f32, 1) * 100 print(f"ok") - print(f"\nDone. {expert_count} expert tensors quantized.") + print(f"\nDone. {expert_count} expert tensors quantized to int{args.ebits}.") print(f"Expert storage: {total_expert_f32/1e9:.1f} GB -> {total_expert_q/1e9:.1f} GB ({ratio:.0f}%)") print(f"Model ready at: {out}") print(f"\nRun: SNAP={out} ./olmoe.exe 32 4 16") diff --git a/c/tools/convert_olmoe_merged.py b/c/tools/convert_olmoe_merged.py new file mode 100644 index 0000000..883b8df --- /dev/null +++ b/c/tools/convert_olmoe_merged.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Convert OLMoE HuggingFace checkpoint to colibri merged int8 format. + +Consolidates gate_proj, up_proj, and down_proj into a single merged tensor per expert. +This allows olmoe.c to load an expert in a single disk read call instead of 3. + +Usage: + python tools/convert_olmoe_merged.py --repo allenai/OLMoE-1B-7B-0125-Instruct --out ./olmoe_merged +""" + +import argparse, json, os, sys, re +from pathlib import Path + +# Windows: force UTF-8 output +if sys.platform == "win32": + for s in (sys.stdout, sys.stderr): + try: s.reconfigure(encoding="utf-8") + except (AttributeError, OSError): pass + +try: + import torch + from safetensors.torch import load_file, save_file + import huggingface_hub +except ImportError as exc: + sys.exit(f"Missing dependencies: {exc}. Install: pip install torch safetensors huggingface_hub") + +EXPERT_KEY_RE = r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight" + +def quantize_row(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Row-wise int8 quantization. Returns (int8_weights, float32_scales).""" + w_f32 = w.float() + row_max = w_f32.abs().amax(dim=1, keepdim=True).clamp(min=1e-12) + scales = row_max / 127.0 + q = (w_f32 / scales).round().clamp(-128, 127).to(torch.int8) + return q, scales.squeeze(1) + +def main(): + ap = argparse.ArgumentParser(description="Convert OLMoE HF checkpoint -> colibri merged int8") + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("--repo", help="HuggingFace repo ID") + src.add_argument("--model", help="Local HF checkpoint directory") + ap.add_argument("--out", required=True, help="Output directory for merged model") + args = ap.parse_args() + + if args.repo: + from huggingface_hub import snapshot_download + from huggingface_hub.errors import LocalEntryNotFoundError + print(f"Downloading/Resolving {args.repo}...") + try: + src_dir = snapshot_download(args.repo, local_files_only=True, max_workers=4) + except LocalEntryNotFoundError: + src_dir = None + if src_dir is None or not any(Path(src_dir).glob("*.safetensors")): + print("Downloading safetensors...") + src_dir = snapshot_download(args.repo, max_workers=4) + else: + src_dir = args.model + + src = Path(src_dir) + if not src.is_dir(): + sys.exit(f"Model directory not found: {src}") + if not (src / "config.json").is_file(): + sys.exit(f"config.json missing in {src}") + + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + + # Copy config.json + import shutil + shutil.copy2(src / "config.json", out / "config.json") + print(f"config.json -> {out}") + + # Process safetensors + shards = sorted(src.glob("*.safetensors")) + if not shards: + sys.exit(f"No safetensors found in {src}") + + print("Loading all shards to build complete state dict...") + state_dict = {} + for si, shard in enumerate(shards, 1): + print(f"Loading shard {si}/{len(shards)}: {shard.name}...") + tensors = load_file(str(shard)) + state_dict.update(tensors) + + # Gather experts + experts = {} + for name in list(state_dict.keys()): + m = re.match(EXPERT_KEY_RE, name) + if m: + layer_idx, expert_idx, proj = m.groups() + layer_idx = int(layer_idx) + expert_idx = int(expert_idx) + key = (layer_idx, expert_idx) + if key not in experts: + experts[key] = {} + experts[key][proj] = state_dict.pop(name) + + print(f"Found {len(experts)} experts to merge.") + + # Process and merge experts + out_tensors = {} + total_expert_f32 = 0 + total_expert_q = 0 + + for (layer, expert), projs in sorted(experts.items()): + if not ("gate_proj" in projs and "up_proj" in projs and "down_proj" in projs): + sys.exit(f"Missing projection for layer {layer} expert {expert}!") + + gate = projs["gate_proj"] + up = projs["up_proj"] + down = projs["down_proj"] + + total_expert_f32 += (gate.numel() + up.numel() + down.numel()) * gate.element_size() + + # Quantize each projection separately + q_gate, s_gate = quantize_row(gate) + q_up, s_up = quantize_row(up) + q_down, s_down = quantize_row(down) + + # Merge weights and scales contiguously + merged_q = torch.cat([q_gate.flatten(), q_up.flatten(), q_down.flatten()]) + merged_scales = torch.cat([s_gate, s_up, s_down]) + + total_expert_q += merged_q.numel() * 1 + merged_scales.numel() * 4 + + # Save to output + out_tensors[f"model.layers.{layer}.mlp.experts.{expert}.merged_weight"] = merged_q + out_tensors[f"model.layers.{layer}.mlp.experts.{expert}.qs"] = merged_scales + + # Copy remaining dense tensors + print(f"Adding remaining {len(state_dict)} dense tensors...") + out_tensors.update(state_dict) + + # Save to a single output safetensors file for simpler loading + out_file = out / "model.safetensors" + print(f"Saving merged safetensors model to {out_file}...") + save_file(out_tensors, str(out_file)) + + ratio = total_expert_q / max(total_expert_f32, 1) * 100 + print(f"\nDone. {len(experts)} experts successfully merged and saved.") + print(f"Expert storage: {total_expert_f32/1e9:.1f} GB -> {total_expert_q/1e9:.1f} GB ({ratio:.0f}%)") + print(f"Model ready at: {out}") + +if __name__ == "__main__": + main() diff --git a/c/tools/efficiency.py b/c/tools/efficiency.py new file mode 100644 index 0000000..775e8cf --- /dev/null +++ b/c/tools/efficiency.py @@ -0,0 +1,458 @@ +"""Efficiency / regression harness for the colibri engine. + +The engine already emits rich telemetry (REPLAY tok/s, PROFILE phase timings, +[PROF] time shares + verdict, CUDA expert-tier utilization). Until now every +consumer of that telemetry — `benchmark_cuda_fixture.py`, `bench_full.sh`, +`bench_ux.sh` — has only *printed* it for a human to eyeball. This module turns +each signal into a parseable field so tests can assert on it. + +Design: + - Reuses SPEED_RE / PROFILE_RE from tools.benchmark_cuda_fixture (no drift). + - parse_run() is pure: stdout+stderr in, dict out. Easy to unit-test against + captured strings (like the existing test_benchmark_cuda_fixture does). + - run_engine() is the subprocess wrapper. Captures stdout and stderr + separately, because the engine splits them: PROFILE/REPLAY/CUDA-tier go to + stdout, the [CUDA]/[PROF]/[prefill] banners go to stderr. + - Floor defaults are module constants (tunable in one place, not scattered). + +No model file is required to import this module; only run_engine() invokes the +binary. parse_run() works on any captured text, so most test surface is covered +by string fixtures without spinning the engine at all. +""" +from __future__ import annotations + +import os +import re +import subprocess +from pathlib import Path +from typing import Optional + +# Reuse the validated regexes from the existing A/B benchmark harness so the +# PROFILE field order (disk, expert_matmul, attention, lm_head, other) and the +# tok/s capture stay identical. Drift here would silently break every consumer. +from tools.benchmark_cuda_fixture import SPEED_RE as _SPEED_RE_REPLAY, PROFILE_RE, PROFILE_KEYS + +# SPEED_RE (from benchmark_cuda_fixture) matches the REPLAY-mode line only: +# "REPLAY decode: ... | 12.34 tok/s | ..." +# run_text / PROMPT mode uses a DIFFERENT format (glm.c:4682): +# "decode N tokens in X.XXs (12.34 tok/s) | expert hit rate ..." +# This alt regex catches the parenthesized form so the full-model report (which +# uses PROMPT mode) gets a real tok/s instead of reporting it missing. +SPEED_RE_TEXT = re.compile(r"decode \d+ tokens in [0-9.]+s \(([0-9.]+) tok/s\)") + + +def _first_speed(stdout: str): + """Find tok/s in whichever run-mode format the engine used.""" + for rx in (_SPEED_RE_REPLAY, SPEED_RE_TEXT): + m = rx.search(stdout) + if m: + return m + return None + + +# Public alias so existing imports keep working (tests reference SPEED_RE). +SPEED_RE = _SPEED_RE_REPLAY + + +# --- additional parsers (formats verified against glm.c printf strings) --- + +# "expert hit rate 88.1%" (summary line) | "expert hit 95.0%" (REPLAY line) +HIT_RE = re.compile(r"expert hit(?:\s+rate)?\s+([0-9.]+)%") + +# "[PROF] time shares: expert-I/O 3% | expert-matmul 12% | attention 56% | lm_head 2% | other 27%" +SHARES_RE = re.compile( + r"\[PROF\] time shares: expert-I/O\s+([0-9.]+)%\s*\|\s*expert-matmul\s+([0-9.]+)%\s*" + r"\|\s*attention\s+([0-9.]+)%\s*\|\s*lm_head\s+([0-9.]+)%\s*\|\s*other\s+([0-9.]+)%" +) + +# "[PROF] verdict: I/O-bound — 60% of the time ..." (also compute-bound / attention-bound / balanced) +VERDICT_RE = re.compile(r"\[PROF\] verdict:\s*(I/O-bound|compute-bound|attention-bound|balanced)") + +# "[PROF] expert I/O: ... hit 95.0% (76 hit / 4 load) | 4.0 loads/token" +LOADS_PER_TOK_RE = re.compile(r"\|\s*([0-9.]+)\s+loads/token") + +# "CUDA expert tier: 111 resident experts (2.36 GB) | 5400 calls served from VRAM" (stdout) +CUDA_TIER_RE = re.compile( + r"CUDA expert tier:\s+(\d+)\s+resident experts\s+\(([0-9.]+)\s+GB\)\s*\|\s+(\d+)\s+calls served from VRAM" +) + +# "[CUDA] device 0: NVIDIA ..., 14.4 GB VRAM, sm_120" (stderr, per device at init) +CUDA_DEVICE_RE = re.compile(r"\[CUDA\] device\s+\d+:") + +# "[CUDA] resident set: 12 tensors, 0.45 GB VRAM" (stderr, cuda_stats_print) +CUDA_RESIDENT_RE = re.compile( + r"\[CUDA\] resident set:\s+(\d+)\s+tensors,\s+([0-9.]+)\s+GB\s+VRAM" +) + +# "PREFILL (teacher-forcing) C vs oracle: 11/32 positions | 1700.4 pos/s" (TF=1 mode, stdout) +TF_MATCH_RE = re.compile(r"PREFILL \(teacher-forcing\).*:\s+(\d+)/(\d+)\s+positions") + +# --- the six deeper signals (added after "are you gathering everything?" audit) --- + +# "ATTENTION: projection/RoPE 0.050s | score-softmax-value 0.009s | output projection 0.011s" +# Sub-breakdown of the attention phase — answers "how is attention being read". +ATTN_BREAKDOWN_RE = re.compile( + r"ATTENTION: projection/RoPE\s+([0-9.]+)s\s*\|\s*score-softmax-value\s+([0-9.]+)s\s*" + r"\|\s*output projection\s+([0-9.]+)s" +) + +# "[PROF] decode forwards: 20 | latency p50 7.3 ms | p90 8.0 ms | p99 8.1 ms | max 8.2 ms | 1.00 tok/forward" +# Per-forward tail latency — a p99 >> p50 means decode stalls (I/O hiccups, KV grow). +LATENCY_RE = re.compile( + r"\[PROF\] decode forwards:\s+(\d+)\s*\| latency p50\s+([0-9.]+)\s*ms\s*\| " + r"p90\s+([0-9.]+)\s*ms\s*\|\s*p99\s+([0-9.]+)\s*ms\s*\|\s*max\s+([0-9.]+)\s*ms" +) + +# "[PROF] expert I/O: 0.004 GB fetched (0.2 MB/token, 0.03 GB/s over the run) | hit 95.0% ... | +# 4.0 loads/token | 0.0s read service / 0.0s felt wait" +# Absolute disk throughput + the felt-wait split (the [PROF] version, more detailed than PROFILE). +EXPERT_IO_RE = re.compile( + r"\[PROF\] expert I/O:\s+([0-9.]+)\s+GB fetched\s+\(([0-9.]+)\s+MB/token,\s*([0-9.]+)\s+GB/s" + r".*?\|\s*([0-9.]+)\s+loads/token\s*\|\s*([0-9.]+)s\s+read service\s*/\s*([0-9.]+)s\s+felt wait" +) + +# "speculation: 1.05 tokens/forward (19 forwards per 20 tokens) | MTP acceptance 44% (7/16)" +# Draft efficiency — is the speculative decoder pulling weight or dead overhead? +SPECULATION_RE = re.compile( + r"speculation:\s+([0-9.]+)\s+tokens/forward\s+\((\d+)\s+forwards per\s+(\d+)\s+tokens\)" + r"\s*\|\s*MTP acceptance\s+([0-9.]+)%" +) + +# "experts loaded/token: 450.0 (per-layer 56.25 across 8; baseline topk=8) | TOPK=0 TOPP=0.00" +# Fuller than loads_per_tok: includes the per-layer spread + the active topk/topp. +EXPERTS_LOADED_RE = re.compile( + r"experts loaded/token:\s+([0-9.]+)\s+\(per-layer\s+([0-9.]+)\s+across\s+(\d+);\s*baseline topk=(\d+)\)" +) + +# "[PROF] machine: Intel(...) | 22 cores (22 omp threads) | RAM 34.1 GB total, 27.1 GB available | backend CUDA" +# Provenance — makes a report reproducible across machines/runs. +MACHINE_RE = re.compile(r"\[PROF\] machine:\s*(.*)\|\s*(\d+)\s+cores.*backend\s+(\S+)") + +# "[PROF] config: RAM_GB=auto 23.9 CTX=4096 | expert cache cap 8/layer ... | DRAFT=0 PIPE=1 DIRECT=0 ..." +# Effective resolved config (after auto-budgeting). Answers "what config actually ran". +CONFIG_RE = re.compile(r"\[PROF\] config:\s*(.*)") + +# "[ORACLE] mismatch pos=7 expected=197 got=22" (TF=1 mode, stderr, per position) +# Captures the engine's *actual* argmax at each position, so two backends can be +# compared DIRECTLY (independent of how each relates to the oracle). +TF_MISMATCH_RE = re.compile(r"\[ORACLE\] mismatch pos=(\d+) expected=\d+ got=(\d+)") + +# --- routing-quality + disk-split + cuda-groups (the deep signals) --- + +# summary suffix: " | swap 3.1% (12/384)" (CACHE_ROUTE inline) +SWAP_RE = re.compile(r"swap\s+([0-9.]+)%\s+\((\d+)/(\d+)\)") + +# summary suffix: " | route_agree 85.0% | route_kl 0.0123" (CACHE_ROUTE/ROUTE_AGREE) +ROUTE_AGREE_RE = re.compile(r"route_agree\s+([0-9.]+)%\s*\|\s*route_kl\s+([0-9.]+)") + +# "disk-load split: draft 8 + absorb 0 + verify/main 3 misses | MTP-layer 0 loads 0.00 GB | +# main-layers 11 loads 0.01 GB (MTP 0.0% of bytes)" (DISK_SPLIT=1) +DISK_SPLIT_RE = re.compile( + r"disk-load split: draft\s+(\d+)\s+\+\s+absorb\s+(\d+)\s+\+\s+verify/main\s+(\d+)\s+misses" + r"\s*\|\s*MTP-layer\s+(\d+)\s+loads\s+([0-9.]+)\s+GB\s*\|\s*main-layers\s+(\d+)\s+loads\s+([0-9.]+)\s+GB" + r"(?:\s+\(MTP\s+([0-9.]+)%\s+of bytes\))?" +) + +# "[CUDA] expert groups: 120 call, 840 expert, 1200 righe (7.00 expert/call)" +CUDA_GROUPS_RE = re.compile( + r"\[CUDA\] expert groups:\s+(\d+)\s+call,\s+(\d+)\s+expert,\s+(\d+)\s+righe\s+\(([0-9.]+)\s+expert/call\)" +) + +# "[CUDA] expert groups timing: H2D 12.3 ms | kernel 45.6 ms | D2H 7.8 ms" (COLI_CUDA_PROFILE=1) +CUDA_GROUPS_TIME_RE = re.compile( + r"\[CUDA\] expert groups timing: H2D\s+([0-9.]+)\s+ms\s*\|\s*kernel\s+([0-9.]+)\s+ms\s*\|\s*D2H\s+([0-9.]+)\s+ms" +) + +# LOOKAHEAD recall block — 4 named rows. (name, pct, hit, tot) +LOOKAHEAD_RE = re.compile( + r"^\s*(.+?)\s+([0-9.]+)%\s+\((\d+)/(\d+)\)\s*$", re.MULTILINE +) + +# "loaded in 0.02s | resident dense: 0.21 MB | layers=5 experts=8 | MTP absent (draft=0)" +LOAD_BANNER_RE = re.compile( + r"loaded in\s+([0-9.]+)s\s*\|\s*resident dense:\s+([0-9.]+)\s+MB\s*\|" + r"\s*layers=(\d+)\s+experts=(\d+)\s*\|\s*MTP\s+(\w+)\s+\(draft=(\d+)\)" +) + + +# --- tunable floors ----------------------------------------------------------- +# These are deliberately generous so they catch *regressions* (broken builds, +# pathological configs, telemetry accounting bugs) without flapping on machine +# noise. The tiny model is fully resident at ~200 tok/s, so a 20 tok/s floor is +# a 10x margin. Tune per-host via env if needed (documented in README). +TINY_TOK_S_FLOOR = float(os.environ.get("COLI_TINY_TOK_S_FLOOR", "20.0")) +# On a fully-resident tiny model the expert-disk wait share should be tiny. +# If it exceeds this, something regressed in the I/O accounting or cache path. +MAX_DISK_WAIT_SHARE = float(os.environ.get("COLI_MAX_DISK_WAIT_SHARE", "0.20")) +# decode wall-time should be roughly the sum of PROFILE phases (other = residual). +PROFILE_SUM_TOLERANCE = float(os.environ.get("COLI_PROFILE_SUM_TOL", "0.05")) +# Minimum direct CPU-vs-CUDA argmax agreement on the tiny TF fixture. The two +# backends use different accumulation orders (SIMD dot vs CUDA kernel), so a +# few near-tied positions flip argmax — that's expected numeric divergence, not +# a kernel bug. Measured baseline ~84% (27/32) on this fixture; the 70% floor +# leaves headroom for machine noise while still catching a catastrophic kernel +# regression (e.g. a wrong GEMM would drop this to ~random = ~4%). +MIN_CPU_CUDA_AGREEMENT = float(os.environ.get("COLI_MIN_CPU_CUDA_AGREE", "0.70")) + + +def parse_run(stdout: str, stderr: str = "") -> dict: + """Parse one engine run's output into a telemetry dict. + + Returns keys: tok_s, hit_pct, profile (dict, seconds), profile_sum, + time_shares (dict, fractions 0..1), verdict, loads_per_tok, cuda (dict), + tf_match (tuple or None), parsed (set of field names found). + + Raises RuntimeError only if the core throughput line is missing — everything + else is optional and absent on some run modes (e.g. [PROF] needs PROF=1, + CUDA tier needs gpu_expert_count>0). + """ + out = dict( + tok_s=None, hit_pct=None, profile=None, profile_sum=None, + time_shares=None, verdict=None, loads_per_tok=None, + cuda=None, tf_match=None, stderr=stderr, + ) + parsed = set() + blob = stdout + "\n" + stderr # [PROF]/[CUDA] live on stderr; scan both. + + m = _first_speed(stdout) + if m: + out["tok_s"] = float(m.group(1)); parsed.add("tok_s") + + m = HIT_RE.search(blob) + if m: + out["hit_pct"] = float(m.group(1)); parsed.add("hit_pct") + + m = PROFILE_RE.search(stdout) + if m: + service, wait, emm, attn, head, other = (float(x) for x in m.groups()) + disk = service + (wait or 0.0) + out["profile"] = dict(zip(PROFILE_KEYS, (disk, emm, attn, head, other))) + out["profile_sum"] = disk + emm + attn + head + other + parsed.add("profile") + + # ATTENTION sub-breakdown: projection/RoPE | score-softmax-value | output. + m = ATTN_BREAKDOWN_RE.search(stdout) + if m: + out["attn_breakdown"] = dict(zip( + ("proj_rope", "score_sm_value", "out_proj"), + (float(x) for x in m.groups()))) + parsed.add("attn_breakdown") + + m = SHARES_RE.search(blob) + if m: + io, emm, attn, head, other = (float(x) / 100.0 for x in m.groups()) + out["time_shares"] = dict(io=io, matmul=emm, attention=attn, head=head, other=other) + parsed.add("time_shares") + + m = VERDICT_RE.search(blob) + if m: + out["verdict"] = m.group(1); parsed.add("verdict") + + # [PROF] decode forwards + latency p50/p90/p99/max. + m = LATENCY_RE.search(blob) + if m: + out["latency"] = dict(zip( + ("forwards", "p50_ms", "p90_ms", "p99_ms", "max_ms"), + (float(x) for x in m.groups()))) + parsed.add("latency") + + # [PROF] expert I/O throughput: GB fetched, MB/token, GB/s, service vs felt wait. + m = EXPERT_IO_RE.search(blob) + if m: + out["expert_io"] = dict(zip( + ("gb_fetched", "mb_per_tok", "gb_per_s", "loads_per_tok", + "read_service_s", "felt_wait_s"), + (float(x) for x in m.groups()))) + parsed.add("expert_io") + + m = LOADS_PER_TOK_RE.search(blob) + if m: + out["loads_per_tok"] = float(m.group(1)); parsed.add("loads_per_tok") + + # experts loaded/token with per-layer spread + baseline topk (run_text summary). + m = EXPERTS_LOADED_RE.search(stdout) + if m: + out["experts_loaded"] = dict( + per_tok=float(m.group(1)), per_layer=float(m.group(2)), + n_sparse_layers=int(m.group(3)), baseline_topk=int(m.group(4))) + parsed.add("experts_loaded") + + # speculation: tokens/forward, forwards, tokens, MTP acceptance%. + m = SPECULATION_RE.search(stdout) + if m: + out["speculation"] = dict(zip( + ("tok_per_fw", "forwards", "tokens", "mtp_accept_pct"), + (float(m.group(1)), int(m.group(2)), int(m.group(3)), float(m.group(4))))) + parsed.add("speculation") + + # routing quality (CACHE_ROUTE inline suffixes on the summary line). + m = SWAP_RE.search(stdout) + if m: + out["swap"] = dict(pct=float(m.group(1)), swaps=int(m.group(2)), slots=int(m.group(3))) + parsed.add("swap") + m = ROUTE_AGREE_RE.search(stdout) + if m: + out["route_agree"] = dict(agree_pct=float(m.group(1)), kl=float(m.group(2))) + parsed.add("route_agree") + + # disk-load split by decode phase (DISK_SPLIT=1). + m = DISK_SPLIT_RE.search(stdout) + if m: + out["disk_split"] = dict(zip( + ("draft", "absorb", "verify_main", "mtp_loads", "mtp_gb", + "main_loads", "main_gb", "mtp_bytes_pct"), + (int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)), + float(m.group(5)), int(m.group(6)), float(m.group(7)), + float(m.group(8)) if m.group(8) else None))) + parsed.add("disk_split") + + # provenance: machine + resolved config. + m = MACHINE_RE.search(blob) + if m: + out["machine"] = dict(cpu=m.group(1).strip(), cores=int(m.group(2)), backend=m.group(3)) + parsed.add("machine") + m = CONFIG_RE.search(blob) + if m: + out["config_str"] = m.group(1).strip(); parsed.add("config") + + # load banner: load time, resident dense MB, layers, experts, MTP status. + m = LOAD_BANNER_RE.search(stdout) + if m: + out["load"] = dict(zip( + ("load_s", "resident_dense_mb", "layers", "experts", "mtp_status", "draft"), + (float(m.group(1)), float(m.group(2)), int(m.group(3)), + int(m.group(4)), m.group(5), int(m.group(6))))) + parsed.add("load") + + # LOOKAHEAD routing-recall block (LOOKA=1): list of {predictor, pct, hit, tot}. + la_block = re.search( + r"LOOKAHEAD routing.*?recall.*?:\n((?:^\s+.+?\s+[0-9.]+%\s+\(\d+/\d+\)\s*$\n?)+)", + blob, re.MULTILINE) + if la_block: + out["lookahead"] = [] + for row in LOOKAHEAD_RE.finditer(la_block.group(1)): + out["lookahead"].append(dict( + predictor=row.group(1).strip(), pct=float(row.group(2)), + hit=int(row.group(3)), tot=int(row.group(4)))) + parsed.add("lookahead") + + cuda = dict(enabled=False, expert_count=None, expert_gb=None, + calls_served=None, resident_tensors=None, resident_gb=None, + groups=None, groups_timing=None) + if CUDA_DEVICE_RE.search(stderr): + cuda["enabled"] = True + m = CUDA_TIER_RE.search(stdout) + if m: + cuda["expert_count"] = int(m.group(1)) + cuda["expert_gb"] = float(m.group(2)) + cuda["calls_served"] = int(m.group(3)) + m = CUDA_RESIDENT_RE.search(stderr) + if m: + cuda["resident_tensors"] = int(m.group(1)) + cuda["resident_gb"] = float(m.group(2)) + m = CUDA_GROUPS_RE.search(stderr) + if m: + cuda["groups"] = dict(zip( + ("calls", "experts", "rows", "experts_per_call"), + (int(m.group(1)), int(m.group(2)), int(m.group(3)), float(m.group(4))))) + m = CUDA_GROUPS_TIME_RE.search(stderr) + if m: + cuda["groups_timing"] = dict(zip( + ("h2d_ms", "kernel_ms", "d2h_ms"), + (float(m.group(1)), float(m.group(2)), float(m.group(3))))) + out["cuda"] = cuda + + m = TF_MATCH_RE.search(stdout) + if m: + out["tf_match"] = (int(m.group(1)), int(m.group(2))); parsed.add("tf_match") + + # Capture per-position argmax divergences from the oracle, keyed by position. + mismatches = {int(mm.group(1)): int(mm.group(2)) + for mm in TF_MISMATCH_RE.finditer(blob)} + if out["tf_match"] is not None: + out["tf_mismatches"] = mismatches + parsed.add("tf_mismatches") + + out["parsed"] = parsed + return out + + +def run_engine( + env_overlay: dict, + *, + engine: Optional[str] = None, + cap: int = 4, + ebits: int = 4, + dbits: int = 4, + timeout: float = 600.0, + snap: Optional[str] = None, +) -> tuple[dict, subprocess.CompletedProcess]: + """Run the engine with an env overlay; return (parsed_telemetry, proc). + + `engine` defaults to ./glm.exe (colibri's Windows host). `snap` defaults to + the bundled tiny model (glm_tiny) so callers can omit it for fast tests. + The positional argv is `cap ebits dbits`, matching the engine's main(). + """ + if engine is None: + engine = str(Path(__file__).resolve().parent.parent / "glm.exe") + env = os.environ.copy() + # Strip CUDA vars by default so a CPU run isn't accidentally GPU-accelerated + # by a leftover env; callers opt in by passing them in env_overlay. + for k in ("COLI_CUDA", "COLI_GPU", "COLI_GPUS", "CUDA_DENSE", "CUDA_EXPERT_GB"): + env.pop(k, None) + env.update(env_overlay) + if snap is not None: + env["SNAP"] = snap + elif "SNAP" not in env: + env["SNAP"] = str(Path(__file__).resolve().parent.parent / "glm_tiny") + + proc = subprocess.run( + [engine, str(cap), str(ebits), str(dbits)], + env=env, capture_output=True, text=True, timeout=timeout, + ) + telemetry = parse_run(proc.stdout, proc.stderr) + telemetry["returncode"] = proc.returncode + telemetry["env"] = {k: env_overlay[k] for k in env_overlay} + return telemetry, proc + + +def disk_wait_share(t: dict) -> Optional[float]: + """Fraction of decode wall-time spent waiting on expert disk reads. + + Preferred source: [PROF] time_shares (the engine's own accounting, which + separates felt-wait from read-service). Falls back to PROFILE disk / sum if + [PROF] wasn't emitted (PROF=0 runs). None if neither is available. + """ + if t.get("time_shares"): + return t["time_shares"]["io"] + if t.get("profile") and t.get("profile_sum"): + return t["profile"]["disk"] / t["profile_sum"] + return None + + +def tf_agreement(cpu: dict, cuda: dict, oracle: list[int]) -> tuple[float, list[int]]: + """Direct CPU-vs-CUDA argmax agreement on the TF fixture. + + Both runs prefilled the SAME oracle sequence; tf_mismatches holds each + backend's actual argmax where it diverged from the oracle. Where a backend + is ABSENT from the mismatch map, its prediction equals the oracle token at + that position. So the reconstructed per-position prediction is: + oracle[i] if i not in mismatches else mismatches[i] + and agreement is the fraction of positions where CPU and CUDA predictions + are identical — independent of how each relates to the oracle. + + `oracle` is ref_glm.json's tf_pred (the per-position oracle argmax). Pass + n_positions = len(oracle). + + Returns (agreement_fraction, list_of_differing_positions). + """ + cm = cpu.get("tf_mismatches") or {} + gm = cuda.get("tf_mismatches") or {} + diff = [] + for i, orc in enumerate(oracle): + cpu_tok = cm.get(i, orc) # matched oracle => oracle token + cuda_tok = gm.get(i, orc) + if cpu_tok != cuda_tok: + diff.append(i) + agree = (len(oracle) - len(diff)) / len(oracle) if oracle else 0.0 + return agree, diff diff --git a/c/tools/iq3xxs_grid.json b/c/tools/iq3xxs_grid.json new file mode 100644 index 0000000..3cde1a2 --- /dev/null +++ b/c/tools/iq3xxs_grid.json @@ -0,0 +1 @@ +[[4, 4, 4, 4], [20, 4, 4, 4], [36, 4, 4, 4], [12, 12, 4, 4], [28, 12, 4, 4], [62, 12, 4, 4], [4, 20, 4, 4], [20, 20, 4, 4], [12, 28, 4, 4], [20, 36, 4, 4], [28, 62, 4, 4], [44, 62, 4, 4], [12, 4, 12, 4], [28, 4, 12, 4], [4, 12, 12, 4], [20, 12, 12, 4], [12, 20, 12, 4], [44, 20, 12, 4], [4, 28, 12, 4], [20, 28, 12, 4], [12, 36, 12, 4], [36, 44, 12, 4], [4, 62, 12, 4], [4, 4, 20, 4], [20, 4, 20, 4], [36, 4, 20, 4], [12, 12, 20, 4], [4, 20, 20, 4], [20, 20, 20, 4], [12, 28, 20, 4], [28, 28, 20, 4], [62, 28, 20, 4], [12, 44, 20, 4], [62, 44, 20, 4], [44, 62, 20, 4], [12, 4, 28, 4], [62, 4, 28, 4], [4, 12, 28, 4], [20, 12, 28, 4], [44, 20, 28, 4], [4, 62, 28, 4], [28, 12, 36, 4], [62, 28, 36, 4], [36, 36, 36, 4], [62, 44, 36, 4], [28, 62, 36, 4], [44, 62, 36, 4], [12, 4, 44, 4], [62, 4, 44, 4], [20, 28, 44, 4], [20, 44, 44, 4], [44, 28, 52, 4], [36, 52, 52, 4], [4, 12, 62, 4], [36, 12, 62, 4], [52, 12, 62, 4], [28, 36, 62, 4], [12, 52, 62, 4], [12, 4, 4, 12], [28, 4, 4, 12], [4, 12, 4, 12], [20, 12, 4, 12], [12, 20, 4, 12], [28, 20, 4, 12], [4, 28, 4, 12], [20, 28, 4, 12], [36, 28, 4, 12], [62, 36, 4, 12], [4, 44, 4, 12], [4, 4, 12, 12], [20, 4, 12, 12], [12, 12, 12, 12], [4, 20, 12, 12], [20, 20, 12, 12], [12, 4, 20, 12], [28, 4, 20, 12], [4, 12, 20, 12], [20, 12, 20, 12], [12, 20, 20, 12], [4, 28, 20, 12], [20, 62, 20, 12], [4, 4, 28, 12], [20, 4, 28, 12], [4, 20, 28, 12], [12, 28, 28, 12], [52, 36, 28, 12], [52, 52, 28, 12], [12, 4, 36, 12], [44, 4, 36, 12], [4, 44, 36, 12], [4, 20, 44, 12], [36, 20, 44, 12], [52, 36, 44, 12], [12, 62, 44, 12], [44, 4, 52, 12], [20, 20, 62, 12], [4, 36, 62, 12], [4, 4, 4, 20], [20, 4, 4, 20], [12, 12, 4, 20], [28, 12, 4, 20], [4, 20, 4, 20], [20, 20, 4, 20], [52, 20, 4, 20], [12, 28, 4, 20], [20, 36, 4, 20], [12, 4, 12, 20], [28, 4, 12, 20], [44, 4, 12, 20], [4, 12, 12, 20], [20, 12, 12, 20], [12, 20, 12, 20], [4, 28, 12, 20], [28, 52, 12, 20], [62, 52, 12, 20], [4, 62, 12, 20], [4, 4, 20, 20], [20, 4, 20, 20], [12, 12, 20, 20], [62, 12, 20, 20], [4, 20, 20, 20], [20, 20, 20, 20], [62, 28, 20, 20], [4, 36, 20, 20], [44, 44, 20, 20], [12, 4, 28, 20], [4, 12, 28, 20], [36, 12, 28, 20], [4, 62, 28, 20], [36, 62, 28, 20], [44, 28, 36, 20], [28, 44, 36, 20], [28, 4, 44, 20], [62, 20, 44, 20], [12, 36, 44, 20], [36, 62, 44, 20], [12, 4, 62, 20], [28, 4, 62, 20], [52, 12, 62, 20], [44, 36, 62, 20], [12, 4, 4, 28], [4, 12, 4, 28], [20, 12, 4, 28], [12, 20, 4, 28], [28, 20, 4, 28], [4, 44, 4, 28], [44, 52, 4, 28], [20, 62, 4, 28], [4, 4, 12, 28], [20, 4, 12, 28], [4, 20, 12, 28], [12, 28, 12, 28], [36, 36, 12, 28], [52, 36, 12, 28], [12, 4, 20, 28], [28, 4, 20, 28], [4, 12, 20, 28], [44, 20, 20, 28], [20, 44, 20, 28], [20, 62, 20, 28], [12, 12, 28, 28], [28, 28, 28, 28], [4, 28, 36, 28], [62, 36, 36, 28], [20, 62, 36, 28], [4, 4, 44, 28], [52, 4, 44, 28], [20, 20, 44, 28], [44, 44, 44, 28], [36, 12, 52, 28], [52, 28, 52, 28], [28, 52, 52, 28], [28, 28, 62, 28], [4, 52, 62, 28], [36, 4, 4, 36], [62, 12, 4, 36], [44, 28, 4, 36], [62, 28, 4, 36], [28, 44, 4, 36], [62, 44, 4, 36], [36, 62, 12, 36], [4, 20, 20, 36], [62, 28, 20, 36], [4, 36, 20, 36], [4, 52, 20, 36], [52, 52, 20, 36], [62, 4, 28, 36], [44, 36, 28, 36], [36, 4, 36, 36], [12, 44, 36, 36], [36, 52, 36, 36], [44, 20, 44, 36], [28, 36, 44, 36], [4, 62, 44, 36], [44, 4, 62, 36], [4, 12, 62, 36], [20, 12, 62, 36], [4, 28, 62, 36], [20, 12, 4, 44], [12, 36, 4, 44], [4, 62, 4, 44], [4, 4, 12, 44], [52, 4, 12, 44], [52, 20, 12, 44], [44, 44, 12, 44], [36, 12, 20, 44], [20, 28, 20, 44], [20, 62, 20, 44], [20, 4, 28, 44], [28, 44, 28, 44], [4, 12, 36, 44], [28, 20, 36, 44], [62, 20, 36, 44], [20, 62, 36, 44], [20, 4, 44, 44], [12, 28, 44, 44], [4, 44, 52, 44], [36, 20, 62, 44], [20, 36, 62, 44], [36, 20, 4, 52], [36, 36, 4, 52], [52, 36, 4, 52], [36, 52, 4, 52], [12, 20, 12, 52], [12, 52, 12, 52], [62, 12, 20, 52], [36, 52, 20, 52], [4, 28, 28, 52], [52, 28, 28, 52], [36, 36, 36, 52], [44, 4, 44, 52], [20, 44, 44, 52], [28, 28, 52, 52], [28, 4, 62, 52], [12, 20, 62, 52], [28, 4, 4, 62], [44, 4, 4, 62], [62, 4, 4, 62], [4, 12, 4, 62], [20, 28, 4, 62], [20, 44, 4, 62], [52, 20, 12, 62], [4, 36, 12, 62], [20, 12, 20, 62], [44, 36, 20, 62], [20, 44, 20, 62], [4, 4, 28, 62], [44, 12, 28, 62], [28, 28, 28, 62], [4, 52, 28, 62], [12, 20, 36, 62], [12, 36, 36, 62], [4, 4, 44, 62], [20, 4, 44, 62], [36, 20, 44, 62], [4, 28, 52, 62]] \ No newline at end of file diff --git a/c/tools/make_olmoe_real_oracle.py b/c/tools/make_olmoe_real_oracle.py new file mode 100644 index 0000000..1e638e4 --- /dev/null +++ b/c/tools/make_olmoe_real_oracle.py @@ -0,0 +1,70 @@ +"""Generate reference token IDs for the real OLMoE-1B-7B model. + +Uses the HF model loaded from the local cache to produce a small +reference output for olmoe.exe validation. Saves to ref_olmoe_real.json. + +Usage: python tools/make_olmoe_real_oracle.py +""" +import json +import sys +from pathlib import Path + +if sys.platform == "win32": + for s in (sys.stdout, sys.stderr): + try: + s.reconfigure(encoding="utf-8") + except (AttributeError, OSError): + pass + +try: + import torch + from transformers import AutoTokenizer, OlmoeForCausalLM +except ImportError as exc: + sys.exit(f"Missing deps: {exc}. Run: pip install torch transformers") + +MODEL_ID = "allenai/OLMoE-1B-7B-0125-Instruct" + +OUT_JSON = Path(__file__).resolve().parent.parent / "ref_olmoe_real.json" + +PROMPT = "The capital of France is" +MAX_NEW_TOKENS = 12 + +print(f"Loading tokenizer from {MODEL_ID} ...") +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) + +print("Encoding prompt ...") +enc = tokenizer(PROMPT, return_tensors="pt") +prompt_ids = enc["input_ids"][0].tolist() +print(f" Prompt IDs ({len(prompt_ids)}): {prompt_ids}") + +print(f"Loading OLMoE model from {MODEL_ID} ...") +print(" (this will use ~14 GB RAM — please be patient)") +model = OlmoeForCausalLM.from_pretrained( + MODEL_ID, + torch_dtype=torch.bfloat16, + device_map="cpu", + low_cpu_mem_usage=True, +) +model.eval() +print(" Model loaded!") + +print(f"Generating {MAX_NEW_TOKENS} tokens ...") +with torch.no_grad(): + out = model.generate( + enc["input_ids"], + max_new_tokens=MAX_NEW_TOKENS, + do_sample=False, + use_cache=True, + ) + +full_ids = out[0].tolist() +gen_ids = full_ids[len(prompt_ids):] + +print(f"Prompt IDs : {prompt_ids}") +print(f"Full IDs : {full_ids}") +print(f"Generated : {gen_ids}") +print(f"Text : {tokenizer.decode(gen_ids, skip_special_tokens=True)!r}") + +payload = {"prompt_ids": prompt_ids, "full_ids": full_ids} +OUT_JSON.write_text(json.dumps(payload, indent=2)) +print(f"\nSaved reference to {OUT_JSON}") diff --git a/c/tools/quant_ablation.py b/c/tools/quant_ablation.py index 689643a..a602ae8 100644 --- a/c/tools/quant_ablation.py +++ b/c/tools/quant_ablation.py @@ -117,11 +117,79 @@ def quantize_param(w, bits, group, rot=False, e8=""): def _grid_or_e8(x, bits, group, e8): + if e8 == "-iq3": + return _quant_iq3(x.float()) if e8: - return _quant_e8(x.float(), group, ball=(e8 == "-e8")) + return _quant_e8(x.float(), group, bits, ball=(e8 == "-e8")) return _quant_last_dim(x, bits, group) +# -------------------------------------------------------------------------------------- +# IQ3_XXS-style codebook (#452 candidate (a)): llama.cpp's deployed 3.06-bpw scheme. +# 4-dim magnitude blocks quantized to a 256-entry lattice-subset grid (magnitudes on the +# odd ladder 4,12,..,62 in half-units), signs factored out per 8 weights with an odd-parity +# constraint (7 stored + 1 derived: a block whose true signs violate parity gets its +# smallest-magnitude sign flipped — modelled here so the ablation pays the real cost). +# Scales: fp16 super-scale per 256 + 4-bit sub-scale per 32, db = d*(0.5+s)*0.5. +# Grid extracted from ggml-common.h (MIT). +# -------------------------------------------------------------------------------------- +_IQ3_GRID = None +def _iq3_grid(device): + global _IQ3_GRID + if _IQ3_GRID is None or _IQ3_GRID.device != device: + import json, os + path = os.path.join(os.path.dirname(__file__), "iq3xxs_grid.json") + _IQ3_GRID = torch.tensor(json.load(open(path)), dtype=torch.float32, device=device) + return _IQ3_GRID # [256,4], half-unit magnitudes (value/2 = weight units) + +def _quant_iq3(x): + orig = x.shape + K = orig[-1] + assert K % 256 == 0, "iq3 needs multiples of 256 along the input dim" + xb = x.reshape(-1, 256) # super-blocks + grid = _iq3_grid(x.device) * 0.5 # weight units + out = torch.empty_like(xb) + signs = torch.sign(xb); signs[signs == 0] = 1.0 + mags = xb.abs() + for sb in range(8): # 8 sub-blocks of 32 + m = mags[:, sb*32:(sb+1)*32] # [N,32] + s = signs[:, sb*32:(sb+1)*32] + # per-8 sign parity: flip the smallest-|w| sign where the product is negative + s8 = s.reshape(-1, 4, 8) + m8 = m.reshape(-1, 4, 8) + viol = (s8.prod(-1) < 0) # odd number of minus signs + idxmin = m8.argmin(-1) + flip = torch.zeros_like(s8) + flip.scatter_(-1, idxmin[..., None], 1.0) + s8 = torch.where(viol[..., None].expand_as(s8) & (flip > 0), -s8, s8) + s = s8.reshape(-1, 32) + # sub-scale search: db candidates from the 4-bit code, super d from block RMS + d = m.pow(2).mean(-1, keepdim=True).sqrt() / 20.0 + 1e-12 # rough anchor + best = None + for code in range(16): + db = d * (0.5 + code) * 0.5 + q = m / db # [N,32] target magnitudes + q4 = q.reshape(-1, 4) # 4-dim grid blocks + # chunked argmin ||q-g||^2 = argmin(|g|^2 - 2 q.g): a full cdist on a + # 100M-param tensor materializes tens of GB — this stays at ~256 MB. + g2 = grid.pow(2).sum(-1) + idx = torch.empty(q4.shape[0], dtype=torch.long, device=q4.device) + CH = 1 << 18 + for i0 in range(0, q4.shape[0], CH): + cc = q4[i0:i0+CH] + idx[i0:i0+CH] = (g2 - 2.0 * (cc @ grid.T)).argmin(-1) + hit = grid[idx].reshape(-1, 8, 4) + rec = (hit.reshape(-1, 32) * db) + err = (rec - m).pow(2).sum(-1, keepdim=True) + if best is None: + best = (err, rec) + else: + take = err < best[0] + best = (torch.where(take, err, best[0]), torch.where(take, rec, best[1])) + out[:, sb*32:(sb+1)*32] = best[1] * s + return out.reshape(orig) + + def _rot_quant(x, bits, group, e8=""): """W -> Qn(W@Q) @ Q^T along the last (input) dim — see rotation() above.""" q = rotation(x.shape[-1], x.device) @@ -169,8 +237,15 @@ def _e8_ball(y, r2=10.0): return p -def _quant_e8(x, group, ball): - """Blocks of 8 along the input dim; per-group scale by MSE search over RMS multiples.""" +_E8_R2_REPORTED = set() +def _e8_radius(bits): + # E8 lattice: points within |p|^2<=r2 grow ~r2^4, so +1 bit (x256 codebook) needs r2 x4. + # Anchor: r2=10 is the ~2^16 E8P ball (2 bits over 8 dims). Scale from there. + return 10.0 * (4.0 ** (bits - 2)) + +def _quant_e8(x, group, bits, ball): + """Blocks of 8 along the input dim; per-group scale by MSE search over RMS multiples. + ball=True clamps to the rate-scaled E8 ball for `bits`; ball=False is the unbounded ideal.""" if x.shape[-1] % 8: raise SystemExit(f"-e8 needs input dim divisible by 8 (got {x.shape[-1]})") g = group or x.shape[-1] @@ -183,7 +258,11 @@ def _quant_e8(x, group, ball): for k in (0.5, 0.7, 0.9, 1.1, 1.4, 1.8, 2.4): s = rms * k yb = (xg / s).reshape(-1, g // 8, 8) - p = _e8_ball(yb) if ball else _e8_nearest(yb) + p = _e8_ball(yb, _e8_radius(bits)) if ball else _e8_nearest(yb) + if ball and bits not in _E8_R2_REPORTED: + _E8_R2_REPORTED.add(bits) + import sys as _sys + _sys.stderr.write(f"[e8] bits={bits}: ball r2={_e8_radius(bits):.1f}\n") out = (p.reshape(-1, g) * s) err = (out - xg).pow(2).sum(-1, keepdim=True) if best_err is None: @@ -195,7 +274,7 @@ def _quant_e8(x, group, ball): return best_out.reshape(shp) -SCHEME_RE = re.compile(r"^int(2|3|4|8)(?:-g(\d+))?(-e8u?)?(-rot)?(-nohead)?$") +SCHEME_RE = re.compile(r"^int(2|3|4|8)(?:-g(\d+))?(-e8u?|-iq3)?(-rot)?(-nohead)?$") def parse_scheme(name): diff --git a/c/tools/repair_mtp_int8.py b/c/tools/repair_mtp_int8.py new file mode 100644 index 0000000..695e99f --- /dev/null +++ b/c/tools/repair_mtp_int8.py @@ -0,0 +1,229 @@ +"""Repair an existing colibri int4 container whose MTP head was quantized at int4. + +WHY THIS EXISTS + `model.layers..eh_proj.weight` [D, 2D] multiplies the MTP concat + [embedding_norm ; hidden_norm], and its two column halves differ in scale by + ~20-30x per row (embedding-half absmax ~0.05, hidden-half ~1.5 on GLM-5.2). + Per-row int4 uses ONE scale (= absmax/7) per row, so every embedding-half + weight lands below half a quantization step and np.rint rounds the ENTIRE + embedding half to exact zeros (packed bytes 0x88). The MTP head then drafts + garbage: acceptance ~0% (issue #8 measured 0-4% at int4; 39-59% at int8 — + which is why `convert_fp8_to_int4.py --mtp` defaults to --ebits 8). + + A container converted (or downloaded) with an int4 MTP head does not need a + full re-conversion: this script re-downloads ONLY the affected dense tensors + (~355 MB of HTTP range reads against the FP8 source repo), requantizes them + at int8 with the converter's exact math, and patches the local shards in + place. Originals are kept beside as *.bak-int4. + +WHAT IT TOUCHES + The MTP layer's dense tensors only (eh_proj, q_a/q_b/kv_a/kv_b/o_proj, + shared_experts.*): the ones that stream into RAM once and stay resident. + Routed experts (model.layers..mlp.experts.*) are NOT touched — they are + statistically like the main layers' experts and int4 is acceptable there. + The engine auto-detects int8 vs int4 by blob size (qt_from_disk), so no + engine or config change is needed. Cost: ~+133 MB on disk / resident RAM. + +USAGE + python3 tools/repair_mtp_int8.py --snap /path/to/glm52_i4 # repair + python3 tools/repair_mtp_int8.py --snap /path/to/glm52_i4 --dry-run # inspect only + + --source-repo defaults to zai-org/GLM-5.2-FP8 (the checkpoint the public + int4 containers were converted from). Requires numpy and network access; + no torch, no HF token (public repo, anonymous range reads). +""" +import argparse, glob, json, os, ssl, struct, sys, urllib.request +import numpy as np + +# macOS python.org builds ship no CA bundle: use certifi when available (Linux +# system Pythons generally have working system certs and skip this). +try: + import certifi + _SSL_CTX = ssl.create_default_context(cafile=certifi.where()) +except ImportError: + _SSL_CTX = ssl.create_default_context() + + +# ---------- HTTP range reads against the source repo ---------- +def http_range(url, start, length, tries=5): + req = urllib.request.Request(url, headers={"User-Agent": "colibri-mtp-repair", + "Range": f"bytes={start}-{start+length-1}"}) + for attempt in range(tries): + try: + with urllib.request.urlopen(req, timeout=30, context=_SSL_CTX) as r: + data = r.read() + if len(data) == length: + return data + except KeyboardInterrupt: + raise + except Exception as ex: + if attempt == tries - 1: + raise RuntimeError(f"range read failed for {url}: {ex}") + raise RuntimeError(f"short range read for {url}") + + +class SourceRepo: + def __init__(self, repo, revision="main"): + self.base = f"https://huggingface.co/{repo}/resolve/{revision}/" + with urllib.request.urlopen(self.base + "model.safetensors.index.json", timeout=30, context=_SSL_CTX) as r: + self.wmap = json.loads(r.read())["weight_map"] + self._hdr = {} + + def _shard_header(self, shard): + if shard not in self._hdr: + n = struct.unpack("> 3) & 0xF + mant = (b & 7).astype(np.float64) + v = (sign * np.where(e > 0, (1 + mant / 8) * np.exp2(e.astype(np.float64) - 7), + mant / 8 * np.exp2(-6.0))).reshape(m["shape"]) + sn = name + "_scale_inv" + sshard = self.wmap[sn] + shdr, sbase = self._shard_header(sshard) + sm = shdr[sn] + so0, so1 = sm["data_offsets"] + sc = np.frombuffer(http_range(self.base + sshard, sbase + so0, so1 - so0), + dtype=np.float32).reshape(sm["shape"]) + O, I = m["shape"] + scf = np.repeat(np.repeat(sc, 128, axis=0)[:O], 128, axis=1)[:, :I] + return (v * scf).astype(np.float32) + raise ValueError(f"{name}: unsupported source dtype {m['dtype']}") + + +# ---------- quantization: identical to convert_fp8_to_int4.quant_int8 ---------- +def quant_int8(w): + amax = np.abs(w).max(axis=1, keepdims=True) + s = np.maximum(amax / 127, 1e-8) + q = np.clip(np.rint(w / s), -128, 127).astype(np.int8) + return q.reshape(-1).view(np.uint8).copy(), s[:, 0].astype(np.float32) + + +# ---------- local safetensors IO (no deps; preserves byte-identity of untouched tensors) ---------- +def read_shard(path): + with open(path, "rb") as fh: + n = struct.unpack(" [tensor names to repair] + already_ok, skipped_experts = [], 0 + for f in sorted(glob.glob(os.path.join(a.snap, "*.safetensors"))): + with open(f, "rb") as fh: + n = struct.unpack(" ..." line +m = re.search(r"C engine\s*:\s*([\d ]+)", r1.stdout) +if not m: + sys.exit("Could not parse 'C engine :' line from output") +gen_ids = [int(x) for x in m.group(1).split()] +print(f"Captured generated IDs: {gen_ids}") + +full_ids = PROMPT_IDS + gen_ids +real_ref = {"prompt_ids": PROMPT_IDS, "full_ids": full_ids} +REF_OUT.write_text(json.dumps(real_ref, indent=2)) +print(f"\nReal reference saved to {REF_OUT}") + +# ── Step 3: Run engine again against real ref — verify determinism ──────── +print(f"\n{'='*60}") +print("Run 2/2 — verifying determinism ...") +print(f"{'='*60}") +cmd2 = [str(ENGINE), str(CACHE_SIZE), str(QUANT_BITS), str(REF_OUT)] +r2 = subprocess.run(cmd2, env=env, capture_output=True, text=True, cwd=str(HERE)) +print(r2.stdout) +if r2.returncode != 0: + print("STDERR:", r2.stderr, file=sys.stderr) + sys.exit(r2.returncode) + +if "Matching tokens: 12/12" in r2.stdout or f"Matching tokens: {MAX_NEW}/{MAX_NEW}" in r2.stdout: + print("✓ Engine is DETERMINISTIC — same output on both runs!") +else: + m2 = re.search(r"Matching tokens: (\d+)/(\d+)", r2.stdout) + if m2: + print(f"⚠ Partial match: {m2.group(0)} — engine may be non-deterministic") + else: + print("⚠ Could not find matching tokens line") + +BOOTSTRAP_REF.unlink(missing_ok=True) diff --git a/c/version.py b/c/version.py new file mode 100644 index 0000000..9951ca1 --- /dev/null +++ b/c/version.py @@ -0,0 +1,3 @@ +"""Single source of truth for the colibrì version number.""" + +__version__ = "1.0.0" diff --git a/colibri/__init__.py b/colibri/__init__.py new file mode 100644 index 0000000..77f7061 --- /dev/null +++ b/colibri/__init__.py @@ -0,0 +1,5 @@ +"""colibrì — tiny engine, immense model.""" + +from colibri._version import __version__ + +__all__ = ["__version__"] diff --git a/colibri/_version.py b/colibri/_version.py new file mode 100644 index 0000000..174442a --- /dev/null +++ b/colibri/_version.py @@ -0,0 +1,23 @@ +"""Version accessor for the pip package. + +The single source of truth is c/version.py (#394): coli --version and the +GitHub Release workflow read it, so the pip metadata must read the SAME file +instead of carrying a second literal that would drift on the first bump. + +From a checkout (the supported install: `pip install -e .`) the file is read +directly. From an installed wheel c/ is not on disk, so fall back to the +package metadata that setuptools baked at build time from that same file. +""" +from pathlib import Path + +try: + _ns = {} + exec((Path(__file__).resolve().parent.parent / "c" / "version.py").read_text(), _ns) + __version__ = _ns["__version__"] +except OSError: + from importlib.metadata import PackageNotFoundError, version + + try: + __version__ = version("colibri-engine") + except PackageNotFoundError: + __version__ = "0.0.0+unknown" diff --git a/colibri/cli.py b/colibri/cli.py new file mode 100644 index 0000000..3f19234 --- /dev/null +++ b/colibri/cli.py @@ -0,0 +1,30 @@ +"""Entry point for `coli` when installed via pip. + +Delegates to the original c/coli script which handles all subcommands. +This wrapper exists so `pip install colibri-engine` creates a `coli` console +script that works without the user having to add c/ to PATH manually. +""" + +import os +import sys +import runpy + + +def main(): + here = os.path.dirname(os.path.abspath(__file__)) + engine_dir = os.path.join(os.path.dirname(here), "c") + coli_script = os.path.join(engine_dir, "coli") + + if not os.path.exists(coli_script): + sys.exit( + "colibri engine directory not found.\n" + "Install from source: git clone + pip install -e ." + ) + + sys.path.insert(0, engine_dir) + sys.argv[0] = coli_script + runpy.run_path(coli_script, run_name="__main__") + + +if __name__ == "__main__": + main() diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..fa639f9 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,15 @@ +FROM debian:stable-slim + +# We install the necessary packages to download and compile the code +RUN apt-get update && \ + apt-get install -y locales git build-essential python3 python3-pip && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir /app + +# Let's clone the repository +WORKDIR /app +RUN git clone https://github.com/JustVugg/colibri.git . +WORKDIR /app/c +# It's time to compile +RUN ./setup.sh && ./coli build + diff --git a/docker/README.IT.md b/docker/README.IT.md new file mode 100644 index 0000000..43c3e31 --- /dev/null +++ b/docker/README.IT.md @@ -0,0 +1,454 @@ +_Read the read me in [English](README.md)._ + + +# Guida a Colibrì - Motore di Inferenza Locale + +Una guida semplice per eseguire **Colibrì**, un motore di inferenza locale basato su GLM 5.2, senza conoscenze di programmazione. Se hai già Docker installato, sei a buon punto. + +--- + +## 📋 Sommario + +- [Cosa è Colibrì?](#cosa-è-colibrì) +- [Cosa serve](#cosa-serve) + - [Hardware](#hardware) + - [Software](#software) +- [Come iniziare](#come-iniziare) + - [Passo 1: Scarica il modello](#passo-1-scarica-il-modello) + - [Passo 2: Scarica il Dockerfile di Colibrì](#passo-2-scarica-il-dockerfile-di-colibrì) + - [Passo 3: Compila l'immagine Docker](#passo-3-compila-limmagine-docker) + - [Passo 4: Avvia Colibrì](#passo-4-avvia-colibr%C3%AC) + - [Cosa significa quel comando?](#cosa-significa-quel-comando) + - [Usare Colibrì](#usare-colibrì) +- [Entrare in una console Linux dentro il container](#entrare-in-una-console-linux-dentro-il-container) +- [Risoluzione dei problemi](#risoluzione-dei-problemi) +- [Note tecniche](#note-tecniche) +- [Domande frequenti](#domande-frequenti) +- [Supporto e contributi](#supporto-e-contributi) +- [Test sul mio PC](#test-sul-mio-pc) + +--- + +## Cosa è Colibrì? + +Colibrì è un'applicazione che ti permette di eseguire un modello di intelligenza artificiale (GLM 5.2) direttamente sul tuo computer, senza connettersi a server esterni. È possbile anche farlo girare in Docker, che isola l'applicazione dal resto del sistema. + +> **Nota importante**: Il modello è molto grande. Attendi anche diversi minuti per una risposta a una domanda semplice, specialmente con poca RAM. Alla fine di questo readme vedrai il risultato sul mio PC (senza scheda grafica discreta) e arrivo a 0.01 token al secondo. + +--- + +## Cosa serve + +### Hardware + +| Memoria RAM | Funziona? | Note | +|:---:|:---:|---| +| < 16 GB | ❌ No | Memoria insufficiente | +| 24 GB | ⚠️ Forse | Possibile, da testare | +| 32 GB | ✅ Sì | Il minimo (ma vedi sezione memoria su Windows) | +| 48+ GB | ✅ Sì | Meglio | + +Inoltre: un **disco SSD veloce** è essenziale. Colibrì usa il disco come memoria aggiuntiva. Con una scheda grafica NVidia è ancora meglio. + +### Software + +- **Docker Desktop** (Windows, Mac, Linux) — [scarica qui](https://www.docker.com/products/docker-desktop/) +- **Python** (solo se vuoi scaricare il modello da casa tua) + - Windows: [python.org](https://www.python.org) oppure Microsoft Store + - Linux: `apt-get install python3 python3-pip` + - Mac: [python.org](https://www.python.org) oppure Homebrew + +Non serve nessun ambiente di compilazione. Tutto avviene dentro il container Docker. + +--- + +## Come iniziare + +### Passo 1: Scarica il modello + +Il modello GLM 5.2 è circa **360 GB**. Scegli uno di questi metodi: + +#### Metodo A: Con Python (consigliato) + +1. **Installa la libreria per Hugging Face:** + ```bash + python -m pip install -U huggingface_hub[cli] + ``` + Su Linux, usa `python3` al posto di `python`. + +2. **Scarica il modello** (apri il terminale nella cartella dove lo vuoi salvare): + ```bash + hf_download mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp --local-dir . + ``` + + **Esempio**: se vuoi salvarlo in `C:\LLM\models\glm-5.2` (Windows): + - Apri PowerShell in quella cartella + - Copia e incolla il comando sopra + - Attendi (molto) + +#### Metodo B: Senza Python (solo se necessario) + +Se sei su Windows e non riesci con Python: +- Scarica manualmente da [Hugging Face](https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp) +- Decomprimi in una cartella (es. `C:\LLM\models\glm-5.2`) + +--- + +### Passo 2: Scarica il Dockerfile di Colibrì + +1. Vai a: https://github.com/JustVugg/colibri/blob/main/docker/Dockerfile +2. Clicca il pulsante **Download** (icona ⬇️) in alto a destra +3. Salva il file in una cartella (es. `C:\LLM\Colibrì`) + +--- + +### Passo 3: Compila l'immagine Docker + +Apri il terminale (PowerShell su Windows, Terminal su Mac/Linux) **nella cartella dove hai salvato il Dockerfile** e digita: + +**Windows:** +```bash +docker build -t colibri-i . +``` + +**Linux/Mac:** +```bash +sudo docker build -t colibri-i . +``` + +Attendi che finisca (pochi minuti). Se tutto va bene, vedrai: `Successfully tagged colibri-i:latest` + +> **Se vuoi recepire gli aggiornamenti del repository**: Cancella prima l'immagine vecchia con `docker rmi colibri-i` e ricompila. + +--- + +### Passo 4: Avvia Colibrì + +Apri il terminale e digita il comando sottostante (sostituisci `C:\LLM\models\glm-5.2` con il percorso reale del tuo PC): + +**Windows** (PowerShell): +```bash +$MODEL_PATH="C:\LLM\models\glm-5.2" +docker run --rm -it --name colibri-c ` + -v "$MODEL_PATH`:/app/glm-5.2" ` + -e COLI_MODEL=/app/glm-5.2 ` + colibri-i ./coli chat +``` + +**Mac/Linux** (Terminal/Bash): +```bash +MODEL_PATH="/path/to/glm-5.2" +docker run --rm -it --name colibri-c \ + -v "$MODEL_PATH:/app/glm-5.2" \ + -e COLI_MODEL=/app/glm-5.2 \ + colibri-i ./coli chat +``` + +**Esempio per Linux:** +```bash +MODEL_PATH="/home/user/LLM/glm-5.2" +docker run --rm -it --name colibri-c \ + -v "$MODEL_PATH:/app/glm-5.2" \ + -e COLI_MODEL=/app/glm-5.2 \ + colibri-i ./coli chat +``` + +--- + +### Cosa significa quel comando? + +| Parte | Spiegazione | +|-------|-------------| +| `docker run` | Avvia un container | +| `--rm` | Cancella il container quando chiudi | +| `-it` | Modalità interattiva (puoi scrivere e leggere) | +| `-v "PERCORSO:/app/glm-5.2"` | Collega il tuo modello dentro il container | +| `-e COLI_MODEL=/app/glm-5.2` | Dice a Colibrì dove trovare il modello | +| `colibri-i` | Nome dell'immagine Docker | +| `./coli chat` | Avvia Colibrì in modalità chat | + +--- + +### Usare Colibrì + +Una volta avviato, vedrai un prompt come questo: + +``` + ────────────────────────────────────────────────────────── + type and press Enter · Ctrl-C stops the answer · :more continues · :reset clears memory · :q exits +``` + +**Comandi utili:** +- `Scrivi una domanda + Invio` → Ricevi la risposta +- `Ctrl + C` → Interrompi la risposta +- `:reset` → Cancella la memoria della conversazione +- `:q` → Esci + +**Esempio di uso:** +``` +› Quanti abitanti ha la Cina? + +La Cina è attualmente il paese più popoloso al mondo. +La popolazione è di circa 1,41 miliardi di abitanti. +``` + +Il modello capisce **italiano, inglese, cinese e altre lingue**, anche se è ottimizzato per inglese e cinese. + +--- + +## Entrare in una console Linux dentro il container + +Se vuoi esplorare il container come fosse una macchina Linux normale: + +```bash +docker run --rm -it --name colibri-c \ + -v "PERCORSO_MODELLO:/app/glm-5.2" \ + -e COLI_MODEL=/app/glm-5.2 \ + colibri-i /bin/bash +``` + +Ora sei dentro Linux. Digita `exit` per uscire. + +--- + +## Risoluzione dei problemi + +### ❌ "Docker non trovato" + +**Causa**: Docker non è installato o il terminale non lo riconosce. + +**Soluzione**: +1. Reinstalla [Docker Desktop](https://www.docker.com/products/docker-desktop/) +2. Riavvia il computer +3. Apri un nuovo terminale e riprova + +--- + +### ❌ "Out of memory" (memoria insufficiente) o container che si chiude subito + +**Causa**: Il tuo computer non ha abbastanza RAM, oppure su Windows, WSL usa meno memoria di quella disponibile. + +**Soluzione per Windows (WSL):** + +1. Apri PowerShell e controlla la memoria disponibile a WSL: + ```bash + wsl + cat /proc/meminfo | grep MemTotal + exit + ``` + Dividi il numero per 1.073.741.824 (è 1024³) per averlo in GB. + +2. Se WSL usa meno di quello che hai, crea un file di configurazione: + - Apri un editor di testo (Notepad va bene) + - Copia questo: + ```ini + [wsl2] + memory=24GB + processors=12 + swap=16GB + ``` + - Salva il file con il nome: `.wslconfig` (con il punto) + - Posizionalo in: `C:\Users\TuoNomeUtente\` + +3. Riavvia WSL da PowerShell: + ```bash + wsl --shutdown + wsl + ``` + +4. Controlla di nuovo: + ```bash + # cat /proc/meminfo | grep MemTotal + # exit + ``` + +**Soluzione per Mac/Linux**: Aumenta la RAM disponibile a Docker dalle impostazioni di Docker Desktop, oppure aggiungi più RAM al computer. + +--- + +### ❌ La risposta è molto lenta + +**Cause possibili**: +1. Il disco è lento +2. Hai poca RAM +3. Colibrì sta usando il disco come memoria aggiuntiva (normale) + +**Come controllare la velocità del disco:** + +**Windows** (PowerShell da amministratore): +```bash +winsat disk -drive C +``` +Cambia `C` con la lettera del tuo disco. + +**Linux/Mac** (Terminal): +```bash +sudo hdparm -Tt /dev/sda +``` +Cambia `/dev/sda` con il tuo disco (vedi con `lsblk` per Linux). + +Un **SSD NVMe moderno** arriva a 15 GB/sec. Se il tuo è sotto 2-3 GB/sec, è lento. + +--- + +### ❌ "Permission denied" su Linux + +**Causa**: Docker richiede permessi da amministratore. + +**Soluzione - Opzione 1** (rapida): +```bash +sudo docker build -t colibri-i . +sudo docker run ... (come sopra, con sudo davanti) +``` + +**Soluzione - Opzione 2** (permanente): +```bash +sudo usermod -aG docker $USER +# Riavvia il computer +docker run ... (senza sudo) +``` + +--- + +### ❌ "Image not found" o errore durante il build + +**Causa**: Il Dockerfile è corrotto o non nella cartella giusta. + +**Soluzione**: +1. Verifica che il Dockerfile sia nella cartella dove apri il terminale: + ```bash + ls Dockerfile # Mac/Linux + dir Dockerfile # Windows + ``` +2. Riscarica il Dockerfile dal repository GitHub +3. Elimina l'immagine vecchia: `docker rmi colibri-i` +4. Riprova il build + +--- + +### ❌ "hf_download: command not found" + +**Causa**: La libreria Hugging Face non è installata correttamente. + +**Soluzione**: +```bash +pip install -U huggingface_hub[cli] +# oppure su Linux/Mac: +pip3 install -U huggingface_hub[cli] +``` + +Poi riprova il comando `hf_download`. + +--- + +### ❌ Il modello non si scarica (timeout o errori di rete) + +**Cause**: Connessione lenta o instabile, Hugging Face temporaneamente non disponibile. + +**Soluzione**: +1. Attendi e riprova il comando `hf_download` +2. Se continua, scarica manualmente da [qui](https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp) +3. Decomprimi il file ZIP nella cartella desiderata + +--- + +## Note tecniche + +### Perché il disco è importante? + +Colibrì usa il disco come "RAM aggiuntiva" virtuale (paging). Un disco **veloce** è cruciale per prestazioni decenti. + +- **SSD NVMe** (consigliato): 1-15 GB/sec +- **SSD SATA**: 0.5-1 GB/sec +- **Hard disk meccanico**: 0.05-0.1 GB/sec ❌ (troppo lento) + +Se il tuo disco è lento, le risposte saranno molto lente anche con molta RAM. + +--- + +### Configurazione di default consigliata per WLS su Windows + +Se hai **esattamente 32 GB di RAM** e usi Windows, è molto probabile che WLS di default sia settato per non consumare più di 16 GB di RAM. Bisogna aumentare questo limite [Risoluzione dei problemi](#risoluzione-dei-problemi) . Nel mio caso ho adottato questa configurazione: +```ini +[wsl2] +memory=24GB +processors=12 +swap=16GB +``` + +Ovvero, nel mio caso, ho lasciato 8 GB di RAM e 4 CPU a Windows e dato 24 GB e 12 processori a WSL + Linux. + +--- + +## Domande frequenti + +**D: E se ho meno di 32 GB di RAM?** +R: Probabile che non funzioni bene. Puoi provare se hai 24 GB, ma non è garantito. + +**D: Posso aumentare la velocità di risposta?** +R: Sì, in parte: +- Usa un SSD NVMe veloce +- Aumenta la RAM +- Riduci la complessità delle domande +- Usa `:reset` per cancellare la memoria e alleggerire il carico + +**D: Posso usare Colibrì senza Docker?** +R: Colibrì è nato così, ma questa guida assume Docker. Per compilare da sorgente, vedi il repository GitHub. + +**D: Quanta connessione internet mi serve dopo aver scaricato il modello?** +R: Zero. Colibrì funziona completamente offline. + +--- + +## Supporto e contributi + +Se trovi errori o hai suggerimenti per migliorare questa guida, aprici una issue o una pull request sul repository GitHub di Colibrì. + +Buon divertimento! 🐦 + +--- + +## Test sul mio PC + +Nel primo caso ho fatto una domanda in italiano, nel secondo in giapponese, e nel terzo ho rifatto la domanda in giapponese ma ho richiesto una risposta in italiano. + +``` +PS C:\quack\llm\colibri\docker> docker run --rm -it --name colibri-c -v "C:\quack\llm\models\glm-5.2:/app/glm-5.2" -e COLI_MODEL=/app/glm-5.2 colibri-i ./coli chat + + ▄▀▀▀▄ ▄ colibrì v1.0 + ▄▄▄▄▀▀▀▀▄▀▀ tiny engine, immense model + ▀▀▀▀▀▀▀ GLM-5.2 · 744B MoE · int4 · streaming CPU + ▀▀▀▀ chat · glm-5.2 · ram -GB · topp off + ▀ + ────────────────────────────────────────────────────────── + type and press Enter · Ctrl-C stops the answer · :more continues · :reset clears memory · :q exits + + ╭────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ › Quanti abitanti ha la Cina? │ + ╰────────────────────────────────────────────────────────────────────────────────────────────────╯ + + ◆ colibrì + La Cina è attualmente il paese più popoloso al mondo (sebbene, secondo alcune stime recenti, sia stata ormai superata dall'India). + + La popolazione totale della Repubblica Popolare Cinese è di circa 1,41 miliardi di abitanti (dati del 2020-2022 circa). + └─ 76 tok · 0.04 tok/s · hit 3% · RSS 15.9 GB · 2012s + + ╭────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ › 漫画「ワンピース」の主人公の名前を教えてください。名前だけで、それ以上のコメントはありません。 │ + ╰────────────────────────────────────────────────────────────────────────────────────────────────╯ + + ◆ colibrì + ルフィ + └─ 2 tok · 0.01 tok/s · hit 1% · RSS 16.7 GB · 260s + + ╭────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ › 漫画「ワンピース」の主人公の名前を教えてください。名前だけで、それ以上のコメントはありません。イタリア語で返信 │ + ╰────────────────────────────────────────────────────────────────────────────────────────────────╯ + + ◆ colibrì + Il nome del protagonista di One Piece è Monkey D. Luffy. + └─ 14 tok · 0.02 tok/s · hit 2% · RSS 17.3 GB · 593s + + ╭────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ › + ``` \ No newline at end of file diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..44d8a19 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,488 @@ +_Leggi il leggimi in [Italiano](README.IT.md)._ + + +# Colibrì Guide - Local Inference Engine + +A simple guide to running **Colibrì**, a local inference engine based on GLM 5.2, without needing programming knowledge. If you already have Docker installed, you are well on your way. + +--- + +## 📋 Table of Contents + +* [What is Colibrì?](https://www.google.com/search?q=#what-is-colibr%C3%AC) +* [Requirements](https://www.google.com/search?q=#requirements) + * [Hardware](https://www.google.com/search?q=#hardware) + * [Software](https://www.google.com/search?q=#software) +* [How to get started](https://www.google.com/search?q=#how-to-get-started) + * [Step 1: Download the model](https://www.google.com/search?q=#step-1-download-the-model) + * [Step 2: Download the Colibrì Dockerfile](https://www.google.com/search?q=#step-2-download-the-colibr%C3%AC-dockerfile) + * [Step 3: Build the Docker image](https://www.google.com/search?q=#step-3-build-the-docker-image) + * [Step 4: Start Colibrì](https://www.google.com/search?q=#step-4-start-colibr%C3%AC) + * [What does that command mean?](https://www.google.com/search?q=#what-does-that-command-mean) + * [Using Colibrì](https://www.google.com/search?q=#using-colibr%C3%AC) +* [Entering a Linux console inside the container](https://www.google.com/search?q=#entering-a-linux-console-inside-the-container) +* [Troubleshooting](https://www.google.com/search?q=#troubleshooting) +* [Technical notes](https://www.google.com/search?q=#technical-notes) +* [Frequently Asked Questions](https://www.google.com/search?q=#frequently-asked-questions) +* [Support and contributions](https://www.google.com/search?q=#support-and-contributions) +* [Tests on my PC](https://www.google.com/search?q=#tests-on-my-pc) + +--- + +## What is Colibrì? + +Colibrì is an application that allows you to run an artificial intelligence model (GLM 5.2) directly on your computer, without connecting to external servers. It is also possible to run it in Docker, which isolates the application from the rest of the system. + +> **Important note**: The model is very large. Expect to wait several minutes for an answer to a simple question, especially with low RAM. At the end of this readme, you will see the result on my PC (without a discrete graphics card), and I reach 0.01 tokens per second. + +--- + +## Requirements + +### Hardware + +| RAM Memory | Works? | Notes | +| --- | --- | --- | +| < 16 GB | ❌ No | Insufficient memory | +| 24 GB | ⚠️ Maybe | Possible, needs testing | +| 32 GB | ✅ Yes | The minimum (but see memory section for Windows) | +| 48+ GB | ✅ Yes | Better | + +Additionally: a **fast SSD** is essential. Colibrì uses the disk as additional memory. Having an NVidia graphics card is even better. + +### Software + +* **Docker Desktop** (Windows, Mac, Linux) — [download here](https://www.google.com/search?q=https://www.docker.com/products/docker-desktop/) +* **Python** (only if you want to download the model yourself) +* Windows: [python.org](https://www.google.com/search?q=https://www.python.org) or Microsoft Store +* Linux: `apt-get install python3 python3-pip` +* Mac: [python.org](https://www.google.com/search?q=https://www.python.org) or Homebrew + +No build environment is needed. Everything happens inside the Docker container. + +--- + +## How to get started + +### Step 1: Download the model + +The GLM 5.2 model is approximately **360 GB**. Choose one of these methods: + +#### Method A: Using Python (recommended) + +1. **Install the library for Hugging Face:** +```bash +python -m pip install -U huggingface_hub[cli] +``` + +On Linux, use `python3` instead of `python`. +2. **Download the model** (open the terminal in the folder where you want to save it): +```bash +hf_download mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp --local-dir . +``` + +**Example**: if you want to save it in `C:\LLM\models\glm-5.2` (Windows): +* Open PowerShell in that folder +* Copy and paste the command above +* Wait (a long time) + +#### Method B: Without Python (only if necessary) + +If you are on Windows and cannot get it to work with Python: + +* Download manually from [Hugging Face](https://www.google.com/search?q=https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp) +* Unzip into a folder (e.g., `C:\LLM\models\glm-5.2`) + +--- + +### Step 2: Download the Colibrì Dockerfile + +1. Go to: [https://github.com/JustVugg/colibri/blob/main/docker/Dockerfile](https://www.google.com/search?q=https://github.com/JustVugg/colibri/blob/main/docker/Dockerfile) +2. Click the **Download** button (⬇️ icon) in the top right +3. Save the file in a folder (e.g., `C:\LLM\Colibrì`) + +--- + +### Step 3: Build the Docker image + +Open the terminal (PowerShell on Windows, Terminal on Mac/Linux) **in the folder where you saved the Dockerfile** and type: + +**Windows:** + +```bash +docker build -t colibri-i . +``` + +**Linux/Mac:** + +```bash +sudo docker build -t colibri-i . +``` + +Wait for it to finish (a few minutes). If everything goes well, you will see: `Successfully tagged colibri-i:latest` + +> **If you want to receive repository updates**: First delete the old image with `docker rmi colibri-i` and rebuild. + +--- + +### Step 4: Start Colibrì + +Open the terminal and type the command below (replace `C:\LLM\models\glm-5.2` with the actual path on your PC): + +**Windows** (PowerShell): + +```bash +$MODEL_PATH="C:\LLM\models\glm-5.2" +docker run --rm -it --name colibri-c ` + -v "$MODEL_PATH`:/app/glm-5.2" ` + -e COLI_MODEL=/app/glm-5.2 ` + colibri-i ./coli chat +``` + +**Mac/Linux** (Terminal/Bash): + +```bash +MODEL_PATH="/path/to/glm-5.2" +docker run --rm -it --name colibri-c \ + -v "$MODEL_PATH:/app/glm-5.2" \ + -e COLI_MODEL=/app/glm-5.2 \ + colibri-i ./coli chat +``` + +**Example for Linux:** + +```bash +MODEL_PATH="/home/user/LLM/glm-5.2" +docker run --rm -it --name colibri-c \ + -v "$MODEL_PATH:/app/glm-5.2" \ + -e COLI_MODEL=/app/glm-5.2 \ + colibri-i ./coli chat +``` + +--- + +### What does that command mean? + +| Part | Explanation | +| --- | --- | +| `docker run` | Starts a container | +| `--rm` | Deletes the container when you close it | +| `-it` | Interactive mode (you can write and read) | +| `-v "PATH:/app/glm-5.2"` | Mounts your model inside the container | +| `-e COLI_MODEL=/app/glm-5.2` | Tells Colibrì where to find the model | +| `colibri-i` | Name of the Docker image | +| `./coli chat` | Starts Colibrì in chat mode | + +--- + +### Using Colibrì + +Once started, you will see a prompt like this: + +``` + ────────────────────────────────────────────────────────── + type and press Enter · Ctrl-C stops the answer · :more continues · :reset clears memory · :q exits + +``` + +**Useful commands:** + +* `Write a question + Enter` → Receive the answer +* `Ctrl + C` → Stop the answer +* `:reset` → Clear conversation memory +* `:q` → Exit + +**Usage example:** + +``` +› How many inhabitants does China have? + +China is currently the most populous country in the world. +The population is approximately 1.41 billion people. + +``` + +The model understands **Italian, English, Chinese, and other languages**, although it is optimized for English and Chinese. + +--- + +## Entering a Linux console inside the container + +If you want to explore the container as if it were a normal Linux machine: + +```bash +docker run --rm -it --name colibri-c \ + -v "MODEL_PATH:/app/glm-5.2" \ + -e COLI_MODEL=/app/glm-5.2 \ + colibri-i /bin/bash +``` + +Now you are inside Linux. Type `exit` to leave. + +--- + +## Troubleshooting + +### ❌ "Docker not found" + +**Cause**: Docker is not installed or the terminal does not recognize it. + +**Solution**: + +1. Reinstall [Docker Desktop](https://www.google.com/search?q=https://www.docker.com/products/docker-desktop/) +2. Restart your computer +3. Open a new terminal and try again + +--- + +### ❌ "Out of memory" or container closes immediately + +**Cause**: Your computer does not have enough RAM, or on Windows, WSL is using less memory than available. + +**Solution for Windows (WSL):** + +1. Open PowerShell and check the memory available to WSL: +```bash +wsl +cat /proc/meminfo | grep MemTotal +exit +``` + + +Divide the number by 1,073,741,824 (which is 1024³) to get it in GB. +2. If WSL uses less than what you have, create a configuration file: +* Open a text editor (Notepad is fine) +* Copy this: +```ini +[wsl2] +memory=24GB +processors=12 +swap=16GB + +``` + +* Save the file with the name: `.wslconfig` (with the dot) +* Place it in: `C:\Users\YourUsername\` + +3. Restart WSL from PowerShell: +```bash +wsl --shutdown +wsl +``` + +4. Check again: +```bash +# cat /proc/meminfo | grep MemTotal +# exit +``` + +**Solution for Mac/Linux**: Increase the RAM available to Docker from the Docker Desktop settings, or add more RAM to the computer. + +--- + +### ❌ The answer is very slow + +**Possible causes**: + +1. The disk is slow +2. You have low RAM +3. Colibrì is using the disk as additional memory (normal) + +**How to check disk speed:** + +**Windows** (PowerShell as administrator): + +```bash +winsat disk -drive C +``` + +Change `C` with your disk letter. + +**Linux/Mac** (Terminal): + +```bash +sudo hdparm -Tt /dev/sda +``` + +Change `/dev/sda` with your disk (check with `lsblk` on Linux). + +A **modern NVMe SSD** reaches 15 GB/sec. If yours is under 2-3 GB/sec, it is slow. + +--- + +### ❌ "Permission denied" on Linux + +**Cause**: Docker requires administrator permissions. + +**Solution - Option 1** (quick): + +```bash +sudo docker build -t colibri-i . +sudo docker run ... (as above, with sudo in front) +``` + +**Solution - Option 2** (permanent): + +```bash +sudo usermod -aG docker $USER +# Restart the computer +docker run ... (without sudo) +``` +--- + +### ❌ "Image not found" or error during build + +**Cause**: The Dockerfile is corrupted or not in the right folder. + +**Solution**: + +1. Verify that the Dockerfile is in the folder where you open the terminal: +```bash +ls Dockerfile # Mac/Linux +dir Dockerfile # Windows +``` + +2. Redownload the Dockerfile from the GitHub repository +3. Delete the old image: `docker rmi colibri-i` +4. Retry the build + +--- + +### ❌ "hf_download: command not found" + +**Cause**: The Hugging Face library is not installed correctly. + +**Solution**: + +```bash +pip install -U huggingface_hub[cli] +# or on Linux/Mac: +pip3 install -U huggingface_hub[cli] +``` + +Then retry the `hf_download` command. + +--- + +### ❌ The model does not download (timeout or network errors) + +**Causes**: Slow or unstable connection, Hugging Face temporarily unavailable. + +**Solution**: + +1. Wait and retry the `hf_download` command +2. If it continues, download manually from [here](https://www.google.com/search?q=https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp) +3. Unzip the ZIP file into the desired folder + +--- + +## Technical notes + +### Why is the disk important? + +Colibrì uses the disk as "additional virtual RAM" (paging). A **fast** disk is crucial for decent performance. + +* **NVMe SSD** (recommended): 1-15 GB/sec +* **SATA SSD**: 0.5-1 GB/sec +* **Rotational hard disk**: 0.05-0.1 GB/sec ❌ (too slow) + +If your disk is slow, the answers will be very slow even with a lot of RAM. + +--- + +### Recommended default configuration for WLS on Windows + +If you have **exactly 32 GB of RAM** and are using Windows, it is very likely that WLS by default is set to consume no more than 16 GB of RAM. We need to increase this limit [Troubleshooting](https://www.google.com/search?q=#troubleshooting) . In my case I adopted this configuration: + +```ini +[wsl2] +memory=24GB +processors=12 +swap=16GB + +``` + +That is, in my case, I left 8 GB of RAM and 4 CPUs to Windows and gave 24 GB and 12 processors to WSL + Linux. + +--- + +## Frequently Asked Questions + +**Q: What if I have less than 32 GB of RAM?** + +A: It is likely that it will not work well. You can try if you have 24 GB, but it is not guaranteed. + +**Q: Can I increase the response speed?** + +A: Yes, partly: + +* Use a fast NVMe SSD +* Increase RAM +* Reduce the complexity of questions +* Use `:reset` to clear memory and lighten the load + +**Q: Can I use Colibrì without Docker?** + +A: Colibrì was born that way, but this guide assumes Docker. To build from source, see the GitHub repository. + +**Q: How much internet connection do I need after downloading the model?** + +A: Zero. Colibrì works completely offline. + +--- + +## Support and contributions + +If you find errors or have suggestions for improving this guide, open an issue or a pull request on the Colibrì GitHub repository. + +Have fun! 🐦 + +--- + +## Tests on my PC + +In the first case, I asked a question in Italian; in the second, in Japanese; and in the third, I repeated the question in Japanese but requested an answer in Italian. + +``` +PS C:\quack\llm\colibri\docker> docker run --rm -it --name colibri-c -v "C:\quack\llm\models\glm-5.2:/app/glm-5.2" -e COLI_MODEL=/app/glm-5.2 colibri-i ./coli chat + + ▄▀▀▀▄ ▄ colibrì v1.0 + ▄▄▄▄▀▀▀▀▄▀▀ tiny engine, immense model + ▀▀▀▀▀▀▀ GLM-5.2 · 744B MoE · int4 · streaming CPU + ▀▀▀▀ chat · glm-5.2 · ram -GB · topp off + ▀ + ────────────────────────────────────────────────────────── + type and press Enter · Ctrl-C stops the answer · :more continues · :reset clears memory · :q exits + + ╭────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ › Quanti abitanti ha la Cina? │ + ╰────────────────────────────────────────────────────────────────────────────────────────────────╯ + + ◆ colibrì + La Cina è attualmente il paese più popoloso al mondo (sebbene, secondo alcune stime recenti, sia stata ormai superata dall'India). + + La popolazione totale della Repubblica Popolare Cinese è di circa 1,41 miliardi di abitanti (dati del 2020-2022 circa). + └─ 76 tok · 0.04 tok/s · hit 3% · RSS 15.9 GB · 2012s + + ╭────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ › 漫画「ワンピース」の主人公の名前を教えてください。名前だけで、それ以上のコメントはありません。 │ + ╰────────────────────────────────────────────────────────────────────────────────────────────────╯ + + ◆ colibrì + ルフィ + └─ 2 tok · 0.01 tok/s · hit 1% · RSS 16.7 GB · 260s + + ╭────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ › 漫画「ワンピース」の主人公の名前を教えてください。名前だけで、それ以上のコメントはありません。イタリア語で返信 │ + ╰────────────────────────────────────────────────────────────────────────────────────────────────╯ + + ◆ colibrì + Il nome del protagonista di One Piece è Monkey D. Luffy. + └─ 14 tok · 0.02 tok/s · hit 2% · RSS 17.3 GB · 593s + + ╭────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ › + +``` + +[source: 1] \ No newline at end of file diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 4bf2d55..c3694c6 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -2,7 +2,7 @@ Reference for the environment variables read by the colibrì engine. -**Generated from `upstream/dev @ 6d3ed7e`** by scanning every `getenv()` site in `c/glm.c`. Defaults and behavior are taken from the source; see [MAINTAINING-DOCS.md](MAINTAINING-DOCS.md) to regenerate this after the code changes. +**Generated from `dev @ d5327e2`** by scanning every `getenv()` site in `c/glm.c` and the other C sources (`c/olmoe.c`, `c/backend_cuda.cu`, `c/backend_metal.mm`). Defaults and behavior are taken from the source; see [MAINTAINING-DOCS.md](MAINTAINING-DOCS.md) to regenerate this after the code changes. ## Which program reads these? @@ -43,6 +43,7 @@ Format: `VAR` — default — effect. | `URING` | `0` (off) | Linux-only queued expert I/O. `URING=1` implies `PIPE=1`, forces cold reads through io-wq (`IOSQE_ASYNC`), replaces blocking loader pthreads and spin waits with batched SQEs/CQEs, and batches `PILOT_REAL` loads on a separate ring. Use `DIRECT=1` for cold NVMe to avoid page-cache copy/readahead limits. Fails clearly if the kernel denies io_uring; incompatible with `COLI_MMAP=1`. | | `DIRECT` | `0` (off) | Use `O_DIRECT`/unbuffered reads for expert slabs. Helps sustained NVMe; keeps the zero-copy GPU path. | | `COLI_NO_OMP_TUNE` | off | **Kill-switch** for the OpenMP hot-thread tuning (`OMP_WAIT_POLICY=active` spin + proc-bind). Set `=1` when the CPU is mostly waiting on the GPU (Metal) so spin doesn't steal the shared power budget. | +| `COLI_NUMA` | auto in generated plans on multi-socket Linux; otherwise off | `COLI_NUMA=1` selectively interleaves large expert and dense slabs across NUMA nodes via `mbind` (raw syscall, no libnuma). Helps multi-socket hosts (+7–40% expert matmul); silent no-op on single-node or non-Linux. Explicit `COLI_NUMA=0` overrides the generated plan. | | `MLOCK` | `-1` (auto: on for macOS) | Wire the streamed expert cache into physical RAM (`mlock`) to dodge the memory compressor. `0` off, `1` force. | | `CAP_RAISE` | `1` (on) | Let the engine raise the expert-cache cap above `topk` when RAM allows (bigger batches). `0` fixes the cap. | | `PREFETCH` | `0` | Prefetch depth for streamed experts. | @@ -54,16 +55,26 @@ Format: `VAR` — default — effect. | `PILOT` | `0` (off) | Router-piloted cross-layer expert prefetch. | | `PILOT_REAL` | `0` (off) | Value-preserving real cross-layer prefetch loads (`PILOT_REAL=1` opts in). | | `PILOT_K` | `6` if `PILOT_REAL` else `8` | Number of experts the pilot prefetches per step. | +| `PILOT_TWO` | `0` (off) | Two-step shared-expert-corrected router prediction for the pilot. | +| `COUPLE` | unset | Path to a coupling-score file driving cross-layer expert prefetch (#176). When set, `couple_load` reads it. | +| `COUPLE_K` | `8` | Top-K coupled experts per layer when `COUPLE` is set. | +| `COUPLE_D` | `1` | Coupling lookahead depth (`1` or `2`) when `COUPLE` is set. | | `CACHE_ROUTE` | `0` (off) | Opt-in max-rank cache-aware MoE routing (pin∪LRU prefer within top-M). See [CACHE_ROUTE.md](CACHE_ROUTE.md). | | `ROUTE_J` | `2` | Sacred top ranks always taken when `CACHE_ROUTE=1`. | | `ROUTE_M` | `12` | Max-rank window for resident preference when `CACHE_ROUTE=1`. | | `ROUTE_P` | `0` | Cumulative mass window for CACHE_ROUTE (`0` = fixed M). | | `ROUTE_ALPHA` | `1` | Scale gate mass of substituted experts before renorm (`1` = off). | | `ROUTE_AGREE` | auto | Overlap% + KL vs true top-K; auto-on when `CACHE_ROUTE=1`. | +| `ROUTE_TRACE` | unset | If set to a path, logs every routing decision there (testing/analysis). | | `ABSORB` | `-1` (auto: absorbed for S≤4) | MLA attention absorption mode. | | `IDOT` | `1` | Integer dot-product kernel. `IDOT=0` uses exact f32 kernels (for A/B numerical checks). | | `COLI_POLICY` | `quality` | Resource policy: `quality`, `balanced`, or `experimental-fast`. | | `PROF` | `0` (off) | Performance profile: a startup header (machine + effective config), then per run — or per turn in serve mode, on stderr — forward-latency percentiles (p50/p90/p99/max), expert-I/O totals and cache-tier fill, phase shares of wall time, and a verdict naming the knob most likely to help on this machine. Output is additive; `PROF` unset changes nothing. | +| `COLI_NO_FUSED_PAIR` | `0` (off) | `=1` disables the fused-pair matmul kernel. | +| `DISK_SPLIT` | `0` (off) | `=1` splits the reported disk-load time across the draft/absorb/forward phases in stats. | +| `I4S` | unset | Engage the int4 `IDOT` kernel only for batch `S>=` (testing). | +| `SPEC_PIN` | `1` (on) | Speculation gate mode. `0` reverts to the legacy S-dependent speculation gates (#163). | +| `COLI_RAM_OVERCOMMIT` | off | `=1` overrides the "projected peak > MemAvailable → exit(2)" guard so a run that risks kernel OOM-kill is allowed to proceed. | --- @@ -77,7 +88,31 @@ Format: `VAR` — default — effect. | `CUDA_EXPERT_GB` | `0` | VRAM budget (GB) for caching experts on the GPU. | | `CUDA_RELEASE_HOST` | auto (`1` if >1 device) | Release host-side copies after upload. | | `COLI_CUDA_ATTN` | off | Run S≤4 attention on the GPU. | +| `COLI_CUDA_ATTN_SHARD` | off | `=1` splits KV-b heads across devices during attention load (multi-GPU). | | `COLI_CUDA_PROFILE` | off | Emit CUDA timing. | +| `COLI_CUDA_PIPE` | `0` (off) | `1` engages the multi-step attention pipeline; `2` enables the pipe2 path. | +| `COLI_CUDA_PIPE_SHARD` | off | `=1` runs the multi-device P2P head-shard attention path (opt-in for NVLink topologies; serializes ~95 MB/layer over a star PCIe topology). | +| `COLI_CUDA_PIPE_S_MIN` | `1` single-GPU, `8` multi-GPU | Minimum prefill batch S to engage the pipe2 CUDA path. | +| `COLI_CUDA_MTP` | `0` (off) | `=1` opts into MTP speculation under CUDA (off by default: cold streaming experts run on CPU where the fused-pair/IDOT kernels diverge in FP order, collapsing draft acceptance, #163/#292). | +| `COLI_CUDA_ASYNC` | on | `=0` forces synchronous `cudaMemcpy` instead of async + pinned host staging. | +| `COLI_CUDA_DUAL_PROJ` | on | `=0` issues gate+up as two separate launches instead of one fused `grouped_hidden_w4_dual`. | +| `COLI_CUDA_W4_PACKED` | on | `=0` disables the grouped packed-int4 path. | +| `COLI_CUDA_TC_INT4` | off | `=1` uses the W4A4 WMMA Tensor Core path (when all expert tensors are int4 and dims divide). | +| `COLI_CUDA_TC_MIN_ROWS` | `8` | Min rows-per-expert to engage the W4A4 Tensor Core path. | +| `COLI_CUDA_TC_W4A16` | off | `=1` uses the lossless W4A16 Tensor Core path (compute capability ≥7). | +| `COLI_CUDA_TC_W4A16_MIN` | `16` | Per-expert row threshold above which W4A16 TC tiles dispatch (smaller batches fall back to the naive kernel). | +| `COLI_CUDA_SHARED_W4A16` | off | `=1` uploads shared-expert weights and runs the shared-MLP W4A16 Tensor Core kernel. | +| `COLI_CUDA_SHARED_W4A16_MIN_ROWS` | `32` | Min row count to engage the shared-MLP W4A16 kernel. | +| `COLI_METAL_UNTRACKED` | off (Metal only) | `=1` sets `MTLResourceHazardTrackingModeUntracked` on Metal buffers (reduces hazard-tracking overhead). | + +> **Windows note.** On Windows, a bare `coli chat` / `coli run` / `coli serve` +> (no `--gpu`/`--vram`/`--auto-tier`) **auto-enables the GPU** when it detects a +> CUDA build (`coli_cuda.dll` next to the engine) and at least one GPU via +> `nvidia-smi`. The expert-tier VRAM budget is then sized automatically from the +> card's free VRAM (same computation as `--auto-tier`). If `nvidia-smi` is not on +> `PATH` the run falls back to CPU with a warning — pass `--vram N` (or add +> `nvidia-smi` to `PATH`) to enable CUDA in that case. `--gpu none` forces +> CPU-only. (Linux/macOS behaviour is unchanged: pass a flag to enable CUDA.) --- @@ -89,8 +124,11 @@ These are for testing, benchmarking, or internal use — not part of the everyda |---|---|---| | `SPEC` | `1` | Speculative decoding on/off. | | `DRAFT` | `-1` (auto: 3 with MTP, else 0) | Number of speculative draft tokens per step. | -| `GRAMMAR` | unset | Path to a GBNF grammar file to constrain generation. | +| `GRAMMAR` | unset | Path to a GBNF grammar file to constrain generation. Takes precedence over `SCHEMA`. | +| `SCHEMA` | unset | Path to a JSON-Schema file compiled to GBNF to constrain generation (consulted only when `GRAMMAR` is empty). | | `GRAMMAR_DRAFT` | unset | Max grammar-forced draft span length. | +| `EXPERT_BUDGET` | `0` (off) | Cap experts loaded per layer (MoE-Spec). **Quarantined:** silently forced to `0` unless `EXPERT_BUDGET_EXPERIMENTAL` is set — every tested value is either no faster or incoherent (issue #303). | +| `EXPERT_BUDGET_EXPERIMENTAL` | unset | Setting it (any value) allows `EXPERT_BUDGET>0` to actually take effect (expect garbage, #294). | | `DSA` | on | Dynamic Sparse Attention indexer. `DSA=0` disables. | | `DSA_FORCE` | `0` | Force the DSA path on. | | `DSA_TOPK` | model value | Override the DSA index top-k (testing). | @@ -101,11 +139,15 @@ These are for testing, benchmarking, or internal use — not part of the everyda | `PIN_FILL` | `0` | Fill the pinned store even without usage data. | | `MTP_DEBUG` / `MTP_PRENORM` / `MTP_SWAP` | off | MTP head debugging / ablations. | | `STATS` | unset | Write an expert-usage histogram to `STATS=` at end of run. | +| `TOKENS` | unset | If set, dumps generated token ids to stderr for A/B comparison. | | `SCORE` | unset | Scoring/eval mode over `SCORE=`. | +| `SCORE_PREFIX` | on | If unset or `≠0`, prepends `[gMASK]` to scoring contexts (GLM-family only). | +| `REPIN_VERBOSE` | off | If set, prints per-swap `[REPIN]` diagnostics during VRAM repin. | | `REF` / `REF_FORCE` | `ref_glm.json` | Reference-output comparison mode. | | `REPLAY` | unset | Replay mode. | | `TF` | unset | Teacher-forcing mode. | | `CHAT_TEMPLATE` | `1` | Apply the GLM chat template (`0` = raw prompt). | +| `PPL` | off (`olmoe.c` only) | `PPL=1` enters teacher-forced NLL/perplexity meter mode in the OLMoE sister engine. | --- @@ -136,7 +178,7 @@ These are read by the Python programs (not the `glm` engine), so they don't appe - `SNAP` — model snapshot directory (required by `glm`; set from `--model`). - `SERVE`, `SERVE_BATCH` — select serve / batched-serve mode. -- `PROMPT` — one-shot text mode. +- `PROMPT` — one-shot text mode (the engine also honors `COLI_PROMPT`, preferred cross-platform; `PROMPT` is ignored on Windows if it contains cmd.exe `$`-metacharacters). - `COLI_OMP_TUNED` — internal sentinel guarding the OMP re-exec (see `COLI_NO_OMP_TUNE`); not user-facing. --- diff --git a/docs/api.md b/docs/api.md index d3b09f1..62010bd 100644 --- a/docs/api.md +++ b/docs/api.md @@ -49,6 +49,82 @@ errors before streaming headers are sent. `GET /health` exposes active/queued/completed/rejected counters, and successful generation responses include `x-colibri-queue-wait-ms`. +## Connect a coding CLI or editor + +The API is OpenAI-compatible, so most coding CLIs and editor extensions work by +pointing them at Colibri as an *OpenAI-compatible* provider. Three settings: + +- **Base URL** — `http://localhost:8000/v1` +- **Model** — `glm-5.2-colibri` (or whatever you pass to `--model-id`) +- **API key** — any non-empty string, e.g. `local` + +Colibri needs **no** API key by default, but many clients refuse to start without +one — give them any dummy value. The key is only enforced if you set `COLI_API_KEY`. + +Smoke-test the endpoint first (no key needed unless you set one): + +```bash +curl http://127.0.0.1:8000/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"glm-5.2-colibri","messages":[{"role":"user","content":"hi"}]}' +``` + +**aider** + +```bash +export OPENAI_API_BASE=http://localhost:8000/v1 +export OPENAI_API_KEY=local +aider --model openai/glm-5.2-colibri # the openai/ prefix routes to OPENAI_API_BASE +``` + +**crush** — add a provider to `crush.json` (`~/.config/crush/crush.json`, or +`%USERPROFILE%\AppData\Local\crush\crush.json` on Windows): + +```json +{ + "$schema": "https://charm.land/crush.json", + "providers": { + "colibri": { + "name": "Colibri", + "type": "openai-compat", + "base_url": "http://localhost:8000/v1/", + "api_key": "local", + "models": [ + { "name": "GLM-5.2 (Colibri)", "id": "glm-5.2-colibri", + "context_window": 131072, "default_max_tokens": 1024 } + ] + } + } +} +``` + +The `"api_key": "local"` dummy is what satisfies clients that demand a key. +`context_window` is only the client's budget display — set it to whatever your +KV configuration actually allows. + +**Continue, Cline / Roo, `llm`, the OpenAI SDKs, …** — set the provider's base +URL to `http://localhost:8000/v1`, the model to `glm-5.2-colibri`, and any dummy +key (`OPENAI_API_KEY` / `OPENAI_BASE_URL` for env-based tools). + +> **Set your expectations before connecting an agentic CLI.** Two costs dominate, +> and the first one is invisible until you know it's there: +> +> 1. **Prefill.** Coding agents (crush, aider in repo-map mode, Cline, …) send a +> large system prompt plus tool definitions — often 10–20k tokens — *before +> your first word*. Prefill on the CPU-streaming path runs at a few tokens per +> second (it is attention-bound, see #153), so a 15k-token agent preamble is +> **an hour of silent "thinking" before the first output token**. The client +> looks hung; it isn't. Smoke-test with the tiny `curl` above first — if that +> answers in about a minute, the pipeline works and what you're paying for is +> prompt size. +> 2. **Decode.** Roughly 1 tok/s for a large model, so multi-turn agent loops +> (which re-pay the growing context every turn) compound the cost. +> +> Practical guidance: single surgical asks with a short context work; iterative +> agent sessions against a disk-streaming 744B model do not resemble a hosted +> API and mostly won't be worth the wait. If your client lets you trim or disable +> its system preamble and tool catalog, do it. + ## Isolated KV contexts `coli serve --kv-slots N` allocates up to 16 independent sequence contexts. diff --git a/docs/benchmarks.md b/docs/benchmarks.md index f9c83f2..09d3752 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -116,8 +116,12 @@ can match an RTX 5090 on expert matmul ([#101](https://github.com/JustVugg/colib so **the GPU tier earns its VRAM only when the CPU is the weak link**. On multi-socket hosts, NUMA placement is a further lever: interleaving the resident weights across nodes measured **+13% (2-socket) and +40% (4-socket CPU-only)** -([#82](https://github.com/JustVugg/colibri/issues/82)) — but never blanket-interleave -a GPU host (measured 10× regression via the DMA staging pages). +([#82](https://github.com/JustVugg/colibri/issues/82)). On a 2-socket Xeon Silver +4510 host with 6× RTX 5090, selective `COLI_NUMA=1` raised effective CPU-expert +bandwidth from **42.42 to 58.26/65.89 GB/s** and greedy decode from **7.66 to +9.02/9.17 tok/s** (64 tokens, `TEMP=0 DRAFT=0`, byte-identical output). Do not +blanket-interleave a GPU host: it also spreads DMA staging pages and has measured +up to a 10× regression; generated plans enable only the selective slab policy. ## Quality benchmark diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 0000000..60d4877 --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,184 @@ +# Quick Start — from zero to a running model + +A step-by-step guide for first-time users on **Linux**, **Windows**, and **macOS**. +No prior experience with C, CUDA, or model conversion is assumed. If you get +stuck, `./coli doctor` (below) tells you exactly what's missing. + +> **What you're setting up:** colibrì runs a very large Mixture-of-Experts model +> (e.g. GLM-5.2, 744B parameters) on a normal machine by streaming the model's +> experts from disk instead of needing them all in RAM. The engine is a single +> C program; Python is only used once, to prepare the model files. + +--- + +## 0. What you need first (prerequisites) + +| | Minimum | Recommended | +|---|---|---| +| **RAM** | ~16 GB | 24 GB+ | +| **Free disk** | ~380 GB for the int4 model | a fast NVMe SSD (streaming speed = your token speed) | +| **OS** | Linux, Windows 10/11, or macOS | any | +| **Tools** | a C compiler + `make` + `git` + `python3` | — | + +You do **not** need a GPU. A GPU only helps if you have one; the engine runs +CPU-only by default. + +--- + +## 1. Install the build tools + +### Linux (Ubuntu / Debian) + +```bash +sudo apt update +sudo apt install -y build-essential git python3 +``` + +`build-essential` gives you `gcc`, `make`, and OpenMP (libgomp) — everything the +engine needs. + +### Windows + +You have two options. + +**Option A — download a prebuilt binary (no compiler needed).** +Grab `colibri--windows-x86_64.zip` from the +[Releases page](https://github.com/JustVugg/colibri/releases) and unzip it. +Inside you'll find: + +| File | What it is | +|---|---| +| `colibri--windows-x86_64.exe` | **the engine** — the C program that actually runs the model | +| `coli` | the command-line launcher (`chat`, `serve`, `convert`, `doctor`, …) | +| `openai_server.py`, `resource_plan.py`, `doctor.py` | Python support for the API server and placement planner | + +Two setup steps: + +1. **Rename the engine to `glm.exe`** so the launcher can find it (it looks for a + binary named `glm`): + ```powershell + Rename-Item colibri-*-windows-x86_64.exe glm.exe + ``` +2. **Install Python 3** from [python.org](https://www.python.org/downloads/) — the + `coli` launcher and the API gateway are Python scripts (the engine itself is + pure C and needs nothing). + +Then continue to [step 3](#3-get-the-model). Prefer to skip the launcher? You can +run the engine directly — `.\glm.exe` reads the model path from the `SNAP` +environment variable (see [docs/windows.md](windows.md)) — but `coli chat` is the +easy path. + +**Option B — build from source with MSYS2.** +Install [MSYS2](https://www.msys2.org/), open the **UCRT64** shell, and run: + +```bash +pacman -S --needed mingw-w64-ucrt-x86_64-gcc make git python +``` + +### macOS + +```bash +xcode-select --install # C compiler (clang) +brew install libomp git python # OpenMP for multithreading +``` + +--- + +## 2. Get the code and build the engine + +```bash +git clone https://github.com/JustVugg/colibri.git +cd colibri/c +./setup.sh +``` + +`setup.sh` checks your compiler and OpenMP, builds the engine, and runs a tiny +self-test. When it prints: + +``` +engine self-test: 32/32 (expected 32/32) +``` + +the engine is working correctly. (On Windows Option A you already have the +binary — you can skip this step.) + +--- + +## 3. Get the model + +You have two paths. + +### Easiest — download a ready-made int4 container + +A pre-converted **GLM-5.2 int4** model is on Hugging Face. **Use the version +with the int8 MTP heads** (the plain int4 heads disable speculative decoding — +see [#8](https://github.com/JustVugg/colibri/issues/8)): + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +Download it into a folder on a fast disk, e.g. `/nvme/glm52_i4` (Linux/macOS) or +`D:\glm52_i4` (Windows). It is about **372 GB**, so make sure you have the space. + +### Or convert it yourself from the FP8 source + +One resumable command downloads and converts the model shard by shard, so it +never needs the full ~756 GB on disk at once: + +```bash +./coli convert --model /nvme/glm52_i4 +``` + +This step uses Python and runs only once. Safe to interrupt and re-run — it +resumes where it left off. + +--- + +## 4. Run it + +Point `COLI_MODEL` at the folder from step 3 and start chatting: + +```bash +# Linux / macOS +COLI_MODEL=/nvme/glm52_i4 ./coli chat + +# Windows (UCRT64 shell) +COLI_MODEL=/d/glm52_i4 ./coli chat +``` + +Useful first commands: + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # read-only check: is everything ready? +COLI_MODEL=/nvme/glm52_i4 ./coli plan # shows where the model will live (RAM/disk/GPU) +COLI_MODEL=/nvme/glm52_i4 ./coli chat --topp 0.85 # faster: reads less from disk, same quality +``` + +> **Tip:** `--topp 0.85` is worth adding on a disk-bound machine — it reads +> fewer expert bytes per token with no quality loss, which directly means more +> tokens per second. + +--- + +## 5. What to expect + +- **First launch loads the resident weights** (~10 GB) — this takes a moment. +- **Speed depends on your disk.** The experts stream from storage, so a fast + NVMe SSD is the single biggest factor in tokens/second. On a slow or shared + disk, generation can be well under 1 token/second — that's expected, and it's + the honest cost of running a 744B model on a small machine. +- **It's still the full model.** Placement only changes speed, never the model's + answers or precision. + +If something doesn't work, run `./coli doctor` — it reports exactly what's +missing (compiler, model files, permissions) and how to fix it. + +--- + +## Where to go next + +| Topic | Doc | +|---|---| +| Windows native build (and CUDA DLL) | [docs/windows.md](windows.md) | +| Tuning: cache, prefetch, speculation | [docs/tuning.md](tuning.md) | +| OpenAI-compatible API + web dashboard | [docs/api.md](api.md) | +| Every environment variable | [docs/ENVIRONMENT.md](ENVIRONMENT.md) | diff --git a/docs/windows.md b/docs/windows.md index 94a4f69..ecb81e7 100644 --- a/docs/windows.md +++ b/docs/windows.md @@ -1,25 +1,107 @@ -# Windows 11 (native, no WSL) +# Windows 11 native install — a complete walkthrough (no WSL) -colibrì builds and runs natively on Windows 11 x86-64 with MinGW-w64. The port -adds a `_WIN32` compatibility layer in `c/compat.h` that maps POSIX I/O to the -Windows API (pread → ReadFile+OVERLAPPED, posix_fadvise no-op, aligned -allocation, MoveFileEx rename, GlobalMemoryStatusEx RAM detection). All platform -differences stay in `compat.h`; the engine source is unchanged. +A start-to-finish, reproducible path from a fresh Windows 11 machine to GLM-5.2 generating tokens, with the GPU tier. Every step and every failure mode below was hit and verified on real hardware: Core Ultra 9 285K (AVX-VNNI) / RTX 5080 (sm_120) / 128 GB RAM / Windows 11 24H2 (issue #306). Steps are ordered so the long downloads run while you build. -**Toolchain:** GCC via [winlibs](https://winlibs.com/) or MSYS2 MinGW-w64. -Tested with GCC 16.1.0 (x86_64-ucrt-posix-seh). +## 0. What you need + +| Piece | Why | Get it | +|---|---|---| +| git, Python 3 | clone + `coli` launcher | winget / python.org | +| MinGW-w64 gcc + make | builds the engine (MSVC can't) | `scoop install mingw-winlibs`, MSYS2, or portable **w64devkit** (no admin, unzip and go) | +| CUDA Toolkit ≥ 12.8 | GPU tier; ≥12.8 required for Blackwell/sm_120 | `winget install Nvidia.CUDA` | +| MSVC Build Tools (C++ workload) | nvcc's host compiler for the CUDA DLL | `winget install Microsoft.VisualStudio.2022.BuildTools` + "Desktop development with C++" | +| ~400 GB free on a local NVMe | the int4 model (~370–384 GB) | NTFS is fine; **never** a network mount | + +RAM: 16 GB minimum, more = bigger expert cache = faster. The build itself needs none of the CUDA/MSVC pieces — do the CPU build first, add the GPU tier later. + +## 1. Start the model download first (it's the long pole) ```powershell -# One-time toolchain install (pick one): -scoop install mingw-winlibs # portable, no shell needed -# or: pacman -S mingw-w64-x86_64-gcc make # via MSYS2 +python -m pip install -U "huggingface_hub[hf_transfer]" +$env:HF_HUB_ENABLE_HF_TRANSFER = "1" +hf download --local-dir D:\glm52_i4 +``` -# Build (from c/ directory): -make glm.exe # GLM-5.2 engine (static, no DLL dependencies) -make olmoe.exe # OLMoE engine (same shims) -make iobench.exe # disk I/O benchmark -make test-c # run C tests -make test-python # run Python tests (requires python) +Use the container recommended in the README (with **int8 MTP heads** — int4 heads silently give 0% draft acceptance). The download is resumable: if it stops, rerun the same command. Expect hours; everything below fits inside them. + +## 2. Build the engine (CPU) + +From a normal PowerShell, in the repo's `c\` directory: + +```powershell +make glm.exe ARCH=native # ARCH=native unlocks AVX-VNNI on Alder Lake+/Arrow Lake +make iobench.exe # disk benchmark, useful before committing to the download +``` + +Warnings about `#pragma comment` and unused variables are normal (MSVC-isms gcc ignores). The engine banner should print `idot: avx-vnni` on VNNI-capable CPUs — if it says avx2, you built without `ARCH=native`. + +### ⚠️ Smart App Control will block your fresh binary + +On Windows 11 machines with **Smart App Control** enforced (`VerifiedAndReputablePolicyState = 1`), running your self-compiled `glm.exe` fails with: + +``` +Program 'glm.exe' failed to run: An Application Control policy has blocked this file +``` + +This is not Defender and not Mark-of-the-Web — SAC blocks *all* unsigned, unknown binaries, which includes anything you compile yourself. **Fix:** Windows Security → App & browser control → Smart App Control settings → **Off**, then **reboot** (the policy only reloads on restart). Note SAC is one-way: re-enabling later requires resetting Windows. If the settings page is missing, the registry equivalent is setting `HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy\VerifiedAndReputablePolicyState` to `0` (admin PowerShell), then rebooting. Check your current state before touching anything: + +```powershell +(Get-ItemProperty "HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy").VerifiedAndReputablePolicyState +# 0 = off, 1 = enforced, 2 = evaluation +``` + +## 3. Build the CUDA DLL (GPU tier) + +nvcc needs MSVC as host compiler, so this one step must run from a shell with the MSVC environment: open **"x64 Native Tools Command Prompt for VS 2022"** from the Start menu (plain PowerShell will fail the `cl` check). Then: + +```cmd +make cuda-dll CUDA_ARCH=sm_120 # match your GPU: sm_120 Blackwell, sm_89 Ada, ... +make glm.exe CUDA_DLL=1 ARCH=native # relink host with the runtime loader +``` + +Two pitfalls, both fixed on current `dev` (#314) but worth knowing on older checkouts: + +- **Spaces in `CUDA_HOME`** (`C:\Program Files\...`) used to break the recipe → fixed; nvcc now comes from PATH and `"$(NVCC)"` is quoted. +- **`make glm.exe CUDA_DLL=1` after a CPU-only build** used to report `up to date` and silently keep the CPU-only binary (GPU tier never engages, no error). Current `dev` has a build-config stamp that forces the relink. On older trees: delete `glm.exe` first. + +Sanity check: first GPU run should print `[CUDA] device 0: , ... sm_XX` and `[CUDA] mode: routed experts + resident dense tensors`. + +## 4. First run + +```powershell +cd \c +$env:OMP_NUM_THREADS = "" +python coli run "Explain what a mixture-of-experts model is." --model D:\glm52_i4 --ngen 48 +``` + +The first run is cold — expect the profile to be dominated by `expert-disk` while the cache warms; hit rate climbs run over run. GPU tier on top: + +```powershell +$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_DENSE="1"; $env:CUDA_EXPERT_GB="4" +python coli run "..." --model D:\glm52_i4 --ngen 64 +``` + +Size `CUDA_EXPERT_GB` so dense (~10 GB) + experts + working set stays under your VRAM. Note MTP speculation is off by default under CUDA (#293, float-accumulation divergence between draft and verify) — `COLI_CUDA_MTP=1` opts back in. + +## 5. Reference numbers from this walkthrough's hardware + +285K / RTX 5080 / 128 GB / NVMe at 5.85 GB/s random-read (19 MB blocks, `iobench`): 0.26 tok/s cold CPU → 0.30 warm CPU (MTP 2.2–2.3 tok/forward) → 0.42 tok/s GPU tier + auto-pin, expert hit 66%, ~65% of wall time in expert-disk. Disk-bound is the expected shape at ~25% expert residency — a faster disk and more RAM move the floor, the GPU moves the compute. + +## Quick failure index + +| Symptom | Cause | Fix | +|---|---|---| +| `An Application Control policy has blocked this file` | Smart App Control | §2 — turn SAC off + **reboot** | +| `cuda-dll ... Error 1` immediately | old tree: spaced CUDA_HOME / MSVC rejects `-Wextra` | update to current `dev` (#314) | +| `glm.exe is up to date` but GPU never engages | old tree: stale CPU-only binary | update to `dev`, or delete `glm.exe` and rebuild | +| `cl.exe (MSVC) not in PATH` | built from plain PowerShell | use the x64 Native Tools prompt | +| `nvcc fatal: unsupported gpu architecture 'sm_120'` | CUDA < 12.8 | install CUDA 12.8+ | +| MTP `0% (0/0)` on CPU path | int4 MTP heads in the container | use the int8-MTP container | +| MTP `draft=0` under CUDA | intended default since #293 | `COLI_CUDA_MTP=1` to opt in | + +--- + +## Reference: build flags & warmup # AVX-VNNI: Intel Alder Lake+ (and Meteor Lake+) CPUs have a 128-bit int8 # dot-product instruction (VPDPBUSD) the engine can use for ~1.3x faster diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..9b788fa --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1784160687, + "narHash": "sha256-iYL/bixrb6FlHFu/gIuBYzq6c6lM5AAXsXNSWXtIgQc=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "4382ed2b7a6839d4280a9b386db49cbc5907414d", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-26.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix index 368b0a7..584a088 100644 --- a/flake.nix +++ b/flake.nix @@ -26,7 +26,9 @@ version = "1.0"; src = ./.; - nativeBuildInputs = [ pkgs.makeWrapper ]; + # python3 is needed by checkPhase: `make test-c` shells out to + # `python3 tools/run_tests.py` (see c/Makefile, PYTHON ?= python3). + nativeBuildInputs = [ pkgs.makeWrapper pkgs.python3 ]; buildInputs = [ pkgs.gcc @@ -44,18 +46,29 @@ installPhase = '' runHook preInstall - mkdir -p $out/bin - cp c/glm $out/bin/glm - # Wrap coli (the Python CLI) so it finds the right python and the engine - mkdir -p $out/share/colibri - cp c/coli $out/share/colibri/coli - chmod +x $out/share/colibri/coli - cp -r c/tools $out/share/colibri/tools + # Self-contained layout under $out/lib/colibri that mirrors the + # source tree `coli` runs in (see the path-resolution logic at the + # top of c/coli): the engine, the coli CLI script, the support + # modules it imports (openai_server.py, resource_plan.py, + # doctor.py), and tools/ all sit next to each other. + mkdir -p $out/lib/colibri/tools $out/bin + cp c/glm $out/lib/colibri/glm + cp c/coli $out/lib/colibri/coli + chmod +x $out/lib/colibri/coli + cp c/openai_server.py c/resource_plan.py c/doctor.py $out/lib/colibri/ + cp -r c/tools/* $out/lib/colibri/tools/ + # $out/bin holds the user-facing entry points. + ln -s ../lib/colibri/glm $out/bin/glm + + # Wrap coli: point it at the bundled engine (COLI_ENGINE) so it is + # found by default, and at the module dir (PYTHONPATH) so + # `import openai_server` / `resource_plan` / `doctor` resolve. makeWrapper ${pythonEnv}/bin/python $out/bin/coli \ - --add-flags "$out/share/colibri/coli" \ - --set PYTHONPATH "${pythonEnv}/${pkgs.python3.sitePackages}" + --add-flags "$out/lib/colibri/coli" \ + --set-default COLI_ENGINE "$out/lib/colibri/glm" \ + --set PYTHONPATH "$out/lib/colibri:${pythonEnv}/${pkgs.python3.sitePackages}" runHook postInstall ''; diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..18f8f85 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,53 @@ +[build-system] +requires = ["setuptools>=68.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "colibri-engine" +dynamic = ["version"] +description = "Tiny engine, immense model — run GLM-5.2 (744B MoE) locally" +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.10" +authors = [ + {name = "JustVugg"}, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: Science/Research", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Programming Language :: Python :: 3", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] + +[project.optional-dependencies] +convert = [ + "numpy", + "huggingface_hub", +] +oracle = [ + "torch>=2.0", + "transformers>=4.40", + "safetensors", +] +bench = [ + "tokenizers", + "datasets", +] + +[project.scripts] +coli = "colibri.cli:main" + +[project.urls] +Homepage = "https://github.com/JustVugg/colibri" +Issues = "https://github.com/JustVugg/colibri/issues" + +[tool.setuptools.dynamic] +version = {attr = "colibri._version.__version__"} + +[tool.setuptools.packages.find] +where = ["."] +include = ["colibri*"] diff --git a/site/colibri-icon.svg b/site/colibri-icon.svg new file mode 100644 index 0000000..76838b9 --- /dev/null +++ b/site/colibri-icon.svg @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/site/colibri.svg b/site/colibri.svg new file mode 100644 index 0000000..4aed0bb --- /dev/null +++ b/site/colibri.svg @@ -0,0 +1,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + colibrì + tiny engine, immense model + GLM-5.2 · 744B MoE · int4 · streaming CPU + diff --git a/site/index.html b/site/index.html new file mode 100644 index 0000000..dd6dac1 --- /dev/null +++ b/site/index.html @@ -0,0 +1,698 @@ + + + + + +colibrì — tiny engine, immense model + + + + + + + + + + + + +
+ +
tiny engine · immense model
+

colibrì

+
a 744-billion-parameter mind, on your desk
+

Run GLM-5.2 — 744B Mixture-of-Experts — on hardware you already own. + Pure C, zero dependencies, experts staged just-in-time across VRAM, RAM and disk. + The galaxy behind this text is real: the model's measured expert atlas.

+
+ ★ Star on GitHub + Quick start → +
+
+
744B
parameters
+
19,456
experts
+
0
dependencies
+
25 GB
min RAM
+
9.2
tok/s measured
+
+
scroll
+
+ +
+ +
+
the vision
+

Frontier models should not be sealed inside datacenters.
+ Anyone curious enough should be able to open one up —
+ run it, watch every expert fire, and make it better.

+
+
⚡

Run it

+

A 744B frontier-class MoE answering on a 25 GB machine — the proven floor — up to + 9+ tok/s on a workstation. No cluster, no cloud bill, no API key.

+
🔬

Study it

+

The engine is a microscope: live routing telemetry, per-expert heat, and the first + published expert atlas of a 700B-class model — measured on machines like yours.

+
🛠

Improve it

+

One C file, readable in an afternoon. Every optimisation this project found came from + someone measuring something — the next one can come from you.

+
+
+ +
+
01 — the algorithm
+

A JIT, but for weights

+
+
+

A compiler JIT never compiles the whole program — it watches what actually runs + and compiles the hot paths, just in time. colibrì applies the same bet to a + 744B parameter space: a token activates only ~40B parameters, and just + ~11 GB of those change from token to token — the routed experts.

+
So parameters are not resident state to be held — they are + data to be staged, exactly when the router proves they are needed.
+

The engine keeps the dense weights resident and treats the 19,456 experts as a + heterogeneous storage hierarchy: measured routing heat decides which experts earn + VRAM, which earn pinned RAM, and which stream from NVMe. The router runs a layer + ahead, so prefetch hides the staging latency — and because routing has + measurable structure (the atlas below), the hierarchy keeps getting the answer right. + Like a JIT, it learns your workload: the more you run, the hotter the right experts get.

+

Everything is validated token-exact against the reference transformers + implementation — speed never buys drift.

+
⚗ Active research: the next algorithmic step is under way — + smarter placement and scheduling beyond LRU + pin, overlap of CPU and GPU expert execution, + and routing-aware speculation. Measured, reviewed, and merged in the open.
+
+
+
+ VRAMhottest experts · grouped GPU matmuls · + six 5090s hold the entire routed set — disk reads hit zero
+
↑ promote by measured heat ↓ demote
+
+ RAMwarm set · pinned & wired, learned from + your usage · AVX-512/NEON int4 kernels + LRU behind it
+
↑ promote ↓ demote
+
+ NVMecold tail · streamed on demand with io_uring, + prefetched a layer ahead — how 744B fits a 25 GB machine at all
+
token → router picks 8 of 256 experts × 75 layers → + gather across tiers → int4 matmul → next token
+
+
+
+ +
+
02 — the engine, live
+

Watch it think

+

A replay of the engine at work, paced to measured decode speeds from real + community hardware. The grid is all 19,456 experts (colour = storage tier, brightness = + routing heat, routed = white flash); the galaxy below it is the measured atlas — + watch different questions light different regions of the mind.

+
+
+
+
+ coli chat —
+
+
+
+
+ the brain — 75 MoE layers × 256 experts + MTP
+
+ +
+
0.0 tok/s
decode
+
—
ttft (measured)
+
—
resident hit
+
+
+ VRAM tier + RAM tier + disk tier + routed now +
+
+
+
+
+ the atlas, live — measured specialists routed by the current topic
+
+ +
routing: —
+
+
+
+

Simulation replays a fixed transcript at each profile's measured decode rate + (#82, #389, + #387, #273); + “—” means that machine's report did not publish the number — nothing here is invented. + Atlas flashes sample real measured affinity (atlas v1) for the active topic.

+
+ +
+
+
03 — the atlas
+

Measured, not assumed

+

Every point below is a real measured expert from the published + canonical atlas v1: + 721 canonical specialists confirmed across three independent kernel families + (AVX-512 VNNI · NEON · AVX2, engine-pinned, replication-gated) plus 637 gate-sensitive ones. + Position is the measured 10-topic affinity vector — a specialist sits at its topic's anchor, + a generalist drifts to the centre. Colour = top topic, large points = canonical. Drag to spin.

+
+
+ +
+
drag to spin · scroll past to release
+
+
+
+ +
+
04 — the ladder
+

Same engine, any hardware

+

Measured decode on real community machines — the hardware only changes + where the experts live. Every row links to its public report.

+ + + + + + + + + + + + + + + + + + + + +
hardwaredecodewhere the experts live
6× RTX 5090 · full residency + selective NUMA (#82)9.0–9.2 tok/s
all in VRAM+RAM · disk 0
2× Xeon Gold 6430 · 1 TB DDR5 · CPU-only (#389)5.42 tok/s
every expert pinned in RAM
DGX Spark GB10 · 121 GB unified (#161)3.33 tok/s
unified memory + CACHE_ROUTE
MacBook Pro M5 Max · 128 GB · Metal (#387)2.0 tok/s
Metal tier + warm pin
Ryzen AI Max+ 395 · 128 GB (#200)1.83 tok/s
hot set pinned in RAM
Ryzen 9 9950X3D2 · RTX 5090 · Gen5 NVMe (#120)1.23 tok/s
28 GB VRAM tier + 11.5 GB/s disk
single RTX 5070 Ti laptop-class (#273)1.07 tok/s
GPU-resident pipeline
25 GB dev box · cold — the proven floor0.05–0.1 tok/s
streamed from NVMe
+

…and 15+ more community machines, from an M4 Pro Mac Mini to a 430 GB EPYC server. + Full tables, methodology and quality ablations: docs/benchmarks.md

+
+ +
+
05 — the models
+

One engine, more minds coming

+

The tiering algorithm is model-agnostic: any Mixture-of-Experts with routed + experts can be staged the same way. GLM-5.2 is live today; support for more open-weight + families is on the roadmap.

+
+
live

GLM-5.2

Z.ai · 744B MoE
+

The flagship target: int4 container, token-exact vs reference, full atlas published.

+
live

OLMoE

Allen AI · 7B MoE
+

The small research workhorse — quantization A/Bs and quality ablations run here first.

+
planned

Kimi K2

Moonshot AI · 1T MoE
+

The next scale step: a trillion-parameter mind on the same tiered engine.

+
planned

Qwen3 MoE

Alibaba Qwen
+

The most widely-deployed open family — broad hardware coverage meets broad adoption.

+
planned

MiniMax

MiniMax · MoE
+

Long-context specialists — a different routing profile for the atlas to map.

+
+
+

Thank you to the teams whose open weights make this project possible — + Z.ai (GLM), Moonshot AI (Kimi), Alibaba Qwen, MiniMax, + Allen AI (OLMoE), and everyone releasing frontier-class models in the open. + An engine is nothing without a mind to run.

+
+
+ +
+
06 — join in
+

Come build the microscope

+

Every optimisation in this project started with someone measuring something on + their own machine — a disk swap, a NUMA experiment, an atlas replication on a Mac Mini. + You don't need a datacenter to move frontier-model research. That is the whole point.

+
+

▸ run a benchmark

+

Any hardware is a datapoint — the community table grows one issue at a time.

 +

▸ pick an issue

+

Kernels, schedulers, the atlas, the dashboard — measured problems, waiting.

 +

▸ read the engine

+

One C file plus headers, zero dependencies. Readable in an afternoon — really.

 +
+
+ +
+ +
+ 🐦 colibrì — MIT license + GitHub + API + Tuning + the hummingbird: tiny, fast, precise. +
+ + + + diff --git a/web/src/App.tsx b/web/src/App.tsx index a33cc88..39c9f3b 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -9,6 +9,7 @@ import { Database, Feather, Gauge, + Globe, HardDrive, KeyRound, Layers, @@ -34,6 +35,7 @@ import { Brain } from "./Brain" import { Profiling } from "./Profiling" import { persistPublicSettings, stored } from "@/lib/storage" import { cn } from "@/lib/utils" +import { useLocale } from "./i18n" const message = (role: ChatMessage["role"], content: string): ChatMessage => { let id: string @@ -42,15 +44,12 @@ const message = (role: ChatMessage["role"], content: string): ChatMessage => { } export default function App() { - // When the page is served by the engine itself (coli web), same-origin is the - // right default: no CORS, no manual endpoint editing. The Vite dev server - // (port 5173) keeps the classic default. + const { t, locale, setLocale, locales } = useLocale() + const servedByEngine = typeof window !== "undefined" && window.location.port !== "5173" && window.location.protocol.startsWith("http") const defaultBase = servedByEngine ? `${window.location.origin}/v1` : "http://127.0.0.1:8000/v1" const [baseUrl, setBaseUrl] = useState(() => { const saved = stored(localStorage, "colibri.baseUrl", defaultBase) - // migrate: a stored FACTORY default pointing at another origin would trip CORS - // when the page is engine-served — upgrade it to same-origin once. if (servedByEngine && saved === "http://127.0.0.1:8000/v1" && defaultBase !== saved) return defaultBase return saved }) @@ -120,7 +119,7 @@ export default function App() { const result = await getHealth(baseUrl, apiKey) if (!disposed) { setHealth(result); setHealthError("") } } catch (cause) { - if (!disposed) setHealthError(cause instanceof Error ? cause.message : "Runtime metrics unavailable") + if (!disposed) setHealthError(cause instanceof Error ? cause.message : "status.runtimeUnavailable") } } const timer = window.setInterval(() => void poll(), 5000) @@ -155,13 +154,13 @@ export default function App() { } catch (cause) { if (!controller.signal.aborted) { setHealth(null) - setHealthError(cause instanceof Error ? cause.message : "Runtime metrics unavailable") + setHealthError(cause instanceof Error ? cause.message : "status.runtimeUnavailable") } } } catch (cause) { if (controller.signal.aborted) return setConnected(false) - setError(cause instanceof Error ? cause.message : "Could not reach the server.") + setError(cause instanceof Error ? cause.message : "status.serverError") } finally { if (probeRef.current === controller) { probeRef.current = null; setConnecting(false) } } @@ -227,7 +226,7 @@ export default function App() { if (controller.signal.aborted) { updateMessages((current) => current.filter((item) => item.id !== assistant.id || item.content)) } else { - setError(cause instanceof Error ? cause.message : "Generation failed.") + setError(cause instanceof Error ? cause.message : "status.generationFailed") updateMessages((current) => current.filter((item) => item.id !== assistant.id || item.content)) } } finally { @@ -241,22 +240,22 @@ export default function App() {
-
ACTIVE MODEL{model}
+
{t("topbar.activeModel")}{model}
- - - + + +
- {loading && tokenCount > 0 ? {tokenCount} tokens : null} - {!loading && tokPerSec != null ? {tokPerSec.toFixed(1)} tok/s : null} + {loading && tokenCount > 0 ? {t("topbar.tokens", { n: tokenCount })} : null} + {!loading && tokPerSec != null ? {t("topbar.tokPerSec", { n: tokPerSec.toFixed(1) })} : null} {!loading && ttft != null ? TTFT {(ttft/1000).toFixed(1)}s : null} {!loading && lastRun?.usage ? {lastRun.usage.prompt_tokens}→{lastRun.usage.completion_tokens} : null} {lastRun?.queueWaitMs != null ? queue {Math.round(lastRun.queueWaitMs)}ms : null} - slot {cacheSlot + 1} - + {t("topbar.slot", { n: cacheSlot + 1 })} +
@@ -335,11 +342,11 @@ export default function App() { {!messages.length ? (
- COLIBRÌ ENGINE -

Ask the giant.
Keep the machine yours.

-

Connect to a local colibrì server and stream responses directly from your hardware. Nothing leaves the endpoint you choose.

+ {t("hero.title")} +

{t("hero.subtitle")}
{t("hero.tagline")}

+

{t("hero.description")}

- {["Explain how expert routing works", "Write a small C benchmark", "Compare RAM and VRAM caching"].map((item) => )} + {[t("prompts.routing"), t("prompts.benchmark"), t("prompts.caching")].map((item) => )}
) : ( @@ -347,7 +354,7 @@ export default function App() { {messages.map((item) => (
{item.role === "user" ? "Y" : }
-
{item.role === "user" ? "You" : "colibrì"}
{item.content || }
+
{item.role === "user" ? t("chat.you") : t("chat.colibri")}
{item.content || }
))}
@@ -356,10 +363,10 @@ export default function App() {
- {error &&
{error}
} + {error &&
{t(error)}
}
-