diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000..bdbc3a3 --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,49 @@ +# CI: run the repo's own dependency-free gate (`make check` = clean + portable +# CPU build + C unit suites + Python stdlib tests) on the three claimed +# platforms. No model downloads, no CUDA, no external deps — by design (#140). +name: check + +on: + push: + branches: [main, dev] + pull_request: + branches: [main, dev] + +jobs: + linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: make check + run: make -C c check + + windows: + # The job that would have caught #68/#137 pre-merge: native MinGW-w64 + # (MSYS2/UCRT64), the exact toolchain the README's Windows port targets. + runs-on: windows-latest + defaults: + run: + shell: msys2 {0} + steps: + - uses: actions/checkout@v4 + - uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: false + install: >- + make + mingw-w64-ucrt-x86_64-gcc + mingw-w64-ucrt-x86_64-python + - name: make check + run: make -C c check + + macos: + # clang; libomp for the threaded path (Makefile falls back to + # single-threaded automatically if it's ever missing). + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - name: install libomp + run: brew install libomp + - name: make check + run: make -C c check diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e18f1b3..72b7149 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,78 @@ jobs: -Xcompiler=-Wall,-Wextra echo "CUDA syntax check passed" + windows-cuda-build: + # The Windows CUDA path has no coverage anywhere: `engine-cuda-syntax` above + # compiles with nvcc's *GCC* host on Linux, and check.yml's windows job is + # MinGW/UCRT64 CPU-only by design (#140). But nvcc on Windows requires MSVC + # as its host compiler — it does not accept MinGW — so `make cuda-dll` runs a + # toolchain nothing else in CI touches. That gap is not theoretical: every + # bug in #158 was a *build* failure on this path (MSVC rejects the GCC-style + # -Xcompiler=-Wall,-Wextra with "D8021 invalid numeric argument '/Wextra'", + # unresolvable CUDA_HOME/NVCC defaults, POSIX setenv in the kernel test), and + # #314 was CUDA_HOME with spaces — the layout the CUDA installer ships by + # default. Both classes are compile-time and need no GPU to catch. + # + # Build-only ON PURPOSE: GitHub's hosted runners have no NVIDIA device, so + # this job proves the Windows+MSVC CUDA build stays buildable, NOT that the + # kernels or the DLL loader behave on real silicon. That still needs hardware + # (see #157). Claiming otherwise would be the false confidence the + # engine-cuda-syntax comment above already warns about. + name: CUDA build (Windows, MSVC host) + # windows-2022, NOT windows-latest: the latest image now ships Visual Studio + # 18 (MSVC 14.5x), and CUDA's crt/host_config.h hard-errors on any host newer + # than VS 2022 ("Only the versions between 2017 and 2022 (inclusive) are + # supported"). That is a real constraint for every CUDA user on Windows, not + # a CI quirk — pinning tracks what the toolkit actually supports. Revisit when + # a CUDA release accepts VS 18; -allow-unsupported-compiler would only mask it. + runs-on: windows-2022 + steps: + - uses: actions/checkout@v4 + - name: MSVC environment (puts cl.exe on PATH for nvcc -ccbin) + uses: ilammy/msvc-dev-cmd@v1 + - name: Install CUDA toolkit (compiler only) + uses: Jimver/cuda-toolkit@v0.2.19 + with: + # Same pin as engine-cuda-syntax: v0.2.19's version table stops at + # 12.6.2. This installs to the default "C:\Program Files\NVIDIA GPU + # Computing Toolkit\..." path, so CUDA_HOME is space-bearing here — + # which is exactly the #314 regression this job would have caught. + cuda: '12.6.2' + method: network + # cudart as well as nvcc: unlike the Linux job (which only needs to + # *compile* backend_cuda.cu), cuda-dll links it, and on Windows the + # runtime headers/import lib ship as a separate installer component — + # with '["nvcc"]' alone this fails at `#include `. + sub-packages: '["nvcc", "cudart"]' + - uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: false + # inherit: cl.exe (msvc-dev-cmd) and nvcc (cuda-toolkit) are added to + # the *Windows* PATH by the steps above; without inheriting it the + # recipe's `command -v` guards fail inside the MSYS2 shell. + path-type: inherit + install: >- + make + mingw-w64-ucrt-x86_64-gcc + - name: make cuda-dll (nvcc + MSVC host) + shell: msys2 {0} + run: | + cd c + # CUDA_ARCH is pinned: the default is `native`, which asks the driver + # what card is present — there is none here, so it must be explicit. + # sm_80 matches engine-cuda-syntax and is supported by the 12.6 pin. + make cuda-dll CUDA_ARCH=sm_80 + test -f coli_cuda.dll || { echo "cuda-dll reported success but produced no DLL" >&2; exit 1; } + echo "coli_cuda.dll built (MSVC host)" + - name: make glm CUDA_DLL=1 (host links backend_loader, not cudart) + shell: msys2 {0} + run: | + cd c + make glm CUDA_DLL=1 + test -f glm.exe || { echo "glm CUDA_DLL=1 reported success but produced no exe" >&2; exit 1; } + echo "glm.exe built against the DLL loader" + web: name: Web UI runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..245d24f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,99 @@ +name: Release + +on: + push: + tags: ['v*'] + +permissions: + contents: write + +jobs: + build: + strategy: + matrix: + include: + - os: ubuntu-latest + name: linux-x86_64 + ext: "" + make_args: "ARCH=x86-64-v3" + - os: macos-latest + name: macos-arm64 + ext: "" + make_args: "" + - os: windows-latest + name: windows-x86_64 + ext: ".exe" + make_args: "" + shell: msys2 + runs-on: ${{ matrix.os }} + defaults: + run: + shell: ${{ matrix.shell || 'bash' }} + steps: + - uses: actions/checkout@v4 + + - if: matrix.shell == 'msys2' + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: false + install: >- + make + mingw-w64-ucrt-x86_64-gcc + + - if: matrix.os == 'macos-latest' + run: brew install libomp + + - name: Build engine + run: | + cd c + make glm ${{ matrix.make_args }} + ls -lh glm${{ matrix.ext }} + + - name: Package + run: | + TAG=${GITHUB_REF#refs/tags/} + mkdir -p dist + cp c/glm${{ matrix.ext }} dist/colibri-${TAG}-${{ matrix.name }}${{ matrix.ext }} + cp c/coli dist/ + cp c/version.py dist/ + cp c/openai_server.py dist/ + cp c/resource_plan.py dist/ + cp c/doctor.py dist/ + cp LICENSE dist/ + cd dist + if [ "${{ matrix.ext }}" = ".exe" ]; then + 7z a colibri-${TAG}-${{ matrix.name }}.zip * + else + tar czf colibri-${TAG}-${{ matrix.name }}.tar.gz * + fi + + - uses: actions/upload-artifact@v4 + with: + name: colibri-${{ matrix.name }} + path: dist/colibri-*.* + + release: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + path: artifacts + merge-multiple: true + + - name: Create GitHub Release + env: + GH_TOKEN: ${{ github.token }} + run: | + TAG=${GITHUB_REF#refs/tags/} + # Extract the latest version section from CHANGELOG + NOTES=$(awk "/^## \\[${TAG#v}\\]/{found=1;next} /^## \\[/{if(found)exit} found{print}" CHANGELOG.md) + if [ -z "$NOTES" ]; then + NOTES="Release ${TAG}" + fi + gh release create "$TAG" artifacts/* \ + --title "colibrì ${TAG}" \ + --notes "$NOTES" diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6177966 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,53 @@ +# Changelog + +All notable changes to colibrì are documented here. +Format follows [Keep a Changelog](https://keepachangelog.com/). + +## [1.0.0] — 2026-07-19 + +First tagged release. The engine has been running in production since late June +2026; this tag marks the baseline for semantic versioning going forward. + +### Highlights + +- **GLM-5.2 (744B MoE)** runs on ~25 GB RAM in pure C, streaming experts from disk +- **Three-tier placement**: VRAM (hot) / RAM (warm) / NVMe (cold), with a learning + cache that pins your workload's hottest experts automatically +- **CUDA backend**: multi-GPU expert tier, dense tensor distribution, batched + ragged attention, resident pipeline (`COLI_CUDA_PIPE=2`) +- **Metal backend** (Apple Silicon): batched expert SwiGLU + fused decode attention + on unified memory GPU +- **MTP speculation**: native GLM-5.2 draft heads, grammar-forced drafts, kernel- + pinned verification (`SPEC_PIN=1`) +- **OpenAI-compatible API**: `coli serve` with SSE streaming, KV slots, bounded + queue, web dashboard (`coli web`) +- **Web UI**: chat with live metrics, expert cortex brain page, profiling breakdown, + expert atlas 3-D galaxy +- **Cross-platform**: Linux, macOS, Windows 11 (native MinGW), PowerPC; CI on all three +- **Auto-tune**: `coli plan --auto-tier` classifies the bottleneck and derives + MTP/PIPE/NUMA/PIN settings with explanations + +### Engine + +- Token-exact validation against `transformers` oracle (teacher-forcing 32/32) +- Compressed MLA KV cache (576 floats/token, 57× smaller), persisted across + restarts (`.coli_kv`, zero re-prefill) +- DSA sparse attention (lightning indexer), faithfully implemented +- Router-lookahead prefetch (`PILOT=1`, 71.6% predictive) +- Async expert I/O pool (`PIPE=1`), io_uring batching (`URING=1`) +- NUMA-aware expert placement (`COLI_NUMA=1`, +13–40% on multi-socket) +- AVX2 / AVX-512 / AVX-VNNI / ARM NEON / NEON-i8mm / POWER VSX kernels +- int4 / int8 / int2 / grouped-int4 (fmt=4) quantization formats + +### Tools + +- `coli convert` — FP8→int4 one-shard-at-a-time converter +- `coli doctor` — read-only setup diagnostics +- `coli plan` — resource planner with auto-tune prescription +- `coli bench` — MMLU / HellaSwag / ARC quality benchmarks +- Expert atlas (`tools/analyze.py --web`) — measured topic affinity for 19,456 experts + +### Community + +- 30+ hardware datapoints in the benchmark tracker +- Contributions from 20+ authors across engine, docs, tooling, and ports diff --git a/README.md b/README.md index 2e22048..797f1ca 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,16 @@ colibrì — tiny engine, immense model

+

+ English · 繁體中文 +

+ **Tiny engine, immense model.** Run **GLM-5.2 (744B-parameter MoE)** on a consumer machine with ~25 GB of RAM — in pure C, with zero dependencies, by streaming experts from disk. -Colibrì is a lightweight, quality-preserving MoE runtime that treats VRAM, -RAM, and storage as one managed memory hierarchy. Insufficient fast memory may -reduce speed, but the default policy never silently changes model precision or -router semantics. +Colibrì is a lightweight, quality-preserving MoE runtime that treats VRAM, RAM, +and storage as one managed memory hierarchy. Insufficient fast memory may reduce +speed, but the default policy **never silently changes model precision or router +semantics**. ``` $ ./coli chat @@ -17,14 +21,14 @@ $ ./coli chat ◆ Ciao! 😊 Come posso aiutarti oggi? ``` - ## See it running

colibrì web dashboard — live metrics, hardware panel, expert tiers

-

The web dashboard (./coli web): a 744B model answering at 4+ tok/s end-to-end on 6× RTX 5090 — -with live token metrics, the hardware panel, and the VRAM/RAM/disk expert tiers.

+

The web dashboard (./coli web): a 744B model at 4 tok/s, TTFT 1.6 s, disk 0 — +full expert residency on 6× RTX 5090, with live token metrics, the per-turn time breakdown, +the VRAM/RAM/disk tier bar and the live mini-brain in the corner.

the Brain page — 19,456 experts as a live cortex @@ -33,613 +37,174 @@ with live token metrics, the hardware panel, and the VRAM/RAM/disk expert tiers. brightness is routing heat, and every expert routed in a turn flashes white. Hovering shows the expert's measured topic affinity.

-## Contents - -- [The idea](#the-idea) -- [See it running](#see-it-running) -- [What's implemented](#whats-implemented) -- [Honest numbers](#honest-numbers-wsl2-12-cores-25-gb-ram-nvme-via-vhdx) -- [Download the model](#download-the-model) -- [Web dashboard](#web-dashboard) -- [Got a better machine?](#got-a-better-machine-try-it--heres-what-to-expect) +

+ the Atlas page — the measured expert atlas as a 3-D galaxy +

+

The Atlas page: the measured expert atlas +as a 3-D galaxy — 13,260 characterised experts, 1,041 replicated specialists clustering by topic +(poetry, law, Chinese, SQL…). Position is measured routing affinity, not a learned embedding. Drag to spin.

## The idea -A 744B Mixture-of-Experts model activates only ~40B parameters per token — and only ~11 GB of those change from token to token (the routed experts). So: +A 744B Mixture-of-Experts model activates only ~40B parameters per token — and +only ~11 GB of those change from token to token (the routed experts): -- the **dense part** (attention, shared experts, embeddings — ~17B params) stays **resident in RAM at int4** (~9.9 GB); -- the **19,456 routed experts** (75 MoE layers × 256 experts + the MTP head, ~19 MB each at int4) live **on disk** (~370 GB) and are **streamed on demand**, with a per-layer LRU cache, an optional pinned hot-store, and the OS page cache as a free L2. +

+ only ~5.4% of parameters are active per token +

-The engine is a single C file (`c/glm.c`) plus small headers. No BLAS, no Python at runtime, no GPU required (an opt-in CUDA tier for pinned experts exists — see below). +So the model doesn't need to *fit* in fast memory — it needs to be **placed**: -## What's implemented +- the **dense part** (attention, shared experts, embeddings — ~17B params) stays + **resident in RAM at int4** (~9.9 GB); +- the **19,456 routed experts** (75 MoE layers × 256 + the MTP head, ~19 MB each + at int4) live **on disk** (~370 GB) and are **streamed on demand**, with a + per-layer LRU cache, a learned pinned hot-store, and an optional VRAM tier. -- **Faithful GLM-5.2 (`glm_moe_dsa`) forward** — validated token-exact against a `transformers` oracle (teacher-forcing 32/32, greedy 20/20 on a tiny-random model with the real architecture). -- **MLA attention** (q/kv-LoRA, interleaved partial RoPE) with **compressed KV-cache**: 576 floats/token instead of 32,768 (57× smaller — GLM-5.2 has 64 heads and no GQA). -- **DeepSeek-V3-style sigmoid router** (noaux_tc, routed_scaling_factor), shared expert, first-3-dense layers. -- **Native MTP speculative decoding** — GLM-5.2's own multi-token-prediction head (layer 78) drafts tokens that the main model verifies in one batched forward. **The head must be int8** (the converter does this by default): at int4 draft acceptance collapses to 0–4% and speculation never engages; at int8 it's 39–59% acceptance, **2.2–2.8 tokens/forward** (community-measured, [#8](https://github.com/JustVugg/colibri/issues/8)). Lossless *in exact arithmetic* — but **not byte-identical to non-speculative greedy in practice** ([#100](https://github.com/JustVugg/colibri/issues/100)). This isn't MTP-specific: colibrì's quantized integer kernels are shape-dependent, so any batched (S>1) or GPU forward rounds slightly differently from the single-token path, and int4 GLM-5.2 sits close enough to argmax ties that such a rounding change can flip a token. MTP, the CUDA expert tier, and batched prefill are three different ways to trip the same sensitivity (community-confirmed in #100: swapping only the kernel family forks greedy output on 3/5 prompts, with **zero speculation**). Every emitted token is still the argmax of a *valid* forward — the continuation stays correct — it just isn't the same stream. For byte-exact reproducibility: `DRAFT=0` (no speculation), plus `IDOT=0 COLI_CUDA=0` if you also want kernel-family/GPU independence. Under sampling, rejection sampling keeps the distribution correct. Honest caveat from the same measurement: on a **cold** cache each verified draft routes to extra experts (~660 → ~1100 expert-loads/token), so speculation can be a net *time* loss until the cache/pin warms up. -- **Grammar-forced speculative drafts** (`GRAMMAR=file.gbnf`, [#48](https://github.com/JustVugg/colibri/issues/48)) — on constrained-output workloads (JSON/NDJSON, function calling, structured extraction) the grammar itself is a third draft source: wherever it admits exactly **one** legal byte (braces, quotes, key names, enum bodies), that forced span is tokenized and injected as pre-accepted drafts with ~1.0 acceptance — no draft head, no lookup table, and it engages even with the int4 MTP head from [#8](https://github.com/JustVugg/colibri/issues/8). It never constrains sampling: forced spans are verified in the same batch-union forward as any draft, so a wrong or out-of-sync grammar cannot change the output — worst case is rejected drafts, and an adaptive guard turns the source off below 50% acceptance. Byte-level GBNF subset (literals, char classes, `| ( ) ? * +`, comments); `GRAMMAR_DRAFT=n` caps the forced span per forward (default 24). Composes with `DRAFT`/MTP, which fill the free-text gaps between forced spans. Full reference — mechanism, measured A/Bs, when it pays, prior art: [docs/grammar-draft.md](docs/grammar-draft.md). -- **True sampling** — temperature + nucleus, defaults tuned for int4 reality (0.7 / 0.90; the official 1.0 / 0.95 samples quantization noise from the tail). -- **Integer-dot kernels** (Q8_0-style int8 activations, AVX2 `maddubs`): int8 matmuls 1.4–2.5× faster (119 GFLOP/s measured), int4 1.8× in batch — routing decided per shape by measurement (int4 single-row stays f32: it measured slower). -- **MLA weight absorption** (DeepSeek trick) for decode: no per-token k/v reconstruction — the query absorbs `kv_b`, context is projected after attention. Validated exact: TF 32/32 and generation 20/20 with absorption forced everywhere. -- **Async expert readahead**: while one block of experts is being multiplied, the kernel is already reading the next (`WILLNEED`). -- **Quantization kernels**: int8 / packed int4 / packed int2, per-row scales, AVX2, dequant-on-use. Packing validated bit-identical to the int8 container. -- **DSA sparse attention** — GLM-5.2's lightning indexer, faithful to the reference `glm_moe_dsa` modeling: per-layer top-2048 causal key selection (full/shared indexer layers), auto-detected from the `out-idx-*` weights (`--indexer` converter mode, ~189 MB extracted from the FP8 repo). Validated exact: forcing the selection to keep every key reproduces dense attention token-for-token. `DSA=0` disables, `DSA_TOPK` overrides. -- **KV-cache persistence** — conversations reopen **warm** across engine restarts: serve mode appends the compressed MLA KV to `.coli_kv` after every turn (~182 KB/token, crash-safe) and resumes it at startup with zero re-prefill. Validated byte-identical to an uninterrupted session. `KVSAVE=0` disables. -- **Router-lookahead prefetch** (`PILOT=1`, experimental) — the next layer's routing is 71.6% predictable from the current layer's post-attention state (measured); a dedicated I/O thread prefetches those experts while the current layer computes. -- **Batch-union MoE**: in prefill (and MTP verification), each unique expert of the batch is read once and applied to every position that routes to it. -- **Byte-level BPE tokenizer in C** (GPT-2-style with Unicode-property regex, 320k merges). -- **RAM safety**: the expert cache is auto-sized from `MemAvailable` at startup — an honest peak projection (working set, KV, MTP row, reconstruction buffers) so the kernel OOM-killer never fires. -- **Offline FP8→int4 converter** (`c/tools/convert_fp8_to_int4.py`): downloads one shard at a time (~5 GB), dequants (128×128 block scales), requantizes to the engine's container, deletes the shard — the 756 GB FP8 checkpoint never needs to exist on disk at once. Resumable. +The engine is a single C file (`c/glm.c`) plus small headers. No BLAS, no Python +at runtime, no GPU required. -## Honest numbers (WSL2, 12 cores, 25 GB RAM, NVMe via VHDX) +## How it works -Detailed GPU experiment: [GLM-5.2 on 6x RTX 5090](docs/experiments/glm52-6x5090-2026-07-12.md) — full expert residency across VRAM+RAM reaches 6.84 tok/s single-request decode. +### The per-token path -| metric | value | -|---|---| -| model on disk (int4 container) | ~370 GB | -| resident RAM (dense, int4) | 9.9 GB | -| load time | ~30 s | -| peak RSS during chat | ~20 GB (auto-capped) | -| cold decode cost | ~11 GB disk reads/token (75 layers × 8 experts) | -| disk ceiling (this dev box's drive) | ~1 GB/s → ~0.05–0.1 tok/s cold | -| MTP speculation (int8 head) | 2.2–2.8 tok/forward measured ([#8](https://github.com/JustVugg/colibri/issues/8)) | +

+ route → union → place → overlap → learn +

-This is not fast. It is a 744B frontier-class model **answering correctly on a machine that costs less than one H100 fan**. Warm cache, pinned hot experts and MTP push the useful-response latency down considerably; the physics of the disk does the rest. +Every layer of every token walks the same five steps. The design goal is that +**placement only ever decides speed** — the router's decisions and the weights' +precision are the same whether an expert answered from VRAM or from disk. -### SSD note -Cold starts are heavy on random reads (~11 GB/token), but reads don't meaningfully wear an SSD — colibrì's streaming is read-only. The real concerns under heavy use are (1) **swap traffic** if the system runs out of RAM (writes do wear the drive — keep a sane `--ram` budget; colibrì's auto-budget is designed to stay clear of swap) and (2) **sustained thermals**: hours at full read duty cycle will heat cheaper drives. Monitor drive temperature and health. +### One memory hierarchy instead of one memory requirement -## Download the model +

+ VRAM / RAM / NVMe three-tier expert residency +

-A pre-converted **GLM-5.2 int4** model for colibrì is available on Hugging Face — **use the version with the int8 MTP heads** (matey-0's clone): +The same engine spans the whole range: on a 25 GB laptop everything streams from +disk (slow but correct); on a large host the entire expert set becomes resident +(`CUDA_EXPERT_GB=auto PIN_GB=all`) and disk drops out of the decode path +entirely. Between the tiers sits a **learning cache**: the engine records which +experts *your* workload routes to (`.coli_usage`, updated every turn) and pins +the hottest ones automatically — colibrì literally gets faster the more you use +it. On multi-socket hosts, `COLI_NUMA=1` interleaves the resident weights across +memory controllers ([#82](https://github.com/JustVugg/colibri/issues/82)). + +### Never wait for the disk twice + +Misses are expensive, so the engine spends most of its cleverness avoiding and +overlapping them: each expert's three matrices are stored adjacent and read in +one `pread`; a bounded async I/O pool (`PIPE=1`, default) loads missing experts +while resident ones compute; batched positions read each unique expert once +(**batch-union**); and a router-lookahead thread (`PILOT=1`) prefetches the next +layer's experts — routing is measurably **71.6% predictable one layer ahead**. +On GPUs, the resident pipeline (`COLI_CUDA_PIPE=2`) keeps the residual stream +on-device across layers so the CPU expert loop runs uninterrupted; on Apple +Silicon an experimental [Metal backend](docs/metal.md) does the batched expert +math on the unified-memory GPU. + +### Faithful model, compressed state + +The forward pass is validated **token-exact against a `transformers` oracle** +(teacher-forcing 32/32). MLA attention stores a compressed KV state — 576 +floats/token instead of 32,768 (**57× smaller**) — and persists it across +restarts (`.coli_kv`): conversations reopen warm with zero re-prefill, +byte-identical to an uninterrupted session. DSA sparse attention (GLM-5.2's +lightning indexer) is implemented faithfully and validated by forcing full-key +selection to reproduce dense attention exactly. + +### Speculative decoding, honestly + +GLM-5.2's native MTP head drafts tokens that the main model verifies in one +batched forward — 2.2–2.8 tokens/forward when it pays. Two hard-won rules ship +as defaults: the MTP head must be **int8** (int4 heads collapse to 0–4% +acceptance, [#8](https://github.com/JustVugg/colibri/issues/8)), and draft and +verify must compute **the same function** — `SPEC_PIN=1` pins both to one +kernel family ([#163](https://github.com/JustVugg/colibri/issues/163) is the +full forensic story). Grammar-forced drafts +([`GRAMMAR=file.gbnf`](docs/grammar-draft.md)) add ~free acceptance on +constrained JSON output. Whether speculation is a net win depends on your +cache temperature — measure, and use `DRAFT=0` when it doesn't pay. + +## What it achieves + +

+ measured decode speed by hardware class +

+ +Same engine, same int4 container — the hardware only changes where the experts +live. Highlights from the [full benchmark tables](docs/benchmarks.md): + +- **6× RTX 5090, full residency:** 5.8–6.8 tok/s decode, TTFT ~13 s + ([experiment log](docs/experiments/glm52-6x5090-2026-07-12.md)); +- **128 GB CPU-only desktop:** ~1.8 tok/s warm ([#200](https://github.com/JustVugg/colibri/issues/200)); +- **single RTX 5070 Ti laptop-class box:** 1.07 tok/s via the GPU-resident + pipeline ([#273](https://github.com/JustVugg/colibri/issues/273)); +- **25 GB dev box:** 0.05–0.1 tok/s cold — the proven floor where this project + started, and still the honest baseline. + +Quality is measured, not assumed: the int4 container's quantization cost and the +scale-granularity/rotation ablations live in +[docs/benchmarks.md](docs/benchmarks.md#quality-benchmark) and +[#108](https://github.com/JustVugg/colibri/issues/108)/[#81](https://github.com/JustVugg/colibri/issues/81). + +## Get started + +### 1. Get the model + +A pre-converted **GLM-5.2 int4** container is on Hugging Face — **use the +version with the int8 MTP heads**: **https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** -> ⚠️ **The MTP head must be int8.** The original mirror ([jlnsrk/GLM-5.2-colibri-int4](https://huggingface.co/jlnsrk/GLM-5.2-colibri-int4)) ships **int4** MTP heads, which give **0% draft acceptance** — speculation silently never engages and you lose the ~2× MTP lever. This is the single most common "why is MTP stuck at 0%?" report ([#8](https://github.com/JustVugg/colibri/issues/8), [#102](https://github.com/JustVugg/colibri/issues/102)). The int8 head gives the measured **39–59% acceptance**. matey-0's clone above is the original int4 model with the three `out-mtp-*` files already swapped to int8 — download that one and you're done. -> -> Check what you have: `ls -l /out-mtp-*` -> · **int8 (correct):** `3527131672 / 5366238584 / 1065950496` -> · **int4 (0% acceptance):** `1765523544 / 2686077736 / 536747200` — if you see these, replace just those three files from the int8 mirror. +> ⚠️ The original mirror ships int4 MTP heads → 0% draft acceptance +> ([#8](https://github.com/JustVugg/colibri/issues/8)). Check yours: +> `ls -l /out-mtp-*` — int8 (correct) is `3527131672 / 5366238584 / 1065950496`. -Download the repository and point `COLI_MODEL` to its directory: +Or convert from the FP8 source yourself — one resumable command that never needs +the full 756 GB on disk at once: ```bash -COLI_MODEL=/path/to/GLM-5.2-colibri-int4-with-int8-mtp ./coli chat +cd c && ./setup.sh # checks gcc/OpenMP, builds, self-tests +./coli convert --model /nvme/glm52_i4 # download+convert shard by shard (python, one-time) ``` -This skips the FP8 → int4 conversion step entirely. Thanks to DatPat for the original mirror and matey-0 for the int8-head clone. - -### Quick start +### 2. Run it ```bash -cd c -./setup.sh # checks gcc/OpenMP, builds, self-tests - -# ONE command does everything model-side: downloads GLM-5.2-FP8 shard by shard -# (never needs the full 756 GB at once), converts to the int4 container, then -# converts the MTP head for speculative decoding. Resumable at any point. -# Conversion (only) needs python with: pip install torch safetensors huggingface_hub numpy -./coli convert --model /nvme/glm52_i4 # ~400 GB free on a real ext4/NVMe path - -# chat — RAM budget, expert cache and MTP are all detected automatically: -COLI_MODEL=/nvme/glm52_i4 ./coli chat +COLI_MODEL=/nvme/glm52_i4 ./coli chat # RAM budget, cache and MTP auto-detected +COLI_MODEL=/nvme/glm52_i4 ./coli plan # inspect the planned VRAM/RAM/disk placement +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # read-only readiness check +./coli web --model /nvme/glm52_i4 # API + web dashboard on one port +./coli serve --model /nvme/glm52_i4 # OpenAI-compatible API only ``` -Inspect the planned storage hierarchy before loading the model: +The engine at runtime is pure C — python is only used by the one-time converter +and the optional API gateway. -```bash -COLI_MODEL=/nvme/glm52_i4 ./coli plan -COLI_MODEL=/nvme/glm52_i4 ./coli plan --gpu 0,1 --ram 128 --vram 48 --json +### 3. Go deeper -# apply the bounded plan to the normal runner -COLI_MODEL=/nvme/glm52_i4 ./coli chat --auto-tier -``` - -`coli plan` reads only safetensors headers and reports the model's exact dense/expert -footprint, runtime RAM reserve, safe expert-cache cap, and bounded VRAM hot tier. Its -versioned JSON output is intended to be shared by the CLI, API server, Web UI, and -desktop shell; it does not allocate model tensors or start inference. -`--auto-tier` applies the same plan to `chat`, `run`, `serve`, and benchmarks. It -sets the RAM budget and context immediately; the VRAM tier is enabled only when -the current `glm` binary is linked with CUDA. Explicit flags and environment -variables keep precedence over automatic values. - -Before loading the model, `coli doctor` performs a read-only readiness check and -explains whether the selected Disk/RAM/VRAM placement is runnable: - -```bash -COLI_MODEL=/nvme/glm52_i4 ./coli doctor -COLI_MODEL=/nvme/glm52_i4 ./coli doctor --gpu 0 --ram 128 --json -``` - -Doctor validates the model directory, config, tokenizer, safetensors headers, -engine executable, available RAM, requested NVIDIA devices, CUDA linkage, and the -same placement budget used by `coli plan`. It never starts `glm`, reads tensor -payloads, imports a model framework, or creates a CUDA context. The versioned JSON -report uses stable check IDs for automation. Warnings keep exit status 0; missing -requirements or an unsafe RAM projection return 1, while invalid CLI values return 2. - -The engine at runtime is pure C — python is only used by the one-time converter. - -### Windows 11 (native, no WSL) - -colibrì builds and runs natively on Windows 11 x86-64 with MinGW-w64. The port adds -a `_WIN32` compatibility layer in `c/compat.h` that maps POSIX I/O to the Windows API -(pread → ReadFile+OVERLAPPED, posix_fadvise no-op, aligned allocation, MoveFileEx rename, -GlobalMemoryStatusEx RAM detection). All platform differences stay in `compat.h`; the -engine source is unchanged. - -**Toolchain:** GCC via [winlibs](https://winlibs.com/) or MSYS2 MinGW-w64. Tested with -GCC 16.1.0 (x86_64-ucrt-posix-seh). - -```powershell -# One-time toolchain install (pick one): -scoop install mingw-winlibs # portable, no shell needed -# or: pacman -S mingw-w64-x86_64-gcc make # via MSYS2 - -# Build (from c/ directory): -make glm.exe # GLM-5.2 engine (static, no DLL dependencies) -make olmoe.exe # OLMoE engine (same shims) -make iobench.exe # disk I/O benchmark -make test-c # run C tests -make test-python # run Python tests (requires python) - -# AVX-VNNI: Intel Alder Lake+ (and Meteor Lake+) CPUs have a 128-bit int8 -# dot-product instruction (VPDPBUSD) the engine can use for ~1.3x faster -# quantized matmul. The x86-64-v3 default (portable AVX2) compiles it out; -# build for THIS machine to enable it: -make glm.exe ARCH=native # banner prints "idot: avx-vnni" - -# Verify (tiny model, 2.4 MB): -pip install torch transformers safetensors huggingface_hub -python tools/make_glm_oracle.py # generate tiny oracle -SNAP=./glm_tiny TF=1 ./glm.exe 64 16 16 # expect "32/32 positions" - -# Run with real model: -SNAP=D:\glm52_i4 ./glm.exe 64 4 16 # batch inference -python coli chat --model D:\glm52_i4 # interactive chat -python coli serve --model D:\glm52_i4 # OpenAI-compatible API -``` - -**Warmup (overnight cache priming):** the engine's expert cache learns from -your workload. The included `warmup.ps1` script runs `coli run` in a loop with -diverse prompts to build the `.coli_usage` histogram unattended, so the next -real session starts with a large, accurate hot-expert pin. Each run saves usage -atomically on clean completion. - -```powershell -.\warmup.ps1 -Rounds 1 -Ngen 32 # ~60-90 min, durable progress -``` - -**NVIDIA GPU (optional, via runtime DLL):** on Windows the engine is built with -MinGW gcc but CUDA kernels require MSVC + nvcc. The split is clean: build the -CUDA backend into a standalone `coli_cuda.dll` (nvcc + MSVC), then the host -`glm.exe` loads it at runtime via `LoadLibrary` (`c/backend_loader.c`). The host -never links cudart directly; if the DLL is absent the engine falls back to CPU -without error. - -```powershell -# Prerequisites: CUDA Toolkit + MSVC Build Tools (cl.exe) + nvcc on PATH. -# Build the DLL from a shell with the MSVC environment set (vcvars64.bat or -# "x64 Native Tools Command Prompt for VS"): -make cuda-dll CUDA_HOME="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" CUDA_ARCH=sm_120 - -# Build the host with the runtime loader (CUDA_DLL=1 adds -DCOLI_CUDA and -# links backend_loader.o instead of cudart): -make glm.exe CUDA_DLL=1 ARCH=native - -# Run with the GPU expert tier (8 GB VRAM budget here; scale to your free VRAM): -$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_EXPERT_GB="8" -python coli chat --model D:\glm52_i4 --topp 0.7 -``` - -The DLL exports 11 `extern "C"` symbols (`coli_cuda_init`, `coli_cuda_matmul`, -etc.); `backend_loader.c` resolves them via `GetProcAddress` on first use. -`ColiCudaTensor*` is opaque to the host (stored, never dereferenced), so the -MSVC-allocated struct is safe across the ABI boundary. `CUDA_ARCH` must match -your GPU's compute capability (e.g. `sm_120` for Blackwell / RTX 50-series, -`sm_89` for Ada / RTX 40-series). - -**Status:** Phase 1 complete (compiles, correct, static-linked). The Windows -GPU tier (runtime `coli_cuda.dll` via `LoadLibrary`) is implemented and -verified on RTX 50-series (sm_120). O_DIRECT (Phase 2) and full-model -validation against the transformers oracle remain separate workstreams. - -### OpenAI-compatible API - -`coli serve` keeps one model process loaded and exposes a text-only OpenAI-compatible -HTTP API. The gateway uses only the Python standard library; inference still runs in -the same dependency-free C engine. - -```bash -cd c -COLI_MODEL=/nvme/glm52_i4 COLI_API_KEY=local-secret ./coli serve \ - --host 127.0.0.1 --port 8000 --model-id glm-5.2-colibri - -curl http://127.0.0.1:8000/v1/chat/completions \ - -H 'Authorization: Bearer local-secret' \ - -H 'Content-Type: application/json' \ - -d '{ - "model": "glm-5.2-colibri", - "messages": [{"role": "user", "content": "Hello"}], - "stream": true - }' -``` - -Implemented endpoints are `GET /v1/models`, `GET /v1/models/{model}`, -`POST /v1/chat/completions`, and legacy `POST /v1/completions`. Chat and -completion requests support JSON responses, SSE streaming, usage counts, -`max_tokens`/`max_completion_tokens`, `temperature`, and `top_p`. The extension -`enable_thinking: true` enables GLM-5.2's reasoning block; the standard -`reasoning_effort` field also enables it unless set to `none`. - -The first version is deliberately text-only and serves one generation at a time: -the 744B model stays in one persistent process, so concurrent HTTP requests queue -instead of loading duplicate model copies. Tools, image/audio input, custom stop -sequences, log probabilities, and token penalties return an explicit error rather -than being silently ignored. The default bind address is localhost; set -`COLI_API_KEY` before exposing the server beyond the machine. - -Browser access from the Vite development server and Tauri local origins is enabled -by default. Repeat `--cors-origin https://your-ui.example` to allow another exact -origin, or use `--cors-origin '*'` only on a trusted local network. - -The engine owns one mutable KV context, so HTTP generation uses a bounded FIFO -admission queue instead of pretending to run unsafe parallel sequences. Configure it -with `--max-queue N` (default 8) and `--queue-timeout SECONDS` (default 300), or the -`COLI_MAX_QUEUE` / `COLI_QUEUE_TIMEOUT` environment variables. Saturated and timed-out -requests receive OpenAI-shaped HTTP 429 errors before streaming headers are sent. -`GET /health` exposes active/queued/completed/rejected counters, and successful -generation responses include `x-colibri-queue-wait-ms`. - -### Isolated KV contexts - -`coli serve --kv-slots N` allocates up to 16 independent sequence contexts. Requests -select one with the optional integer `cache_slot` field; ordinary OpenAI clients omit -it and keep the original slot 0 behavior. - -```json -{ - "model": "glm-5.2-colibri", - "messages": [{"role": "user", "content": "Continue this conversation"}], - "cache_slot": 1 -} -``` - -Each slot owns its token history, compressed MLA/DSA KV memory, MTP window, and -crash-safe persistence file (`.coli_kv`, `.coli_kv.1`, ...). The engine still executes -one sequence at a time; this establishes explicit KV ownership without pretending that -threaded HTTP is continuous batching. RAM admission accounts for every configured slot. -Use `COLI_KV_SLOTS=N` as the environment equivalent. Start with a small value: at the -default 4096-token context, every slot costs hundreds of MB. - -### Experimental Metal backend (Apple Silicon) - -On Apple Silicon the decode profile is matmul-bound, and unified memory removes the -PCIe copy tax that keeps CUDA's streaming experts on the CPU — so colibrì has an -opt-in Metal backend that runs the **routed-expert SwiGLU (batched, zero-copy from -the RAM slabs)**, the **fused decode attention** (full MLA layer in one command -buffer, S≤4), and **prefill's large GEMMs** on the GPU. Token-exact vs the CPU path. - -```bash -cd c -make glm METAL=1 # macOS only; no Xcode needed (shader compiles at runtime) -make metal-test # standalone kernel/attention correctness vs CPU reference -COLI_METAL=1 COLI_MODEL=/path/glm52_i4 ./coli chat --ram 96 -``` - -Measured on an M4 Max (128 GB, warm cache, MTP on): CPU 0.30 → Metal **0.42 tok/s (~1.4×)** -(best config adds `DIRECT=1`; ~3× vs this machine's first cold run). -Key design points: Metal's ~5 ms submit latency makes per-matmul dispatch a loss — -everything is batched into few command buffers per layer, and the resident experts' -GPU work is submitted *before* the missed experts' disk reads so I/O and compute -overlap. `COLI_METAL_GEMM_MIN` tunes the prefill GEMM row threshold (default 16). -Streaming, cache, MTP, DSA and the persistence formats are unchanged; every GPU -path falls back to the CPU per-block on any fault. Numerics are dequant→f32-MAC -(same as the CUDA tier); greedy outputs are byte-identical to the CPU engine. - -### Experimental resident CUDA backend - -colibrì includes an opt-in CUDA backend for model-resident tensors. Streaming -experts deliberately remain on the original CPU path for now: copying an expert -from NVMe to the GPU on every use would only replace the disk bottleneck with a -PCIe bottleneck. Resident quantized tensors are uploaded lazily once and reused. - -```bash -cd c -make cuda-test CUDA=1 # q8/q4/q2/f32 kernel correctness -make CUDA=1 -# optional dense-path experiment (hot experts are configured below) -COLI_CUDA=1 COLI_GPU=0 CUDA_DENSE=1 SNAP=/nvme/glm52_i4 ./glm 64 4 4 -``` - -Requirements: Linux, an NVIDIA driver, and a CUDA Toolkit under -`/usr/local/cuda` (override with `CUDA_HOME=/path/to/cuda`). `CUDA_ARCH=native` -builds for the GPU in the current machine; set an explicit architecture when -cross-compiling. Requesting CUDA with a CPU-only binary, an invalid device, or -an unavailable runtime fails at startup instead of silently falling back. - -The normal `make` build and runtime behavior are unchanged. CUDA defaults to an -expert-only accelerator. `CUDA_DENSE=1` additionally distributes resident -dense/attention projection tensors round-robin across the selected devices; -their projected footprint is reserved before the expert tier is placed. On six -RTX 5090s with a 150 GB expert tier, a warmed two-request/64-token GLM-5.2 run -improved from 1.650 to 2.157 aggregate tok/s (+30.8%) while retaining the full -expert tier. Treat this as an opt-in until the projected dense set and the 2 GB -per-device runtime reserve fit the target GPUs. -A measured `PIN` profile can promote its hottest experts into the persistent -VRAM tier while keeping the rest in RAM: - -```bash -STATS=stats.txt SNAP=/nvme/glm52_i4 ./glm 64 4 4 # collect routing frequencies first -COLI_CUDA=1 COLI_GPU=0 CUDA_EXPERT_GB=16 \ -PIN=stats.txt PIN_GB=160 SNAP=/nvme/glm52_i4 ./glm 64 4 4 -# multi-GPU expert tier, 150 GB total budget across six 32 GB devices -COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=150 \ -CUDA_DENSE=1 PIN=stats.txt PIN_GB=300 RAM_GB=226 \ -SNAP=/nvme/glm52_i4 ./glm 64 4 4 -# large-RAM host: fill safe VRAM, then keep every remaining expert in RAM -COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=auto \ -CUDA_DENSE=1 COLI_CUDA_ATTN=1 PIN=stats.txt PIN_GB=all RAM_GB=auto \ -SNAP=/nvme/glm52_i4 ./glm 64 4 4 -``` - -Selected experts are uploaded during startup, so capacity failures occur before -inference and the log reports their exact tensor footprint. The budget is clamped -against free VRAM after reserving the projected dense resident set and 2 GB of -runtime headroom per selected device. With `COLI_GPUS`, `CUDA_EXPERT_GB` is a -total budget across the device set; experts are assigned whole to the -least-loaded device that can hold them. Multi-GPU runs also default to -`PIN_FILL=1`: the measured hot set is placed first, then unused VRAM is filled -with zero-heat experts. `CUDA_RELEASE_HOST=1` (the multi-GPU default) releases -the RAM copy after a successful upload and reloads it from disk only if CUDA -later fails. Set either variable to `0` to restore the conservative behavior. -When host backing is released, placement is disjoint and staged: the hottest -prefix is loaded, uploaded to VRAM, and freed before the next-ranked suffix is -loaded into RAM. `PIN_GB` therefore describes the combined ranked set rather -than duplicate RAM and VRAM copies. On a 256 GB dual-socket host, moving from a -150 GB VRAM + 130 GB RAM placement to 150 GB VRAM + 150 GB RAM raised fixed-token -replay from 1.87 to 2.16 tok/s (+15.7%), reduced expert disk wait from 5.144s to -3.948s, and kept the projected RAM peak below `RAM_GB=226`. The cache cap adjusts -down automatically (54 to 40 in that run) so the larger pinned tier does not exceed -the process budget. Start lower on hosts with less available RAM. - -`CUDA_EXPERT_GB=auto` fills each selected device only up to its measured free -memory minus projected dense tensors and 2 GB of runtime headroom. `PIN_GB=all` -then loads every remaining routed expert into RAM, eliminating decode-time disk -misses when the host budget permits it. The regular `RAM_GB` guard still clamps -the per-layer working cache and rejects unsafe projections; this mode is intended -for dedicated high-memory inference hosts, not desktops running other workloads. -On a dedicated 251 GiB host with six RTX 5090s, this mode selected a 176.7 GB -VRAM expert tier and a 191.3 GB RAM tier (all 19,456 experts resident). The -mode also adapts the VRAM tier every 16 emitted tokens by swapping hot RAM -experts into existing GPU slots. A real 64-token greedy GLM-5.2 generation -measured **6.00 tok/s decode**, up from -2.20 tok/s end-to-end with the earlier 150 GB tier; expert hit rate was 100% -and disk wait was zero. Prompt prefill is reported separately. This is a -host-specific capacity result, not a portable default. - -Text-mode timing reports prefill separately from decode. The decode rate starts -after the prompt KV is built, so it is comparable to `REPLAY` throughput without -hiding time-to-first-token. -MTP speculation defaults off on CUDA because cold draft routes increase expert -traffic; an explicit `DRAFT=n` still overrides the default. - -On six RTX 5090 32 GB cards with GLM-5.2 int4, a 150 GB hot-first tier sustained -0.94 token/s over a 64-token varied prompt (87.8% expert hit rate), and reached -1.64 token/s on a warmed short prompt (99.3% hit rate). The same capacity filled -without routing heat managed only 0.29 token/s, so profile quality matters more -than raw VRAM capacity. These are single-run engineering measurements, not a -portable performance guarantee. - -Current limitations: devices use independent contexts and synchronous -host-staged activation copies—there is no P2P/NCCL dependency yet. Independent -expert groups execute concurrently across devices, but a single expert is not -sharded. The kernels are correctness-first custom kernels rather than -cuBLAS/Tensor Core kernels. - -For a reproducible backend A/B without the full checkpoint, generate the -deterministic 313M-parameter `glm_moe_dsa` fixture and run fixed-token replay: - -```bash -cd c -python tools/make_glm_bench_model.py --output /nvme/colibri-bench-medium --device cuda -python tools/benchmark_cuda_fixture.py --model /nvme/colibri-bench-medium --gpu 0 -``` - -The fixture has random weights and is not a language model. It exists only to -preserve the real MLA/MoE/streaming shapes and compare CPU streaming, dense-only -CUDA, CPU hot-store, and CUDA hot-expert execution with identical replay tokens. - -### Web interface - -`web/` contains a community-contributed browser UI (React + TypeScript, a pure -API client — it never touches the engine directly): - -```bash -cd web -npm ci && npm run dev # then point it at an OpenAI-compatible endpoint -``` - -It speaks the standard OpenAI Chat Completions protocol with SSE streaming, so it -works against the colibrì OpenAI-compatible server (in review, #21) or any other -compatible endpoint. Nothing leaves the endpoint you configure. The terminal -`coli chat` remains the first-class interface. - -Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens per answer (`:more` in chat continues a truncated one), `--repin N` adapt RAM/VRAM hot experts every N emitted tokens, `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `GRAMMAR=g.gbnf` grammar-forced drafts for constrained JSON/NDJSON output (`GRAMMAR_DRAFT=n` caps the forced span), `TF=1` teacher-forcing validation, `PILOT=1` router-lookahead disk prefetch (experimental — see below), `URING=1` Linux-only batched expert I/O (implies `PIPE=1`; also batches `PILOT_REAL`), `PIPE=0` disable the async expert-load pool (**default ON on Windows** — overlaps expert `pread` with the matmul so the CPU isn't idle waiting on the SSD; measured −18% disk service time), `RAM_GB=` claim more RAM for the expert cache than the conservative auto-detect (e.g. `RAM_GB=31` on a 32 GB host raises the cache cap and hit rate measurably), `CAP_RAISE=0` don't auto-grow the expert cache. - -### Resource policy - -`coli plan` reports the planned hot (VRAM), warm (RAM), and cold backing -(disk) tiers, the reason for each placement, and the expected bottleneck. The -default `--policy quality` and `--policy balanced` modes preserve checkpoint -quantization and router decisions unless `--topk` or `--topp` is passed; those -explicit lossy overrides print a warning and proceed. - -Auto-tier plans size OpenMP from physical cores and bind workers across cores. -Memory-bound quantized kernels can regress sharply when SMT siblings compete -for limited memory channels; explicit `OMP_*` settings always take precedence. - -```bash -coli plan --model /models/glm52_i4 --policy quality -coli run --auto-tier --policy quality "Explain MoE offloading" -# Explicit research-only router reduction: -coli run --policy experimental-fast --topk 4 "Benchmark prompt" -``` - -Disk is an immutable recovery source, not a normal decode target. If the plan -leaves cold expert bytes on disk, speed depends on cache hit rate; output -quality does not. - -Cold expert reads can use a deferred pipeline: resident RAM/VRAM experts execute -while missing experts are loaded in a bounded background I/O pool, then the -cold results join before the layer completes. The pool engages only under -`PIPE=1`; `PIPE_WORKERS=n` sets its worker count (default 8). Profiling reports -both disk service time and the smaller foreground-visible wait time so overlap -is explicit rather than credited as unexplained speedup. - -`--policy balanced` enables lossless live placement (`REPIN=64`). At safe -request boundaries, a per-layer LFRU score combines decaying session frequency -with recent access and replaces at most four sufficiently colder pinned -experts. `--policy quality` leaves live replacement off by default; `REPIN=0` -always disables it. Persistent `.coli_usage` history and session-local LFRU -state remain separate. - -For single-token q4 CPU experts, gate and up projections share one OpenMP -dispatch while retaining the same per-row AVX2/NEON arithmetic. This removes -one thread-team launch per RAM expert without activation requantization or a -lower-precision fallback. It is a stepping stone toward a persistent native -CPU expert pool, not a replacement for one. - -**The expert cache auto-sizes to your RAM** (since 2026-07-10): the engine now *raises* the LRU cap to fill your `--ram` budget instead of only lowering it. Before this fix a 128 GB machine ran with the same 8-experts/layer cache as a 16 GB one (issue #12) — **if you benchmarked colibrì before this date, rerun: your numbers were capped.** - -**Router-lookahead prefetch** (`PILOT=1`, experimental): GLM-5.2's expert routing is measurably predictable *ahead of time* — applying layer L+1's router to layer L's post-attention state recalls **71.6%** of the true top-8 (vs 41.3% for "same experts as last token"). `PILOT=1` uses this to issue next-layer expert readahead from a dedicated I/O thread while the current layer computes. On our dev box the disk is already ~80% saturated, so it measures neutral; on machines where compute and disk are balanced (like the Ryzen AI 9 in issue #12: 43% disk / 46% matmul) it should overlap real work — measurements welcome. - -**The learning cache**: the engine records which experts your usage actually routes to (`.coli_usage` next to the model, updated every turn) and at startup automatically pins the hottest ones in spare RAM. colibrì literally gets faster the more you use it. - -**Live tier adaptation** (`--repin N`, opt-in): at safe turn boundaries, a decaying -session heat map replaces cold pinned experts with hotter streamed experts. Replacement -loads the expert from disk into the existing RAM slot; GPU-backed slots immediately -refresh the same VRAM tier budget. A 25% hysteresis and a four-swap limit prevent tier -thrashing. Persistent `.coli_usage` remains the long-term signal and is not decayed. - -**Conversations reopen warm** (`.coli_kv`, since 2026-07-10): `coli chat` persists the compressed MLA KV-cache to disk after every turn (~182 KB/token, appended incrementally, crash-safe). Close the chat, reopen it tomorrow — the model still remembers the whole conversation and **zero re-prefill happens**: validated byte-identical to an uninterrupted session. `:reset` clears it, `KVSAVE=0` disables it. - -## Web dashboard - -One command serves the OpenAI-compatible API **and** the web console on the same port, then opens your browser when the engine is ready: - -```bash -cd web && npm install && npm run build # once -./coli web --model -``` - -What you get: - -- **Chat** with live metrics: a flashing token counter while generating, then tok/s, time-to-first-token, prompt→completion counts and queue wait; -- **Runtime panel**: your hardware (CPU, GPUs + VRAM, RAM, cores), the scheduler, and the live expert-tier bar — how many of the 19,456 experts sit in VRAM / RAM / disk right now; -- **Brain**: the whole model as a 76×256 cortex, one cell per expert. Colour = tier, brightness = routing heat, and the experts routed in each turn flash white and decay — you watch the model think. Hover any cell for its tier, heat and [measured topic affinity](https://github.com/JustVugg/colibri/issues/175) (specialists for code, Chinese, math, law… live in layers 11–22). - -The dashboard talks to the engine over two tiny protocol lines (`TIERS`, `EMAP`/`HITS`) and plain JSON endpoints — nothing heavier than the engine itself. - -## Got a better machine? Try it — here's what to expect - -colibrì was built on deliberately humble hardware (12 cores, 25 GB RAM, an older DRAM-less NVMe behind a WSL2 VHDX that measured ~1 GB/s random on *this* drive — note WSL2 VHDX is not inherently slow: a community 5090 box measured 10.5 GB/s O_DIRECT through one, [#101](https://github.com/JustVugg/colibri/issues/101)). **Every one of those constraints is a knob your machine can turn up.** The engine needs: Linux (or WSL2), macOS, or **Windows 11 natively (MinGW-w64)**; gcc with OpenMP, AVX2, ≥16 GB RAM, and the ~370 GB int4 model on a local NVMe (ext4/NTFS — never a network/9p mount). - -**How to test it, in order:** - -```bash -cd c && ./setup.sh # build + architecture self-test (expects 32/32) - -# 1) measure YOUR disk the way the engine uses it (parallel 19 MB random reads): -gcc -O2 -fopenmp iobench.c -o iobench -./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 0 # buffered, 8 threads -./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 1 # O_DIRECT (bypass cache) -# Caveat (#86): iobench reads a bounded ~1 GB shard, so buffered reads on a big-RAM box -# report the PAGE CACHE, not the disk. Use the O_DIRECT run (arg 1) for a true number, and -# run it on a shard you haven't touched this session (a prior buffered run caches its pages). -# On macOS there is no O_DIRECT — iobench uses F_NOCACHE, which stops *new* caching but can't -# evict pages a prior buffered run already resident-mapped, so a macOS "O_DIRECT" figure right -# after a buffered run still reads cache. Reboot or use a fresh shard for a real cold read. - -# 2) chat; watch the per-turn stats line (tok/s, expert hit-rate, RSS): -COLI_MODEL=/path/to/glm52_i4 ./coli chat - -# 3) record expert usage, then pin the hottest experts in your spare RAM: -STATS=stats.txt ./coli chat -PIN=stats.txt PIN_GB=20 ./coli chat # scale PIN_GB to your free RAM - -# 4) quality benchmarks (MMLU/HellaSwag/ARC): -./coli bench -``` - -**Back-of-envelope predictions** (decode is disk-bound: a cold token costs ~11.4 GB of expert reads; MTP speculation roughly halves the effective cost *once the cache is warm*; RAM turns cold reads into free cache hits): - -| machine | expected | +| topic | doc | |---|---| -| this dev box (WSL2 VHDX, ~1 GB/s, 25 GB RAM) | ~0.05–0.1 tok/s cold — proven baseline | -| native Linux, PCIe4 NVMe (~3–5 GB/s random), 32 GB | ~0.5–1 tok/s | -| PCIe5 NVMe or 2×NVMe RAID0 (~8–12 GB/s), 64 GB (PIN ~40 GB of hot experts) | ~2–4 tok/s | -| 128–256 GB RAM, 12 cores (hot experts cached) | ~2–4 tok/s — matmul-bound: ~80 GFLOP/token vs ~250 GFLOP/s of our AVX2 kernels | -| same RAM + 24–32 cores, or AVX-512/VNNI kernels | ~5–15 tok/s — interactive; kernel work is the multiplier | - -These are estimates, not measurements — if you run colibrì on serious hardware, **please open an issue with your numbers**: real datapoints from better machines are exactly what this project needs next. - -### Community benchmarks (measured) - -Real numbers from real machines, stock build (`setup.sh`, gcc 13), greedy decoding, `--ngen 32`, MTP active: - -| machine | disk (iobench, 19 MB × 64, 8 threads) | config | measured | -|---|---|---|---| -| Intel Core Ultra 7 270K Plus (24 threads) · WSL2 · 24 GB RAM · NVMe VHDX ([#2](https://github.com/JustVugg/colibri/issues/2)) | 1.96 GB/s buffered · 2.74 GB/s O_DIRECT | default | 0.07 tok/s · expert hit 3–4% · RSS 14.1 GB | -| 〃 | 〃 | `--topp 0.7` | **0.11 tok/s** · expert hit 11% · RSS 14.7 GB | -| Apple M5 Max (18 cores) · macOS · 128 GB unified · internal SSD ([#4](https://github.com/JustVugg/colibri/issues/4), [#5](https://github.com/JustVugg/colibri/issues/5)) | ~4 GB/s cold (the 14.2 GB/s reading was cache-influenced — see note) | default, MTP off | **1.06 tok/s** · expert hit 23% · RSS 21.8 GB | -| Apple M5 Max · macOS · 128 GB unified · 2 TB SSD · **Metal backend** ([#72](https://github.com/JustVugg/colibri/pull/72), [#87](https://github.com/JustVugg/colibri/issues/87)) | (macOS O_DIRECT figure unreliable — see note) | Metal on · `--ram 96` · 39.7 GB warm pin · MTP off | **1.83 tok/s** · expert hit 66% · warmed 1.11 → 1.83 over the run | -| 〃 · 46.9 GB pin (2.94M-selection history) · `--ram 110`, 1024-token run ([#103](https://github.com/JustVugg/colibri/issues/103)) | 〃 | Metal on (experts + attention) · MTP off | **2.06 tok/s** · hit 72.5% · coherent output · fastest datapoint yet (still on the pre-rebase Metal branch) | -| Mac Mini M4 Pro · macOS · **48 GB** unified · **Metal backend** ([#107](https://github.com/JustVugg/colibri/issues/107)) | 6.59 GB/s F_NOCACHE (fresh shard) | Metal on · `--ram 38` | **0.30 tok/s** (vs 0.18 CPU-only) — entry Apple Silicon on a third the RAM beats the 32-core 9950X row | -| Epyc 9654 ES · Linux · 4x16GB DDR5-4800-rdimm · Samsung PCIe Gen3 x4 NVME SSD | — | `MTP=1 DIRECT=1` | 0.31 tok/s · expert hit 35% · RSS 21.52 GB | -| Ryzen AI 9 HX 370 (Framework 13) · Arch Linux · 128 GB · WD SN850X, BTRFS zstd ([#12](https://github.com/JustVugg/colibri/issues/12)) | — | int8 MTP head · `--cap 32` · 46.7 GB auto-learned PIN | **0.37 tok/s** · expert hit 66% · MTP acceptance 52% (2.59 tok/fw) · RSS 105 GB | -| Ryzen 9 9950X (32 threads) · Linux · 123 GB · Crucial P3 QLC Gen3 ([#31](https://github.com/JustVugg/colibri/issues/31)) | 1.51 GB/s buffered | default, 2 runs from cold | 0.10 tok/s · hit 53% · profile 66% disk | -| 〃 same machine, model moved to a Samsung 9100 PRO PCIe 5.0 ([#31](https://github.com/JustVugg/colibri/issues/31)) | **8.81 GB/s** O_DIRECT | 〃 (usage history retained) | **0.28 tok/s** · hit 57% · profile flips: 32% disk / **57% matmul** | -| Ryzen AI Max+ 395 (Framework Desktop) · Ubuntu · 128 GB LPDDR5x · Intel Optane 905p PCIe 3.0 ([#39](https://github.com/JustVugg/colibri/issues/39)) | 3.27 GB/s buffered | int8 MTP head · fresh history (pure LRU, auto-raised cap 65) | 0.16 tok/s · hit 57% · profile 49% disk / 47% matmul | -| 〃 five runs later — learned pin 47.6 GB ([#39](https://github.com/JustVugg/colibri/issues/39)) | 〃 | `--temp 0.7 --topp 0.7` | **0.40 tok/s** · hit 71% · fastest non-Apple datapoint | -| Ryzen 7 9800X3D (16T) · WSL2 · 70 GB RAM · Samsung 9100 PRO PCIe 5.0 · RTX 5090 ([#101](https://github.com/JustVugg/colibri/issues/101)) | **10.51 GB/s** O_DIRECT | MTP off · learned pin 24 GB · hit 54% · OMP hot-team on | **0.41 tok/s** · disk-bound (36.5 s disk vs 24.0 s matmul) · **CUDA expert tier ≈ 0%** (AVX-512 CPU matches the 5090) · `--topp 0.7` → **0.52 tok/s** | -| EPYC 7443 (24C/48T, Zen3 AVX2) · Linux · **430 GB RAM** · NVMe RAID-Z1 via TrueNAS VM ([#104](https://github.com/JustVugg/colibri/issues/104)) | ~1 GB/s (VM overhead) | 77.5 GB pin · cap auto-raised to 194/layer · MTP off | **1.00 tok/s** · **hit 98%** · disk eliminated → **RAM-bandwidth + matmul bound** (no AVX-512/VNNI on Zen3) | -| Intel i5-12600K (10C/16T, AVX2) · **native Windows 11, no WSL** · 32 GB · MinGW GCC 16.1 ([#113](https://github.com/JustVugg/colibri/issues/113)) | buffered (no O_DIRECT on MinGW) | int8 MTP head · cold, small-RAM (cap ~2/layer) | **0.08 tok/s** · hit 3.7% · **MTP 57% acceptance** — first native-Windows datapoint, port validated | -| Ryzen 9 9950X3D2 (16C/32T, avx512-vnni) · native Linux · 121 GB · Samsung 9100 PRO **PCIe Gen5** · RTX 5090 (28 GB expert tier, 1475 pinned) ([#120](https://github.com/JustVugg/colibri/issues/120)) | **11.48 GB/s** O_DIRECT | `MTP=0 DIRECT=1 PIPE_WORKERS=16 PREFETCH=1` | **1.23 tok/s** · MTP-off wins disk-bound · fastest x86 datapoint yet | -| Ryzen AI Max+ 395 (Strix Halo, 16C/32T Zen5, avx512-vnni) · Arch Linux · 128 GB unified LPDDR5x · SK hynix P41 PCIe 4.0 ([#124](https://github.com/JustVugg/colibri/issues/124)) | — | `DIRECT=1 PIPE=1 --topp 0.7` · auto-pin | 0.06 cold → **1.10 tok/s** sustained · first Strix Halo / gfx1151 datapoint (unified memory: no discrete VRAM tier) | -| Intel Core Ultra 9 185H (16C/22T, avx-vnni) · **native Windows 11, no WSL** · 32 GB · Crucial P3 QLC NTFS · RTX 5070 Ti (unused) ([#128](https://github.com/JustVugg/colibri/issues/128)) | — | int8 MTP head · **with [#131](https://github.com/JustVugg/colibri/pull/131) (pipe + RAM fixes), warm cache, no GPU** | 0.03 cold → **0.5 tok/s** warm (~7-prompt warmup) · cache-warming on native Windows once the portability blockers are fixed — stock main hung on the `\r\n` READY sentinel before #131 | -| Dell Pro Max GB10 (DGX Spark: Grace 10×X925 + 10×A725, **aarch64 i8mm/sve2**) · Linux · 121 GB unified LPDDR5x · Dell OEM 4 TB NVMe · GB10 sm_121 ([#136](https://github.com/JustVugg/colibri/issues/136)) | **5.58 GB/s** O_DIRECT (NVIDIA-OEM unit in #76 was 10.74 — same platform, different SSD) | int8 MTP head · warm cache | 0.21 cold → **0.50 tok/s** warm · hit 83% · MTP 73% (3.20 tok/fw) · **matmul-bound** (matmul 130 s vs disk 58 s) — unified memory, CUDA placement tier neutral; the lever here is an i8mm compute kernel, not placement | - -Takeaways: with 24 GB of RAM the engine auto-caps the expert cache to 2 slots/layer, so decode stays cold even on a disk 2–2.7× faster than the dev box — **on small-RAM machines the RAM cap, not the disk, is the binding constraint**, exactly as the table above predicts; `--topp 0.7` alone bought a clean 1.6× end-to-end speedup. The M5 Max datapoint lands right on the table's second row: **~1 tok/s of a 744B model on a laptop SSD** — and its 14 GB/s disk shifts the bottleneck back to RAM budget and kernels. The Framework 13 rows are the cache thesis proven end-to-end on one machine: 0.29 → 0.37 tok/s (hit 28% → 66%, speculation finally engaging at 52% acceptance) just by giving the cache its RAM — int8 MTP head + a bigger cap + the learned pin. The cap part is now automatic (cap auto-raise, 2026-07-10). The 9950X pair is the cleanest bottleneck experiment yet — same machine, same history, only the disk swapped: ×5.8 disk bandwidth bought ×2.9 tokens, and the profile **flipped from 66% disk to 57% matmul**. But the crossover depends on the CPU kernel: the 9800X3D row ([#101](https://github.com/JustVugg/colibri/issues/101)) shows that with the OMP hot-team tuning on, the AVX-512 CPU matmul is fast enough that even a **10 GB/s NVMe stays disk-bound** — and there the **CUDA expert tier buys ≈ 0%**, because the CPU already matches the 5090 on expert matmul. The GPU tier earns its VRAM only when the CPU is the weak link, not by default. (Honest correction from #101: an earlier version of that report ran with the OMP tuning off, which manufactured a false matmul-bound crossover and a false +14% for CUDA — neither survived a clean re-run.) - -## Quality benchmark — help wanted - -**First measurement is in** ([#108](https://github.com/JustVugg/colibri/issues/108), thanks dnnspaul): the int4 container scored **62.5% mean acc_norm** on hellaswag/arc/mmlu (0-shot log-likelihood, n=40) — below the 85–95% published for full-precision GLM-5.2, but **the gap is not yet attributable to quantization.** Two confounds sit in the way: (1) 0-shot log-likelihood MC scoring badly underserves a *reasoning* model like GLM-5.2 (it never gets to think), so a large gap is expected even at fp16; (2) n=40 is ±14pp. The **decisive experiment** is the OLMoE fp16-vs-int4 A/B under this same harness (small enough to run both precisions) — that delta *is* the quantization cost with the scoring protocol cancelled out. Until it's run, 62.5% is a datapoint, not a verdict. - -The code is here and ready; one command runs it end to end (it auto-downloads the datasets on first use): - -```bash -cd c -pip install tokenizers datasets # in addition to the convert deps above -./coli bench # hellaswag, arc_challenge, mmlu — 40 questions each -./coli bench hellaswag --limit 200 # one task, more questions -./coli bench mmlu arc_challenge --ram 100 # pick tasks, set a RAM budget -``` - -It prints per-task accuracy (log-likelihood scoring, EleutherAI-harness style). **If you can run the OLMoE fp16-vs-int4 A/B (or a large-n GLM run), please open an issue with the numbers** — it's the measurement that turns 62.5% into either "int4 is fine, scoring artifact" or "quantization is the ceiling, grouped-scale is the priority." +| Benchmarks, community datapoints, quality measurements | [docs/benchmarks.md](docs/benchmarks.md) | +| Tuning knobs, policies, the learning cache, prefetch | [docs/tuning.md](docs/tuning.md) | +| Windows 11 native build (+ CUDA DLL) | [docs/windows.md](docs/windows.md) | +| CUDA backend, VRAM expert tier, full residency | [docs/cuda.md](docs/cuda.md) | +| Apple Silicon Metal backend | [docs/metal.md](docs/metal.md) | +| OpenAI-compatible API, KV slots, web dashboard | [docs/api.md](docs/api.md) | +| Grammar-forced drafts (structured output) | [docs/grammar-draft.md](docs/grammar-draft.md) | +| Environment variable inventory | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | ## Supporting the project -colibrì is a one-person project, written and tested entirely on a 12-core laptop with 25 GB of RAM — the numbers above are the ceiling of what I can measure at home. If this project is useful or interesting to you and you'd like to support its development (better test hardware translates *directly* into a faster engine for everyone: real NVMe scaling data, bigger pinned caches, int2/int3 quality sweeps on real benchmarks), you can: +colibrì started as a one-person project on a 12-core laptop with 25 GB of RAM; +today its numbers come from a community of real machines. If it's useful to you: - ⭐ star the repo and share it; -- 🐛 open issues with benchmark numbers from your hardware; -- 💬 reach out via GitHub issues if you'd like to sponsor development or donate hardware. - -Every contribution, from a datapoint to a disk, moves the ceiling. +- 🐛 open issues with benchmark numbers from your hardware — datapoints move + this project more than anything else; +- 💬 reach out via GitHub issues to sponsor development or donate hardware. ## Repo layout @@ -656,20 +221,20 @@ c/ ├── tools/ offline conversion, fixtures and benchmarks ├── scripts/ long-running conversion helpers └── tests/ dependency-free C and Python tests -web/ browser UI (pure OpenAI-API client, community-maintained) +web/ browser UI (pure OpenAI-API client) desktop/ Tauri v2 desktop shell wrapping the web UI +docs/ reference docs, experiments, media ``` The runtime path intentionally stays flat and readable: `glm.c` plus its small -headers. Auxiliary Python and shell tooling is grouped separately and is never a -runtime dependency of the engine. - -From the repository root, `make`, `make check`, and `make clean` delegate to the -engine Makefile. Existing commands run from `c/` continue to work unchanged. +headers. From the repository root, `make`, `make check`, and `make clean` +delegate to the engine Makefile. ## Why "colibrì" -The hummingbird weighs a few grams, hovers in place, and visits a thousand flowers a day. This engine keeps a 744-billion-parameter giant alive on hummingbird rations: 25 GB of RAM, twelve CPU cores, and a lot of disk patience. +The hummingbird weighs a few grams, hovers in place, and visits a thousand +flowers a day. This engine keeps a 744-billion-parameter giant alive on +hummingbird rations: 25 GB of RAM, twelve CPU cores, and a lot of disk patience. ## License diff --git a/README.zh-TW.md b/README.zh-TW.md new file mode 100644 index 0000000..1662617 --- /dev/null +++ b/README.zh-TW.md @@ -0,0 +1,233 @@ +

+ colibrì——小巧引擎，龐大模型 +

+ +

+ English · 繁體中文 +

+ +**小巧引擎，龐大模型。**只要約 25 GB 記憶體，就能在消費級電腦上執行 **GLM-5.2（744B 參數的 MoE）**——以零相依套件的純 C 實作，從硬碟串流載入專家。 + +Colibrì 是一套輕量、維持品質的 MoE 執行環境，將 VRAM、RAM +與儲存裝置視為統一管理的記憶體階層。高速記憶體不足可能降低速度， +但預設策略**絕不會在未告知的情況下改變模型精度或路由語意**。 + +``` +$ ./coli chat + 🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU + ✓ ready in 32s · resident 9.9 GB + › ciao! + ◆ Ciao! 😊 Come posso aiutarti oggi? +``` + +## 實際運行畫面 + +

+ colibrì 網頁儀表板——即時指標、硬體面板與專家儲存層級 +

+

網頁儀表板（./coli web）：744B 模型達到 4 tok/s、TTFT 1.6 秒、硬碟讀取 0—— +在 6× RTX 5090 上讓所有專家常駐，並即時顯示 token 指標、每輪耗時明細、 +VRAM／RAM／硬碟層級長條，以及角落的即時迷你大腦。

+ +

+ 大腦頁面——以即時皮質呈現 19,456 個專家 +

+

大腦（Brain）頁面：將全部 19,456 個專家呈現為活的皮質——顏色代表儲存層級， +亮度代表路由熱度，每輪被路由到的專家都會閃白。將游標停在專家上，即可查看其 +實測主題親和度。

+ +

+ 圖譜頁面——以 3D 星系呈現實測專家圖譜 +

+

圖譜（Atlas）頁面：將實測專家圖譜 +呈現為 3D 星系——共 13,260 個已分析專家，其中 1,041 個可重現的專門專家會按主題聚集 +（詩歌、法律、中文、SQL……）。位置取自實測路由親和度，而非學習出的嵌入向量。拖曳即可旋轉。

+ +## 核心概念 + +744B 的專家混合（Mixture-of-Experts）模型，每個 token 只會啟用約 40B 參數—— +其中每個 token 之間會變動的只有約 11 GB（被路由到的專家）： + +

+ 每個 token 只會啟用約 5.4% 的參數 +

+ +所以模型不必完整**放進**高速記憶體，而是需要正確**配置位置**： + +- **稠密部分**（注意力、共享專家、嵌入——約 17B 參數）以 int4 + **常駐 RAM**（約 9.9 GB）； +- **19,456 個路由專家**（75 個 MoE 層 × 256，加上 MTP head；每個在 int4 下約 19 MB） + **存放在硬碟**（約 370 GB），並**隨需串流載入**，搭配逐層 LRU 快取、 + 會學習的熱門專家固定儲存區，以及選用的 VRAM 層級。 + +引擎由單一 C 檔（`c/glm.c`）與少量標頭檔組成。不需要 BLAS， +執行階段不需要 Python，也不需要 GPU。 + +## 運作方式 + +### 每個 token 的處理路徑 + +

+ 路由 → 聯集 → 配置 → 重疊執行 → 學習 +

+ +每個 token 的每一層都會走過相同的五個步驟。設計目標是讓 +**配置只決定速度**——無論專家是從 VRAM 或硬碟回應，路由器的決策與權重精度都完全相同。 + +### 統一記憶體階層，取代單一記憶體門檻 + +

+ VRAM／RAM／NVMe 三層專家常駐架構 +

+ +同一套引擎涵蓋完整硬體範圍：在 25 GB 筆電上，一切都從硬碟串流載入 +（慢，但結果正確）；在大型主機上，則可讓整組專家常駐 +（`CUDA_EXPERT_GB=auto PIN_GB=all`），讓硬碟完全退出解碼路徑。 +兩端之間有一層**學習型快取**：引擎會記錄*你的*工作負載路由到哪些專家 +（`.coli_usage`，每輪更新），並自動固定最熱門的專家——colibrì 確實會越用越快。 +在多插槽主機上，`COLI_NUMA=1` 會將常駐權重交錯分配到各記憶體控制器 +（[#82](https://github.com/JustVugg/colibri/issues/82)）。 + +### 絕不為同一次硬碟讀取等待兩遍 + +快取未命中的成本很高，因此引擎大部分的巧思都用來避免或重疊處理這些讀取： +每個專家的三個矩陣相鄰儲存，並以一次 `pread` 讀取；有界非同步 I/O pool +（`PIPE=1`，預設啟用）會在常駐專家運算時載入缺少的專家；批次位置只讀取每個 +不重複專家一次（**批次聯集**）；路由前瞻執行緒（`PILOT=1`）則預先載入下一層專家—— +實測顯示，路由結果提前一層時有 **71.6% 的可預測性**。 +在 GPU 上，常駐管線（`COLI_CUDA_PIPE=2`）讓殘差流跨層保留在裝置端， +使 CPU 專家迴圈不中斷；在 Apple Silicon 上，實驗性的 +[Metal 後端](docs/metal.md)會用統一記憶體 GPU 執行批次專家運算。 + +### 忠實模型，壓縮狀態 + +前向傳遞已透過 `transformers` oracle 驗證為**逐 token 完全一致** +（teacher-forcing 32/32）。MLA 注意力儲存壓縮後的 KV 狀態——每個 token 為 576 個 +浮點數，而非 32,768 個（**縮小 57×**）——並跨重新啟動持久保存 +（`.coli_kv`）：對話可暖啟恢復，不需重新 prefill，結果與不中斷的工作階段 +逐位元組相同。DSA 稀疏注意力（GLM-5.2 的 lightning indexer）已忠實實作， +並透過強制選取所有 key，驗證可精確重現稠密注意力。 + +### 如實呈現推測式解碼 + +GLM-5.2 原生 MTP head 會起草 token，再由主模型以一次批次前向傳遞驗證—— +條件合適時每次 forward 可產生 2.2–2.8 個 token。兩條得來不易的規則已成為預設值： +MTP head 必須是 **int8**（int4 head 的接受率會崩落到 0–4%，見 +[#8](https://github.com/JustVugg/colibri/issues/8)），且草稿與驗證必須計算 +**相同函數**——`SPEC_PIN=1` 會把兩者固定在同一 kernel family +（完整鑑識過程見 [#163](https://github.com/JustVugg/colibri/issues/163)）。 +文法強制草稿（[`GRAMMAR=file.gbnf`](docs/grammar-draft.md)）可在受限 JSON 輸出中， +以近乎免費的成本提高接受率。推測式解碼是否帶來淨收益取決於快取熱度——請實測， +若不划算就使用 `DRAFT=0`。 + +## 實際成果 + +

+ 各硬體等級的實測解碼速度 +

+ +同一套引擎、同一個 int4 容器——硬體只會改變專家的存放位置。 +[完整 benchmark 表格](docs/benchmarks.md)中的重點如下： + +- **6× RTX 5090，全部常駐：**解碼 5.8–6.8 tok/s，TTFT 約 13 秒 + （[實驗紀錄](docs/experiments/glm52-6x5090-2026-07-12.md)）； +- **128 GB、僅使用 CPU 的桌上型電腦：**暖機後約 1.8 tok/s + （[#200](https://github.com/JustVugg/colibri/issues/200)）； +- **單張 RTX 5070 Ti 的筆電級電腦：**透過 GPU 常駐管線達到 1.07 tok/s + （[#273](https://github.com/JustVugg/colibri/issues/273)）； +- **25 GB 開發機：**冷啟動 0.05–0.1 tok/s——這是專案起步時已證實的下限， + 也仍是如實呈現的基準。 + +品質來自測量，而非假設：int4 容器的量化成本，以及 scale granularity／rotation +消融實驗，收錄於 [docs/benchmarks.md](docs/benchmarks.md#quality-benchmark)、 +[#108](https://github.com/JustVugg/colibri/issues/108) 與 +[#81](https://github.com/JustVugg/colibri/issues/81)。 + +## 開始使用 + +### 1. 取得模型 + +Hugging Face 上已有預先轉換的 **GLM-5.2 int4** 容器——請務必使用 +**含 int8 MTP head 的版本**： + +**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp** + +> ⚠️ 原始鏡像使用 int4 MTP head → 草稿接受率為 0% +>（[#8](https://github.com/JustVugg/colibri/issues/8)）。請檢查你的版本： +> `ls -l /out-mtp-*`——正確的 int8 大小為 `3527131672 / 5366238584 / 1065950496`。 + +你也可以自行從 FP8 來源轉換——只需一條可續傳的指令，且任何時候都不需要 +在硬碟上同時存放完整的 756 GB： + +```bash +cd c && ./setup.sh # 檢查 gcc/OpenMP、建置並執行自我測試 +./coli convert --model /nvme/glm52_i4 # 逐 shard 下載並轉換（僅此一次需要 python） +``` + +### 2. 執行 + +```bash +COLI_MODEL=/nvme/glm52_i4 ./coli chat # 自動偵測 RAM 預算、快取與 MTP +COLI_MODEL=/nvme/glm52_i4 ./coli plan # 檢視規劃的 VRAM／RAM／硬碟配置 +COLI_MODEL=/nvme/glm52_i4 ./coli doctor # 唯讀就緒檢查 +./coli web --model /nvme/glm52_i4 # 在同一個連接埠提供 API 與網頁儀表板 +./coli serve --model /nvme/glm52_i4 # 僅提供 OpenAI 相容 API +``` + +引擎執行階段是純 C——python 只供單次轉換工具與選用的 API gateway 使用。 + +### 3. 深入了解 + +| 主題 | 文件 | +|---|---| +| Benchmark、社群實測數據、品質測量 | [docs/benchmarks.md](docs/benchmarks.md) | +| 調校選項、策略、學習型快取、預先載入 | [docs/tuning.md](docs/tuning.md) | +| Windows 11 原生建置（含 CUDA DLL） | [docs/windows.md](docs/windows.md) | +| CUDA 後端、VRAM 專家層級、全部常駐 | [docs/cuda.md](docs/cuda.md) | +| Apple Silicon Metal 後端 | [docs/metal.md](docs/metal.md) | +| OpenAI 相容 API、KV slots、網頁儀表板 | [docs/api.md](docs/api.md) | +| 文法強制草稿（結構化輸出） | [docs/grammar-draft.md](docs/grammar-draft.md) | +| 環境變數完整清單 | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | + +## 支持專案 + +colibrì 最初是由一人使用 12 核心、25 GB RAM 的筆電開發； +如今它的數據來自社群中的各種真實機器。如果這個專案對你有用： + +- ⭐ 為儲存庫加星並分享； +- 🐛 以 issue 提交你的硬體 benchmark 數據——實測資料比任何其他事都更能推動專案； +- 💬 若想贊助開發或捐贈硬體，請透過 GitHub issues 聯絡。 + +## 儲存庫結構 + +``` +Makefile 根目錄建置／檢查入口 +c/ +├── glm.c 單檔 GLM 引擎 +├── st.h, tok.h, json.h 執行階段標頭檔 +├── backend_cuda.* 選用的 CUDA 層級 +├── Makefile 建置與本機檢查 +├── coli 使用者介面 CLI +├── openai_server.py OpenAI 相容 HTTP gateway +├── setup.sh 單一指令完成本機設定 +├── tools/ 離線轉換、fixtures 與 benchmarks +├── scripts/ 長時間轉換輔助工具 +└── tests/ 零相依套件的 C 與 Python 測試 +web/ 瀏覽器 UI（純 OpenAI API client） +desktop/ 包裝網頁 UI 的 Tauri v2 桌面 shell +docs/ 參考文件、實驗與媒體檔 +``` + +執行階段路徑刻意維持扁平、易讀：`glm.c` 加上少量標頭檔。 +在儲存庫根目錄執行 `make`、`make check` 與 `make clean`， +都會轉交給引擎的 Makefile。 + +## 為什麼叫做「colibrì」 + +蜂鳥只有幾公克重，能在原地懸停，並在一天內造訪上千朵花。 +這套引擎只用蜂鳥般的配給，就能讓 744B 參數的巨人運轉： +25 GB RAM、十二個 CPU 核心，以及對硬碟的大量耐心。 + +## 授權條款 + +Apache 2.0。GLM-5.2 權重由 Z.ai 以 MIT 授權發布。 diff --git a/c/.gitignore b/c/.gitignore index 782930b..d8e1f88 100644 --- a/c/.gitignore +++ b/c/.gitignore @@ -2,3 +2,4 @@ glm_tiny/ olmoe_hf/ olmoe_i4/ +.build-config diff --git a/c/Makefile b/c/Makefile index ccf0a3c..1f8583b 100644 --- a/c/Makefile +++ b/c/Makefile @@ -166,7 +166,7 @@ else PYTHON ?= python3 endif CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) ifneq (,$(LINUX)) TEST_BINS += tests/test_uring$(EXE) endif @@ -212,11 +212,25 @@ all: glm$(EXE) # phony 'glm' → 'glm.exe' on Windows (so 'make glm' and 'coli build' work on every platform) glm: glm$(EXE) -glm$(EXE): glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h $(CUDA_OBJ) $(METAL_OBJ) +# Config stamp: make only tracks file timestamps, not flag changes. Without this, +# `make glm.exe CUDA_DLL=1` after a prior CPU-only build reports "up to date" and +# silently keeps the CPU-only binary (no CUDA loader) — a build that looks like it +# worked but isn't. We record the build-affecting flags in .build-config and rewrite +# it ONLY when they change (evaluated here at parse time, so the file's timestamp +# moves exactly when the config moves). glm.exe and the CUDA/loader objects depend +# on it, so they relink on a config change and stay put otherwise. (#306) +BUILD_CONFIG := $(CC)|$(CFLAGS)|$(LDFLAGS)|CUDA=$(CUDA)|CUDA_DLL=$(CUDA_DLL)|ARCH=$(ARCH)|CUDA_ARCH=$(CUDA_ARCH)|METAL=$(METAL) +BUILD_CONFIG_OLD := $(shell cat .build-config 2>/dev/null) +ifneq "$(BUILD_CONFIG)" "$(BUILD_CONFIG_OLD)" +$(shell printf '%s' '$(BUILD_CONFIG)' > .build-config) +endif +.build-config: ; + +glm$(EXE): glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h $(CUDA_OBJ) $(METAL_OBJ) .build-config $(CC) $(CFLAGS) glm.c $(CUDA_OBJ) $(METAL_OBJ) -o glm$(EXE) $(LDFLAGS) # Windows runtime loader object: resolves coli_cuda_* from coli_cuda.dll. -backend_loader.o: backend_loader.c backend_cuda.h compat.h +backend_loader.o: backend_loader.c backend_cuda.h compat.h .build-config $(CC) $(CFLAGS) -c backend_loader.c -o $@ # Windows CUDA DLL: compile backend_cuda.cu with nvcc (+MSVC cl.exe as host @@ -225,15 +239,15 @@ backend_loader.o: backend_loader.c backend_cuda.h compat.h # "x64 Native Tools Command Prompt"). COLI_CUDA_BUILDING_DLL enables # __declspec(dllexport) so the 15 API symbols are exported. cuda-dll: backend_cuda.cu backend_cuda.h - @command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } + @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } @command -v cl >/dev/null 2>&1 || { echo "cl.exe (MSVC) not in PATH — run vcvars64.bat first" >&2; exit 1; } - $(NVCC) $(NVCCFLAGS) -shared -DCOLI_CUDA_BUILDING_DLL \ + "$(NVCC)" $(NVCCFLAGS) -shared -DCOLI_CUDA_BUILDING_DLL \ -L"$(CUDA_HOME)/lib/x64" -lcudart \ backend_cuda.cu -o coli_cuda.dll -backend_cuda.o: backend_cuda.cu backend_cuda.h - @command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } - $(NVCC) $(NVCCFLAGS) -c backend_cuda.cu -o $@ +backend_cuda.o: backend_cuda.cu backend_cuda.h .build-config + @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } + "$(NVCC)" $(NVCCFLAGS) -c backend_cuda.cu -o $@ backend_metal.o: backend_metal.mm backend_metal.h $(METALXX) -c backend_metal.mm -o $@ @@ -243,13 +257,13 @@ metal-test: tests/test_backend_metal.mm backend_metal.mm backend_metal.h ./backend_metal_test cuda-test: backend_cuda.cu backend_cuda.h tests/test_backend_cuda.cu - @command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } - $(NVCC) $(NVCCFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) + @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } + "$(NVCC)" $(NVCCFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE) ./backend_cuda_test$(EXE) cuda-bench: backend_cuda.cu backend_cuda.h tests/bench_tensor_core.cu - @command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } - $(NVCC) $(NVCCFLAGS) backend_cuda.cu tests/bench_tensor_core.cu -o backend_cuda_bench$(EXE) + @command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } + "$(NVCC)" $(NVCCFLAGS) backend_cuda.cu tests/bench_tensor_core.cu -o backend_cuda_bench$(EXE) ./backend_cuda_bench$(EXE) olmoe$(EXE): olmoe.c st.h json.h compat.h @@ -279,6 +293,9 @@ iobench$(EXE): iobench.c compat.h tests/test_json$(EXE): tests/test_json.c json.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_st_pread$(EXE): tests/test_st_pread.c st.h json.h compat.h + $(CC) $(CFLAGS) -DST_PREAD_CHUNK=7 $< -o $@ $(LDFLAGS) + tests/test_st$(EXE): tests/test_st.c st.h json.h compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) @@ -297,15 +314,43 @@ tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h tests/test_idot$(EXE): tests/test_idot.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_stops$(EXE): tests/test_stops.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_topp$(EXE): tests/test_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +# bench_topp is a microbenchmark (old qsort vs new heap partial-select, #335), NOT a test +# gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_topp +tests/bench_topp$(EXE): tests/bench_topp.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_sample_nan$(EXE): tests/test_sample_nan.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_logit_nan$(EXE): tests/test_logit_nan.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_dsa_select$(EXE): tests/test_dsa_select.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +# bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356), +# NOT a test gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_dsa_select +tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + tests/test_uring$(EXE): tests/test_uring.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu index 9ce142f..413e53b 100644 --- a/c/backend_cuda.cu +++ b/c/backend_cuda.cu @@ -8,12 +8,21 @@ #include #include +struct RaggedKVEntry { + const void *key; + const float *host_l,*host_r; + float *latent,*rope; + int length,capacity,K,R; +}; + struct ColiCudaTensor { void *weights; float *scales; size_t weight_bytes; int fmt, I, O, device; int tracked; + RaggedKVEntry ragged[512]; + int ragged_count; }; typedef struct { @@ -23,7 +32,7 @@ typedef struct { size_t x_cap, y_cap, gate_cap, up_cap; uint8_t *qx; float *qscale; size_t qx_cap, qscale_cap; - float *host_x,*host_y; size_t host_x_cap,host_y_cap; + float *host_x,*host_y,*host_kv; size_t host_x_cap,host_y_cap,host_kv_cap; float *aq,*al,*ar,*ac; size_t aq_cap,al_cap,ar_cap,ac_cap; float *pipe_buf[24]; size_t pipe_cap[24]; /* scratch persistenti del resident pipeline */ cudaStream_t stream; @@ -338,6 +347,49 @@ __global__ static void attention_absorb_batch_kernel(float *ctx,const float *q, ctx[((size_t)s*H+h)*V+v]=a*(fmt?wscale[row]:1.f);} } +/* Independent device-resident KV sequence per row. lengths selects the valid + * prefix; latent/rope point at paged caches updated by the host wrapper. */ +__global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q, + const float *const *latent,const float *const *rope,const int *lengths, + const void *weights,const float *wscale,int fmt,int S,int H,int Q,int R, + int V,int K,int T,float scale){ + int s=blockIdx.y,h=blockIdx.x,tid=threadIdx.x,nt=lengths[s],rbase=h*(Q+V); + if(s>=S||nt<1||nt>T)return; + extern __shared__ float sm[];float *qa=sm,*cl=qa+K,*scores=cl+K,*red=scores+T; + const float *qs=q+((size_t)s*H+h)*(Q+R); + const float *ls=latent[s],*rs=rope[s]; + for(int k=tid;k>1;n;n>>=1){if(tid>1;n;n>>=1){if(tid= bytes) return 1; if (*ptr) cudaFree(*ptr); @@ -406,16 +458,17 @@ extern "C" void coli_cuda_shutdown(void) { for(int b=0;b<24;b++) if(ctx->pipe_buf[b]) cudaFree(ctx->pipe_buf[b]); if (ctx->host_x) cudaFreeHost(ctx->host_x); if (ctx->host_y) cudaFreeHost(ctx->host_y); + if (ctx->host_kv) cudaFreeHost(ctx->host_kv); if (ctx->stream) cudaStreamDestroy(ctx->stream); if (ctx->group_desc) cudaFree(ctx->group_desc); ctx->x = ctx->y = ctx->gate = ctx->up = nullptr; ctx->qx=nullptr; ctx->qscale=nullptr; ctx->aq=ctx->al=ctx->ar=ctx->ac=nullptr; - ctx->host_x=ctx->host_y=nullptr;ctx->stream=nullptr; + ctx->host_x=ctx->host_y=ctx->host_kv=nullptr;ctx->stream=nullptr; ctx->x_cap = ctx->y_cap = ctx->gate_cap = ctx->up_cap = 0; ctx->qx_cap=ctx->qscale_cap=0; ctx->aq_cap=ctx->al_cap=ctx->ar_cap=ctx->ac_cap=0; - ctx->host_x_cap=ctx->host_y_cap=0; + ctx->host_x_cap=ctx->host_y_cap=ctx->host_kv_cap=0; ctx->group_desc=nullptr; ctx->group_desc_cap=0; } g_nctx = 0; @@ -770,6 +823,89 @@ extern "C" int coli_cuda_attention_project_batch(ColiCudaTensor *w,ColiCudaTenso return attention_absorb_batch_run(w,proj,out,q,latent,rope,S,H,Q,R,V,K,T,scale); } +extern "C" int coli_cuda_attention_project_ragged(ColiCudaTensor *w,ColiCudaTensor *proj, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int T,float scale){ + if(!w||!proj||!out||!q||!keys||!latent||!rope||!lengths||S<1||S>512||T<1||T>8192|| + H<1||Q<1||R<1||V<1||K<1||K>512||w->I!=K||w->O!=H*(Q+V)|| + proj->device!=w->device||proj->I!=H*V)return 0; + DeviceContext *dc=find_ctx(w->device); + if(!select_ctx(dc))return 0; + float **dl=(float**)std::malloc((size_t)S*sizeof(*dl)); + float **dr=(float**)std::malloc((size_t)S*sizeof(*dr)); + int *old=(int*)std::malloc((size_t)S*sizeof(*old)); + int *add=(int*)std::malloc((size_t)S*sizeof(*add)); + int *off=(int*)std::malloc((size_t)S*sizeof(*off));int packed_n=0; + if(!dl||!dr||!old||!add||!off){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;} + for(int s=0;sT){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;} + RaggedKVEntry *e=nullptr; + for(int i=0;iragged_count;i++)if(w->ragged[i].key==keys[s]){e=&w->ragged[i];break;} + if(!e){ + if(w->ragged_count>=512){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;} + e=&w->ragged[w->ragged_count++];std::memset(e,0,sizeof(*e));e->key=keys[s]; + } + if(e->K!=K||e->R!=R||e->host_l!=latent[s]||e->host_r!=rope[s]||lengths[s]length){ + if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope); + e->latent=e->rope=nullptr;e->length=e->capacity=0; + e->K=K;e->R=R;e->host_l=latent[s];e->host_r=rope[s]; + } + if(lengths[s]>e->capacity){ + int cap=(lengths[s]+63)&~63;float *nl=nullptr,*nr=nullptr; + if(!cuda_ok(cudaMalloc(&nl,(size_t)cap*K*sizeof(float)),"ragged KV latent page")|| + !cuda_ok(cudaMalloc(&nr,(size_t)cap*R*sizeof(float)),"ragged KV rope page")){ + if(nl)cudaFree(nl);if(nr)cudaFree(nr);std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0; + } + if(e->length){ + cudaMemcpyAsync(nl,e->latent,(size_t)e->length*K*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream); + cudaMemcpyAsync(nr,e->rope,(size_t)e->length*R*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream); + } + if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope); + e->latent=nl;e->rope=nr;e->capacity=cap; + } + dl[s]=e->latent;dr[s]=e->rope;old[s]=e->length;add[s]=lengths[s]-e->length; + off[s]=packed_n;packed_n+=add[s]*(K+R); + } + size_t qb=(size_t)S*H*(Q+R)*sizeof(float); + size_t cb=(size_t)S*H*V*sizeof(float),ob=(size_t)S*proj->O*sizeof(float); + size_t pb=(size_t)packed_n*sizeof(float); + size_t desc=(size_t)S*(2*sizeof(float*)+4*sizeof(int)); + int ok=reserve(&dc->aq,&dc->aq_cap,qb)&&reserve(&dc->ac,&dc->ac_cap,cb)&& + reserve(&dc->y,&dc->y_cap,ob)&&reserve_bytes(&dc->group_desc,&dc->group_desc_cap,desc)&& + (!pb||(reserve(&dc->al,&dc->al_cap,pb)&&reserve_pinned(&dc->host_kv,&dc->host_kv_cap,pb))); + char *db=(char*)dc->group_desc;float **ddl=(float**)db,**ddr=ddl+S; + int *dn=(int*)(ddr+S),*dold=dn+S,*dadd=dold+S,*doff=dadd+S; + if(ok&&pb){ + for(int s=0;shost_kv+off[s]; + std::memcpy(p,latent[s]+(size_t)old[s]*K,(size_t)add[s]*K*sizeof(float)); + std::memcpy(p+(size_t)add[s]*K,rope[s]+(size_t)old[s]*R,(size_t)add[s]*R*sizeof(float)); + } + ok=cuda_ok(cudaMemcpyAsync(dc->al,dc->host_kv,pb,cudaMemcpyHostToDevice,dc->stream),"ragged KV append upload"); + } + if(ok)ok=cuda_ok(cudaMemcpyAsync(dc->aq,q,qb,cudaMemcpyHostToDevice,dc->stream),"ragged q upload")&& + cuda_ok(cudaMemcpyAsync(ddl,dl,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged latent pointers")&& + cuda_ok(cudaMemcpyAsync(ddr,dr,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged rope pointers")&& + cuda_ok(cudaMemcpyAsync(dn,lengths,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged lengths upload")&& + cuda_ok(cudaMemcpyAsync(dold,old,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged old lengths")&& + cuda_ok(cudaMemcpyAsync(dadd,add,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append lengths")&& + cuda_ok(cudaMemcpyAsync(doff,off,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append offsets"); + if(ok&&pb)ragged_kv_append<<stream>>>(ddl,ddr,dc->al,dold,dadd,doff,K,R); + if(ok)for(int s=0;sragged_count;i++)if(w->ragged[i].key==keys[s]){w->ragged[i].length=lengths[s];break;} + } + std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);if(!ok)return 0; + size_t shared=(size_t)(2*K+T+256)*sizeof(float); + attention_absorb_ragged_kernel<<stream>>>(dc->ac,dc->aq,ddl,ddr, + dn,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale); + quant_matmul<<O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights, + proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I)); + return cuda_ok(cudaGetLastError(),"ragged attention launch")&& + cuda_ok(cudaMemcpyAsync(out,dc->y,ob,cudaMemcpyDeviceToHost,dc->stream),"ragged output download")&& + cuda_ok(cudaStreamSynchronize(dc->stream),"ragged attention synchronize"); +} + extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) { if (!tensor) return; DeviceContext *ctx = find_ctx(tensor->device); @@ -781,6 +917,10 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) { } if (tensor->weights) cudaFree(tensor->weights); if (tensor->scales) cudaFree(tensor->scales); + for(int i=0;iragged_count;i++){ + if(tensor->ragged[i].latent)cudaFree(tensor->ragged[i].latent); + if(tensor->ragged[i].rope)cudaFree(tensor->ragged[i].rope); + } std::free(tensor); } diff --git a/c/backend_cuda.h b/c/backend_cuda.h index acbe4bc..63c1f11 100644 --- a/c/backend_cuda.h +++ b/c/backend_cuda.h @@ -14,6 +14,7 @@ #define COLI_CUDA_DLLEXPORT #endif + #ifdef __cplusplus extern "C" { #endif @@ -92,6 +93,11 @@ COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,C const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale); +COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale); + COLI_CUDA_DLLEXPORT void coli_cuda_tensor_free(ColiCudaTensor *tensor); COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor); COLI_CUDA_DLLEXPORT int coli_cuda_tensor_device(const ColiCudaTensor *tensor); @@ -143,4 +149,3 @@ COLI_CUDA_DLLEXPORT int coli_cuda_pipe_sync(int device); #endif #endif - diff --git a/c/backend_loader.c b/c/backend_loader.c index b743d1b..eedbd50 100644 --- a/c/backend_loader.c +++ b/c/backend_loader.c @@ -61,6 +61,10 @@ typedef int (*fn_attention_absorb_batch)(ColiCudaTensor *kv_b,float *ctx,const f typedef int (*fn_attention_absorb_batch_dev)(ColiCudaTensor *kv_b_shard,float *ctx_dev, const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); typedef int (*fn_attention_absorb_kvdev)(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale); typedef int (*fn_attention_project_batch)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q,const float *latent, const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale); +typedef int (*fn_attention_project_ragged)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale); typedef int (*fn_attention_project_batch_dev)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); typedef int (*fn_attention_project_batch_dev_out)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale); typedef int (*fn_pipe_add)(int device,float *x_dev,const float *t_dev,size_t n); @@ -107,6 +111,7 @@ static struct { fn_attention_absorb_batch_dev attention_absorb_batch_dev; fn_attention_absorb_kvdev attention_absorb_kvdev; fn_attention_project_batch attention_project_batch; + fn_attention_project_ragged attention_project_ragged; fn_attention_project_batch_dev attention_project_batch_dev; fn_attention_project_batch_dev_out attention_project_batch_dev_out; fn_pipe_add pipe_add; @@ -200,6 +205,7 @@ static int coli_cuda_load(void){ RESOLVE(attention_absorb_batch_dev, fn_attention_absorb_batch_dev) RESOLVE(attention_absorb_kvdev, fn_attention_absorb_kvdev) RESOLVE(attention_project_batch, fn_attention_project_batch) + RESOLVE(attention_project_ragged, fn_attention_project_ragged) RESOLVE(attention_project_batch_dev, fn_attention_project_batch_dev) RESOLVE(attention_project_batch_dev_out, fn_attention_project_batch_dev_out) RESOLVE(pipe_add, fn_pipe_add) @@ -342,6 +348,15 @@ int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,ColiCudaTensor *o_pro return g_cuda.attention_project_batch(kv_b, o_proj, out, q, latent, rope, S, H, Q, R, V, K, T, attention_scale); } +int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, + float *out,const float *q,const void *const *keys, + const float *const *latent,const float *const *rope, + const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale){ + if(!coli_cuda_load()) return 0; + return g_cuda.attention_project_ragged(kv_b,o_proj,out,q,keys,latent,rope,lengths, + S,H,Q,R,V,K,max_t,attention_scale); +} + int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){ if(!g_cuda.available){ return 0; } return g_cuda.attention_project_batch_dev(kv_b, o_proj, out, q_dev, latent_dev, rope_dev, S, H, Q, R, V, K, T, scale); diff --git a/c/coli b/c/coli index 4c27415..803c22b 100755 --- a/c/coli +++ b/c/coli @@ -20,7 +20,7 @@ Configuration through environment variables or flags (also valid after the subco --topp P adaptive expert top-p --topk N fixed top-k --ngen N maximum response tokens --cap N cache slots/layer """ -import os, sys, subprocess, argparse, json, time, signal, shutil, threading, re, codecs, tempfile, textwrap +import os, sys, subprocess, argparse, json, time, signal, shutil, threading, re, codecs, tempfile, textwrap, struct # The engine mmaps every shard (144+ files); macOS default RLIMIT_NOFILE is 256. if sys.platform != "win32": @@ -40,6 +40,8 @@ if sys.platform == "win32": except (AttributeError, OSError): pass HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +from version import __version__ as _version # Run-in-place (source checkout, "cd c && ./coli ..."): the engine, the # support modules (resource_plan.py, doctor.py, openai_server.py) and @@ -115,7 +117,7 @@ def sprite_lines(): def banner(sub=""): sp=sprite_lines() txt=[ - f"{C.teal}{C.b}colibrì{C.r} {C.dim}v1.0{C.r}", + f"{C.teal}{C.b}colibrì{C.r} {C.dim}v{_version}{C.r}", f"{C.dim}tiny engine, immense model{C.r}", f"{C.gray}GLM-5.2 · 744B MoE · int4 · streaming CPU{C.r}", f"{C.dgray}{sub}{C.r}" if sub else "", @@ -141,12 +143,24 @@ def need_model(model): sys.exit(f"{C.yel}engine is not built.{C.r} Run: coli build") def cuda_binary(): - if not os.path.exists(GLM) or sys.platform != "linux": return False - try: - linked=subprocess.run(["ldd",GLM],capture_output=True,text=True,timeout=3) - return any("libcudart" in line and "not found" not in line - for line in linked.stdout.splitlines()) - except (OSError,subprocess.SubprocessError): return False + if not os.path.exists(GLM): return False + if sys.platform == "linux": + try: + linked=subprocess.run(["ldd",GLM],capture_output=True,text=True,timeout=3) + return any("libcudart" in line and "not found" not in line + for line in linked.stdout.splitlines()) + except (OSError,subprocess.SubprocessError): return False + if sys.platform == "win32": + # Windows CUDA_DLL=1 builds never link libcudart directly: glm.exe loads + # coli_cuda.dll at runtime via LoadLibrary (backend_loader.c), so there's no + # import-table entry for ldd/dumpbin to see. Detect the COLI_CUDA build via a + # marker string baked into glm.c's #ifdef COLI_CUDA block instead, and require + # coli_cuda.dll to actually sit next to glm.exe (else CUDA init fails at startup). + try: + with open(GLM,"rb") as f: built=b"[CUDA] mode: routed experts" in f.read() + except OSError: return False + return built and os.path.exists(os.path.join(os.path.dirname(GLM),"coli_cuda.dll")) + return False def resource_request(a, env): ctx=a.ctx or int(env.get("CTX",4096)) @@ -480,13 +494,141 @@ def cmd_run(a): need_model(a.model) prompt=" ".join(a.prompt) if a.prompt else sys.exit('usage: coli run "your prompt"') banner("run") - # template ufficiale GLM-5.2: niente \n dopo i ruoli; = risposta diretta (nothink) - e=env_for(a); e["PROMPT"]=f"[gMASK]<|user|>{prompt}<|assistant|>" + # template ufficiale GLM-5.2: niente \n dopo i ruoli; = risposta diretta (nothink). + # THINK=1 lascia aperto, stessa convenzione del serve mode (glm.c). EN: THINK=1 leaves + # open so the engine emits its reasoning block; the default stays nothink. + tk="" if os.environ.get("THINK","0")=="1" else "" + e=env_for(a); e["PROMPT"]=f"[gMASK]<|user|>{prompt}<|assistant|>{tk}" sys.exit(subprocess.call([GLM, str(a.cap)], env=e)) +def server_probe(base, api_key=None, timeout=1.5): + """Is a coli serve alive at `base`? Returns its model_id, or None. + Probes /health then /v1/models — both cheap, neither touches the engine.""" + import urllib.request, urllib.error + def get(path): + req=urllib.request.Request(base.rstrip("/")+path) + if api_key: req.add_header("Authorization", f"Bearer {api_key}") + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read().decode("utf-8","replace")) + try: + if get("/health").get("status")!="ok": return None + data=get("/v1/models").get("data") or [] + return data[0]["id"] if data else None + except Exception: + return None + +def chat_attached(a, base, model_id): + """The chat REPL over HTTP against a running `coli serve`. + + Why this exists (the cold-chat cost, measured): spawning a private engine + pays 34-136 s of resident load on EVERY start, and begins with an empty + expert cache — hit rate 4% cold vs 55% warm, a ~10x on early decode. A + resident server pays load once and keeps the LRU warm across sessions; + its KV slots reuse the conversation prefix, so a continued chat skips + re-prefill too. The engine byte-protocol stays untouched — this is plain + OpenAI SSE over localhost, stdlib only.""" + import urllib.request + print(f" {C.grn}✦ attached{C.r} {C.dim}to {base} · model {model_id} · the engine stays warm after you quit{C.r}") + print(f" {C.dim}type and press Enter · Ctrl-C stops the answer · :reset starts a new conversation · :q exits{C.r}\n") + msgs=[] + w=term_w()-4 + while True: + if TTY: + print(f" {C.dgray}╭{'─'*w}╮{C.r}") + try: msg=input(f" {C.dgray}│{C.r} {C.teal}{C.b}›{C.r} ") + except EOFError: print(); break + print(f" {C.dgray}╰{'─'*w}╯{C.r}") + else: + try: msg=input() + except EOFError: break + msg=msg.strip() + if msg in (":q",":quit","exit"): break + if not msg: continue + if msg==":reset": msgs=[]; print(f" {C.dim}✦ new conversation{C.r}\n"); continue + msgs.append({"role":"user","content":msg}) + body=json.dumps({"model":model_id,"messages":msgs,"stream":True, + "max_tokens":a.ngen}).encode() + req=urllib.request.Request(base.rstrip("/")+"/v1/chat/completions", data=body, + headers={"Content-Type":"application/json"}) + if a.api_key: req.add_header("Authorization", f"Bearer {a.api_key}") + print(f"\n {C.teal}◆ colibrì{C.r}") + sp=Spinner("thinking…"); sp.start() + md=MDStream(" "); reply=[]; first=True; t0=time.time(); interrupted=False + try: + with urllib.request.urlopen(req) as r: + for raw in r: + line=raw.decode("utf-8","replace").strip() + if not line.startswith("data: "): continue + data=line[6:] + if data=="[DONE]": break + try: ev=json.loads(data) + except ValueError: continue + for ch in ev.get("choices",[]): + d=ch.get("delta",{}) + txt=d.get("content") + if not txt: continue # ping/ruolo/reasoning: non è testo + if first: sp.stop(); first=False + md.feed(txt); reply.append(txt) + except KeyboardInterrupt: + interrupted=True # il server annulla la richiesta alla disconnessione + except OSError as e: + sp.stop() + print(f"\n {C.yel}[server unreachable: {e}]{C.r}"); break + sp.stop(); md.close() + if reply: msgs.append({"role":"assistant","content":"".join(reply)}) + else: msgs.pop() # turno vuoto: non sporcare la history + el=time.time()-t0 + note=" · ⏹ interrupted" if interrupted else "" + print(f"\r {C.dgray}└─ ~{len(''.join(reply))//4} tok · {el:.0f}s{note}{C.r}\n") + print(f" {C.dim}goodbye — the engine keeps running for the next chat 🐦{C.r}") + +def kv_resume_notice(model_dir): + """SERVE mode silently resumes .coli_kv from disk (glm.c kv_disk_load): a chat + started today continues a conversation from days ago, with `first=0` so the + turn is appended WITHOUT the [gMASK] prefix. The engine does announce it + on stderr — but nothing here ever shows that: the drain thread's + p.stderr.read() blocks until EOF, so on a healthy start errlog is still empty + when the status lines are printed. The warning only appeared once the engine + DIED, which is exactly when it no longer mattered. + + Measured cost of the silence: a chat inherited 670 tokens of an old Italian + session ("il mio numero preferito e 7, ricordalo!"). Every later reply came + back in Italian, and "explain fibonacci in short" was answered about the + number 7 — the model was being coherent with a context nobody could see, and + it read as a quantization bug for a day. + + So say it here, in Python, from the file itself: no pipe, no thread, no + Windows deadlock risk (see the stderr comment below).""" + p=os.path.join(model_dir, ".coli_kv") + try: + with open(p,"rb") as f: + if f.read(8)!=b"COLIKV1\0": return + h=struct.unpack("<8i", f.read(32)) + n=h[6] + if n<1: return + age=time.time()-os.path.getmtime(p) + when=f"{age/86400:.0f}d ago" if age>86400 else f"{age/3600:.0f}h ago" if age>3600 else "just now" + print(f" {C.yel}↺ resuming a saved conversation: {n} tokens, last written {when}{C.r}") + print(f" {C.dgray} it steers tone, language and topic. :reset clears it · " + f"KVSAVE=0 disables saving · delete {p} to start clean{C.r}") + except (OSError, struct.error): pass + def cmd_chat(a): + # ATTACH: a running `coli serve` beats a private engine every time — the load + # (34-136 s) and the cache warmth survive between sessions. Explicit --attach + # wins; otherwise probe localhost quietly and use it if it's there. --no-attach + # forces the old behaviour. The probe costs ~1 ms when nothing is listening. + if not getattr(a,"no_attach",False): + base=getattr(a,"attach",None) or "http://127.0.0.1:8000" + mid=server_probe(base, getattr(a,"api_key",None)) + if mid: + banner(f"chat · {mid} · attached") + chat_attached(a, base, mid); return + if getattr(a,"attach",None): + sys.exit(f"--attach: no coli serve answering at {base} (start one with: coli serve --model )") need_model(a.model) banner(f"chat · {os.path.basename(a.model)} · ram {a.ram or '-'}GB · topp {a.topp or 'off'}") + kv_resume_notice(a.model) errlog=tempfile.NamedTemporaryFile(mode="w+", suffix=".log", delete=False) e=env_for(a); e["SERVE"]="1" # stderr -> PIPE, NOT stderr=errlog (file). On Windows/MinGW, pointing the @@ -619,12 +761,65 @@ def cmd_chat(a): except Exception: pass print(f" {C.teal}goodbye{C.r} {C.dim}— the hummingbird returns to its nest{C.r} 🐦\n") +def serve_pidfile(port): return os.path.join(tempfile.gettempdir(), f"coli-serve-{port}.pid") + def cmd_serve(a): need_model(a.model) + # pidfile: cosi' `coli stop` spegne tutto con un comando, senza pkill a mano. + # EN: pidfile so `coli stop` can shut everything down without manual pkill. + try: + with open(serve_pidfile(a.port),"w") as f: f.write(f"{os.getpid()} {a.model}\n") + except OSError: pass from openai_server import serve - serve(a.model, a.host, a.port, a.model_id, a.api_key, - a.cap,a.ngen,GLM,env_for(a),a.cors_origin, - a.max_queue,a.queue_timeout,a.kv_slots) + try: + serve(a.model, a.host, a.port, a.model_id, a.api_key, + a.cap,a.ngen,GLM,env_for(a),a.cors_origin, + a.max_queue,a.queue_timeout,a.kv_slots) + finally: + try: os.unlink(serve_pidfile(a.port)) + except OSError: pass + +def cmd_stop(a): + """Shut down a running `coli serve` AND its engine — one command, no pkill. + The engine re-execs itself for OMP tuning, so its process is named `exe`, + not `glm`: every `pkill -x glm` in history silently killed nothing (that is + how two 17+5 GB ghost engines OOM'd this box on 2026-07-16). This finds the + real processes: the pidfile first, then /proc by cmdline/environ — only + processes that are demonstrably ours (SERVE=1 + our SNAP, or `coli serve` + in the command line).""" + banner("stop") + targets=[] # (pid, descrizione) + pf=serve_pidfile(a.port) + try: + pid=int(open(pf).read().split()[0]) + os.kill(pid,0); targets.append((pid,f"coli serve (pidfile, port {a.port})")) + except (OSError,ValueError,IndexError): pass + for pd in os.listdir("/proc"): + if not pd.isdigit(): continue + pid=int(pd) + try: + cmd=open(f"/proc/{pd}/cmdline","rb").read().replace(b"\0",b" ").decode("utf-8","replace") + if "coli" in cmd and " serve" in cmd and pid!=os.getpid(): + if not any(p==pid for p,_ in targets): targets.append((pid,"coli serve (cmdline)")) + comm=open(f"/proc/{pd}/comm").read().strip() + if comm in ("glm","exe","olmoe"): + env=open(f"/proc/{pd}/environ","rb").read().replace(b"\0",b"\n").decode("utf-8","replace") + if "SERVE=1" in env: targets.append((pid,f"engine `{comm}` (SERVE=1)")) + except (OSError,PermissionError): continue + if not targets: + print(f" nothing running — no serve on port {a.port}, no SERVE engines"); return + for pid,desc in targets: print(f" {'would stop' if a.dry_run else 'stopping'} {pid}: {desc}") + if a.dry_run: return + for pid,_ in targets: + try: os.kill(pid, signal.SIGTERM) + except OSError: pass + time.sleep(2.0) + for pid,_ in targets: + try: os.kill(pid, signal.SIGKILL); print(f" {pid}: forced (SIGKILL)") + except OSError: pass # gia' morto: bene + try: os.unlink(pf) + except OSError: pass + print(f" {C.grn}✓ stopped{C.r} — RAM released") def cmd_web(a): """serve + open the dashboard in the browser once the API answers.""" @@ -712,6 +907,7 @@ def main(): common.add_argument("--topp", type=float, default=0); common.add_argument("--topk", type=int, default=0) common.add_argument("--temp", type=float, default=None) # temperatura token (0=greedy, default 1.0+nucleus .95) ap=argparse.ArgumentParser(prog="coli", parents=[common], description="colibrì — run GLM-5.2 locally") + ap.add_argument("--version", action="version", version=f"colibrì {_version}") sub=ap.add_subparsers(dest="cmd") sub.add_parser("build", parents=[common]); sub.add_parser("info", parents=[common]) pp=sub.add_parser("plan",parents=[common]) @@ -719,7 +915,14 @@ def main(): pd=sub.add_parser("doctor",parents=[common]) pd.add_argument("--json",action="store_true",help="emit a versioned JSON report") pr=sub.add_parser("run", parents=[common]); pr.add_argument("prompt", nargs="*") - sub.add_parser("chat", parents=[common]) + pc=sub.add_parser("chat", parents=[common]) + pc.add_argument("--attach", nargs="?", const="http://127.0.0.1:8000", default=None, + help="chat against a running `coli serve` instead of spawning an engine " + "(keeps the model loaded and the expert cache warm across chat sessions). " + "Bare --attach probes localhost:8000.") + pc.add_argument("--no-attach", action="store_true", + help="never auto-attach, always spawn a private engine") + pc.add_argument("--api-key", default=os.environ.get("COLI_API_KEY")) ps=sub.add_parser("serve", parents=[common]) ps.add_argument("--host",default="127.0.0.1"); ps.add_argument("--port",type=int,default=8000) ps.add_argument("--model-id",default=os.environ.get("COLI_MODEL_ID","glm-5.2-colibri")) @@ -728,6 +931,8 @@ def main(): ps.add_argument("--max-queue",type=int,default=int(os.environ.get("COLI_MAX_QUEUE","8"))) ps.add_argument("--queue-timeout",type=float,default=float(os.environ.get("COLI_QUEUE_TIMEOUT","300"))) ps.add_argument("--kv-slots",type=int,default=int(os.environ.get("COLI_KV_SLOTS","1"))) + pst=sub.add_parser("stop", parents=[common], help="shut down a running coli serve and its engine") + pst.add_argument("--port",type=int,default=8000); pst.add_argument("--dry-run",action="store_true") pw=sub.add_parser("web", parents=[common], help="serve + open the dashboard in a browser") for arg,kw in (("--host",dict(default="127.0.0.1")),("--port",dict(type=int,default=8000)), ("--model-id",dict(default=os.environ.get("COLI_MODEL_ID","glm-5.2-colibri"))), @@ -751,7 +956,7 @@ def main(): pc.add_argument("--no-mtp",action="store_true",help="skip the MTP head (no speculative drafts)") a=ap.parse_args() handler={"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"doctor":cmd_doctor, - "run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"bench":cmd_bench, + "run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"stop":cmd_stop,"bench":cmd_bench, "convert":cmd_convert,"web":cmd_web}.get(a.cmd) if handler: sys.exit(handler(a) or 0) banner(); print(__doc__) diff --git a/c/compat.h b/c/compat.h index 82de8b6..1b56f5f 100644 --- a/c/compat.h +++ b/c/compat.h @@ -80,6 +80,7 @@ static inline int compat_open_direct(const char *path){ #endif #include #include +#include /* _mkdir (for the mkdtemp shim below) */ #include #include #include @@ -143,6 +144,12 @@ static inline int compat_fadvise(int fd, off_t off, off_t len, int advice){ * Thread-safe (no shared seek position). Gestisce offset >4 GB e chunking * per letture >2 GB (anche se i tensori individuali sono nell'ordine dei * MB-centinaia di MB, il wrapper e' robusto per ogni taglia). */ +/* Ultimo GetLastError() di una ReadFile fallita, per thread: il chiamante + * (pread_full in glm.c) lo stampa accanto a strerror. Senza questo, OGNI + * fallimento Windows collassa in "EIO -> Input/output error" e la diagnosi + * dal campo diventa un tirare a indovinare (#307: tre giri di ipotesi tra + * tre persone perche' il codice vero non compariva da nessuna parte). */ +static __thread DWORD compat_pread_lasterr __attribute__((unused)); static inline ssize_t compat_pread(int fd, void *buf, size_t n, off_t off){ intptr_t osfh = _get_osfhandle(fd); if(osfh == -1 || osfh == -2){ errno = EBADF; return -1; } @@ -158,6 +165,7 @@ static inline ssize_t compat_pread(int fd, void *buf, size_t n, off_t off){ if(!ReadFile(h, (char*)buf + total, chunk32, &rd, &ov)){ DWORD err = GetLastError(); if(err == ERROR_HANDLE_EOF) break; /* past EOF → return bytes read (0 if none, matching POSIX pread) */ + compat_pread_lasterr = err; /* preserva il codice VERO per il report (#307) */ if(err == ERROR_INVALID_HANDLE || err == ERROR_INVALID_FUNCTION) errno = EBADF; else errno = EIO; return -1; @@ -237,7 +245,9 @@ static inline int compat_rename(const char *old, const char *new){ /* --- rss_gb: getrusage -> GetProcessMemoryInfo --- * ru_maxrss in KB (come Linux): rss_gb() divide per 1e6 → GB corretti. */ #include -#pragma comment(lib, "psapi.lib") +#ifdef _MSC_VER +#pragma comment(lib, "psapi.lib") /* MSVC: link psapi; MinGW/GCC uses -lpsapi */ +#endif struct rusage { long ru_maxrss; }; #define RUSAGE_SELF 0 static inline int getrusage(int who, struct rusage *r){ @@ -304,8 +314,52 @@ static inline int compat_setenv(const char *name, const char *value, int overwri } #define setenv(name,value,overwrite) compat_setenv(name,value,overwrite) +/* --- getenv_utf8: read an env var as UTF-8, not through the ANSI codepage --- + * Plain getenv()/_environ are populated by the CRT from the ANSI-codepage view + * of the process environment block, not UTF-8. A parent that hands the child a + * Unicode value via CreateProcessW's wide env block (e.g. Python's subprocess + * module, which coli uses to pass the chat prompt) round-trips correctly only + * through GetEnvironmentVariableW; going through narrow getenv() re-encodes it + * via CP_ACP first, so any non-ASCII prompt text (Cyrillic, CJK, ...) comes out + * corrupted before the byte-level tokenizer ever sees it. Read the wide value + * directly and convert straight to UTF-8, bypassing the ANSI codepage entirely. + * Returned buffer is intentionally leaked: called a handful of times at + * startup, lives for the process. */ +static inline const char *compat_getenv_utf8(const char *name){ + wchar_t wname[64]; + if(MultiByteToWideChar(CP_UTF8, 0, name, -1, wname, 64) <= 0) return getenv(name); + DWORD need = GetEnvironmentVariableW(wname, NULL, 0); + if(!need) return NULL; + wchar_t *wval = (wchar_t*)malloc(need * sizeof(wchar_t)); + if(!wval) return NULL; + GetEnvironmentVariableW(wname, wval, need); + int blen = WideCharToMultiByte(CP_UTF8, 0, wval, -1, NULL, 0, NULL, NULL); + char *val = blen>0 ? (char*)malloc((size_t)blen) : NULL; + if(val) WideCharToMultiByte(CP_UTF8, 0, wval, -1, val, blen, NULL, NULL); + free(wval); + return val; +} +#define getenv_utf8(name) compat_getenv_utf8(name) + +/* --- mkdtemp -> _mktemp + _mkdir (POSIX mkdtemp assente su Windows) --- + * Test binaries (test_stops.c) create a scratch dir in the CWD via a + * "name_XXXXXX" template; POSIX mkdtemp fills the X's and mkdirs 0700. The + * Windows CRT has _mktemp (in-place, same XXXXXX contract) so we compose it. + * Returns the template pointer on success, NULL on failure — matching POSIX. */ +static inline char *compat_mkdtemp(char *tmpl){ + if(!tmpl) return NULL; + if(!_mktemp(tmpl)) return NULL; /* fills the trailing X's in place */ + if(_mkdir(tmpl) != 0) return NULL; /* EEXIST is impossible post-_mktemp */ + return tmpl; +} +#define mkdtemp(tmpl) compat_mkdtemp(tmpl) + #endif /* _WIN32 */ +#ifndef getenv_utf8 +#define getenv_utf8(name) getenv(name) +#endif + /* --- compat_aligned_free su piattaforme diverse da Windows --- * Su Linux/macOS, posix_memalign usa free() normale. */ #ifndef compat_aligned_free diff --git a/c/doctor.py b/c/doctor.py index 4cb5a42..cbe9c87 100644 --- a/c/doctor.py +++ b/c/doctor.py @@ -19,16 +19,33 @@ def _check(identifier, status, summary, **details): def cuda_linkage(engine_path): """Return CUDA linkage state without loading the executable or CUDA runtime.""" - if not Path(engine_path).is_file() or os.name != "posix": + engine = Path(engine_path) + if not engine.is_file(): return {"linked": False, "missing": False} - try: - result = subprocess.run(["ldd", str(engine_path)], capture_output=True, text=True, - timeout=3, check=False) - except (OSError, subprocess.SubprocessError): - return {"linked": False, "missing": False} - lines = [line for line in result.stdout.splitlines() if "libcudart" in line] - return {"linked": any("not found" not in line for line in lines), - "missing": any("not found" in line for line in lines)} + if os.name == "posix": + try: + result = subprocess.run(["ldd", str(engine)], capture_output=True, text=True, + timeout=3, check=False) + except (OSError, subprocess.SubprocessError): + return {"linked": False, "missing": False} + lines = [line for line in result.stdout.splitlines() if "libcudart" in line] + return {"linked": any("not found" not in line for line in lines), + "missing": any("not found" in line for line in lines)} + if sys.platform == "win32": + # Windows CUDA_DLL=1 builds never link libcudart directly: glm.exe loads + # coli_cuda.dll at runtime via LoadLibrary (backend_loader.c), so there's no + # import-table entry for ldd/dumpbin to see. Detect the COLI_CUDA build via a + # marker string baked into glm.c's #ifdef COLI_CUDA block instead, and require + # coli_cuda.dll to actually sit next to glm.exe (else CUDA init fails at startup). + try: + built = b"[CUDA] mode: routed experts" in engine.read_bytes() + except OSError: + return {"linked": False, "missing": False} + if not built: + return {"linked": False, "missing": False} + dll_present = (engine.parent / "coli_cuda.dll").is_file() + return {"linked": dll_present, "missing": not dll_present} + return {"linked": False, "missing": False} def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *, diff --git a/c/glm.c b/c/glm.c index 00c7076..a3e7ab7 100644 --- a/c/glm.c +++ b/c/glm.c @@ -34,9 +34,15 @@ #if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) #include #include /* mlock: inchioda le pagine in RAM / wire pages into RAM */ +#ifdef __linux__ +#include /* COLI_NUMA: mbind degli slab expert / expert-slab interleave */ +#endif #include /* fstat per mmap degli shard (COLI_MMAP) */ #include /* SIGINT = stop morbido del turno in serve mode */ #endif +#ifdef __linux__ +#include /* statfs: real fs-type check for the 9p warning (below) */ +#endif #if defined(_WIN32) && (defined(__x86_64__) || defined(__i386__)) #include /* hwinfo_emit: CPU brand string senza /proc */ #endif @@ -190,12 +196,18 @@ typedef struct { uint64_t mtp_prop, mtp_acc; /* statistica acceptance */ int **eroute; int *enr; /* metodo C: routing dell'ULTIMO token per layer */ uint64_t eclock, hits, miss, ereq; + uint64_t hit_pin, hit_ecache; /* split di hits per tier (#336): pin vs LRU ecache */ uint64_t gpu_expert_calls; int gpu_expert_count; int64_t gpu_expert_bytes; uint64_t n_fw, n_emit; /* metodo E: forward di decode / token emessi */ uint64_t route_slots, route_swaps; /* CACHE_ROUTE: slots chosen / substituted vs true top-K */ uint64_t route_agree_hit, route_agree_tot; /* ROUTE_AGREE: |chosen ∩ true top-K| / K */ double route_kl_sum; uint64_t route_kl_n; /* mean KL(true||chosen) on gate mass */ - double t_edisk, t_ewait, t_emm, t_attn, t_kvb, t_head;/* profiling: dove va il tempo */ + double t_ewait, t_emm, t_ecpu, t_egpu, t_route, t_p2p, t_attn, t_kvb, t_head; + uint64_t n_p2p; /* P0 execution profile: tier split + residual hops */ + uint64_t cpu_expert_rows; int64_t cpu_expert_bytes; + /* profiling: dove va il tempo (wall del + * thread di compute; il servizio disco + * overlappato vive in g_edisk_ns) */ double t_aproj,t_acore,t_aout; /* attention breakdown */ int64_t resident_bytes; /* DISK_SPLIT=1: split dei DISK LOAD (miss LRU -> expert_load) per contesto e per tipo @@ -212,6 +224,7 @@ static void ehit_mark(Model *m, int layer, int eid); static void emap_emit(Model *m); static void hits_emit(Model *m); static void hwinfo_emit(Model *m); +static int64_t expert_bytes_probe(Model *m, int ebits); /* PROF: tier sizes in the report */ static int g_repin; static uint64_t g_last_repin; #ifdef COLI_CUDA @@ -277,6 +290,42 @@ static double rss_gb(void){ struct rusage r; getrusage(RUSAGE_SELF,&r); return r.ru_maxrss/(1024.0*1024.0); /* Linux: in KB */ #endif } +/* ---- PROF=1: opt-in performance profile ---------------------------------- + * Records per-forward decode latency and expert-file bytes fetched, then + * reports percentiles, I/O totals, phase shares and a tuning verdict next to + * the existing PROFILE line. Additive only: with PROF unset the output of + * every mode stays byte-identical. */ +static int g_prof=0; +static _Atomic int64_t g_prof_io; /* bytes pread()/faulted from expert files */ +/* Disk service: wall time inside expert_load on whichever thread runs the read + * (PIPE I/O workers, OMP loaders, the speculative pilot). It overlaps compute, + * so it is NOT a wall-time phase — the stall the compute thread actually felt + * is m->t_ewait. Thread-seconds, so it can exceed wall time under parallel + * reads; wait << service means overlap/parallelism is hiding the reads, + * wait ~ service means the loads block the compute thread. */ +static _Atomic int64_t g_edisk_ns; +static double edisk_s(void){ return atomic_load_explicit(&g_edisk_ns,memory_order_relaxed)*1e-9; } +#define PROF_LAT_CAP 32768 +static double g_prof_lat[PROF_LAT_CAP]; /* per-forward decode wall clock (ring) */ +static uint64_t g_prof_nlat; /* forwards recorded (monotonic) */ +static void prof_lat(double s){ g_prof_lat[g_prof_nlat++ % PROF_LAT_CAP]=s; } +/* snapshot for windowed reports (serve mode: one report per turn) */ +typedef struct { + double edisk,ewait,emm,ecpu,egpu,route,p2p,attn,head; + int64_t io,cpu_bytes; uint64_t hits,miss,ereq,n_fw,n_emit,nlat,n_p2p,cpu_rows; + uint64_t hit_pin,hit_ecache; +} ProfBase; +static void prof_base(Model *m, ProfBase *b){ + b->edisk=edisk_s(); b->ewait=m->t_ewait; b->emm=m->t_emm; + b->ecpu=m->t_ecpu; b->egpu=m->t_egpu; b->route=m->t_route; b->p2p=m->t_p2p; + b->attn=m->t_attn; b->head=m->t_head; + b->io=atomic_load_explicit(&g_prof_io,memory_order_relaxed); + b->hits=m->hits; b->miss=m->miss; b->ereq=m->ereq; + b->hit_pin=m->hit_pin; b->hit_ecache=m->hit_ecache; + b->n_fw=m->n_fw; b->n_emit=m->n_emit; b->nlat=g_prof_nlat; b->n_p2p=m->n_p2p; + b->cpu_bytes=m->cpu_expert_bytes;b->cpu_rows=m->cpu_expert_rows; +} + static float *falloc(int64_t n){ /* guardia anti-wrap (report PR #25): n assurdo da file modello ostili non deve * diventare una malloc piccola. Niente calloc: il memset nel percorso caldo costa. */ @@ -1159,13 +1208,50 @@ static int g_disk_split=0; /* DISK_SPLIT=1: contatori che spezzano i DISK LOAD ( * non vengono stampate. Solo misura: nessun effetto sull'output. */ /* Aligned allocator for dense QT weights/scales: under METAL, page-align + register so the * GPU reads them zero-copy (no upload duplicate). Plain malloc otherwise. */ +/* ---- COLI_NUMA=1 (#82): interleave the expert slabs across NUMA nodes ---- + * On multi-socket hosts first-touch parks nearly the whole pin+LRU on the loader + * thread's node (measured: node0 766MB free / node1 idle), and every far-socket + * core then streams weights over the interconnect. Interleaving ONLY the expert + * slabs recruits all memory controllers: +7%/-14% expert-matmul on 2 sockets, + * +40% on a 4-socket (#82). Blanket `numactl --interleave=all` is NOT equivalent: + * it also interleaves the CUDA pinned staging buffers and cost a 4-socket GPU host + * 10x (#82) — hence per-region mbind here and nothing else. Raw syscall, no libnuma + * dependency; MPOL_MF_MOVE migrates pages of reused heap chunks too. Linux-only, + * silent no-op elsewhere or on single-node hosts. */ +#ifdef __linux__ +static int g_numa_nodes=0; /* only touched under __linux__; off-Linux NUMA is a no-op */ +#endif +static void numa_slab_bind(void *p, size_t n){ +#ifdef __linux__ + if(g_numa_nodes<2 || !p || !n) return; + unsigned long mask=(1UL<=2) fprintf(stderr,"[NUMA] expert slabs interleaved across %d nodes\n",g_numa_nodes); + else fprintf(stderr,"[NUMA] single node: COLI_NUMA ignored\n"); +#endif +} + static void *qalloc(size_t n){ #ifdef COLI_METAL if(g_metal_enabled){ void *p; size_t r=(n+16383)&~(size_t)16383; if(posix_memalign(&p,16384,r)){fprintf(stderr,"OOM qalloc\n");exit(1);} coli_metal_register(p,r); return p; } #endif - return malloc(n); + void *p=malloc(n); + if(n>=(size_t)1<<20) numa_slab_bind(p,n); /* resident dense weights too (#82: attention/shared stream from RAM every token) */ + return p; } static float *qsalloc(int O){ return (float*)qalloc((size_t)O*sizeof(float)); } static int g_pilot_real=0;/* PILOT_REAL=1: il pilota fa LOAD VERI cross-layer dentro ecache[L+1] @@ -1277,6 +1363,35 @@ static void load_cfg(Cfg *c, const char *snap){ if(eo){ if(eo->t==J_NUM) c->stop_ids[c->n_stop++]=(int)eo->num; else if(eo->t==J_ARR) for(int i=0;ilen && c->n_stop<8;i++) c->stop_ids[c->n_stop++]=(int)eo->kids[i]->num; } + /* generation_config.json e' il file AUTOREVOLE per la generazione secondo HuggingFace: + * config.json ne porta spesso una copia legacy o parziale. Un tool di conversione che + * rigenera un config.json ridotto lascia il motore fermo su MENO stop del dovuto, e i + * token di controllo che restano finiscono stampati in chat come testo (woolcoxm, #298: + * "the stop token being printed to chat", verificato sui token id). Unione dei due: + * uno stop in piu' non fa danno, uno in meno si' -- e chi converte i pesi non siamo noi. + * EN: generation_config.json is HF's authority for generation; config.json often carries + * a partial legacy copy. Union both -- an extra stop is harmless, a missing one is not. */ + { char gp[2100]; snprintf(gp,sizeof(gp),"%s/generation_config.json",snap); + FILE *gf=fopen(gp,"rb"); /* assente = nessun problema: e' opzionale */ + if(gf){ + fseek(gf,0,SEEK_END); long gn=ftell(gf); fseek(gf,0,SEEK_SET); + if(gn>0){ + char *gb=malloc(gn+1); size_t gg=fread(gb,1,gn,gf); gb[gg]=0; + char *ga=NULL; jval *gr=json_parse(gb,&ga); + jval *ge=gr?json_get(gr,"eos_token_id"):NULL; + if(ge){ + int add[8], na=0; + if(ge->t==J_NUM) add[na++]=(int)ge->num; + else if(ge->t==J_ARR) for(int i=0;ilen && na<8;i++) add[na++]=(int)ge->kids[i]->num; + for(int i=0;in_stop<8;i++){ + int dup=0; for(int j=0;jn_stop;j++) if(c->stop_ids[j]==add[i]) dup=1; + if(!dup) c->stop_ids[c->n_stop++]=add[i]; + } + } + free(ga); free(gb); + } + fclose(gf); + } } /* DSA lightning indexer: parametri + tipo per-layer (lista esplicita o formula freq/offset) */ c->index_topk=gi(r,"index_topk"); c->index_nh=gi(r,"index_n_heads"); c->index_hd=gi(r,"index_head_dim"); { jval *it=json_get(r,"indexer_types"); @@ -1542,7 +1657,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits c->index_topk, c->index_topk); } } - m->hlast=falloc(D); m->h_all=falloc((int64_t)64*D); + m->hlast=falloc(D); m->h_all=falloc((int64_t)512*D); /* byte della parte DENSA residente (embed+lm_head+attn+mlp densa+shared+norme) */ int64_t rb=qt_bytes(&m->embed)+qt_bytes(&m->lm_head); @@ -1630,15 +1745,21 @@ static int pread_full(int fd, void *buf, int64_t n, int64_t off, const char *tag while(gotg); qt_from_disk(m,nm[1],I,D,b,g_drop,&s->u); qt_from_disk(m,nm[2],D,I,b,g_drop,&s->d); + atomic_fetch_add_explicit(&g_prof_io, + st_nbytes(&m->S,nm[0])+st_nbytes(&m->S,nm[1])+st_nbytes(&m->S,nm[2]),memory_order_relaxed); s->eid=eid; return 0; } st_tensor *tw[3], *tq[3]; @@ -1709,6 +1832,7 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){ * leak locked pages for every GPU-tier expert). See pin_wire() below: it wires * the final resident set only, after GPU release has already nulled out the * pointers for anything that isn't genuinely RAM-tier. */ + atomic_fetch_add_explicit(&g_prof_io,(int64_t)(n+nq),memory_order_relaxed); } s->eid=eid; return 0; } @@ -1730,6 +1854,7 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){ compat_aligned_free(s->slab); if(posix_memalign((void**)&s->slab,4096,wtot+8192)){fprintf(stderr,"OOM slab\n"); if(fatal) exit(1); s->slab=NULL; s->slab_cap=0; return -1;} s->slab_cap=wtot+8192; + numa_slab_bind(s->slab,(size_t)s->slab_cap); #endif } if(!s->fslab || ftot > s->fslab_cap){ @@ -1761,6 +1886,7 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){ } } s->fslab_cap=ftot; + numa_slab_bind(s->fslab,(size_t)ftot*sizeof(float)); #endif } int ord[3]={0,1,2}; /* ordina per offset nel file */ @@ -1796,6 +1922,7 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){ for(int k=0;k<3;k++){ if(pread_full(tq[k]->fd, (char*)(s->fslab+fo), tq[k]->nbytes, tq[k]->off, "pread qs")){ if(fatal) exit(1); return -1; } fp[k]=s->fslab+fo; fo+=tq[k]->nbytes/4; } + atomic_fetch_add_explicit(&g_prof_io,wtot+fo*4,memory_order_relaxed); if(g_drop){ /* scarta subito le pagine: evita che la page * cache in pressione strangoli il throughput */ posix_fadvise(tw[ord[0]]->fd, tw[ord[0]]->off, wtot, POSIX_FADV_DONTNEED); @@ -1813,6 +1940,14 @@ static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){ } s->eid=eid; return 0; } +/* Every expert read goes through here: time the whole load (pread/fault + + * bookkeeping) on the thread that runs it, into the disk-service counter. */ +static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){ + double t0=now_s(); + int rc=expert_load_impl(m,layer,eid,s,fatal); + atomic_fetch_add_explicit(&g_edisk_ns,(int64_t)((now_s()-t0)*1e9),memory_order_relaxed); + return rc; +} #ifdef __linux__ /* io_uring expert batches. One owner prepares all reads for a block, submits @@ -2157,6 +2292,18 @@ static void expert_prefetch(Model *m, int layer, int eid){ static void qt_addrow(const QT *t, int row, float coef, float *acc){ int I=t->I; if(t->fmt==0){ const float *w=t->qf+(int64_t)row*I; for(int i=0;ifmt==4){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); + int gs=t->gs, ng=(I+gs-1)/gs; const float *scl=t->s+(int64_t)row*ng; + for(int i=0;i+1>1]; + acc[i] +=coef*scl[i/gs] *((int)(b&0xF)-8); + acc[i+1]+=coef*scl[(i+1)/gs]*((int)(b>>4)-8); } + if(I&1){ uint8_t b=w[I>>1]; acc[I-1]+=coef*scl[(I-1)/gs]*((int)(b&0xF)-8); } return; } float c=coef*t->s[row]; if(t->fmt==1){ const int8_t *w=t->q8+(int64_t)row*I; for(int i=0;ifmt==2){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); @@ -2175,6 +2322,13 @@ static void qt_matvec_rows(const QT *t, int r0, int n, const float *x, float *y) else if(t->fmt==2){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); float s=t->s[row]; float acc=0; for(int i=0;i+1>1]; acc+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } if(I&1){ uint8_t b=w[I>>1]; acc+=((int)(b&0xF)-8)*x[I-1]; } a=acc*s; } + else if(t->fmt==4){ /* per-gruppo, come matmul_i4_grouped / per-group, as matmul_i4_grouped */ + const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); + int gs=t->gs, ng=(I+gs-1)/gs; const float *scl=t->s+(int64_t)row*ng; + for(int g=0; g*gsI?I:base+gs; float acc=0; + for(int i=base;i>1]; + acc+=(float)((i&1)?((int)(b>>4)-8):((int)(b&0xF)-8))*x[i]; } + a+=(double)acc*scl[g]; } } else { const uint8_t *w=t->q4+(int64_t)row*((I+3)/4); float s=t->s[row]; float acc=0; for(int i=0;i>2]; acc+=((int)((b>>((i&3)*2))&3)-2)*x[i]; } a=acc*s; } y[j]=(float)a; @@ -2188,6 +2342,43 @@ static int g_dsa_force=0; /* DSA_FORCE=1: selezione sempre attiva (test: top-min static int cmp_fdesc(const void *a,const void *b){ float x=*(const float*)a, y=*(const float*)b; return xy?-1:0; } +/* PARTIAL SELECT (quickselect, Hoare partition, DESCending). After this call the k + * LARGEST elements of a[0..n) are in a[0..k) in unspecified order; the (k+1)-th and + * beyond are untouched-or-smaller. O(n) average, O(n^2) pathological (mitigated by + * median-of-three below) — and unlike a full qsort it never orders more than needed. + * + * Why this exists (#356): the DSA top-keep in attention_rows previously full-qsorted + * all nk context scores (O(nk log nk)) per layer per token just to read ONE value -- + * the keep-th largest (the threshold). quickselect finds that pivot in O(nk) average, + * and the position-order scans that build dst[] are unchanged, so the kept set is + * bit-identical. Mirrors the sampling-side fix in #335 (heap partial-select there). + * + * NOT a stable partition: callers must derive the threshold and then re-scan the + * ORIGINAL array (the DSA code does exactly this) rather than reading a[0..k). */ +static void partial_select_desc(float *a, int n, int k){ + if(k<=0) return; + if(k>=n) return; /* nothing to partition: all kept */ + int lo=0, hi=n-1; + while(lo>1); + if(a[mid]>a[lo]){ float t=a[lo]; a[lo]=a[mid]; a[mid]=t; } + if(a[hi]>a[lo]){ float t=a[lo]; a[lo]=a[hi]; a[hi]=t; } + if(a[mid]>a[hi]){ float t=a[hi]; a[hi]=a[mid]; a[mid]=t; } + float piv=a[hi]; + int i=lo, j=hi; + for(;;){ + while(a[i]>piv) i++; /* desc: large values go left */ + while(j>lo && a[j]=j) break; + float t=a[i]; a[i]=a[j]; a[j]=t; i++; if(i>j) break; j--; + } + /* partition point: a[lo..i) are all >= piv, a[i..hi] are all <= piv */ + if(k<=i-1) hi=i-1; /* the k-th largest is in the left partition */ + else lo=i; /* it's in the right partition */ + } +} + /* attenzione MLA con KV-cache compressa, su token nuovi x[S,hidden], pos_base = pos del primo */ /* kvs/pos describe a ragged decode batch: each row may belong to a different * sequence. NULL keeps the original contiguous, currently-bound KV path. */ @@ -2470,10 +2661,14 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p } isc[t]=a*wsc; } - /* top-keep: soglia via qsort desc, poi scan in ordine di posizione */ + /* top-keep: threshold via PARTIAL SELECT (#356), poi scan in ordine di posizione. + * Era un qsort completo su nk (O(nk log nk)); quickselect estrae solo il + * keep-esimo valore piu' grande in O(nk) medio. La soglia (= min del blocco + * dei keep maggiori) e' identica a tmp[keep-1] del vecchio qsort, quindi i + * due scan qui sotto costruiscono dst[] bit-identical. */ float *tmp=falloc(nk); memcpy(tmp,isc,nk*sizeof(float)); - qsort(tmp,nk,sizeof(float),cmp_fdesc); - float thr=tmp[keep-1]; + partial_select_desc(tmp,nk,keep); + float thr=tmp[0]; for(int t=1;tdsa_sel+(int64_t)s*dtopk, nd=0; for(int t=0;tthr) dst[nd++]=t; for(int t=0;tn_kv_b_shard>1){ + if(kvs&&g_cuda_enabled&&getenv("COLI_CUDA_ATTN")&&atoi(getenv("COLI_CUDA_ATTN"))&& + !dnsel&&l->kv_b.cuda_eligible&&l->o.cuda_eligible&& + qt_cuda_upload(&l->kv_b)&&qt_cuda_upload(&l->o)){ + const float **rl=malloc((size_t)S*sizeof(*rl)),**rr=malloc((size_t)S*sizeof(*rr)); + const void **rk=malloc((size_t)S*sizeof(*rk)); + int *rn=malloc((size_t)S*sizeof(*rn)); int mt=0; + if(rk&&rl&&rr&&rn){ + for(int s=0;skv_start[layer]; rn[s]=pos+1-st0; + rk[s]=kvs[s]; + rl[s]=coli_kv_row(kvs[s]->Lc[layer],st0,kvl); + rr[s]=coli_kv_row(kvs[s]->Rc[layer],st0,c->qk_rope); + if(rn[s]>mt)mt=rn[s]; + } + cuda_core=cuda_projected=coli_cuda_attention_project_ragged(l->kv_b.cuda,l->o.cuda, + out,Q,rk,rl,rr,rn,S,H,c->qk_nope,c->qk_rope,vh,kvl,mt,c->attn_scale); + } + free(rk);free(rl);free(rr);free(rn); + } else if(cuda_absorb&&l->n_kv_b_shard>1){ int n=l->n_kv_b_shard,st0=m->kv_start[layer],nt=pos_base+S-st0,ok=1; float *qs=falloc((int64_t)S*H*qh),*cs=falloc((int64_t)S*H*vh); for(int d=0;dt_route+=now_s()-route_t0; if(g_route_fp) g_route_call++; if(g_couple && cp_pred && S<=8) for(int s2=0;s2pin[layer]; - for(int z=0;znpin[layer];z++) if(P[z].eid==eid){ m->hits++; use[j]=&P[z]; break; } + for(int z=0;znpin[layer];z++) if(P[z].eid==eid){ m->hits++; m->hit_pin++; use[j]=&P[z]; break; } if(!use[j]){ ESlot *Sl=m->ecache[layer]; int nn=m->ecn[layer]; - for(int z=0;zhits++; Sl[z].used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); use[j]=&Sl[z]; break; } } + for(int z=0;zhits++; m->hit_ecache++; Sl[z].used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); use[j]=&Sl[z]; break; } } if(!use[j]){ qof[j]=nmiss; use[j]=&m->ws[nmiss]; missk[nmiss++]=j; m->miss++; if(g_disk_split){ if(m->ld_ctx==1) m->miss_draft++; else if(m->ld_ctx==2) m->miss_absorb++; } } } @@ -3011,11 +3226,12 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int double t0=now_s(); int eids[64]; for(int q=0;qt_edisk += now_s()-t0; /* dispatch only; real reads hide behind matmul */ + m->t_ewait += now_s()-t0; /* dispatch only; the reads overlap matmul and + * are timed as service inside expert_load */ } else { double t0=now_s(); /* ORIGINALE: blocking parallel load */ #pragma omp parallel for schedule(dynamic,1) for(int q=0;qws[q],1); - m->t_ewait += now_s()-t0; } /* blocking: whole load stalls compute */ + m->t_ewait += now_s()-t0; } /* compute thread blocked for the whole load */ } /* I/O ASINCRONO: readahead (WILLNEED) del blocco SUCCESSIVO mentre calcoliamo * questo — il kernel legge in background, le pread dopo trovano cache calda */ @@ -3044,7 +3260,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int * correct (and free) when a subset falls back to the CPU. */ if(g_pipe && nmiss){ double tw=now_s(); for(int q=0;qt_ewait += now_s()-tw; } /* blocking: drain stalled compute */ + m->t_ewait += now_s()-tw; } MB_BUILD(1, 0); /* missed experts, now loaded */ if(nbb>0){ double t0=now_s(); @@ -3068,7 +3284,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int * Stays ABOVE the METAL skip: a subset that fell back to the CPU still needs its * slot drained here, and under METAL the block-level drain above already ran (this * spin is then a no-op). */ - if(g_pipe && qof[j]>=0){ double tw=now_s(); pipe_wait(qof[j]); m->t_ewait += now_s()-tw; } /* blocking */ + if(g_pipe && qof[j]>=0){ double tw=now_s(); pipe_wait(qof[j]); m->t_ewait += now_s()-tw; } #ifdef COLI_METAL /* skip the subsets already computed on GPU */ if(g_metal_enabled && ((is_miss[j] && !cpu_miss) || (!is_miss[j] && !cpu_res))) continue; @@ -3094,7 +3310,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int coli_cuda_expert_mlp(e->g.cuda,e->u.cuda,e->d.cuda,hh,xg,nr)){ for(int r=0;rt_emm+=now_s()-t0; continue; + double dt=now_s()-t0;m->t_emm+=dt;if(g_prof)m->t_egpu+=dt;continue; } if(!e->slab) expert_host_ensure(m,layer,e); #endif @@ -3103,7 +3319,9 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int matmul_qt(hh, gg, &e->d, nr); for(int r=0;rt_emm += now_s()-t0; + double dt=now_s()-t0;m->t_emm+=dt;if(g_prof){m->t_ecpu+=dt; + m->cpu_expert_bytes+=qt_bytes(&e->g)+qt_bytes(&e->u)+qt_bytes(&e->d); + m->cpu_expert_rows+=(uint64_t)nr;} } #ifdef COLI_CUDA ColiCudaTensor *dev_g[COLI_CUDA_MAX_DEVICES][64],*dev_u[COLI_CUDA_MAX_DEVICES][64]; @@ -3111,6 +3329,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int int dev_rows[COLI_CUDA_MAX_DEVICES][64],dev_which[COLI_CUDA_MAX_DEVICES][64]; int dev_nc[COLI_CUDA_MAX_DEVICES]={0},dev_total[COLI_CUDA_MAX_DEVICES]={0}; int dev_off[COLI_CUDA_MAX_DEVICES]={0},dev_ok[COLI_CUDA_MAX_DEVICES]={0}; + double dev_time[COLI_CUDA_MAX_DEVICES]={0}; for(int di=0;dig.cuda_device==g_cuda_devices[di]) dev_total[di]+=group_n[q]; for(int di=1;di1) schedule(static) - for(int di=0;dig.cuda,e->u.cuda,e->d.cuda,hh,xg,nr)){ expert_host_ensure(m,layer,e); expert_gate_up(gg,uu,xg,&e->g,&e->u,nr); for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z]; matmul_qt(hh,gg,&e->d,nr); + if(g_prof){m->cpu_expert_bytes+=qt_bytes(&e->g)+qt_bytes(&e->u)+qt_bytes(&e->d); + m->cpu_expert_rows+=(uint64_t)nr;} } + if(g_prof)m->t_ecpu+=now_s()-tc; } float *src=dev_ok[di]?group_y+(int64_t)off*D:hh; for(int r=0;rmx)mx=dev_time[di];m->t_egpu+=mx;} m->t_emm+=now_s()-tg; #endif /* No drain barrier: the per-expert pipe_wait(qof[j]) above (issued for every @@ -3756,7 +3983,11 @@ static void layers_forward_rows(Model *m, float *x, int S, int pos_base, float *dst=coli_cuda_pipe_scratch(dev,15,xb); if(dst){ if(x_dev_on<0) ok=coli_cuda_pipe_upload(dev,dst,x,xb); - else if(x_dev_on!=dev){ ok=coli_cuda_pipe_peer_copy(dev,dst,x_dev_on,x_dev,xb); } + else if(x_dev_on!=dev){ + double tp=g_prof?now_s():0; + ok=coli_cuda_pipe_peer_copy(dev,dst,x_dev_on,x_dev,xb); + if(g_prof){m->t_p2p+=now_s()-tp;m->n_p2p++;} + } else dst=x_dev; if(ok){ x_dev=dst; x_dev_on=dev; @@ -3852,7 +4083,7 @@ static float *step_all(Model *m, const int *ids, int S, int pos_base){ float *x=falloc((int64_t)S*D); for(int s=0;sh_all) memcpy(m->h_all, x, (int64_t)S*D*sizeof(float)); /* hidden di TUTTE le pos (S<=64) */ + if(m->h_all) memcpy(m->h_all, x, (int64_t)S*D*sizeof(float)); /* hidden di TUTTE le pos (S<=512) */ if(m->hlast) memcpy(m->hlast, x+(int64_t)(S-1)*D, D*sizeof(float)); float *lo=falloc((int64_t)S*c->vocab), *row=falloc(D); for(int s=0;sfinal_norm, D, c->eps); @@ -3865,8 +4096,8 @@ static float *step_all(Model *m, const int *ids, int S, int pos_base){ static float *step_decode_batch(Model *m, const DecodeRow *rows, int S){ Cfg *c=&m->c; int D=c->hidden; /* Ragged KV currently uses MLA absorption; the stack kernel is sized to 512. */ - if(!rows || S<1 || S>64 || c->kv_lora>512) return NULL; - KVState *kvs[64]; int positions[64]; + if(!rows || S<1 || S>512 || c->kv_lora>512) return NULL; + KVState *kvs[512]; int positions[512]; float *x=falloc((int64_t)S*D); for(int s=0;sLc || !rows[s].kv->Rc || !rows[s].kv->kv_start || @@ -3982,7 +4213,11 @@ static void mtp_absorb(Model *m, const int *next_ids, const float *x, int S, int } static inline int argmax_v(const float *lo, int V){ - int b=0; float bv=lo[0]; for(int i=1;ibv){bv=lo[i];b=i;} return b; + /* skip NaN (x==x is false for NaN) so a poisoned logit can't pin the argmax + * to index 0 — pick the max finite/+Inf entry instead. */ + int b=-1; float bv=-INFINITY; + for(int i=0;ibv){ bv=x; b=i; } } + return b<0?0:b; } /* ---- METODO F: draft grammaticale (#48) ---- @@ -4069,24 +4304,61 @@ static uint64_t g_rng=0x9E3779B97F4A7C15ULL; static inline double rndu(void){ g_rng^=g_rng<<13; g_rng^=g_rng>>7; g_rng^=g_rng<<17; return (double)(g_rng>>11)*(1.0/9007199254740992.0); } static float *g_pbuf=NULL; static int *g_pidx=NULL; /* buffer riusati (decode single-thread) */ -static int cmp_pdesc(const void *a,const void *b){ - float pa=g_pbuf[*(const int*)a], pb=g_pbuf[*(const int*)b]; - return papb ? -1 : 0; } -/* costruisce in g_pbuf la distribuzione target: softmax(lo/temp) troncata a top-p g_nuc */ +/* sift-down su max-heap in h[0..n), chiave = g_pbuf[h[i]] (#335: partial top-p select). + * Versione "a buco": porta il valore di radice e lo deposita solo alla fine, cosi' + * heapify e' O(V) e ogni pop e' O(log n) senza qsort sull'intero vocabolario. */ +static void topp_siftdown(int *h, int n, int i){ + int iv=h[i]; float kv=g_pbuf[iv]; + for(;;){ int l=2*i+1; + if(l>=n) break; /* foglia */ + int b=l; if(l+1g_pbuf[h[l]]) b=l+1; /* figlio maggiore */ + if(g_pbuf[h[b]]<=kv) break; /* nessun figlio supera la radice -> ferma */ + h[i]=h[b]; i=b; } + h[i]=iv; +} +/* costruisce in g_pbuf la distribuzione target: softmax(lo/temp) troncata a top-p g_nuc. + * Invariante per dist_sample: g_pbuf resta INDICIZZATO per token-id (mai riordinato); + * la coda troncata va AZZERATA in g_pbuf (dist_sample la legge direttamente per id). */ static void dist_build(const float *lo, int V){ if(!g_pbuf){ g_pbuf=falloc(V); g_pidx=malloc(V*sizeof(int)); } - float mx=lo[0]; for(int i=1;imx) mx=lo[i]; + /* Un solo logit NaN/+Inf avvelenava tutto (#369): +Inf diventava mx, NaN/Inf-mx + * -> expf NaN -> s NaN -> ogni prob NaN -> dist_sample cade sul fallback + * `g_pbuf[i]>0` (NaN>0 e' falso ovunque) e ritorna 0 PER SEMPRE, in silenzio. + * Difesa: mx solo sui finiti; un logit non finito contribuisce prob 0; + * se la distribuzione degenera (tutti non finiti / somma non valida) si + * ripiega sull'argmax dei finiti e si avvisa UNA volta, mai in silenzio. */ + int mxi=-1; float mx=0; + for(int i=0;imx)){ mx=lo[i]; mxi=i; } double s=0; float invt=1.f/(g_temp>1e-4f?g_temp:1e-4f); - for(int i=0;i=0){ + for(int i=0;i=0)?mxi:0; /* mxi = argmax dei logit FINITI (robusto + * anche se lo[0] e' NaN, dove argmax_v fallirebbe) */ + for(int i=0;i0 && g_nuc<1.f){ for(int i=0;i=g_nuc){ keep=i+1; break; } } - double s2=0; for(int i=keep;i=0;i--) topp_siftdown(g_pidx,V,i); /* heapify O(V) */ + /* pop verso la coda: i vincitori (testa top-p) cadono in g_pidx[out..V-1] in ordine + * DECRESCENTE, come il vecchio qsort, quindi s2 accumula nello stesso ordine -> + * head bit-identical sui casi senza pareggi (i pareggi erano gia' non specificati + * sotto il qsort instabile e restano tali). Il prefisso g_pidx[0..out-1) e' la coda. */ + double s2=0, cum=0; int out=V; + do{ int root=g_pidx[0]; /* massimo corrente */ + g_pidx[0]=g_pidx[--out]; g_pidx[out]=root; /* sposta il max in coda */ + s2+=g_pbuf[root]; cum+=g_pbuf[root]; + if(out>0) topp_siftdown(g_pidx,out,0); + } while(cum0); + for(int i=0;i=0 -> quel token e' escluso (rinormalizzando al volo) */ @@ -4107,18 +4379,34 @@ static int pick_tok(const float *lo, int V, int ban){ /* stop-set attivo (popolato da run_text/run_serve dal config; vuoto in validazione, * dove si genera un numero fisso di token da confrontare con l'oracolo) */ -static int g_stop[9], g_nstop=0; +static int g_stop[64], g_nstop=0; /* config eos + ogni added-token "special" del tokenizer */ static void repin_pass_limit(Model *m,int limit); static void repin_pass(Model *m){ repin_pass_limit(m,16); } static inline int is_stop(int t){ for(int i=0;i solo gli stop del config (validazione/oracolo, dove il tokenizer non serve). */ +static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){ g_nstop=0; - for(int i=0;in_stop;i++) g_stop[g_nstop++]=c->stop_ids[i]; - if(tok_eos>=0 && !is_stop(tok_eos)) g_stop[g_nstop++]=tok_eos; + for(int i=0;in_stop && g_nstop<64;i++) g_stop[g_nstop++]=c->stop_ids[i]; + if(tok_eos>=0 && !is_stop(tok_eos) && g_nstop<64) g_stop[g_nstop++]=tok_eos; + int nsp=0; + /* DIFESA IN PROFONDITA' (woolcoxm, #298): il tokenizer marca "special":true i token di + * CONTROLLO -- <|user|>, <|assistant|>, <|observation|>, , [gMASK], i marker + * image/video/audio. Nessuno di questi e' contenuto legittimo di una risposta: se il + * modello ne emette uno, il turno e' finito (infatti GLM ne elenca tre fra gli eos + * ufficiali). Senza questo, uno di quei token non elencato nel config veniva + * DETOKENIZZATO E STAMPATO IN CHAT come testo, e la generazione proseguiva oltre la + * fine reale -- l'"added stuff on the end" riportato su un checkpoint convertito. + * Fidarsi del config di pesi convertiti da terzi e' precisamente cio' che non + * possiamo controllare; il flag del tokenizer lo possiamo leggere. + * NB: // hanno "special":false e restano contenuto vero. */ + if(T) for(int id=0; idn_ids && g_nstop<64; id++) + if(T->id_special[id] && !is_stop(id)){ g_stop[g_nstop++]=id; nsp++; } fprintf(stderr,"[stop] %d stop tokens:",g_nstop); for(int i=0;in_fw++; + if(g_prof) prof_lat(now_s()-tf0); int k=0; /* verifica: accetta finche' coincide */ if(g>0 && getenv("MTP_DEBUG")){ int veri=argmax_v(lo,V); fprintf(stderr,"[mtpdbg] draft0=%d verified=%d %s\n", draft[0], veri, draft[0]==veri?"HIT":"miss"); } @@ -4347,12 +4637,17 @@ static void generate(Model *m, const int *prompt, int np, int n_new, int *out){ } static void profile_print(Model *m, double elapsed){ - double accounted=m->t_edisk+m->t_ewait+m->t_emm+m->t_attn+m->t_head; + double accounted=m->t_ewait+m->t_emm+m->t_attn+m->t_head; printf("PROFILE: expert-disk %.3fs service / %.3fs wait | expert-matmul %.3fs | attention %.3fs " "(including kvb %.3fs) | lm_head %.3fs | other %.3fs\n", - m->t_edisk,m->t_ewait,m->t_emm,m->t_attn,m->t_kvb,m->t_head,elapsed-accounted); + edisk_s(),m->t_ewait,m->t_emm,m->t_attn,m->t_kvb,m->t_head,elapsed-accounted); printf("ATTENTION: projection/RoPE %.3fs | score-softmax-value %.3fs | output projection %.3fs\n", m->t_aproj,m->t_acore,m->t_aout); + if(g_prof)printf("P0-EXEC: routed CPU %.3fs / %.2f GB/s (%llu row) | routed GPU critical %.3fs | router %.3fs | residual P2P %.3fs / %llu hop | orchestration %.3fs\n", + m->t_ecpu,m->t_ecpu>0?m->cpu_expert_bytes/1e9/m->t_ecpu:0.0, + (unsigned long long)m->cpu_expert_rows,m->t_egpu,m->t_route,m->t_p2p,(unsigned long long)m->n_p2p, + elapsed-m->t_ewait-m->t_emm-m->t_attn-m->t_head-m->t_route-m->t_p2p>0? + elapsed-m->t_ewait-m->t_emm-m->t_attn-m->t_head-m->t_route-m->t_p2p:0); #ifdef COLI_METAL if(g_metal_enabled){ uint64_t ok=0,fb=0,ex=0; double su=0,gp=0,sc=0; coli_metal_moe_counts(&ok,&fb,&ex); coli_metal_moe_times(&su,&gp,&sc); @@ -4365,8 +4660,87 @@ static void profile_print(Model *m, double elapsed){ } static void profile_reset(Model *m){ - m->t_edisk=m->t_ewait=m->t_emm=m->t_attn=m->t_kvb=m->t_head=0; + m->t_ewait=m->t_emm=m->t_attn=m->t_kvb=m->t_head=0; + m->t_ecpu=m->t_egpu=m->t_route=m->t_p2p=0;m->n_p2p=0; + m->cpu_expert_bytes=0;m->cpu_expert_rows=0; m->t_aproj=m->t_acore=m->t_aout=0; + atomic_store_explicit(&g_edisk_ns,0,memory_order_relaxed); +} + +/* PROF=1 report: forward-latency percentiles, expert I/O totals, phase shares + * of wall time, and a plain-language verdict naming the knob most likely to + * move tok/s on THIS machine with THIS config. `b` marks the window start + * (serve mode reports per turn; batch modes snapshot right after reset). */ +static int prof_cmp_d(const void *a,const void *b){ + double x=*(const double*)a, y=*(const double*)b; return (x>y)-(xc; if(elapsed<1e-9) elapsed=1e-9; + uint64_t nw=g_prof_nlat-b->nlat; if(nw>PROF_LAT_CAP) nw=PROF_LAT_CAP; /* ring keeps the tail */ + uint64_t nfw=m->n_fw-b->n_fw, nem=m->n_emit-b->n_emit; + if(nw){ + double *v=malloc((size_t)nw*sizeof(double)); + if(v){ + for(uint64_t i=0;i=32 && p99>3*p50) + fprintf(f,"[PROF] tail: p99 is %.1fx p50 — the slow forwards are cold-cache expert loads; " + "a warm-up turn or a pinned hot-store (PIN / AUTOPIN history) shrinks them\n",p99/p50); + free(v); + } + } + int64_t io=atomic_load_explicit(&g_prof_io,memory_order_relaxed)-b->io; + uint64_t dh=m->hits-b->hits, dm=m->miss-b->miss, dq=m->ereq-b->ereq; + double hitp=(dh+dm)?100.0*dh/(dh+dm):100.0; + double eb=(double)expert_bytes_probe(m,m->ebits); + int pinned=0,lru=0; + for(int i=0;i<=c->n_layers;i++){ if(m->npin)pinned+=m->npin[i]; if(m->ecn)lru+=m->ecn[i]; } + double io_w=m->t_ewait-b->ewait; /* stall the compute thread felt */ + double io_svc=edisk_s()-b->edisk; /* read service on the loading threads (overlaps compute) */ + uint64_t dhp=m->hit_pin-b->hit_pin, dhe=m->hit_ecache-b->hit_ecache; /* split #336 */ + fprintf(f,"[PROF] expert I/O: %.3f GB fetched (%.1f MB/token, %.2f GB/s over the run%s) | " + "hit %.1f%% (%llu pin + %llu lru / %llu load) | %.1f loads/token | %.1fs read service / %.1fs felt wait\n", + io/1e9, tokens>0?io/1e6/tokens:0.0, io/1e9/elapsed, + g_mmap?"; COLI_MMAP=1: page cache may serve part":"", + hitp,(unsigned long long)dhp,(unsigned long long)dhe,(unsigned long long)dm, tokens>0?(double)dq/tokens:0.0, + io_svc,io_w); + fprintf(f,"[PROF] resident experts: %d pinned (%.1f GB) + %d in LRU (%.1f GB, cap %d/layer)\n", + pinned,pinned*eb/1e9,lru,lru*eb/1e9,m->ecap); + double emm=m->t_emm-b->emm, ecpu=m->t_ecpu-b->ecpu, egpu=m->t_egpu-b->egpu; + double route=m->t_route-b->route,p2p=m->t_p2p-b->p2p; + uint64_t np2p=m->n_p2p-b->n_p2p; + int64_t cpu_bytes=m->cpu_expert_bytes-b->cpu_bytes; + uint64_t cpu_rows=m->cpu_expert_rows-b->cpu_rows; + double attn=m->t_attn-b->attn, head=m->t_head-b->head; + double other=elapsed-io_w-emm-attn-head-route-p2p; if(other<0) other=0; + double f_io=io_w/elapsed, f_emm=emm/elapsed, f_attn=attn/elapsed; + fprintf(f,"[PROF] time shares: expert-I/O %.0f%% | expert-matmul %.0f%% | attention %.0f%% | lm_head %.0f%% | other %.0f%%\n", + 100*f_io,100*f_emm,100*f_attn,100*head/elapsed,100*other/elapsed); + double slow=ecpu>egpu?ecpu:egpu,fast=ecpu0?cpu_bytes/1e9/ecpu:0.0,(unsigned long long)cpu_rows, + egpu,fast>1e-9?slow/fast:0.0,route,p2p,(unsigned long long)np2p, + np2p?p2p*1e3/np2p:0.0,other); + if(f_io>=0.30){ + fprintf(f,"[PROF] verdict: I/O-bound — %.0f%% of the time waits on expert reads (hit %.0f%%).",100*f_io,hitp); + if(hitp<90) fprintf(f," More cache is the lever: raise RAM_GB (or add RAM)."); + else fprintf(f," The cache is already warm — the routed working set streams from disk; a faster disk or a bigger pinned tier (PIN_GB) is the lever."); + if(!g_pipe) fprintf(f," Try PIPE=1 (overlap reads with matmul)."); + if(!g_direct) fprintf(f," On NVMe try DIRECT=1."); + fprintf(f,"\n"); + } else if(f_emm>=0.40){ + fprintf(f,"[PROF] verdict: compute-bound in expert matmuls (%.0f%%) — more cores/threads help; keep IDOT=1, or move hot experts to a GPU tier (COLI_CUDA / COLI_METAL).%s\n", + 100*f_emm, g_mmap?" Note: with COLI_MMAP=1 page-fault I/O is accounted inside matmul.":""); + } else if(f_attn>=0.35){ + fprintf(f,"[PROF] verdict: attention-bound (%.0f%%) — context length is the cost (DSA %s). A lower CTX helps if the workload allows.\n", + 100*f_attn, m->has_dsa?"on":"not available for this model"); + } else { + fprintf(f,"[PROF] verdict: balanced — no phase dominates (I/O %.0f%%, matmul %.0f%%, attention %.0f%%); this config is a reasonable fit for this machine.\n", + 100*f_io,100*f_emm,100*f_attn); + } } /* Fixed-token decode benchmark: prefill all but the prompt's last token, then @@ -4376,16 +4750,20 @@ static void run_replay(Model *m, const int *full, int nfull, int np){ if(np<2||nfull<=np){ fprintf(stderr,"REPLAY requires a non-empty prompt and continuation\n"); return; } kv_alloc(m,nfull+2); float *logit=step(m,full,np-1,0); free(logit); - m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; + m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; m->hit_pin=m->hit_ecache=0; profile_reset(m); + ProfBase pb; prof_base(m,&pb); double t0=now_s(); int steps=0; for(int i=np-1;in_fw++; m->n_emit++; } } double dt=now_s()-t0, tot=m->hits+m->miss; printf("REPLAY decode: %d tokens in %.3fs | %.2f tok/s | expert hit %.1f%%\n", steps,dt,steps/dt,tot?100.0*m->hits/tot:0.0); profile_print(m,dt); + if(g_prof) prof_report(m,&pb,dt,steps,stdout); #ifdef COLI_CUDA if(m->gpu_expert_count) printf("CUDA expert tier: %d resident experts (%.2f GB) | %llu calls served from VRAM\n", m->gpu_expert_count,m->gpu_expert_bytes/1e9,(unsigned long long)m->gpu_expert_calls); @@ -4399,7 +4777,7 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ Cfg *c=&m->c; char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap); Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); - stops_arm(&m->c, eos); + stops_arm_tok(&m->c, eos, &T); grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */ if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della * distribuzione int4 e' rumore di quantizzazione */ @@ -4422,10 +4800,11 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ } prefill_t=now_s()-prefill_t; printf("PROFILO PREFILL (%.2fs):\n",prefill_t); profile_print(m,prefill_t); - m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; + m->hits=m->miss=m->ereq=m->gpu_expert_calls=0; m->hit_pin=m->hit_ecache=0; m->n_emit=m->n_fw=0; g_last_repin=0; profile_reset(m); + ProfBase pb; prof_base(m,&pb); double t=now_s(); EmitStream es={&T,m,t,0,0}; grammar_reset(); @@ -4434,8 +4813,9 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ double tot=m->hits+m->miss; int nsp=0; for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; printf("\n---\nprefill %d tokens in %.2fs | decode %d tokens in %.2fs (%.2f tok/s) | " - "expert hit rate %.1f%% | RSS %.2f GB", - np,prefill_t,produced,dt,produced/dt,tot?100.0*m->hits/tot:0.0,rss_gb()); + "expert hit rate %.1f%% (pin %.1f%% + lru %.1f%%) | RSS %.2f GB", /* split #336: quale tier serve gli hit */ + np,prefill_t,produced,dt,produced/dt,tot?100.0*m->hits/tot:0.0, + tot?100.0*m->hit_pin/tot:0.0, tot?100.0*m->hit_ecache/tot:0.0, rss_gb()); if(g_cache_route && m->route_slots) printf(" | swap %.1f%% (%llu/%llu)", 100.0*m->route_swaps/m->route_slots, @@ -4469,6 +4849,7 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ if(g_cuda_enabled) cuda_stats_print(); #endif profile_print(m,dt); + if(g_prof) prof_report(m,&pb,dt,produced,stdout); if(g_pilot_real) printf("PILOT_REAL: %ld load cross-layer completati, %ld scartati (main gia' sul layer) | PILOT_K=%d\n", (long)atomic_load_explicit(&g_pilot_loads,memory_order_relaxed), (long)atomic_load_explicit(&g_pilot_drops,memory_order_relaxed), g_pilot_k); @@ -4543,7 +4924,68 @@ static int repin_pick(Model *m, RepinCand *out, int maxc){ } return nb; } +/* ---- RSS GUARD (#403) ----------------------------------------------------- + * La proiezione di cap_for_ram e' una STIMA: sul GB10 (#403) le generazioni + * lunghe l'hanno sforata di ~40 GB (proiettato 74.4, reale 115.6 -> 3 kill del + * kernel). La run D dell'issue prova che un cap piu' basso CONTIENE la crescita: + * questa guardia lo fa da sola, sull'RSS MISURATO invece che sul proiettato. + * Al safe point (stessa sede di repin: nessun moe in volo), ogni ~16 token + * emessi: se l'RSS supera il budget, svuota gli slot LRU meno usati e abbassa + * ecap perche' non ricrescano. Gli slab sono >128KB (mmap'd da glibc): la free + * restituisce le pagine al kernel subito, quindi l'RSS scende davvero. + * Lo slot NON viene compattato via: resta al suo posto con eid=-1/used=0 (primo + * candidato al riuso), perche' con PILOT_REAL il worker tiene puntatori dentro + * ecache[] durante i suoi pread e uno spostamento li invaliderebbe; per lo + * stesso motivo gli slot eid<0 (riservati/in caricamento) non si toccano e la + * selezione avviene sotto g_pilot_mx. resident_bytes resta invariato: gli slot + * LRU non sono mai contati li' (solo pin e densa). + * EN: evict = free the slab in place (eid=-1, used=0, never compact: PILOT_REAL + * EN: holds pointers into ecache[] across its preads), skip eid<0 reservations, + * EN: select under g_pilot_mx. RSS_GUARD_GB= forces an explicit ceiling. */ +static double g_ram_budget_gb=0; /* budget risolto, scritto da cap_for_ram */ +static uint64_t g_rssg_last=0; +static void rss_guard(Model *m){ + double lim = getenv("RSS_GUARD_GB") ? atof(getenv("RSS_GUARD_GB")) : g_ram_budget_gb; + if(lim<=0) return; + if(m->n_emit - g_rssg_last < 16) return; + g_rssg_last = m->n_emit; + double rss=rss_gb(); + if(rss <= lim*1.02+0.3) return; /* tolleranza: 2% + 300MB */ + Cfg *c=&m->c; + int64_t need=(int64_t)((rss-lim)*1e9), freed=0; int dropped=0; + for(int pass=0; pass<8 && freedn_layers && freedecache || !m->ecache[l]) continue; + pthread_mutex_lock(&g_pilot_mx); + int nn=m->ecn[l], lru=-1; + for(int z=0;zecache[l][z]; + if(cand->eid<0 || !cand->slab) continue; + if(lru<0 || cand->usedecache[l][lru].used) lru=z; + } + if(lru<0){ pthread_mutex_unlock(&g_pilot_mx); continue; } + ESlot *s=&m->ecache[l][lru]; + s->eid=-1; /* nascosto: nessun hit/evict altrui */ + pthread_mutex_unlock(&g_pilot_mx); + int64_t sb=s->slab_cap + s->fslab_cap*4; +#ifdef COLI_METAL + if(s->slab && g_metal_enabled) coli_metal_unregister(s->slab); +#endif + compat_aligned_free(s->slab); free(s->fslab); + s->slab=NULL; s->fslab=NULL; s->slab_cap=s->fslab_cap=0; + QT *q[3]={&s->g,&s->u,&s->d}; + for(int k=0;k<3;k++){ q[k]->qf=NULL; q[k]->q8=NULL; q[k]->q4=NULL; q[k]->s=NULL; } + s->used=0; /* primo candidato al riuso */ + freed += sb; dropped++; + } + if(m->ecap>2) m->ecap--; /* il tetto scende: niente ricrescita */ + } + if(dropped) + fprintf(stderr,"[RAM-GUARD] RSS %.1f GB over the %.1f GB budget (#403): " + "dropped %d cached experts, cap -> %d\n", rss, lim, dropped, m->ecap); +} static void repin_pass_limit(Model *m,int limit){ + rss_guard(m); /* #403: il budget si fa rispettare sull'RSS MISURATO */ if(g_repin<=0) return; if(m->n_emit - g_last_repin < (uint64_t)g_repin) return; g_last_repin = m->n_emit; @@ -4775,6 +5217,8 @@ typedef struct { float temp, top_p; double started; uint64_t hits0, miss0; + ProfBase pb; /* phase-time window start (same convention as hits0): + feeds the PROF protocol line and the PROF=1 report */ } ServeReq; static void mux_data(Tok *T, unsigned long long id, int token){ @@ -4791,10 +5235,26 @@ static void mux_done(Model *m, ServeCtx *sc, ServeReq *r){ tiers_emit(m); emap_emit(m); hits_emit(m); + /* PROF: per-turn phase timings for the dashboard profiling page — + * "PROF ". + * edisk = disk service (expert_load wall on the reading threads, overlaps + * compute); ewait = the stall the compute thread felt — only ewait belongs + * in a wall-time breakdown. With KV_SLOTS>1 concurrent slots share the + * batched forwards, so the shares describe the whole engine over the + * window, not the single request (same convention as the STAT hit% below). */ + printf("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %llu\n",dt, + r->prompt_tokens,r->emitted, + edisk_s()-r->pb.edisk,m->t_ewait-r->pb.ewait,m->t_emm-r->pb.emm, + m->t_attn-r->pb.attn,m->t_head-r->pb.head, + (unsigned long long)(m->n_fw-r->pb.n_fw)); printf("DONE %llu STAT %d %.2f %.1f %.2f %d %d\n",r->id,r->emitted, r->emitted/dt,(dh+dm)>0?100.0*dh/(dh+dm):0.0,rss_gb(), r->prompt_tokens,r->length_limited); fflush(stdout); kv_bind(m,&sc->kv); kv_disk_append(m,sc->hist,sc->len); + /* PROF window = this request's lifetime; with KV_SLOTS>1 concurrent slots + * share the batched forwards, so the shares describe the engine, not the + * single request (same convention as the STAT hit%% above). */ + if(g_prof) prof_report(m,&r->pb,dt,r->emitted,stderr); r->active=0; } @@ -4857,6 +5317,7 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, int nctx, ServeReq *r=&req[sub.slot]; memset(r,0,sizeof(*r)); r->id=sub.id; r->maximum=sub.max_tokens; r->temp=sub.temperature; r->top_p=sub.top_p; r->prompt_tokens=nt; r->started=now_s(); r->hits0=m->hits; r->miss0=m->miss; + prof_base(m,&r->pb); /* a few loads: cheap enough to always track */ int room=maxctx-sc->len-1; if(r->maximum>room){r->maximum=room; r->length_limited=1;} g_temp=r->temp; g_nuc=r->top_p; int next=pick_tok(logit,m->c.vocab,-1); free(logit); @@ -4869,11 +5330,11 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, int nctx, static void run_serve_mux(Model *m, const char *snap){ char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap); - Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); stops_arm(&m->c,eos); + Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); stops_arm_tok(&m->c,eos,&T); g_draft=0; /* one scheduler owns every forward; MTP/speculation is not ragged-safe */ int maxctx=getenv("CTX")?atoi(getenv("CTX")):4096; int nctx=getenv("KV_SLOTS")?atoi(getenv("KV_SLOTS")):1; - if(nctx<1||nctx>16){fprintf(stderr,"KV_SLOTS deve essere tra 1 e 16\n");exit(2);} + if(nctx<1||nctx>512){fprintf(stderr,"KV_SLOTS must be between 1 and 512\n");exit(2);} g_kvsave=getenv("KVSAVE")?atoi(getenv("KVSAVE")):1; KVState *initial=m->kv; free(initial->kv_start); free(initial); ServeCtx *ctx=calloc(nctx,sizeof(*ctx)); ServeReq *req=calloc(nctx,sizeof(*req)); @@ -4929,12 +5390,14 @@ static void run_serve_mux(Model *m, const char *snap){ } active=0; for(int i=0;in_fw++; + if(g_prof) prof_lat(now_s()-tf0); for(int s=0;slen++; g_temp=r->temp; g_nuc=r->top_p; @@ -4970,7 +5433,7 @@ static void run_serve(Model *m, const char *snap){ char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap); Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); - stops_arm(&m->c, eos); + stops_arm_tok(&m->c, eos, &T); grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */ if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della * distribuzione int4 e' rumore di quantizzazione */ @@ -5004,6 +5467,7 @@ static void run_serve(Model *m, const char *snap){ if(len<1){ printf("\x01\x01" "END" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f\n", rss_gb()); fflush(stdout); continue; } int cur=ngen; if(len+cur+g_draft+2>=maxctx) cur=maxctx-len-g_draft-2; uint64_t h0=m->hits, ms0=m->miss; double tt0=now_s(); + ProfBase pb; if(g_prof) prof_base(m,&pb); float *logit=step(m,hist+len-1,1,len-1); EmitStream es={&T,m,now_s(),0,1}; int prod=0; @@ -5013,7 +5477,9 @@ static void run_serve(Model *m, const char *snap){ double dh=(double)(m->hits-h0), dm=(double)(m->miss-ms0); printf("\n\x01\x01" "END" "\x01\x01\n"); printf("STAT %d %.2f %.1f %.2f\n", prod, prod/tdt, (dh+dm)>0?100.0*dh/(dh+dm):0.0, rss_gb()); - fflush(stdout); kv_disk_append(m,hist,len); repin_pass(m); continue; } /* RFC: re-pin a caldo tra i turni / live re-pin between turns */ + fflush(stdout); + if(g_prof) prof_report(m,&pb,tdt,prod,stderr); /* per-turn window; stdout is the framed protocol */ + kv_disk_append(m,hist,len); repin_pass(m); continue; } /* RFC: re-pin a caldo tra i turni / live re-pin between turns */ if(nr<1){ printf("\x01\x01" "END" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f\n", rss_gb()); fflush(stdout); continue; } /* API mode: an exact, length-prefixed prompt. Unlike the interactive * line protocol this accepts newlines. The tokenized prompt is matched @@ -5081,6 +5547,7 @@ static void run_serve(Model *m, const char *snap){ uint64_t agh0=m->route_agree_hit, agt0=m->route_agree_tot; uint64_t kln0=m->route_kl_n; double kls0=m->route_kl_sum; double tt0=now_s(); + ProfBase pb; if(g_prof) prof_base(m,&pb); float *logit; if(k>0){ logit=step(m,hist+len,k,len); len+=k; } else logit=step(m,hist+len-1,1,len-1); /* prompt identico/prefisso: rigenera i logits */ @@ -5107,6 +5574,7 @@ static void run_serve(Model *m, const char *snap){ agree_pct,kl_mean); printf("\n"); fflush(stdout); + if(g_prof) prof_report(m,&pb,tdt,prod,stderr); /* per-turn window; stdout is the framed protocol */ free(raw); g_temp=base_temp; g_nuc=base_nuc; usage_save(m); /* la cache che impara: storia aggiornata a ogni turno */ kv_disk_append(m,hist,len); /* KV su disco: il prossimo avvio riparte da qui */ @@ -5167,10 +5635,10 @@ static void ehit_mark(Model *m, int layer, int eid){ } /* HWINFO: hardware snapshot for the web dashboard — emitted once at READY. */ -static void hwinfo_emit(Model *m){ - Cfg *c=&m->c; (void)c; /* silence -Wunused on builds without /proc (#148 report) */ - /* CPU */ - char cpu[256]=""; +/* CPU model + cores + RAM (GB); empty/zero where unavailable. + * Shared by the dashboard HWINFO line and the PROF=1 header. */ +static void hw_probe(char *cpu, size_t cn, int *cores, double *ram_total, double *ram_avail){ + cpu[0]=0; #ifdef _WIN32 /* niente /proc su Windows: brand string via CPUID (0x80000002..4), zero * dipendenze extra. La dashboard mostrava "0 GB RAM / 0 cores" perche' @@ -5180,7 +5648,7 @@ static void hwinfo_emit(Model *m){ for(unsigned int f=0x80000002u; f<=0x80000004u; f++,w+=4) __get_cpuid(f,&w[0],&w[1],&w[2],&w[3]); char *b=(char*)r; b[47]=0; while(*b==' ')b++; - snprintf(cpu,sizeof(cpu),"%s",b); } + snprintf(cpu,cn,"%s",b); } #endif #else FILE *ci=fopen("/proc/cpuinfo","r"); @@ -5188,27 +5656,32 @@ static void hwinfo_emit(Model *m){ while(fgets(ln,sizeof(ln),ci)) if(!strncmp(ln,"model name",10)){ char *p=strchr(ln,':'); if(p){ p++; while(*p==' ')p++; int n=(int)strlen(p); if(n>0&&p[n-1]=='\n')p[--n]=0; - snprintf(cpu,sizeof(cpu),"%s",p); } break; } + snprintf(cpu,cn,"%s",p); } break; } fclose(ci); } #endif - int cores=0; + *cores=0; #ifdef _WIN32 - { SYSTEM_INFO si; GetSystemInfo(&si); cores=(int)si.dwNumberOfProcessors; } + { SYSTEM_INFO si; GetSystemInfo(&si); *cores=(int)si.dwNumberOfProcessors; } #elif defined(_SC_NPROCESSORS_ONLN) - cores=(int)sysconf(_SC_NPROCESSORS_ONLN); + *cores=(int)sysconf(_SC_NPROCESSORS_ONLN); #endif - /* RAM */ - double ram_total=0,ram_avail=0; + *ram_total=*ram_avail=0; #ifdef _WIN32 - compat_meminfo(&ram_total,&ram_avail); /* GlobalMemoryStatusEx, gia' in compat.h */ + compat_meminfo(ram_total,ram_avail); /* GlobalMemoryStatusEx, gia' in compat.h */ #else FILE *mi=fopen("/proc/meminfo","r"); if(mi){ char ln[256]; double mt=0,ma=0; while(fgets(ln,sizeof(ln),mi)){ - if(sscanf(ln,"MemTotal: %lf",&mt)==1) ram_total=mt/1e6; - if(sscanf(ln,"MemAvailable: %lf",&ma)==1) ram_avail=ma/1e6; + if(sscanf(ln,"MemTotal: %lf",&mt)==1) *ram_total=mt/1e6; + if(sscanf(ln,"MemAvailable: %lf",&ma)==1) *ram_avail=ma/1e6; } fclose(mi); } #endif +} + +static void hwinfo_emit(Model *m){ + Cfg *c=&m->c; (void)c; /* silence -Wunused on builds without /proc (#148 report) */ + char cpu[256]; int cores; double ram_total,ram_avail; + hw_probe(cpu,sizeof(cpu),&cores,&ram_total,&ram_avail); /* GPU */ int ngpu=0; double vram_total=0; char gpu_name[128]=""; @@ -5612,6 +6085,7 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){ if(auto_b){ ram_gb = g_mem_avail_boot*0.88; /* misurata PRIMA del load: il residente gia' * allocato viene sottratto sotto, non due volte */ if(ram_gb<4){ fprintf(stderr,"[RAM] MemAvailable is unreadable or too low; assuming 8 GB\n"); ram_gb=8; } } + g_ram_budget_gb = ram_gb; /* #403: la RSS-guard usa il budget RISOLTO */ /* slack ONESTO, non forfettario (l'OOM del 2026-07-04 veniva da qui): * ws[64] slab del working-set (si materializzano TUTTI nel prefill batch-union), * KV cache a max_ctx, kvb_all della ricostruzione k/v in attention, @@ -5701,9 +6175,9 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){ * self-test, and would "generate" from "$P$G". So on Windows a PROMPT carrying * cmd's $-metacodes is ignored; set COLI_PROMPT to pass a real prompt from cmd. */ static const char *coli_user_prompt(void){ - const char *p = getenv("COLI_PROMPT"); + const char *p = getenv_utf8("COLI_PROMPT"); if(p) return p; - p = getenv("PROMPT"); + p = getenv_utf8("PROMPT"); #ifdef _WIN32 if(p) for(const char *q=p; q[0]; q++) if(q[0]=='$' && q[1] && strchr("ABCDEFGHLNPQSTV_+|$", q[1]&~0x20)){ p=NULL; break; } @@ -5711,6 +6185,34 @@ static const char *coli_user_prompt(void){ return p; } +/* PROF=1 startup header: one self-describing block so a saved log answers + * "what machine, what config" when comparing runs after changing RAM_GB, + * knobs, or moving to another host. */ +static void prof_config(Model *m, double ram_env, int est_ctx){ + Cfg *c=&m->c; + char cpu[256]; int cores; double rt,ra; + hw_probe(cpu,sizeof(cpu),&cores,&rt,&ra); + const char *backend="CPU"; +#ifdef COLI_CUDA + if(g_cuda_enabled) backend="CUDA"; +#endif +#ifdef COLI_METAL + if(g_metal_enabled) backend="Metal"; +#endif + int nsp=0; for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; + int rows=nsp+(m->has_mtp?2:0); /* stessa convenzione di cap_for_ram (MTP int8 = 2x) */ + int pinned=0; for(int i=0;i<=c->n_layers;i++) if(m->npin) pinned+=m->npin[i]; + double eb=(double)expert_bytes_probe(m,m->ebits); + fprintf(stderr,"[PROF] machine: %s | %d cores (%d omp threads) | RAM %.1f GB total, %.1f GB available | backend %s\n", + cpu[0]?cpu:"unknown CPU",cores,omp_get_max_threads(),rt,ra,backend); + fprintf(stderr,"[PROF] config: RAM_GB=%s%.1f CTX=%d | expert cache cap %d/layer (up to %.1f GB) | pinned %d (%.1f GB) | " + "DRAFT=%d PIPE=%d DIRECT=%d MMAP=%d IDOT=%d DSA=%s PILOT=%d CACHE_ROUTE=%d\n", + ram_env<=0?"auto ":"",ram_env<=0?g_mem_avail_boot*0.88:ram_env,est_ctx, + m->ecap,(double)m->ecap*rows*eb/1e9,pinned,pinned*eb/1e9, + g_draft,g_pipe,g_direct,g_mmap,g_idot, + (m->has_dsa&&c->index_topk)?"on":"off",g_pilot,g_cache_route); +} + int main(int argc, char **argv){ /* ---- Permanent OpenMP hot-thread tuning. The per-expert matmul regions are * tiny and back-to-back; with the default passive wait policy libgomp parks @@ -5735,6 +6237,14 @@ int main(int argc, char **argv){ !getenv("COLI_CUDA") && !getenv("COLI_METAL")){ setenv("OMP_WAIT_POLICY","active",0); /* keep the team hot across the tiny per-expert matmul regions */ setenv("GOMP_SPINCOUNT","200000",0); /* spin briefly, then yield so long disk waits don't burn a core */ + /* LLVM libomp (clang builds: FreeBSD cc, macOS, some Linux setups) does not + * read GOMP_*: with OMP_WAIT_POLICY=active it sets KMP_BLOCKTIME=infinite, + * so the idle team SPINS FOREVER once generation ends — a serve-mode engine + * parked on stdin burns ~100% x nthreads (#341, measured 3000% on FreeBSD). + * 200 ms of blocktime keeps the team hot across back-to-back expert matmuls + * and lets it sleep at the prompt. libgomp ignores KMP_*; overwrite=0 keeps + * the user's own setting authoritative. */ + setenv("KMP_BLOCKTIME","200",0); setenv("OMP_PROC_BIND","close",0); /* pack the team onto adjacent cores for cache locality */ setenv("OMP_DYNAMIC","FALSE",0); /* fixed team size: no per-region thread-count churn */ setenv("COLI_OMP_TUNED","1",1); @@ -5749,6 +6259,9 @@ int main(int argc, char **argv){ perror("[OMP] execv self-reexec failed, running untuned"); #endif } +#ifdef _WIN32 + _setmode(fileno(stdout), O_BINARY); +#endif #if defined(__AVX512F__) && defined(__AVX512BW__) if(getenv("I4_ACC512")) g_i4_acc512=atoi(getenv("I4_ACC512"))!=0; if(getenv("I4_ACC512_TEST")){ @@ -5762,6 +6275,7 @@ int main(int argc, char **argv){ g_prefetch = getenv("PREFETCH")?atoi(getenv("PREFETCH")):0; g_mmap = getenv("COLI_MMAP")?atoi(getenv("COLI_MMAP")):0; if(g_mmap) fprintf(stderr,"[MMAP] expert = viste zero-copy nei file (page cache = cache)\n"); + numa_init(); /* COLI_NUMA=1: expert-slab interleave (#82) */ g_topk = getenv("TOPK")?atoi(getenv("TOPK")):0; g_topp = getenv("TOPP")?atof(getenv("TOPP")):0; /* EXPERT_BUDGET e' sotto quarantena: la finestra operativa e' misurata VUOTA. @@ -5884,8 +6398,9 @@ int main(int argc, char **argv){ int cap = argc>1?atoi(argv[1]):64; int ebits= argc>2?atoi(argv[2]):8; int dbits= argc>3?atoi(argv[3]):ebits; - if(getenv("SERVE") && (kv_slot_count()<1 || kv_slot_count()>16)){ - fprintf(stderr,"KV_SLOTS must be between 1 and 16\n"); return 2; + int kv_limit=(getenv("SERVE_BATCH")&&atoi(getenv("SERVE_BATCH")))?512:16; + if(getenv("SERVE") && (kv_slot_count()<1 || kv_slot_count()>kv_limit)){ + fprintf(stderr,"KV_SLOTS must be between 1 and %d\n",kv_limit); return 2; } #ifdef COLI_CUDA if(getenv("COLI_CUDA") && atoi(getenv("COLI_CUDA"))){ @@ -5956,14 +6471,32 @@ int main(int argc, char **argv){ #endif } if(getenv("DSA_TOPK")) m.c.index_topk=atoi(getenv("DSA_TOPK")); /* override per test */ + /* Il path MUX (SERVE_BATCH=1, cioe' `coli serve`) forza g_draft=0 sotto — + * la speculazione non e' ragged-safe nel batch multi-slot. Segnalarlo QUI, + * altrimenti "MTP active (draft=8)" mentirebbe: il messaggio e' stampato + * prima della scelta del path (run_serve_mux, sotto), e con DRAFT=8 diceva + * "active" per poi disabilitarlo in silenzio (#358, LordMZTE). */ + int mux_will_disable_mtp = getenv("SERVE") && getenv("SERVE_BATCH") && atoi(getenv("SERVE_BATCH")); + int eff_draft = mux_will_disable_mtp ? 0 : g_draft; printf("loaded in %.2fs | resident dense: %.2f MB | layers=%d experts=%d | MTP %s (draft=%d)\n", now_s()-t0, m.resident_bytes/(1024.0*1024.0), m.c.n_layers, m.c.n_experts, - m.has_mtp?"ACTIVE":"absent", g_draft); + m.has_mtp?(mux_will_disable_mtp?"DISABLED (multiplexed serve)":"ACTIVE"):"absent", eff_draft); /* anche su stderr: e' il canale che le UI (coli) mostrano all'utente */ - fprintf(stderr,"[MTP] %s (draft=%d)\n", m.has_mtp?"active: native speculative decoding":"absent", g_draft); - if(!strncmp(snap,"/mnt/",5)) - fprintf(stderr,"WARNING: the model is on %s (slow 9p/Windows filesystem; fadvise is ineffective).\n" - " Keep it on ext4 (for example, /home/...) for memory efficiency and speed.\n", snap); + if(mux_will_disable_mtp && m.has_mtp) + fprintf(stderr,"[MTP] disabled in multiplexed serve (SERVE_BATCH=1): speculation is not " + "ragged-safe across KV slots. Single-client interactive use (`coli chat`) keeps MTP.\n"); + else + fprintf(stderr,"[MTP] %s (draft=%d)\n", m.has_mtp?"active: native speculative decoding":"absent", eff_draft); +#ifdef __linux__ + { /* Only warn for a GENUINE 9p mount (WSL Windows drives, magic 0x01021997), where + * fadvise is a no-op. The old check was `snap` starting with "/mnt/", which + * false-positives on native-Linux ZFS/ext4/xfs/NFS mounts that also live under /mnt. */ + struct statfs sfb; + if(statfs(snap,&sfb)==0 && (unsigned long)sfb.f_type==0x01021997UL) + fprintf(stderr,"WARNING: the model is on %s (9p/Windows filesystem; fadvise is ineffective).\n" + " Keep it on a native Linux fs (ext4/xfs/zfs) for memory efficiency and speed.\n", snap); + } +#endif /* HOT-STORE: PIN= [PIN_GB=g] -> top expert per frequenza fissi in RAM. * Va PRIMA di cap_for_ram: i pinnati contano nel residente. */ if(getenv("PIN")){ @@ -6015,7 +6548,9 @@ int main(int argc, char **argv){ } /* SEMPRE: senza clamp la LRU cresce fino a cap*76 layer = decine di GB -> OOM-kill. * RAM_GB assente o <=0 = budget automatico da MemAvailable. */ - cap_for_ram(&m, ram_env, ebits, est_ctx); } + cap_for_ram(&m, ram_env, ebits, est_ctx); + g_prof = getenv("PROF")?atoi(getenv("PROF")):0; /* PROF=1: opt-in performance profile */ + if(g_prof) prof_config(&m, ram_env, est_ctx); } const char *stats=getenv("STATS"); /* STATS= -> istogramma uso expert a fine run */ /* modo scoring per benchmark: SCORE= -> log-likelihood per riga */ @@ -6092,6 +6627,7 @@ int main(int argc, char **argv){ return 0; } int *out=malloc((np+n_new)*sizeof(int)); + ProfBase pb; prof_base(&m,&pb); double t=now_s(); generate(&m,prompt,np,n_new,out); double dt=now_s()-t; int match=0; printf("\nReference (oracle): "); for(int i=np;ic; + m->max_t = nfull; + m->K = calloc(c->n_layers, sizeof(float*)); m->V = calloc(c->n_layers, sizeof(float*)); + for (int i = 0; i < c->n_layers; i++) { + m->K[i] = falloc((int64_t)c->n_heads * m->max_t * c->head_dim); + m->V[i] = falloc((int64_t)c->n_heads * m->max_t * c->head_dim); + } + double nll = 0; int scored = 0; + float *logit = step(m, full, np, 0); /* prefill on the prompt */ + for (int i = np; i < nfull; i++) { + /* log softmax(logit)[full[i]] without materializing the softmax */ + float mx = logit[0]; for (int v = 1; v < c->vocab; v++) if (logit[v] > mx) mx = logit[v]; + double Z = 0; for (int v = 0; v < c->vocab; v++) Z += exp((double)logit[v] - mx); + nll += -((double)logit[full[i]] - mx - log(Z)); + scored++; + free(logit); logit = NULL; + if (i == nfull - 1) break; + logit = step(m, &full[i], 1, i); /* teacher forcing */ + } + if (logit) free(logit); + *nll_out = nll / scored; + return scored; +} + /* ---------- lettura ref.json ---------- */ static int *read_int_array(jval *o, const char *key, int *n_out) { jval *a = json_get(o, key); @@ -430,6 +461,19 @@ int main(int argc, char **argv) { Model m; model_init(&m, snap, cap, bits); printf("resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb()); + if (getenv("PPL") && atoi(getenv("PPL")) == 1) { /* loss-meter mode: teacher-forced NLL */ + double nll; double t = now_s(); + int scored = tf_nll(&m, full, nfull, np, &nll); + double dt = now_s() - t; + double tot = m.hits + m.miss; + printf("TF-NLL: %.4f nats/token over %d tokens | ppl = %.2f\n", nll, scored, exp(nll)); + printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0, + (unsigned long long)m.hits, (unsigned long long)m.miss); + printf("Speed: %.2f tok/s (%.1fs for %d tokens) | PEAK RSS: %.2f GB\n", scored/dt, dt, scored, rss_gb()); + free(buf); free(arena); + return 0; + } + int *out = malloc((np + n_new) * sizeof(int)); double t = now_s(); generate(&m, prompt, np, n_new, out); diff --git a/c/openai_server.py b/c/openai_server.py index ba19e43..3900ba6 100644 --- a/c/openai_server.py +++ b/c/openai_server.py @@ -27,6 +27,7 @@ HERE = Path(__file__).resolve().parent END = b"\x01\x01END\x01\x01\n" READY = b"\x01\x01READY\x01\x01\n" MAX_BODY = 4 << 20 +PROFILE_TURNS = 120 # rolling window of per-turn PROF snapshots kept for /profile DEFAULT_CORS_ORIGINS = ( "http://127.0.0.1:8000", "http://localhost:8000", @@ -270,6 +271,14 @@ def parse_tool_calls(reply, tools=None): salvaged.append(name) calls.append({"id": "call_" + uuid.uuid4().hex[:24], "type": "function", "function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)}}) + if tools and not calls and re.search(r"||", reply): + # Diagnosi per la #401: il client ha dichiarato i tools e il modello ha PROVATO la + # sintassi, ma il parse rigoroso non ha agganciato nulla (tipico output int4 storpiato). + # EN: #401 field diagnosis: tools were declared and the model attempted the syntax, + # EN: but the strict parse matched nothing (typically quantization-mangled output). + sys.stderr.write("[api] tools declared and tool-call markers present, but no call " + "parsed -- output may be quantization-mangled; try COLI_TOOL_SALVAGE=1\n") + sys.stderr.flush() text = _BOX_RE.sub("", reply) if THINK_CLOSE in text: text = text.split(THINK_CLOSE, 1)[1] @@ -405,7 +414,10 @@ def generation_options(body, limit): maximum = body.get("max_tokens") maximum_param = "max_tokens" if maximum is None: - maximum = min(256, limit) + # Client omitted max_tokens: honor the operator's configured budget (--max-tokens / + # --ngen), not an arbitrary 256 — `coli serve --ngen 32768` must mean 32768 (#382). + # Generation still ends at EOS, so this is a cap, not a target. + maximum = limit temperature = body.get("temperature") top_p = body.get("top_p") temperature = 0.7 if temperature is None else temperature @@ -473,6 +485,8 @@ class Engine: self.emap = None self.hits = None self.hits_seq = 0 # latest "TIERS" snapshot from the engine + self.profile = collections.deque(maxlen=PROFILE_TURNS) # per-turn phase timings + self.profile_seq = 0 read_engine_turn(self.process.stdout, READY, lambda _: None) self.dispatcher = threading.Thread(target=self._dispatch_stdout, name="colibri-stdout", daemon=True) @@ -550,6 +564,20 @@ class Engine: elif kind == "HITS" and len(fields) == 4: self.hits = fields[3] self.hits_seq += 1 + elif kind == "PROF" and len(fields) >= 10: + # per-turn phase timings: where the engine spent this turn's wall time + self.profile.append({ + "wall_s": float(fields[1]), + "prompt_tokens": int(fields[2]), + "completion_tokens": int(fields[3]), + "expert_disk_s": float(fields[4]), + "expert_wait_s": float(fields[5]), + "expert_matmul_s": float(fields[6]), + "attention_s": float(fields[7]), + "lm_head_s": float(fields[8]), + "forwards": int(fields[9]), + }) + self.profile_seq += 1 elif kind == "TIERS" and len(fields) >= 6: self.tiers = {"vram": int(fields[1]), "ram": int(fields[2]), "disk": int(fields[3]), "vram_gb": float(fields[4]), @@ -782,6 +810,12 @@ class APIHandler(BaseHTTPRequestHandler): payload["seq"] = eng.hits_seq self.send_json(200, payload, request_id) return + if path == "/profile": + eng = self.server.engine + payload = {"seq": getattr(eng, "profile_seq", 0) if eng else 0, + "turns": list(getattr(eng, "profile", ()) or ()) if eng else []} + self.send_json(200, payload, request_id) + return if self.serve_static(path): return self.require_auth() diff --git a/c/resource_plan.py b/c/resource_plan.py index 6b3f2fe..5f17a6a 100644 --- a/c/resource_plan.py +++ b/c/resource_plan.py @@ -203,6 +203,22 @@ def physical_cpu_count(): return os.cpu_count() or 1 +def cpu_socket_count(): + """Return the number of physical CPU sockets visible to this process.""" + if not sys.platform.startswith("linux"): + return 1 + try: + result = subprocess.run(["lscpu", "-p=socket"], text=True, + capture_output=True, check=True, timeout=5) + sockets = {int(line) for line in result.stdout.splitlines() + if line and not line.startswith("#")} + if sockets: + return len(sockets) + except (OSError, ValueError, subprocess.SubprocessError): + pass + return 1 + + POLICIES = { "quality": {"preserve_quantization": True, "preserve_router": True}, "balanced": {"preserve_quantization": True, "preserve_router": True}, @@ -212,11 +228,12 @@ POLICIES = { def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, available_memory=None, available_disk=None, gpus=None, - policy="quality", physical_cpus=None): + policy="quality", physical_cpus=None, cpu_sockets=None): if policy not in POLICIES: raise ValueError(f"unknown policy: {policy}") info = analyze_model(model) physical_cpus = physical_cpu_count() if physical_cpus is None else physical_cpus + cpu_sockets = cpu_socket_count() if cpu_sockets is None else cpu_sockets cfg = info["config"] available_memory = memory_available() if available_memory is None else available_memory if available_disk is None: @@ -286,6 +303,7 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, "quality_preserving": policy != "experimental-fast"}, "model": {key: value for key, value in info.items() if key != "config"}, "cpu": {"physical_cores": max(1, int(physical_cpus)), + "sockets": max(1, int(cpu_sockets)), "thread_policy": "physical-cores"}, "tiers": { "disk": {"role": "cold-backing", "model_bytes": info["model_bytes"], @@ -318,6 +336,10 @@ def environment_for_plan(plan, env=None, cuda_enabled=True): # ("Affinity not supported on this configuration"): non impostarle li'. result.setdefault("OMP_PROC_BIND", "spread") result.setdefault("OMP_PLACES", "cores") + if sys.platform.startswith("linux") and plan["cpu"].get("sockets", 1) > 1: + # Selectively interleave large expert/dense slabs across memory controllers. + # Unlike blanket numactl interleave, this leaves CUDA staging buffers local. + result.setdefault("COLI_NUMA", "1") if plan["policy"]["name"] == "balanced": result.setdefault("REPIN", "64") ram = plan["tiers"]["ram"] diff --git a/c/st.h b/c/st.h index 6b4a710..efa6e5a 100644 --- a/c/st.h +++ b/c/st.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -104,6 +105,38 @@ static int st_direct_fd(shards *S, int fd) { } /* indicizza tutti i model-*.safetensors in snap_dir */ +/* pread completo: chunk-loop (una singola pread si ferma a ~2^31 byte su Linux + * — i tensori bf16 grandi la superano), riprova su EINTR e riporta un errore + * ONESTO: perror stampava "Success" su una short-read (errno resta 0), lo + * stesso sintomo corretto in glm.c per #236. ST_PREAD_CHUNK e' sovrascrivibile + * per i test. EN: full pread — chunk loop (one pread caps at ~2^31 bytes and + * big bf16 tensors exceed it), EINTR retry, honest short-read errors. + * Exits on failure, like every st.h reader. */ +#ifndef ST_PREAD_CHUNK +#define ST_PREAD_CHUNK (1u << 30) +#endif +static void st_pread_full(int fd, void *buf, int64_t n, int64_t off, const char *tag) { + char *p = (char *)buf; + int64_t got = 0; + while (got < n) { + int64_t want = n - got; + if (want > (int64_t)ST_PREAD_CHUNK) want = ST_PREAD_CHUNK; + ssize_t r = pread(fd, p + got, (size_t)want, off + got); + if (r < 0) { + if (errno == EINTR) continue; + fprintf(stderr, "%s: %s (off %lld, %lld/%lld bytes)\n", tag, strerror(errno), + (long long)off, (long long)got, (long long)n); + exit(1); + } + if (r == 0) { + fprintf(stderr, "%s: short read at EOF (off %lld, %lld/%lld bytes) — truncated file?\n", + tag, (long long)off, (long long)got, (long long)n); + exit(1); + } + got += r; + } +} + static void st_init(shards *S, const char *snap_dir) { memset(S, 0, sizeof(*S)); S->cap = 4096; S->t = calloc(S->cap, sizeof(st_tensor)); @@ -128,7 +161,7 @@ static void st_init(shards *S, const char *snap_dir) { if (fstat(fd, &sst) != 0) { perror("fstat shard"); exit(1); } int64_t fsz = (int64_t)sst.st_size; uint64_t hlen; - if (pread(fd, &hlen, 8, 0) != 8) { perror("pread hlen"); exit(1); } + st_pread_full(fd, &hlen, 8, 0, "pread hlen"); /* file malevolo/troncato: hlen deve stare nel file dopo gli 8 byte di * prefisso e sotto il tetto. Senza questo bound hlen+1 puo' andare in * overflow (malloc(0) e poi hdr[hlen]=0 fuori limiti) o forzare una @@ -138,7 +171,7 @@ static void st_init(shards *S, const char *snap_dir) { files[fi], (unsigned long long)hlen, (long long)fsz); exit(1); } char *hdr = malloc(hlen + 1); if (!hdr) { perror("malloc safetensors header"); exit(1); } - if (pread(fd, hdr, hlen, 8) != (ssize_t)hlen) { perror("pread hdr"); exit(1); } + st_pread_full(fd, hdr, (int64_t)hlen, 8, "pread hdr"); hdr[hlen] = 0; int64_t data_start = 8 + (int64_t)hlen; char *arena = NULL; @@ -218,7 +251,7 @@ static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) { if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); } void *raw = malloc(t->nbytes); if (!raw) { fprintf(stderr, "malloc %lld bytes for tensor %s failed\n", (long long)t->nbytes, name); exit(1); } - if (pread(t->fd, raw, t->nbytes, t->off) != t->nbytes) { perror("pread data"); exit(1); } + st_pread_full(t->fd, raw, t->nbytes, t->off, "pread data"); if (t->dtype == 2) { memcpy(out, raw, t->nbytes); } else if (t->dtype == 0) { @@ -243,7 +276,7 @@ static int64_t st_nbytes(shards *S, const char *name) { static void st_read_raw(shards *S, const char *name, void *out, int drop) { st_tensor *t = st_find(S, name); if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); } - if (pread(t->fd, out, t->nbytes, t->off) != t->nbytes) { perror("pread raw"); exit(1); } + st_pread_full(t->fd, out, t->nbytes, t->off, "pread raw"); if (drop) posix_fadvise(t->fd, t->off, t->nbytes, POSIX_FADV_DONTNEED); } @@ -256,7 +289,7 @@ static void st_read_slice_f32(shards *S, const char *name, int64_t elem_off, int int esz = (t->dtype == 2) ? 4 : 2; int64_t boff = t->off + elem_off * esz, nb = n_elems * esz; void *raw = malloc(nb); - if (pread(t->fd, raw, nb, boff) != nb) { perror("pread slice"); exit(1); } + st_pread_full(t->fd, raw, nb, boff, "pread slice"); if (t->dtype == 2) memcpy(out, raw, nb); else if (t->dtype == 0) { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = bf16_to_f32(p[i]); } else { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = f16_to_f32(p[i]); } diff --git a/c/tests/bench_dsa_select.c b/c/tests/bench_dsa_select.c new file mode 100644 index 0000000..a1f2014 --- /dev/null +++ b/c/tests/bench_dsa_select.c @@ -0,0 +1,165 @@ +/* Microbenchmark: old (full-qsort) vs new (quickselect partial-select) DSA top-keep. + * + * This is NOT a unit test -- test_dsa_select.c proves correctness. This measures the + * headline claim of #356: that replacing the O(nk log nk) qsort over all nk context + * scores with an O(nk) partial_select_desc is materially faster per call, which is + * the win the issue was opened for -- and that the win GROWS with context length + * (because quickselect is linear average, qsort is n-log-n). + * + * It re-implements the OLD top-keep inline (qsort + threshold + scans) on a private + * buffer so the A/B runs in one process, same inputs, same warm caches -- a controlled + * comparison. It calls the REAL (new) partial_select_desc via the include-glm.c + * pattern, replicating the production threshold derivation + scans. + * + * Methodology (chosen to be honest, not to flatter the change): + * - keep = 2048 (the real GLM-5.2 index_topk), nk swept across context lengths from + * the 2049 activation boundary up to 65536 (a long conversation). + * - Three score shapes: (a) realistic peaked -- a few hot keys, long tail, the shape + * real DSA attention scores take; (b) uniform random -- no structure; (c) a plateau + * of ties, to exercise the boundary-membership path. + * - Each (shape, nk) is timed over N_REPEAT=2000 iterations, with the scores frozen + * so both algorithms do IDENTICAL work. We report median ns/call and the new/old + * ratio. A warmup pass primes caches before timing. + * + * Run: make tests/bench_dsa_select && ./tests/bench_dsa_select (not in TEST_BINS) + */ +#define main coli_glm_main_unused +#include "../glm.c" +#undef main + +#include +#include + +/* ---- the OLD algorithm, verbatim from dev before #356, on a private buffer ---- */ +static int cmp_pdesc_old(const void *a, const void *b){ + float x=*(const float*)a, y=*(const float*)b; return xy?-1:0; } +static void keep_old(const float *isc, int nk, int keep, int *dst, int *nd_out){ + float *tmp=malloc((size_t)nk*sizeof(float)); + memcpy(tmp,isc,(size_t)nk*sizeof(float)); + qsort(tmp,(size_t)nk,sizeof(float),cmp_pdesc_old); + float thr=tmp[keep-1]; int nd=0; + for(int t=0;tthr) dst[nd++]=t; + for(int t=0;t=0 && ts[b]>k){ ts[b+1]=ts[b]; b--; } ts[b+1]=k; } + free(dst); + return ts[N_REPEAT/2]; +} + +/* Sort an array of doubles ascending (median-of-medians aggregation below). */ +static void dsort(double *a, int n){ + for(int s=1;s=0 && a[b]>k){ a[b+1]=a[b]; b--; } a[b+1]=k; } +} + +/* the NEW algorithm calls the real partial_select_desc + replicates the production + * threshold derivation and position scans. */ +static void keep_new(const float *isc, int nk, int keep, int *dst, int *nd_out){ + float *tmp=malloc((size_t)nk*sizeof(float)); + memcpy(tmp,isc,(size_t)nk*sizeof(float)); + partial_select_desc(tmp,nk,keep); + float thr=tmp[0]; for(int t=1;tthr) dst[nd++]=t; + for(int t=0;t> 17; brng ^= brng << 5; + return (double)(brng >> 8) * (1.0 / 16777216.0); } +static void fill_realistic(float *isc, int nk){ /* few hot, long distinct tail */ + for(int i=0;i boundary path */ + for(int i=0;i +#include + +/* ---- the OLD algorithm, verbatim from dev before #354, on a private buffer ---- */ +static float *s_pbuf; static int *s_pidx; static double *s_ref; +static int cmp_pdesc_old(const void *a, const void *b){ + double pa = s_ref[*(const int*)a], pb = s_ref[*(const int*)b]; + return pa < pb ? 1 : pa > pb ? -1 : 0; } +static void dist_build_old(const float *lo, int V, double temp, double nuc){ + double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i]; + double s = 0, invt = 1.0 / (temp > 1e-4 ? temp : 1e-4); + for (int i = 0; i < V; i++){ s_ref[i] = exp((lo[i]-mx)*invt); s += s_ref[i]; } + for (int i = 0; i < V; i++) s_ref[i] /= s; + if (nuc > 0 && nuc < 1.0){ + for (int i = 0; i < V; i++) s_pidx[i] = i; + qsort(s_pidx, V, sizeof(int), cmp_pdesc_old); + double cum = 0; int keep = V; + for (int i = 0; i < V; i++){ cum += s_ref[s_pidx[i]]; if (cum >= nuc){ keep = i+1; break; } } + double s2 = 0; + for (int i = keep; i < V; i++) s_ref[s_pidx[i]] = 0; + for (int i = 0; i < keep; i++) s2 += s_ref[s_pidx[i]]; + for (int i = 0; i < keep; i++) s_ref[s_pidx[i]] /= s2; + } + (void)s_pbuf; +} + +/* ---- timing: median of N_REPEAT runs in ns/call, sorted ascending ---- */ +#define N_REPEAT 2000 +static double bench_ns(void (*fn)(const float*,int,double,double), + const float *lo, int V, double temp, double nuc){ + static double ts[N_REPEAT]; + for (int r = 0; r < N_REPEAT; r++){ + double t0 = now_s(); + fn(lo, V, temp, nuc); + ts[r] = (now_s() - t0) * 1e9; + } + /* insertion sort the N_REPEAT samples (small), take median */ + for (int a = 1; a < N_REPEAT; a++){ double k = ts[a]; int b = a-1; + while (b >= 0 && ts[b] > k){ ts[b+1] = ts[b]; b--; } ts[b+1] = k; } + return ts[N_REPEAT/2]; +} + +/* the NEW algorithm is the real dist_build, but it writes g_pbuf (not a private buf). + * Wrap it so the bench signature matches, and set the globals it reads. */ +static void dist_build_new(const float *lo, int V, double temp, double nuc){ + g_temp = (float)temp; g_nuc = (float)nuc; + dist_build(lo, V); +} + +/* deterministic logit fill for three shapes */ +static uint32_t brng = 0xA5A5A5A5u; +static double brand(void){ brng ^= brng << 13; brng ^= brng >> 17; brng ^= brng << 5; + return (double)(brng >> 8) * (1.0 / 16777216.0); } +static void fill_realistic(float *lo, int V){ /* one hot, exponential tail -- like real logits */ + for (int i = 0; i < V; i++) lo[i] = (float)(-4.0 * brand() - (double)i * 0.0001); + lo[0] = 6.f; lo[V/50] = 4.f; lo[V/200] = 3.f; +} +static void fill_uniform(float *lo, int V){ /* worst case for the heap: max pop count */ + for (int i = 0; i < V; i++) lo[i] = 0.f; +} +static void fill_plateau(float *lo, int V){ /* ties: blocks of equal value */ + for (int i = 0; i < V; i++) lo[i] = (float)(-(double)(i / 50)); +} + +int main(void){ + int V = 151936; + float *lo = malloc((size_t)V * sizeof(float)); + s_ref = malloc((size_t)V * sizeof(double)); + s_pidx = malloc((size_t)V * sizeof(int)); + /* force the new dist_build to allocate g_pbuf/g_pidx at full V once */ + g_temp = 0.7f; g_nuc = 0.9f; dist_build(lo, V); + + double temp = 0.7; + struct { const char *name; void (*fill)(float*,int); } shapes[] = { + { "realistic", fill_realistic }, + { "uniform", fill_uniform }, + { "plateau", fill_plateau }, + }; + double nucs[] = { 0.5, 0.9, 0.95, 0.99 }; + + printf("bench_topp: top-p truncation, old (qsort) vs new (heap) V=%d temp=%.2f\n", V, temp); + printf("%-12s %6s %14s %14s %9s %9s\n", "shape", "nuc", "old ns/call", "new ns/call", "speedup", "keep"); + printf("-----------------------------------------------------------------------------\n"); + + for (size_t sh = 0; sh < sizeof(shapes)/sizeof(shapes[0]); sh++){ + shapes[sh].fill(lo, V); + for (size_t ni = 0; ni < sizeof(nucs)/sizeof(nucs[0]); ni++){ + double nuc = nucs[ni]; + /* warmup both paths so caches/branch predictors are primed */ + for (int w = 0; w < 50; w++){ dist_build_old(lo, V, temp, nuc); dist_build_new(lo, V, temp, nuc); } + double t_old = bench_ns(dist_build_old, lo, V, temp, nuc); + double t_new = bench_ns(dist_build_new, lo, V, temp, nuc); + /* keep count = non-zero entries the new path leaves (== old's keep) */ + int keep = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) keep++; + printf("%-12s %6.2f %14.0f %14.0f %8.2fx %9d\n", + shapes[sh].name, nuc, t_old, t_new, t_old / t_new, keep); + } + printf("\n"); + } + + printf("bench_topp: done (lower ns is better; speedup = old/new)\n"); + free(lo); free(s_ref); free(s_pidx); + return 0; +} diff --git a/c/tests/test_benchmark_cuda_fixture.py b/c/tests/test_benchmark_cuda_fixture.py index a704052..3feaf11 100644 --- a/c/tests/test_benchmark_cuda_fixture.py +++ b/c/tests/test_benchmark_cuda_fixture.py @@ -1,11 +1,12 @@ import unittest -from tools.benchmark_cuda_fixture import parse_output +from tools.benchmark_cuda_fixture import parse_output, parse_p0 SAMPLE = """ REPLAY decode: 4 tokens | 12.34 tok/s PROFILE: expert-disk 1.25s | expert-matmul 2.50s | attention 0.75s | lm_head 0.10s | other -0.05s +P0-EXEC: routed CPU 1.200s / 123.40 GB/s (456 row) | routed GPU critical 0.150s | router 0.200s | residual P2P 0.030s / 75 hop | orchestration 0.100s """ @@ -19,6 +20,9 @@ class ParseOutputTest(unittest.TestCase): with self.assertRaisesRegex(RuntimeError, "benchmark output missing"): parse_output("REPLAY decode: 4 tokens | 12.34 tok/s", "engine failed") + def test_extracts_p0_profile(self): + self.assertEqual(parse_p0(SAMPLE), [1.2, 123.4, 456.0, 0.15, 0.2, 0.03, 75.0, 0.1]) + if __name__ == "__main__": unittest.main() diff --git a/c/tests/test_dsa_select.c b/c/tests/test_dsa_select.c new file mode 100644 index 0000000..99fecb0 --- /dev/null +++ b/c/tests/test_dsa_select.c @@ -0,0 +1,223 @@ +/* DSA top-keep partial-select: the quickselect rewrite (#356) must produce a + * BIT-IDENTICAL kept-position set to the old full-vocab qsort, for every score + * shape the attention indexer can see. + * + * Why this test exists (#356): attention_rows() selects the top-`keep` context + * keys (index_topk=2048 on GLM-5.2) to attend to. It previously did this by + * full-qsorting all `nk` scores (O(nk log nk)) and reading tmp[keep-1] as the + * threshold. It now does a partial_select_desc (quickselect, O(nk) average) and + * takes the threshold as the min of the selected top-keep block. The contract is + * subtle but STRONGER than test_topp's: + * + * The two position-order scans that build dst[] -- + * for t: if isc[t] > thr -> keep (strictly above threshold) + * for t: if isc[t] == thr -> keep (ties, in position order) + * -- are UNCHANGED by the rewrite. So if the threshold value is identical, + * the kept-position set is identical element-by-element (not just as a + * multiset, which is all the unstable sampling heap in #335 could promise). + * + * Strategy: drive the REAL partial_select_desc (via the include-glm.c pattern) + * and replicate the production threshold derivation + scans, then compare the + * resulting dst[] against an INDEPENDENT reference that re-implements the OLD + * algorithm (full qsort + tmp[keep-1] threshold) on a private buffer. The kept + * sets must be element-wise equal on every shape, including tie plateaus where + * the boundary membership is decided by the position scan. + * + * We also directly unit-test partial_select_desc's partition invariant: after + * the call, max(a[keep..n)) <= min(a[0..keep)) -- i.e. the keep largest really + * did land in the prefix. This catches a broken quickselect even before the + * end-to-end comparison. + * + * In-memory only (no scratch files), so it builds clean on the Windows MinGW CI + * job without the unmerged compat shim. */ +#define main coli_glm_main_unused +#include "../glm.c" +#undef main + +#include + +static int g_nfails = 0; + +#define FAIL(fmt, ...) do { \ + fprintf(stderr, " FAIL [%s nk=%d keep=%d shape=%s]: " fmt "\n", \ + label, nk, keep, shape_name, ##__VA_ARGS__); \ + g_nfails++; \ + return; \ +} while (0) + +/* ---- independent reference: the OLD algorithm (full qsort + tmp[keep-1]) ---- */ +/* qsort comparator matching the production cmp_fdesc exactly (desc, unstable). */ +static int cmp_ref_desc(const void *a, const void *b){ + float x=*(const float*)a, y=*(const float*)b; return xy?-1:0; } + +/* Reproduce the OLD glm.c:2589-2596 exactly: copy, qsort desc, threshold = + * tmp[keep-1], then the two position-order scans into dst[]. Returns nd. */ +static int keep_old(const float *isc, int nk, int keep, int *dst){ + float *tmp=malloc((size_t)nk*sizeof(float)); + memcpy(tmp,isc,(size_t)nk*sizeof(float)); + qsort(tmp,(size_t)nk,sizeof(float),cmp_ref_desc); + float thr=tmp[keep-1]; + int nd=0; + for(int t=0;tthr) dst[nd++]=t; + for(int t=0;tthr) dst[nd++]=t; + for(int t=0;t= every element of tmp[keep..n). + * (>=, not >: equal values may sit on either side of the partition boundary, + * which is fine -- the threshold is the MIN of the prefix, and the position + * scan handles ties.) */ + float top_min=INFINITY, tail_max=-INFINITY; + for(int i=0;itail_max) tail_max=tmp[i]; + if(!(top_min >= tail_max)) + FAIL("partition invariant violated: top_min=%.9g < tail_max=%.9g", top_min, tail_max); + free(tmp); +} + +/* ---- end-to-end: old vs new kept-set must be element-wise identical ---- */ +static void check_case(const char *label, int nk, int keep, const char *shape_name, + const float *isc){ + int *da=malloc((size_t)nk*sizeof(int)); + int *db=malloc((size_t)nk*sizeof(int)); + int na=keep_old(isc,nk,keep,da); + int nb=keep_new(isc,nk,keep,db); + + /* 1. both keep exactly `keep` positions (the contract: keep the top-keep by + * count). A count mismatch is a real bug, not a tie artifact. */ + if(na!=keep) FAIL("old kept %d, expected %d (old path is the reference)", na, keep); + if(nb!=keep) FAIL("new kept %d, expected %d", nb, keep); + if(na!=nb) FAIL("keep-count mismatch: old=%d new=%d", na, nb); + + /* 2. element-wise identical dst[]. This is the strong contract: because the + * threshold is derived identically and the position-order scans are byte- + * for-byte the same, the kept SET and its ORDER must match exactly. (This + * is what makes #356 cleaner than #335, which was multiset-only.) */ + int first_diff=-1; + for(int i=0;i=0) + FAIL("kept-set differs at index %d: old dst[%d]=%d new dst[%d]=%d", + first_diff, first_diff, da[first_diff], first_diff, db[first_diff]); + + /* 3. also check the partition invariant directly (catches a subtly broken + * quickselect even if the threshold happened to come out right). */ + check_partition(label,isc,nk,keep,shape_name); + + free(da); free(db); + printf(" ok [nk=%d keep=%d shape=%s]\n", nk, keep, shape_name); +} +#undef FAIL + +/* deterministic xorshift32 RNG (matches the test_i4_grouped.c / test_topp.c convention) */ +static uint32_t rng_state = 0x12345678u; +static uint32_t xr(void){ rng_state ^= rng_state << 13; rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; return rng_state; } +static double frand(void){ return (xr() >> 8) * (1.0 / 16777216.0); } /* [0,1) */ + +/* fill scores for a given shape. Shapes stress the threshold boundary and the + * quickselect's median-of-three pivot differently. */ +static void fill_shape(float *isc, int nk, int shape){ + switch(shape){ + case 0: /* uniform random distinct (no ties): the clean contract case */ + for(int i=0;i3) isc[nk/3]=1.f; if(nk>2) isc[nk/2]=0.5f; break; + case 2: /* strictly decreasing geometric (no ties): sorted input -- worst case + * for a naive quickselect; median-of-three must handle it */ + for(int i=0;i boundary membership decided + * entirely by the position scan (exercises the ==thr path) */ + for(int i=0;i degenerate threshold, all kept + * via the ==thr scan; quickselect must not infinite-loop or corrupt */ + for(int i=0;i=n / k==1 / k==n edges. */ + int nks[] = {1, 2, 8, 64, 2049, 4097, 8193}; + int keeps[] = {1, 8, 256, 1024, 2048}; + int n_shapes = 6; + + int cases = 0; + for(size_t ni=0; nin, but skip the plateau/geometric edge if nk<7) */ + fill_shape(isc,nk,shape); + for(size_t ki=0; kink) continue; /* keep<=nk invariant of the production code */ + if(keep<=0) continue; + char label[40]; snprintf(label,sizeof(label),"nk[%zu]/keep[%zu]/shape[%d]",ni,ki,shape); + const char *sn=(const char*[]){"random","peaked","decreasing","increasing","plateau","all-equal"}[shape]; + check_case(label,nk,keep,sn,isc); + cases++; + } + } + free(isc); + } + + /* edge: keep == nk (nothing to partition; both paths keep everything) */ + { + int nk=100, keep=100; float isc[100]; + for(int i=0;i every kept slot is a tie; + * the position scan must pick positions 0..keep-1 deterministically */ + { + int nk=1000, keep=500; float isc[1000]; + for(int i=0;i positions 0..%d]\n",keep,keep-1); cases++; + free(db); + } + + printf("\ntest_dsa_select: %d cases run, %d failure(s)\n", cases, g_nfails); + if(g_nfails){ printf("test_dsa_select: FAIL\n"); return 1; } + printf("test_dsa_select: ok\n"); + return 0; +} diff --git a/c/tests/test_i4_grouped.c b/c/tests/test_i4_grouped.c new file mode 100644 index 0000000..2a9521e --- /dev/null +++ b/c/tests/test_i4_grouped.c @@ -0,0 +1,201 @@ +/* Exactness test for the grouped-int4 kernel (fmt=4, one f32 scale per `gs` + * elements along I) against a plain-C reference that dequantizes and multiplies + * in double. + * + * Why this test exists: matmul_i4_grouped is the REFERENCE the CUDA fmt=4 path + * (#298) is expected to reproduce, and it had no test of its own. Debugging a + * backend against an unverified oracle means two moving targets. Anyone porting + * fmt=4 to a new backend can now diff against a kernel that is known exact here. + * + * Covers: I a clean multiple of gs, I with a partial last group (the `glen` + * clamp), odd I (the nibble tail), gs larger than I (single group), and the + * nibble edges 0 and 15 (which decode to -8 and +7 — an offset encoding, NOT + * two's complement; getting this backwards is silent and looks like noise). + * + * FP note: the kernel sums each group in f32 (AVX2 accumulator + scalar tail) + * while the reference sums in double, so we compare against a relative epsilon + * rather than bit-exactly. The tolerance is tight enough that a wrong scale + * index, a wrong group boundary or a swapped nibble cannot hide under it — + * those are O(1) relative errors, not O(1e-6). */ +#define main coli_glm_main_unused +#include "../glm.c" +#undef main + +static uint32_t rng_state=0xC0FFEEu; +static uint32_t xr(void){ rng_state^=rng_state<<13; rng_state^=rng_state>>17; rng_state^=rng_state<<5; return rng_state; } +static float frand(void){ return (float)((int)(xr()%2001)-1000)/1000.0f; } + +/* Reference: dequantize nibble -> (v-8)*scale[group], accumulate in double. + * Deliberately the dumbest possible expression of the format. */ +static void ref_grouped(double *y, double *mag, const float *x, const uint8_t *q4, + const float *scale, int S, int I, int O, int gs){ + int rb=(I+1)/2, ng=(I+gs-1)/gs; + for(int o=0;o>1]; + int nib=(i&1)?(int)(byte>>4):(int)(byte&0xF); + double term=(double)xs[i] * (double)(nib-8) * (double)scl[i/gs]; + a += term; m += fabs(term); + } + y[(int64_t)s*O+o]=a; + /* Sum of |terms|: the scale the f32 rounding error actually lives on. + * Comparing against |result| instead would flag pure cancellation -- + * a dot product of signed terms can land near zero, and then a 1e-6 + * absolute error reads as a 1e-3 relative one. That is the accumulator's + * precision, not a kernel defect. A wrong scale index or group boundary + * shifts the result by a fraction OF THE TERMS, so it is caught here. */ + mag[(int64_t)s*O+o]=m; + } + } +} + +static int check(const char *name, int S, int I, int O, int gs, int fill_edges){ + int rb=(I+1)/2, ng=(I+gs-1)/gs; + uint8_t *q4=malloc((size_t)O*rb); + float *scale=malloc((size_t)O*ng*sizeof(float)); + float *x=malloc((size_t)S*I*sizeof(float)); + float *y=malloc((size_t)S*O*sizeof(float)); + double *yr=malloc((size_t)S*O*sizeof(double)); + double *ym=malloc((size_t)S*O*sizeof(double)); + if(!q4||!scale||!x||!y||!yr||!ym){ fprintf(stderr,"%s: OOM\n",name); return 1; } + + for(size_t i=0;i<(size_t)O*rb;i++) q4[i]=(uint8_t)(xr()&0xFF); + if(fill_edges){ + /* nibble extremes: 0x0F -> +7, 0x00 -> -8. A two's-complement misread + * turns 15 into -1 instead of +7 and the error is data-dependent noise. */ + for(size_t i=0;i<(size_t)O*rb && i<64;i++) q4[i]=(i&1)?0x00:0xFF; + } + /* scales span a few orders of magnitude: a wrong group index shows up big */ + for(int i=0;i1e-30 ? d/ym[i] : d; /* error relative to the summed magnitude */ + if(rel>worst) worst=rel; + if(rel>1e-6){ + if(bad<3) fprintf(stderr,"%s: [%d] got %.9g want %.9g (|terms| %.3g, rel %.3g)\n", + name,i,(double)y[i],yr[i],ym[i],rel); + bad++; + } + } + free(q4);free(scale);free(x);free(y);free(yr);free(ym); + if(bad){ fprintf(stderr,"%s: FAIL (%d/%d mismatched, worst rel %.3g)\n",name,bad,S*O,worst); return 1; } + printf(" %-42s ok (S=%d I=%d O=%d gs=%d ng=%d, worst rel %.2g)\n",name,S,I,O,gs,ng,worst); + return 0; +} + +/* matmul_i4_grouped_pair (fused gate+up, #298) reads x once instead of twice. + * Checked two ways, because "identical" is only true where it can be: + * + * - Correctness, always: both outputs must match the double reference within + * the same magnitude-relative epsilon as the unfused kernel. + * - Bit-exactness, only when I % gs == 0: then every group is covered by the + * AVX2 body, whose accumulation order is identical to the unfused kernel, so + * the results agree to the last bit. This is the shape the real g64 + * checkpoints have (I = 2048 / 6144, gs = 64), i.e. the production path. + * + * With a PARTIAL last group the group tail falls to scalar code, and the + * compiler is free to contract/reassociate the fused body differently from the + * single-matrix one. The results then differ by ~1e-7 -- rounding, not logic + * (which of gate/up "differs" is arbitrary, the tell that it is FP luck). + * Demanding bit-exactness there would report a compiler artifact as a bug. */ +#ifdef COLI_HAVE_GROUPED_PAIR +static int check_pair(const char *name, int S, int I, int O, int gs){ + int rb=(I+1)/2, ng=(I+gs-1)/gs; + uint8_t *qg=malloc((size_t)O*rb), *qu=malloc((size_t)O*rb); + float *sg=malloc((size_t)O*ng*sizeof(float)), *su=malloc((size_t)O*ng*sizeof(float)); + float *x=malloc((size_t)S*I*sizeof(float)); + float *yg=malloc((size_t)S*O*sizeof(float)), *yu=malloc((size_t)S*O*sizeof(float)); + float *rg=malloc((size_t)S*O*sizeof(float)), *ru=malloc((size_t)S*O*sizeof(float)); + double *dg=malloc((size_t)S*O*sizeof(double)), *du=malloc((size_t)S*O*sizeof(double)); + double *mg=malloc((size_t)S*O*sizeof(double)), *mu=malloc((size_t)S*O*sizeof(double)); + if(!qg||!qu||!sg||!su||!x||!yg||!yu||!rg||!ru||!dg||!du||!mg||!mu){ fprintf(stderr,"%s: OOM\n",name); return 1; } + + for(size_t i=0;i<(size_t)O*rb;i++){ qg[i]=(uint8_t)(xr()&0xFF); qu[i]=(uint8_t)(xr()&0xFF); } + for(int i=0;i1e-30 ? fabs((double)yg[i]-dg[i])/mg[i] : fabs((double)yg[i]-dg[i]); + double eu = mu[i]>1e-30 ? fabs((double)yu[i]-du[i])/mu[i] : fabs((double)yu[i]-du[i]); + if(eg>worst) worst=eg; + if(eu>worst) worst=eu; + if(eg>1e-6||eu>1e-6){ + if(bad<3) fprintf(stderr,"%s: [%d] gate %.9g/%.9g up %.9g/%.9g (rel %.3g/%.3g)\n", + name,i,(double)yg[i],dg[i],(double)yu[i],du[i],eg,eu); + bad++; + } + if(yg[i]!=rg[i]||yu[i]!=ru[i]) exact=0; + } + /* Aligned shapes run entirely through the AVX2 body: same order as unfused, + * so bit-exactness is a real invariant there and worth asserting. */ + if(I%gs==0 && !exact){ + fprintf(stderr,"%s: FAIL fused != unfused bitwise on an ALIGNED shape " + "(no scalar tail runs here; the orders must match)\n",name); + bad++; + } + free(qg);free(qu);free(sg);free(su);free(x);free(yg);free(yu);free(rg);free(ru); + free(dg);free(du);free(mg);free(mu); + if(bad){ fprintf(stderr,"%s: FAIL (%d mismatched)\n",name,bad); return 1; } + printf(" %-42s ok (S=%d I=%d O=%d gs=%d, worst rel %.2g%s)\n",name,S,I,O,gs,worst, + I%gs==0?", bit-exact vs unfused":""); + return 0; +} +#endif + +int main(void){ + int fail=0; + printf("test_i4_grouped: matmul_i4_grouped vs plain-C dequant reference\n"); + + /* the shape the g64 checkpoints actually use */ + fail|=check("gs=64, I multiple of gs", 2, 512, 8, 64, 0); + fail|=check("gs=64, single row single token", 1, 128, 1, 64, 0); + fail|=check("gs=64, nibble edges (0x00/0xFF)", 1, 256, 4, 64, 1); + + /* partial last group: glen clamp, the classic off-by-one */ + fail|=check("gs=64, partial last group (I=200)", 2, 200, 4, 64, 0); + fail|=check("gs=64, I just over a group (I=65)", 1, 65, 3, 64, 0); + fail|=check("gs=64, I one under a group (I=63)", 1, 63, 3, 64, 0); + + /* odd I: the scalar nibble tail (i+1 == I) */ + fail|=check("gs=64, odd I (I=201)", 2, 201, 4, 64, 0); + fail|=check("gs=16, odd I (I=33)", 1, 33, 2, 16, 0); + + /* gs > I: everything in one group */ + fail|=check("gs=128 > I=64 (single group)", 1, 64, 4, 128, 0); + + /* the other documented group size */ + fail|=check("gs=128, I multiple of gs", 2, 512, 4, 128, 0); + + /* batch: S>1 exercises the per-s inner loop against a shared scale row */ + fail|=check("gs=64, batch S=8", 8, 320, 6, 64, 0); + +#ifdef COLI_HAVE_GROUPED_PAIR + printf("test_i4_grouped: matmul_i4_grouped_pair (fused gate+up) vs two separate calls\n"); + fail|=check_pair("pair: gs=64, I multiple of gs", 2, 512, 8, 64); + fail|=check_pair("pair: gs=64, partial last group",2, 200, 4, 64); + fail|=check_pair("pair: gs=64, odd I (I=201)", 2, 201, 4, 64); + fail|=check_pair("pair: gs=64, decode S=1", 1, 320, 6, 64); + fail|=check_pair("pair: gs=128, I=512", 2, 512, 4, 128); +#endif + + if(fail){ printf("test_i4_grouped: FAIL\n"); return 1; } + printf("test_i4_grouped: ok\n"); + return 0; +} diff --git a/c/tests/test_logit_nan.c b/c/tests/test_logit_nan.c new file mode 100644 index 0000000..981d74d --- /dev/null +++ b/c/tests/test_logit_nan.c @@ -0,0 +1,70 @@ +/* Regression for non-finite-logit poisoning of sampling. + * + * A single NaN or +Inf in the logits (a bad streamed expert tile, or an fp + * overflow in the matmul at a low-RAM eviction boundary) used to make softmax + * produce an all-NaN g_pbuf; dist_sample then never satisfied cum>=u and fell + * through to return token 0 — so the engine silently emitted an unbroken run of + * token 0 with no error, on the DEFAULT serve path (TEMP>0, 0 +#include +#define main coli_glm_main_unused +#include "../glm.c" +#undef main + +static int approx1(double x){ return x > 0.999 && x < 1.001; } + +int main(void){ + /* --- argmax_v must skip NaN (greedy decode + speculative-verify paths) --- */ + { float lo[8]={NAN,1.f,5.f,2.f,NAN,-3.f,4.f,0.f}; + assert(argmax_v(lo,8)==2 && "pick max finite (idx2=5.0), not NaN-pinned idx0"); } + { float lo[8]={3.f,INFINITY,1.f,2.f,0.f,-1.f,2.5f,1.5f}; + assert(argmax_v(lo,8)==1 && "pick the +Inf position"); } + { float lo[8]; for(int i=0;i<8;i++) lo[i]=NAN; + assert(argmax_v(lo,8)==0 && "all-NaN: no crash, defined fallback"); } + + g_temp=0.7f; g_nuc=0.9f; /* the default serve/chat sampling path */ + + /* --- dist_build: a NaN logit must yield a finite one-hot, not all-NaN --- */ + { float lo[8]={0.5f,1.f,NAN,8.f,0.2f,-1.f,0.f,0.3f}; /* max finite = idx3 (8.0) */ + dist_build(lo,8); + double sum=0; int nan=0; + for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; } + assert(!nan && "g_pbuf must be finite after a NaN logit"); + assert(approx1(sum) && "g_pbuf must normalize to 1"); + assert(approx1(g_pbuf[3]) && "mass must land on the max finite logit (idx3)"); + assert(dist_sample(8,-1)==3 && "sampler emits the finite argmax, not token 0"); } + + /* --- NaN at index 0: poisons the max scan itself (the old mx=lo[0] seed), + * the failure mode that starts before the sum (review note on #369) --- */ + { float lo[8]={NAN,1.f,0.5f,6.f,0.2f,-1.f,0.f,0.3f}; /* max finite = idx3 (6.0) */ + dist_build(lo,8); + double sum=0; int nan=0; + for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; } + assert(!nan && "NaN at lo[0] must not poison via the mx seed"); + assert(approx1(sum) && "still normalizes to 1"); + assert(dist_sample(8,-1)==3 && "emits the max finite logit, not token 0"); } + + /* --- all-NaN logits: worst case — must stay finite, no crash --- */ + { float lo[8]; for(int i=0;i<8;i++) lo[i]=NAN; + dist_build(lo,8); + for(int i=0;i<8;i++) assert(g_pbuf[i]==g_pbuf[i] && "all-NaN: g_pbuf stays finite"); } + + /* --- regression: clean logits still produce a valid distribution --- */ + { float lo[8]={0.1f,0.2f,3.0f,0.4f,0.5f,0.6f,0.7f,0.8f}; /* peak = idx2 */ + dist_build(lo,8); + double sum=0; int nan=0; + for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; } + assert(!nan && "clean softmax stays finite"); + assert(approx1(sum) && "clean softmax must sum to 1"); + assert(g_pbuf[2]>=g_pbuf[0] && "peak token keeps the most mass"); } + + printf("OK test_logit_nan: argmax_v NaN-skip + dist_build finite-collapse\n"); + return 0; +} diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py index f8aec86..4cda9ef 100644 --- a/c/tests/test_openai_server.py +++ b/c/tests/test_openai_server.py @@ -380,6 +380,25 @@ class DispatcherTest(unittest.TestCase): engine.close() self.assertEqual(chunks, ["é"]) + def test_records_profile_snapshots_from_prof_lines(self): + def respond(process, frame): + request_id = frame.split()[1] + process.stdout.feed(b"DATA " + request_id + b" 2\nok\n") + process.stdout.feed(b"PROF 2.500 7 12 0.400 0.100 0.900 0.600 0.200 15\n") + process.stdout.feed(b"DONE " + request_id + b" STAT 12 4.8 0 1.0 7 0\n") + + process = FakeProcess(respond) + with patch("openai_server.subprocess.Popen", return_value=process): + engine = Engine("glm", "model") + engine.generate("hello", 16, 0.7, 0.9, lambda _: None) + engine.close() + self.assertEqual(engine.profile_seq, 1) + self.assertEqual(list(engine.profile), [{ + "wall_s": 2.5, "prompt_tokens": 7, "completion_tokens": 12, + "expert_disk_s": 0.4, "expert_wait_s": 0.1, "expert_matmul_s": 0.9, + "attention_s": 0.6, "lm_head_s": 0.2, "forwards": 15, + }]) + def test_cancels_generation_after_consumer_disconnects(self): request_id = None @@ -443,6 +462,20 @@ class HTTPTest(unittest.TestCase): self.assertIn("queued", scheduler) self.assertEqual(health["kv_slots"], 2) + def test_profile_reports_recent_turns_without_auth(self): + with urlopen(self.base + "/profile", timeout=2) as response: + self.assertEqual(json.load(response), {"seq": 0, "turns": []}) + turn = {"wall_s": 2.5, "prompt_tokens": 7, "completion_tokens": 12, + "expert_disk_s": 0.4, "expert_wait_s": 0.1, "expert_matmul_s": 0.9, + "attention_s": 0.6, "lm_head_s": 0.2, "forwards": 15} + self.engine.profile = [turn] + self.engine.profile_seq = 1 + try: + with urlopen(self.base + "/profile", timeout=2) as response: + self.assertEqual(json.load(response), {"seq": 1, "turns": [turn]}) + finally: + del self.engine.profile, self.engine.profile_seq + def test_browser_preflight(self): request = Request(self.base + "/v1/chat/completions", method="OPTIONS", headers={ "Origin": "http://localhost:5173", diff --git a/c/tests/test_openai_tools_e2e.py b/c/tests/test_openai_tools_e2e.py new file mode 100644 index 0000000..d921843 --- /dev/null +++ b/c/tests/test_openai_tools_e2e.py @@ -0,0 +1,182 @@ +"""End-to-end tool-calling test for the OpenAI gateway (#401). + +Unlike the unit tests in test_openai_server.py (which call parse_tool_calls / +render_chat directly), this suite runs openai_server.py as a real subprocess +against a mock engine that speaks the actual SERVE wire protocol +(READY / SUBMIT / DATA / DONE), then talks to it over real HTTP. It pins down +the full path a coding client exercises: tool declaration rendering, marker +suppression in streamed deltas (across chunk boundaries), tool_calls in both +response shapes, and the <|observation|> round trip. +""" +import json +import os +import socket +import subprocess +import sys +import tempfile +import unittest +import urllib.request +from pathlib import Path + +SERVER = Path(__file__).resolve().parent.parent / "openai_server.py" +MODEL_ID = "glm-5.2-colibri" + +# Mock engine: replies are keyed on the prompt so one process covers every case. +# Prompts received are appended to MOCK_LOG for assertions on the rendering. +MOCK_ENGINE = r'''#!/usr/bin/env python3 +import sys, os +out, inp = sys.stdout.buffer, sys.stdin.buffer +out.write(b"\x01\x01READY\x01\x01\n" + b"STAT 0 0 0 0 0\n"); out.flush() + +def reply(rid, text, chunks=1): + data = text.encode("utf-8") + n = max(1, len(data) // chunks) + for i in range(0, len(data), n): + part = data[i:i+n] + out.write(("DATA %s %d\n" % (rid, len(part))).encode() + part + b"\n"); out.flush() + out.write(("DONE %s STAT %d 1.0 50.0 10.0 42 0\n" % (rid, len(text.split()))).encode()) + out.flush() + +while True: + line = inp.readline() + if not line: break + f = line.decode().strip().split() + if not f or f[0] != "SUBMIT": continue + rid, plen = f[1], int(f[3]) + prompt = inp.read(plen).decode("utf-8", "replace"); inp.read(1) + with open(os.environ["MOCK_LOG"], "a") as log: + log.write(prompt + "\n\x00\n") + if "" in prompt: + reply(rid, "25 degrees and sunny in Rome.") + elif "weather in Rome" in prompt: + reply(rid, "get_weathercity" + "Rome") + elif "weather in Milan" in prompt: + # split across many tiny DATA chunks: streamed marker suppression must + # hold even when a marker straddles a chunk boundary + reply(rid, "Checking. get_weathercity" + "Milan", chunks=20) + else: + reply(rid, "Hello from the mock engine.") +''' + +TOOLS = [{"type": "function", "function": { + "name": "get_weather", + "description": "Current weather for a city", + "parameters": {"type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"]}}}] + + +@unittest.skipUnless(os.name == "posix", + "the mock engine is a shebang script the gateway execs directly; " + "Windows CreateProcess cannot run it. The gateway logic under test " + "is platform-independent and covered by the POSIX CI jobs.") +class ToolCallingE2E(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + mock = Path(cls.tmp.name) / "mock_engine.py" + mock.write_text(MOCK_ENGINE) + mock.chmod(0o755) + cls.mock_log = Path(cls.tmp.name) / "prompts.log" + cls.mock_log.touch() + with socket.socket() as probe: # free port, then hand it to the server + probe.bind(("127.0.0.1", 0)) + cls.port = probe.getsockname()[1] + env = dict(os.environ, MOCK_LOG=str(cls.mock_log)) + cls.server = subprocess.Popen( + [sys.executable, str(SERVER), "--model", cls.tmp.name, + "--engine", str(mock), "--port", str(cls.port)], + env=env, stderr=subprocess.DEVNULL) + cls.base = f"http://127.0.0.1:{cls.port}/v1" + for _ in range(100): + try: + urllib.request.urlopen(cls.base + "/models", timeout=2) + return + except OSError: + if cls.server.poll() is not None: + raise RuntimeError("gateway exited during startup") + import time + time.sleep(0.1) + raise RuntimeError("gateway did not come up") + + @classmethod + def tearDownClass(cls): + cls.server.terminate() + cls.server.wait(timeout=5) + cls.tmp.cleanup() + + def post(self, body, stream=False): + req = urllib.request.Request( + self.base + "/chat/completions", json.dumps(body).encode(), + {"Content-Type": "application/json"}) + resp = urllib.request.urlopen(req, timeout=30) + if not stream: + return json.loads(resp.read()) + events = [] + for raw in resp: + line = raw.decode().strip() + if line.startswith("data: ") and line != "data: [DONE]": + events.append(json.loads(line[6:])) + return events + + def test_tool_call_non_stream(self): + r = self.post({"model": MODEL_ID, "tools": TOOLS, + "messages": [{"role": "user", + "content": "What is the weather in Rome?"}]}) + choice = r["choices"][0] + self.assertEqual(choice["finish_reason"], "tool_calls") + calls = choice["message"]["tool_calls"] + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0]["function"]["name"], "get_weather") + self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"city": "Rome"}) + self.assertNotIn("", choice["message"].get("content") or "") + + def test_tool_call_streamed_markers_suppressed(self): + events = self.post({"model": MODEL_ID, "tools": TOOLS, "stream": True, + "messages": [{"role": "user", + "content": "What is the weather in Milan?"}]}, + stream=True) + deltas = [e["choices"][0]["delta"] for e in events if e["choices"]] + text = "".join(d.get("content") or "" for d in deltas) + self.assertNotIn("", text) + self.assertNotIn("", text) + calls = [d["tool_calls"] for d in deltas if d.get("tool_calls")] + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0][0]["function"]["name"], "get_weather") + self.assertEqual(json.loads(calls[0][0]["function"]["arguments"]), + {"city": "Milan"}) + finish = [e["choices"][0]["finish_reason"] for e in events + if e["choices"] and e["choices"][0].get("finish_reason")] + self.assertEqual(finish, ["tool_calls"]) + + def test_tool_result_round_trip(self): + r = self.post({"model": MODEL_ID, "tools": TOOLS, "messages": [ + {"role": "user", "content": "What is the weather in Rome?"}, + {"role": "assistant", "content": None, "tool_calls": [ + {"id": "call_x", "type": "function", + "function": {"name": "get_weather", + "arguments": "{\"city\": \"Rome\"}"}}]}, + {"role": "tool", "tool_call_id": "call_x", + "content": "25 degrees, sunny"}]}) + choice = r["choices"][0] + self.assertEqual(choice["finish_reason"], "stop") + self.assertFalse(choice["message"].get("tool_calls")) + self.assertIn("25 degrees", choice["message"]["content"]) + rendered = self.mock_log.read_text().split("\x00")[-2] + self.assertIn("<|observation|>25 degrees, sunny", + rendered) + self.assertIn("# Tools", rendered) + self.assertIn('"get_weather"', rendered) + + def test_no_tools_plain_text(self): + r = self.post({"model": MODEL_ID, + "messages": [{"role": "user", "content": "Hi!"}]}) + choice = r["choices"][0] + self.assertEqual(choice["finish_reason"], "stop") + self.assertIn("mock engine", choice["message"]["content"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_ragged_attention.cu b/c/tests/test_ragged_attention.cu new file mode 100644 index 0000000..dd1e2f8 --- /dev/null +++ b/c/tests/test_ragged_attention.cu @@ -0,0 +1,39 @@ +#include "../backend_cuda.h" + +#include +#include +#include + +int main(){ + int dev=0;if(!coli_cuda_init(&dev,1))return 77; + constexpr int S=3,H=2,Q=2,R=1,V=2,K=3,D=H*V,O=3,T=3; + std::vector w(H*(Q+V)*K),p(O*D),q(S*H*(Q+R)); + for(size_t i=0;i> l(S),r(S); + const float *lp[S],*rp[S]; + const void *keys[S]; + for(int s=0;s0` e' falso su NaN ovunque). + * Ora la distribuzione degenere ripiega sull'argmax dei logit FINITI e avvisa una volta. + * + * Il test verifica: (a) con logit sani il campionamento resta corretto; (b) con NaN/+Inf + * iniettato il token scelto e' l'argmax dei FINITI (mai 0 per default), su ogni posizione + * del NaN inclusa lo[0]; (c) nessun NaN sopravvive in g_pbuf. */ +#define main coli_glm_main_unused +#include "../glm.c" +#undef main +#include + +static uint32_t rs=0x1234abcd; static uint32_t xr(){rs^=rs<<13;rs^=rs>>17;rs^=rs<<5;return rs;} + +static int pbuf_has_nan(int V){ for(int i=0;i 0 e nessun NaN nel buffer */ + for(int i=0;ipv){pv=g_pbuf[i];picked=i;} + if(b<2 && picked!=amax){ /* NaN e +Inf: delta esatto su amax */ + printf(" FAIL: %s @ %d -> picked %d, atteso argmax finito %d\n",bn[b],at,picked,amax); fail=1; + } + } + } + if(!fail) printf(" NaN/+Inf iniettato: argmax dei finiti vince, mai 0/NaN ok\n"); + + /* (c) caso estremo: TUTTI non finiti -> non deve crashare, buffer valido */ + for(int i=0;i +#include +#include +#ifndef _WIN32 +#include +#include +#endif + +#include "../st.h" + +#define CHECK(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "%s:%d: check failed: %s\n", __FILE__, __LINE__, #condition); \ + return 1; \ + } \ +} while (0) + +static void write_snap(const char *dir, int truncate_bytes) { + char path[512]; + snprintf(path, sizeof(path), "%s/model.safetensors", dir); + unsigned char data[96]; + for (int i = 0; i < 96; i++) data[i] = (unsigned char)(i * 7 + 3); + const char *hdr = "{\"t\":{\"dtype\":\"U8\",\"shape\":[96],\"data_offsets\":[0,96]}}"; + uint64_t hlen = strlen(hdr); + FILE *f = fopen(path, "wb"); + fwrite(&hlen, 8, 1, f); + fwrite(hdr, 1, hlen, f); + fwrite(data, 1, (size_t)(96 - truncate_bytes), f); + fclose(f); +} + +int main(void) { + /* relative to the CWD, per test_stops: MinGW .exe files resolve Windows + * paths and "/tmp" is not one */ + char dir[] = "test_st_pread_XXXXXX"; + if (!mkdtemp(dir)) { perror("mkdtemp"); return 1; } + + /* 1) chunk loop: 96-byte tensor read 7 bytes at a time, content exact */ + write_snap(dir, 0); + shards S; st_init(&S, dir); + unsigned char out[96] = {0}; + st_read_raw(&S, "t", out, 0); + for (int i = 0; i < 96; i++) CHECK(out[i] == (unsigned char)(i * 7 + 3)); + +#ifndef _WIN32 + /* 2) shard truncated AFTER st_init (init validates static bounds, so the + * pread path only fires when the file shrinks underneath a live handle): + * child must exit(1) with an honest message, not perror's "Success" */ + char shard[512]; snprintf(shard, sizeof(shard), "%s/model.safetensors", dir); + struct stat sb; CHECK(stat(shard, &sb) == 0); + CHECK(truncate(shard, sb.st_size - 40) == 0); + int pipefd[2]; CHECK(pipe(pipefd) == 0); + pid_t pid = fork(); CHECK(pid >= 0); + if (pid == 0) { + dup2(pipefd[1], 2); close(pipefd[0]); close(pipefd[1]); + unsigned char buf[96]; + st_read_raw(&S, "t", buf, 0); /* inherited handles; must exit(1) inside */ + _exit(42); /* reaching here = bug */ + } + close(pipefd[1]); + char err[512] = {0}; + ssize_t n = read(pipefd[0], err, sizeof(err)-1); (void)n; + close(pipefd[0]); + int status = 0; waitpid(pid, &status, 0); + CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 1); + CHECK(strstr(err, "short read") != NULL); + CHECK(strstr(err, "Success") == NULL); +#else + /* fork/pipe/truncate are POSIX; Windows still runs the chunk-loop check */ + printf("test_st_pread: truncation subtest skipped on Windows\n"); +#endif + + char cmd[600]; +#ifdef _WIN32 + snprintf(cmd, sizeof(cmd), "rmdir /s /q %s", dir); +#else + snprintf(cmd, sizeof(cmd), "rm -rf %s", dir); +#endif + if (system(cmd)) {} + printf("test_st_pread: chunk loop + honest truncation error: ok\n"); + return 0; +} diff --git a/c/tests/test_stops.c b/c/tests/test_stops.c new file mode 100644 index 0000000..00f922b --- /dev/null +++ b/c/tests/test_stops.c @@ -0,0 +1,136 @@ +/* Stop-token arming: the engine must not depend on a converted checkpoint's + * config.json being complete. + * + * Why this test exists (#298, #307): GLM-5.2 declares THREE eos ids + * (<|endoftext|>, <|user|>, <|observation|>). A conversion tool that rewrites + * config.json with a reduced eos_token_id list leaves the engine stopping on + * fewer tokens than the model actually emits -- and the ones it misses get + * DETOKENIZED AND PRINTED INTO THE CHAT as literal text, with generation + * continuing past the real end of the turn. @woolcoxm hit exactly this on a + * g64 checkpoint and confirmed it by comparing token ids. + * + * Two independent defenses, one test each: + * 1. eos_token_id is unioned from generation_config.json (HuggingFace's + * authority for generation) as well as config.json. + * 2. every added-token the TOKENIZER marks "special":true is a stop, whatever + * the configs say. Those are control tokens (<|user|>, , ...) and are + * never legitimate content. / are "special":false and + * must NOT be swept up -- they are real output. + * + * Defense 2 is what makes this robust against checkpoints we don't control: + * even with BOTH configs mutilated, a control token cannot leak into a reply. */ +#define main coli_glm_main_unused +#include "../glm.c" +#undef main + +static const char *TOKJSON = +"{\"model\":{\"vocab\":{\"a\":0,\"b\":1,\"c\":2},\"merges\":[[\"a\",\"b\"]]}," +" \"added_tokens\":[" +" {\"id\":100,\"content\":\"<|endoftext|>\",\"special\":true}," +" {\"id\":101,\"content\":\"<|user|>\",\"special\":true}," +" {\"id\":102,\"content\":\"<|observation|>\",\"special\":true}," +" {\"id\":103,\"content\":\"<|assistant|>\",\"special\":true}," +" {\"id\":104,\"content\":\"\",\"special\":true}," +" {\"id\":110,\"content\":\"\",\"special\":false}," +" {\"id\":111,\"content\":\"\",\"special\":false}" +"]}"; + +/* minimal config.json that survives load_cfg's range validation */ +static void write_cfg(const char *dir, const char *fname, const char *eos_json){ + char p[512]; snprintf(p,sizeof(p),"%s/%s",dir,fname); + FILE *f=fopen(p,"w"); if(!f){ perror(p); exit(1); } + fprintf(f,"{\"hidden_size\":64,\"num_hidden_layers\":2,\"num_attention_heads\":4," + "\"n_routed_experts\":8,\"num_experts_per_tok\":2,\"moe_intermediate_size\":32," + "\"intermediate_size\":64,\"first_k_dense_replace\":1,\"q_lora_rank\":0," + "\"kv_lora_rank\":16,\"qk_nope_head_dim\":8,\"qk_rope_head_dim\":8," + "\"v_head_dim\":8,\"n_shared_experts\":1,\"vocab_size\":200," + "\"n_group\":1,\"topk_group\":1,\"rope_theta\":10000.0"); + if(eos_json) fprintf(f,",\"eos_token_id\":%s",eos_json); + fprintf(f,"}\n"); fclose(f); +} +static void write_tok(const char *dir){ + char p[512]; snprintf(p,sizeof(p),"%s/tokenizer.json",dir); + FILE *f=fopen(p,"w"); if(!f){ perror(p); exit(1); } + fputs(TOKJSON,f); fclose(f); +} +static void rm_file(const char *dir, const char *fname){ + char p[512]; snprintf(p,sizeof(p),"%s/%s",dir,fname); remove(p); +} + +static int expect(const char *what, int id, int want){ + int got=is_stop(id); + if(got!=want){ fprintf(stderr," FAIL %s: token %d stop=%d, expected %d\n",what,id,got,want); return 1; } + return 0; +} + +int main(void){ + int fail=0; + /* Relative to the CWD, like test_compat_direct's TMPF — NOT "/tmp/...". + * These binaries are built by MinGW into native Windows .exe files, which + * resolve Windows paths: "/tmp" is not one, so mkdtemp there fails ENOENT + * and the whole `make check` goes red on the windows job (and only there). */ + char dir[]="test_stops_XXXXXX"; + if(!mkdtemp(dir)){ perror("mkdtemp"); return 1; } + write_tok(dir); + char tkp[512]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",dir); + Tok T; tok_load(&T,tkp); + printf("test_stops: stop arming vs incomplete checkpoint metadata\n"); + + /* 1. the tokenizer's "special" flag must survive loading, and must NOT + * swallow /, which are real content. */ + if(!T.id_special[101]){ fprintf(stderr," FAIL <|user|> not marked special\n"); fail=1; } + if(!T.id_special[104]){ fprintf(stderr," FAIL not marked special\n"); fail=1; } + if(T.id_special[110]){ fprintf(stderr," FAIL wrongly marked special\n"); fail=1; } + if(T.id_special[111]){ fprintf(stderr," FAIL wrongly marked special\n"); fail=1; } + if(!fail) printf(" tokenizer: special flag parsed, / excluded ok\n"); + + /* 2. config.json mutilated to one eos, generation_config.json intact: + * the union must recover all three. This is @woolcoxm's case. */ + { Cfg c; memset(&c,0,sizeof c); + write_cfg(dir,"config.json","[100]"); + write_cfg(dir,"generation_config.json","[100,101,102]"); + load_cfg(&c,dir); + if(c.n_stop!=3){ fprintf(stderr," FAIL union: expected 3 stops, got %d\n",c.n_stop); fail=1; } + else printf(" config eos=[100] + generation_config=[100,101,102] -> 3 stops ok\n"); } + + /* 3. no generation_config.json at all: must not crash, config alone wins */ + { Cfg c; memset(&c,0,sizeof c); + write_cfg(dir,"config.json","[100,101,102]"); + rm_file(dir,"generation_config.json"); + load_cfg(&c,dir); + if(c.n_stop!=3){ fprintf(stderr," FAIL no-genconfig: expected 3, got %d\n",c.n_stop); fail=1; } + else printf(" generation_config.json absent -> config alone, no crash ok\n"); } + + /* 4. BOTH configs mutilated: the tokenizer's special set must still stop the + * turn on every control token, and must still leave alone. */ + { Cfg c; memset(&c,0,sizeof c); + write_cfg(dir,"config.json","[100]"); + write_cfg(dir,"generation_config.json","[100]"); + load_cfg(&c,dir); + stops_arm_tok(&c,-1,&T); + fail|=expect("endoftext (config)", 100,1); + fail|=expect("<|user|> (tokenizer)", 101,1); + fail|=expect("<|observation|>", 102,1); + fail|=expect("<|assistant|>", 103,1); + fail|=expect("", 104,1); + fail|=expect(" must NOT stop",110,0); + fail|=expect(" must NOT stop",111,0); + fail|=expect("plain token must NOT stop",2,0); + if(!fail) printf(" both configs mutilated -> tokenizer still stops all 5 control tokens ok\n"); } + + /* 5. T=NULL (validation/oracle path): config stops only, unchanged behaviour */ + { Cfg c; memset(&c,0,sizeof c); + write_cfg(dir,"config.json","[100]"); + rm_file(dir,"generation_config.json"); + load_cfg(&c,dir); + stops_arm_tok(&c,-1,NULL); + fail|=expect("T=NULL: config stop",100,1); + fail|=expect("T=NULL: no tokenizer sweep",101,0); + if(!fail) printf(" T=NULL -> config stops only (validation path untouched) ok\n"); } + + rm_file(dir,"config.json"); rm_file(dir,"generation_config.json"); rm_file(dir,"tokenizer.json"); + rmdir(dir); + if(fail){ printf("test_stops: FAIL\n"); return 1; } + printf("test_stops: ok\n"); + return 0; +} diff --git a/c/tests/test_topp.c b/c/tests/test_topp.c new file mode 100644 index 0000000..9276507 --- /dev/null +++ b/c/tests/test_topp.c @@ -0,0 +1,280 @@ +/* Top-p (nucleus) truncation in dist_build: the partial-select rewrite (#335) must be + * indistinguishable from the old full-vocab qsort for every shape dist_sample can see. + * + * Why this test exists (#335): dist_build() previously qsort-ed the entire 151936-entry + * vocab on every sampled token to find the few-hundred-token head whose cumulative mass + * reaches g_nuc. It now heapifies (O(V)) and pops only the head (k * O(log V)). The win + * is structural; the risk is a silent sampling-distribution change, because the contract + * is subtle: + * + * dist_sample() iterates g_pbuf[0..V-1] BY TOKEN ID and sums probabilities directly. + * So dist_build MUST leave g_pbuf indexed by id (never reordered) AND must zero every + * truncated tail entry -- merely excluding the tail from the head would leave mass on + * it and the sampled distribution would drift with no crash and no error. + * + * Strategy: drive the REAL dist_build (via the test_stops.c include-glm.c pattern) on a + * sweep of distributions and g_nuc values, and compare against an INDEPENDENT reference + * that re-implements the OLD algorithm (full qsort + zero-tail + renorm) in double on a + * private buffer. On shapes with no ties the renormalized head must be BIT-IDENTICAL to + * the reference (the issue's stated invariant: s2 accumulates in the same descending + * order). On tie shapes, where the unstable qsort already left ordering unspecified, we + * check multiset equality instead. Every shape also checks: exact-zero tails, head sums + * to 1.0, and a sane keep-count. + * + * No scratch files: the test runs entirely in memory (no mkdtemp), so it builds clean on + * the Windows MinGW CI job without the unmerged compat shim (#352). */ +#define main coli_glm_main_unused +#include "../glm.c" +#undef main + +#include + +static int g_nfails = 0; + +/* pointer set by ref_build so cmp_ref_desc can read the current reference buffer + * (the qsort comparator gets no user-data argument in C). */ +static const double *g_ref_p = NULL; + +#define FAIL(fmt, ...) do { \ + fprintf(stderr, " FAIL [%s V=%d nuc=%.3f shape=%s]: " fmt "\n", \ + label, V, nuc, shape_name, ##__VA_ARGS__); \ + g_nfails++; \ + return; \ +} while (0) + +/* ---- independent reference: the OLD algorithm, in double, on a private buffer ------- */ +/* Stable qsort by descending probability (ties broken by ascending index, which makes + * the reference deterministic regardless of the production comparator). */ +static int cmp_ref_desc(const void *a, const void *b){ + double pa = ((const double *)g_ref_p)[*(const int*)a]; + double pb = ((const double *)g_ref_p)[*(const int*)b]; + if (pa < pb) return 1; + if (pa > pb) return -1; + /* tie -> lower index first (stable, unlike the production comparator) */ + return *(const int*)a - *(const int*)b; +} + +/* Build the reference distribution into out[0..V-1] (indexed by token id), mirroring the + * old dist_build: softmax(lo/temp) truncated to top-p nuc, tail zeroed, head renormalized. + * Returns the keep-count through *keep_out. */ +static void ref_build(const float *lo, int V, double temp, double nuc, + double *out, int *pidx, int *keep_out){ + double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i]; + double s = 0, invt = 1.0 / (temp > 1e-4 ? temp : 1e-4); + for (int i = 0; i < V; i++){ out[i] = exp((lo[i]-mx)*invt); s += out[i]; } + for (int i = 0; i < V; i++) out[i] /= s; + + if (nuc > 0 && nuc < 1.0){ + for (int i = 0; i < V; i++) pidx[i] = i; + qsort(pidx, V, sizeof(int), cmp_ref_desc); + double cum = 0; int keep = V; + for (int i = 0; i < V; i++){ cum += out[pidx[i]]; if (cum >= nuc){ keep = i+1; break; } } + double s2 = 0; + for (int i = keep; i < V; i++) out[pidx[i]] = 0; + for (int i = 0; i < keep; i++) s2 += out[pidx[i]]; + for (int i = 0; i < keep; i++) out[pidx[i]] /= s2; + *keep_out = keep; + } else { + *keep_out = V; + } +} + +/* count how many production g_pbuf entries are non-zero == the head size */ +static int head_count(int V){ + int n = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) n++; return n; +} + +/* Run one case: load logits into g_pbuf via the real dist_build, compare to reference. + * shape_name is for diagnostics only. */ +static void check_case(const char *label, int V, double nuc, const char *shape_name, + const float *lo){ + /* reference on a private buffer */ + double *ref = malloc((size_t)V * sizeof(double)); + int *ridx = malloc((size_t)V * sizeof(int)); + int ref_keep = 0; + g_ref_p = ref; /* cmp_ref_desc reads this */ + ref_build(lo, V, g_temp, nuc, ref, ridx, &ref_keep); + + /* production: drive the real dist_build (writes the global g_pbuf) */ + g_nuc = (float)nuc; + dist_build(lo, V); + + int got_keep = head_count(V); + + /* 1. keep-count must match the reference exactly. The partial select and the old + * qsort keep the same NUMBER of tokens by construction (same cumulative-mass rule); + * a count divergence is a real bug, not a tie artifact. */ + if (got_keep != ref_keep) + FAIL("keep-count mismatch: got %d, ref %d", got_keep, ref_keep); + + /* 2. Detect ties across the WHOLE pre-truncation distribution, not just the kept set. + * A tie at the head/tail boundary makes which-side-a-token-lands-on interchangeable: + * both algorithms keep the right count but may keep different MEMBERS. So any input + * with a duplicated softmax value needs the relaxed multiset comparison below. We + * detect this on the reference softmax (pre-truncation) by sorting all V values. */ + int has_ties = 0; + { + double *all = malloc((size_t)V * sizeof(double)); + /* reconstruct the pre-truncation softmax the same way ref_build does */ + double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i]; + double s = 0, invt = 1.0 / (g_temp > 1e-4 ? g_temp : 1e-4); + for (int i = 0; i < V; i++){ all[i] = exp((lo[i]-mx)*invt); s += all[i]; } + for (int i = 0; i < V; i++) all[i] /= s; + for (int a = 1; a < V; a++){ double k = all[a]; int b = a-1; + while (b >= 0 && all[b] > k){ all[b+1] = all[b]; b--; } all[b+1] = k; } + for (int a = 1; a < V; a++) if (all[a] == all[a-1]){ has_ties = 1; break; } + free(all); + } + + if (has_ties){ + /* Multiset equality of the non-zero (head) values. Ties make membership + * interchangeable, so we compare sorted value-multisets, not id-aligned values. + * Tolerance is 1e-6 relative -- the engine uses float arithmetic, the reference + * double, so sub-ULP noise is expected (matches test_i4_grouped.c's convention). */ + double *got = malloc((size_t)ref_keep * sizeof(double)); + int gm = 0; + for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) got[gm++] = (double)g_pbuf[i]; + if (gm != ref_keep) + FAIL("tie-shape head size mismatch: got %d non-zero, ref %d", gm, ref_keep); + for (int a = 1; a < gm; a++){ double k = got[a]; int b = a-1; + while (b >= 0 && got[b] > k){ got[b+1] = got[b]; b--; } got[b+1] = k; } + double *rsort = malloc((size_t)ref_keep * sizeof(double)); + int rm = 0; + for (int i = 0; i < V; i++) if (ref[i] != 0.0) rsort[rm++] = ref[i]; + for (int a = 1; a < rm; a++){ double k = rsort[a]; int b = a-1; + while (b >= 0 && rsort[b] > k){ rsort[b+1] = rsort[b]; b--; } rsort[b+1] = k; } + int mm = 0; double worst = 0; + for (int i = 0; i < gm; i++){ + double d = fabs(got[i] - rsort[i]); + double rel = rsort[i] > 1e-30 ? d / rsort[i] : d; + if (rel > worst) worst = rel; + if (rel > 1e-6) mm++; + } + free(got); free(rsort); + if (mm) FAIL("tie-shape multiset mismatch: %d/%d head values differ beyond 1e-6 rel (worst %.3g)", + mm, ref_keep, worst); + } else { + /* No ties anywhere: membership is forced, so compare id-aligned head values. The + * engine computes in float (g_pbuf /= (float)s2) while the reference uses double, + * so the comparison is relative-tolerance (1e-6), not bit-exact -- the partial + * select and qsort accumulate s2 in the same descending order, so any difference + * is pure float-rounding noise, not an ordering bug. */ + int bad = 0; int first_id = -1; float gv = 0, rv = 0; double worst = 0; + for (int i = 0; i < V; i++){ + if (ref[i] == 0.0) continue; /* tail */ + float want = (float)ref[i]; + double d = fabs((double)g_pbuf[i] - (double)want); + double rel = fabs((double)want) > 1e-30 ? d / fabs((double)want) : d; + if (rel > worst) worst = rel; + if (rel > 1e-6){ + bad++; if (first_id < 0){ first_id = i; gv = g_pbuf[i]; rv = want; } + if (bad > 3) break; + } + } + if (bad) + FAIL("head not within 1e-6 rel of reference: %d entries differ (first id %d: got %.9g want %.9g, worst %.3g)", + bad, first_id, (double)gv, (double)rv, worst); + } + + /* 3. head must renormalize to 1.0 (within float epsilon) */ + double sum = 0; for (int i = 0; i < V; i++) sum += g_pbuf[i]; + if (fabs(sum - 1.0) > 1e-5) + FAIL("head does not sum to 1.0: sum=%.12g (keep=%d)", sum, got_keep); + + free(ref); free(ridx); + printf(" ok [V=%d nuc=%.3f shape=%s keep=%d%s sum=%.10f]\n", + V, nuc, shape_name, got_keep, has_ties ? " (ties)" : "", sum); +} +#undef FAIL + +/* deterministic xorshift32 RNG (matches the test_i4_grouped.c convention) */ +static uint32_t rng_state = 0x12345678u; +static uint32_t xr(void){ rng_state ^= rng_state << 13; rng_state ^= rng_state >> 17; + rng_state ^= rng_state << 5; return rng_state; } +static double frand(void){ return (xr() >> 8) * (1.0 / 16777216.0); } /* [0,1) */ + +/* fill logits for a given shape. Shapes chosen to stress the comparator and the head/tail + * boundary differently. */ +static void fill_shape(float *lo, int V, int shape){ + switch (shape){ + case 0: /* uniform -> every token equal probability -> massive tie plateau */ + for (int i = 0; i < V; i++) lo[i] = 0.f; break; + case 1: /* peaked: one dominant token, rest small and distinct (no ties). + * The fixed hot-spots are clamped to V-1 so small V (incl. V=1) doesn't + * write out of bounds and corrupt heap metadata on the later free(lo). */ + for (int i = 0; i < V; i++) lo[i] = (float)(-1.0 - frand()*4.0); + lo[0] = 3.f; lo[V/3 comparator tie handling */ + for (int i = 0; i < V; i++) lo[i] = (float)(-(double)(i / 7)); /* 7-wide plateaus */ + break; + case 4: /* sharp-tail: a few hot, then a long flat floor (small tie at the floor). + * Hot count is min(12,V) so V<12 (incl. V=1) stays in bounds. */ + for (int i = 0; i < V; i++) lo[i] = -8.f; + { int hot = V<12 ? V : 12; for (int i = 0; i < hot; i++) lo[i] = (float)(2.0 - frand()); } break; + } +} + +int main(void){ + /* sizes: small for exhaustive tie detection up to near-production scale */ + int sizes[] = {1, 2, 8, 64, 257, 1519}; /* 1519 ~= V/100 of GLM-5.2 */ + double nucs[] = {0.001, 0.5, 0.9, 0.999}; /* tight -> almost-everything */ + int n_shapes = 5; + + /* temperature used by dist_build: pick a normal serving value */ + g_temp = 0.7f; + + int cases = 0; + for (size_t si = 0; si < sizeof(sizes)/sizeof(sizes[0]); si++){ + int V = sizes[si]; + /* dist_build allocates g_pbuf/g_pidx ONCE and reuses them (single-V invariant in + * real serving, where V is the constant model vocab). This sweep varies V, so free + * and force a reallocation per size -- otherwise a later, larger V would overflow + * the buffer sized for the first (smallest) V. */ + free(g_pbuf); g_pbuf = NULL; free(g_pidx); g_pidx = NULL; + float *lo = malloc((size_t)V * sizeof(float)); + for (int shape = 0; shape < n_shapes; shape++){ + fill_shape(lo, V, shape); + for (size_t ni = 0; ni < sizeof(nucs)/sizeof(nucs[0]); ni++){ + char label[32]; snprintf(label, sizeof(label), "size[%zu]/shape[%d]", si, shape); + const char *sn = (const char*[]){"uniform","peaked","geometric","plateau","sharptail"}[shape]; + check_case(label, V, nucs[ni], sn, lo); + cases++; + } + } + free(lo); + } + + /* guard-off path: g_nuc >= 1 must skip truncation entirely (full softmax kept) */ + { + int V = 256; float lo[256]; + for (int i = 0; i < V; i++) lo[i] = (float)(frand()*4 - 2); + g_nuc = 1.0f; dist_build(lo, V); + int nz = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) nz++; + if (nz != V){ fprintf(stderr, " FAIL [guard-off nuc=1.0]: %d/%d entries kept, expected all\n", nz, V); g_nfails++; } + else printf(" ok [guard-off nuc=1.0 keep=%d]\n", nz); + cases++; + + g_nuc = 0.0f; dist_build(lo, V); + nz = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) nz++; + if (nz != V){ fprintf(stderr, " FAIL [guard-off nuc=0.0]: %d/%d entries kept, expected all\n", nz, V); g_nfails++; } + else printf(" ok [guard-off nuc=0.0 keep=%d]\n", nz); + cases++; + } + + /* extreme tie edge case: V=1, single token -> keep=1 regardless of nuc */ + { + float lo[1] = {5.f}; + g_nuc = 0.5f; dist_build(lo, 1); + if (g_pbuf[0] == 0.f || !(fabs((double)g_pbuf[0] - 1.0) < 1e-6)){ + fprintf(stderr, " FAIL [V=1]: g_pbuf[0]=%.9g, expected 1.0\n", (double)g_pbuf[0]); g_nfails++; + } else printf(" ok [V=1 keep=1]\n"); + cases++; + } + + printf("\ntest_topp: %d cases run, %d failure(s)\n", cases, g_nfails); + if (g_nfails){ printf("test_topp: FAIL\n"); return 1; } + printf("test_topp: ok\n"); + return 0; +} diff --git a/c/tok.h b/c/tok.h index f61011e..7d1140b 100644 --- a/c/tok.h +++ b/c/tok.h @@ -42,6 +42,11 @@ typedef struct { hmap vocab; /* stringa byte-level -> id */ hmap merges; /* "left\0right" -> rank */ char **id2str; int *id_added; int n_ids; /* id -> stringa; id_added=1 se added-token (output letterale) */ + int *id_special; /* 1 = added-token con "special":true nel tokenizer: + * token di CONTROLLO (<|user|>, <|assistant|>, , ...), + * mai contenuto legittimo di una risposta. Distinto da + * id_added, che copre anche / ("special" + * false), i quali sono testo vero e vanno renderizzati. */ Special *sp; int nsp; /* added tokens, ordinati per lunghezza decrescente */ uint32_t byte2cp[256]; int byte2cp_len[256]; char byte2str[256][3]; int16_t cp2byte[1024]; @@ -106,6 +111,7 @@ static void tok_load(Tok *T, const char *path){ T->n_ids=maxid+1; T->id2str=calloc(T->n_ids,sizeof(char*)); T->id_added=calloc(T->n_ids,sizeof(int)); + T->id_special=calloc(T->n_ids,sizeof(int)); /* vocab: stringa -> id (capacita' potenza di 2, ~2-3x) */ int vc=1; while(vc < vocab->len*2) vc<<=1; @@ -133,6 +139,8 @@ static void tok_load(Tok *T, const char *path){ char *content=json_get(a,"content")->str; int id=(int)json_get(a,"id")->num; T->sp[i].str=content; T->sp[i].len=(int)strlen(content); T->sp[i].id=id; T->id2str[id]=content; T->id_added[id]=1; + jval *sf=json_get(a,"special"); /* "special": true/false */ + if(sf && sf->t==J_BOOL && sf->boolean) T->id_special[id]=1; } qsort(T->sp,T->nsp,sizeof(Special),cmp_sp_len); /* match piu' lungo per primo */ } diff --git a/c/tools/benchmark_cuda_fixture.py b/c/tools/benchmark_cuda_fixture.py index bced18e..26512dd 100644 --- a/c/tools/benchmark_cuda_fixture.py +++ b/c/tools/benchmark_cuda_fixture.py @@ -17,6 +17,12 @@ PROFILE_RE = re.compile( r"\| attention ([0-9.]+)s .* lm_head ([0-9.]+)s \| other ([0-9.-]+)s" ) PROFILE_KEYS = ("disk", "expert_matmul", "attention", "lm_head", "other") +P0_RE = re.compile( + r"P0-EXEC: routed CPU ([0-9.]+)s / ([0-9.]+) GB/s \(([0-9]+) row\) \| routed GPU critical ([0-9.]+)s \| " + r"router ([0-9.]+)s \| residual P2P ([0-9.]+)s / ([0-9]+) hop \| orchestration ([0-9.]+)s" +) +P0_KEYS = ("routed_cpu", "routed_cpu_gb_s", "routed_cpu_rows", "routed_gpu_critical", + "router", "p2p", "p2p_hops", "orchestration") def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]: @@ -30,11 +36,20 @@ def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]: return float(speed.group(1)), [disk] + [float(value) for value in rest] -def execute(engine: str, env: dict[str, str]) -> tuple[float, list[float]]: +def parse_p0(stdout: str) -> list[float]: + """Extract the optional PROF=1 execution-layer breakdown.""" + row = P0_RE.search(stdout) + if not row: + raise RuntimeError("benchmark output missing P0-EXEC profile") + return [float(value) for value in row.groups()] + + +def execute(engine: str, env: dict[str, str]) -> tuple[float, list[float], list[float]]: run = subprocess.run( [engine, "4", "4", "4"], env=env, text=True, capture_output=True, check=True ) - return parse_output(run.stdout, run.stderr) + speed, profile = parse_output(run.stdout, run.stderr) + return speed, profile, parse_p0(run.stdout) def main() -> None: @@ -63,6 +78,8 @@ def main() -> None: OMP_NUM_THREADS=str(args.threads), OMP_PROC_BIND="spread", OMP_PLACES="cores", + DRAFT="0", + PROF="1", ) execute(args.engine, base | {"STATS": str(stats)}) @@ -86,13 +103,15 @@ def main() -> None: execute(args.engine, base | extra) # warm-up speeds = {name: [] for name in modes} profiles = {name: [] for name in modes} + p0_profiles = {name: [] for name in modes} names = list(modes) for run_index in range(args.runs): order = names[run_index % len(names):] + names[:run_index % len(names)] for name in order: - speed, profile = execute(args.engine, base | modes[name]) + speed, profile, p0 = execute(args.engine, base | modes[name]) speeds[name].append(speed) profiles[name].append(profile) + p0_profiles[name].append(p0) result = {} for name in names: @@ -103,6 +122,10 @@ def main() -> None: key: statistics.median(row[index] for row in profiles[name]) for index, key in enumerate(PROFILE_KEYS) }, + "median_p0": { + key: statistics.median(row[index] for row in p0_profiles[name]) + for index, key in enumerate(P0_KEYS) + }, } print(json.dumps(result, indent=2)) diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py index 382125e..eac51f9 100644 --- a/c/tools/convert_fp8_to_int4.py +++ b/c/tools/convert_fp8_to_int4.py @@ -286,6 +286,17 @@ def main(): # testa MTP a int4 = acceptance ~0-4% (misurato, issue #8): il draft sbaglia sempre # e la speculazione non parte mai. A int8: 39-59%, 2.2-2.8 token/forward. a.ebits = 8 if (a.mtp or a.indexer) else 4 + if a.mtp and a.ebits < 8 and a.group_size <= 0: + # Non solo lossy: eh_proj ha ~20-30x di asimmetria di scala fra le due meta' di + # colonna, quindi l'int4 per-riga (UNA scala per riga) arrotonda a ZERO l'intera + # meta' embedding -> il draft non vede il token -> acceptance ~0% (issue #8). + # EN: not merely lossy: eh_proj has ~20-30x column-scale asymmetry, so per-row + # EN: int4 rounds its ENTIRE embedding half to exact zeros -> the draft cannot + # EN: see the input token -> ~0% acceptance (issue #8). A container converted + # EN: this way is repairable in place with tools/repair_mtp_int8.py. + print(f"WARNING: --mtp with --ebits {a.ebits} and per-row scales ZEROES eh_proj's " + "embedding half -> MTP acceptance ~0% (issue #8). Use the default --ebits 8, " + "or add --group-size 128 for group-scaled int4.") if a.xbits is None: a.xbits = a.ebits # Build per-type bits map. If a type-specific arg is set, use it; otherwise the @@ -299,6 +310,16 @@ def main(): if bits_map: print(f"[MIXED] precision map: " + ", ".join(f"{k}={v}bit" for k,v in sorted(bits_map.items()))) + # Il PIANO risolto, PRIMA di toccare qualunque cosa (#383): --mtp/--indexer cambiano il + # default di ebits a 8 (testa int4 = acceptance ~0%, issue #8) e il ramo grouped e' + # gated su bits<=4 — combinazioni sorprendenti devono mostrarsi al secondo 1 di un job + # da ore, non nel size-check dopo. EN: print the RESOLVED plan before doing anything. + mode = "MTP head only" if a.mtp else "DSA indexer only" if a.indexer else "main model" + grp = f"grouped gs={a.group_size} (fmt=4)" if (a.group_size and a.ebits <= 4) else \ + (f"PER-ROW (grouped branch needs bits<=4; ebits={a.ebits} disables it)" if a.group_size else "per-row") + print(f"[PLAN] mode: {mode} | source: {'local ' + a.indir if a.indir else 'download ' + a.repo} | " + f"experts {a.ebits}-bit, embed/lm_head {a.io_bits}-bit, x {a.xbits}-bit | {grp}") + if a.selftest_nvfp4: import torch # 1) LUT e2m1: i 16 codici devono decodificare esattamente ai valori attesi. @@ -379,14 +400,99 @@ def main(): if a.indir: # conversione locale (test) shards = sorted(glob.glob(os.path.join(a.indir, "*.safetensors"))) from safetensors.numpy import save_file + # #383: se l'indice c'e', i passaggi --mtp/--indexer convertono SOLO gli shard + # che contengono i tensori richiesti (3 invece di scandire tutti i 141 — ogni + # scansione a vuoto apre comunque uno shard da 5 GB). Senza indice: scansione + # completa come prima. + # EN: #383: when the index is present, the --mtp/--indexer passes convert ONLY + # the shards that hold the requested tensors (3 instead of scanning all 141 — + # every empty scan still opens a 5 GB shard). Without the index: full scan as + # before. + if a.mtp or a.indexer: + idxp = os.path.join(a.indir, "model.safetensors.index.json") + if os.path.exists(idxp): + wmap = json.load(open(idxp))["weight_map"] + if a.mtp: + want = {v for k, v in wmap.items() if k.startswith(f"model.layers.{a.n_layers}.")} + else: + want = {v for k, v in wmap.items() if "indexer" in k and 0 <= layer_idx(k) < a.n_layers} + keep = [sp for sp in shards if os.path.basename(sp) in want] + print(f"[PLAN] index: {len(keep)}/{len(shards)} local shard(s) hold the requested tensors") + shards = keep + # BUG #355: questo ramo ignorava --mtp/--indexer. Con --mtp scriveva + # out-NNNNN (gli STESSI nomi di una conversione normale) in ebits=8 e + # keep_mtp=False -> il "secondo passaggio MTP" nella stessa outdir + # SOVRASCRIVEVA il container gia' finito con una riconversione int8 + # completa, in silenzio (137/141 shard distrutti prima di accorgersene). + # Ora il ramo locale rispecchia il download path: prefisso corretto, + # flag passate, shard vuoti saltati. + prefix = "out-mtp-" if a.mtp else "out-idx-" if a.indexer else "out-" + # RIPRESA (#383): i nomi out-NNNNN contano gli shard EMESSI, non l'indice di + # input (gli shard senza tensori rilevanti non producono file), quindi "il + # file esiste" non basta per saltare il lavoro gia' fatto. Un manifest + # sidecar ricorda input -> output (o "vuoto") e con quali parametri: la + # ripresa salta solo cio' che combacia, e parametri diversi sulla stessa + # outdir vengono rifiutati invece di mescolare container (il modo #355). + # EN: RESUME (#383): out-NNNNN names count EMITTED shards, not the input + # EN: index (shards with no relevant tensors emit no file), so "the file + # EN: exists" is not enough to skip completed work. A sidecar manifest + # EN: records input -> output (or "empty") plus the conversion parameters: + # EN: resume skips only what matches, and different parameters on the same + # EN: outdir are refused instead of mixing containers (the #355 failure mode). + params = {"ebits": a.ebits, "io_bits": a.io_bits, "xbits": a.xbits, + "group_size": a.group_size, "n_layers": a.n_layers, "bits_map": bits_map} + prog_path = os.path.join(a.outdir, f".{prefix}progress.json") + prog = {} + if os.path.exists(prog_path): + try: prog = json.loads(open(prog_path).read()) + except (OSError, ValueError): prog = {} + if prog and prog.get("params") != params: + print(f"ERROR: {prog_path} records a conversion with {prog.get('params')};\n" + f" this run uses {params}. Refusing to mix conversions in the same " + f"outdir — use a fresh --outdir (or delete the manifest and the " + f"{prefix}*.safetensors shards to redo).") + return + done = prog.setdefault("shards", {}); prog["params"] = params + n = 0; fresh = 0; skipped = 0 for i, sp in enumerate(shards): - out = {}; convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits, group_size=a.group_size, bits_map=bits_map) - save_file(out, os.path.join(a.outdir, f"out-{i:05d}.safetensors")) - # copia config + tokenizer - for fn in ["config.json"]: - src = os.path.join(a.indir, fn) - if os.path.exists(src): shutil.copy(src, a.outdir) - print(f"converted {len(shards)} shards -> {a.outdir}") + key = os.path.basename(sp) + prev = done.get(key) # None = mai visto; "" = visto, vuoto; nome = emesso + if prev is not None and (prev == "" or os.path.exists(os.path.join(a.outdir, prev))): + if prev: n += 1 + skipped += 1 + continue + out = {} + convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits, + keep_mtp=a.mtp, keep_idx=a.indexer, + group_size=a.group_size, bits_map=bits_map) + if not out: # shard senza MTP/idx: niente file (come il download path) + done[key] = "" + else: + name = f"{prefix}{n:05d}.safetensors" + save_file(out, os.path.join(a.outdir, name)) + done[key] = name; n += 1; fresh += 1 + tmp_prog = prog_path + ".tmp" # scrittura atomica: una ripresa non vede mai un manifest mezzo scritto + with open(tmp_prog, "w") as f: json.dump(prog, f, indent=1) # EN: atomic write: a resume never sees a half-written manifest + os.replace(tmp_prog, prog_path) + if skipped: print(f"[RESUME] {skipped} shard(s) already done in {a.outdir}, skipped") + # metadati per la conversione principale: gli stessi quattro file del download + # path — senza tokenizer.json chat/serve non partono. I passaggi mtp/idx vanno + # nella stessa outdir di un container gia' completo di metadati. + # EN: metadata for the main pass: the same four files as the download path — + # EN: chat/serve won't start without tokenizer.json. The mtp/idx passes target + # EN: an outdir whose container already has its metadata. + if not a.mtp and not a.indexer: + copied, missing = [], [] + for fn in ["config.json", "tokenizer.json", "tokenizer_config.json", "generation_config.json"]: + src = os.path.join(a.indir, fn) + if os.path.exists(src): shutil.copy(src, a.outdir); copied.append(fn) + else: missing.append(fn) + print(f"[META] copied from {a.indir}: {', '.join(copied) if copied else 'nothing'}") + if missing: + print(f"[META] WARNING: not found in {a.indir}: {', '.join(missing)}" + + (" — chat/serve need tokenizer.json" if "tokenizer.json" in missing else "")) + tag = "MTP" if a.mtp else "indexer" if a.indexer else "main" + print(f"converted {fresh} {tag} shard(s), {n} in container -> {a.outdir} ({prefix}NNNNN)") return # reale: scarica shard per shard, converte, cancella diff --git a/c/tools/convert_olmoe.py b/c/tools/convert_olmoe.py index dd45806..dc25f6a 100644 --- a/c/tools/convert_olmoe.py +++ b/c/tools/convert_olmoe.py @@ -3,7 +3,10 @@ Downloads or converts a local OLMoE checkpoint (e.g., allenai/OLMoE-1B-7B-0125-Instruct). Dense weights stay as-is (engine reads BF16/F16 → F32 on load). -Expert weights get row-wise int8 quantization with float32 scales. +Expert weights get row-wise symmetric quantization to --ebits bits (default 4) +with float32 scales. Storage stays one value per int8 byte regardless of bits, +matching the engine's expert layout (olmoe.c quantize_rows) — for 4 bits the +values are simply confined to [-8, 7] with scales computed against qmax=7. Usage: python tools/convert_olmoe.py --repo allenai/OLMoE-1B-7B-0125-Instruct --out ./olmoe_i4 @@ -29,12 +32,21 @@ except ImportError as exc: EXPERT_KEY_RE = r"model\.layers\.\d+\.mlp\.experts\.\d+\.(gate_proj|up_proj|down_proj)\.weight" -def quantize_row(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: - """Row-wise int8 quantization. Returns (int8_weights, float32_scales).""" +def quantize_row(w: torch.Tensor, bits: int = 8) -> tuple[torch.Tensor, torch.Tensor]: + """Row-wise symmetric quantization to `bits` (2..8). + + Returns (int8_weights, float32_scales). Storage is one value per int8 byte + for every bit width — the engine dequantizes as q*scale and never assumes + the full int8 range — mirroring olmoe.c quantize_rows(): + qmax = 2**(bits-1) - 1 (8 -> 127, 4 -> 7, 2 -> 1) + scale = amax(|w|, row) / qmax + q = clamp(round(w / scale), -qmax-1, qmax) + """ + qmax = (1 << (bits - 1)) - 1 w_f32 = w.float() row_max = w_f32.abs().amax(dim=1, keepdim=True).clamp(min=1e-12) - scales = row_max / 127.0 - q = (w_f32 / scales).round().clamp(-128, 127).to(torch.int8) + scales = row_max / qmax + q = (w_f32 / scales).round().clamp(-qmax - 1, qmax).to(torch.int8) return q, scales.squeeze(1) @@ -50,9 +62,12 @@ def main(): src.add_argument("--model", help="Local HF checkpoint directory") ap.add_argument("--out", required=True, help="Output directory for int4 model") ap.add_argument("--ebits", type=int, default=4, - help="Expert quant bits (4 or 8, default 4)") + help="Expert quant bits (2..8, default 4)") args = ap.parse_args() + if not 2 <= args.ebits <= 8: # storage is int8_t; engine rejects the same range (olmoe.c) + sys.exit(f"--ebits must be 2..8 (got {args.ebits})") + if args.repo: from huggingface_hub import snapshot_download from huggingface_hub.errors import LocalEntryNotFoundError @@ -96,7 +111,7 @@ def main(): for name, tensor in tensors.items(): if is_expert_weight(name): expert_count += 1 - q, scales = quantize_row(tensor) + q, scales = quantize_row(tensor, args.ebits) total_expert_f32 += tensor.numel() * tensor.element_size() total_expert_q += q.numel() * 1 + scales.numel() * 4 out_tensors[name] = q @@ -109,7 +124,7 @@ def main(): ratio = total_expert_q / max(total_expert_f32, 1) * 100 print(f"ok") - print(f"\nDone. {expert_count} expert tensors quantized.") + print(f"\nDone. {expert_count} expert tensors quantized to int{args.ebits}.") print(f"Expert storage: {total_expert_f32/1e9:.1f} GB -> {total_expert_q/1e9:.1f} GB ({ratio:.0f}%)") print(f"Model ready at: {out}") print(f"\nRun: SNAP={out} ./olmoe.exe 32 4 16") diff --git a/c/tools/make_glm_oracle.py b/c/tools/make_glm_oracle.py index b623faf..5e16f51 100644 --- a/c/tools/make_glm_oracle.py +++ b/c/tools/make_glm_oracle.py @@ -17,6 +17,45 @@ EN: computed AFTER the FP8 round-trip, so the reference matches exactly what the EN: ingests. Default: bf16 (original oracle unchanged).""" import json, sys, argparse from pathlib import Path + +# --- Version gate (must run BEFORE the heavy `from transformers import ...`, +# which triggers transformers' lazy-loading and can in turn reset the in-memory +# __version__ attribute; importlib.metadata reads the installed package version +# directly and is immune to that). --- +# +# GLM-5.2's MLA attention uses interleaved (DeepSeek-style) RoPE, which is what +# the C engine implements. transformers < 5.11.0 applied split-half (Llama-style) +# RoPE in GlmMoeDsa* instead; an oracle built on those versions drifts and the +# engine then scores 25/32 instead of the documented 32/32 (issue #281). Weights +# come out identical across versions — only the forward pass differs — so there +# is no safe "partial" run: a too-old transformers silently produces an invalid +# ref_glm.json. Hard-fail rather than warn. EN: same. +import transformers +from importlib.metadata import version as _pkg_version, PackageNotFoundError + +_MIN_TRANSFORMERS = (5, 11) +def _tf_version_tuple(): + try: + v = _pkg_version("transformers") # authoritative: installed dist metadata + except PackageNotFoundError: + v = getattr(transformers, "__version__", "0") # fallback (editable/src installs) + out = [] + for part in v.split(".")[:2]: # major.minor only + out.append(int("".join(c for c in part if c.isdigit()) or "0")) + while len(out) < 2: + out.append(0) + return tuple(out[:2]) + +_tf_ver = _tf_version_tuple() +if _tf_ver < _MIN_TRANSFORMERS: + sys.exit( + f"\nERROR: make_glm_oracle.py requires transformers >= " + f"{'.'.join(map(str, _MIN_TRANSFORMERS))}.0 (found {_tf_ver[0]}.{_tf_ver[1]}). " + f"GLM-5.2 MLA uses interleaved RoPE; older versions apply split-half RoPE " + f"and silently produce an oracle the engine scores 25/32 against (issue #281). " + f"Upgrade: pip install -U 'transformers>=5.11'\n" + ) + import torch from transformers import GlmMoeDsaConfig, GlmMoeDsaForCausalLM @@ -115,6 +154,7 @@ print("tf_pred:", tf_pred) sd = model.state_dict() unfuse_experts(sd) +Path("glm_tiny").mkdir(parents=True, exist_ok=True) # safetensors/json won't create the dir themselves if args.fp8: n_fp8, n_tot = save_fp8_safetensors(sd, "glm_tiny/model.safetensors") print(f"\nsaved FP8: {n_fp8} e4m3 tensors (+{n_tot - n_fp8} scale_inv sidecars / f32) " diff --git a/c/tools/quant_ablation.py b/c/tools/quant_ablation.py index 689643a..dd08715 100644 --- a/c/tools/quant_ablation.py +++ b/c/tools/quant_ablation.py @@ -118,7 +118,7 @@ def quantize_param(w, bits, group, rot=False, e8=""): def _grid_or_e8(x, bits, group, e8): if e8: - return _quant_e8(x.float(), group, ball=(e8 == "-e8")) + return _quant_e8(x.float(), group, bits, ball=(e8 == "-e8")) return _quant_last_dim(x, bits, group) @@ -169,8 +169,15 @@ def _e8_ball(y, r2=10.0): return p -def _quant_e8(x, group, ball): - """Blocks of 8 along the input dim; per-group scale by MSE search over RMS multiples.""" +_E8_R2_REPORTED = set() +def _e8_radius(bits): + # E8 lattice: points within |p|^2<=r2 grow ~r2^4, so +1 bit (x256 codebook) needs r2 x4. + # Anchor: r2=10 is the ~2^16 E8P ball (2 bits over 8 dims). Scale from there. + return 10.0 * (4.0 ** (bits - 2)) + +def _quant_e8(x, group, bits, ball): + """Blocks of 8 along the input dim; per-group scale by MSE search over RMS multiples. + ball=True clamps to the rate-scaled E8 ball for `bits`; ball=False is the unbounded ideal.""" if x.shape[-1] % 8: raise SystemExit(f"-e8 needs input dim divisible by 8 (got {x.shape[-1]})") g = group or x.shape[-1] @@ -183,7 +190,11 @@ def _quant_e8(x, group, ball): for k in (0.5, 0.7, 0.9, 1.1, 1.4, 1.8, 2.4): s = rms * k yb = (xg / s).reshape(-1, g // 8, 8) - p = _e8_ball(yb) if ball else _e8_nearest(yb) + p = _e8_ball(yb, _e8_radius(bits)) if ball else _e8_nearest(yb) + if ball and bits not in _E8_R2_REPORTED: + _E8_R2_REPORTED.add(bits) + import sys as _sys + _sys.stderr.write(f"[e8] bits={bits}: ball r2={_e8_radius(bits):.1f}\n") out = (p.reshape(-1, g) * s) err = (out - xg).pow(2).sum(-1, keepdim=True) if best_err is None: diff --git a/c/tools/repair_mtp_int8.py b/c/tools/repair_mtp_int8.py new file mode 100644 index 0000000..695e99f --- /dev/null +++ b/c/tools/repair_mtp_int8.py @@ -0,0 +1,229 @@ +"""Repair an existing colibri int4 container whose MTP head was quantized at int4. + +WHY THIS EXISTS + `model.layers..eh_proj.weight` [D, 2D] multiplies the MTP concat + [embedding_norm ; hidden_norm], and its two column halves differ in scale by + ~20-30x per row (embedding-half absmax ~0.05, hidden-half ~1.5 on GLM-5.2). + Per-row int4 uses ONE scale (= absmax/7) per row, so every embedding-half + weight lands below half a quantization step and np.rint rounds the ENTIRE + embedding half to exact zeros (packed bytes 0x88). The MTP head then drafts + garbage: acceptance ~0% (issue #8 measured 0-4% at int4; 39-59% at int8 — + which is why `convert_fp8_to_int4.py --mtp` defaults to --ebits 8). + + A container converted (or downloaded) with an int4 MTP head does not need a + full re-conversion: this script re-downloads ONLY the affected dense tensors + (~355 MB of HTTP range reads against the FP8 source repo), requantizes them + at int8 with the converter's exact math, and patches the local shards in + place. Originals are kept beside as *.bak-int4. + +WHAT IT TOUCHES + The MTP layer's dense tensors only (eh_proj, q_a/q_b/kv_a/kv_b/o_proj, + shared_experts.*): the ones that stream into RAM once and stay resident. + Routed experts (model.layers..mlp.experts.*) are NOT touched — they are + statistically like the main layers' experts and int4 is acceptable there. + The engine auto-detects int8 vs int4 by blob size (qt_from_disk), so no + engine or config change is needed. Cost: ~+133 MB on disk / resident RAM. + +USAGE + python3 tools/repair_mtp_int8.py --snap /path/to/glm52_i4 # repair + python3 tools/repair_mtp_int8.py --snap /path/to/glm52_i4 --dry-run # inspect only + + --source-repo defaults to zai-org/GLM-5.2-FP8 (the checkpoint the public + int4 containers were converted from). Requires numpy and network access; + no torch, no HF token (public repo, anonymous range reads). +""" +import argparse, glob, json, os, ssl, struct, sys, urllib.request +import numpy as np + +# macOS python.org builds ship no CA bundle: use certifi when available (Linux +# system Pythons generally have working system certs and skip this). +try: + import certifi + _SSL_CTX = ssl.create_default_context(cafile=certifi.where()) +except ImportError: + _SSL_CTX = ssl.create_default_context() + + +# ---------- HTTP range reads against the source repo ---------- +def http_range(url, start, length, tries=5): + req = urllib.request.Request(url, headers={"User-Agent": "colibri-mtp-repair", + "Range": f"bytes={start}-{start+length-1}"}) + for attempt in range(tries): + try: + with urllib.request.urlopen(req, timeout=30, context=_SSL_CTX) as r: + data = r.read() + if len(data) == length: + return data + except KeyboardInterrupt: + raise + except Exception as ex: + if attempt == tries - 1: + raise RuntimeError(f"range read failed for {url}: {ex}") + raise RuntimeError(f"short range read for {url}") + + +class SourceRepo: + def __init__(self, repo, revision="main"): + self.base = f"https://huggingface.co/{repo}/resolve/{revision}/" + with urllib.request.urlopen(self.base + "model.safetensors.index.json", timeout=30, context=_SSL_CTX) as r: + self.wmap = json.loads(r.read())["weight_map"] + self._hdr = {} + + def _shard_header(self, shard): + if shard not in self._hdr: + n = struct.unpack("> 3) & 0xF + mant = (b & 7).astype(np.float64) + v = (sign * np.where(e > 0, (1 + mant / 8) * np.exp2(e.astype(np.float64) - 7), + mant / 8 * np.exp2(-6.0))).reshape(m["shape"]) + sn = name + "_scale_inv" + sshard = self.wmap[sn] + shdr, sbase = self._shard_header(sshard) + sm = shdr[sn] + so0, so1 = sm["data_offsets"] + sc = np.frombuffer(http_range(self.base + sshard, sbase + so0, so1 - so0), + dtype=np.float32).reshape(sm["shape"]) + O, I = m["shape"] + scf = np.repeat(np.repeat(sc, 128, axis=0)[:O], 128, axis=1)[:, :I] + return (v * scf).astype(np.float32) + raise ValueError(f"{name}: unsupported source dtype {m['dtype']}") + + +# ---------- quantization: identical to convert_fp8_to_int4.quant_int8 ---------- +def quant_int8(w): + amax = np.abs(w).max(axis=1, keepdims=True) + s = np.maximum(amax / 127, 1e-8) + q = np.clip(np.rint(w / s), -128, 127).astype(np.int8) + return q.reshape(-1).view(np.uint8).copy(), s[:, 0].astype(np.float32) + + +# ---------- local safetensors IO (no deps; preserves byte-identity of untouched tensors) ---------- +def read_shard(path): + with open(path, "rb") as fh: + n = struct.unpack(" [tensor names to repair] + already_ok, skipped_experts = [], 0 + for f in sorted(glob.glob(os.path.join(a.snap, "*.safetensors"))): + with open(f, "rb") as fh: + n = struct.unpack("=` (testing). | +| `SPEC_PIN` | `1` (on) | Speculation gate mode. `0` reverts to the legacy S-dependent speculation gates (#163). | +| `COLI_RAM_OVERCOMMIT` | off | `=1` overrides the "projected peak > MemAvailable → exit(2)" guard so a run that risks kernel OOM-kill is allowed to proceed. | --- @@ -76,7 +88,22 @@ Format: `VAR` — default — effect. | `CUDA_EXPERT_GB` | `0` | VRAM budget (GB) for caching experts on the GPU. | | `CUDA_RELEASE_HOST` | auto (`1` if >1 device) | Release host-side copies after upload. | | `COLI_CUDA_ATTN` | off | Run S≤4 attention on the GPU. | +| `COLI_CUDA_ATTN_SHARD` | off | `=1` splits KV-b heads across devices during attention load (multi-GPU). | | `COLI_CUDA_PROFILE` | off | Emit CUDA timing. | +| `COLI_CUDA_PIPE` | `0` (off) | `1` engages the multi-step attention pipeline; `2` enables the pipe2 path. | +| `COLI_CUDA_PIPE_SHARD` | off | `=1` runs the multi-device P2P head-shard attention path (opt-in for NVLink topologies; serializes ~95 MB/layer over a star PCIe topology). | +| `COLI_CUDA_PIPE_S_MIN` | `1` single-GPU, `8` multi-GPU | Minimum prefill batch S to engage the pipe2 CUDA path. | +| `COLI_CUDA_MTP` | `0` (off) | `=1` opts into MTP speculation under CUDA (off by default: cold streaming experts run on CPU where the fused-pair/IDOT kernels diverge in FP order, collapsing draft acceptance, #163/#292). | +| `COLI_CUDA_ASYNC` | on | `=0` forces synchronous `cudaMemcpy` instead of async + pinned host staging. | +| `COLI_CUDA_DUAL_PROJ` | on | `=0` issues gate+up as two separate launches instead of one fused `grouped_hidden_w4_dual`. | +| `COLI_CUDA_W4_PACKED` | on | `=0` disables the grouped packed-int4 path. | +| `COLI_CUDA_TC_INT4` | off | `=1` uses the W4A4 WMMA Tensor Core path (when all expert tensors are int4 and dims divide). | +| `COLI_CUDA_TC_MIN_ROWS` | `8` | Min rows-per-expert to engage the W4A4 Tensor Core path. | +| `COLI_CUDA_TC_W4A16` | off | `=1` uses the lossless W4A16 Tensor Core path (compute capability ≥7). | +| `COLI_CUDA_TC_W4A16_MIN` | `16` | Per-expert row threshold above which W4A16 TC tiles dispatch (smaller batches fall back to the naive kernel). | +| `COLI_CUDA_SHARED_W4A16` | off | `=1` uploads shared-expert weights and runs the shared-MLP W4A16 Tensor Core kernel. | +| `COLI_CUDA_SHARED_W4A16_MIN_ROWS` | `32` | Min row count to engage the shared-MLP W4A16 kernel. | +| `COLI_METAL_UNTRACKED` | off (Metal only) | `=1` sets `MTLResourceHazardTrackingModeUntracked` on Metal buffers (reduces hazard-tracking overhead). | --- @@ -88,8 +115,11 @@ These are for testing, benchmarking, or internal use — not part of the everyda |---|---|---| | `SPEC` | `1` | Speculative decoding on/off. | | `DRAFT` | `-1` (auto: 3 with MTP, else 0) | Number of speculative draft tokens per step. | -| `GRAMMAR` | unset | Path to a GBNF grammar file to constrain generation. | +| `GRAMMAR` | unset | Path to a GBNF grammar file to constrain generation. Takes precedence over `SCHEMA`. | +| `SCHEMA` | unset | Path to a JSON-Schema file compiled to GBNF to constrain generation (consulted only when `GRAMMAR` is empty). | | `GRAMMAR_DRAFT` | unset | Max grammar-forced draft span length. | +| `EXPERT_BUDGET` | `0` (off) | Cap experts loaded per layer (MoE-Spec). **Quarantined:** silently forced to `0` unless `EXPERT_BUDGET_EXPERIMENTAL` is set — every tested value is either no faster or incoherent (issue #303). | +| `EXPERT_BUDGET_EXPERIMENTAL` | unset | Setting it (any value) allows `EXPERT_BUDGET>0` to actually take effect (expect garbage, #294). | | `DSA` | on | Dynamic Sparse Attention indexer. `DSA=0` disables. | | `DSA_FORCE` | `0` | Force the DSA path on. | | `DSA_TOPK` | model value | Override the DSA index top-k (testing). | @@ -100,11 +130,15 @@ These are for testing, benchmarking, or internal use — not part of the everyda | `PIN_FILL` | `0` | Fill the pinned store even without usage data. | | `MTP_DEBUG` / `MTP_PRENORM` / `MTP_SWAP` | off | MTP head debugging / ablations. | | `STATS` | unset | Write an expert-usage histogram to `STATS=` at end of run. | +| `TOKENS` | unset | If set, dumps generated token ids to stderr for A/B comparison. | | `SCORE` | unset | Scoring/eval mode over `SCORE=`. | +| `SCORE_PREFIX` | on | If unset or `≠0`, prepends `[gMASK]` to scoring contexts (GLM-family only). | +| `REPIN_VERBOSE` | off | If set, prints per-swap `[REPIN]` diagnostics during VRAM repin. | | `REF` / `REF_FORCE` | `ref_glm.json` | Reference-output comparison mode. | | `REPLAY` | unset | Replay mode. | | `TF` | unset | Teacher-forcing mode. | | `CHAT_TEMPLATE` | `1` | Apply the GLM chat template (`0` = raw prompt). | +| `PPL` | off (`olmoe.c` only) | `PPL=1` enters teacher-forced NLL/perplexity meter mode in the OLMoE sister engine. | --- @@ -135,7 +169,7 @@ These are read by the Python programs (not the `glm` engine), so they don't appe - `SNAP` — model snapshot directory (required by `glm`; set from `--model`). - `SERVE`, `SERVE_BATCH` — select serve / batched-serve mode. -- `PROMPT` — one-shot text mode. +- `PROMPT` — one-shot text mode (the engine also honors `COLI_PROMPT`, preferred cross-platform; `PROMPT` is ignored on Windows if it contains cmd.exe `$`-metacharacters). - `COLI_OMP_TUNED` — internal sentinel guarding the OMP re-exec (see `COLI_NO_OMP_TUNE`); not user-facing. --- diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..62010bd --- /dev/null +++ b/docs/api.md @@ -0,0 +1,186 @@ +# OpenAI-compatible API, KV contexts & web UI + +## `coli serve` + +`coli serve` keeps one model process loaded and exposes a text-only +OpenAI-compatible HTTP API. The gateway uses only the Python standard library; +inference still runs in the same dependency-free C engine. + +```bash +cd c +COLI_MODEL=/nvme/glm52_i4 COLI_API_KEY=local-secret ./coli serve \ + --host 127.0.0.1 --port 8000 --model-id glm-5.2-colibri + +curl http://127.0.0.1:8000/v1/chat/completions \ + -H 'Authorization: Bearer local-secret' \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "glm-5.2-colibri", + "messages": [{"role": "user", "content": "Hello"}], + "stream": true + }' +``` + +Implemented endpoints are `GET /v1/models`, `GET /v1/models/{model}`, +`POST /v1/chat/completions`, and legacy `POST /v1/completions`. Chat and +completion requests support JSON responses, SSE streaming, usage counts, +`max_tokens`/`max_completion_tokens`, `temperature`, and `top_p`. The extension +`enable_thinking: true` enables GLM-5.2's reasoning block; the standard +`reasoning_effort` field also enables it unless set to `none`. + +The server is deliberately text-only and serves one generation at a time: the +744B model stays in one persistent process, so concurrent HTTP requests queue +instead of loading duplicate model copies. Tools, image/audio input, custom +stop sequences, log probabilities, and token penalties return an explicit error +rather than being silently ignored. The default bind address is localhost; set +`COLI_API_KEY` before exposing the server beyond the machine. + +Browser access from the Vite development server and Tauri local origins is +enabled by default. Repeat `--cors-origin https://your-ui.example` to allow +another exact origin, or use `--cors-origin '*'` only on a trusted local +network. + +The engine owns its KV contexts, so HTTP generation uses a bounded FIFO +admission queue instead of pretending to run unsafe parallel sequences. +Configure it with `--max-queue N` (default 8) and `--queue-timeout SECONDS` +(default 300), or the `COLI_MAX_QUEUE` / `COLI_QUEUE_TIMEOUT` environment +variables. Saturated and timed-out requests receive OpenAI-shaped HTTP 429 +errors before streaming headers are sent. `GET /health` exposes +active/queued/completed/rejected counters, and successful generation responses +include `x-colibri-queue-wait-ms`. + +## Connect a coding CLI or editor + +The API is OpenAI-compatible, so most coding CLIs and editor extensions work by +pointing them at Colibri as an *OpenAI-compatible* provider. Three settings: + +- **Base URL** — `http://localhost:8000/v1` +- **Model** — `glm-5.2-colibri` (or whatever you pass to `--model-id`) +- **API key** — any non-empty string, e.g. `local` + +Colibri needs **no** API key by default, but many clients refuse to start without +one — give them any dummy value. The key is only enforced if you set `COLI_API_KEY`. + +Smoke-test the endpoint first (no key needed unless you set one): + +```bash +curl http://127.0.0.1:8000/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"glm-5.2-colibri","messages":[{"role":"user","content":"hi"}]}' +``` + +**aider** + +```bash +export OPENAI_API_BASE=http://localhost:8000/v1 +export OPENAI_API_KEY=local +aider --model openai/glm-5.2-colibri # the openai/ prefix routes to OPENAI_API_BASE +``` + +**crush** — add a provider to `crush.json` (`~/.config/crush/crush.json`, or +`%USERPROFILE%\AppData\Local\crush\crush.json` on Windows): + +```json +{ + "$schema": "https://charm.land/crush.json", + "providers": { + "colibri": { + "name": "Colibri", + "type": "openai-compat", + "base_url": "http://localhost:8000/v1/", + "api_key": "local", + "models": [ + { "name": "GLM-5.2 (Colibri)", "id": "glm-5.2-colibri", + "context_window": 131072, "default_max_tokens": 1024 } + ] + } + } +} +``` + +The `"api_key": "local"` dummy is what satisfies clients that demand a key. +`context_window` is only the client's budget display — set it to whatever your +KV configuration actually allows. + +**Continue, Cline / Roo, `llm`, the OpenAI SDKs, …** — set the provider's base +URL to `http://localhost:8000/v1`, the model to `glm-5.2-colibri`, and any dummy +key (`OPENAI_API_KEY` / `OPENAI_BASE_URL` for env-based tools). + +> **Set your expectations before connecting an agentic CLI.** Two costs dominate, +> and the first one is invisible until you know it's there: +> +> 1. **Prefill.** Coding agents (crush, aider in repo-map mode, Cline, …) send a +> large system prompt plus tool definitions — often 10–20k tokens — *before +> your first word*. Prefill on the CPU-streaming path runs at a few tokens per +> second (it is attention-bound, see #153), so a 15k-token agent preamble is +> **an hour of silent "thinking" before the first output token**. The client +> looks hung; it isn't. Smoke-test with the tiny `curl` above first — if that +> answers in about a minute, the pipeline works and what you're paying for is +> prompt size. +> 2. **Decode.** Roughly 1 tok/s for a large model, so multi-turn agent loops +> (which re-pay the growing context every turn) compound the cost. +> +> Practical guidance: single surgical asks with a short context work; iterative +> agent sessions against a disk-streaming 744B model do not resemble a hosted +> API and mostly won't be worth the wait. If your client lets you trim or disable +> its system preamble and tool catalog, do it. + +## Isolated KV contexts + +`coli serve --kv-slots N` allocates up to 16 independent sequence contexts. +Requests select one with the optional integer `cache_slot` field; ordinary +OpenAI clients omit it and keep the original slot 0 behavior. + +```json +{ + "model": "glm-5.2-colibri", + "messages": [{"role": "user", "content": "Continue this conversation"}], + "cache_slot": 1 +} +``` + +Each slot owns its token history, compressed MLA/DSA KV memory, MTP window, and +crash-safe persistence file (`.coli_kv`, `.coli_kv.1`, ...). The engine matches +each request's tokenized prompt against the slot's history and reuses the common +KV prefix, so stateless HTTP turns keep their cache across requests and even +across engine restarts. Use `COLI_KV_SLOTS=N` as the environment equivalent. +Start small: at the default 4096-token context, every slot costs hundreds of MB. + +## Web dashboard + +One command serves the OpenAI-compatible API **and** the web console on the +same port, then opens your browser when the engine is ready: + +```bash +cd web && npm install && npm run build # once +./coli web --model +``` + +What you get: + +- **Chat** with live metrics: a flashing token counter while generating, then + tok/s, time-to-first-token, prompt→completion counts and queue wait; +- **Runtime panel**: your hardware (CPU, GPUs + VRAM, RAM, cores), the + scheduler, and the live expert-tier bar — how many of the 19,456 experts sit + in VRAM / RAM / disk right now; +- **Brain**: the whole model as a 76×256 cortex, one cell per expert. Colour = + tier, brightness = routing heat, and the experts routed in each turn flash + white and decay — you watch the model think. Hover any cell for its tier, + heat and [measured topic affinity](https://github.com/JustVugg/colibri/issues/175); +- **Atlas**: the measured expert atlas as a 3-D galaxy (publish `experts.json` + from `tools/expert_atlas/analyze.py --web`). + +The dashboard talks to the engine over a small line protocol and plain JSON +endpoints — nothing heavier than the engine itself. `web/` is a pure OpenAI-API +client (React + TypeScript) and also works against any other compatible +endpoint; the terminal `coli chat` remains the first-class interface. + +The layout is responsive down to phone widths, and the sidebar carries the full +telemetry stack — hardware, scheduler, tier bar, per-turn time breakdown, tok/s +trend and per-GPU expert counts: + +

+ the dashboard on a phone-sized viewport +    + the telemetry sidebar +

diff --git a/docs/benchmarks.md b/docs/benchmarks.md new file mode 100644 index 0000000..09d3752 --- /dev/null +++ b/docs/benchmarks.md @@ -0,0 +1,144 @@ +# Benchmarks & measured numbers + +Everything on this page is a measurement, not a promise. If you run colibrì on +hardware not listed here, **please open an issue with your numbers** — real +datapoints are what move this project. + +## Reference numbers (the original dev box: WSL2, 12 cores, 25 GB RAM, NVMe via VHDX) + +Detailed GPU experiment: [GLM-5.2 on 6× RTX 5090](experiments/glm52-6x5090-2026-07-12.md) — +full expert residency across VRAM+RAM reaches **6.84 tok/s** single-request decode. + +| metric | value | +|---|---| +| model on disk (int4 container) | ~370 GB | +| resident RAM (dense, int4) | 9.9 GB | +| load time | ~30 s | +| peak RSS during chat | ~20 GB (auto-capped) | +| cold decode cost | ~11 GB disk reads/token (75 layers × 8 experts) | +| disk ceiling (this dev box's drive) | ~1 GB/s → ~0.05–0.1 tok/s cold | +| MTP speculation (int8 head) | 2.2–2.8 tok/forward measured ([#8](https://github.com/JustVugg/colibri/issues/8)) | + +This is not fast. It is a 744B frontier-class model **answering correctly on a +machine that costs less than one H100 fan**. Warm cache, pinned hot experts and +MTP push the useful-response latency down considerably; the physics of the disk +does the rest. + +### SSD note + +Cold starts are heavy on random reads (~11 GB/token), but reads don't +meaningfully wear an SSD — colibrì's streaming is read-only. The real concerns +under heavy use are (1) **swap traffic** if the system runs out of RAM (writes +do wear the drive — keep a sane `--ram` budget; colibrì's auto-budget is designed +to stay clear of swap) and (2) **sustained thermals**: hours at full read duty +cycle will heat cheaper drives. Monitor drive temperature and health. + +## Test your machine, in order + +```bash +cd c && ./setup.sh # build + architecture self-test (expects 32/32) + +# 1) measure YOUR disk the way the engine uses it (parallel 19 MB random reads): +gcc -O2 -fopenmp iobench.c -o iobench +./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 0 # buffered, 8 threads +./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 1 # O_DIRECT (bypass cache) +# Caveat (#86): iobench reads a bounded ~1 GB shard, so buffered reads on a big-RAM box +# report the PAGE CACHE, not the disk. Use the O_DIRECT run (arg 1) for a true number, and +# run it on a shard you haven't touched this session (a prior buffered run caches its pages). +# On macOS there is no O_DIRECT — iobench uses F_NOCACHE, which stops *new* caching but can't +# evict pages a prior buffered run already resident-mapped, so a macOS "O_DIRECT" figure right +# after a buffered run still reads cache. Reboot or use a fresh shard for a real cold read. + +# 2) chat; watch the per-turn stats line (tok/s, expert hit-rate, RSS): +COLI_MODEL=/path/to/glm52_i4 ./coli chat + +# 3) record expert usage, then pin the hottest experts in your spare RAM: +STATS=stats.txt ./coli chat +PIN=stats.txt PIN_GB=20 ./coli chat # scale PIN_GB to your free RAM + +# 4) quality benchmarks (MMLU/HellaSwag/ARC): +./coli bench +``` + +## Back-of-envelope predictions + +Decode is disk-bound: a cold token costs ~11.4 GB of expert reads; MTP +speculation roughly halves the effective cost *once the cache is warm*; RAM +turns cold reads into free cache hits. + +| machine | expected | +|---|---| +| the dev box (WSL2 VHDX, ~1 GB/s, 25 GB RAM) | ~0.05–0.1 tok/s cold — proven baseline | +| native Linux, PCIe4 NVMe (~3–5 GB/s random), 32 GB | ~0.5–1 tok/s | +| PCIe5 NVMe or 2×NVMe RAID0 (~8–12 GB/s), 64 GB (PIN ~40 GB of hot experts) | ~2–4 tok/s | +| 128–256 GB RAM, 12 cores (hot experts cached) | ~2–4 tok/s — matmul-bound: ~80 GFLOP/token vs ~250 GFLOP/s of our AVX2 kernels | +| same RAM + 24–32 cores, or AVX-512/VNNI kernels | ~5–15 tok/s — interactive; kernel work is the multiplier | + +These are estimates, not measurements. + +## Community benchmarks (measured) + +Real numbers from real machines, stock build (`setup.sh`, gcc 13), greedy decoding, `--ngen 32`, MTP active: + +| machine | disk (iobench, 19 MB × 64, 8 threads) | config | measured | +|---|---|---|---| +| Intel Core Ultra 7 270K Plus (24 threads) · WSL2 · 24 GB RAM · NVMe VHDX ([#2](https://github.com/JustVugg/colibri/issues/2)) | 1.96 GB/s buffered · 2.74 GB/s O_DIRECT | default | 0.07 tok/s · expert hit 3–4% · RSS 14.1 GB | +| 〃 | 〃 | `--topp 0.7` | **0.11 tok/s** · expert hit 11% · RSS 14.7 GB | +| Apple M5 Max (18 cores) · macOS · 128 GB unified · internal SSD ([#4](https://github.com/JustVugg/colibri/issues/4), [#5](https://github.com/JustVugg/colibri/issues/5)) | ~4 GB/s cold (the 14.2 GB/s reading was cache-influenced — see note) | default, MTP off | **1.06 tok/s** · expert hit 23% · RSS 21.8 GB | +| Apple M5 Max · macOS · 128 GB unified · 2 TB SSD · **Metal backend** ([#72](https://github.com/JustVugg/colibri/pull/72), [#87](https://github.com/JustVugg/colibri/issues/87)) | (macOS O_DIRECT figure unreliable — see note) | Metal on · `--ram 96` · 39.7 GB warm pin · MTP off | **1.83 tok/s** · expert hit 66% · warmed 1.11 → 1.83 over the run | +| 〃 · 46.9 GB pin (2.94M-selection history) · `--ram 110`, 1024-token run ([#103](https://github.com/JustVugg/colibri/issues/103)) | 〃 | Metal on (experts + attention) · MTP off | **2.06 tok/s** · hit 72.5% · coherent output | +| Mac Mini M4 Pro · macOS · **48 GB** unified · **Metal backend** ([#107](https://github.com/JustVugg/colibri/issues/107)) | 6.59 GB/s F_NOCACHE (fresh shard) | Metal on · `--ram 38` | **0.30 tok/s** (vs 0.18 CPU-only) | +| Epyc 9654 ES · Linux · 4x16GB DDR5-4800-rdimm · Samsung PCIe Gen3 x4 NVME SSD | — | `MTP=1 DIRECT=1` | 0.31 tok/s · expert hit 35% · RSS 21.52 GB | +| Ryzen AI 9 HX 370 (Framework 13) · Arch Linux · 128 GB · WD SN850X, BTRFS zstd ([#12](https://github.com/JustVugg/colibri/issues/12)) | — | int8 MTP head · `--cap 32` · 46.7 GB auto-learned PIN | **0.37 tok/s** · expert hit 66% · MTP acceptance 52% (2.59 tok/fw) · RSS 105 GB | +| Ryzen 9 9950X (32 threads) · Linux · 123 GB · Crucial P3 QLC Gen3 ([#31](https://github.com/JustVugg/colibri/issues/31)) | 1.51 GB/s buffered | default, 2 runs from cold | 0.10 tok/s · hit 53% · profile 66% disk | +| 〃 same machine, model moved to a Samsung 9100 PRO PCIe 5.0 ([#31](https://github.com/JustVugg/colibri/issues/31)) | **8.81 GB/s** O_DIRECT | 〃 (usage history retained) | **0.28 tok/s** · hit 57% · profile flips: 32% disk / **57% matmul** | +| Ryzen AI Max+ 395 (Framework Desktop) · Ubuntu · 128 GB LPDDR5x · Intel Optane 905p PCIe 3.0 ([#39](https://github.com/JustVugg/colibri/issues/39)) | 3.27 GB/s buffered | int8 MTP head · fresh history (pure LRU, auto-raised cap 65) | 0.16 tok/s · hit 57% · profile 49% disk / 47% matmul | +| 〃 five runs later — learned pin 47.6 GB ([#39](https://github.com/JustVugg/colibri/issues/39)) | 〃 | `--temp 0.7 --topp 0.7` | **0.40 tok/s** · hit 71% | +| Ryzen 7 9800X3D (16T) · WSL2 · 70 GB RAM · Samsung 9100 PRO PCIe 5.0 · RTX 5090 ([#101](https://github.com/JustVugg/colibri/issues/101)) | **10.51 GB/s** O_DIRECT | MTP off · learned pin 24 GB · hit 54% · OMP hot-team on | **0.41 tok/s** · disk-bound (36.5 s disk vs 24.0 s matmul) · **CUDA expert tier ≈ 0%** (AVX-512 CPU matches the 5090) · `--topp 0.7` → **0.52 tok/s** | +| EPYC 7443 (24C/48T, Zen3 AVX2) · Linux · **430 GB RAM** · NVMe RAID-Z1 via TrueNAS VM ([#104](https://github.com/JustVugg/colibri/issues/104)) | ~1 GB/s (VM overhead) | 77.5 GB pin · cap auto-raised to 194/layer · MTP off | **1.00 tok/s** · **hit 98%** · disk eliminated → **RAM-bandwidth + matmul bound** | +| Intel i5-12600K (10C/16T, AVX2) · **native Windows 11, no WSL** · 32 GB · MinGW GCC 16.1 ([#113](https://github.com/JustVugg/colibri/issues/113)) | buffered (no O_DIRECT on MinGW) | int8 MTP head · cold, small-RAM (cap ~2/layer) | **0.08 tok/s** · hit 3.7% · **MTP 57% acceptance** — first native-Windows datapoint | +| Ryzen 9 9950X3D2 (16C/32T, avx512-vnni) · native Linux · 121 GB · Samsung 9100 PRO **PCIe Gen5** · RTX 5090 (28 GB expert tier, 1475 pinned) ([#120](https://github.com/JustVugg/colibri/issues/120)) | **11.48 GB/s** O_DIRECT | `MTP=0 DIRECT=1 PIPE_WORKERS=16 PREFETCH=1` | **1.23 tok/s** | +| Ryzen AI Max+ 395 (Strix Halo, 16C/32T Zen5, avx512-vnni) · Arch Linux · 128 GB unified LPDDR5x · SK hynix P41 PCIe 4.0 ([#124](https://github.com/JustVugg/colibri/issues/124)) | — | `DIRECT=1 PIPE=1 --topp 0.7` · auto-pin | 0.06 cold → **1.10 tok/s** sustained · later **1.83 tok/s** on current dev with `DIRECT=1 PIPE=1 PILOT_REAL=1 PILOT_TWO=1` ([#200](https://github.com/JustVugg/colibri/issues/200)) | +| Intel Core Ultra 9 185H (16C/22T, avx-vnni) · **native Windows 11, no WSL** · 32 GB · Crucial P3 QLC NTFS · RTX 5070 Ti ([#128](https://github.com/JustVugg/colibri/issues/128), [#273](https://github.com/JustVugg/colibri/issues/273)) | — | int8 MTP head · warm cache · GPU-resident pipeline at decode | 0.03 cold → 0.5 warm CPU → **1.07 tok/s** with the pipe2 decode gate (#274) | +| Dell Pro Max GB10 (DGX Spark: Grace, **aarch64 i8mm/sve2**) · Linux · 121 GB unified LPDDR5x · GB10 sm_121 ([#136](https://github.com/JustVugg/colibri/issues/136), [#161](https://github.com/JustVugg/colibri/issues/161)) | **5.58 GB/s** O_DIRECT | int8 MTP head · warm cache | 0.50 tok/s warm · **2.4 tok/s full-k8**, **3.33 tok/s** with `CACHE_ROUTE` (#199) | +| **6 × RTX 5090 · dual Xeon Silver 4510 · 251 GB** (author's rig, [experiment log](experiments/glm52-6x5090-2026-07-12.md)) | NVMe | `CUDA_EXPERT_GB=auto PIN_GB=all` full residency · `COLI_CUDA_PIPE=2 TC_W4A16` · DRAFT=0 | **5.8–6.8 tok/s** decode · TTFT ~13 s · hit 89–100% | + +### Takeaways + +With 24 GB of RAM the engine auto-caps the expert cache to 2 slots/layer, so +decode stays cold even on a fast disk — **on small-RAM machines the RAM cap, not +the disk, is the binding constraint**; `--topp 0.7` alone bought a clean 1.6× +end-to-end speedup. The 9950X pair is the cleanest bottleneck experiment: same +machine, same history, only the disk swapped — ×5.8 disk bandwidth bought ×2.9 +tokens, and the profile **flipped from 66% disk to 57% matmul**. But the +crossover depends on the CPU kernel: with OMP hot-team tuning on, an AVX-512 CPU +can match an RTX 5090 on expert matmul ([#101](https://github.com/JustVugg/colibri/issues/101)), +so **the GPU tier earns its VRAM only when the CPU is the weak link**. On +multi-socket hosts, NUMA placement is a further lever: interleaving the resident +weights across nodes measured **+13% (2-socket) and +40% (4-socket CPU-only)** +([#82](https://github.com/JustVugg/colibri/issues/82)). On a 2-socket Xeon Silver +4510 host with 6× RTX 5090, selective `COLI_NUMA=1` raised effective CPU-expert +bandwidth from **42.42 to 58.26/65.89 GB/s** and greedy decode from **7.66 to +9.02/9.17 tok/s** (64 tokens, `TEMP=0 DRAFT=0`, byte-identical output). Do not +blanket-interleave a GPU host: it also spreads DMA staging pages and has measured +up to a 10× regression; generated plans enable only the selective slab policy. + +## Quality benchmark + +**Measured** ([#108](https://github.com/JustVugg/colibri/issues/108)): the int4 +container scored **62.5% mean acc_norm** on hellaswag/arc/mmlu (0-shot +log-likelihood, n=40) — but 0-shot MC scoring underserves a reasoning model, and +the OLMoE fp16-vs-int4 A/B under the same harness measured the pure quantization +cost at **-8.2pp**, concentrated on the hardest task (per-row int4 scales erode +the small logit margins hard questions depend on — grouped scales recover ~63% +of that loss, see [#225](https://github.com/JustVugg/colibri/issues/225)). The +scale-granularity/rotation/lattice ablation lives in +`tools/quant_ablation.py` ([#81](https://github.com/JustVugg/colibri/issues/81)). + +```bash +cd c +pip install tokenizers datasets +./coli bench # hellaswag, arc_challenge, mmlu — 40 questions each +./coli bench hellaswag --limit 200 # one task, more questions +./coli bench mmlu arc_challenge --ram 100 # pick tasks, set a RAM budget +``` diff --git a/docs/cuda.md b/docs/cuda.md new file mode 100644 index 0000000..fab477d --- /dev/null +++ b/docs/cuda.md @@ -0,0 +1,104 @@ +# CUDA backend (Linux) + +colibrì includes an opt-in CUDA backend for model-resident tensors. Streaming +experts deliberately remain on the original CPU path: copying an expert from +NVMe to the GPU on every use would only replace the disk bottleneck with a PCIe +bottleneck. Resident quantized tensors are uploaded lazily once and reused. + +```bash +cd c +make cuda-test CUDA=1 # q8/q4/q2/f32 kernel correctness +make CUDA=1 +# optional dense-path experiment (hot experts are configured below) +COLI_CUDA=1 COLI_GPU=0 CUDA_DENSE=1 SNAP=/nvme/glm52_i4 ./glm 64 4 4 +``` + +Requirements: Linux, an NVIDIA driver, and a CUDA Toolkit under +`/usr/local/cuda` (override with `CUDA_HOME=/path/to/cuda`). +`CUDA_ARCH=native` builds for the GPU in the current machine. Requesting CUDA +with a CPU-only binary, an invalid device, or an unavailable runtime fails at +startup instead of silently falling back. For Windows, see +[windows.md](windows.md) (runtime DLL path). + +## The VRAM expert tier + +A measured `PIN` profile promotes its hottest experts into a persistent VRAM +tier while keeping the rest in RAM: + +```bash +STATS=stats.txt SNAP=/nvme/glm52_i4 ./glm 64 4 4 # collect routing frequencies first +COLI_CUDA=1 COLI_GPU=0 CUDA_EXPERT_GB=16 \ +PIN=stats.txt PIN_GB=160 SNAP=/nvme/glm52_i4 ./glm 64 4 4 + +# multi-GPU expert tier, 150 GB total budget across six 32 GB devices +COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=150 \ +CUDA_DENSE=1 PIN=stats.txt PIN_GB=300 RAM_GB=226 \ +SNAP=/nvme/glm52_i4 ./glm 64 4 4 + +# large-RAM host: fill safe VRAM, then keep every remaining expert in RAM +COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=auto \ +CUDA_DENSE=1 COLI_CUDA_ATTN=1 PIN=stats.txt PIN_GB=all RAM_GB=auto \ +SNAP=/nvme/glm52_i4 ./glm 64 4 4 +``` + +Selected experts are uploaded during startup, so capacity failures occur before +inference. The budget is clamped against free VRAM after reserving the projected +dense resident set and 2 GB of runtime headroom per device. With `COLI_GPUS`, +`CUDA_EXPERT_GB` is a total budget across the device set; experts are assigned +whole to the least-loaded device that can hold them. Multi-GPU runs default to +`PIN_FILL=1` (measured hot set first, then unused VRAM filled with zero-heat +experts) and `CUDA_RELEASE_HOST=1` (RAM copy released after upload, reloaded +from disk only if CUDA later fails). + +`CUDA_EXPERT_GB=auto` fills each device up to measured free memory minus +projected dense tensors and headroom. `PIN_GB=all` then loads the remaining +routed experts into RAM **up to the `--ram` budget** (it clamps — [#229](https://github.com/JustVugg/colibri/issues/229)), +eliminating decode-time disk misses when capacity permits. This mode is intended +for dedicated high-memory inference hosts. + +### Full-residency reference result (6× RTX 5090, 251 GiB host) + +`CUDA_EXPERT_GB=auto PIN_GB=all` selected a 176.7 GB VRAM tier + 191.3 GB RAM +tier (all 19,456 experts resident), adapting the VRAM tier every 16 tokens. +With the GPU-resident pipeline (`COLI_CUDA_PIPE=2`) and Tensor-Core W4A16 +dispatch (`COLI_CUDA_TC_W4A16=1`), 96-token greedy decode measured +**5.8–6.8 tok/s** (TTFT ~13 s; 1571-token prefill ~122 s then 4.2 tok/s). +Full experiment log: [experiments/glm52-6x5090-2026-07-12.md](experiments/glm52-6x5090-2026-07-12.md). +These are host-specific capacity results, not portable defaults. + +## The GPU-resident pipeline (`COLI_CUDA_PIPE`) + +`COLI_CUDA_PIPE=2` keeps the residual stream on-device across layers: rmsnorms, +residual adds, router GEMMs and the shared expert run on the GPU while the CPU +expert loop runs uninterrupted, with batched attention and grouped expert +uploads at prefill. On a single-GPU host this also pays at decode (S=1): +**+49%** measured on a 5070 Ti ([#273](https://github.com/JustVugg/colibri/issues/273)/#274); +on multi-GPU hosts the per-layer P2P hops cancel the gain, so the decode gate is +device-count aware. `COLI_CUDA_TC_W4A16=1` enables Tensor-Core int4×fp16 mixed +dispatch for batched rows (pays at ≥16 rows). + +## Notes and limitations + +- Text-mode timing reports prefill separately from decode. +- MTP speculation defaults off on CUDA (cold draft routes increase expert + traffic); explicit `DRAFT=n` overrides. Since #294, `SPEC_PIN=1` keeps + draft/verify kernels consistent when speculation is on. +- Devices use independent contexts; a single expert is not sharded. Kernels are + correctness-first custom kernels. +- Profile quality matters more than raw VRAM capacity: the same 150 GB tier + measured 0.94–1.64 tok/s hot-first vs 0.29 tok/s filled without routing heat. +- The GPU tier earns its VRAM only when the CPU is the weak link — a tuned + AVX-512 CPU can match a 5090 on expert matmul + ([#101](https://github.com/JustVugg/colibri/issues/101)). + +## Reproducible backend A/B without the full checkpoint + +```bash +cd c +python tools/make_glm_bench_model.py --output /nvme/colibri-bench-medium --device cuda +python tools/benchmark_cuda_fixture.py --model /nvme/colibri-bench-medium --gpu 0 +``` + +The 313M-parameter fixture has random weights and is not a language model. It +preserves the real MLA/MoE/streaming shapes to compare CPU streaming, dense-only +CUDA, CPU hot-store, and CUDA hot-expert execution with identical replay tokens. diff --git a/docs/media/colibri-atlas.png b/docs/media/colibri-atlas.png new file mode 100644 index 0000000..5e5e76e Binary files /dev/null and b/docs/media/colibri-atlas.png differ diff --git a/docs/media/colibri-brain.png b/docs/media/colibri-brain.png index 0cd6c90..9ac5676 100644 Binary files a/docs/media/colibri-brain.png and b/docs/media/colibri-brain.png differ diff --git a/docs/media/colibri-dashboard.png b/docs/media/colibri-dashboard.png index 123fd3e..d51aa08 100644 Binary files a/docs/media/colibri-dashboard.png and b/docs/media/colibri-dashboard.png differ diff --git a/docs/media/colibri-metrics.png b/docs/media/colibri-metrics.png new file mode 100644 index 0000000..853d686 Binary files /dev/null and b/docs/media/colibri-metrics.png differ diff --git a/docs/media/colibri-mobile.png b/docs/media/colibri-mobile.png new file mode 100644 index 0000000..5c1c63c Binary files /dev/null and b/docs/media/colibri-mobile.png differ diff --git a/docs/media/ladder.png b/docs/media/ladder.png new file mode 100644 index 0000000..cd272fc Binary files /dev/null and b/docs/media/ladder.png differ diff --git a/docs/media/sparse.png b/docs/media/sparse.png new file mode 100644 index 0000000..0d7a4a1 Binary files /dev/null and b/docs/media/sparse.png differ diff --git a/docs/media/tiers.png b/docs/media/tiers.png new file mode 100644 index 0000000..9452587 Binary files /dev/null and b/docs/media/tiers.png differ diff --git a/docs/media/token-path.png b/docs/media/token-path.png new file mode 100644 index 0000000..4aebd39 Binary files /dev/null and b/docs/media/token-path.png differ diff --git a/docs/metal.md b/docs/metal.md new file mode 100644 index 0000000..3c94e5c --- /dev/null +++ b/docs/metal.md @@ -0,0 +1,30 @@ +# Metal backend (Apple Silicon, experimental) + +On Apple Silicon the decode profile is matmul-bound, and unified memory removes +the PCIe copy tax that keeps CUDA's streaming experts on the CPU — so colibrì +has an opt-in Metal backend that runs the **routed-expert SwiGLU (batched, +zero-copy from the RAM slabs)**, the **fused decode attention** (full MLA layer +in one command buffer, S≤4), and **prefill's large GEMMs** on the GPU. +Token-exact vs the CPU path. + +```bash +cd c +make glm METAL=1 # macOS only; no Xcode needed (shader compiles at runtime) +make metal-test # standalone kernel/attention correctness vs CPU reference +COLI_METAL=1 COLI_MODEL=/path/glm52_i4 ./coli chat --ram 96 +``` + +Measured on an M4 Max (128 GB, warm cache, MTP on): CPU 0.30 → Metal +**0.42 tok/s (~1.4×)** (best config adds `DIRECT=1`; ~3× vs this machine's +first cold run). An M5 Max with a 46.9 GB learned pin reached **2.06 tok/s** +([#103](https://github.com/JustVugg/colibri/issues/103); see also the +[M5 Max performance report](METAL-M5MAX-PERF-REPORT.md)). + +Key design points: Metal's ~5 ms submit latency makes per-matmul dispatch a +loss — everything is batched into few command buffers per layer, and the +resident experts' GPU work is submitted *before* the missed experts' disk reads +so I/O and compute overlap. `COLI_METAL_GEMM_MIN` tunes the prefill GEMM row +threshold (default 16). Streaming, cache, MTP, DSA and the persistence formats +are unchanged; every GPU path falls back to the CPU per-block on any fault. +Numerics are dequant→f32-MAC (same as the CUDA tier); greedy outputs are +byte-identical to the CPU engine. diff --git a/docs/serve_protocol.md b/docs/serve_protocol.md new file mode 100644 index 0000000..b0eb61e --- /dev/null +++ b/docs/serve_protocol.md @@ -0,0 +1,107 @@ +# The serve protocols — engine ⇄ server wire format + +The engine speaks two line-oriented protocols over stdin/stdout. Both are plain text +plus byte-counted payload frames; every outbound line is written with a trailing +`fflush`, one line per write. On Windows both ends of the pipe are switched to binary +mode at startup — the CRT's CRLF translation otherwise corrupts the sentinels and +stalls byte-counted reads (#195). + +| protocol | entry | selected by | used by | +|---|---|---|---| +| **mux** (continuous batching, up to 16 KV slots) | `run_serve_mux` | `SERVE_BATCH=1` | `openai_server.py`, `coli web` | +| **legacy** (single slot, interactive) | `run_serve` | `SERVE=1` (without `SERVE_BATCH`) | `coli chat` | + +This document is the reference for the **mux** protocol; the legacy protocol is +summarized at the end. Line formats below are quoted from the emitting `printf`s in +`glm.c` — if this document and the code disagree, the code wins and this file needs a PR. + +## Startup handshake (engine → server) + +``` +\x01\x01READY\x01\x01 +STAT 0 0.00 0.0 +HWINFO | +TIERS +EMAP +``` + +The server must not send requests before `READY`. `HWINFO`/`TIERS`/`EMAP` are +telemetry (see below) and may grow — **servers must ignore line kinds they do not +recognize**; that is the protocol's forward-compatibility rule. + +## Requests (server → engine) + +``` +SUBMIT \n\n +CANCEL \n +``` + +- `id` — non-zero u64, unique among in-flight requests. +- `slot` — KV slot index, `0 … KV_SLOTS-1` (`KV_SLOTS` env, 1–16, default 1). A slot + holds one conversation's KV; the engine matches the tokenized payload against the + slot's history and reuses the common prefix (truncate-and-extend), so stateless + HTTP turns keep their cache. +- `bytes` — exact byte length of `payload` (UTF-8, may contain newlines). The engine + reads exactly that many bytes after the header line, then one trailing `\n`. +- `payload` — the fully rendered prompt (the server owns the chat template). +- EOF on stdin = graceful shutdown: in-flight requests finish first. + +Prefill is serial; decode is continuously batched — every active slot contributes +one row per forward. + +## Responses (engine → server) + +Per request, in order: + +``` +DATA \n\n # a decoded token's text; repeated +TOPK 5 ... ×5 # candidates for the sampled token (SERVE_TOPK=1) +HITS # ~every 6 tokens: routed-expert bitmap since last HITS +REPIN # live re-pin swap events, as they happen +... +DONE STAT +``` + +Errors replace the stream: `ERROR ` with codes `BAD_FRAME`, `BAD_REQUEST`, +`SLOT_BUSY`, `DUPLICATE_ID`, `EMPTY_PROMPT`, `NOT_FOUND` (CANCEL of unknown id), +`CANCELLED`. A `CANCEL` is acknowledged by `ERROR CANCELLED` after the slot's KV +is persisted. + +Immediately before each `DONE` the engine emits a telemetry block for the finished +turn: `HWINFO`, `PERF`, `ENTROPY`, `GPUS`, `TIERS`, `EMAP`, `HITS` (formats below). +`.coli_usage` is persisted at every turn end, not only at exit. + +## Telemetry lines + +| line | format | meaning | +|---|---|---| +| `TIERS` | `TIERS ` | expert count per tier + resident bytes | +| `HWINFO` | `HWINFO \|` | host snapshot (GBs are floats) | +| `EMAP` | `EMAP ` | one byte per expert, row-major over `rows×cols` (sparse layers +MTP × experts): `byte = (tier<<6) \| heat` — 2-bit tier (0 disk / 1 RAM / 2 VRAM), 6-bit log₂-bucketed usage heat | +| `HITS` | `HITS ` | 1 bit per expert, experts routed since the previous `HITS` | +| `PERF` | `PERF
` | this turn's PROFILO deltas, seconds | +| `ENTROPY` | `ENTROPY

…` | per-sparse-layer routing entropy of the turn, bits | +| `GPUS` | `GPUS ( )×n` | per-device VRAM + resident expert count (CUDA builds) | +| `TOPK` | `TOPK 5 ( )×5` | token text hex-encoded so the line stays line-shaped | +| `REPIN` | `REPIN ` | one line per hot-store swap (`REPIN=n` mode) | + +All telemetry is advisory: servers render what they know and skip the rest. + +## HTTP surface (`openai_server.py`) + +- `POST /v1/chat/completions` — OpenAI-compatible; streaming responses emit one extra + SSE frame `data: {"colibri": {stats, perf, topk, entropy, gpus, repin}}` immediately + before `data: [DONE]`; non-streaming responses attach the same object as a + `"colibri"` field. +- `GET /experts` — the latest `EMAP`/`HITS` state: `{rows, cols, map, hits, seq, + gpus, entropy, repin}`. +- `GET /*` — static hosting of `web/dist` (SPA fallback, path-traversal-safe), plus + `experts.json` if published there (the measured expert atlas, #175/#218). + +## Legacy protocol (`run_serve`, `coli chat`) + +Interactive lines are prompts; control frames: `\x02RESET` (clear history), +`\x02MORE` (continue an NGEN-truncated answer), and +`\x02PROMPT [kv_slot]\n\n` +(the pre-mux API mode). Responses are raw text terminated by `\x01\x01END\x01\x01` +plus a `STAT` line. New integrations should use the mux protocol. diff --git a/docs/tuning.md b/docs/tuning.md new file mode 100644 index 0000000..c95eca8 --- /dev/null +++ b/docs/tuning.md @@ -0,0 +1,110 @@ +# Tuning & runtime knobs + +Everything here is opt-in; the defaults are chosen so a plain `./coli chat` +is safe on any machine. See also [SETTINGS.md](SETTINGS.md) and +[ENVIRONMENT.md](ENVIRONMENT.md) for the full variable inventory. + +## The knobs that matter most + +| knob | what it does | +|---|---| +| `--temp T` | token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy) | +| `--topp 0.7` | adaptive expert top-p (30–40% less disk; lossy — prints a warning) | +| `--ngen N` | max tokens per answer (`:more` in chat continues a truncated one) | +| `--repin N` | adapt RAM/VRAM hot experts every N emitted tokens | +| `RAM_GB=` | claim more RAM for the expert cache than the conservative auto-detect | +| `PIN=stats PIN_GB=g` | pin the hottest experts from a measured usage profile | +| `DRAFT=n` | MTP draft depth (0 disables speculation) | +| `GRAMMAR=g.gbnf` | grammar-forced drafts for constrained JSON/NDJSON output ([docs](grammar-draft.md)) | +| `THINK=1` | enable GLM-5.2's reasoning block | +| `PILOT=1` | router-lookahead disk prefetch (see below) | +| `URING=1` | Linux-only batched expert I/O (implies `PIPE=1`) | +| `PIPE=0` | disable the async expert-load pool (default ON — overlaps `pread` with matmul, −18% disk service) | +| `DIRECT=1` | O_DIRECT expert reads (measured **+65%** alone on a Strix Halo, [#200](https://github.com/JustVugg/colibri/issues/200)) | +| `COLI_NUMA=1` | interleave resident weights across NUMA nodes on multi-socket hosts ([#82](https://github.com/JustVugg/colibri/issues/82)) | +| `CACHE_ROUTE=1` | cache-aware max-rank routing (opt-in, [#199](https://github.com/JustVugg/colibri/issues/199)) | +| `AUTOPIN=0` | disable the learning cache's auto-pin | +| `CAP_RAISE=0` | don't auto-grow the expert cache | +| `KVSAVE=0` | disable KV-cache persistence | +| `TF=1` | teacher-forcing validation | + +## Resource policy + +`coli plan` reports the planned hot (VRAM), warm (RAM), and cold backing (disk) +tiers, the reason for each placement, and the expected bottleneck. The default +`--policy quality` and `--policy balanced` modes preserve checkpoint quantization +and router decisions unless `--topk` or `--topp` is passed; those explicit lossy +overrides print a warning and proceed. + +Auto-tier plans size OpenMP from physical cores and bind workers across cores. +Memory-bound quantized kernels can regress sharply when SMT siblings compete for +limited memory channels; explicit `OMP_*` settings always take precedence. + +```bash +coli plan --model /models/glm52_i4 --policy quality +coli run --auto-tier --policy quality "Explain MoE offloading" +# Explicit research-only router reduction: +coli run --policy experimental-fast --topk 4 "Benchmark prompt" +``` + +Disk is an immutable recovery source, not a normal decode target. If the plan +leaves cold expert bytes on disk, speed depends on cache hit rate; output quality +does not. + +Cold expert reads can use a deferred pipeline: resident RAM/VRAM experts execute +while missing experts are loaded in a bounded background I/O pool, then the cold +results join before the layer completes. The pool engages only under `PIPE=1`; +`PIPE_WORKERS=n` sets its worker count (default 8). Profiling reports both disk +service time and the smaller foreground-visible wait time so overlap is explicit. + +`--policy balanced` enables lossless live placement (`REPIN=64`). At safe request +boundaries, a per-layer LFRU score combines decaying session frequency with recent +access and replaces at most four sufficiently colder pinned experts. `--policy +quality` leaves live replacement off by default; `REPIN=0` always disables it. + +## The learning cache + +The engine records which experts your usage actually routes to (`.coli_usage` +next to the model, updated every turn) and at startup automatically pins the +hottest ones in spare RAM — colibrì literally gets faster the more you use it. +`PIN=auto` seeds the pin directly from the live usage history +([#301](https://github.com/JustVugg/colibri/pull/301)). + +**The expert cache auto-sizes to your RAM** (since 2026-07-10): the engine +*raises* the LRU cap to fill your `--ram` budget instead of only lowering it. +If you benchmarked colibrì before that date, rerun — your numbers were capped. + +**Live tier adaptation** (`--repin N`, opt-in): at safe turn boundaries, a +decaying session heat map replaces cold pinned experts with hotter streamed +experts. A 25% hysteresis and a four-swap limit prevent tier thrashing. +Persistent `.coli_usage` remains the long-term signal and is not decayed. + +## Router-lookahead prefetch (`PILOT=1`, experimental) + +GLM-5.2's expert routing is measurably predictable *ahead of time* — applying +layer L+1's router to layer L's post-attention state recalls **71.6%** of the +true top-8 (vs 41.3% for "same experts as last token"). `PILOT=1` issues +next-layer expert readahead from a dedicated I/O thread while the current layer +computes. `PILOT_REAL=1` moves the prefetched loads off the critical path +(measured +11pp hit rate on a big-cache host), and `PILOT_TWO=1` folds the +computed shared-expert into the prediction (+3% recall, +[#200](https://github.com/JustVugg/colibri/issues/200)). On disk-saturated +hosts hint-only PILOT can be net negative — measure on yours. + +## Speculation and reproducibility + +Speculative decoding requires that the draft and verify paths compute the same +function — `SPEC_PIN=1` (default since [#294](https://github.com/JustVugg/colibri/pull/294)) +pins every forward issued while drafts are live to the platform's S=1 kernel +family. For byte-exact reproducibility across runs: `DRAFT=0`, plus `IDOT=0 +COLI_CUDA=0` if you also want kernel-family/GPU independence. Acceptance +percentages are not comparable across engine versions under `--topp` +([#163](https://github.com/JustVugg/colibri/issues/163) has the full story). + +## Conversations reopen warm + +`coli chat` persists the compressed MLA KV-cache to disk after every turn +(`.coli_kv`, ~182 KB/token, appended incrementally, crash-safe). Close the chat, +reopen it tomorrow — the model still remembers the whole conversation and **zero +re-prefill happens**: validated byte-identical to an uninterrupted session. +`:reset` clears it, `KVSAVE=0` disables it. diff --git a/docs/windows.md b/docs/windows.md new file mode 100644 index 0000000..ecb81e7 --- /dev/null +++ b/docs/windows.md @@ -0,0 +1,172 @@ +# Windows 11 native install — a complete walkthrough (no WSL) + +A start-to-finish, reproducible path from a fresh Windows 11 machine to GLM-5.2 generating tokens, with the GPU tier. Every step and every failure mode below was hit and verified on real hardware: Core Ultra 9 285K (AVX-VNNI) / RTX 5080 (sm_120) / 128 GB RAM / Windows 11 24H2 (issue #306). Steps are ordered so the long downloads run while you build. + +## 0. What you need + +| Piece | Why | Get it | +|---|---|---| +| git, Python 3 | clone + `coli` launcher | winget / python.org | +| MinGW-w64 gcc + make | builds the engine (MSVC can't) | `scoop install mingw-winlibs`, MSYS2, or portable **w64devkit** (no admin, unzip and go) | +| CUDA Toolkit ≥ 12.8 | GPU tier; ≥12.8 required for Blackwell/sm_120 | `winget install Nvidia.CUDA` | +| MSVC Build Tools (C++ workload) | nvcc's host compiler for the CUDA DLL | `winget install Microsoft.VisualStudio.2022.BuildTools` + "Desktop development with C++" | +| ~400 GB free on a local NVMe | the int4 model (~370–384 GB) | NTFS is fine; **never** a network mount | + +RAM: 16 GB minimum, more = bigger expert cache = faster. The build itself needs none of the CUDA/MSVC pieces — do the CPU build first, add the GPU tier later. + +## 1. Start the model download first (it's the long pole) + +```powershell +python -m pip install -U "huggingface_hub[hf_transfer]" +$env:HF_HUB_ENABLE_HF_TRANSFER = "1" +hf download --local-dir D:\glm52_i4 +``` + +Use the container recommended in the README (with **int8 MTP heads** — int4 heads silently give 0% draft acceptance). The download is resumable: if it stops, rerun the same command. Expect hours; everything below fits inside them. + +## 2. Build the engine (CPU) + +From a normal PowerShell, in the repo's `c\` directory: + +```powershell +make glm.exe ARCH=native # ARCH=native unlocks AVX-VNNI on Alder Lake+/Arrow Lake +make iobench.exe # disk benchmark, useful before committing to the download +``` + +Warnings about `#pragma comment` and unused variables are normal (MSVC-isms gcc ignores). The engine banner should print `idot: avx-vnni` on VNNI-capable CPUs — if it says avx2, you built without `ARCH=native`. + +### ⚠️ Smart App Control will block your fresh binary + +On Windows 11 machines with **Smart App Control** enforced (`VerifiedAndReputablePolicyState = 1`), running your self-compiled `glm.exe` fails with: + +``` +Program 'glm.exe' failed to run: An Application Control policy has blocked this file +``` + +This is not Defender and not Mark-of-the-Web — SAC blocks *all* unsigned, unknown binaries, which includes anything you compile yourself. **Fix:** Windows Security → App & browser control → Smart App Control settings → **Off**, then **reboot** (the policy only reloads on restart). Note SAC is one-way: re-enabling later requires resetting Windows. If the settings page is missing, the registry equivalent is setting `HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy\VerifiedAndReputablePolicyState` to `0` (admin PowerShell), then rebooting. Check your current state before touching anything: + +```powershell +(Get-ItemProperty "HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy").VerifiedAndReputablePolicyState +# 0 = off, 1 = enforced, 2 = evaluation +``` + +## 3. Build the CUDA DLL (GPU tier) + +nvcc needs MSVC as host compiler, so this one step must run from a shell with the MSVC environment: open **"x64 Native Tools Command Prompt for VS 2022"** from the Start menu (plain PowerShell will fail the `cl` check). Then: + +```cmd +make cuda-dll CUDA_ARCH=sm_120 # match your GPU: sm_120 Blackwell, sm_89 Ada, ... +make glm.exe CUDA_DLL=1 ARCH=native # relink host with the runtime loader +``` + +Two pitfalls, both fixed on current `dev` (#314) but worth knowing on older checkouts: + +- **Spaces in `CUDA_HOME`** (`C:\Program Files\...`) used to break the recipe → fixed; nvcc now comes from PATH and `"$(NVCC)"` is quoted. +- **`make glm.exe CUDA_DLL=1` after a CPU-only build** used to report `up to date` and silently keep the CPU-only binary (GPU tier never engages, no error). Current `dev` has a build-config stamp that forces the relink. On older trees: delete `glm.exe` first. + +Sanity check: first GPU run should print `[CUDA] device 0: , ... sm_XX` and `[CUDA] mode: routed experts + resident dense tensors`. + +## 4. First run + +```powershell +cd \c +$env:OMP_NUM_THREADS = "" +python coli run "Explain what a mixture-of-experts model is." --model D:\glm52_i4 --ngen 48 +``` + +The first run is cold — expect the profile to be dominated by `expert-disk` while the cache warms; hit rate climbs run over run. GPU tier on top: + +```powershell +$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_DENSE="1"; $env:CUDA_EXPERT_GB="4" +python coli run "..." --model D:\glm52_i4 --ngen 64 +``` + +Size `CUDA_EXPERT_GB` so dense (~10 GB) + experts + working set stays under your VRAM. Note MTP speculation is off by default under CUDA (#293, float-accumulation divergence between draft and verify) — `COLI_CUDA_MTP=1` opts back in. + +## 5. Reference numbers from this walkthrough's hardware + +285K / RTX 5080 / 128 GB / NVMe at 5.85 GB/s random-read (19 MB blocks, `iobench`): 0.26 tok/s cold CPU → 0.30 warm CPU (MTP 2.2–2.3 tok/forward) → 0.42 tok/s GPU tier + auto-pin, expert hit 66%, ~65% of wall time in expert-disk. Disk-bound is the expected shape at ~25% expert residency — a faster disk and more RAM move the floor, the GPU moves the compute. + +## Quick failure index + +| Symptom | Cause | Fix | +|---|---|---| +| `An Application Control policy has blocked this file` | Smart App Control | §2 — turn SAC off + **reboot** | +| `cuda-dll ... Error 1` immediately | old tree: spaced CUDA_HOME / MSVC rejects `-Wextra` | update to current `dev` (#314) | +| `glm.exe is up to date` but GPU never engages | old tree: stale CPU-only binary | update to `dev`, or delete `glm.exe` and rebuild | +| `cl.exe (MSVC) not in PATH` | built from plain PowerShell | use the x64 Native Tools prompt | +| `nvcc fatal: unsupported gpu architecture 'sm_120'` | CUDA < 12.8 | install CUDA 12.8+ | +| MTP `0% (0/0)` on CPU path | int4 MTP heads in the container | use the int8-MTP container | +| MTP `draft=0` under CUDA | intended default since #293 | `COLI_CUDA_MTP=1` to opt in | + +--- + +## Reference: build flags & warmup + +# AVX-VNNI: Intel Alder Lake+ (and Meteor Lake+) CPUs have a 128-bit int8 +# dot-product instruction (VPDPBUSD) the engine can use for ~1.3x faster +# quantized matmul. The x86-64-v3 default (portable AVX2) compiles it out; +# build for THIS machine to enable it: +make glm.exe ARCH=native # banner prints "idot: avx-vnni" + +# Verify (tiny model, 2.4 MB): +pip install torch transformers safetensors huggingface_hub +python tools/make_glm_oracle.py # generate tiny oracle +SNAP=./glm_tiny TF=1 ./glm.exe 64 16 16 # expect "32/32 positions" + +# Run with real model: +SNAP=D:\glm52_i4 ./glm.exe 64 4 16 # batch inference +python coli chat --model D:\glm52_i4 # interactive chat +python coli serve --model D:\glm52_i4 # OpenAI-compatible API +``` + +> Windows Store's `python` alias stub is the single most common native-Windows +> trap: install real Python (python.org or `winget install Python.Python.3.12`) +> or disable the alias under *Settings → Apps → App execution aliases*. + +## Warmup (overnight cache priming) + +The engine's expert cache learns from your workload. The included `warmup.ps1` +script runs `coli run` in a loop with diverse prompts to build the +`.coli_usage` histogram unattended, so the next real session starts with a +large, accurate hot-expert pin. Each run saves usage atomically on clean +completion. + +```powershell +.\warmup.ps1 -Rounds 1 -Ngen 32 # ~60-90 min, durable progress +``` + +## NVIDIA GPU (optional, via runtime DLL) + +On Windows the engine is built with MinGW gcc but CUDA kernels require MSVC + +nvcc. The split is clean: build the CUDA backend into a standalone +`coli_cuda.dll` (nvcc + MSVC), then the host `glm.exe` loads it at runtime via +`LoadLibrary` (`c/backend_loader.c`). The host never links cudart directly; if +the DLL is absent the engine falls back to CPU without error. + +```powershell +# Prerequisites: CUDA Toolkit + MSVC Build Tools (cl.exe) + nvcc on PATH. +# Build the DLL from a shell with the MSVC environment set (vcvars64.bat or +# "x64 Native Tools Command Prompt for VS"): +make cuda-dll CUDA_HOME="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" CUDA_ARCH=sm_120 + +# Build the host with the runtime loader (CUDA_DLL=1 adds -DCOLI_CUDA and +# links backend_loader.o instead of cudart): +make glm.exe CUDA_DLL=1 ARCH=native + +# Run with the GPU expert tier (8 GB VRAM budget here; scale to your free VRAM): +$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_EXPERT_GB="8" +python coli chat --model D:\glm52_i4 --topp 0.7 +``` + +The DLL exports the full `extern "C"` surface (including the #111 pipeline ABI); +`backend_loader.c` resolves symbols via `GetProcAddress` on first use. +`ColiCudaTensor*` is opaque to the host (stored, never dereferenced), so the +MSVC-allocated struct is safe across the ABI boundary. `CUDA_ARCH` must match +your GPU's compute capability (e.g. `sm_120` for Blackwell / RTX 50-series, +`sm_89` for Ada / RTX 40-series). A one-shot `build_cuda.bat` wrapper is also +available. + +**Measured on a single RTX 5070 Ti + Core Ultra 9 (32 GB RAM):** CPU-only 0.63 +→ CUDA attention+dense 0.72 → **1.07 tok/s** with the GPU-resident pipeline at +decode ([#273](https://github.com/JustVugg/colibri/issues/273), merged in #274). diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..9b788fa --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1784160687, + "narHash": "sha256-iYL/bixrb6FlHFu/gIuBYzq6c6lM5AAXsXNSWXtIgQc=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "4382ed2b7a6839d4280a9b386db49cbc5907414d", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-26.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix index 368b0a7..584a088 100644 --- a/flake.nix +++ b/flake.nix @@ -26,7 +26,9 @@ version = "1.0"; src = ./.; - nativeBuildInputs = [ pkgs.makeWrapper ]; + # python3 is needed by checkPhase: `make test-c` shells out to + # `python3 tools/run_tests.py` (see c/Makefile, PYTHON ?= python3). + nativeBuildInputs = [ pkgs.makeWrapper pkgs.python3 ]; buildInputs = [ pkgs.gcc @@ -44,18 +46,29 @@ installPhase = '' runHook preInstall - mkdir -p $out/bin - cp c/glm $out/bin/glm - # Wrap coli (the Python CLI) so it finds the right python and the engine - mkdir -p $out/share/colibri - cp c/coli $out/share/colibri/coli - chmod +x $out/share/colibri/coli - cp -r c/tools $out/share/colibri/tools + # Self-contained layout under $out/lib/colibri that mirrors the + # source tree `coli` runs in (see the path-resolution logic at the + # top of c/coli): the engine, the coli CLI script, the support + # modules it imports (openai_server.py, resource_plan.py, + # doctor.py), and tools/ all sit next to each other. + mkdir -p $out/lib/colibri/tools $out/bin + cp c/glm $out/lib/colibri/glm + cp c/coli $out/lib/colibri/coli + chmod +x $out/lib/colibri/coli + cp c/openai_server.py c/resource_plan.py c/doctor.py $out/lib/colibri/ + cp -r c/tools/* $out/lib/colibri/tools/ + # $out/bin holds the user-facing entry points. + ln -s ../lib/colibri/glm $out/bin/glm + + # Wrap coli: point it at the bundled engine (COLI_ENGINE) so it is + # found by default, and at the module dir (PYTHONPATH) so + # `import openai_server` / `resource_plan` / `doctor` resolve. makeWrapper ${pythonEnv}/bin/python $out/bin/coli \ - --add-flags "$out/share/colibri/coli" \ - --set PYTHONPATH "${pythonEnv}/${pkgs.python3.sitePackages}" + --add-flags "$out/lib/colibri/coli" \ + --set-default COLI_ENGINE "$out/lib/colibri/glm" \ + --set PYTHONPATH "$out/lib/colibri:${pythonEnv}/${pkgs.python3.sitePackages}" runHook postInstall ''; diff --git a/web/README.md b/web/README.md index c867d01..fe502ae 100644 --- a/web/README.md +++ b/web/README.md @@ -17,9 +17,14 @@ npm test npm run build ``` +Besides Chat and Brain, the **Profiling** tab charts where the engine spent +each turn's wall time (I/O wait, expert matmul, attention, LM head) from the +server's `/profile` endpoint — a rolling window of per-turn `PROF` snapshots +emitted by the engine. + The test suite stays browser-light: API requests use a mocked `fetch`, while runtime capability and storage behavior are covered through pure helpers. It -checks that `/health` is resolved next to (not below) the OpenAI `/v1` prefix, +checks that `/health` and `/profile` are resolved next to (not below) the OpenAI `/v1` prefix, supports both boolean and numeric `scheduler.active` responses, and sends the colibrì-specific `cache_slot` field only when KV-slot support was advertised. diff --git a/web/src/App.tsx b/web/src/App.tsx index d4e0e41..a33cc88 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -31,6 +31,7 @@ import { Textarea } from "@/components/ui/textarea" import { getHealth, listModels, streamChat, type ChatMessage, type HealthResponse, type StreamChatResult } from "@/lib/api" import { activeRequests, supportsCacheSlots } from "@/lib/runtime" import { Brain } from "./Brain" +import { Profiling } from "./Profiling" import { persistPublicSettings, stored } from "@/lib/storage" import { cn } from "@/lib/utils" @@ -73,7 +74,7 @@ export default function App() { const [totalTokens, setTotalTokens] = useState({ prompt: 0, completion: 0 }) const [connecting, setConnecting] = useState(false) const [connected, setConnected] = useState(false) - const [view, setView] = useState<"chat" | "brain">("chat") + const [view, setView] = useState<"chat" | "brain" | "profiling">("chat") const [error, setError] = useState("") const autoConnected = useRef(false) const abortRef = useRef(null) @@ -314,6 +315,7 @@ export default function App() {
+
{loading && tokenCount > 0 ? {tokenCount} tokens : null} @@ -326,7 +328,8 @@ export default function App() {
- {view === "brain" ? : <> + {view === "brain" ? + : view === "profiling" ? : <>
{!messages.length ? ( diff --git a/web/src/Profiling.tsx b/web/src/Profiling.tsx new file mode 100644 index 0000000..9fdaebb --- /dev/null +++ b/web/src/Profiling.tsx @@ -0,0 +1,165 @@ +import { useEffect, useState } from "react" +import { Activity, Gauge, HardDrive, Timer } from "lucide-react" + +import { getProfile, type ProfileTurn } from "@/lib/api" + +/* Wall-time phases stacked per turn. The order is the palette's CVD-safe slot + * order (validated as a set on this surface) — identity never leans on colour + * alone: segments keep 2px gaps, the legend is always shown and the table + * carries the exact numbers. Disk *service* time is reported separately: it + * runs on I/O threads overlapped with compute, so only the stall the compute + * thread actually felt (I/O wait) belongs inside the wall-time stack. */ +const PHASES = [ + { key: "expert_wait_s", name: "I/O wait", color: "#3987e5" }, + { key: "expert_matmul_s", name: "Expert matmul", color: "#199e70" }, + { key: "attention_s", name: "Attention", color: "#c98500" }, + { key: "lm_head_s", name: "LM head", color: "#008300" }, + { key: "other_s", name: "Other", color: "#9085e9" }, +] as const + +interface Turn extends ProfileTurn { other_s: number; toks: number } + +const derive = (turn: ProfileTurn): Turn => ({ + ...turn, + other_s: Math.max(0, turn.wall_s - turn.expert_wait_s - turn.expert_matmul_s - turn.attention_s - turn.lm_head_s), + toks: turn.wall_s > 0 ? turn.completion_tokens / turn.wall_s : 0, +}) + +const seconds = (value: number) => (value >= 10 ? value.toFixed(1) : value.toFixed(2)) + "s" + +function ShareBar({ label, turns }: { label: string; turns: Turn[] }) { + const total = turns.reduce((sum, turn) => sum + turn.wall_s, 0) + const parts = PHASES.map((phase) => ({ ...phase, value: turns.reduce((sum, turn) => sum + turn[phase.key], 0) })) + return ( +
+
{label}{seconds(total)}
+
`${part.name} ${seconds(part.value)}`).join(", ")}> + {parts.map((part) => { + const share = total > 0 ? part.value / total : 0 + return share > 0.001 ? ( + + {share >= 0.09 ? `${Math.round(100 * share)}%` : ""} + + ) : null + })} +
+
+ ) +} + +/* Column chart over the recent turns; oldest on the left. Stacked mode draws the + * wall-time composition, plain mode a single series (no legend — the title names it). */ +function TurnColumns({ turns, stacked, height, format }: { turns: Turn[]; stacked: boolean; height: number; format: (turn: Turn) => string }) { + const [hover, setHover] = useState(null) + const peak = Math.max(...turns.map((turn) => (stacked ? turn.wall_s : turn.toks)), 1e-9) + const gap = 2 + const width = Math.max(1, (100 - gap * (turns.length - 1)) / turns.length) + return ( +
setHover(null)}> + +
+ {turns.length > 1 ? `${turns.length} turns · oldest → newest` : "1 turn"} + {hover !== null && turns[hover] ? format(turns[hover]) : `peak ${stacked ? seconds(peak) : peak.toFixed(1) + " tok/s"}`} +
+
+ ) +} + +export function Profiling({ baseUrl, apiKey, connected }: { baseUrl: string; apiKey: string; connected: boolean }) { + const [turns, setTurns] = useState([]) + + useEffect(() => { + if (!connected) return + let disposed = false + const poll = async () => { + if (document.visibilityState === "hidden") return + try { + const result = await getProfile(baseUrl, apiKey) + if (!disposed) setTurns(result.turns.map(derive)) + } catch { /* engine busy or restarting — keep the last snapshot */ } + } + void poll() + const timer = window.setInterval(() => void poll(), 2000) + return () => { disposed = true; window.clearInterval(timer) } + }, [baseUrl, apiKey, connected]) + + const latest = turns[turns.length - 1] + const recent = turns.slice(-40) + const diskService = turns.reduce((sum, turn) => sum + turn.expert_disk_s, 0) + + return ( +
+
+
Profiling — where the engine spends each turn
+
+ {PHASES.map((phase) => {phase.name})} +
+
+ + {!latest ? ( +

{connected ? "No profiled turns yet — send a chat message and the breakdown appears here." : "Connect to the engine to collect per-turn timings."}

+ ) : ( + <> +
+
Last turn{latest.toks.toFixed(1)}tok/s
+
Wall time{seconds(latest.wall_s)}{latest.prompt_tokens} → {latest.completion_tokens} tokens
+
Batching{latest.forwards > 0 ? (latest.completion_tokens / latest.forwards).toFixed(2) : "—"}tokens / forward
+
Disk service{seconds(latest.expert_disk_s)}overlapped with compute
+
+ +
+ + {turns.length > 1 ? : null} +
+ +
+
+
Throughput per turn (tok/s)
+ `${turn.toks.toFixed(1)} tok/s · ${turn.completion_tokens} tokens`} /> +
+
+
Turn wall time by phase (s)
+ `${seconds(turn.wall_s)} · ${PHASES.map((phase) => `${phase.name} ${seconds(turn[phase.key])}`).join(" · ")}`} /> +
+
+ +
+ + {PHASES.map((phase) => )} + + {recent.slice().reverse().map((turn, index) => ( + + + + + + {PHASES.map((phase) => )} + + + ))} + +
TurnTokenstok/sWall{phase.name}Disk service
{turns.length - index}{turn.prompt_tokens} → {turn.completion_tokens}{turn.toks.toFixed(1)}{seconds(turn.wall_s)}{seconds(turn[phase.key])}{seconds(turn.expert_disk_s)}
+ {diskService > 0 ?

Disk service is time spent reading experts on I/O threads; it overlaps with compute, so only the I/O wait the compute thread felt counts inside the wall-time stack. With multiple KV sessions the shares describe the whole engine over the turn's window.

: null} +
+ + )} +
+ ) +} diff --git a/web/src/index.css b/web/src/index.css index b428805..8d777f4 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -151,3 +151,39 @@ button:focus-visible, input:focus-visible, textarea:focus-visible, select:focus- .brain-tip-spec { color: var(--primary); font-weight: 700; } .brain-tip-spec small, .brain-tip-aff { color: #8b9aa3; font-weight: 400; } .brain-tip-aff { font-size: 10px; } + +/* ---- Profiling page ---- */ +.prof-page { flex: 1; display: flex; flex-direction: column; gap: 16px; padding: 18px 22px; overflow: auto; } +.prof-head { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 10px; } +.prof-legend { display: flex; flex-wrap: wrap; gap: 12px; font-size: 11px; color: #a5b0b4; align-items: center; } +.prof-legend i { width: 9px; height: 9px; border-radius: 2px; display: inline-block; margin-right: 5px; vertical-align: -1px; } +.prof-tiles { display: grid; grid-template-columns: repeat(4, 1fr); gap: 10px; } +.prof-tiles > div { display: grid; gap: 4px; padding: 12px 14px; border: 1px solid var(--border); border-radius: 11px; background: var(--card); } +.prof-tiles span { display: flex; align-items: center; gap: 5px; color: #7f8d92; font-size: 9px; font-weight: 700; letter-spacing: .08em; text-transform: uppercase; } +.prof-tiles strong { font: 600 22px ui-monospace, SFMono-Regular, monospace; font-variant-numeric: tabular-nums; } +.prof-tiles small { color: var(--muted-foreground); font-size: 10px; } +.prof-shares { display: grid; gap: 10px; padding: 14px; border: 1px solid var(--border); border-radius: 11px; background: var(--card); } +.prof-share { display: grid; gap: 6px; } +.prof-share-head { display: flex; justify-content: space-between; color: #a5b0b4; font-size: 11px; font-weight: 600; } +.prof-share-head code { color: var(--foreground); font-size: 11px; font-variant-numeric: tabular-nums; } +.prof-share-bar { display: flex; gap: 2px; height: 22px; border-radius: 6px; overflow: hidden; } +.prof-share-bar span { display: grid; place-items: center; min-width: 0; overflow: hidden; color: #06090b; font-size: 10px; font-weight: 700; font-variant-numeric: tabular-nums; transition: width .5s ease; } +.prof-charts { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; } +.prof-chart { display: grid; gap: 8px; padding: 12px 14px; border: 1px solid var(--border); border-radius: 11px; background: var(--card); } +.prof-chart-title { color: #a5b0b4; font-size: 11px; font-weight: 600; } +.prof-plot svg { display: block; width: 100%; height: 120px; } +.prof-grid { stroke: var(--border); stroke-width: .3; } +.prof-plot-foot { display: flex; justify-content: space-between; gap: 8px; margin-top: 6px; color: #7f8d92; font-size: 10px; } +.prof-plot-foot code { color: var(--foreground); font-variant-numeric: tabular-nums; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } +.prof-table-wrap { overflow-x: auto; border: 1px solid var(--border); border-radius: 11px; background: var(--card); } +.prof-table { width: 100%; border-collapse: collapse; font-size: 11px; font-variant-numeric: tabular-nums; } +.prof-table th { padding: 9px 12px; border-bottom: 1px solid var(--border); color: #7f8d92; font-size: 9px; font-weight: 700; letter-spacing: .07em; text-transform: uppercase; text-align: left; white-space: nowrap; } +.prof-table th i { width: 8px; height: 8px; border-radius: 2px; display: inline-block; margin-right: 5px; vertical-align: -1px; } +.prof-table td { padding: 7px 12px; border-bottom: 1px solid var(--border); color: #c3ccd0; white-space: nowrap; } +.prof-table tbody tr:last-child td { border-bottom: 0; } +.prof-note { margin: 0; padding: 10px 12px; color: var(--muted-foreground); font-size: 10px; line-height: 1.5; border-top: 1px solid var(--border); } +@media (max-width: 900px) { + .prof-page { padding: 10px; } + .prof-tiles { grid-template-columns: 1fr 1fr; } + .prof-charts { grid-template-columns: 1fr; } +} diff --git a/web/src/lib/api.test.ts b/web/src/lib/api.test.ts index 29e7ede..9239828 100644 --- a/web/src/lib/api.test.ts +++ b/web/src/lib/api.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest" -import { extractSSE, getHealth, serverEndpoint, streamChat } from "./api" +import { extractSSE, getHealth, getProfile, serverEndpoint, streamChat } from "./api" afterEach(() => vi.unstubAllGlobals()) @@ -36,6 +36,19 @@ describe("runtime API", () => { headers: expect.objectContaining({ Authorization: "Bearer secret" }), })) }) + + it("requests the profiling history next to the OpenAI v1 prefix", async () => { + const turn = { + wall_s: 2.5, prompt_tokens: 7, completion_tokens: 12, + expert_disk_s: 0.4, expert_wait_s: 0.1, expert_matmul_s: 0.9, + attention_s: 0.6, lm_head_s: 0.2, forwards: 15, + } + const fetchMock = vi.fn().mockResolvedValue(new Response(JSON.stringify({ seq: 1, turns: [turn] }))) + vi.stubGlobal("fetch", fetchMock) + + await expect(getProfile("http://localhost:8000/v1/")).resolves.toEqual({ seq: 1, turns: [turn] }) + expect(fetchMock).toHaveBeenCalledWith("http://localhost:8000/profile", expect.anything()) + }) }) describe("chat request extensions", () => { diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 195f138..11bb9c5 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -49,6 +49,23 @@ export interface HealthResponse { hwinfo?: HwinfoHealth } +export interface ProfileTurn { + wall_s: number + prompt_tokens: number + completion_tokens: number + expert_disk_s: number + expert_wait_s: number + expert_matmul_s: number + attention_s: number + lm_head_s: number + forwards: number +} + +export interface ProfileResponse { + seq: number + turns: ProfileTurn[] +} + export interface TokenUsage { prompt_tokens: number completion_tokens: number @@ -100,6 +117,12 @@ export async function getHealth(baseUrl: string, apiKey = "", signal?: AbortSign return (await response.json()) as HealthResponse } +export async function getProfile(baseUrl: string, apiKey = "", signal?: AbortSignal): Promise { + const response = await fetch(serverEndpoint(baseUrl, "profile"), { headers: headers(apiKey), signal }) + if (!response.ok) throw new Error(await responseError(response)) + return (await response.json()) as ProfileResponse +} + export function extractSSE(buffer: string) { const frames = buffer.split(/\r?\n\r?\n/) const rest = frames.pop() || ""