Merge remote-tracking branch 'origin/dev' into pr437
# Conflicts: # c/colibri.c
This commit is contained in:
JustVugg
@@ -0,0 +1,10 @@
|
||||
BasedOnStyle: LLVM
|
||||
IndentWidth: 4
|
||||
ColumnLimit: 120
|
||||
AllowShortFunctionsOnASingleLine: All
|
||||
AllowShortIfStatementsOnASingleLine: AllIfsAndElse
|
||||
AllowShortLoopsOnASingleLine: true
|
||||
BreakBeforeBraces: Attach
|
||||
PointerAlignment: Right
|
||||
SpaceAfterCStyleCast: false
|
||||
SortIncludes: false
|
||||
@@ -0,0 +1,24 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
|
||||
[*.{c,h,cu}]
|
||||
indent_size = 4
|
||||
|
||||
[*.{ts,tsx,js,json,css}]
|
||||
indent_size = 2
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
|
||||
[*.yml]
|
||||
indent_size = 2
|
||||
|
||||
[*.md]
|
||||
trim_trailing_whitespace = false
|
||||
@@ -0,0 +1,49 @@
|
||||
# CI: run the repo's own dependency-free gate (`make check` = clean + portable
|
||||
# CPU build + C unit suites + Python stdlib tests) on the three claimed
|
||||
# platforms. No model downloads, no CUDA, no external deps — by design (#140).
|
||||
name: check
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, dev]
|
||||
pull_request:
|
||||
branches: [main, dev]
|
||||
|
||||
jobs:
|
||||
linux:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: make check
|
||||
run: make -C c check
|
||||
|
||||
windows:
|
||||
# The job that would have caught #68/#137 pre-merge: native MinGW-w64
|
||||
# (MSYS2/UCRT64), the exact toolchain the README's Windows port targets.
|
||||
runs-on: windows-latest
|
||||
defaults:
|
||||
run:
|
||||
shell: msys2 {0}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
msystem: UCRT64
|
||||
update: false
|
||||
install: >-
|
||||
make
|
||||
mingw-w64-ucrt-x86_64-gcc
|
||||
mingw-w64-ucrt-x86_64-python
|
||||
- name: make check
|
||||
run: make -C c check
|
||||
|
||||
macos:
|
||||
# clang; libomp for the threaded path (Makefile falls back to
|
||||
# single-threaded automatically if it's ever missing).
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: install libomp
|
||||
run: brew install libomp
|
||||
- name: make check
|
||||
run: make -C c check
|
||||
@@ -12,8 +12,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Build glm
|
||||
run: cd c && make glm
|
||||
- name: Build colibri
|
||||
run: cd c && make colibri
|
||||
- name: C test suite
|
||||
run: cd c && make test-c
|
||||
|
||||
@@ -41,6 +41,78 @@ jobs:
|
||||
-Xcompiler=-Wall,-Wextra
|
||||
echo "CUDA syntax check passed"
|
||||
|
||||
windows-cuda-build:
|
||||
# The Windows CUDA path has no coverage anywhere: `engine-cuda-syntax` above
|
||||
# compiles with nvcc's *GCC* host on Linux, and check.yml's windows job is
|
||||
# MinGW/UCRT64 CPU-only by design (#140). But nvcc on Windows requires MSVC
|
||||
# as its host compiler — it does not accept MinGW — so `make cuda-dll` runs a
|
||||
# toolchain nothing else in CI touches. That gap is not theoretical: every
|
||||
# bug in #158 was a *build* failure on this path (MSVC rejects the GCC-style
|
||||
# -Xcompiler=-Wall,-Wextra with "D8021 invalid numeric argument '/Wextra'",
|
||||
# unresolvable CUDA_HOME/NVCC defaults, POSIX setenv in the kernel test), and
|
||||
# #314 was CUDA_HOME with spaces — the layout the CUDA installer ships by
|
||||
# default. Both classes are compile-time and need no GPU to catch.
|
||||
#
|
||||
# Build-only ON PURPOSE: GitHub's hosted runners have no NVIDIA device, so
|
||||
# this job proves the Windows+MSVC CUDA build stays buildable, NOT that the
|
||||
# kernels or the DLL loader behave on real silicon. That still needs hardware
|
||||
# (see #157). Claiming otherwise would be the false confidence the
|
||||
# engine-cuda-syntax comment above already warns about.
|
||||
name: CUDA build (Windows, MSVC host)
|
||||
# windows-2022, NOT windows-latest: the latest image now ships Visual Studio
|
||||
# 18 (MSVC 14.5x), and CUDA's crt/host_config.h hard-errors on any host newer
|
||||
# than VS 2022 ("Only the versions between 2017 and 2022 (inclusive) are
|
||||
# supported"). That is a real constraint for every CUDA user on Windows, not
|
||||
# a CI quirk — pinning tracks what the toolkit actually supports. Revisit when
|
||||
# a CUDA release accepts VS 18; -allow-unsupported-compiler would only mask it.
|
||||
runs-on: windows-2022
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: MSVC environment (puts cl.exe on PATH for nvcc -ccbin)
|
||||
uses: ilammy/msvc-dev-cmd@v1
|
||||
- name: Install CUDA toolkit (compiler only)
|
||||
uses: Jimver/cuda-toolkit@v0.2.19
|
||||
with:
|
||||
# Same pin as engine-cuda-syntax: v0.2.19's version table stops at
|
||||
# 12.6.2. This installs to the default "C:\Program Files\NVIDIA GPU
|
||||
# Computing Toolkit\..." path, so CUDA_HOME is space-bearing here —
|
||||
# which is exactly the #314 regression this job would have caught.
|
||||
cuda: '12.6.2'
|
||||
method: network
|
||||
# cudart as well as nvcc: unlike the Linux job (which only needs to
|
||||
# *compile* backend_cuda.cu), cuda-dll links it, and on Windows the
|
||||
# runtime headers/import lib ship as a separate installer component —
|
||||
# with '["nvcc"]' alone this fails at `#include <cuda_runtime.h>`.
|
||||
sub-packages: '["nvcc", "cudart"]'
|
||||
- uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
msystem: UCRT64
|
||||
update: false
|
||||
# inherit: cl.exe (msvc-dev-cmd) and nvcc (cuda-toolkit) are added to
|
||||
# the *Windows* PATH by the steps above; without inheriting it the
|
||||
# recipe's `command -v` guards fail inside the MSYS2 shell.
|
||||
path-type: inherit
|
||||
install: >-
|
||||
make
|
||||
mingw-w64-ucrt-x86_64-gcc
|
||||
- name: make cuda-dll (nvcc + MSVC host)
|
||||
shell: msys2 {0}
|
||||
run: |
|
||||
cd c
|
||||
# CUDA_ARCH is pinned: the default is `native`, which asks the driver
|
||||
# what card is present — there is none here, so it must be explicit.
|
||||
# sm_80 matches engine-cuda-syntax and is supported by the 12.6 pin.
|
||||
make cuda-dll CUDA_ARCH=sm_80
|
||||
test -f coli_cuda.dll || { echo "cuda-dll reported success but produced no DLL" >&2; exit 1; }
|
||||
echo "coli_cuda.dll built (MSVC host)"
|
||||
- name: make colibri CUDA_DLL=1 (host links backend_loader, not cudart)
|
||||
shell: msys2 {0}
|
||||
run: |
|
||||
cd c
|
||||
make colibri CUDA_DLL=1
|
||||
test -f colibri.exe || { echo "colibri CUDA_DLL=1 reported success but produced no exe" >&2; exit 1; }
|
||||
echo "colibri.exe built against the DLL loader"
|
||||
|
||||
web:
|
||||
name: Web UI
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ['v*']
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
name: linux-x86_64
|
||||
ext: ""
|
||||
make_args: "ARCH=x86-64-v3"
|
||||
- os: macos-latest
|
||||
name: macos-arm64
|
||||
ext: ""
|
||||
make_args: ""
|
||||
- os: windows-latest
|
||||
name: windows-x86_64
|
||||
ext: ".exe"
|
||||
make_args: ""
|
||||
# GitHub shells are format strings: 'msys2 {0}', never bare 'msys2'
|
||||
# (the v1.0.0 tag build failed on exactly this). Same as ci.yml.
|
||||
shell: "msys2 {0}"
|
||||
runs-on: ${{ matrix.os }}
|
||||
defaults:
|
||||
run:
|
||||
shell: ${{ matrix.shell || 'bash' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- if: matrix.os == 'windows-latest'
|
||||
uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
msystem: UCRT64
|
||||
update: false
|
||||
# inherit: the Package step calls the runner's 7z.exe, which lives on
|
||||
# the Windows PATH; the default minimal path would not see it.
|
||||
path-type: inherit
|
||||
install: >-
|
||||
make
|
||||
mingw-w64-ucrt-x86_64-gcc
|
||||
|
||||
- if: matrix.os == 'macos-latest'
|
||||
run: brew install libomp
|
||||
|
||||
- name: Build engine
|
||||
run: |
|
||||
cd c
|
||||
make glm ${{ matrix.make_args }}
|
||||
ls -lh glm${{ matrix.ext }}
|
||||
|
||||
- name: Package
|
||||
run: |
|
||||
TAG=${GITHUB_REF#refs/tags/}
|
||||
mkdir -p dist
|
||||
cp c/glm${{ matrix.ext }} dist/colibri-${TAG}-${{ matrix.name }}${{ matrix.ext }}
|
||||
cp c/coli dist/
|
||||
cp c/version.py dist/
|
||||
cp c/openai_server.py dist/
|
||||
cp c/resource_plan.py dist/
|
||||
cp c/doctor.py dist/
|
||||
cp LICENSE dist/
|
||||
cd dist
|
||||
if [ "${{ matrix.ext }}" = ".exe" ]; then
|
||||
7z a colibri-${TAG}-${{ matrix.name }}.zip *
|
||||
else
|
||||
tar czf colibri-${TAG}-${{ matrix.name }}.tar.gz *
|
||||
fi
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: colibri-${{ matrix.name }}
|
||||
path: dist/colibri-*.*
|
||||
|
||||
release:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
merge-multiple: true
|
||||
|
||||
- name: Create GitHub Release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
TAG=${GITHUB_REF#refs/tags/}
|
||||
# Extract the latest version section from CHANGELOG
|
||||
NOTES=$(awk "/^## \\[${TAG#v}\\]/{found=1;next} /^## \\[/{if(found)exit} found{print}" CHANGELOG.md)
|
||||
if [ -z "$NOTES" ]; then
|
||||
NOTES="Release ${TAG}"
|
||||
fi
|
||||
gh release create "$TAG" artifacts/* \
|
||||
--title "colibrì ${TAG}" \
|
||||
--notes "$NOTES"
|
||||
@@ -10,6 +10,8 @@ desktop/src-tauri/target/
|
||||
desktop/src-tauri/gen/
|
||||
|
||||
# binari compilati (si rigenerano con make / coli build)
|
||||
c/colibri
|
||||
c/colibri.exe
|
||||
c/glm
|
||||
c/glm.exe
|
||||
c/olmoe
|
||||
@@ -68,3 +70,7 @@ c/tests/test_decode_batch
|
||||
c/tests/test_i4_acc512
|
||||
c/tests/test_idot
|
||||
c/tests/test_uring
|
||||
olmoe_merged/
|
||||
olmoe_i4/
|
||||
c/olmoe_merged/
|
||||
c/olmoe_i4/
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to colibrì are documented here.
|
||||
Format follows [Keep a Changelog](https://keepachangelog.com/).
|
||||
|
||||
## [1.0.0] — 2026-07-19
|
||||
|
||||
First tagged release. The engine has been running in production since late June
|
||||
2026; this tag marks the baseline for semantic versioning going forward.
|
||||
|
||||
### Highlights
|
||||
|
||||
- **GLM-5.2 (744B MoE)** runs on ~25 GB RAM in pure C, streaming experts from disk
|
||||
- **Three-tier placement**: VRAM (hot) / RAM (warm) / NVMe (cold), with a learning
|
||||
cache that pins your workload's hottest experts automatically
|
||||
- **CUDA backend**: multi-GPU expert tier, dense tensor distribution, batched
|
||||
ragged attention, resident pipeline (`COLI_CUDA_PIPE=2`)
|
||||
- **Metal backend** (Apple Silicon): batched expert SwiGLU + fused decode attention
|
||||
on unified memory GPU
|
||||
- **MTP speculation**: native GLM-5.2 draft heads, grammar-forced drafts, kernel-
|
||||
pinned verification (`SPEC_PIN=1`)
|
||||
- **OpenAI-compatible API**: `coli serve` with SSE streaming, KV slots, bounded
|
||||
queue, web dashboard (`coli web`)
|
||||
- **Web UI**: chat with live metrics, expert cortex brain page, profiling breakdown,
|
||||
expert atlas 3-D galaxy
|
||||
- **Cross-platform**: Linux, macOS, Windows 11 (native MinGW), PowerPC; CI on all three
|
||||
- **Auto-tune**: `coli plan --auto-tier` classifies the bottleneck and derives
|
||||
MTP/PIPE/NUMA/PIN settings with explanations
|
||||
|
||||
### Engine
|
||||
|
||||
- Token-exact validation against `transformers` oracle (teacher-forcing 32/32)
|
||||
- Compressed MLA KV cache (576 floats/token, 57× smaller), persisted across
|
||||
restarts (`.coli_kv`, zero re-prefill)
|
||||
- DSA sparse attention (lightning indexer), faithfully implemented
|
||||
- Router-lookahead prefetch (`PILOT=1`, 71.6% predictive)
|
||||
- Async expert I/O pool (`PIPE=1`), io_uring batching (`URING=1`)
|
||||
- NUMA-aware expert placement (`COLI_NUMA=1`, +13–40% on multi-socket)
|
||||
- AVX2 / AVX-512 / AVX-VNNI / ARM NEON / NEON-i8mm / POWER VSX kernels
|
||||
- int4 / int8 / int2 / grouped-int4 (fmt=4) quantization formats
|
||||
|
||||
### Tools
|
||||
|
||||
- `coli convert` — FP8→int4 one-shard-at-a-time converter
|
||||
- `coli doctor` — read-only setup diagnostics
|
||||
- `coli plan` — resource planner with auto-tune prescription
|
||||
- `coli bench` — MMLU / HellaSwag / ARC quality benchmarks
|
||||
- Expert atlas (`tools/analyze.py --web`) — measured topic affinity for 19,456 experts
|
||||
|
||||
### Community
|
||||
|
||||
- 30+ hardware datapoints in the benchmark tracker
|
||||
- Contributions from 20+ authors across engine, docs, tooling, and ports
|
||||
+260
@@ -0,0 +1,260 @@
|
||||
<p align="center">
|
||||
<img src="assets/colibri.svg" width="500" alt="colibrì — motore piccolo, modello immenso">
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="README.md">English</a> · <a href="README.zh-CN.md">简体中文</a> · <a href="README.zh-TW.md">繁體中文</a> · Italiano
|
||||
</p>
|
||||
|
||||
**Motore piccolo, modello immenso.** Esegui **GLM-5.2 (744 miliardi di parametri, MoE)** su un computer consumer con ~25 GB di RAM — in C puro, zero dipendenze, caricando gli expert dal disco in streaming.
|
||||
|
||||
Colibrì è un runtime MoE leggero e che preserva la qualità: tratta VRAM, RAM e
|
||||
disco come un'unica gerarchia di memoria gestita. Se la memoria veloce non basta
|
||||
il modello rallenta, ma la policy predefinita **non cambia mai silenziosamente la
|
||||
precisione del modello né la semantica del router**.
|
||||
|
||||
```
|
||||
$ ./coli chat
|
||||
🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU
|
||||
✓ ready in 32s · resident 9.9 GB
|
||||
› ciao!
|
||||
◆ Ciao! 😊 Come posso aiutarti oggi?
|
||||
```
|
||||
|
||||
## Guardalo in azione
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/colibri-dashboard.png" width="900" alt="dashboard web di colibrì — metriche live, pannello hardware, livelli degli expert">
|
||||
</p>
|
||||
<p align="center"><em>La dashboard web (<code>./coli web</code>): un modello da 744B a <strong>4 tok/s, TTFT 1.6 s, disco 0</strong> —
|
||||
residenza completa degli expert su 6× RTX 5090, con metriche token in tempo reale, breakdown dei tempi per turno,
|
||||
la barra dei livelli VRAM/RAM/disco e il mini-cervello live nell'angolo.</em></p>
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/colibri-brain.png" width="900" alt="la pagina Brain — 19.456 expert come una corteccia vivente">
|
||||
</p>
|
||||
<p align="center"><em>La pagina <strong>Brain</strong>: tutti i 19.456 expert come una corteccia vivente — il colore indica
|
||||
il livello di archiviazione, la luminosità il calore di routing, e ogni expert instradato in un turno
|
||||
lampeggia bianco. Passando il cursore si vede l'<a href="https://github.com/JustVugg/colibri/issues/175">affinità
|
||||
tematica misurata</a> dell'expert.</em></p>
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/colibri-atlas.png" width="900" alt="la pagina Atlas — l'atlante misurato degli expert come una galassia 3D">
|
||||
</p>
|
||||
<p align="center"><em>La pagina <strong>Atlas</strong>: l'<a href="https://github.com/JustVugg/colibri/issues/175">atlante
|
||||
misurato degli expert</a> come una galassia 3D — 13.260 expert caratterizzati, 1.041 specialisti
|
||||
replicabili che si raggruppano per argomento (poesia, legge, cinese, SQL…). La posizione deriva
|
||||
dall'affinità di routing misurata, non da un embedding appreso. Trascinare per ruotare.</em></p>
|
||||
|
||||
## L'idea
|
||||
|
||||
Un modello Mixture-of-Experts da 744B attiva solo ~40B parametri per token — e
|
||||
solo ~11 GB di quelli cambiano da un token all'altro (gli expert instradati):
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/sparse.png" width="880" alt="solo ~5.4% dei parametri è attivo per token">
|
||||
</p>
|
||||
|
||||
Il modello non ha bisogno di *stare* in memoria veloce — ha bisogno di essere
|
||||
**piazzato**:
|
||||
|
||||
- la **parte densa** (attenzione, expert condivisi, embedding — ~17B parametri)
|
||||
resta **residente in RAM a int4** (~9.9 GB);
|
||||
- i **19.456 expert instradati** (75 layer MoE × 256 + la testa MTP, ~19 MB
|
||||
ciascuno a int4) stanno **su disco** (~370 GB) e vengono **caricati on demand
|
||||
in streaming**, con una cache LRU per layer, un hot-store pinnato che impara,
|
||||
e un livello VRAM opzionale.
|
||||
|
||||
Il motore è un singolo file C (`c/colibri.c`) più header piccoli. Niente BLAS,
|
||||
niente Python a runtime, niente GPU obbligatoria.
|
||||
|
||||
## Come funziona
|
||||
|
||||
### Il percorso di ogni token
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/token-path.png" width="880" alt="instrada → unione → piazza → sovrapponi → impara">
|
||||
</p>
|
||||
|
||||
Ogni layer di ogni token percorre gli stessi cinque passi. L'obiettivo
|
||||
progettuale è che **il piazzamento decide solo la velocità** — le decisioni
|
||||
del router e la precisione dei pesi sono identiche sia che un expert risponda
|
||||
dalla VRAM sia dal disco.
|
||||
|
||||
### Una gerarchia di memoria, non un requisito di memoria
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/tiers.png" width="880" alt="residenza expert a tre livelli: VRAM / RAM / NVMe">
|
||||
</p>
|
||||
|
||||
Lo stesso motore copre l'intero spettro: su un portatile da 25 GB tutto viene
|
||||
caricato dal disco in streaming (lento, ma corretto); su un host grande l'intero
|
||||
set di expert diventa residente (`CUDA_EXPERT_GB=auto PIN_GB=all`) e il disco
|
||||
esce completamente dal percorso di decode. Tra i livelli c'è una **cache che
|
||||
impara**: il motore registra quali expert il *tuo* carico di lavoro instrada
|
||||
(`.coli_usage`, aggiornato a ogni turno) e fissa automaticamente i più caldi —
|
||||
colibrì diventa letteralmente più veloce man mano che lo usi. Sugli host
|
||||
multi-socket, `COLI_NUMA=1` interlaccia i pesi residenti tra i controller di
|
||||
memoria ([#82](https://github.com/JustVugg/colibri/issues/82)).
|
||||
|
||||
### Mai aspettare il disco due volte
|
||||
|
||||
I miss nella cache costano caro, quindi il motore investe la maggior parte
|
||||
della sua astuzia per evitarli e sovrapporli: le tre matrici di ogni expert sono
|
||||
memorizzate contigue e lette con un unico `pread`; un pool I/O asincrono
|
||||
limitato (`PIPE=1`, attivo per default) carica gli expert mancanti mentre quelli
|
||||
residenti calcolano; le posizioni in batch leggono ogni expert unico una sola
|
||||
volta (**batch-union**); un thread di lookahead del router (`PILOT=1`) fa il
|
||||
prefetch degli expert del layer successivo — il routing è misurabilmente
|
||||
**prevedibile al 71.6% un layer in anticipo**. Sulle GPU, la pipeline residente
|
||||
(`COLI_CUDA_PIPE=2`) mantiene il flusso residuo on-device tra i layer, così il
|
||||
loop CPU degli expert procede senza interruzioni; su Apple Silicon un backend
|
||||
[Metal](docs/metal.md) sperimentale esegue la matmul batch degli expert sulla
|
||||
GPU a memoria unificata.
|
||||
|
||||
### Modello fedele, stato compresso
|
||||
|
||||
Il forward pass è validato **token-esatto contro un oracle `transformers`**
|
||||
(teacher-forcing 32/32). L'attenzione MLA memorizza uno stato KV compresso — 576
|
||||
float/token invece di 32.768 (**57× più piccolo**) — e lo persiste tra i
|
||||
riavvii (`.coli_kv`): le conversazioni riaprono "calde", senza alcun re-prefill,
|
||||
byte-identiche a una sessione ininterrotta. L'attenzione sparsa DSA (il
|
||||
lightning indexer di GLM-5.2) è implementata fedelmente e validata forzando la
|
||||
selezione di tutte le chiavi per riprodurre esattamente l'attenzione densa.
|
||||
|
||||
### Decodifica speculativa, onestamente
|
||||
|
||||
La testa MTP nativa di GLM-5.2 propone token che il modello principale verifica
|
||||
in un unico forward batch — 2.2–2.8 token/forward quando conviene. Due regole
|
||||
conquistate a caro prezzo sono i default: la testa MTP deve essere **int8** (le
|
||||
teste int4 crollano al 0–4% di accettazione,
|
||||
[#8](https://github.com/JustVugg/colibri/issues/8)), e draft e verifica devono
|
||||
calcolare **la stessa funzione** — `SPEC_PIN=1` fissa entrambi sulla stessa
|
||||
famiglia di kernel ([#163](https://github.com/JustVugg/colibri/issues/163)
|
||||
contiene l'intera indagine forense). I draft forzati da grammatica
|
||||
([`GRAMMAR=file.gbnf`](docs/grammar-draft.md)) aggiungono accettazione quasi
|
||||
gratuita sull'output JSON vincolato. Se la speculazione conviene dipende dalla
|
||||
temperatura della cache — misura, e usa `DRAFT=0` quando non paga.
|
||||
|
||||
## Cosa ottiene
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/ladder.png" width="880" alt="velocità di decode misurata per classe hardware">
|
||||
</p>
|
||||
|
||||
Stesso motore, stesso container int4 — cambia solo dove risiedono gli expert.
|
||||
Punti salienti dalle [tabelle benchmark complete](docs/benchmarks.md):
|
||||
|
||||
- **6× RTX 5090, residenza completa:** 5.8–6.8 tok/s in decode, TTFT ~13 s
|
||||
([log dell'esperimento](docs/experiments/glm52-6x5090-2026-07-12.md));
|
||||
- **desktop solo-CPU da 128 GB:** ~1.8 tok/s a cache calda
|
||||
([#200](https://github.com/JustVugg/colibri/issues/200));
|
||||
- **singola RTX 5070 Ti, classe laptop:** 1.07 tok/s tramite la pipeline
|
||||
GPU-residente ([#273](https://github.com/JustVugg/colibri/issues/273));
|
||||
- **macchina di sviluppo da 25 GB:** 0.05–0.1 tok/s a freddo — il punto di
|
||||
partenza dimostrato da cui è nato il progetto, e ancora oggi la baseline onesta.
|
||||
|
||||
La qualità è misurata, non presunta: il costo di quantizzazione del container
|
||||
int4 e le ablazioni su granularità delle scale e rotazione sono in
|
||||
[docs/benchmarks.md](docs/benchmarks.md#quality-benchmark) e
|
||||
[#108](https://github.com/JustVugg/colibri/issues/108)/[#81](https://github.com/JustVugg/colibri/issues/81).
|
||||
|
||||
## Per iniziare
|
||||
|
||||
### 1. Scarica il modello
|
||||
|
||||
Un container **GLM-5.2 int4** pre-convertito è su Hugging Face — **usa la
|
||||
versione con le teste MTP int8**:
|
||||
|
||||
**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp**
|
||||
|
||||
> ⚠️ Il mirror originale contiene teste MTP int4 → accettazione dei draft allo 0%
|
||||
> ([#8](https://github.com/JustVugg/colibri/issues/8)). Verifica la tua versione:
|
||||
> `ls -l <modello>/out-mtp-*` — int8 (corretto) è `3527131672 / 5366238584 / 1065950496`.
|
||||
|
||||
Oppure converti tu stesso dalla sorgente FP8 — un unico comando riprendibile che
|
||||
non richiede mai i 756 GB completi su disco contemporaneamente:
|
||||
|
||||
```bash
|
||||
cd c && ./setup.sh # verifica gcc/OpenMP, compila, autotest
|
||||
./coli convert --model /nvme/glm52_i4 # scarica e converti shard per shard (python, una tantum)
|
||||
```
|
||||
|
||||
### 2. Esegui
|
||||
|
||||
```bash
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli chat # budget RAM, cache e MTP rilevati automaticamente
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli plan # mostra il piazzamento pianificato VRAM/RAM/disco
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli doctor # controllo di idoneità (sola lettura)
|
||||
./coli web --model /nvme/glm52_i4 # API + dashboard web sulla stessa porta
|
||||
./coli serve --model /nvme/glm52_i4 # solo API compatibile OpenAI
|
||||
```
|
||||
|
||||
Il motore a runtime è puro C — python si usa solo per il convertitore (una tantum)
|
||||
e per il gateway API opzionale.
|
||||
|
||||
### 3. Approfondisci
|
||||
|
||||
| argomento | documento |
|
||||
|---|---|
|
||||
| Benchmark, dati dalla comunità, misurazioni di qualità | [docs/benchmarks.md](docs/benchmarks.md) |
|
||||
| Parametri di tuning, policy, cache che impara, prefetch | [docs/tuning.md](docs/tuning.md) |
|
||||
| Build nativa su Windows 11 (con CUDA DLL) | [docs/windows.md](docs/windows.md) |
|
||||
| Backend CUDA, livello expert in VRAM, residenza completa | [docs/cuda.md](docs/cuda.md) |
|
||||
| Backend Metal per Apple Silicon | [docs/metal.md](docs/metal.md) |
|
||||
| API compatibile OpenAI, KV slot, dashboard web | [docs/api.md](docs/api.md) |
|
||||
| Draft forzati da grammatica (output strutturato) | [docs/grammar-draft.md](docs/grammar-draft.md) |
|
||||
| Inventario delle variabili d'ambiente | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) |
|
||||
|
||||
## Sostenere il progetto
|
||||
|
||||
colibrì è nato come progetto di una sola persona su un portatile con 12 core
|
||||
e 25 GB di RAM; oggi i suoi numeri arrivano da una comunità di macchine reali.
|
||||
Se ti è utile:
|
||||
|
||||
- ⭐ metti una stella al repository e condividilo;
|
||||
- 🐛 apri issue con i numeri di benchmark del tuo hardware — i datapoint
|
||||
fanno avanzare questo progetto più di qualsiasi altra cosa;
|
||||
- 💬 contattaci via GitHub issues per sponsorizzare lo sviluppo o donare hardware.
|
||||
|
||||
## Struttura del repository
|
||||
|
||||
```
|
||||
Makefile punto d'ingresso root per build/check
|
||||
c/
|
||||
├── colibri.c motore principale
|
||||
├── quant.h kernel matmul quantizzati (SIMD multi-architettura)
|
||||
├── sample.h campionamento, RNG, set di stop
|
||||
├── kv_persist.h persistenza KV su disco (.coli_kv)
|
||||
├── telemetry.h protocollo dashboard, statistiche, usage
|
||||
├── st.h, tok.h, json.h header di runtime
|
||||
├── backend_cuda.* livello CUDA opzionale
|
||||
├── Makefile build e check locali
|
||||
├── coli CLI utente
|
||||
├── openai_server.py gateway HTTP compatibile OpenAI
|
||||
├── setup.sh setup locale in un solo comando
|
||||
├── tools/ conversione offline, fixture e benchmark
|
||||
├── scripts/ helper per conversioni lunghe
|
||||
└── tests/ test C e Python senza dipendenze
|
||||
web/ UI browser (puro client API OpenAI)
|
||||
desktop/ shell desktop Tauri v2 che racchiude la web UI
|
||||
docs/ documentazione di riferimento, esperimenti, media
|
||||
```
|
||||
|
||||
Il percorso a runtime resta intenzionalmente piatto e leggibile: `colibri.c`
|
||||
più i suoi header. Dalla radice del repository, `make`, `make check` e
|
||||
`make clean` delegano al Makefile del motore.
|
||||
|
||||
## Perché "colibrì"
|
||||
|
||||
Il colibrì pesa pochi grammi, sta sospeso nel vuoto e visita un migliaio di
|
||||
fiori al giorno. Questo motore tiene in vita un gigante da 744 miliardi di
|
||||
parametri con le razioni di un colibrì: 25 GB di RAM, dodici core CPU e
|
||||
tanta pazienza col disco.
|
||||
|
||||
Il nome è rimasto in italiano perché questa è la lingua in cui è stato scritto
|
||||
il primo prototipo — i commenti nel codice lo testimoniano ancora.
|
||||
|
||||
## Licenza
|
||||
|
||||
Apache 2.0. I pesi di GLM-5.2 sono rilasciati da Z.ai sotto licenza MIT.
|
||||
@@ -2,12 +2,16 @@
|
||||
<img src="assets/colibri.svg" width="500" alt="colibrì — tiny engine, immense model">
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
English · <a href="README.zh-CN.md">简体中文</a> · <a href="README.zh-TW.md">繁體中文</a> · <a href="README.it.md">Italiano</a>
|
||||
</p>
|
||||
|
||||
**Tiny engine, immense model.** Run **GLM-5.2 (744B-parameter MoE)** on a consumer machine with ~25 GB of RAM — in pure C, with zero dependencies, by streaming experts from disk.
|
||||
|
||||
Colibrì is a lightweight, quality-preserving MoE runtime that treats VRAM,
|
||||
RAM, and storage as one managed memory hierarchy. Insufficient fast memory may
|
||||
reduce speed, but the default policy never silently changes model precision or
|
||||
router semantics.
|
||||
Colibrì is a lightweight, quality-preserving MoE runtime that treats VRAM, RAM,
|
||||
and storage as one managed memory hierarchy. Insufficient fast memory may reduce
|
||||
speed, but the default policy **never silently changes model precision or router
|
||||
semantics**.
|
||||
|
||||
```
|
||||
$ ./coli chat
|
||||
@@ -17,14 +21,14 @@ $ ./coli chat
|
||||
◆ Ciao! 😊 Come posso aiutarti oggi?
|
||||
```
|
||||
|
||||
|
||||
## See it running
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/colibri-dashboard.png" width="900" alt="colibrì web dashboard — live metrics, hardware panel, expert tiers">
|
||||
</p>
|
||||
<p align="center"><em>The web dashboard (<code>./coli web</code>): a 744B model answering at 4+ tok/s end-to-end on 6× RTX 5090 —
|
||||
with live token metrics, the hardware panel, and the VRAM/RAM/disk expert tiers.</em></p>
|
||||
<p align="center"><em>The web dashboard (<code>./coli web</code>): a 744B model at <strong>4 tok/s, TTFT 1.6 s, disk 0</strong> —
|
||||
full expert residency on 6× RTX 5090, with live token metrics, the per-turn time breakdown,
|
||||
the VRAM/RAM/disk tier bar and the live mini-brain in the corner.</em></p>
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/colibri-brain.png" width="900" alt="the Brain page — 19,456 experts as a live cortex">
|
||||
@@ -33,613 +37,182 @@ with live token metrics, the hardware panel, and the VRAM/RAM/disk expert tiers.
|
||||
brightness is routing heat, and every expert routed in a turn flashes white. Hovering shows the expert's
|
||||
<a href="https://github.com/JustVugg/colibri/issues/175">measured topic affinity</a>.</em></p>
|
||||
|
||||
## Contents
|
||||
|
||||
- [The idea](#the-idea)
|
||||
- [See it running](#see-it-running)
|
||||
- [What's implemented](#whats-implemented)
|
||||
- [Honest numbers](#honest-numbers-wsl2-12-cores-25-gb-ram-nvme-via-vhdx)
|
||||
- [Download the model](#download-the-model)
|
||||
- [Web dashboard](#web-dashboard)
|
||||
- [Got a better machine?](#got-a-better-machine-try-it--heres-what-to-expect)
|
||||
<p align="center">
|
||||
<img src="docs/media/colibri-atlas.png" width="900" alt="the Atlas page — the measured expert atlas as a 3-D galaxy">
|
||||
</p>
|
||||
<p align="center"><em>The <strong>Atlas</strong> page: the <a href="https://github.com/JustVugg/colibri/issues/175">measured expert atlas</a>
|
||||
as a 3-D galaxy — 13,260 characterised experts, 1,041 replicated specialists clustering by topic
|
||||
(poetry, law, Chinese, SQL…). Position is measured routing affinity, not a learned embedding. Drag to spin.</em></p>
|
||||
|
||||
## The idea
|
||||
|
||||
A 744B Mixture-of-Experts model activates only ~40B parameters per token — and only ~11 GB of those change from token to token (the routed experts). So:
|
||||
A 744B Mixture-of-Experts model activates only ~40B parameters per token — and
|
||||
only ~11 GB of those change from token to token (the routed experts):
|
||||
|
||||
- the **dense part** (attention, shared experts, embeddings — ~17B params) stays **resident in RAM at int4** (~9.9 GB);
|
||||
- the **19,456 routed experts** (75 MoE layers × 256 experts + the MTP head, ~19 MB each at int4) live **on disk** (~370 GB) and are **streamed on demand**, with a per-layer LRU cache, an optional pinned hot-store, and the OS page cache as a free L2.
|
||||
<p align="center">
|
||||
<img src="docs/media/sparse.png" width="880" alt="only ~5.4% of parameters are active per token">
|
||||
</p>
|
||||
|
||||
The engine is a single C file (`c/glm.c`) plus small headers. No BLAS, no Python at runtime, no GPU required (an opt-in CUDA tier for pinned experts exists — see below).
|
||||
So the model doesn't need to *fit* in fast memory — it needs to be **placed**:
|
||||
|
||||
## What's implemented
|
||||
- the **dense part** (attention, shared experts, embeddings — ~17B params) stays
|
||||
**resident in RAM at int4** (~9.9 GB);
|
||||
- the **19,456 routed experts** (75 MoE layers × 256 + the MTP head, ~19 MB each
|
||||
at int4) live **on disk** (~370 GB) and are **streamed on demand**, with a
|
||||
per-layer LRU cache, a learned pinned hot-store, and an optional VRAM tier.
|
||||
|
||||
- **Faithful GLM-5.2 (`glm_moe_dsa`) forward** — validated token-exact against a `transformers` oracle (teacher-forcing 32/32, greedy 20/20 on a tiny-random model with the real architecture).
|
||||
- **MLA attention** (q/kv-LoRA, interleaved partial RoPE) with **compressed KV-cache**: 576 floats/token instead of 32,768 (57× smaller — GLM-5.2 has 64 heads and no GQA).
|
||||
- **DeepSeek-V3-style sigmoid router** (noaux_tc, routed_scaling_factor), shared expert, first-3-dense layers.
|
||||
- **Native MTP speculative decoding** — GLM-5.2's own multi-token-prediction head (layer 78) drafts tokens that the main model verifies in one batched forward. **The head must be int8** (the converter does this by default): at int4 draft acceptance collapses to 0–4% and speculation never engages; at int8 it's 39–59% acceptance, **2.2–2.8 tokens/forward** (community-measured, [#8](https://github.com/JustVugg/colibri/issues/8)). Lossless *in exact arithmetic* — but **not byte-identical to non-speculative greedy in practice** ([#100](https://github.com/JustVugg/colibri/issues/100)). This isn't MTP-specific: colibrì's quantized integer kernels are shape-dependent, so any batched (S>1) or GPU forward rounds slightly differently from the single-token path, and int4 GLM-5.2 sits close enough to argmax ties that such a rounding change can flip a token. MTP, the CUDA expert tier, and batched prefill are three different ways to trip the same sensitivity (community-confirmed in #100: swapping only the kernel family forks greedy output on 3/5 prompts, with **zero speculation**). Every emitted token is still the argmax of a *valid* forward — the continuation stays correct — it just isn't the same stream. For byte-exact reproducibility: `DRAFT=0` (no speculation), plus `IDOT=0 COLI_CUDA=0` if you also want kernel-family/GPU independence. Under sampling, rejection sampling keeps the distribution correct. Honest caveat from the same measurement: on a **cold** cache each verified draft routes to extra experts (~660 → ~1100 expert-loads/token), so speculation can be a net *time* loss until the cache/pin warms up.
|
||||
- **Grammar-forced speculative drafts** (`GRAMMAR=file.gbnf`, [#48](https://github.com/JustVugg/colibri/issues/48)) — on constrained-output workloads (JSON/NDJSON, function calling, structured extraction) the grammar itself is a third draft source: wherever it admits exactly **one** legal byte (braces, quotes, key names, enum bodies), that forced span is tokenized and injected as pre-accepted drafts with ~1.0 acceptance — no draft head, no lookup table, and it engages even with the int4 MTP head from [#8](https://github.com/JustVugg/colibri/issues/8). It never constrains sampling: forced spans are verified in the same batch-union forward as any draft, so a wrong or out-of-sync grammar cannot change the output — worst case is rejected drafts, and an adaptive guard turns the source off below 50% acceptance. Byte-level GBNF subset (literals, char classes, `| ( ) ? * +`, comments); `GRAMMAR_DRAFT=n` caps the forced span per forward (default 24). Composes with `DRAFT`/MTP, which fill the free-text gaps between forced spans. Full reference — mechanism, measured A/Bs, when it pays, prior art: [docs/grammar-draft.md](docs/grammar-draft.md).
|
||||
- **True sampling** — temperature + nucleus, defaults tuned for int4 reality (0.7 / 0.90; the official 1.0 / 0.95 samples quantization noise from the tail).
|
||||
- **Integer-dot kernels** (Q8_0-style int8 activations, AVX2 `maddubs`): int8 matmuls 1.4–2.5× faster (119 GFLOP/s measured), int4 1.8× in batch — routing decided per shape by measurement (int4 single-row stays f32: it measured slower).
|
||||
- **MLA weight absorption** (DeepSeek trick) for decode: no per-token k/v reconstruction — the query absorbs `kv_b`, context is projected after attention. Validated exact: TF 32/32 and generation 20/20 with absorption forced everywhere.
|
||||
- **Async expert readahead**: while one block of experts is being multiplied, the kernel is already reading the next (`WILLNEED`).
|
||||
- **Quantization kernels**: int8 / packed int4 / packed int2, per-row scales, AVX2, dequant-on-use. Packing validated bit-identical to the int8 container.
|
||||
- **DSA sparse attention** — GLM-5.2's lightning indexer, faithful to the reference `glm_moe_dsa` modeling: per-layer top-2048 causal key selection (full/shared indexer layers), auto-detected from the `out-idx-*` weights (`--indexer` converter mode, ~189 MB extracted from the FP8 repo). Validated exact: forcing the selection to keep every key reproduces dense attention token-for-token. `DSA=0` disables, `DSA_TOPK` overrides.
|
||||
- **KV-cache persistence** — conversations reopen **warm** across engine restarts: serve mode appends the compressed MLA KV to `.coli_kv` after every turn (~182 KB/token, crash-safe) and resumes it at startup with zero re-prefill. Validated byte-identical to an uninterrupted session. `KVSAVE=0` disables.
|
||||
- **Router-lookahead prefetch** (`PILOT=1`, experimental) — the next layer's routing is 71.6% predictable from the current layer's post-attention state (measured); a dedicated I/O thread prefetches those experts while the current layer computes.
|
||||
- **Batch-union MoE**: in prefill (and MTP verification), each unique expert of the batch is read once and applied to every position that routes to it.
|
||||
- **Byte-level BPE tokenizer in C** (GPT-2-style with Unicode-property regex, 320k merges).
|
||||
- **RAM safety**: the expert cache is auto-sized from `MemAvailable` at startup — an honest peak projection (working set, KV, MTP row, reconstruction buffers) so the kernel OOM-killer never fires.
|
||||
- **Offline FP8→int4 converter** (`c/tools/convert_fp8_to_int4.py`): downloads one shard at a time (~5 GB), dequants (128×128 block scales), requantizes to the engine's container, deletes the shard — the 756 GB FP8 checkpoint never needs to exist on disk at once. Resumable.
|
||||
The engine is a single C file (`c/glm.c`) plus small headers. No BLAS, no Python
|
||||
at runtime, no GPU required.
|
||||
|
||||
## Honest numbers (WSL2, 12 cores, 25 GB RAM, NVMe via VHDX)
|
||||
## How it works
|
||||
|
||||
Detailed GPU experiment: [GLM-5.2 on 6x RTX 5090](docs/experiments/glm52-6x5090-2026-07-12.md) — full expert residency across VRAM+RAM reaches 6.84 tok/s single-request decode.
|
||||
### The per-token path
|
||||
|
||||
| metric | value |
|
||||
|---|---|
|
||||
| model on disk (int4 container) | ~370 GB |
|
||||
| resident RAM (dense, int4) | 9.9 GB |
|
||||
| load time | ~30 s |
|
||||
| peak RSS during chat | ~20 GB (auto-capped) |
|
||||
| cold decode cost | ~11 GB disk reads/token (75 layers × 8 experts) |
|
||||
| disk ceiling (this dev box's drive) | ~1 GB/s → ~0.05–0.1 tok/s cold |
|
||||
| MTP speculation (int8 head) | 2.2–2.8 tok/forward measured ([#8](https://github.com/JustVugg/colibri/issues/8)) |
|
||||
<p align="center">
|
||||
<img src="docs/media/token-path.png" width="880" alt="route → union → place → overlap → learn">
|
||||
</p>
|
||||
|
||||
This is not fast. It is a 744B frontier-class model **answering correctly on a machine that costs less than one H100 fan**. Warm cache, pinned hot experts and MTP push the useful-response latency down considerably; the physics of the disk does the rest.
|
||||
Every layer of every token walks the same five steps. The design goal is that
|
||||
**placement only ever decides speed** — the router's decisions and the weights'
|
||||
precision are the same whether an expert answered from VRAM or from disk.
|
||||
|
||||
### SSD note
|
||||
Cold starts are heavy on random reads (~11 GB/token), but reads don't meaningfully wear an SSD — colibrì's streaming is read-only. The real concerns under heavy use are (1) **swap traffic** if the system runs out of RAM (writes do wear the drive — keep a sane `--ram` budget; colibrì's auto-budget is designed to stay clear of swap) and (2) **sustained thermals**: hours at full read duty cycle will heat cheaper drives. Monitor drive temperature and health.
|
||||
### One memory hierarchy instead of one memory requirement
|
||||
|
||||
## Download the model
|
||||
<p align="center">
|
||||
<img src="docs/media/tiers.png" width="880" alt="VRAM / RAM / NVMe three-tier expert residency">
|
||||
</p>
|
||||
|
||||
A pre-converted **GLM-5.2 int4** model for colibrì is available on Hugging Face — **use the version with the int8 MTP heads** (matey-0's clone):
|
||||
The same engine spans the whole range: on a 25 GB laptop everything streams from
|
||||
disk (slow but correct); on a large host the entire expert set becomes resident
|
||||
(`CUDA_EXPERT_GB=auto PIN_GB=all`) and disk drops out of the decode path
|
||||
entirely. Between the tiers sits a **learning cache**: the engine records which
|
||||
experts *your* workload routes to (`.coli_usage`, updated every turn) and pins
|
||||
the hottest ones automatically — colibrì literally gets faster the more you use
|
||||
it. On multi-socket hosts, `COLI_NUMA=1` interleaves the resident weights across
|
||||
memory controllers ([#82](https://github.com/JustVugg/colibri/issues/82)).
|
||||
|
||||
### Never wait for the disk twice
|
||||
|
||||
Misses are expensive, so the engine spends most of its cleverness avoiding and
|
||||
overlapping them: each expert's three matrices are stored adjacent and read in
|
||||
one `pread`; a bounded async I/O pool (`PIPE=1`, default) loads missing experts
|
||||
while resident ones compute; batched positions read each unique expert once
|
||||
(**batch-union**); and a router-lookahead thread (`PILOT=1`) prefetches the next
|
||||
layer's experts — routing is measurably **71.6% predictable one layer ahead**.
|
||||
On GPUs, the resident pipeline (`COLI_CUDA_PIPE=2`) keeps the residual stream
|
||||
on-device across layers so the CPU expert loop runs uninterrupted; on Apple
|
||||
Silicon an experimental [Metal backend](docs/metal.md) does the batched expert
|
||||
math on the unified-memory GPU.
|
||||
|
||||
### Faithful model, compressed state
|
||||
|
||||
The forward pass is validated **token-exact against a `transformers` oracle**
|
||||
(teacher-forcing 32/32). MLA attention stores a compressed KV state — 576
|
||||
floats/token instead of 32,768 (**57× smaller**) — and persists it across
|
||||
restarts (`.coli_kv`): conversations reopen warm with zero re-prefill,
|
||||
byte-identical to an uninterrupted session. DSA sparse attention (GLM-5.2's
|
||||
lightning indexer) is implemented faithfully and validated by forcing full-key
|
||||
selection to reproduce dense attention exactly.
|
||||
|
||||
### Speculative decoding, honestly
|
||||
|
||||
GLM-5.2's native MTP head drafts tokens that the main model verifies in one
|
||||
batched forward — 2.2–2.8 tokens/forward when it pays. Two hard-won rules ship
|
||||
as defaults: the MTP head must be **int8** (int4 heads collapse to 0–4%
|
||||
acceptance, [#8](https://github.com/JustVugg/colibri/issues/8)), and draft and
|
||||
verify must compute **the same function** — `SPEC_PIN=1` pins both to one
|
||||
kernel family ([#163](https://github.com/JustVugg/colibri/issues/163) is the
|
||||
full forensic story). Grammar-forced drafts
|
||||
([`GRAMMAR=file.gbnf`](docs/grammar-draft.md)) add ~free acceptance on
|
||||
constrained JSON output. Whether speculation is a net win depends on your
|
||||
cache temperature — measure, and use `DRAFT=0` when it doesn't pay.
|
||||
|
||||
## What it achieves
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/ladder.png" width="880" alt="measured decode speed by hardware class">
|
||||
</p>
|
||||
|
||||
Same engine, same int4 container — the hardware only changes where the experts
|
||||
live. Highlights from the [full benchmark tables](docs/benchmarks.md):
|
||||
|
||||
- **6× RTX 5090, full residency:** 5.8–6.8 tok/s decode, TTFT ~13 s
|
||||
([experiment log](docs/experiments/glm52-6x5090-2026-07-12.md));
|
||||
- **128 GB CPU-only desktop:** ~1.8 tok/s warm ([#200](https://github.com/JustVugg/colibri/issues/200));
|
||||
- **single RTX 5070 Ti laptop-class box:** 1.07 tok/s via the GPU-resident
|
||||
pipeline ([#273](https://github.com/JustVugg/colibri/issues/273));
|
||||
- **25 GB dev box:** 0.05–0.1 tok/s cold — the proven floor where this project
|
||||
started, and still the honest baseline.
|
||||
|
||||
Quality is measured, not assumed: the int4 container's quantization cost and the
|
||||
scale-granularity/rotation ablations live in
|
||||
[docs/benchmarks.md](docs/benchmarks.md#quality-benchmark) and
|
||||
[#108](https://github.com/JustVugg/colibri/issues/108)/[#81](https://github.com/JustVugg/colibri/issues/81).
|
||||
|
||||
## Get started
|
||||
|
||||
> **New here?** The [Quick Start guide](docs/quickstart.md) walks through
|
||||
> install → build → model → first chat step by step for Linux, Windows, and
|
||||
> macOS, with copy-paste commands and no assumed background.
|
||||
|
||||
### 1. Get the model
|
||||
|
||||
A pre-converted **GLM-5.2 int4** container is on Hugging Face — **use the
|
||||
version with the int8 MTP heads**:
|
||||
|
||||
**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp**
|
||||
|
||||
> ⚠️ **The MTP head must be int8.** The original mirror ([jlnsrk/GLM-5.2-colibri-int4](https://huggingface.co/jlnsrk/GLM-5.2-colibri-int4)) ships **int4** MTP heads, which give **0% draft acceptance** — speculation silently never engages and you lose the ~2× MTP lever. This is the single most common "why is MTP stuck at 0%?" report ([#8](https://github.com/JustVugg/colibri/issues/8), [#102](https://github.com/JustVugg/colibri/issues/102)). The int8 head gives the measured **39–59% acceptance**. matey-0's clone above is the original int4 model with the three `out-mtp-*` files already swapped to int8 — download that one and you're done.
|
||||
>
|
||||
> Check what you have: `ls -l <model>/out-mtp-*`
|
||||
> · **int8 (correct):** `3527131672 / 5366238584 / 1065950496`
|
||||
> · **int4 (0% acceptance):** `1765523544 / 2686077736 / 536747200` — if you see these, replace just those three files from the int8 mirror.
|
||||
> ⚠️ The original mirror ships int4 MTP heads → 0% draft acceptance
|
||||
> ([#8](https://github.com/JustVugg/colibri/issues/8)). Check yours:
|
||||
> `ls -l <model>/out-mtp-*` — int8 (correct) is `3527131672 / 5366238584 / 1065950496`.
|
||||
|
||||
Download the repository and point `COLI_MODEL` to its directory:
|
||||
Or convert from the FP8 source yourself — one resumable command that never needs
|
||||
the full 756 GB on disk at once:
|
||||
|
||||
```bash
|
||||
COLI_MODEL=/path/to/GLM-5.2-colibri-int4-with-int8-mtp ./coli chat
|
||||
cd c && ./setup.sh # checks gcc/OpenMP, builds, self-tests
|
||||
./coli convert --model /nvme/glm52_i4 # download+convert shard by shard (python, one-time)
|
||||
```
|
||||
|
||||
This skips the FP8 → int4 conversion step entirely. Thanks to DatPat for the original mirror and matey-0 for the int8-head clone.
|
||||
|
||||
### Quick start
|
||||
### 2. Run it
|
||||
|
||||
```bash
|
||||
cd c
|
||||
./setup.sh # checks gcc/OpenMP, builds, self-tests
|
||||
|
||||
# ONE command does everything model-side: downloads GLM-5.2-FP8 shard by shard
|
||||
# (never needs the full 756 GB at once), converts to the int4 container, then
|
||||
# converts the MTP head for speculative decoding. Resumable at any point.
|
||||
# Conversion (only) needs python with: pip install torch safetensors huggingface_hub numpy
|
||||
./coli convert --model /nvme/glm52_i4 # ~400 GB free on a real ext4/NVMe path
|
||||
|
||||
# chat — RAM budget, expert cache and MTP are all detected automatically:
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli chat
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli chat # RAM budget, cache and MTP auto-detected
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli plan # inspect the planned VRAM/RAM/disk placement
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli doctor # read-only readiness check
|
||||
./coli web --model /nvme/glm52_i4 # API + web dashboard on one port
|
||||
./coli serve --model /nvme/glm52_i4 # OpenAI-compatible API only
|
||||
```
|
||||
|
||||
Inspect the planned storage hierarchy before loading the model:
|
||||
The engine at runtime is pure C — python is only used by the one-time converter
|
||||
and the optional API gateway.
|
||||
|
||||
```bash
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli plan
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli plan --gpu 0,1 --ram 128 --vram 48 --json
|
||||
Prefer a `coli` command on your PATH? From a checkout, `pip install -e .`
|
||||
registers it (the engine itself still lives in `c/` — this is an editable
|
||||
install from the clone, not a standalone wheel).
|
||||
|
||||
# apply the bounded plan to the normal runner
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli chat --auto-tier
|
||||
```
|
||||
### 3. Go deeper
|
||||
|
||||
`coli plan` reads only safetensors headers and reports the model's exact dense/expert
|
||||
footprint, runtime RAM reserve, safe expert-cache cap, and bounded VRAM hot tier. Its
|
||||
versioned JSON output is intended to be shared by the CLI, API server, Web UI, and
|
||||
desktop shell; it does not allocate model tensors or start inference.
|
||||
`--auto-tier` applies the same plan to `chat`, `run`, `serve`, and benchmarks. It
|
||||
sets the RAM budget and context immediately; the VRAM tier is enabled only when
|
||||
the current `glm` binary is linked with CUDA. Explicit flags and environment
|
||||
variables keep precedence over automatic values.
|
||||
|
||||
Before loading the model, `coli doctor` performs a read-only readiness check and
|
||||
explains whether the selected Disk/RAM/VRAM placement is runnable:
|
||||
|
||||
```bash
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli doctor
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli doctor --gpu 0 --ram 128 --json
|
||||
```
|
||||
|
||||
Doctor validates the model directory, config, tokenizer, safetensors headers,
|
||||
engine executable, available RAM, requested NVIDIA devices, CUDA linkage, and the
|
||||
same placement budget used by `coli plan`. It never starts `glm`, reads tensor
|
||||
payloads, imports a model framework, or creates a CUDA context. The versioned JSON
|
||||
report uses stable check IDs for automation. Warnings keep exit status 0; missing
|
||||
requirements or an unsafe RAM projection return 1, while invalid CLI values return 2.
|
||||
|
||||
The engine at runtime is pure C — python is only used by the one-time converter.
|
||||
|
||||
### Windows 11 (native, no WSL)
|
||||
|
||||
colibrì builds and runs natively on Windows 11 x86-64 with MinGW-w64. The port adds
|
||||
a `_WIN32` compatibility layer in `c/compat.h` that maps POSIX I/O to the Windows API
|
||||
(pread → ReadFile+OVERLAPPED, posix_fadvise no-op, aligned allocation, MoveFileEx rename,
|
||||
GlobalMemoryStatusEx RAM detection). All platform differences stay in `compat.h`; the
|
||||
engine source is unchanged.
|
||||
|
||||
**Toolchain:** GCC via [winlibs](https://winlibs.com/) or MSYS2 MinGW-w64. Tested with
|
||||
GCC 16.1.0 (x86_64-ucrt-posix-seh).
|
||||
|
||||
```powershell
|
||||
# One-time toolchain install (pick one):
|
||||
scoop install mingw-winlibs # portable, no shell needed
|
||||
# or: pacman -S mingw-w64-x86_64-gcc make # via MSYS2
|
||||
|
||||
# Build (from c/ directory):
|
||||
make glm.exe # GLM-5.2 engine (static, no DLL dependencies)
|
||||
make olmoe.exe # OLMoE engine (same shims)
|
||||
make iobench.exe # disk I/O benchmark
|
||||
make test-c # run C tests
|
||||
make test-python # run Python tests (requires python)
|
||||
|
||||
# AVX-VNNI: Intel Alder Lake+ (and Meteor Lake+) CPUs have a 128-bit int8
|
||||
# dot-product instruction (VPDPBUSD) the engine can use for ~1.3x faster
|
||||
# quantized matmul. The x86-64-v3 default (portable AVX2) compiles it out;
|
||||
# build for THIS machine to enable it:
|
||||
make glm.exe ARCH=native # banner prints "idot: avx-vnni"
|
||||
|
||||
# Verify (tiny model, 2.4 MB):
|
||||
pip install torch transformers safetensors huggingface_hub
|
||||
python tools/make_glm_oracle.py # generate tiny oracle
|
||||
SNAP=./glm_tiny TF=1 ./glm.exe 64 16 16 # expect "32/32 positions"
|
||||
|
||||
# Run with real model:
|
||||
SNAP=D:\glm52_i4 ./glm.exe 64 4 16 # batch inference
|
||||
python coli chat --model D:\glm52_i4 # interactive chat
|
||||
python coli serve --model D:\glm52_i4 # OpenAI-compatible API
|
||||
```
|
||||
|
||||
**Warmup (overnight cache priming):** the engine's expert cache learns from
|
||||
your workload. The included `warmup.ps1` script runs `coli run` in a loop with
|
||||
diverse prompts to build the `.coli_usage` histogram unattended, so the next
|
||||
real session starts with a large, accurate hot-expert pin. Each run saves usage
|
||||
atomically on clean completion.
|
||||
|
||||
```powershell
|
||||
.\warmup.ps1 -Rounds 1 -Ngen 32 # ~60-90 min, durable progress
|
||||
```
|
||||
|
||||
**NVIDIA GPU (optional, via runtime DLL):** on Windows the engine is built with
|
||||
MinGW gcc but CUDA kernels require MSVC + nvcc. The split is clean: build the
|
||||
CUDA backend into a standalone `coli_cuda.dll` (nvcc + MSVC), then the host
|
||||
`glm.exe` loads it at runtime via `LoadLibrary` (`c/backend_loader.c`). The host
|
||||
never links cudart directly; if the DLL is absent the engine falls back to CPU
|
||||
without error.
|
||||
|
||||
```powershell
|
||||
# Prerequisites: CUDA Toolkit + MSVC Build Tools (cl.exe) + nvcc on PATH.
|
||||
# Build the DLL from a shell with the MSVC environment set (vcvars64.bat or
|
||||
# "x64 Native Tools Command Prompt for VS"):
|
||||
make cuda-dll CUDA_HOME="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" CUDA_ARCH=sm_120
|
||||
|
||||
# Build the host with the runtime loader (CUDA_DLL=1 adds -DCOLI_CUDA and
|
||||
# links backend_loader.o instead of cudart):
|
||||
make glm.exe CUDA_DLL=1 ARCH=native
|
||||
|
||||
# Run with the GPU expert tier (8 GB VRAM budget here; scale to your free VRAM):
|
||||
$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_EXPERT_GB="8"
|
||||
python coli chat --model D:\glm52_i4 --topp 0.7
|
||||
```
|
||||
|
||||
The DLL exports 11 `extern "C"` symbols (`coli_cuda_init`, `coli_cuda_matmul`,
|
||||
etc.); `backend_loader.c` resolves them via `GetProcAddress` on first use.
|
||||
`ColiCudaTensor*` is opaque to the host (stored, never dereferenced), so the
|
||||
MSVC-allocated struct is safe across the ABI boundary. `CUDA_ARCH` must match
|
||||
your GPU's compute capability (e.g. `sm_120` for Blackwell / RTX 50-series,
|
||||
`sm_89` for Ada / RTX 40-series).
|
||||
|
||||
**Status:** Phase 1 complete (compiles, correct, static-linked). The Windows
|
||||
GPU tier (runtime `coli_cuda.dll` via `LoadLibrary`) is implemented and
|
||||
verified on RTX 50-series (sm_120). O_DIRECT (Phase 2) and full-model
|
||||
validation against the transformers oracle remain separate workstreams.
|
||||
|
||||
### OpenAI-compatible API
|
||||
|
||||
`coli serve` keeps one model process loaded and exposes a text-only OpenAI-compatible
|
||||
HTTP API. The gateway uses only the Python standard library; inference still runs in
|
||||
the same dependency-free C engine.
|
||||
|
||||
```bash
|
||||
cd c
|
||||
COLI_MODEL=/nvme/glm52_i4 COLI_API_KEY=local-secret ./coli serve \
|
||||
--host 127.0.0.1 --port 8000 --model-id glm-5.2-colibri
|
||||
|
||||
curl http://127.0.0.1:8000/v1/chat/completions \
|
||||
-H 'Authorization: Bearer local-secret' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model": "glm-5.2-colibri",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"stream": true
|
||||
}'
|
||||
```
|
||||
|
||||
Implemented endpoints are `GET /v1/models`, `GET /v1/models/{model}`,
|
||||
`POST /v1/chat/completions`, and legacy `POST /v1/completions`. Chat and
|
||||
completion requests support JSON responses, SSE streaming, usage counts,
|
||||
`max_tokens`/`max_completion_tokens`, `temperature`, and `top_p`. The extension
|
||||
`enable_thinking: true` enables GLM-5.2's reasoning block; the standard
|
||||
`reasoning_effort` field also enables it unless set to `none`.
|
||||
|
||||
The first version is deliberately text-only and serves one generation at a time:
|
||||
the 744B model stays in one persistent process, so concurrent HTTP requests queue
|
||||
instead of loading duplicate model copies. Tools, image/audio input, custom stop
|
||||
sequences, log probabilities, and token penalties return an explicit error rather
|
||||
than being silently ignored. The default bind address is localhost; set
|
||||
`COLI_API_KEY` before exposing the server beyond the machine.
|
||||
|
||||
Browser access from the Vite development server and Tauri local origins is enabled
|
||||
by default. Repeat `--cors-origin https://your-ui.example` to allow another exact
|
||||
origin, or use `--cors-origin '*'` only on a trusted local network.
|
||||
|
||||
The engine owns one mutable KV context, so HTTP generation uses a bounded FIFO
|
||||
admission queue instead of pretending to run unsafe parallel sequences. Configure it
|
||||
with `--max-queue N` (default 8) and `--queue-timeout SECONDS` (default 300), or the
|
||||
`COLI_MAX_QUEUE` / `COLI_QUEUE_TIMEOUT` environment variables. Saturated and timed-out
|
||||
requests receive OpenAI-shaped HTTP 429 errors before streaming headers are sent.
|
||||
`GET /health` exposes active/queued/completed/rejected counters, and successful
|
||||
generation responses include `x-colibri-queue-wait-ms`.
|
||||
|
||||
### Isolated KV contexts
|
||||
|
||||
`coli serve --kv-slots N` allocates up to 16 independent sequence contexts. Requests
|
||||
select one with the optional integer `cache_slot` field; ordinary OpenAI clients omit
|
||||
it and keep the original slot 0 behavior.
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "glm-5.2-colibri",
|
||||
"messages": [{"role": "user", "content": "Continue this conversation"}],
|
||||
"cache_slot": 1
|
||||
}
|
||||
```
|
||||
|
||||
Each slot owns its token history, compressed MLA/DSA KV memory, MTP window, and
|
||||
crash-safe persistence file (`.coli_kv`, `.coli_kv.1`, ...). The engine still executes
|
||||
one sequence at a time; this establishes explicit KV ownership without pretending that
|
||||
threaded HTTP is continuous batching. RAM admission accounts for every configured slot.
|
||||
Use `COLI_KV_SLOTS=N` as the environment equivalent. Start with a small value: at the
|
||||
default 4096-token context, every slot costs hundreds of MB.
|
||||
|
||||
### Experimental Metal backend (Apple Silicon)
|
||||
|
||||
On Apple Silicon the decode profile is matmul-bound, and unified memory removes the
|
||||
PCIe copy tax that keeps CUDA's streaming experts on the CPU — so colibrì has an
|
||||
opt-in Metal backend that runs the **routed-expert SwiGLU (batched, zero-copy from
|
||||
the RAM slabs)**, the **fused decode attention** (full MLA layer in one command
|
||||
buffer, S≤4), and **prefill's large GEMMs** on the GPU. Token-exact vs the CPU path.
|
||||
|
||||
```bash
|
||||
cd c
|
||||
make glm METAL=1 # macOS only; no Xcode needed (shader compiles at runtime)
|
||||
make metal-test # standalone kernel/attention correctness vs CPU reference
|
||||
COLI_METAL=1 COLI_MODEL=/path/glm52_i4 ./coli chat --ram 96
|
||||
```
|
||||
|
||||
Measured on an M4 Max (128 GB, warm cache, MTP on): CPU 0.30 → Metal **0.42 tok/s (~1.4×)**
|
||||
(best config adds `DIRECT=1`; ~3× vs this machine's first cold run).
|
||||
Key design points: Metal's ~5 ms submit latency makes per-matmul dispatch a loss —
|
||||
everything is batched into few command buffers per layer, and the resident experts'
|
||||
GPU work is submitted *before* the missed experts' disk reads so I/O and compute
|
||||
overlap. `COLI_METAL_GEMM_MIN` tunes the prefill GEMM row threshold (default 16).
|
||||
Streaming, cache, MTP, DSA and the persistence formats are unchanged; every GPU
|
||||
path falls back to the CPU per-block on any fault. Numerics are dequant→f32-MAC
|
||||
(same as the CUDA tier); greedy outputs are byte-identical to the CPU engine.
|
||||
|
||||
### Experimental resident CUDA backend
|
||||
|
||||
colibrì includes an opt-in CUDA backend for model-resident tensors. Streaming
|
||||
experts deliberately remain on the original CPU path for now: copying an expert
|
||||
from NVMe to the GPU on every use would only replace the disk bottleneck with a
|
||||
PCIe bottleneck. Resident quantized tensors are uploaded lazily once and reused.
|
||||
|
||||
```bash
|
||||
cd c
|
||||
make cuda-test CUDA=1 # q8/q4/q2/f32 kernel correctness
|
||||
make CUDA=1
|
||||
# optional dense-path experiment (hot experts are configured below)
|
||||
COLI_CUDA=1 COLI_GPU=0 CUDA_DENSE=1 SNAP=/nvme/glm52_i4 ./glm 64 4 4
|
||||
```
|
||||
|
||||
Requirements: Linux, an NVIDIA driver, and a CUDA Toolkit under
|
||||
`/usr/local/cuda` (override with `CUDA_HOME=/path/to/cuda`). `CUDA_ARCH=native`
|
||||
builds for the GPU in the current machine; set an explicit architecture when
|
||||
cross-compiling. Requesting CUDA with a CPU-only binary, an invalid device, or
|
||||
an unavailable runtime fails at startup instead of silently falling back.
|
||||
|
||||
The normal `make` build and runtime behavior are unchanged. CUDA defaults to an
|
||||
expert-only accelerator. `CUDA_DENSE=1` additionally distributes resident
|
||||
dense/attention projection tensors round-robin across the selected devices;
|
||||
their projected footprint is reserved before the expert tier is placed. On six
|
||||
RTX 5090s with a 150 GB expert tier, a warmed two-request/64-token GLM-5.2 run
|
||||
improved from 1.650 to 2.157 aggregate tok/s (+30.8%) while retaining the full
|
||||
expert tier. Treat this as an opt-in until the projected dense set and the 2 GB
|
||||
per-device runtime reserve fit the target GPUs.
|
||||
A measured `PIN` profile can promote its hottest experts into the persistent
|
||||
VRAM tier while keeping the rest in RAM:
|
||||
|
||||
```bash
|
||||
STATS=stats.txt SNAP=/nvme/glm52_i4 ./glm 64 4 4 # collect routing frequencies first
|
||||
COLI_CUDA=1 COLI_GPU=0 CUDA_EXPERT_GB=16 \
|
||||
PIN=stats.txt PIN_GB=160 SNAP=/nvme/glm52_i4 ./glm 64 4 4
|
||||
# multi-GPU expert tier, 150 GB total budget across six 32 GB devices
|
||||
COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=150 \
|
||||
CUDA_DENSE=1 PIN=stats.txt PIN_GB=300 RAM_GB=226 \
|
||||
SNAP=/nvme/glm52_i4 ./glm 64 4 4
|
||||
# large-RAM host: fill safe VRAM, then keep every remaining expert in RAM
|
||||
COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=auto \
|
||||
CUDA_DENSE=1 COLI_CUDA_ATTN=1 PIN=stats.txt PIN_GB=all RAM_GB=auto \
|
||||
SNAP=/nvme/glm52_i4 ./glm 64 4 4
|
||||
```
|
||||
|
||||
Selected experts are uploaded during startup, so capacity failures occur before
|
||||
inference and the log reports their exact tensor footprint. The budget is clamped
|
||||
against free VRAM after reserving the projected dense resident set and 2 GB of
|
||||
runtime headroom per selected device. With `COLI_GPUS`, `CUDA_EXPERT_GB` is a
|
||||
total budget across the device set; experts are assigned whole to the
|
||||
least-loaded device that can hold them. Multi-GPU runs also default to
|
||||
`PIN_FILL=1`: the measured hot set is placed first, then unused VRAM is filled
|
||||
with zero-heat experts. `CUDA_RELEASE_HOST=1` (the multi-GPU default) releases
|
||||
the RAM copy after a successful upload and reloads it from disk only if CUDA
|
||||
later fails. Set either variable to `0` to restore the conservative behavior.
|
||||
When host backing is released, placement is disjoint and staged: the hottest
|
||||
prefix is loaded, uploaded to VRAM, and freed before the next-ranked suffix is
|
||||
loaded into RAM. `PIN_GB` therefore describes the combined ranked set rather
|
||||
than duplicate RAM and VRAM copies. On a 256 GB dual-socket host, moving from a
|
||||
150 GB VRAM + 130 GB RAM placement to 150 GB VRAM + 150 GB RAM raised fixed-token
|
||||
replay from 1.87 to 2.16 tok/s (+15.7%), reduced expert disk wait from 5.144s to
|
||||
3.948s, and kept the projected RAM peak below `RAM_GB=226`. The cache cap adjusts
|
||||
down automatically (54 to 40 in that run) so the larger pinned tier does not exceed
|
||||
the process budget. Start lower on hosts with less available RAM.
|
||||
|
||||
`CUDA_EXPERT_GB=auto` fills each selected device only up to its measured free
|
||||
memory minus projected dense tensors and 2 GB of runtime headroom. `PIN_GB=all`
|
||||
then loads every remaining routed expert into RAM, eliminating decode-time disk
|
||||
misses when the host budget permits it. The regular `RAM_GB` guard still clamps
|
||||
the per-layer working cache and rejects unsafe projections; this mode is intended
|
||||
for dedicated high-memory inference hosts, not desktops running other workloads.
|
||||
On a dedicated 251 GiB host with six RTX 5090s, this mode selected a 176.7 GB
|
||||
VRAM expert tier and a 191.3 GB RAM tier (all 19,456 experts resident). The
|
||||
mode also adapts the VRAM tier every 16 emitted tokens by swapping hot RAM
|
||||
experts into existing GPU slots. A real 64-token greedy GLM-5.2 generation
|
||||
measured **6.00 tok/s decode**, up from
|
||||
2.20 tok/s end-to-end with the earlier 150 GB tier; expert hit rate was 100%
|
||||
and disk wait was zero. Prompt prefill is reported separately. This is a
|
||||
host-specific capacity result, not a portable default.
|
||||
|
||||
Text-mode timing reports prefill separately from decode. The decode rate starts
|
||||
after the prompt KV is built, so it is comparable to `REPLAY` throughput without
|
||||
hiding time-to-first-token.
|
||||
MTP speculation defaults off on CUDA because cold draft routes increase expert
|
||||
traffic; an explicit `DRAFT=n` still overrides the default.
|
||||
|
||||
On six RTX 5090 32 GB cards with GLM-5.2 int4, a 150 GB hot-first tier sustained
|
||||
0.94 token/s over a 64-token varied prompt (87.8% expert hit rate), and reached
|
||||
1.64 token/s on a warmed short prompt (99.3% hit rate). The same capacity filled
|
||||
without routing heat managed only 0.29 token/s, so profile quality matters more
|
||||
than raw VRAM capacity. These are single-run engineering measurements, not a
|
||||
portable performance guarantee.
|
||||
|
||||
Current limitations: devices use independent contexts and synchronous
|
||||
host-staged activation copies—there is no P2P/NCCL dependency yet. Independent
|
||||
expert groups execute concurrently across devices, but a single expert is not
|
||||
sharded. The kernels are correctness-first custom kernels rather than
|
||||
cuBLAS/Tensor Core kernels.
|
||||
|
||||
For a reproducible backend A/B without the full checkpoint, generate the
|
||||
deterministic 313M-parameter `glm_moe_dsa` fixture and run fixed-token replay:
|
||||
|
||||
```bash
|
||||
cd c
|
||||
python tools/make_glm_bench_model.py --output /nvme/colibri-bench-medium --device cuda
|
||||
python tools/benchmark_cuda_fixture.py --model /nvme/colibri-bench-medium --gpu 0
|
||||
```
|
||||
|
||||
The fixture has random weights and is not a language model. It exists only to
|
||||
preserve the real MLA/MoE/streaming shapes and compare CPU streaming, dense-only
|
||||
CUDA, CPU hot-store, and CUDA hot-expert execution with identical replay tokens.
|
||||
|
||||
### Web interface
|
||||
|
||||
`web/` contains a community-contributed browser UI (React + TypeScript, a pure
|
||||
API client — it never touches the engine directly):
|
||||
|
||||
```bash
|
||||
cd web
|
||||
npm ci && npm run dev # then point it at an OpenAI-compatible endpoint
|
||||
```
|
||||
|
||||
It speaks the standard OpenAI Chat Completions protocol with SSE streaming, so it
|
||||
works against the colibrì OpenAI-compatible server (in review, #21) or any other
|
||||
compatible endpoint. Nothing leaves the endpoint you configure. The terminal
|
||||
`coli chat` remains the first-class interface.
|
||||
|
||||
Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens per answer (`:more` in chat continues a truncated one), `--repin N` adapt RAM/VRAM hot experts every N emitted tokens, `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `GRAMMAR=g.gbnf` grammar-forced drafts for constrained JSON/NDJSON output (`GRAMMAR_DRAFT=n` caps the forced span), `TF=1` teacher-forcing validation, `PILOT=1` router-lookahead disk prefetch (experimental — see below), `URING=1` Linux-only batched expert I/O (implies `PIPE=1`; also batches `PILOT_REAL`), `PIPE=0` disable the async expert-load pool (**default ON on Windows** — overlaps expert `pread` with the matmul so the CPU isn't idle waiting on the SSD; measured −18% disk service time), `RAM_GB=<n>` claim more RAM for the expert cache than the conservative auto-detect (e.g. `RAM_GB=31` on a 32 GB host raises the cache cap and hit rate measurably), `CAP_RAISE=0` don't auto-grow the expert cache.
|
||||
|
||||
### Resource policy
|
||||
|
||||
`coli plan` reports the planned hot (VRAM), warm (RAM), and cold backing
|
||||
(disk) tiers, the reason for each placement, and the expected bottleneck. The
|
||||
default `--policy quality` and `--policy balanced` modes preserve checkpoint
|
||||
quantization and router decisions unless `--topk` or `--topp` is passed; those
|
||||
explicit lossy overrides print a warning and proceed.
|
||||
|
||||
Auto-tier plans size OpenMP from physical cores and bind workers across cores.
|
||||
Memory-bound quantized kernels can regress sharply when SMT siblings compete
|
||||
for limited memory channels; explicit `OMP_*` settings always take precedence.
|
||||
|
||||
```bash
|
||||
coli plan --model /models/glm52_i4 --policy quality
|
||||
coli run --auto-tier --policy quality "Explain MoE offloading"
|
||||
# Explicit research-only router reduction:
|
||||
coli run --policy experimental-fast --topk 4 "Benchmark prompt"
|
||||
```
|
||||
|
||||
Disk is an immutable recovery source, not a normal decode target. If the plan
|
||||
leaves cold expert bytes on disk, speed depends on cache hit rate; output
|
||||
quality does not.
|
||||
|
||||
Cold expert reads can use a deferred pipeline: resident RAM/VRAM experts execute
|
||||
while missing experts are loaded in a bounded background I/O pool, then the
|
||||
cold results join before the layer completes. The pool engages only under
|
||||
`PIPE=1`; `PIPE_WORKERS=n` sets its worker count (default 8). Profiling reports
|
||||
both disk service time and the smaller foreground-visible wait time so overlap
|
||||
is explicit rather than credited as unexplained speedup.
|
||||
|
||||
`--policy balanced` enables lossless live placement (`REPIN=64`). At safe
|
||||
request boundaries, a per-layer LFRU score combines decaying session frequency
|
||||
with recent access and replaces at most four sufficiently colder pinned
|
||||
experts. `--policy quality` leaves live replacement off by default; `REPIN=0`
|
||||
always disables it. Persistent `.coli_usage` history and session-local LFRU
|
||||
state remain separate.
|
||||
|
||||
For single-token q4 CPU experts, gate and up projections share one OpenMP
|
||||
dispatch while retaining the same per-row AVX2/NEON arithmetic. This removes
|
||||
one thread-team launch per RAM expert without activation requantization or a
|
||||
lower-precision fallback. It is a stepping stone toward a persistent native
|
||||
CPU expert pool, not a replacement for one.
|
||||
|
||||
**The expert cache auto-sizes to your RAM** (since 2026-07-10): the engine now *raises* the LRU cap to fill your `--ram` budget instead of only lowering it. Before this fix a 128 GB machine ran with the same 8-experts/layer cache as a 16 GB one (issue #12) — **if you benchmarked colibrì before this date, rerun: your numbers were capped.**
|
||||
|
||||
**Router-lookahead prefetch** (`PILOT=1`, experimental): GLM-5.2's expert routing is measurably predictable *ahead of time* — applying layer L+1's router to layer L's post-attention state recalls **71.6%** of the true top-8 (vs 41.3% for "same experts as last token"). `PILOT=1` uses this to issue next-layer expert readahead from a dedicated I/O thread while the current layer computes. On our dev box the disk is already ~80% saturated, so it measures neutral; on machines where compute and disk are balanced (like the Ryzen AI 9 in issue #12: 43% disk / 46% matmul) it should overlap real work — measurements welcome.
|
||||
|
||||
**The learning cache**: the engine records which experts your usage actually routes to (`.coli_usage` next to the model, updated every turn) and at startup automatically pins the hottest ones in spare RAM. colibrì literally gets faster the more you use it.
|
||||
|
||||
**Live tier adaptation** (`--repin N`, opt-in): at safe turn boundaries, a decaying
|
||||
session heat map replaces cold pinned experts with hotter streamed experts. Replacement
|
||||
loads the expert from disk into the existing RAM slot; GPU-backed slots immediately
|
||||
refresh the same VRAM tier budget. A 25% hysteresis and a four-swap limit prevent tier
|
||||
thrashing. Persistent `.coli_usage` remains the long-term signal and is not decayed.
|
||||
|
||||
**Conversations reopen warm** (`.coli_kv`, since 2026-07-10): `coli chat` persists the compressed MLA KV-cache to disk after every turn (~182 KB/token, appended incrementally, crash-safe). Close the chat, reopen it tomorrow — the model still remembers the whole conversation and **zero re-prefill happens**: validated byte-identical to an uninterrupted session. `:reset` clears it, `KVSAVE=0` disables it.
|
||||
|
||||
## Web dashboard
|
||||
|
||||
One command serves the OpenAI-compatible API **and** the web console on the same port, then opens your browser when the engine is ready:
|
||||
|
||||
```bash
|
||||
cd web && npm install && npm run build # once
|
||||
./coli web --model <model-dir>
|
||||
```
|
||||
|
||||
What you get:
|
||||
|
||||
- **Chat** with live metrics: a flashing token counter while generating, then tok/s, time-to-first-token, prompt→completion counts and queue wait;
|
||||
- **Runtime panel**: your hardware (CPU, GPUs + VRAM, RAM, cores), the scheduler, and the live expert-tier bar — how many of the 19,456 experts sit in VRAM / RAM / disk right now;
|
||||
- **Brain**: the whole model as a 76×256 cortex, one cell per expert. Colour = tier, brightness = routing heat, and the experts routed in each turn flash white and decay — you watch the model think. Hover any cell for its tier, heat and [measured topic affinity](https://github.com/JustVugg/colibri/issues/175) (specialists for code, Chinese, math, law… live in layers 11–22).
|
||||
|
||||
The dashboard talks to the engine over two tiny protocol lines (`TIERS`, `EMAP`/`HITS`) and plain JSON endpoints — nothing heavier than the engine itself.
|
||||
|
||||
## Got a better machine? Try it — here's what to expect
|
||||
|
||||
colibrì was built on deliberately humble hardware (12 cores, 25 GB RAM, an older DRAM-less NVMe behind a WSL2 VHDX that measured ~1 GB/s random on *this* drive — note WSL2 VHDX is not inherently slow: a community 5090 box measured 10.5 GB/s O_DIRECT through one, [#101](https://github.com/JustVugg/colibri/issues/101)). **Every one of those constraints is a knob your machine can turn up.** The engine needs: Linux (or WSL2), macOS, or **Windows 11 natively (MinGW-w64)**; gcc with OpenMP, AVX2, ≥16 GB RAM, and the ~370 GB int4 model on a local NVMe (ext4/NTFS — never a network/9p mount).
|
||||
|
||||
**How to test it, in order:**
|
||||
|
||||
```bash
|
||||
cd c && ./setup.sh # build + architecture self-test (expects 32/32)
|
||||
|
||||
# 1) measure YOUR disk the way the engine uses it (parallel 19 MB random reads):
|
||||
gcc -O2 -fopenmp iobench.c -o iobench
|
||||
./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 0 # buffered, 8 threads
|
||||
./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 1 # O_DIRECT (bypass cache)
|
||||
# Caveat (#86): iobench reads a bounded ~1 GB shard, so buffered reads on a big-RAM box
|
||||
# report the PAGE CACHE, not the disk. Use the O_DIRECT run (arg 1) for a true number, and
|
||||
# run it on a shard you haven't touched this session (a prior buffered run caches its pages).
|
||||
# On macOS there is no O_DIRECT — iobench uses F_NOCACHE, which stops *new* caching but can't
|
||||
# evict pages a prior buffered run already resident-mapped, so a macOS "O_DIRECT" figure right
|
||||
# after a buffered run still reads cache. Reboot or use a fresh shard for a real cold read.
|
||||
|
||||
# 2) chat; watch the per-turn stats line (tok/s, expert hit-rate, RSS):
|
||||
COLI_MODEL=/path/to/glm52_i4 ./coli chat
|
||||
|
||||
# 3) record expert usage, then pin the hottest experts in your spare RAM:
|
||||
STATS=stats.txt ./coli chat
|
||||
PIN=stats.txt PIN_GB=20 ./coli chat # scale PIN_GB to your free RAM
|
||||
|
||||
# 4) quality benchmarks (MMLU/HellaSwag/ARC):
|
||||
./coli bench
|
||||
```
|
||||
|
||||
**Back-of-envelope predictions** (decode is disk-bound: a cold token costs ~11.4 GB of expert reads; MTP speculation roughly halves the effective cost *once the cache is warm*; RAM turns cold reads into free cache hits):
|
||||
|
||||
| machine | expected |
|
||||
| topic | doc |
|
||||
|---|---|
|
||||
| this dev box (WSL2 VHDX, ~1 GB/s, 25 GB RAM) | ~0.05–0.1 tok/s cold — proven baseline |
|
||||
| native Linux, PCIe4 NVMe (~3–5 GB/s random), 32 GB | ~0.5–1 tok/s |
|
||||
| PCIe5 NVMe or 2×NVMe RAID0 (~8–12 GB/s), 64 GB (PIN ~40 GB of hot experts) | ~2–4 tok/s |
|
||||
| 128–256 GB RAM, 12 cores (hot experts cached) | ~2–4 tok/s — matmul-bound: ~80 GFLOP/token vs ~250 GFLOP/s of our AVX2 kernels |
|
||||
| same RAM + 24–32 cores, or AVX-512/VNNI kernels | ~5–15 tok/s — interactive; kernel work is the multiplier |
|
||||
|
||||
These are estimates, not measurements — if you run colibrì on serious hardware, **please open an issue with your numbers**: real datapoints from better machines are exactly what this project needs next.
|
||||
|
||||
### Community benchmarks (measured)
|
||||
|
||||
Real numbers from real machines, stock build (`setup.sh`, gcc 13), greedy decoding, `--ngen 32`, MTP active:
|
||||
|
||||
| machine | disk (iobench, 19 MB × 64, 8 threads) | config | measured |
|
||||
|---|---|---|---|
|
||||
| Intel Core Ultra 7 270K Plus (24 threads) · WSL2 · 24 GB RAM · NVMe VHDX ([#2](https://github.com/JustVugg/colibri/issues/2)) | 1.96 GB/s buffered · 2.74 GB/s O_DIRECT | default | 0.07 tok/s · expert hit 3–4% · RSS 14.1 GB |
|
||||
| 〃 | 〃 | `--topp 0.7` | **0.11 tok/s** · expert hit 11% · RSS 14.7 GB |
|
||||
| Apple M5 Max (18 cores) · macOS · 128 GB unified · internal SSD ([#4](https://github.com/JustVugg/colibri/issues/4), [#5](https://github.com/JustVugg/colibri/issues/5)) | ~4 GB/s cold (the 14.2 GB/s reading was cache-influenced — see note) | default, MTP off | **1.06 tok/s** · expert hit 23% · RSS 21.8 GB |
|
||||
| Apple M5 Max · macOS · 128 GB unified · 2 TB SSD · **Metal backend** ([#72](https://github.com/JustVugg/colibri/pull/72), [#87](https://github.com/JustVugg/colibri/issues/87)) | (macOS O_DIRECT figure unreliable — see note) | Metal on · `--ram 96` · 39.7 GB warm pin · MTP off | **1.83 tok/s** · expert hit 66% · warmed 1.11 → 1.83 over the run |
|
||||
| 〃 · 46.9 GB pin (2.94M-selection history) · `--ram 110`, 1024-token run ([#103](https://github.com/JustVugg/colibri/issues/103)) | 〃 | Metal on (experts + attention) · MTP off | **2.06 tok/s** · hit 72.5% · coherent output · fastest datapoint yet (still on the pre-rebase Metal branch) |
|
||||
| Mac Mini M4 Pro · macOS · **48 GB** unified · **Metal backend** ([#107](https://github.com/JustVugg/colibri/issues/107)) | 6.59 GB/s F_NOCACHE (fresh shard) | Metal on · `--ram 38` | **0.30 tok/s** (vs 0.18 CPU-only) — entry Apple Silicon on a third the RAM beats the 32-core 9950X row |
|
||||
| Epyc 9654 ES · Linux · 4x16GB DDR5-4800-rdimm · Samsung PCIe Gen3 x4 NVME SSD | — | `MTP=1 DIRECT=1` | 0.31 tok/s · expert hit 35% · RSS 21.52 GB |
|
||||
| Ryzen AI 9 HX 370 (Framework 13) · Arch Linux · 128 GB · WD SN850X, BTRFS zstd ([#12](https://github.com/JustVugg/colibri/issues/12)) | — | int8 MTP head · `--cap 32` · 46.7 GB auto-learned PIN | **0.37 tok/s** · expert hit 66% · MTP acceptance 52% (2.59 tok/fw) · RSS 105 GB |
|
||||
| Ryzen 9 9950X (32 threads) · Linux · 123 GB · Crucial P3 QLC Gen3 ([#31](https://github.com/JustVugg/colibri/issues/31)) | 1.51 GB/s buffered | default, 2 runs from cold | 0.10 tok/s · hit 53% · profile 66% disk |
|
||||
| 〃 same machine, model moved to a Samsung 9100 PRO PCIe 5.0 ([#31](https://github.com/JustVugg/colibri/issues/31)) | **8.81 GB/s** O_DIRECT | 〃 (usage history retained) | **0.28 tok/s** · hit 57% · profile flips: 32% disk / **57% matmul** |
|
||||
| Ryzen AI Max+ 395 (Framework Desktop) · Ubuntu · 128 GB LPDDR5x · Intel Optane 905p PCIe 3.0 ([#39](https://github.com/JustVugg/colibri/issues/39)) | 3.27 GB/s buffered | int8 MTP head · fresh history (pure LRU, auto-raised cap 65) | 0.16 tok/s · hit 57% · profile 49% disk / 47% matmul |
|
||||
| 〃 five runs later — learned pin 47.6 GB ([#39](https://github.com/JustVugg/colibri/issues/39)) | 〃 | `--temp 0.7 --topp 0.7` | **0.40 tok/s** · hit 71% · fastest non-Apple datapoint |
|
||||
| Ryzen 7 9800X3D (16T) · WSL2 · 70 GB RAM · Samsung 9100 PRO PCIe 5.0 · RTX 5090 ([#101](https://github.com/JustVugg/colibri/issues/101)) | **10.51 GB/s** O_DIRECT | MTP off · learned pin 24 GB · hit 54% · OMP hot-team on | **0.41 tok/s** · disk-bound (36.5 s disk vs 24.0 s matmul) · **CUDA expert tier ≈ 0%** (AVX-512 CPU matches the 5090) · `--topp 0.7` → **0.52 tok/s** |
|
||||
| EPYC 7443 (24C/48T, Zen3 AVX2) · Linux · **430 GB RAM** · NVMe RAID-Z1 via TrueNAS VM ([#104](https://github.com/JustVugg/colibri/issues/104)) | ~1 GB/s (VM overhead) | 77.5 GB pin · cap auto-raised to 194/layer · MTP off | **1.00 tok/s** · **hit 98%** · disk eliminated → **RAM-bandwidth + matmul bound** (no AVX-512/VNNI on Zen3) |
|
||||
| Intel i5-12600K (10C/16T, AVX2) · **native Windows 11, no WSL** · 32 GB · MinGW GCC 16.1 ([#113](https://github.com/JustVugg/colibri/issues/113)) | buffered (no O_DIRECT on MinGW) | int8 MTP head · cold, small-RAM (cap ~2/layer) | **0.08 tok/s** · hit 3.7% · **MTP 57% acceptance** — first native-Windows datapoint, port validated |
|
||||
| Ryzen 9 9950X3D2 (16C/32T, avx512-vnni) · native Linux · 121 GB · Samsung 9100 PRO **PCIe Gen5** · RTX 5090 (28 GB expert tier, 1475 pinned) ([#120](https://github.com/JustVugg/colibri/issues/120)) | **11.48 GB/s** O_DIRECT | `MTP=0 DIRECT=1 PIPE_WORKERS=16 PREFETCH=1` | **1.23 tok/s** · MTP-off wins disk-bound · fastest x86 datapoint yet |
|
||||
| Ryzen AI Max+ 395 (Strix Halo, 16C/32T Zen5, avx512-vnni) · Arch Linux · 128 GB unified LPDDR5x · SK hynix P41 PCIe 4.0 ([#124](https://github.com/JustVugg/colibri/issues/124)) | — | `DIRECT=1 PIPE=1 --topp 0.7` · auto-pin | 0.06 cold → **1.10 tok/s** sustained · first Strix Halo / gfx1151 datapoint (unified memory: no discrete VRAM tier) |
|
||||
| Intel Core Ultra 9 185H (16C/22T, avx-vnni) · **native Windows 11, no WSL** · 32 GB · Crucial P3 QLC NTFS · RTX 5070 Ti (unused) ([#128](https://github.com/JustVugg/colibri/issues/128)) | — | int8 MTP head · **with [#131](https://github.com/JustVugg/colibri/pull/131) (pipe + RAM fixes), warm cache, no GPU** | 0.03 cold → **0.5 tok/s** warm (~7-prompt warmup) · cache-warming on native Windows once the portability blockers are fixed — stock main hung on the `\r\n` READY sentinel before #131 |
|
||||
| Dell Pro Max GB10 (DGX Spark: Grace 10×X925 + 10×A725, **aarch64 i8mm/sve2**) · Linux · 121 GB unified LPDDR5x · Dell OEM 4 TB NVMe · GB10 sm_121 ([#136](https://github.com/JustVugg/colibri/issues/136)) | **5.58 GB/s** O_DIRECT (NVIDIA-OEM unit in #76 was 10.74 — same platform, different SSD) | int8 MTP head · warm cache | 0.21 cold → **0.50 tok/s** warm · hit 83% · MTP 73% (3.20 tok/fw) · **matmul-bound** (matmul 130 s vs disk 58 s) — unified memory, CUDA placement tier neutral; the lever here is an i8mm compute kernel, not placement |
|
||||
|
||||
Takeaways: with 24 GB of RAM the engine auto-caps the expert cache to 2 slots/layer, so decode stays cold even on a disk 2–2.7× faster than the dev box — **on small-RAM machines the RAM cap, not the disk, is the binding constraint**, exactly as the table above predicts; `--topp 0.7` alone bought a clean 1.6× end-to-end speedup. The M5 Max datapoint lands right on the table's second row: **~1 tok/s of a 744B model on a laptop SSD** — and its 14 GB/s disk shifts the bottleneck back to RAM budget and kernels. The Framework 13 rows are the cache thesis proven end-to-end on one machine: 0.29 → 0.37 tok/s (hit 28% → 66%, speculation finally engaging at 52% acceptance) just by giving the cache its RAM — int8 MTP head + a bigger cap + the learned pin. The cap part is now automatic (cap auto-raise, 2026-07-10). The 9950X pair is the cleanest bottleneck experiment yet — same machine, same history, only the disk swapped: ×5.8 disk bandwidth bought ×2.9 tokens, and the profile **flipped from 66% disk to 57% matmul**. But the crossover depends on the CPU kernel: the 9800X3D row ([#101](https://github.com/JustVugg/colibri/issues/101)) shows that with the OMP hot-team tuning on, the AVX-512 CPU matmul is fast enough that even a **10 GB/s NVMe stays disk-bound** — and there the **CUDA expert tier buys ≈ 0%**, because the CPU already matches the 5090 on expert matmul. The GPU tier earns its VRAM only when the CPU is the weak link, not by default. (Honest correction from #101: an earlier version of that report ran with the OMP tuning off, which manufactured a false matmul-bound crossover and a false +14% for CUDA — neither survived a clean re-run.)
|
||||
|
||||
## Quality benchmark — help wanted
|
||||
|
||||
**First measurement is in** ([#108](https://github.com/JustVugg/colibri/issues/108), thanks dnnspaul): the int4 container scored **62.5% mean acc_norm** on hellaswag/arc/mmlu (0-shot log-likelihood, n=40) — below the 85–95% published for full-precision GLM-5.2, but **the gap is not yet attributable to quantization.** Two confounds sit in the way: (1) 0-shot log-likelihood MC scoring badly underserves a *reasoning* model like GLM-5.2 (it never gets to think), so a large gap is expected even at fp16; (2) n=40 is ±14pp. The **decisive experiment** is the OLMoE fp16-vs-int4 A/B under this same harness (small enough to run both precisions) — that delta *is* the quantization cost with the scoring protocol cancelled out. Until it's run, 62.5% is a datapoint, not a verdict.
|
||||
|
||||
The code is here and ready; one command runs it end to end (it auto-downloads the datasets on first use):
|
||||
|
||||
```bash
|
||||
cd c
|
||||
pip install tokenizers datasets # in addition to the convert deps above
|
||||
./coli bench # hellaswag, arc_challenge, mmlu — 40 questions each
|
||||
./coli bench hellaswag --limit 200 # one task, more questions
|
||||
./coli bench mmlu arc_challenge --ram 100 # pick tasks, set a RAM budget
|
||||
```
|
||||
|
||||
It prints per-task accuracy (log-likelihood scoring, EleutherAI-harness style). **If you can run the OLMoE fp16-vs-int4 A/B (or a large-n GLM run), please open an issue with the numbers** — it's the measurement that turns 62.5% into either "int4 is fine, scoring artifact" or "quantization is the ceiling, grouped-scale is the priority."
|
||||
| Benchmarks, community datapoints, quality measurements | [docs/benchmarks.md](docs/benchmarks.md) |
|
||||
| Tuning knobs, policies, the learning cache, prefetch | [docs/tuning.md](docs/tuning.md) |
|
||||
| Windows 11 native build (+ CUDA DLL) | [docs/windows.md](docs/windows.md) |
|
||||
| CUDA backend, VRAM expert tier, full residency | [docs/cuda.md](docs/cuda.md) |
|
||||
| Apple Silicon Metal backend | [docs/metal.md](docs/metal.md) |
|
||||
| OpenAI-compatible API, KV slots, web dashboard | [docs/api.md](docs/api.md) |
|
||||
| Grammar-forced drafts (structured output) | [docs/grammar-draft.md](docs/grammar-draft.md) |
|
||||
| Environment variable inventory | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) |
|
||||
|
||||
## Supporting the project
|
||||
|
||||
colibrì is a one-person project, written and tested entirely on a 12-core laptop with 25 GB of RAM — the numbers above are the ceiling of what I can measure at home. If this project is useful or interesting to you and you'd like to support its development (better test hardware translates *directly* into a faster engine for everyone: real NVMe scaling data, bigger pinned caches, int2/int3 quality sweeps on real benchmarks), you can:
|
||||
colibrì started as a one-person project on a 12-core laptop with 25 GB of RAM;
|
||||
today its numbers come from a community of real machines. If it's useful to you:
|
||||
|
||||
- ⭐ star the repo and share it;
|
||||
- 🐛 open issues with benchmark numbers from your hardware;
|
||||
- 💬 reach out via GitHub issues if you'd like to sponsor development or donate hardware.
|
||||
|
||||
Every contribution, from a datapoint to a disk, moves the ceiling.
|
||||
- 🐛 open issues with benchmark numbers from your hardware — datapoints move
|
||||
this project more than anything else;
|
||||
- 💬 reach out via GitHub issues to sponsor development or donate hardware.
|
||||
|
||||
## Repo layout
|
||||
|
||||
@@ -656,20 +229,20 @@ c/
|
||||
├── tools/ offline conversion, fixtures and benchmarks
|
||||
├── scripts/ long-running conversion helpers
|
||||
└── tests/ dependency-free C and Python tests
|
||||
web/ browser UI (pure OpenAI-API client, community-maintained)
|
||||
web/ browser UI (pure OpenAI-API client)
|
||||
desktop/ Tauri v2 desktop shell wrapping the web UI
|
||||
docs/ reference docs, experiments, media
|
||||
```
|
||||
|
||||
The runtime path intentionally stays flat and readable: `glm.c` plus its small
|
||||
headers. Auxiliary Python and shell tooling is grouped separately and is never a
|
||||
runtime dependency of the engine.
|
||||
|
||||
From the repository root, `make`, `make check`, and `make clean` delegate to the
|
||||
engine Makefile. Existing commands run from `c/` continue to work unchanged.
|
||||
headers. From the repository root, `make`, `make check`, and `make clean`
|
||||
delegate to the engine Makefile.
|
||||
|
||||
## Why "colibrì"
|
||||
|
||||
The hummingbird weighs a few grams, hovers in place, and visits a thousand flowers a day. This engine keeps a 744-billion-parameter giant alive on hummingbird rations: 25 GB of RAM, twelve CPU cores, and a lot of disk patience.
|
||||
The hummingbird weighs a few grams, hovers in place, and visits a thousand
|
||||
flowers a day. This engine keeps a 744-billion-parameter giant alive on
|
||||
hummingbird rations: 25 GB of RAM, twelve CPU cores, and a lot of disk patience.
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+237
@@ -0,0 +1,237 @@
|
||||
<p align="center">
|
||||
<img src="assets/colibri.svg" width="500" alt="colibrì——小巧引擎，庞大模型">
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="README.md">English</a> · 简体中文 · <a href="README.zh-TW.md">繁體中文</a> · <a href="README.it.md">Italiano</a>
|
||||
</p>
|
||||
|
||||
**小巧引擎，庞大模型。**只需约 25 GB 内存，就能在消费级电脑上运行 **GLM-5.2（744B 参数的 MoE）**——以零依赖的纯 C 实现，从磁盘流式加载专家。
|
||||
|
||||
Colibrì 是一套轻量、保持模型质量的 MoE 运行时，将 VRAM、RAM
|
||||
与存储设备视为统一管理的内存层级。高速内存不足可能降低速度，
|
||||
但默认策略**绝不会在未告知的情况下改变模型精度或路由语义**。
|
||||
|
||||
```
|
||||
$ ./coli chat
|
||||
🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU
|
||||
✓ ready in 32s · resident 9.9 GB
|
||||
› ciao!
|
||||
◆ Ciao! 😊 Come posso aiutarti oggi?
|
||||
```
|
||||
|
||||
## 实际运行效果
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/colibri-dashboard.png" width="900" alt="colibrì 网页仪表盘——实时指标、硬件面板与专家存储层级">
|
||||
</p>
|
||||
<p align="center"><em>网页仪表盘（<code>./coli web</code>）：744B 模型达到 <strong>4 tok/s、TTFT 1.6 秒、磁盘读取 0</strong>——
|
||||
在 6× RTX 5090 上让所有专家常驻，并实时显示 token 指标、每轮耗时明细、
|
||||
VRAM／RAM／磁盘层级条，以及角落的实时迷你大脑。</em></p>
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/colibri-brain.png" width="900" alt="大脑页面——以实时皮层呈现 19,456 个专家">
|
||||
</p>
|
||||
<p align="center"><em><strong>大脑（Brain）</strong>页面：将全部 19,456 个专家呈现为活的皮层——颜色代表存储层级，
|
||||
亮度代表路由热度，每轮被路由到的专家都会闪白。将光标停在专家上，即可查看其
|
||||
<a href="https://github.com/JustVugg/colibri/issues/175">实测主题亲和度</a>。</em></p>
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/colibri-atlas.png" width="900" alt="图谱页面——以 3D 星系呈现实测专家图谱">
|
||||
</p>
|
||||
<p align="center"><em><strong>图谱（Atlas）</strong>页面：将<a href="https://github.com/JustVugg/colibri/issues/175">实测专家图谱</a>
|
||||
呈现为 3D 星系——共 13,260 个已分析专家，其中 1,041 个可复现的专门专家会按主题聚集
|
||||
（诗歌、法律、中文、SQL……）。位置取自实测路由亲和度，而非学习出的嵌入向量。拖拽即可旋转。</em></p>
|
||||
|
||||
## 核心概念
|
||||
|
||||
744B 的专家混合（Mixture-of-Experts）模型，每个 token 只会激活约 40B 参数——
|
||||
其中每个 token 之间会变动的只有约 11 GB（被路由到的专家）：
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/sparse.png" width="880" alt="每个 token 只会激活约 5.4% 的参数">
|
||||
</p>
|
||||
|
||||
所以模型不必完整**装进**高速内存，而是需要正确**放置**：
|
||||
|
||||
- **稠密部分**（注意力、共享专家、嵌入——约 17B 参数）以 int4
|
||||
**常驻 RAM**（约 9.9 GB）；
|
||||
- **19,456 个路由专家**（75 个 MoE 层 × 256，加上 MTP head；每个在 int4 下约 19 MB）
|
||||
**存放在磁盘**（约 370 GB），并**按需流式加载**，配合逐层 LRU 缓存、
|
||||
会学习的热门专家固定存储区，以及可选的 VRAM 层级。
|
||||
|
||||
引擎是一个 C 主文件（`c/colibri.c`）加上若干头文件。不需要 BLAS，
|
||||
运行时不需要 Python，也不需要 GPU。
|
||||
|
||||
## 工作原理
|
||||
|
||||
### 每个 token 的处理路径
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/token-path.png" width="880" alt="路由 → 并集 → 放置 → 重叠执行 → 学习">
|
||||
</p>
|
||||
|
||||
每个 token 的每一层都会经过相同的五个步骤。设计目标是让
|
||||
**放置只决定速度**——无论专家是从 VRAM 还是磁盘响应，路由器的决策与权重精度都完全相同。
|
||||
|
||||
### 统一内存层级，取代单一内存门槛
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/tiers.png" width="880" alt="VRAM／RAM／NVMe 三层专家常驻架构">
|
||||
</p>
|
||||
|
||||
同一套引擎覆盖完整硬件范围：在 25 GB 笔记本上，一切都从磁盘流式加载
|
||||
（慢，但结果正确）；在大内存主机上，则可让整组专家常驻
|
||||
（`CUDA_EXPERT_GB=auto PIN_GB=all`），让磁盘完全退出解码路径。
|
||||
两端之间有一层**学习型缓存**：引擎会记录*你的*工作负载路由到哪些专家
|
||||
（`.coli_usage`，每轮更新），并自动固定最热门的专家——colibrì 确实会越用越快。
|
||||
在多路主机上，`COLI_NUMA=1` 会将常驻权重交错分配到各内存控制器
|
||||
（[#82](https://github.com/JustVugg/colibri/issues/82)）。
|
||||
|
||||
### 绝不为同一次磁盘读取等待两遍
|
||||
|
||||
缓存未命中的代价很高，因此引擎大部分的巧思都用来避免或重叠这些读取：
|
||||
每个专家的三个矩阵相邻存储，并以一次 `pread` 读取；有界异步 I/O 池
|
||||
（`PIPE=1`，默认启用）会在常驻专家计算时加载缺失的专家；批量位置只读取每个
|
||||
不重复专家一次（**批量并集**）；路由前瞻线程（`PILOT=1`）则预取下一层专家——
|
||||
实测显示，路由结果提前一层时有 **71.6% 的可预测性**。
|
||||
在 GPU 上，常驻管线（`COLI_CUDA_PIPE=2`）让残差流跨层保留在设备端，
|
||||
使 CPU 专家循环不中断；在 Apple Silicon 上，实验性的
|
||||
[Metal 后端](docs/metal.md)会用统一内存 GPU 执行批量专家运算。
|
||||
|
||||
### 忠实模型，压缩状态
|
||||
|
||||
前向传播已通过 `transformers` oracle 验证为**逐 token 完全一致**
|
||||
（teacher-forcing 32/32）。MLA 注意力存储压缩后的 KV 状态——每个 token 为 576 个
|
||||
浮点数，而非 32,768 个（**缩小 57×**）——并跨重启持久保存
|
||||
（`.coli_kv`）：对话可暖启恢复，不需重新 prefill，结果与不中断的会话
|
||||
逐字节相同。DSA 稀疏注意力（GLM-5.2 的 lightning indexer）已忠实实现，
|
||||
并通过强制选取所有 key，验证可精确复现稠密注意力。
|
||||
|
||||
### 诚实的推测解码
|
||||
|
||||
GLM-5.2 原生 MTP head 会起草 token，再由主模型以一次批量前向传播验证——
|
||||
条件合适时每次 forward 可产生 2.2–2.8 个 token。两条来之不易的规则已成为默认值：
|
||||
MTP head 必须是 **int8**（int4 head 的接受率会崩塌到 0–4%，见
|
||||
[#8](https://github.com/JustVugg/colibri/issues/8)），且草稿与验证必须计算
|
||||
**相同函数**——`SPEC_PIN=1` 会把两者固定在同一 kernel family
|
||||
（完整取证过程见 [#163](https://github.com/JustVugg/colibri/issues/163)）。
|
||||
语法强制草稿（[`GRAMMAR=file.gbnf`](docs/grammar-draft.md)）可在受限 JSON 输出中，
|
||||
以近乎免费的代价提高接受率。推测解码是否带来净收益取决于缓存热度——请实测，
|
||||
若不划算就使用 `DRAFT=0`。
|
||||
|
||||
## 实际成果
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/ladder.png" width="880" alt="各硬件级别的实测解码速度">
|
||||
</p>
|
||||
|
||||
同一套引擎、同一个 int4 容器——硬件只会改变专家的存放位置。
|
||||
[完整 benchmark 表格](docs/benchmarks.md)中的重点如下：
|
||||
|
||||
- **6× RTX 5090，全部常驻：**解码 5.8–6.8 tok/s，TTFT 约 13 秒
|
||||
（[实验记录](docs/experiments/glm52-6x5090-2026-07-12.md)）；
|
||||
- **128 GB、仅使用 CPU 的台式机：**热缓存后约 1.8 tok/s
|
||||
（[#200](https://github.com/JustVugg/colibri/issues/200)）；
|
||||
- **单张 RTX 5070 Ti 的笔记本级主机：**通过 GPU 常驻管线达到 1.07 tok/s
|
||||
（[#273](https://github.com/JustVugg/colibri/issues/273)）；
|
||||
- **25 GB 开发机：**冷启动 0.05–0.1 tok/s——这是项目起步时已证实的下限，
|
||||
也仍是诚实的基准。
|
||||
|
||||
质量来自测量，而非假设：int4 容器的量化损失，以及 scale granularity／rotation
|
||||
消融实验，收录于 [docs/benchmarks.md](docs/benchmarks.md#quality-benchmark)、
|
||||
[#108](https://github.com/JustVugg/colibri/issues/108) 与
|
||||
[#81](https://github.com/JustVugg/colibri/issues/81)。
|
||||
|
||||
## 开始使用
|
||||
|
||||
### 1. 获取模型
|
||||
|
||||
Hugging Face 上已有预转换的 **GLM-5.2 int4** 容器——请务必使用
|
||||
**含 int8 MTP head 的版本**：
|
||||
|
||||
**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp**
|
||||
|
||||
> ⚠️ 原始镜像使用 int4 MTP head → 草稿接受率为 0%
|
||||
>（[#8](https://github.com/JustVugg/colibri/issues/8)）。请检查你的版本：
|
||||
> `ls -l <model>/out-mtp-*`——正确的 int8 大小为 `3527131672 / 5366238584 / 1065950496`。
|
||||
|
||||
你也可以自行从 FP8 源转换——只需一条可断点续传的命令，且任何时候都不需要
|
||||
在磁盘上同时存放完整的 756 GB：
|
||||
|
||||
```bash
|
||||
cd c && ./setup.sh # 检查 gcc/OpenMP、构建并运行自测
|
||||
./coli convert --model /nvme/glm52_i4 # 逐 shard 下载并转换（仅此一次需要 python）
|
||||
```
|
||||
|
||||
### 2. 运行
|
||||
|
||||
```bash
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli chat # 自动检测 RAM 预算、缓存与 MTP
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli plan # 查看规划的 VRAM／RAM／磁盘配置
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli doctor # 只读就绪检查
|
||||
./coli web --model /nvme/glm52_i4 # 在同一端口提供 API 与网页仪表盘
|
||||
./coli serve --model /nvme/glm52_i4 # 仅提供 OpenAI 兼容 API
|
||||
```
|
||||
|
||||
引擎运行时是纯 C——python 只供一次性转换工具与可选的 API gateway 使用。
|
||||
|
||||
### 3. 深入了解
|
||||
|
||||
| 主题 | 文档 |
|
||||
|---|---|
|
||||
| Benchmark、社区实测数据、质量测量 | [docs/benchmarks.md](docs/benchmarks.md) |
|
||||
| 调优选项、策略、学习型缓存、预取 | [docs/tuning.md](docs/tuning.md) |
|
||||
| Windows 11 原生构建（含 CUDA DLL） | [docs/windows.md](docs/windows.md) |
|
||||
| CUDA 后端、VRAM 专家层级、全部常驻 | [docs/cuda.md](docs/cuda.md) |
|
||||
| Apple Silicon Metal 后端 | [docs/metal.md](docs/metal.md) |
|
||||
| OpenAI 兼容 API、KV slots、网页仪表盘 | [docs/api.md](docs/api.md) |
|
||||
| 语法强制草稿（结构化输出） | [docs/grammar-draft.md](docs/grammar-draft.md) |
|
||||
| 环境变量完整清单 | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) |
|
||||
|
||||
## 支持项目
|
||||
|
||||
colibrì 最初由一人使用 12 核心、25 GB RAM 的笔记本开发；
|
||||
如今它的数据来自社区中各种真实机器。如果这个项目对你有用：
|
||||
|
||||
- ⭐ 为仓库加星并分享；
|
||||
- 🐛 以 issue 提交你的硬件 benchmark 数据——实测数据比任何其他事都更能推动项目；
|
||||
- 💬 若想赞助开发或捐赠硬件，请通过 GitHub issues 联系。
|
||||
|
||||
## 仓库结构
|
||||
|
||||
```
|
||||
Makefile 根目录构建／检查入口
|
||||
c/
|
||||
├── colibri.c 引擎主文件
|
||||
├── quant.h 量化 matmul 内核（SIMD 多架构）
|
||||
├── sample.h 采样与 stop-set 管理
|
||||
├── kv_persist.h .coli_kv 磁盘持久化
|
||||
├── telemetry.h 仪表盘协议、统计与用量持久化
|
||||
├── st.h, tok.h, json.h 运行时头文件
|
||||
├── backend_cuda.* 可选的 CUDA 层级
|
||||
├── Makefile 构建与本地检查
|
||||
├── coli 用户界面 CLI
|
||||
├── openai_server.py OpenAI 兼容 HTTP gateway
|
||||
├── setup.sh 一条命令完成本地设置
|
||||
├── tools/ 离线转换、fixtures 与 benchmarks
|
||||
├── scripts/ 长时间转换辅助工具
|
||||
└── tests/ 零依赖的 C 与 Python 测试
|
||||
web/ 浏览器 UI（纯 OpenAI API client）
|
||||
desktop/ 封装网页 UI 的 Tauri v2 桌面 shell
|
||||
docs/ 参考文档、实验与媒体文件
|
||||
```
|
||||
|
||||
运行时路径刻意保持扁平、易读：`colibri.c` 加上若干头文件。
|
||||
在仓库根目录执行 `make`、`make check` 与 `make clean`，
|
||||
都会转发给引擎的 Makefile。
|
||||
|
||||
## 为什么叫"colibrì"
|
||||
|
||||
蜂鸟只有几克重，能在原地悬停，并在一天内造访上千朵花。
|
||||
这套引擎只用蜂鸟般的配给，就能让 744B 参数的巨人运转：
|
||||
25 GB RAM、十二个 CPU 核心，以及对磁盘的大量耐心。
|
||||
|
||||
## 许可证
|
||||
|
||||
Apache 2.0。GLM-5.2 权重由 Z.ai 以 MIT 许可发布。
|
||||
+237
@@ -0,0 +1,237 @@
|
||||
<p align="center">
|
||||
<img src="assets/colibri.svg" width="500" alt="colibrì——小巧引擎，龐大模型">
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="README.md">English</a> · <a href="README.zh-CN.md">简体中文</a> · 繁體中文 · <a href="README.it.md">Italiano</a>
|
||||
</p>
|
||||
|
||||
**小巧引擎，龐大模型。**只要約 25 GB 記憶體，就能在消費級電腦上執行 **GLM-5.2（744B 參數的 MoE）**——以零相依套件的純 C 實作，從硬碟串流載入專家。
|
||||
|
||||
Colibrì 是一套輕量、維持品質的 MoE 執行環境，將 VRAM、RAM
|
||||
與儲存裝置視為統一管理的記憶體階層。高速記憶體不足可能降低速度，
|
||||
但預設策略**絕不會在未告知的情況下改變模型精度或路由語意**。
|
||||
|
||||
```
|
||||
$ ./coli chat
|
||||
🐦 colibrì v1.0 — GLM-5.2 · 744B MoE · int4 · streaming CPU
|
||||
✓ ready in 32s · resident 9.9 GB
|
||||
› ciao!
|
||||
◆ Ciao! 😊 Come posso aiutarti oggi?
|
||||
```
|
||||
|
||||
## 實際運行畫面
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/colibri-dashboard.png" width="900" alt="colibrì 網頁儀表板——即時指標、硬體面板與專家儲存層級">
|
||||
</p>
|
||||
<p align="center"><em>網頁儀表板（<code>./coli web</code>）：744B 模型達到 <strong>4 tok/s、TTFT 1.6 秒、硬碟讀取 0</strong>——
|
||||
在 6× RTX 5090 上讓所有專家常駐，並即時顯示 token 指標、每輪耗時明細、
|
||||
VRAM／RAM／硬碟層級長條，以及角落的即時迷你大腦。</em></p>
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/colibri-brain.png" width="900" alt="大腦頁面——以即時皮質呈現 19,456 個專家">
|
||||
</p>
|
||||
<p align="center"><em><strong>大腦（Brain）</strong>頁面：將全部 19,456 個專家呈現為活的皮質——顏色代表儲存層級，
|
||||
亮度代表路由熱度，每輪被路由到的專家都會閃白。將游標停在專家上，即可查看其
|
||||
<a href="https://github.com/JustVugg/colibri/issues/175">實測主題親和度</a>。</em></p>
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/colibri-atlas.png" width="900" alt="圖譜頁面——以 3D 星系呈現實測專家圖譜">
|
||||
</p>
|
||||
<p align="center"><em><strong>圖譜（Atlas）</strong>頁面：將<a href="https://github.com/JustVugg/colibri/issues/175">實測專家圖譜</a>
|
||||
呈現為 3D 星系——共 13,260 個已分析專家，其中 1,041 個可重現的專門專家會按主題聚集
|
||||
（詩歌、法律、中文、SQL……）。位置取自實測路由親和度，而非學習出的嵌入向量。拖曳即可旋轉。</em></p>
|
||||
|
||||
## 核心概念
|
||||
|
||||
744B 的專家混合（Mixture-of-Experts）模型，每個 token 只會啟用約 40B 參數——
|
||||
其中每個 token 之間會變動的只有約 11 GB（被路由到的專家）：
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/sparse.png" width="880" alt="每個 token 只會啟用約 5.4% 的參數">
|
||||
</p>
|
||||
|
||||
所以模型不必完整**放進**高速記憶體，而是需要正確**配置位置**：
|
||||
|
||||
- **稠密部分**（注意力、共享專家、嵌入——約 17B 參數）以 int4
|
||||
**常駐 RAM**（約 9.9 GB）；
|
||||
- **19,456 個路由專家**（75 個 MoE 層 × 256，加上 MTP head；每個在 int4 下約 19 MB）
|
||||
**存放在硬碟**（約 370 GB），並**隨需串流載入**，搭配逐層 LRU 快取、
|
||||
會學習的熱門專家固定儲存區，以及選用的 VRAM 層級。
|
||||
|
||||
引擎由主 C 檔（`c/colibri.c`）與多個標頭檔模組組成。不需要 BLAS，
|
||||
執行階段不需要 Python，也不需要 GPU。
|
||||
|
||||
## 運作方式
|
||||
|
||||
### 每個 token 的處理路徑
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/token-path.png" width="880" alt="路由 → 聯集 → 配置 → 重疊執行 → 學習">
|
||||
</p>
|
||||
|
||||
每個 token 的每一層都會走過相同的五個步驟。設計目標是讓
|
||||
**配置只決定速度**——無論專家是從 VRAM 或硬碟回應，路由器的決策與權重精度都完全相同。
|
||||
|
||||
### 統一記憶體階層，取代單一記憶體門檻
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/tiers.png" width="880" alt="VRAM／RAM／NVMe 三層專家常駐架構">
|
||||
</p>
|
||||
|
||||
同一套引擎涵蓋完整硬體範圍：在 25 GB 筆電上，一切都從硬碟串流載入
|
||||
（慢，但結果正確）；在大型主機上，則可讓整組專家常駐
|
||||
（`CUDA_EXPERT_GB=auto PIN_GB=all`），讓硬碟完全退出解碼路徑。
|
||||
兩端之間有一層**學習型快取**：引擎會記錄*你的*工作負載路由到哪些專家
|
||||
（`.coli_usage`，每輪更新），並自動固定最熱門的專家——colibrì 確實會越用越快。
|
||||
在多插槽主機上，`COLI_NUMA=1` 會將常駐權重交錯分配到各記憶體控制器
|
||||
（[#82](https://github.com/JustVugg/colibri/issues/82)）。
|
||||
|
||||
### 絕不為同一次硬碟讀取等待兩遍
|
||||
|
||||
快取未命中的成本很高，因此引擎大部分的巧思都用來避免或重疊處理這些讀取：
|
||||
每個專家的三個矩陣相鄰儲存，並以一次 `pread` 讀取；有界非同步 I/O pool
|
||||
（`PIPE=1`，預設啟用）會在常駐專家運算時載入缺少的專家；批次位置只讀取每個
|
||||
不重複專家一次（**批次聯集**）；路由前瞻執行緒（`PILOT=1`）則預先載入下一層專家——
|
||||
實測顯示，路由結果提前一層時有 **71.6% 的可預測性**。
|
||||
在 GPU 上，常駐管線（`COLI_CUDA_PIPE=2`）讓殘差流跨層保留在裝置端，
|
||||
使 CPU 專家迴圈不中斷；在 Apple Silicon 上，實驗性的
|
||||
[Metal 後端](docs/metal.md)會用統一記憶體 GPU 執行批次專家運算。
|
||||
|
||||
### 忠實模型，壓縮狀態
|
||||
|
||||
前向傳遞已透過 `transformers` oracle 驗證為**逐 token 完全一致**
|
||||
（teacher-forcing 32/32）。MLA 注意力儲存壓縮後的 KV 狀態——每個 token 為 576 個
|
||||
浮點數，而非 32,768 個（**縮小 57×**）——並跨重新啟動持久保存
|
||||
（`.coli_kv`）：對話可暖啟恢復，不需重新 prefill，結果與不中斷的工作階段
|
||||
逐位元組相同。DSA 稀疏注意力（GLM-5.2 的 lightning indexer）已忠實實作，
|
||||
並透過強制選取所有 key，驗證可精確重現稠密注意力。
|
||||
|
||||
### 如實呈現推測式解碼
|
||||
|
||||
GLM-5.2 原生 MTP head 會起草 token，再由主模型以一次批次前向傳遞驗證——
|
||||
條件合適時每次 forward 可產生 2.2–2.8 個 token。兩條得來不易的規則已成為預設值：
|
||||
MTP head 必須是 **int8**（int4 head 的接受率會崩落到 0–4%，見
|
||||
[#8](https://github.com/JustVugg/colibri/issues/8)），且草稿與驗證必須計算
|
||||
**相同函數**——`SPEC_PIN=1` 會把兩者固定在同一 kernel family
|
||||
（完整鑑識過程見 [#163](https://github.com/JustVugg/colibri/issues/163)）。
|
||||
文法強制草稿（[`GRAMMAR=file.gbnf`](docs/grammar-draft.md)）可在受限 JSON 輸出中，
|
||||
以近乎免費的成本提高接受率。推測式解碼是否帶來淨收益取決於快取熱度——請實測，
|
||||
若不划算就使用 `DRAFT=0`。
|
||||
|
||||
## 實際成果
|
||||
|
||||
<p align="center">
|
||||
<img src="docs/media/ladder.png" width="880" alt="各硬體等級的實測解碼速度">
|
||||
</p>
|
||||
|
||||
同一套引擎、同一個 int4 容器——硬體只會改變專家的存放位置。
|
||||
[完整 benchmark 表格](docs/benchmarks.md)中的重點如下：
|
||||
|
||||
- **6× RTX 5090，全部常駐：**解碼 5.8–6.8 tok/s，TTFT 約 13 秒
|
||||
（[實驗紀錄](docs/experiments/glm52-6x5090-2026-07-12.md)）；
|
||||
- **128 GB、僅使用 CPU 的桌上型電腦：**暖機後約 1.8 tok/s
|
||||
（[#200](https://github.com/JustVugg/colibri/issues/200)）；
|
||||
- **單張 RTX 5070 Ti 的筆電級電腦：**透過 GPU 常駐管線達到 1.07 tok/s
|
||||
（[#273](https://github.com/JustVugg/colibri/issues/273)）；
|
||||
- **25 GB 開發機：**冷啟動 0.05–0.1 tok/s——這是專案起步時已證實的下限，
|
||||
也仍是如實呈現的基準。
|
||||
|
||||
品質來自測量，而非假設：int4 容器的量化成本，以及 scale granularity／rotation
|
||||
消融實驗，收錄於 [docs/benchmarks.md](docs/benchmarks.md#quality-benchmark)、
|
||||
[#108](https://github.com/JustVugg/colibri/issues/108) 與
|
||||
[#81](https://github.com/JustVugg/colibri/issues/81)。
|
||||
|
||||
## 開始使用
|
||||
|
||||
### 1. 取得模型
|
||||
|
||||
Hugging Face 上已有預先轉換的 **GLM-5.2 int4** 容器——請務必使用
|
||||
**含 int8 MTP head 的版本**：
|
||||
|
||||
**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp**
|
||||
|
||||
> ⚠️ 原始鏡像使用 int4 MTP head → 草稿接受率為 0%
|
||||
>（[#8](https://github.com/JustVugg/colibri/issues/8)）。請檢查你的版本：
|
||||
> `ls -l <model>/out-mtp-*`——正確的 int8 大小為 `3527131672 / 5366238584 / 1065950496`。
|
||||
|
||||
你也可以自行從 FP8 來源轉換——只需一條可續傳的指令，且任何時候都不需要
|
||||
在硬碟上同時存放完整的 756 GB：
|
||||
|
||||
```bash
|
||||
cd c && ./setup.sh # 檢查 gcc/OpenMP、建置並執行自我測試
|
||||
./coli convert --model /nvme/glm52_i4 # 逐 shard 下載並轉換（僅此一次需要 python）
|
||||
```
|
||||
|
||||
### 2. 執行
|
||||
|
||||
```bash
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli chat # 自動偵測 RAM 預算、快取與 MTP
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli plan # 檢視規劃的 VRAM／RAM／硬碟配置
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli doctor # 唯讀就緒檢查
|
||||
./coli web --model /nvme/glm52_i4 # 在同一個連接埠提供 API 與網頁儀表板
|
||||
./coli serve --model /nvme/glm52_i4 # 僅提供 OpenAI 相容 API
|
||||
```
|
||||
|
||||
引擎執行階段是純 C——python 只供單次轉換工具與選用的 API gateway 使用。
|
||||
|
||||
### 3. 深入了解
|
||||
|
||||
| 主題 | 文件 |
|
||||
|---|---|
|
||||
| Benchmark、社群實測數據、品質測量 | [docs/benchmarks.md](docs/benchmarks.md) |
|
||||
| 調校選項、策略、學習型快取、預先載入 | [docs/tuning.md](docs/tuning.md) |
|
||||
| Windows 11 原生建置（含 CUDA DLL） | [docs/windows.md](docs/windows.md) |
|
||||
| CUDA 後端、VRAM 專家層級、全部常駐 | [docs/cuda.md](docs/cuda.md) |
|
||||
| Apple Silicon Metal 後端 | [docs/metal.md](docs/metal.md) |
|
||||
| OpenAI 相容 API、KV slots、網頁儀表板 | [docs/api.md](docs/api.md) |
|
||||
| 文法強制草稿（結構化輸出） | [docs/grammar-draft.md](docs/grammar-draft.md) |
|
||||
| 環境變數完整清單 | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) |
|
||||
|
||||
## 支持專案
|
||||
|
||||
colibrì 最初是由一人使用 12 核心、25 GB RAM 的筆電開發；
|
||||
如今它的數據來自社群中的各種真實機器。如果這個專案對你有用：
|
||||
|
||||
- ⭐ 為儲存庫加星並分享；
|
||||
- 🐛 以 issue 提交你的硬體 benchmark 數據——實測資料比任何其他事都更能推動專案；
|
||||
- 💬 若想贊助開發或捐贈硬體，請透過 GitHub issues 聯絡。
|
||||
|
||||
## 儲存庫結構
|
||||
|
||||
```
|
||||
Makefile 根目錄建置／檢查入口
|
||||
c/
|
||||
├── colibri.c GLM 引擎主檔
|
||||
├── quant.h 量化 matmul kernel
|
||||
├── sample.h 取樣與 stop-set
|
||||
├── kv_persist.h .coli_kv 磁碟持久化
|
||||
├── telemetry.h 儀表板協定、統計
|
||||
├── st.h, tok.h, json.h 執行階段標頭檔
|
||||
├── backend_cuda.* 選用的 CUDA 層級
|
||||
├── Makefile 建置與本機檢查
|
||||
├── coli 使用者介面 CLI
|
||||
├── openai_server.py OpenAI 相容 HTTP gateway
|
||||
├── setup.sh 單一指令完成本機設定
|
||||
├── tools/ 離線轉換、fixtures 與 benchmarks
|
||||
├── scripts/ 長時間轉換輔助工具
|
||||
└── tests/ 零相依套件的 C 與 Python 測試
|
||||
web/ 瀏覽器 UI（純 OpenAI API client）
|
||||
desktop/ 包裝網頁 UI 的 Tauri v2 桌面 shell
|
||||
docs/ 參考文件、實驗與媒體檔
|
||||
```
|
||||
|
||||
執行階段路徑刻意維持扁平、易讀：`colibri.c` 加上模組化標頭檔。
|
||||
在儲存庫根目錄執行 `make`、`make check` 與 `make clean`，
|
||||
都會轉交給引擎的 Makefile。
|
||||
|
||||
## 為什麼叫做「colibrì」
|
||||
|
||||
蜂鳥只有幾公克重，能在原地懸停，並在一天內造訪上千朵花。
|
||||
這套引擎只用蜂鳥般的配給，就能讓 744B 參數的巨人運轉：
|
||||
25 GB RAM、十二個 CPU 核心，以及對硬碟的大量耐心。
|
||||
|
||||
## 授權條款
|
||||
|
||||
Apache 2.0。GLM-5.2 權重由 Z.ai 以 MIT 授權發布。
|
||||
@@ -2,3 +2,4 @@
|
||||
glm_tiny/
|
||||
olmoe_hf/
|
||||
olmoe_i4/
|
||||
.build-config
|
||||
|
||||
+71
-25
@@ -56,7 +56,7 @@ OMPL =
|
||||
endif
|
||||
CFLAGS = -O3 $(OMPC) -Wall -Wextra -Wno-unused-parameter -Wno-misleading-indentation -Wno-unused-function
|
||||
# Opt-in: ARCH=native appends -mcpu=native (arm64 clang uses -mcpu, not -march),
|
||||
# which unlocks the i8mm SMMLA int8/int4 dot kernels in glm.c. ARCH unset ->
|
||||
# which unlocks the i8mm SMMLA int8/int4 dot kernels in colibri.c. ARCH unset ->
|
||||
# no -mcpu, default build byte-identical. Apple clang knows apple-m4 / native.
|
||||
ifneq ($(ARCH),)
|
||||
CFLAGS += -mcpu=$(ARCH)
|
||||
@@ -70,7 +70,7 @@ else ifneq ($(IS_WIN),)
|
||||
# ARCH default = x86-64-v3 (portable binary with AVX2). For max speed on THIS
|
||||
# machine use ARCH=native: on AVX-VNNI CPUs (Intel Alder Lake+, Meteor Lake+)
|
||||
# it also unlocks the 128-bit VPDPBUSD int8/int4 dot kernel (dot_i8i8/dot_i4i8),
|
||||
# which the x86-64-v3 baseline does not define. The #ifdef guards in glm.c mean
|
||||
# which the x86-64-v3 baseline does not define. The #ifdef guards in colibri.c mean
|
||||
# a v3 build simply compiles out the VNNI path - safe on any x86-64.
|
||||
CC = gcc
|
||||
ARCH ?= x86-64-v3
|
||||
@@ -166,7 +166,7 @@ else
|
||||
PYTHON ?= python3
|
||||
endif
|
||||
CUDA_OBJ =
|
||||
TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE)
|
||||
TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE)
|
||||
ifneq (,$(LINUX))
|
||||
TEST_BINS += tests/test_uring$(EXE)
|
||||
endif
|
||||
@@ -207,16 +207,31 @@ LDFLAGS += -framework Metal -framework Foundation -lc++
|
||||
METAL_OBJ = backend_metal.o
|
||||
endif
|
||||
|
||||
all: glm$(EXE)
|
||||
all: colibri$(EXE)
|
||||
|
||||
# phony 'glm' → 'glm.exe' on Windows (so 'make glm' and 'coli build' work on every platform)
|
||||
glm: glm$(EXE)
|
||||
# phony targets — 'glm' kept for backward compatibility
|
||||
colibri: colibri$(EXE)
|
||||
glm: colibri$(EXE)
|
||||
|
||||
glm$(EXE): glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h $(CUDA_OBJ) $(METAL_OBJ)
|
||||
$(CC) $(CFLAGS) glm.c $(CUDA_OBJ) $(METAL_OBJ) -o glm$(EXE) $(LDFLAGS)
|
||||
# Config stamp: make only tracks file timestamps, not flag changes. Without this,
|
||||
# `make colibri.exe CUDA_DLL=1` after a prior CPU-only build reports "up to date"
|
||||
# and silently keeps the CPU-only binary (no CUDA loader) — a build that looks like
|
||||
# it worked but isn't. We record the build-affecting flags in .build-config and
|
||||
# rewrite it ONLY when they change (evaluated here at parse time, so the file's
|
||||
# timestamp moves exactly when the config moves). The binary and CUDA/loader objects depend
|
||||
# on it, so they relink on a config change and stay put otherwise. (#306)
|
||||
BUILD_CONFIG := $(CC)|$(CFLAGS)|$(LDFLAGS)|CUDA=$(CUDA)|CUDA_DLL=$(CUDA_DLL)|ARCH=$(ARCH)|CUDA_ARCH=$(CUDA_ARCH)|METAL=$(METAL)
|
||||
BUILD_CONFIG_OLD := $(shell cat .build-config 2>/dev/null)
|
||||
ifneq "$(BUILD_CONFIG)" "$(BUILD_CONFIG_OLD)"
|
||||
$(shell printf '%s' '$(BUILD_CONFIG)' > .build-config)
|
||||
endif
|
||||
.build-config: ;
|
||||
|
||||
colibri$(EXE): colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h quant.h sample.h kv_persist.h telemetry.h $(CUDA_OBJ) $(METAL_OBJ) .build-config
|
||||
$(CC) $(CFLAGS) colibri.c $(CUDA_OBJ) $(METAL_OBJ) -o colibri$(EXE) $(LDFLAGS)
|
||||
|
||||
# Windows runtime loader object: resolves coli_cuda_* from coli_cuda.dll.
|
||||
backend_loader.o: backend_loader.c backend_cuda.h compat.h
|
||||
backend_loader.o: backend_loader.c backend_cuda.h compat.h .build-config
|
||||
$(CC) $(CFLAGS) -c backend_loader.c -o $@
|
||||
|
||||
# Windows CUDA DLL: compile backend_cuda.cu with nvcc (+MSVC cl.exe as host
|
||||
@@ -225,15 +240,15 @@ backend_loader.o: backend_loader.c backend_cuda.h compat.h
|
||||
# "x64 Native Tools Command Prompt"). COLI_CUDA_BUILDING_DLL enables
|
||||
# __declspec(dllexport) so the 15 API symbols are exported.
|
||||
cuda-dll: backend_cuda.cu backend_cuda.h
|
||||
@command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
|
||||
@command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
|
||||
@command -v cl >/dev/null 2>&1 || { echo "cl.exe (MSVC) not in PATH — run vcvars64.bat first" >&2; exit 1; }
|
||||
$(NVCC) $(NVCCFLAGS) -shared -DCOLI_CUDA_BUILDING_DLL \
|
||||
"$(NVCC)" $(NVCCFLAGS) -shared -DCOLI_CUDA_BUILDING_DLL \
|
||||
-L"$(CUDA_HOME)/lib/x64" -lcudart \
|
||||
backend_cuda.cu -o coli_cuda.dll
|
||||
|
||||
backend_cuda.o: backend_cuda.cu backend_cuda.h
|
||||
@command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
|
||||
$(NVCC) $(NVCCFLAGS) -c backend_cuda.cu -o $@
|
||||
backend_cuda.o: backend_cuda.cu backend_cuda.h .build-config
|
||||
@command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
|
||||
"$(NVCC)" $(NVCCFLAGS) -c backend_cuda.cu -o $@
|
||||
|
||||
backend_metal.o: backend_metal.mm backend_metal.h
|
||||
$(METALXX) -c backend_metal.mm -o $@
|
||||
@@ -243,13 +258,13 @@ metal-test: tests/test_backend_metal.mm backend_metal.mm backend_metal.h
|
||||
./backend_metal_test
|
||||
|
||||
cuda-test: backend_cuda.cu backend_cuda.h tests/test_backend_cuda.cu
|
||||
@command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
|
||||
$(NVCC) $(NVCCFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE)
|
||||
@command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
|
||||
"$(NVCC)" $(NVCCFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test$(EXE)
|
||||
./backend_cuda_test$(EXE)
|
||||
|
||||
cuda-bench: backend_cuda.cu backend_cuda.h tests/bench_tensor_core.cu
|
||||
@command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
|
||||
$(NVCC) $(NVCCFLAGS) backend_cuda.cu tests/bench_tensor_core.cu -o backend_cuda_bench$(EXE)
|
||||
@command -v "$(NVCC)" >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
|
||||
"$(NVCC)" $(NVCCFLAGS) backend_cuda.cu tests/bench_tensor_core.cu -o backend_cuda_bench$(EXE)
|
||||
./backend_cuda_bench$(EXE)
|
||||
|
||||
olmoe$(EXE): olmoe.c st.h json.h compat.h
|
||||
@@ -271,7 +286,7 @@ PORTABLE_ARCH = native
|
||||
endif
|
||||
|
||||
portable:
|
||||
$(MAKE) glm$(EXE) ARCH=$(PORTABLE_ARCH)
|
||||
$(MAKE) colibri$(EXE) ARCH=$(PORTABLE_ARCH)
|
||||
|
||||
iobench$(EXE): iobench.c compat.h
|
||||
$(CC) $(CFLAGS) iobench.c -o iobench$(EXE) $(LDFLAGS)
|
||||
@@ -279,6 +294,9 @@ iobench$(EXE): iobench.c compat.h
|
||||
tests/test_json$(EXE): tests/test_json.c json.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
tests/test_st_pread$(EXE): tests/test_st_pread.c st.h json.h compat.h
|
||||
$(CC) $(CFLAGS) -DST_PREAD_CHUNK=7 $< -o $@ $(LDFLAGS)
|
||||
|
||||
tests/test_st$(EXE): tests/test_st.c st.h json.h compat.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
@@ -294,10 +312,30 @@ tests/test_schema_gbnf$(EXE): tests/test_schema_gbnf.c schema_gbnf.h grammar.h j
|
||||
tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
tests/test_idot$(EXE): tests/test_idot.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
|
||||
tests/test_idot$(EXE): tests/test_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
|
||||
tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
tests/test_stops$(EXE): tests/test_stops.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
tests/test_topp$(EXE): tests/test_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
# bench_topp is a microbenchmark (old qsort vs new heap partial-select, #335), NOT a test
|
||||
# gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_topp
|
||||
tests/bench_topp$(EXE): tests/bench_topp.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
tests/test_sample_nan$(EXE): tests/test_sample_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
tests/test_kv_alloc$(EXE): tests/test_kv_alloc.c colibri.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
tests/test_logit_nan$(EXE): tests/test_logit_nan.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c
|
||||
@@ -306,7 +344,15 @@ tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c
|
||||
tests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
tests/test_uring$(EXE): tests/test_uring.c glm.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
|
||||
tests/test_dsa_select$(EXE): tests/test_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
# bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356),
|
||||
# NOT a test gate -- intentionally absent from TEST_BINS. Build on demand: make tests/bench_dsa_select
|
||||
tests/bench_dsa_select$(EXE): tests/bench_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
tests/test_uring$(EXE): tests/test_uring.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
test-c: $(TEST_BINS)
|
||||
@@ -323,12 +369,12 @@ check:
|
||||
$(MAKE) portable
|
||||
$(MAKE) test
|
||||
|
||||
install: glm$(EXE) olmoe$(EXE)
|
||||
install: colibri$(EXE) olmoe$(EXE)
|
||||
$(INSTALL) -d $(DESTDIR)$(BINDIR)
|
||||
$(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)
|
||||
$(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)/tools
|
||||
$(INSTALL) -m 755 coli $(DESTDIR)$(BINDIR)/coli
|
||||
$(INSTALL) -m 755 glm$(EXE) $(DESTDIR)$(LIBEXECDIR)/glm$(EXE)
|
||||
$(INSTALL) -m 755 colibri$(EXE) $(DESTDIR)$(LIBEXECDIR)/colibri$(EXE)
|
||||
$(INSTALL) -m 755 olmoe$(EXE) $(DESTDIR)$(LIBEXECDIR)/olmoe$(EXE)
|
||||
$(INSTALL) -m 644 resource_plan.py doctor.py openai_server.py $(DESTDIR)$(LIBEXECDIR)/
|
||||
$(INSTALL) -m 644 tools/*.py $(DESTDIR)$(LIBEXECDIR)/tools/
|
||||
@@ -342,4 +388,4 @@ clean:
|
||||
|
||||
bench: iobench$(EXE)
|
||||
@if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) — run: ./iobench$(EXE) <file> <MB> <iters> <threads> <direct 0|1>"; fi
|
||||
.PHONY: all glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench
|
||||
.PHONY: all colibri glm cuda-test cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench
|
||||
|
||||
+143
-3
@@ -8,12 +8,21 @@
|
||||
#include <cstring>
|
||||
#include <mutex>
|
||||
|
||||
struct RaggedKVEntry {
|
||||
const void *key;
|
||||
const float *host_l,*host_r;
|
||||
float *latent,*rope;
|
||||
int length,capacity,K,R;
|
||||
};
|
||||
|
||||
struct ColiCudaTensor {
|
||||
void *weights;
|
||||
float *scales;
|
||||
size_t weight_bytes;
|
||||
int fmt, I, O, device;
|
||||
int tracked;
|
||||
RaggedKVEntry ragged[512];
|
||||
int ragged_count;
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
@@ -23,7 +32,7 @@ typedef struct {
|
||||
size_t x_cap, y_cap, gate_cap, up_cap;
|
||||
uint8_t *qx; float *qscale;
|
||||
size_t qx_cap, qscale_cap;
|
||||
float *host_x,*host_y; size_t host_x_cap,host_y_cap;
|
||||
float *host_x,*host_y,*host_kv; size_t host_x_cap,host_y_cap,host_kv_cap;
|
||||
float *aq,*al,*ar,*ac; size_t aq_cap,al_cap,ar_cap,ac_cap;
|
||||
float *pipe_buf[24]; size_t pipe_cap[24]; /* scratch persistenti del resident pipeline */
|
||||
cudaStream_t stream;
|
||||
@@ -338,6 +347,49 @@ __global__ static void attention_absorb_batch_kernel(float *ctx,const float *q,
|
||||
ctx[((size_t)s*H+h)*V+v]=a*(fmt?wscale[row]:1.f);}
|
||||
}
|
||||
|
||||
/* Independent device-resident KV sequence per row. lengths selects the valid
|
||||
* prefix; latent/rope point at paged caches updated by the host wrapper. */
|
||||
__global__ static void attention_absorb_ragged_kernel(float *ctx,const float *q,
|
||||
const float *const *latent,const float *const *rope,const int *lengths,
|
||||
const void *weights,const float *wscale,int fmt,int S,int H,int Q,int R,
|
||||
int V,int K,int T,float scale){
|
||||
int s=blockIdx.y,h=blockIdx.x,tid=threadIdx.x,nt=lengths[s],rbase=h*(Q+V);
|
||||
if(s>=S||nt<1||nt>T)return;
|
||||
extern __shared__ float sm[];float *qa=sm,*cl=qa+K,*scores=cl+K,*red=scores+T;
|
||||
const float *qs=q+((size_t)s*H+h)*(Q+R);
|
||||
const float *ls=latent[s],*rs=rope[s];
|
||||
for(int k=tid;k<K;k+=blockDim.x){float a=0;for(int d=0;d<Q;d++)
|
||||
a+=qs[d]*weight_at(weights,fmt,(size_t)(rbase+d)*row_bytes(fmt,K),k)*
|
||||
(fmt?wscale[rbase+d]:1.f);qa[k]=a;}
|
||||
__syncthreads();
|
||||
for(int t=tid;t<nt;t+=blockDim.x){float a=0;const float *lt=ls+(size_t)t*K;
|
||||
const float *rt=rs+(size_t)t*R;for(int k=0;k<K;k++)a+=qa[k]*lt[k];
|
||||
for(int d=0;d<R;d++)a+=qs[Q+d]*rt[d];scores[t]=a*scale;}
|
||||
__syncthreads();
|
||||
float local=-3.402823466e+38F;for(int t=tid;t<nt;t+=blockDim.x)local=fmaxf(local,scores[t]);
|
||||
red[tid]=local;__syncthreads();
|
||||
for(int n=blockDim.x>>1;n;n>>=1){if(tid<n)red[tid]=fmaxf(red[tid],red[tid+n]);__syncthreads();}
|
||||
float mx=red[0];local=0;for(int t=tid;t<nt;t+=blockDim.x){float e=expf(scores[t]-mx);scores[t]=e;local+=e;}
|
||||
red[tid]=local;__syncthreads();
|
||||
for(int n=blockDim.x>>1;n;n>>=1){if(tid<n)red[tid]+=red[tid+n];__syncthreads();}
|
||||
float inv=1.f/red[0];for(int t=tid;t<nt;t+=blockDim.x)scores[t]*=inv;
|
||||
__syncthreads();
|
||||
for(int k=tid;k<K;k+=blockDim.x){float a=0;for(int t=0;t<nt;t++)a+=scores[t]*ls[(size_t)t*K+k];cl[k]=a;}
|
||||
__syncthreads();
|
||||
for(int v=tid;v<V;v+=blockDim.x){int row=rbase+Q+v;float a=0;size_t rb=row_bytes(fmt,K);
|
||||
for(int k=0;k<K;k++)a+=cl[k]*weight_at(weights,fmt,(size_t)row*rb,k);
|
||||
ctx[((size_t)s*H+h)*V+v]=a*(fmt?wscale[row]:1.f);}
|
||||
}
|
||||
|
||||
__global__ static void ragged_kv_append(float *const *latent,float *const *rope,
|
||||
const float *packed,const int *old_len,const int *add,const int *offset,int K,int R){
|
||||
int s=blockIdx.x,n=add[s],base=offset[s];
|
||||
for(int i=threadIdx.x;i<n*(K+R);i+=blockDim.x){
|
||||
if(i<n*K)latent[s][(size_t)old_len[s]*K+i]=packed[base+i];
|
||||
else rope[s][(size_t)old_len[s]*R+i-n*K]=packed[base+i];
|
||||
}
|
||||
}
|
||||
|
||||
static int reserve(float **ptr, size_t *cap, size_t bytes) {
|
||||
if (*cap >= bytes) return 1;
|
||||
if (*ptr) cudaFree(*ptr);
|
||||
@@ -406,16 +458,17 @@ extern "C" void coli_cuda_shutdown(void) {
|
||||
for(int b=0;b<24;b++) if(ctx->pipe_buf[b]) cudaFree(ctx->pipe_buf[b]);
|
||||
if (ctx->host_x) cudaFreeHost(ctx->host_x);
|
||||
if (ctx->host_y) cudaFreeHost(ctx->host_y);
|
||||
if (ctx->host_kv) cudaFreeHost(ctx->host_kv);
|
||||
if (ctx->stream) cudaStreamDestroy(ctx->stream);
|
||||
if (ctx->group_desc) cudaFree(ctx->group_desc);
|
||||
ctx->x = ctx->y = ctx->gate = ctx->up = nullptr;
|
||||
ctx->qx=nullptr; ctx->qscale=nullptr;
|
||||
ctx->aq=ctx->al=ctx->ar=ctx->ac=nullptr;
|
||||
ctx->host_x=ctx->host_y=nullptr;ctx->stream=nullptr;
|
||||
ctx->host_x=ctx->host_y=ctx->host_kv=nullptr;ctx->stream=nullptr;
|
||||
ctx->x_cap = ctx->y_cap = ctx->gate_cap = ctx->up_cap = 0;
|
||||
ctx->qx_cap=ctx->qscale_cap=0;
|
||||
ctx->aq_cap=ctx->al_cap=ctx->ar_cap=ctx->ac_cap=0;
|
||||
ctx->host_x_cap=ctx->host_y_cap=0;
|
||||
ctx->host_x_cap=ctx->host_y_cap=ctx->host_kv_cap=0;
|
||||
ctx->group_desc=nullptr; ctx->group_desc_cap=0;
|
||||
}
|
||||
g_nctx = 0;
|
||||
@@ -770,6 +823,89 @@ extern "C" int coli_cuda_attention_project_batch(ColiCudaTensor *w,ColiCudaTenso
|
||||
return attention_absorb_batch_run(w,proj,out,q,latent,rope,S,H,Q,R,V,K,T,scale);
|
||||
}
|
||||
|
||||
extern "C" int coli_cuda_attention_project_ragged(ColiCudaTensor *w,ColiCudaTensor *proj,
|
||||
float *out,const float *q,const void *const *keys,
|
||||
const float *const *latent,const float *const *rope,
|
||||
const int *lengths,int S,int H,int Q,int R,int V,int K,int T,float scale){
|
||||
if(!w||!proj||!out||!q||!keys||!latent||!rope||!lengths||S<1||S>512||T<1||T>8192||
|
||||
H<1||Q<1||R<1||V<1||K<1||K>512||w->I!=K||w->O!=H*(Q+V)||
|
||||
proj->device!=w->device||proj->I!=H*V)return 0;
|
||||
DeviceContext *dc=find_ctx(w->device);
|
||||
if(!select_ctx(dc))return 0;
|
||||
float **dl=(float**)std::malloc((size_t)S*sizeof(*dl));
|
||||
float **dr=(float**)std::malloc((size_t)S*sizeof(*dr));
|
||||
int *old=(int*)std::malloc((size_t)S*sizeof(*old));
|
||||
int *add=(int*)std::malloc((size_t)S*sizeof(*add));
|
||||
int *off=(int*)std::malloc((size_t)S*sizeof(*off));int packed_n=0;
|
||||
if(!dl||!dr||!old||!add||!off){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;}
|
||||
for(int s=0;s<S;s++){
|
||||
if(!keys[s]||lengths[s]<1||lengths[s]>T){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;}
|
||||
RaggedKVEntry *e=nullptr;
|
||||
for(int i=0;i<w->ragged_count;i++)if(w->ragged[i].key==keys[s]){e=&w->ragged[i];break;}
|
||||
if(!e){
|
||||
if(w->ragged_count>=512){std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;}
|
||||
e=&w->ragged[w->ragged_count++];std::memset(e,0,sizeof(*e));e->key=keys[s];
|
||||
}
|
||||
if(e->K!=K||e->R!=R||e->host_l!=latent[s]||e->host_r!=rope[s]||lengths[s]<e->length){
|
||||
if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope);
|
||||
e->latent=e->rope=nullptr;e->length=e->capacity=0;
|
||||
e->K=K;e->R=R;e->host_l=latent[s];e->host_r=rope[s];
|
||||
}
|
||||
if(lengths[s]>e->capacity){
|
||||
int cap=(lengths[s]+63)&~63;float *nl=nullptr,*nr=nullptr;
|
||||
if(!cuda_ok(cudaMalloc(&nl,(size_t)cap*K*sizeof(float)),"ragged KV latent page")||
|
||||
!cuda_ok(cudaMalloc(&nr,(size_t)cap*R*sizeof(float)),"ragged KV rope page")){
|
||||
if(nl)cudaFree(nl);if(nr)cudaFree(nr);std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);return 0;
|
||||
}
|
||||
if(e->length){
|
||||
cudaMemcpyAsync(nl,e->latent,(size_t)e->length*K*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream);
|
||||
cudaMemcpyAsync(nr,e->rope,(size_t)e->length*R*sizeof(float),cudaMemcpyDeviceToDevice,dc->stream);
|
||||
}
|
||||
if(e->latent)cudaFree(e->latent);if(e->rope)cudaFree(e->rope);
|
||||
e->latent=nl;e->rope=nr;e->capacity=cap;
|
||||
}
|
||||
dl[s]=e->latent;dr[s]=e->rope;old[s]=e->length;add[s]=lengths[s]-e->length;
|
||||
off[s]=packed_n;packed_n+=add[s]*(K+R);
|
||||
}
|
||||
size_t qb=(size_t)S*H*(Q+R)*sizeof(float);
|
||||
size_t cb=(size_t)S*H*V*sizeof(float),ob=(size_t)S*proj->O*sizeof(float);
|
||||
size_t pb=(size_t)packed_n*sizeof(float);
|
||||
size_t desc=(size_t)S*(2*sizeof(float*)+4*sizeof(int));
|
||||
int ok=reserve(&dc->aq,&dc->aq_cap,qb)&&reserve(&dc->ac,&dc->ac_cap,cb)&&
|
||||
reserve(&dc->y,&dc->y_cap,ob)&&reserve_bytes(&dc->group_desc,&dc->group_desc_cap,desc)&&
|
||||
(!pb||(reserve(&dc->al,&dc->al_cap,pb)&&reserve_pinned(&dc->host_kv,&dc->host_kv_cap,pb)));
|
||||
char *db=(char*)dc->group_desc;float **ddl=(float**)db,**ddr=ddl+S;
|
||||
int *dn=(int*)(ddr+S),*dold=dn+S,*dadd=dold+S,*doff=dadd+S;
|
||||
if(ok&&pb){
|
||||
for(int s=0;s<S;s++)if(add[s]){
|
||||
float *p=dc->host_kv+off[s];
|
||||
std::memcpy(p,latent[s]+(size_t)old[s]*K,(size_t)add[s]*K*sizeof(float));
|
||||
std::memcpy(p+(size_t)add[s]*K,rope[s]+(size_t)old[s]*R,(size_t)add[s]*R*sizeof(float));
|
||||
}
|
||||
ok=cuda_ok(cudaMemcpyAsync(dc->al,dc->host_kv,pb,cudaMemcpyHostToDevice,dc->stream),"ragged KV append upload");
|
||||
}
|
||||
if(ok)ok=cuda_ok(cudaMemcpyAsync(dc->aq,q,qb,cudaMemcpyHostToDevice,dc->stream),"ragged q upload")&&
|
||||
cuda_ok(cudaMemcpyAsync(ddl,dl,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged latent pointers")&&
|
||||
cuda_ok(cudaMemcpyAsync(ddr,dr,(size_t)S*sizeof(float*),cudaMemcpyHostToDevice,dc->stream),"ragged rope pointers")&&
|
||||
cuda_ok(cudaMemcpyAsync(dn,lengths,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged lengths upload")&&
|
||||
cuda_ok(cudaMemcpyAsync(dold,old,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged old lengths")&&
|
||||
cuda_ok(cudaMemcpyAsync(dadd,add,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append lengths")&&
|
||||
cuda_ok(cudaMemcpyAsync(doff,off,(size_t)S*sizeof(int),cudaMemcpyHostToDevice,dc->stream),"ragged append offsets");
|
||||
if(ok&&pb)ragged_kv_append<<<S,256,0,dc->stream>>>(ddl,ddr,dc->al,dold,dadd,doff,K,R);
|
||||
if(ok)for(int s=0;s<S;s++){
|
||||
for(int i=0;i<w->ragged_count;i++)if(w->ragged[i].key==keys[s]){w->ragged[i].length=lengths[s];break;}
|
||||
}
|
||||
std::free(dl);std::free(dr);std::free(old);std::free(add);std::free(off);if(!ok)return 0;
|
||||
size_t shared=(size_t)(2*K+T+256)*sizeof(float);
|
||||
attention_absorb_ragged_kernel<<<dim3(H,S),256,shared,dc->stream>>>(dc->ac,dc->aq,ddl,ddr,
|
||||
dn,w->weights,w->scales,w->fmt,S,H,Q,R,V,K,T,scale);
|
||||
quant_matmul<<<dim3(proj->O,S),256,0,dc->stream>>>(dc->y,dc->ac,proj->weights,
|
||||
proj->scales,proj->fmt,S,proj->I,proj->O,row_bytes(proj->fmt,proj->I));
|
||||
return cuda_ok(cudaGetLastError(),"ragged attention launch")&&
|
||||
cuda_ok(cudaMemcpyAsync(out,dc->y,ob,cudaMemcpyDeviceToHost,dc->stream),"ragged output download")&&
|
||||
cuda_ok(cudaStreamSynchronize(dc->stream),"ragged attention synchronize");
|
||||
}
|
||||
|
||||
extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {
|
||||
if (!tensor) return;
|
||||
DeviceContext *ctx = find_ctx(tensor->device);
|
||||
@@ -781,6 +917,10 @@ extern "C" void coli_cuda_tensor_free(ColiCudaTensor *tensor) {
|
||||
}
|
||||
if (tensor->weights) cudaFree(tensor->weights);
|
||||
if (tensor->scales) cudaFree(tensor->scales);
|
||||
for(int i=0;i<tensor->ragged_count;i++){
|
||||
if(tensor->ragged[i].latent)cudaFree(tensor->ragged[i].latent);
|
||||
if(tensor->ragged[i].rope)cudaFree(tensor->ragged[i].rope);
|
||||
}
|
||||
std::free(tensor);
|
||||
}
|
||||
|
||||
|
||||
+6
-1
@@ -14,6 +14,7 @@
|
||||
#define COLI_CUDA_DLLEXPORT
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
@@ -92,6 +93,11 @@ COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,C
|
||||
const float *rope,int S,int H,int Q,int R,
|
||||
int V,int K,int T,float attention_scale);
|
||||
|
||||
COLI_CUDA_DLLEXPORT int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
|
||||
float *out,const float *q,const void *const *keys,
|
||||
const float *const *latent,const float *const *rope,
|
||||
const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale);
|
||||
|
||||
COLI_CUDA_DLLEXPORT void coli_cuda_tensor_free(ColiCudaTensor *tensor);
|
||||
COLI_CUDA_DLLEXPORT size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor);
|
||||
COLI_CUDA_DLLEXPORT int coli_cuda_tensor_device(const ColiCudaTensor *tensor);
|
||||
@@ -143,4 +149,3 @@ COLI_CUDA_DLLEXPORT int coli_cuda_pipe_sync(int device);
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
@@ -61,6 +61,10 @@ typedef int (*fn_attention_absorb_batch)(ColiCudaTensor *kv_b,float *ctx,const f
|
||||
typedef int (*fn_attention_absorb_batch_dev)(ColiCudaTensor *kv_b_shard,float *ctx_dev, const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale);
|
||||
typedef int (*fn_attention_absorb_kvdev)(ColiCudaTensor *kv_b,float *ctx,const float *q, const float *latent_dev,const float *rope_dev,int H,int Q,int R,int V,int K,int T, float scale);
|
||||
typedef int (*fn_attention_project_batch)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q,const float *latent, const float *rope,int S,int H,int Q,int R, int V,int K,int T,float attention_scale);
|
||||
typedef int (*fn_attention_project_ragged)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
|
||||
float *out,const float *q,const void *const *keys,
|
||||
const float *const *latent,const float *const *rope,
|
||||
const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale);
|
||||
typedef int (*fn_attention_project_batch_dev)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale);
|
||||
typedef int (*fn_attention_project_batch_dev_out)(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out_dev,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale);
|
||||
typedef int (*fn_pipe_add)(int device,float *x_dev,const float *t_dev,size_t n);
|
||||
@@ -107,6 +111,7 @@ static struct {
|
||||
fn_attention_absorb_batch_dev attention_absorb_batch_dev;
|
||||
fn_attention_absorb_kvdev attention_absorb_kvdev;
|
||||
fn_attention_project_batch attention_project_batch;
|
||||
fn_attention_project_ragged attention_project_ragged;
|
||||
fn_attention_project_batch_dev attention_project_batch_dev;
|
||||
fn_attention_project_batch_dev_out attention_project_batch_dev_out;
|
||||
fn_pipe_add pipe_add;
|
||||
@@ -200,6 +205,7 @@ static int coli_cuda_load(void){
|
||||
RESOLVE(attention_absorb_batch_dev, fn_attention_absorb_batch_dev)
|
||||
RESOLVE(attention_absorb_kvdev, fn_attention_absorb_kvdev)
|
||||
RESOLVE(attention_project_batch, fn_attention_project_batch)
|
||||
RESOLVE(attention_project_ragged, fn_attention_project_ragged)
|
||||
RESOLVE(attention_project_batch_dev, fn_attention_project_batch_dev)
|
||||
RESOLVE(attention_project_batch_dev_out, fn_attention_project_batch_dev_out)
|
||||
RESOLVE(pipe_add, fn_pipe_add)
|
||||
@@ -342,6 +348,15 @@ int coli_cuda_attention_project_batch(ColiCudaTensor *kv_b,ColiCudaTensor *o_pro
|
||||
return g_cuda.attention_project_batch(kv_b, o_proj, out, q, latent, rope, S, H, Q, R, V, K, T, attention_scale);
|
||||
}
|
||||
|
||||
int coli_cuda_attention_project_ragged(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj,
|
||||
float *out,const float *q,const void *const *keys,
|
||||
const float *const *latent,const float *const *rope,
|
||||
const int *lengths,int S,int H,int Q,int R,int V,int K,int max_t,float attention_scale){
|
||||
if(!coli_cuda_load()) return 0;
|
||||
return g_cuda.attention_project_ragged(kv_b,o_proj,out,q,keys,latent,rope,lengths,
|
||||
S,H,Q,R,V,K,max_t,attention_scale);
|
||||
}
|
||||
|
||||
int coli_cuda_attention_project_batch_dev(ColiCudaTensor *kv_b,ColiCudaTensor *o_proj, float *out,const float *q_dev,const float *latent_dev,const float *rope_dev, int S,int H,int Q,int R,int V,int K,int T,float scale){
|
||||
if(!g_cuda.available){ return 0; }
|
||||
return g_cuda.attention_project_batch_dev(kv_b, o_proj, out, q_dev, latent_dev, rope_dev, S, H, Q, R, V, K, T, scale);
|
||||
|
||||
@@ -20,7 +20,7 @@ Configuration through environment variables or flags (also valid after the subco
|
||||
--topp P adaptive expert top-p --topk N fixed top-k
|
||||
--ngen N maximum response tokens --cap N cache slots/layer
|
||||
"""
|
||||
import os, sys, subprocess, argparse, json, time, signal, shutil, threading, re, codecs, tempfile, textwrap
|
||||
import os, sys, subprocess, argparse, json, time, signal, shutil, threading, re, codecs, tempfile, textwrap, struct
|
||||
|
||||
# The engine mmaps every shard (144+ files); macOS default RLIMIT_NOFILE is 256.
|
||||
if sys.platform != "win32":
|
||||
@@ -40,6 +40,8 @@ if sys.platform == "win32":
|
||||
except (AttributeError, OSError): pass
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
from version import __version__ as _version
|
||||
|
||||
# Run-in-place (source checkout, "cd c && ./coli ..."): the engine, the
|
||||
# support modules (resource_plan.py, doctor.py, openai_server.py) and
|
||||
@@ -52,16 +54,20 @@ HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
# guess is right (e.g. a custom packaging layout).
|
||||
_EXE = ".exe" if sys.platform == "win32" else ""
|
||||
_LIBEXEC = os.path.join(os.path.dirname(HERE), "libexec", "colibri")
|
||||
_here_colibri = os.path.join(HERE, "colibri" + _EXE)
|
||||
_here_glm = os.path.join(HERE, "glm" + _EXE)
|
||||
|
||||
if os.environ.get("COLI_ENGINE"):
|
||||
GLM = os.environ["COLI_ENGINE"]
|
||||
TOOLS = os.path.join(os.path.dirname(GLM), "tools")
|
||||
elif os.path.exists(_here_colibri):
|
||||
GLM = _here_colibri
|
||||
TOOLS = os.path.join(HERE, "tools")
|
||||
elif os.path.exists(_here_glm):
|
||||
GLM = _here_glm
|
||||
TOOLS = os.path.join(HERE, "tools")
|
||||
else:
|
||||
GLM = os.path.join(_LIBEXEC, "glm" + _EXE)
|
||||
GLM = os.path.join(_LIBEXEC, "colibri" + _EXE)
|
||||
TOOLS = os.path.join(_LIBEXEC, "tools")
|
||||
sys.path.insert(0, _LIBEXEC) # so `import resource_plan`, `doctor`, `openai_server` still resolve
|
||||
|
||||
@@ -115,7 +121,7 @@ def sprite_lines():
|
||||
def banner(sub=""):
|
||||
sp=sprite_lines()
|
||||
txt=[
|
||||
f"{C.teal}{C.b}colibrì{C.r} {C.dim}v1.0{C.r}",
|
||||
f"{C.teal}{C.b}colibrì{C.r} {C.dim}v{_version}{C.r}",
|
||||
f"{C.dim}tiny engine, immense model{C.r}",
|
||||
f"{C.gray}GLM-5.2 · 744B MoE · int4 · streaming CPU{C.r}",
|
||||
f"{C.dgray}{sub}{C.r}" if sub else "",
|
||||
@@ -141,12 +147,24 @@ def need_model(model):
|
||||
sys.exit(f"{C.yel}engine is not built.{C.r} Run: coli build")
|
||||
|
||||
def cuda_binary():
|
||||
if not os.path.exists(GLM) or sys.platform != "linux": return False
|
||||
try:
|
||||
linked=subprocess.run(["ldd",GLM],capture_output=True,text=True,timeout=3)
|
||||
return any("libcudart" in line and "not found" not in line
|
||||
for line in linked.stdout.splitlines())
|
||||
except (OSError,subprocess.SubprocessError): return False
|
||||
if not os.path.exists(GLM): return False
|
||||
if sys.platform == "linux":
|
||||
try:
|
||||
linked=subprocess.run(["ldd",GLM],capture_output=True,text=True,timeout=3)
|
||||
return any("libcudart" in line and "not found" not in line
|
||||
for line in linked.stdout.splitlines())
|
||||
except (OSError,subprocess.SubprocessError): return False
|
||||
if sys.platform == "win32":
|
||||
# Windows CUDA_DLL=1 builds never link libcudart directly: glm.exe loads
|
||||
# coli_cuda.dll at runtime via LoadLibrary (backend_loader.c), so there's no
|
||||
# import-table entry for ldd/dumpbin to see. Detect the COLI_CUDA build via a
|
||||
# marker string baked into glm.c's #ifdef COLI_CUDA block instead, and require
|
||||
# coli_cuda.dll to actually sit next to glm.exe (else CUDA init fails at startup).
|
||||
try:
|
||||
with open(GLM,"rb") as f: built=b"[CUDA] mode: routed experts" in f.read()
|
||||
except OSError: return False
|
||||
return built and os.path.exists(os.path.join(os.path.dirname(GLM),"coli_cuda.dll"))
|
||||
return False
|
||||
|
||||
def resource_request(a, env):
|
||||
ctx=a.ctx or int(env.get("CTX",4096))
|
||||
@@ -227,13 +245,13 @@ def env_for(a):
|
||||
e["COLI_CUDA"]="0"; e.pop("CUDA_EXPERT_GB",None); e.pop("CUDA_DENSE",None)
|
||||
else:
|
||||
if not cuda_binary():
|
||||
sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} make glm CUDA=1 (this binary is CPU-only)")
|
||||
sys.exit(f"{C.yel}--gpu needs the CUDA build:{C.r} make colibri CUDA=1 (this binary is CPU-only)")
|
||||
e["COLI_CUDA"]="1"
|
||||
if a.gpu!="auto": e["COLI_GPUS"]=a.gpu
|
||||
e.setdefault("CUDA_DENSE","1")
|
||||
if a.vram and a.gpu!="none":
|
||||
if not cuda_binary():
|
||||
sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} make glm CUDA=1 (this binary is CPU-only)")
|
||||
sys.exit(f"{C.yel}--vram needs the CUDA build:{C.r} make colibri CUDA=1 (this binary is CPU-only)")
|
||||
e["COLI_CUDA"]="1"; e["CUDA_EXPERT_GB"]=str(a.vram)
|
||||
return e
|
||||
|
||||
@@ -407,8 +425,8 @@ def cmd_build(a):
|
||||
banner("build")
|
||||
if not os.path.exists(os.path.join(HERE, "Makefile")):
|
||||
sys.exit(f"{C.yel}coli build{C.r} only works from a source checkout (this is an installed copy).\n"
|
||||
f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c glm.")
|
||||
sys.exit(subprocess.call(["make","-C",HERE,"glm"]))
|
||||
f" Clone https://github.com/JustVugg/colibri and run ./setup.sh, or make -C c colibri.")
|
||||
sys.exit(subprocess.call(["make","-C",HERE,"colibri"]))
|
||||
|
||||
def cmd_info(a):
|
||||
banner("info")
|
||||
@@ -480,13 +498,141 @@ def cmd_run(a):
|
||||
need_model(a.model)
|
||||
prompt=" ".join(a.prompt) if a.prompt else sys.exit('usage: coli run "your prompt"')
|
||||
banner("run")
|
||||
# template ufficiale GLM-5.2: niente \n dopo i ruoli; <think></think> = risposta diretta (nothink)
|
||||
e=env_for(a); e["PROMPT"]=f"[gMASK]<sop><|user|>{prompt}<|assistant|><think></think>"
|
||||
# template ufficiale GLM-5.2: niente \n dopo i ruoli; <think></think> = risposta diretta (nothink).
|
||||
# THINK=1 lascia <think> aperto, stessa convenzione del serve mode (glm.c). EN: THINK=1 leaves
|
||||
# <think> open so the engine emits its reasoning block; the default stays nothink.
|
||||
tk="<think>" if os.environ.get("THINK","0")=="1" else "<think></think>"
|
||||
e=env_for(a); e["PROMPT"]=f"[gMASK]<sop><|user|>{prompt}<|assistant|>{tk}"
|
||||
sys.exit(subprocess.call([GLM, str(a.cap)], env=e))
|
||||
|
||||
def server_probe(base, api_key=None, timeout=1.5):
|
||||
"""Is a coli serve alive at `base`? Returns its model_id, or None.
|
||||
Probes /health then /v1/models — both cheap, neither touches the engine."""
|
||||
import urllib.request, urllib.error
|
||||
def get(path):
|
||||
req=urllib.request.Request(base.rstrip("/")+path)
|
||||
if api_key: req.add_header("Authorization", f"Bearer {api_key}")
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return json.loads(r.read().decode("utf-8","replace"))
|
||||
try:
|
||||
if get("/health").get("status")!="ok": return None
|
||||
data=get("/v1/models").get("data") or []
|
||||
return data[0]["id"] if data else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def chat_attached(a, base, model_id):
|
||||
"""The chat REPL over HTTP against a running `coli serve`.
|
||||
|
||||
Why this exists (the cold-chat cost, measured): spawning a private engine
|
||||
pays 34-136 s of resident load on EVERY start, and begins with an empty
|
||||
expert cache — hit rate 4% cold vs 55% warm, a ~10x on early decode. A
|
||||
resident server pays load once and keeps the LRU warm across sessions;
|
||||
its KV slots reuse the conversation prefix, so a continued chat skips
|
||||
re-prefill too. The engine byte-protocol stays untouched — this is plain
|
||||
OpenAI SSE over localhost, stdlib only."""
|
||||
import urllib.request
|
||||
print(f" {C.grn}✦ attached{C.r} {C.dim}to {base} · model {model_id} · the engine stays warm after you quit{C.r}")
|
||||
print(f" {C.dim}type and press Enter · Ctrl-C stops the answer · :reset starts a new conversation · :q exits{C.r}\n")
|
||||
msgs=[]
|
||||
w=term_w()-4
|
||||
while True:
|
||||
if TTY:
|
||||
print(f" {C.dgray}╭{'─'*w}╮{C.r}")
|
||||
try: msg=input(f" {C.dgray}│{C.r} {C.teal}{C.b}›{C.r} ")
|
||||
except EOFError: print(); break
|
||||
print(f" {C.dgray}╰{'─'*w}╯{C.r}")
|
||||
else:
|
||||
try: msg=input()
|
||||
except EOFError: break
|
||||
msg=msg.strip()
|
||||
if msg in (":q",":quit","exit"): break
|
||||
if not msg: continue
|
||||
if msg==":reset": msgs=[]; print(f" {C.dim}✦ new conversation{C.r}\n"); continue
|
||||
msgs.append({"role":"user","content":msg})
|
||||
body=json.dumps({"model":model_id,"messages":msgs,"stream":True,
|
||||
"max_tokens":a.ngen}).encode()
|
||||
req=urllib.request.Request(base.rstrip("/")+"/v1/chat/completions", data=body,
|
||||
headers={"Content-Type":"application/json"})
|
||||
if a.api_key: req.add_header("Authorization", f"Bearer {a.api_key}")
|
||||
print(f"\n {C.teal}◆ colibrì{C.r}")
|
||||
sp=Spinner("thinking…"); sp.start()
|
||||
md=MDStream(" "); reply=[]; first=True; t0=time.time(); interrupted=False
|
||||
try:
|
||||
with urllib.request.urlopen(req) as r:
|
||||
for raw in r:
|
||||
line=raw.decode("utf-8","replace").strip()
|
||||
if not line.startswith("data: "): continue
|
||||
data=line[6:]
|
||||
if data=="[DONE]": break
|
||||
try: ev=json.loads(data)
|
||||
except ValueError: continue
|
||||
for ch in ev.get("choices",[]):
|
||||
d=ch.get("delta",{})
|
||||
txt=d.get("content")
|
||||
if not txt: continue # ping/ruolo/reasoning: non è testo
|
||||
if first: sp.stop(); first=False
|
||||
md.feed(txt); reply.append(txt)
|
||||
except KeyboardInterrupt:
|
||||
interrupted=True # il server annulla la richiesta alla disconnessione
|
||||
except OSError as e:
|
||||
sp.stop()
|
||||
print(f"\n {C.yel}[server unreachable: {e}]{C.r}"); break
|
||||
sp.stop(); md.close()
|
||||
if reply: msgs.append({"role":"assistant","content":"".join(reply)})
|
||||
else: msgs.pop() # turno vuoto: non sporcare la history
|
||||
el=time.time()-t0
|
||||
note=" · ⏹ interrupted" if interrupted else ""
|
||||
print(f"\r {C.dgray}└─ ~{len(''.join(reply))//4} tok · {el:.0f}s{note}{C.r}\n")
|
||||
print(f" {C.dim}goodbye — the engine keeps running for the next chat 🐦{C.r}")
|
||||
|
||||
def kv_resume_notice(model_dir):
|
||||
"""SERVE mode silently resumes .coli_kv from disk (glm.c kv_disk_load): a chat
|
||||
started today continues a conversation from days ago, with `first=0` so the
|
||||
turn is appended WITHOUT the [gMASK]<sop> prefix. The engine does announce it
|
||||
on stderr — but nothing here ever shows that: the drain thread's
|
||||
p.stderr.read() blocks until EOF, so on a healthy start errlog is still empty
|
||||
when the status lines are printed. The warning only appeared once the engine
|
||||
DIED, which is exactly when it no longer mattered.
|
||||
|
||||
Measured cost of the silence: a chat inherited 670 tokens of an old Italian
|
||||
session ("il mio numero preferito e 7, ricordalo!"). Every later reply came
|
||||
back in Italian, and "explain fibonacci in short" was answered about the
|
||||
number 7 — the model was being coherent with a context nobody could see, and
|
||||
it read as a quantization bug for a day.
|
||||
|
||||
So say it here, in Python, from the file itself: no pipe, no thread, no
|
||||
Windows deadlock risk (see the stderr comment below)."""
|
||||
p=os.path.join(model_dir, ".coli_kv")
|
||||
try:
|
||||
with open(p,"rb") as f:
|
||||
if f.read(8)!=b"COLIKV1\0": return
|
||||
h=struct.unpack("<8i", f.read(32))
|
||||
n=h[6]
|
||||
if n<1: return
|
||||
age=time.time()-os.path.getmtime(p)
|
||||
when=f"{age/86400:.0f}d ago" if age>86400 else f"{age/3600:.0f}h ago" if age>3600 else "just now"
|
||||
print(f" {C.yel}↺ resuming a saved conversation: {n} tokens, last written {when}{C.r}")
|
||||
print(f" {C.dgray} it steers tone, language and topic. :reset clears it · "
|
||||
f"KVSAVE=0 disables saving · delete {p} to start clean{C.r}")
|
||||
except (OSError, struct.error): pass
|
||||
|
||||
def cmd_chat(a):
|
||||
# ATTACH: a running `coli serve` beats a private engine every time — the load
|
||||
# (34-136 s) and the cache warmth survive between sessions. Explicit --attach
|
||||
# wins; otherwise probe localhost quietly and use it if it's there. --no-attach
|
||||
# forces the old behaviour. The probe costs ~1 ms when nothing is listening.
|
||||
if not getattr(a,"no_attach",False):
|
||||
base=getattr(a,"attach",None) or "http://127.0.0.1:8000"
|
||||
mid=server_probe(base, getattr(a,"api_key",None))
|
||||
if mid:
|
||||
banner(f"chat · {mid} · attached")
|
||||
chat_attached(a, base, mid); return
|
||||
if getattr(a,"attach",None):
|
||||
sys.exit(f"--attach: no coli serve answering at {base} (start one with: coli serve --model <dir>)")
|
||||
need_model(a.model)
|
||||
banner(f"chat · {os.path.basename(a.model)} · ram {a.ram or '-'}GB · topp {a.topp or 'off'}")
|
||||
kv_resume_notice(a.model)
|
||||
errlog=tempfile.NamedTemporaryFile(mode="w+", suffix=".log", delete=False)
|
||||
e=env_for(a); e["SERVE"]="1"
|
||||
# stderr -> PIPE, NOT stderr=errlog (file). On Windows/MinGW, pointing the
|
||||
@@ -619,12 +765,65 @@ def cmd_chat(a):
|
||||
except Exception: pass
|
||||
print(f" {C.teal}goodbye{C.r} {C.dim}— the hummingbird returns to its nest{C.r} 🐦\n")
|
||||
|
||||
def serve_pidfile(port): return os.path.join(tempfile.gettempdir(), f"coli-serve-{port}.pid")
|
||||
|
||||
def cmd_serve(a):
|
||||
need_model(a.model)
|
||||
# pidfile: cosi' `coli stop` spegne tutto con un comando, senza pkill a mano.
|
||||
# EN: pidfile so `coli stop` can shut everything down without manual pkill.
|
||||
try:
|
||||
with open(serve_pidfile(a.port),"w") as f: f.write(f"{os.getpid()} {a.model}\n")
|
||||
except OSError: pass
|
||||
from openai_server import serve
|
||||
serve(a.model, a.host, a.port, a.model_id, a.api_key,
|
||||
a.cap,a.ngen,GLM,env_for(a),a.cors_origin,
|
||||
a.max_queue,a.queue_timeout,a.kv_slots)
|
||||
try:
|
||||
serve(a.model, a.host, a.port, a.model_id, a.api_key,
|
||||
a.cap,a.ngen,GLM,env_for(a),a.cors_origin,
|
||||
a.max_queue,a.queue_timeout,a.kv_slots)
|
||||
finally:
|
||||
try: os.unlink(serve_pidfile(a.port))
|
||||
except OSError: pass
|
||||
|
||||
def cmd_stop(a):
|
||||
"""Shut down a running `coli serve` AND its engine — one command, no pkill.
|
||||
The engine re-execs itself for OMP tuning, so its process is named `exe`,
|
||||
not `glm`: every `pkill -x glm` in history silently killed nothing (that is
|
||||
how two 17+5 GB ghost engines OOM'd this box on 2026-07-16). This finds the
|
||||
real processes: the pidfile first, then /proc by cmdline/environ — only
|
||||
processes that are demonstrably ours (SERVE=1 + our SNAP, or `coli serve`
|
||||
in the command line)."""
|
||||
banner("stop")
|
||||
targets=[] # (pid, descrizione)
|
||||
pf=serve_pidfile(a.port)
|
||||
try:
|
||||
pid=int(open(pf).read().split()[0])
|
||||
os.kill(pid,0); targets.append((pid,f"coli serve (pidfile, port {a.port})"))
|
||||
except (OSError,ValueError,IndexError): pass
|
||||
for pd in os.listdir("/proc"):
|
||||
if not pd.isdigit(): continue
|
||||
pid=int(pd)
|
||||
try:
|
||||
cmd=open(f"/proc/{pd}/cmdline","rb").read().replace(b"\0",b" ").decode("utf-8","replace")
|
||||
if "coli" in cmd and " serve" in cmd and pid!=os.getpid():
|
||||
if not any(p==pid for p,_ in targets): targets.append((pid,"coli serve (cmdline)"))
|
||||
comm=open(f"/proc/{pd}/comm").read().strip()
|
||||
if comm in ("colibri","glm","exe","olmoe"):
|
||||
env=open(f"/proc/{pd}/environ","rb").read().replace(b"\0",b"\n").decode("utf-8","replace")
|
||||
if "SERVE=1" in env: targets.append((pid,f"engine `{comm}` (SERVE=1)"))
|
||||
except (OSError,PermissionError): continue
|
||||
if not targets:
|
||||
print(f" nothing running — no serve on port {a.port}, no SERVE engines"); return
|
||||
for pid,desc in targets: print(f" {'would stop' if a.dry_run else 'stopping'} {pid}: {desc}")
|
||||
if a.dry_run: return
|
||||
for pid,_ in targets:
|
||||
try: os.kill(pid, signal.SIGTERM)
|
||||
except OSError: pass
|
||||
time.sleep(2.0)
|
||||
for pid,_ in targets:
|
||||
try: os.kill(pid, signal.SIGKILL); print(f" {pid}: forced (SIGKILL)")
|
||||
except OSError: pass # gia' morto: bene
|
||||
try: os.unlink(pf)
|
||||
except OSError: pass
|
||||
print(f" {C.grn}✓ stopped{C.r} — RAM released")
|
||||
|
||||
def cmd_web(a):
|
||||
"""serve + open the dashboard in the browser once the API answers."""
|
||||
@@ -712,6 +911,7 @@ def main():
|
||||
common.add_argument("--topp", type=float, default=0); common.add_argument("--topk", type=int, default=0)
|
||||
common.add_argument("--temp", type=float, default=None) # temperatura token (0=greedy, default 1.0+nucleus .95)
|
||||
ap=argparse.ArgumentParser(prog="coli", parents=[common], description="colibrì — run GLM-5.2 locally")
|
||||
ap.add_argument("--version", action="version", version=f"colibrì {_version}")
|
||||
sub=ap.add_subparsers(dest="cmd")
|
||||
sub.add_parser("build", parents=[common]); sub.add_parser("info", parents=[common])
|
||||
pp=sub.add_parser("plan",parents=[common])
|
||||
@@ -719,7 +919,14 @@ def main():
|
||||
pd=sub.add_parser("doctor",parents=[common])
|
||||
pd.add_argument("--json",action="store_true",help="emit a versioned JSON report")
|
||||
pr=sub.add_parser("run", parents=[common]); pr.add_argument("prompt", nargs="*")
|
||||
sub.add_parser("chat", parents=[common])
|
||||
pc=sub.add_parser("chat", parents=[common])
|
||||
pc.add_argument("--attach", nargs="?", const="http://127.0.0.1:8000", default=None,
|
||||
help="chat against a running `coli serve` instead of spawning an engine "
|
||||
"(keeps the model loaded and the expert cache warm across chat sessions). "
|
||||
"Bare --attach probes localhost:8000.")
|
||||
pc.add_argument("--no-attach", action="store_true",
|
||||
help="never auto-attach, always spawn a private engine")
|
||||
pc.add_argument("--api-key", default=os.environ.get("COLI_API_KEY"))
|
||||
ps=sub.add_parser("serve", parents=[common])
|
||||
ps.add_argument("--host",default="127.0.0.1"); ps.add_argument("--port",type=int,default=8000)
|
||||
ps.add_argument("--model-id",default=os.environ.get("COLI_MODEL_ID","glm-5.2-colibri"))
|
||||
@@ -728,6 +935,8 @@ def main():
|
||||
ps.add_argument("--max-queue",type=int,default=int(os.environ.get("COLI_MAX_QUEUE","8")))
|
||||
ps.add_argument("--queue-timeout",type=float,default=float(os.environ.get("COLI_QUEUE_TIMEOUT","300")))
|
||||
ps.add_argument("--kv-slots",type=int,default=int(os.environ.get("COLI_KV_SLOTS","1")))
|
||||
pst=sub.add_parser("stop", parents=[common], help="shut down a running coli serve and its engine")
|
||||
pst.add_argument("--port",type=int,default=8000); pst.add_argument("--dry-run",action="store_true")
|
||||
pw=sub.add_parser("web", parents=[common], help="serve + open the dashboard in a browser")
|
||||
for arg,kw in (("--host",dict(default="127.0.0.1")),("--port",dict(type=int,default=8000)),
|
||||
("--model-id",dict(default=os.environ.get("COLI_MODEL_ID","glm-5.2-colibri"))),
|
||||
@@ -751,7 +960,7 @@ def main():
|
||||
pc.add_argument("--no-mtp",action="store_true",help="skip the MTP head (no speculative drafts)")
|
||||
a=ap.parse_args()
|
||||
handler={"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"doctor":cmd_doctor,
|
||||
"run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"bench":cmd_bench,
|
||||
"run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"stop":cmd_stop,"bench":cmd_bench,
|
||||
"convert":cmd_convert,"web":cmd_web}.get(a.cmd)
|
||||
if handler: sys.exit(handler(a) or 0)
|
||||
banner(); print(__doc__)
|
||||
|
||||
+856
-1255
File diff suppressed because it is too large
Load Diff
+55
-1
@@ -80,6 +80,7 @@ static inline int compat_open_direct(const char *path){
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#include <io.h>
|
||||
#include <direct.h> /* _mkdir (for the mkdtemp shim below) */
|
||||
#include <process.h>
|
||||
#include <malloc.h>
|
||||
#include <fcntl.h>
|
||||
@@ -143,6 +144,12 @@ static inline int compat_fadvise(int fd, off_t off, off_t len, int advice){
|
||||
* Thread-safe (no shared seek position). Gestisce offset >4 GB e chunking
|
||||
* per letture >2 GB (anche se i tensori individuali sono nell'ordine dei
|
||||
* MB-centinaia di MB, il wrapper e' robusto per ogni taglia). */
|
||||
/* Ultimo GetLastError() di una ReadFile fallita, per thread: il chiamante
|
||||
* (pread_full in glm.c) lo stampa accanto a strerror. Senza questo, OGNI
|
||||
* fallimento Windows collassa in "EIO -> Input/output error" e la diagnosi
|
||||
* dal campo diventa un tirare a indovinare (#307: tre giri di ipotesi tra
|
||||
* tre persone perche' il codice vero non compariva da nessuna parte). */
|
||||
static __thread DWORD compat_pread_lasterr __attribute__((unused));
|
||||
static inline ssize_t compat_pread(int fd, void *buf, size_t n, off_t off){
|
||||
intptr_t osfh = _get_osfhandle(fd);
|
||||
if(osfh == -1 || osfh == -2){ errno = EBADF; return -1; }
|
||||
@@ -158,6 +165,7 @@ static inline ssize_t compat_pread(int fd, void *buf, size_t n, off_t off){
|
||||
if(!ReadFile(h, (char*)buf + total, chunk32, &rd, &ov)){
|
||||
DWORD err = GetLastError();
|
||||
if(err == ERROR_HANDLE_EOF) break; /* past EOF → return bytes read (0 if none, matching POSIX pread) */
|
||||
compat_pread_lasterr = err; /* preserva il codice VERO per il report (#307) */
|
||||
if(err == ERROR_INVALID_HANDLE || err == ERROR_INVALID_FUNCTION) errno = EBADF;
|
||||
else errno = EIO;
|
||||
return -1;
|
||||
@@ -237,7 +245,9 @@ static inline int compat_rename(const char *old, const char *new){
|
||||
/* --- rss_gb: getrusage -> GetProcessMemoryInfo ---
|
||||
* ru_maxrss in KB (come Linux): rss_gb() divide per 1e6 → GB corretti. */
|
||||
#include <psapi.h>
|
||||
#pragma comment(lib, "psapi.lib")
|
||||
#ifdef _MSC_VER
|
||||
#pragma comment(lib, "psapi.lib") /* MSVC: link psapi; MinGW/GCC uses -lpsapi */
|
||||
#endif
|
||||
struct rusage { long ru_maxrss; };
|
||||
#define RUSAGE_SELF 0
|
||||
static inline int getrusage(int who, struct rusage *r){
|
||||
@@ -304,8 +314,52 @@ static inline int compat_setenv(const char *name, const char *value, int overwri
|
||||
}
|
||||
#define setenv(name,value,overwrite) compat_setenv(name,value,overwrite)
|
||||
|
||||
/* --- getenv_utf8: read an env var as UTF-8, not through the ANSI codepage ---
|
||||
* Plain getenv()/_environ are populated by the CRT from the ANSI-codepage view
|
||||
* of the process environment block, not UTF-8. A parent that hands the child a
|
||||
* Unicode value via CreateProcessW's wide env block (e.g. Python's subprocess
|
||||
* module, which coli uses to pass the chat prompt) round-trips correctly only
|
||||
* through GetEnvironmentVariableW; going through narrow getenv() re-encodes it
|
||||
* via CP_ACP first, so any non-ASCII prompt text (Cyrillic, CJK, ...) comes out
|
||||
* corrupted before the byte-level tokenizer ever sees it. Read the wide value
|
||||
* directly and convert straight to UTF-8, bypassing the ANSI codepage entirely.
|
||||
* Returned buffer is intentionally leaked: called a handful of times at
|
||||
* startup, lives for the process. */
|
||||
static inline const char *compat_getenv_utf8(const char *name){
|
||||
wchar_t wname[64];
|
||||
if(MultiByteToWideChar(CP_UTF8, 0, name, -1, wname, 64) <= 0) return getenv(name);
|
||||
DWORD need = GetEnvironmentVariableW(wname, NULL, 0);
|
||||
if(!need) return NULL;
|
||||
wchar_t *wval = (wchar_t*)malloc(need * sizeof(wchar_t));
|
||||
if(!wval) return NULL;
|
||||
GetEnvironmentVariableW(wname, wval, need);
|
||||
int blen = WideCharToMultiByte(CP_UTF8, 0, wval, -1, NULL, 0, NULL, NULL);
|
||||
char *val = blen>0 ? (char*)malloc((size_t)blen) : NULL;
|
||||
if(val) WideCharToMultiByte(CP_UTF8, 0, wval, -1, val, blen, NULL, NULL);
|
||||
free(wval);
|
||||
return val;
|
||||
}
|
||||
#define getenv_utf8(name) compat_getenv_utf8(name)
|
||||
|
||||
/* --- mkdtemp -> _mktemp + _mkdir (POSIX mkdtemp assente su Windows) ---
|
||||
* Test binaries (test_stops.c) create a scratch dir in the CWD via a
|
||||
* "name_XXXXXX" template; POSIX mkdtemp fills the X's and mkdirs 0700. The
|
||||
* Windows CRT has _mktemp (in-place, same XXXXXX contract) so we compose it.
|
||||
* Returns the template pointer on success, NULL on failure — matching POSIX. */
|
||||
static inline char *compat_mkdtemp(char *tmpl){
|
||||
if(!tmpl) return NULL;
|
||||
if(!_mktemp(tmpl)) return NULL; /* fills the trailing X's in place */
|
||||
if(_mkdir(tmpl) != 0) return NULL; /* EEXIST is impossible post-_mktemp */
|
||||
return tmpl;
|
||||
}
|
||||
#define mkdtemp(tmpl) compat_mkdtemp(tmpl)
|
||||
|
||||
#endif /* _WIN32 */
|
||||
|
||||
#ifndef getenv_utf8
|
||||
#define getenv_utf8(name) getenv(name)
|
||||
#endif
|
||||
|
||||
/* --- compat_aligned_free su piattaforme diverse da Windows ---
|
||||
* Su Linux/macOS, posix_memalign usa free() normale. */
|
||||
#ifndef compat_aligned_free
|
||||
|
||||
+26
-9
@@ -19,16 +19,33 @@ def _check(identifier, status, summary, **details):
|
||||
|
||||
def cuda_linkage(engine_path):
|
||||
"""Return CUDA linkage state without loading the executable or CUDA runtime."""
|
||||
if not Path(engine_path).is_file() or os.name != "posix":
|
||||
engine = Path(engine_path)
|
||||
if not engine.is_file():
|
||||
return {"linked": False, "missing": False}
|
||||
try:
|
||||
result = subprocess.run(["ldd", str(engine_path)], capture_output=True, text=True,
|
||||
timeout=3, check=False)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return {"linked": False, "missing": False}
|
||||
lines = [line for line in result.stdout.splitlines() if "libcudart" in line]
|
||||
return {"linked": any("not found" not in line for line in lines),
|
||||
"missing": any("not found" in line for line in lines)}
|
||||
if os.name == "posix":
|
||||
try:
|
||||
result = subprocess.run(["ldd", str(engine)], capture_output=True, text=True,
|
||||
timeout=3, check=False)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return {"linked": False, "missing": False}
|
||||
lines = [line for line in result.stdout.splitlines() if "libcudart" in line]
|
||||
return {"linked": any("not found" not in line for line in lines),
|
||||
"missing": any("not found" in line for line in lines)}
|
||||
if sys.platform == "win32":
|
||||
# Windows CUDA_DLL=1 builds never link libcudart directly: glm.exe loads
|
||||
# coli_cuda.dll at runtime via LoadLibrary (backend_loader.c), so there's no
|
||||
# import-table entry for ldd/dumpbin to see. Detect the COLI_CUDA build via a
|
||||
# marker string baked into glm.c's #ifdef COLI_CUDA block instead, and require
|
||||
# coli_cuda.dll to actually sit next to glm.exe (else CUDA init fails at startup).
|
||||
try:
|
||||
built = b"[CUDA] mode: routed experts" in engine.read_bytes()
|
||||
except OSError:
|
||||
return {"linked": False, "missing": False}
|
||||
if not built:
|
||||
return {"linked": False, "missing": False}
|
||||
dll_present = (engine.parent / "coli_cuda.dll").is_file()
|
||||
return {"linked": dll_present, "missing": not dll_present}
|
||||
return {"linked": False, "missing": False}
|
||||
|
||||
|
||||
def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *,
|
||||
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
/* kv_persist.h — .coli_kv on-disk KV cache persistence.
|
||||
* Conversations reopen warm across engine restarts: the compressed MLA KV-cache
|
||||
* is appended incrementally after every turn, crash-safe (nrec written last).
|
||||
* Include after Model/KVState/Cfg are defined; requires now_s() and g_draft. */
|
||||
#ifndef KV_PERSIST_H
|
||||
#define KV_PERSIST_H
|
||||
|
||||
static int g_kvsave=1;
|
||||
#define KV_MAGIC "COLIKV1\0"
|
||||
|
||||
static void kv_hdr(Model *m, int32_t *h, int nrec){
|
||||
Cfg *c=&m->c; int nic=0;
|
||||
for(int i=0;i<c->n_layers;i++) if(m->Ic && m->Ic[i]) nic++;
|
||||
h[0]=c->n_layers; h[1]=c->kv_lora; h[2]=c->qk_rope;
|
||||
h[3]=m->has_dsa?c->index_hd:0; h[4]=nic; h[5]=c->vocab; h[6]=nrec; h[7]=0;
|
||||
}
|
||||
|
||||
static int64_t kv_rec_bytes(Model *m){
|
||||
Cfg *c=&m->c;
|
||||
int64_t rec = 4 + (int64_t)c->n_layers*(c->kv_lora+c->qk_rope)*4;
|
||||
if(m->has_dsa) for(int i=0;i<c->n_layers;i++) if(m->Ic[i]) rec+=(int64_t)c->index_hd*4;
|
||||
return rec;
|
||||
}
|
||||
|
||||
static int kv_disk_open(Model *m){
|
||||
KVState *k=m->kv;
|
||||
if(k->disk_fp) return 1;
|
||||
k->disk_fp=fopen(k->disk_path,"r+b");
|
||||
if(!k->disk_fp){
|
||||
k->disk_fp=fopen(k->disk_path,"wb");
|
||||
if(!k->disk_fp) return 0;
|
||||
int32_t h[8]; kv_hdr(m,h,0);
|
||||
fwrite(KV_MAGIC,1,8,k->disk_fp); fwrite(h,4,8,k->disk_fp);
|
||||
fflush(k->disk_fp);
|
||||
fclose(k->disk_fp);
|
||||
k->disk_fp=fopen(k->disk_path,"r+b");
|
||||
if(!k->disk_fp) return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static void kv_disk_truncate(Model *m, int nrec){
|
||||
if(!g_kvsave) return;
|
||||
KVState *k=m->kv;
|
||||
if(k->disk_fp){ fclose(k->disk_fp); k->disk_fp=NULL; }
|
||||
FILE *f=fopen(k->disk_path,"r+b");
|
||||
if(!f){ k->disk_nrec=0; return; }
|
||||
k->disk_nrec=nrec;
|
||||
int32_t nr=nrec; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f);
|
||||
fflush(f); fclose(f);
|
||||
}
|
||||
|
||||
static void kv_disk_reset(Model *m){ kv_disk_truncate(m,0); }
|
||||
|
||||
static void kv_disk_append(Model *m, const int *hist, int len){
|
||||
KVState *k=m->kv;
|
||||
if(!g_kvsave || len<=k->disk_nrec) return;
|
||||
Cfg *c=&m->c;
|
||||
if(!kv_disk_open(m)) return;
|
||||
FILE *f=k->disk_fp;
|
||||
int64_t rec = kv_rec_bytes(m);
|
||||
if(rec > k->disk_buf_cap){
|
||||
uint8_t *nb=realloc(k->disk_buf, rec);
|
||||
if(!nb) return;
|
||||
k->disk_buf=nb; k->disk_buf_cap=rec;
|
||||
}
|
||||
fseek(f, 8+8*4 + (int64_t)k->disk_nrec*rec, SEEK_SET);
|
||||
for(int p=k->disk_nrec;p<len;p++){
|
||||
uint8_t *b=k->disk_buf;
|
||||
*(int32_t*)b = hist[p]; b+=4;
|
||||
for(int i=0;i<c->n_layers;i++){
|
||||
memcpy(b, m->Lc[i]+(int64_t)p*c->kv_lora, (size_t)c->kv_lora*4); b+=c->kv_lora*4;
|
||||
memcpy(b, m->Rc[i]+(int64_t)p*c->qk_rope,(size_t)c->qk_rope*4); b+=c->qk_rope*4;
|
||||
}
|
||||
if(m->has_dsa) for(int i=0;i<c->n_layers;i++) if(m->Ic[i]){
|
||||
memcpy(b, m->Ic[i]+(int64_t)p*c->index_hd, (size_t)c->index_hd*4); b+=c->index_hd*4;
|
||||
}
|
||||
fwrite(k->disk_buf, 1, (size_t)rec, f);
|
||||
}
|
||||
fflush(f);
|
||||
int32_t nr=len; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f);
|
||||
fflush(f);
|
||||
k->disk_nrec=len;
|
||||
}
|
||||
|
||||
static int kv_disk_load(Model *m, int *hist, int maxctx){
|
||||
if(!g_kvsave) return 0;
|
||||
KVState *k=m->kv;
|
||||
Cfg *c=&m->c;
|
||||
FILE *f=fopen(k->disk_path,"rb"); if(!f) return 0;
|
||||
char mg[8]; int32_t h[8], w[8]; kv_hdr(m,w,0);
|
||||
if(fread(mg,1,8,f)!=8 || memcmp(mg,KV_MAGIC,8) || fread(h,4,8,f)!=8 ||
|
||||
h[0]!=w[0]||h[1]!=w[1]||h[2]!=w[2]||h[3]!=w[3]||h[4]!=w[4]||h[5]!=w[5]){
|
||||
fprintf(stderr,"[KV] ignoring .coli_kv from a different model or version\n"); fclose(f); return 0; }
|
||||
int nrec=h[6];
|
||||
if(nrec<1){ fclose(f); return 0; }
|
||||
if(nrec>=maxctx-8-g_draft){
|
||||
fprintf(stderr,"[KV] saved conversation (%d tokens) exceeds the context: starting over\n",nrec);
|
||||
fclose(f); return 0; }
|
||||
double t0=now_s();
|
||||
for(int p=0;p<nrec;p++){
|
||||
int32_t tk; if(fread(&tk,4,1,f)!=1){ nrec=p; break; } hist[p]=tk;
|
||||
for(int i=0;i<c->n_layers;i++){
|
||||
if(fread(m->Lc[i]+(int64_t)p*c->kv_lora, 4, c->kv_lora, f)!=(size_t)c->kv_lora ||
|
||||
fread(m->Rc[i]+(int64_t)p*c->qk_rope, 4, c->qk_rope, f)!=(size_t)c->qk_rope){ nrec=p; goto out; }
|
||||
}
|
||||
if(m->has_dsa) for(int i=0;i<c->n_layers;i++) if(m->Ic[i])
|
||||
if(fread(m->Ic[i]+(int64_t)p*c->index_hd, 4, c->index_hd, f)!=(size_t)c->index_hd){ nrec=p; goto out; }
|
||||
}
|
||||
out:
|
||||
fclose(f);
|
||||
if(nrec>0){
|
||||
if(m->has_mtp) m->kv_start[c->n_layers]=-1;
|
||||
fprintf(stderr,"[KV] resumed conversation from disk: %d tokens in %.1fs (no re-prefill)\n",
|
||||
nrec, now_s()-t0);
|
||||
}
|
||||
k->disk_nrec=nrec;
|
||||
return nrec;
|
||||
}
|
||||
|
||||
#endif /* KV_PERSIST_H */
|
||||
@@ -5,6 +5,15 @@
|
||||
* Densa (embed, attn, router, norme, lm_head) residente in RAM (float32).
|
||||
* Expert letti dal disco on-demand via pread+fadvise(DONTNEED), cache LRU per-layer.
|
||||
* Matmul multi-thread con OpenMP (niente BLAS).
|
||||
*
|
||||
* ENV VARS:
|
||||
* PILOT=0/1/2/3 : 0=no prefetch, 1=1-layer lookahead, 2=2-layer, 3=3-layer lookahead
|
||||
* HOT=N : pin top-N hot experts per layer permanently (never evict)
|
||||
* WARMUP=N : tokens before hot pinning activates (default 5)
|
||||
* WIDE=N : prefetch top-K*N candidates (default 1, try 2 or 3)
|
||||
* SMOOTH=F : EMA coefficient for routing momentum (default 0.3, range 0.0-0.95)
|
||||
* CONF_LIMIT=F : cumulative gate probability threshold for prefetch cutoff (default 0.92)
|
||||
* (expert queue is sorted by eid for SSD read locality)
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include <stdio.h>
|
||||
@@ -12,11 +21,22 @@
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
#include <time.h>
|
||||
#include <pthread.h>
|
||||
#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__)
|
||||
#include <sys/resource.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
#include "st.h"
|
||||
|
||||
#ifdef _WIN32
|
||||
#include <windows.h>
|
||||
#define sleep_ms(ms) Sleep(ms)
|
||||
#else
|
||||
#define sleep_ms(ms) usleep((ms) * 1000)
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
/* ---------- config ---------- */
|
||||
typedef struct {
|
||||
int hidden, n_layers, n_heads, n_kv_heads, head_dim;
|
||||
@@ -33,22 +53,61 @@ typedef struct {
|
||||
* Ogni weight [out,in] tenuto come int8 (per-riga) + scala float per riga.
|
||||
* Cosi' la RAM-cache scende da 4 byte/param (f32) a 1 byte/param: e' il
|
||||
* meccanismo che fa stare GLM-5.2 nei 15 GB. dequant-on-use nel matmul. */
|
||||
typedef struct { int eid; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot;
|
||||
/* pinned=1 means this slot is strongly preferred to keep (hot expert); it will
|
||||
* not be evicted during normal LRU eviction, but may be displaced under extreme
|
||||
* cache pressure when all slots are pinned or in-flight. */
|
||||
typedef struct { int eid; int pinned; int8_t *g, *u, *d; float *gs, *us, *ds; uint64_t used; } Slot;
|
||||
typedef struct { Slot *slots; int n, cap; } LCache;
|
||||
|
||||
typedef struct {
|
||||
Cfg c;
|
||||
shards S;
|
||||
int quant_bits; /* bit di quantizzazione degli expert (2..8); storage int8, niente f32 (#134) */
|
||||
int quant_bits;
|
||||
float *embed, *lm_head, *final_norm;
|
||||
Layer *L;
|
||||
LCache *cache; /* [n_layers] */
|
||||
uint64_t clock, hits, miss;
|
||||
/* kv-cache per-layer: K,V come [H * maxT * head_dim] */
|
||||
float **K, **V; int kv_len, max_t;
|
||||
double dense_load_s;
|
||||
/* IMPROVEMENT 2: expert frequency heatmap */
|
||||
uint32_t *freq;
|
||||
int freq_token_count, hot_pinned, hot_n, warmup_tokens;
|
||||
int token_count;
|
||||
/* PREDICTION IMPROVEMENT A: per-layer EMA of gate logits across tokens.
|
||||
* momentum_logits[l*E .. (l+1)*E-1] = EMA of gate outputs for layer l.
|
||||
* Used exclusively by the PILOT prefetcher to stabilise routing predictions
|
||||
* across tokens; does NOT affect actual MoE routing (pr is unchanged). */
|
||||
float *momentum_logits; /* [n_layers * n_experts], EMA of gate logits */
|
||||
float pilot_smooth; /* SMOOTH env: EMA coefficient 0.0-0.9 (default 0.3) */
|
||||
uint8_t *is_pinned; /* [n_layers * n_experts], 1 if expert is globally pinned */
|
||||
uint8_t *is_queued; /* [n_layers * n_experts], 1 if expert is currently in the prefetch queue */
|
||||
float pilot_conf_limit; /* CONF_LIMIT env: cumulative gate probability threshold (e.g. 0.92) */
|
||||
} Model;
|
||||
|
||||
static pthread_mutex_t g_pilot_mx = PTHREAD_MUTEX_INITIALIZER;
|
||||
static struct { int l, e; } pilot_q[4096];
|
||||
static volatile unsigned pilot_r = 0, pilot_w = 0;
|
||||
static Model *pilot_m = NULL;
|
||||
static int g_pilot = 0;
|
||||
static int g_wide = 1; /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */
|
||||
|
||||
static void pilot_prefetch(Model *m, int lnext, const float *x, int S);
|
||||
static void *pilot_worker(void *arg);
|
||||
static void ensure_pilot_worker_started(Model *m);
|
||||
static void slot_ensure_allocated(Model *m, Slot *s);
|
||||
|
||||
static void ensure_pilot_worker_started(Model *m) {
|
||||
if (!pilot_m) {
|
||||
pilot_m = m;
|
||||
pthread_t t;
|
||||
if (pthread_create(&t, NULL, pilot_worker, NULL) != 0) {
|
||||
fprintf(stderr, "Error: Failed to create pilot prefetch worker thread\n");
|
||||
exit(1);
|
||||
}
|
||||
pthread_detach(t);
|
||||
}
|
||||
}
|
||||
|
||||
/* ---------- utility ---------- */
|
||||
static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec*1e-9; }
|
||||
#if defined(__APPLE__)
|
||||
@@ -210,51 +269,224 @@ static void model_init(Model *m, const char *snap, int cap, int bits) {
|
||||
#undef LD
|
||||
}
|
||||
m->cache = calloc(c->n_layers, sizeof(LCache));
|
||||
for (int i = 0; i < c->n_layers; i++) { m->cache[i].cap = cap; m->cache[i].slots = calloc(cap, sizeof(Slot)); }
|
||||
for (int i = 0; i < c->n_layers; i++) {
|
||||
m->cache[i].cap = cap;
|
||||
m->cache[i].slots = calloc(cap, sizeof(Slot));
|
||||
}
|
||||
/* IMPROVEMENT 2: frequency heatmap for hot expert pinning */
|
||||
m->freq = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint32_t));
|
||||
m->hot_pinned = 0; m->freq_token_count = 0;
|
||||
m->hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0;
|
||||
m->warmup_tokens = getenv("WARMUP") ? atoi(getenv("WARMUP")) : 5;
|
||||
m->token_count = 0;
|
||||
/* PREDICTION A: routing momentum — EMA of gate logits across tokens.
|
||||
* Initialized to zero; first token sets EMA = fresh logits. */
|
||||
m->momentum_logits = calloc((size_t)c->n_layers * c->n_experts, sizeof(float));
|
||||
float sv = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f;
|
||||
if (sv < 0.f) sv = 0.f; if (sv > 0.95f) sv = 0.95f;
|
||||
m->pilot_smooth = sv;
|
||||
m->is_pinned = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t));
|
||||
m->is_queued = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t));
|
||||
float cl = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f;
|
||||
if (cl < 0.1f) cl = 0.1f; if (cl > 1.0f) cl = 1.0f;
|
||||
m->pilot_conf_limit = cl;
|
||||
m->dense_load_s = now_s() - t0;
|
||||
|
||||
// Persistent Hot Pinning: try to load hot_pinned.bin
|
||||
char pinpath[512];
|
||||
snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap);
|
||||
FILE *pinf = fopen(pinpath, "rb");
|
||||
if (pinf) {
|
||||
size_t expected_size = (size_t)c->n_layers * c->n_experts;
|
||||
if (fread(m->is_pinned, 1, expected_size, pinf) == expected_size) {
|
||||
m->hot_pinned = 1;
|
||||
printf("[HOT] Loaded persistent pinning from %s\n", pinpath);
|
||||
|
||||
if (g_pilot) {
|
||||
ensure_pilot_worker_started(m);
|
||||
for (int l = 0; l < c->n_layers; l++) {
|
||||
for (int e = 0; e < c->n_experts; e++) {
|
||||
if (m->is_pinned[l * c->n_experts + e]) {
|
||||
unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED);
|
||||
unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE);
|
||||
if (w - r < 4096) {
|
||||
pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = e;
|
||||
pthread_mutex_lock(&g_pilot_mx);
|
||||
m->is_queued[l * c->n_experts + e] = 1;
|
||||
pthread_mutex_unlock(&g_pilot_mx);
|
||||
__atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
printf("[HOT] Pre-loading pinned experts into cache...\n");
|
||||
double t_wait = now_s();
|
||||
while (1) {
|
||||
unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE);
|
||||
unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE);
|
||||
if (r == w) break;
|
||||
sleep_ms(2);
|
||||
}
|
||||
printf("[HOT] Pre-loaded in %.1fs!\n", now_s() - t_wait);
|
||||
}
|
||||
}
|
||||
fclose(pinf);
|
||||
}
|
||||
}
|
||||
|
||||
/* legge un weight dal disco (streaming) e lo quantizza in q[O,I]+scale[O].
|
||||
* Container pre-quantizzato (convert_olmoe.py: int8 + scale f32 in "name.qs"):
|
||||
* lettura raw diretta — meta' I/O e zero quantize_rows a runtime. Prima di
|
||||
* questa patch il container int8 causava SIGBUS (st_read_f32 su tensori I8). */
|
||||
static void load_expert_w(Model *m, const char *name, int8_t *q, float *scale, int O, int I, float *tmp) {
|
||||
st_tensor *t = st_find(&m->S, name);
|
||||
if (t && t->dtype == 3) { /* I8/U8: container colibri */
|
||||
char qs[300]; snprintf(qs, sizeof(qs), "%s.qs", name);
|
||||
st_read_raw(&m->S, name, q, 1);
|
||||
st_read_f32(&m->S, qs, scale, 1);
|
||||
return;
|
||||
static void slot_ensure_allocated(Model *m, Slot *s) {
|
||||
if (s->g) return;
|
||||
Cfg *c = &m->c;
|
||||
int64_t ng = (int64_t)c->inter * c->hidden;
|
||||
int64_t nd = (int64_t)c->hidden * c->inter;
|
||||
int8_t *w_block = malloc(ng + ng + nd);
|
||||
if (!w_block) {
|
||||
fprintf(stderr, "Error: Out of memory allocating slot weights block\n");
|
||||
exit(1);
|
||||
}
|
||||
st_read_f32(&m->S, name, tmp, 1); /* pread + fadvise DONTNEED */
|
||||
quantize_rows(tmp, q, scale, O, I, m->quant_bits);
|
||||
s->g = w_block;
|
||||
s->u = w_block + ng;
|
||||
s->d = w_block + ng + ng;
|
||||
float *s_block = falloc(c->inter + c->inter + c->hidden);
|
||||
s->gs = s_block;
|
||||
s->us = s_block + c->inter;
|
||||
s->ds = s_block + c->inter + c->inter;
|
||||
s->pinned = 0;
|
||||
}
|
||||
|
||||
static void load_expert_merged(Model *m, int layer, int eid, Slot *s) {
|
||||
char nm[256], qsnm[256];
|
||||
snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.merged_weight", layer, eid);
|
||||
snprintf(qsnm, sizeof(qsnm), "model.layers.%d.mlp.experts.%d.qs", layer, eid);
|
||||
st_read_raw(&m->S, nm, s->g, 1);
|
||||
st_read_f32(&m->S, qsnm, s->gs, 0); /* scales are F32; use typed reader for dtype safety */
|
||||
}
|
||||
|
||||
/* ---------- cache expert: ritorna i pesi quantizzati (q+scale) da cache o disco ---------- */
|
||||
static void expert_get(Model *m, int layer, int eid, Slot **out) {
|
||||
LCache *lc = &m->cache[layer];
|
||||
pthread_mutex_lock(&g_pilot_mx);
|
||||
for (int i = 0; i < lc->n; i++) if (lc->slots[i].eid == eid) {
|
||||
m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i]; return;
|
||||
m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i];
|
||||
pthread_mutex_unlock(&g_pilot_mx);
|
||||
return;
|
||||
}
|
||||
m->miss++;
|
||||
Cfg *c = &m->c;
|
||||
int64_t ng = (int64_t)c->inter * c->hidden, nd = (int64_t)c->hidden * c->inter;
|
||||
Slot *s;
|
||||
if (lc->n < lc->cap) {
|
||||
s = &lc->slots[lc->n++];
|
||||
s->g = malloc(ng); s->u = malloc(ng); s->d = malloc(nd);
|
||||
s->gs = falloc(c->inter); s->us = falloc(c->inter); s->ds = falloc(c->hidden);
|
||||
} else { int lru = 0; for (int i = 1; i < lc->n; i++) if (lc->slots[i].used < lc->slots[lru].used) lru = i; s = &lc->slots[lru]; }
|
||||
float *tmp = falloc(ng > nd ? ng : nd);
|
||||
char nm[256];
|
||||
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.gate_proj.weight",layer,eid); load_expert_w(m,nm,s->g,s->gs,c->inter,c->hidden,tmp);
|
||||
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.up_proj.weight", layer,eid); load_expert_w(m,nm,s->u,s->us,c->inter,c->hidden,tmp);
|
||||
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.down_proj.weight",layer,eid); load_expert_w(m,nm,s->d,s->ds,c->hidden,c->inter,tmp);
|
||||
free(tmp);
|
||||
s->eid = eid; s->used = ++m->clock;
|
||||
slot_ensure_allocated(m, s);
|
||||
} else {
|
||||
/* LRU eviction — skip pinned and in-flight (eid==-1) slots */
|
||||
int lru = -1;
|
||||
for (int i = 0; i < lc->n; i++) {
|
||||
if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue;
|
||||
if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i;
|
||||
}
|
||||
if (lru < 0) {
|
||||
/* All slots are pinned or in-flight; find oldest non-in-flight slot
|
||||
* (may be pinned, but never select one currently being loaded). */
|
||||
for (int i = 0; i < lc->n; i++) {
|
||||
if (lc->slots[i].eid < 0) continue; /* never evict in-flight */
|
||||
if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i;
|
||||
}
|
||||
}
|
||||
if (lru < 0) lru = 0; /* absolute last resort: all in-flight, evict slot 0 */
|
||||
s = &lc->slots[lru];
|
||||
s->pinned = 0;
|
||||
}
|
||||
s->eid = -1;
|
||||
s->used = ++m->clock;
|
||||
pthread_mutex_unlock(&g_pilot_mx);
|
||||
|
||||
load_expert_merged(m, layer, eid, s);
|
||||
|
||||
pthread_mutex_lock(&g_pilot_mx);
|
||||
s->eid = eid;
|
||||
s->pinned = m->is_pinned[layer * c->n_experts + eid];
|
||||
s->used = ++m->clock;
|
||||
*out = s;
|
||||
pthread_mutex_unlock(&g_pilot_mx);
|
||||
}
|
||||
|
||||
/* ---------- IMPROVEMENT 2: pin top-N hot experts per layer ---------- */
|
||||
static void pin_hot_experts(Model *m) {
|
||||
Cfg *c = &m->c;
|
||||
if (m->hot_n <= 0 || m->hot_pinned) return;
|
||||
m->hot_pinned = 1;
|
||||
|
||||
int is_dynamic = (m->hot_n >= 100);
|
||||
double thresh = is_dynamic ? (double)m->hot_n / 1000.0 : 0.0;
|
||||
|
||||
int pinned_total = 0;
|
||||
for (int l = 0; l < c->n_layers; l++) {
|
||||
uint32_t *freq_l = m->freq + (int64_t)l * c->n_experts;
|
||||
|
||||
uint64_t layer_total = 0;
|
||||
for (int e = 0; e < c->n_experts; e++) layer_total += freq_l[e];
|
||||
if (layer_total == 0) continue;
|
||||
|
||||
int max_pin = m->cache[l].cap - 8;
|
||||
if (max_pin < 4) max_pin = 4;
|
||||
|
||||
int hn = is_dynamic ? max_pin : (m->hot_n < c->n_experts ? m->hot_n : c->n_experts);
|
||||
if (hn > 256) hn = 256;
|
||||
int hot_eids[256];
|
||||
int actual_hn = 0;
|
||||
|
||||
for (int k = 0; k < hn; k++) {
|
||||
int best = -1; uint32_t bv = 0;
|
||||
for (int e = 0; e < c->n_experts; e++) {
|
||||
int already = 0;
|
||||
for (int j = 0; j < k; j++) if (hot_eids[j] == e) { already = 1; break; }
|
||||
if (!already && freq_l[e] > bv) { bv = freq_l[e]; best = e; }
|
||||
}
|
||||
if (best < 0 || bv == 0) break;
|
||||
if (is_dynamic && bv < thresh * layer_total) break;
|
||||
hot_eids[k] = best;
|
||||
actual_hn++;
|
||||
}
|
||||
|
||||
for (int k = 0; k < actual_hn; k++) {
|
||||
int eid = hot_eids[k];
|
||||
m->is_pinned[l * c->n_experts + eid] = 1;
|
||||
|
||||
LCache *lc = &m->cache[l];
|
||||
int found = 0;
|
||||
pthread_mutex_lock(&g_pilot_mx);
|
||||
for (int i = 0; i < lc->n; i++) {
|
||||
if (lc->slots[i].eid == eid) { lc->slots[i].pinned = 1; found = 1; break; }
|
||||
}
|
||||
pthread_mutex_unlock(&g_pilot_mx);
|
||||
if (!found && g_pilot > 0) {
|
||||
/* Only enqueue when the prefetch worker is active (PILOT>0). */
|
||||
ensure_pilot_worker_started(m);
|
||||
unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED);
|
||||
unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE);
|
||||
int gidx = l * c->n_experts + eid;
|
||||
pthread_mutex_lock(&g_pilot_mx);
|
||||
int already = m->is_queued[gidx];
|
||||
if (!already && w - r < 4096) {
|
||||
pilot_q[w & 4095].l = l; pilot_q[w & 4095].e = eid;
|
||||
m->is_queued[gidx] = 1;
|
||||
__atomic_store_n(&pilot_w, w + 1, __ATOMIC_RELEASE);
|
||||
}
|
||||
pthread_mutex_unlock(&g_pilot_mx);
|
||||
}
|
||||
pinned_total++;
|
||||
}
|
||||
}
|
||||
if (is_dynamic) {
|
||||
printf("[HOT] Dynamic Pinned %d experts total (thresh=%.1f%%) after %d warmup tokens\n",
|
||||
pinned_total, thresh * 100.0, m->freq_token_count);
|
||||
} else {
|
||||
printf("[HOT] Pinned %d experts (top-%d/layer) after %d warmup tokens\n",
|
||||
pinned_total, m->hot_n, m->freq_token_count);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* ---------- RoPE su un vettore di una testa (head_dim) a posizione assoluta pos ---------- */
|
||||
static void rope_head(float *x, int pos, const Cfg *c) {
|
||||
int h = c->head_dim / 2;
|
||||
@@ -325,6 +557,19 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
|
||||
float *g = falloc(I), *u = falloc(I), *hh = falloc(D);
|
||||
for (int s = 0; s < S; s++) {
|
||||
float *pr = logits + (int64_t)s*E;
|
||||
if (m->momentum_logits && m->pilot_smooth > 0.f) {
|
||||
float *ema = m->momentum_logits + (int64_t)layer * E;
|
||||
int is_zero = 1;
|
||||
for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } }
|
||||
if (is_zero) {
|
||||
for (int e = 0; e < E; e++) ema[e] = pr[e];
|
||||
} else {
|
||||
for (int e = 0; e < E; e++) {
|
||||
ema[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
softmax_row(pr, E);
|
||||
/* top-K indici (selezione parziale) */
|
||||
int idx[64]; float val[64];
|
||||
@@ -337,6 +582,11 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
|
||||
idx[kk] = best; val[kk] = bv;
|
||||
}
|
||||
if (c->norm_topk) { float sm=0; for(int kk=0;kk<K;kk++) sm+=val[kk]; for(int kk=0;kk<K;kk++) val[kk]/=sm; }
|
||||
/* IMPROVEMENT 2: update activation heatmap (before pinning activates) */
|
||||
if (!m->hot_pinned && m->freq) {
|
||||
uint32_t *freq_l = m->freq + (int64_t)layer * E;
|
||||
for (int kk = 0; kk < K; kk++) if (idx[kk] >= 0) freq_l[idx[kk]]++;
|
||||
}
|
||||
const float *xs = x + (int64_t)s*D;
|
||||
for (int kk = 0; kk < K; kk++) {
|
||||
Slot *e; expert_get(m, layer, idx[kk], &e);
|
||||
@@ -352,9 +602,20 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
|
||||
free(logits); free(g); free(u); free(hh);
|
||||
}
|
||||
|
||||
/* un passo: token nuovi ids[S] a posizione pos_base. Ritorna logits dell'ultimo token (malloc'd). */
|
||||
static float *step(Model *m, const int *ids, int S, int pos_base) {
|
||||
Cfg *c = &m->c; int D = c->hidden;
|
||||
if (g_pilot && m->token_count > 0) {
|
||||
/* Flush stale prefetch requests: clear is_queued so pilot_realload
|
||||
* will skip any entries still sitting in pilot_q for the previous
|
||||
* token. We deliberately do NOT move pilot_w backwards; that would
|
||||
* break the ring-buffer invariant (pilot_r could exceed pilot_w if
|
||||
* the worker consumed an entry concurrently). The worker will drain
|
||||
* the stale slots harmlessly because pilot_realload already exits
|
||||
* early when the expert is already cached or is_queued is clear. */
|
||||
pthread_mutex_lock(&g_pilot_mx);
|
||||
memset(m->is_queued, 0, (size_t)c->n_layers * c->n_experts);
|
||||
pthread_mutex_unlock(&g_pilot_mx);
|
||||
}
|
||||
float *x = falloc((int64_t)S*D);
|
||||
for (int s = 0; s < S; s++) memcpy(x + (int64_t)s*D, m->embed + (int64_t)ids[s]*D, D*sizeof(float));
|
||||
float *nrm = falloc((int64_t)S*D), *tmp = falloc((int64_t)S*D);
|
||||
@@ -363,12 +624,26 @@ static float *step(Model *m, const int *ids, int S, int pos_base) {
|
||||
for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->in_ln, D, c->eps);
|
||||
attention(m, l, i, nrm, S, pos_base, tmp);
|
||||
for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j];
|
||||
/* IMPROVEMENT 1: PILOT=1 -> 1-layer lookahead */
|
||||
if (g_pilot >= 1 && S <= 8 && i + 1 < c->n_layers)
|
||||
pilot_prefetch(m, i + 1, x, S);
|
||||
for (int s = 0; s < S; s++) rmsnorm_row(nrm + (int64_t)s*D, x + (int64_t)s*D, l->post_ln, D, c->eps);
|
||||
moe(m, l, i, nrm, S, tmp);
|
||||
for (int64_t j = 0; j < (int64_t)S*D; j++) x[j] += tmp[j];
|
||||
|
||||
/* PREDICTION IMPROVEMENT C (Residual gate trick):
|
||||
* PILOT=2 -> prefetch layer i+2 using completed state x (containing MoE residual). */
|
||||
if (g_pilot >= 2 && S <= 8 && i + 2 < c->n_layers)
|
||||
pilot_prefetch(m, i + 2, x, S);
|
||||
if (g_pilot >= 3 && S <= 8 && i + 3 < c->n_layers)
|
||||
pilot_prefetch(m, i + 3, x, S);
|
||||
|
||||
}
|
||||
/* count actual tokens processed (S>1 during prefill) */
|
||||
m->token_count += S; m->freq_token_count += S;
|
||||
if (!m->hot_pinned && m->hot_n > 0 && m->freq_token_count >= m->warmup_tokens)
|
||||
pin_hot_experts(m);
|
||||
m->kv_len = pos_base + S;
|
||||
/* solo l'ultimo token -> logits */
|
||||
float *last = falloc(D);
|
||||
rmsnorm_row(last, x + (int64_t)(S-1)*D, m->final_norm, D, c->eps);
|
||||
float *logit = falloc(c->vocab);
|
||||
@@ -377,6 +652,192 @@ static float *step(Model *m, const int *ids, int S, int pos_base) {
|
||||
return logit;
|
||||
}
|
||||
|
||||
static void pilot_realload(Model *m, int layer, int eid) {
|
||||
LCache *lc = &m->cache[layer];
|
||||
Cfg *c = &m->c;
|
||||
|
||||
pthread_mutex_lock(&g_pilot_mx);
|
||||
/* Early-exit if entry was flushed (is_queued cleared) while waiting. */
|
||||
if (!m->is_queued[layer * c->n_experts + eid]) {
|
||||
pthread_mutex_unlock(&g_pilot_mx);
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < lc->n; i++) {
|
||||
if (lc->slots[i].eid == eid) {
|
||||
m->is_queued[layer * c->n_experts + eid] = 0;
|
||||
pthread_mutex_unlock(&g_pilot_mx);
|
||||
return;
|
||||
}
|
||||
}
|
||||
Slot *s;
|
||||
if (lc->n < lc->cap) {
|
||||
s = &lc->slots[lc->n++];
|
||||
slot_ensure_allocated(m, s);
|
||||
} else {
|
||||
/* LRU eviction — skip pinned and in-flight (eid==-1) slots */
|
||||
int lru = -1;
|
||||
for (int i = 0; i < lc->n; i++) {
|
||||
if (lc->slots[i].pinned || lc->slots[i].eid < 0) continue;
|
||||
if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i;
|
||||
}
|
||||
if (lru < 0) {
|
||||
m->is_queued[layer * c->n_experts + eid] = 0;
|
||||
pthread_mutex_unlock(&g_pilot_mx);
|
||||
return; /* all pinned/in-flight, skip */
|
||||
}
|
||||
s = &lc->slots[lru]; s->pinned = 0;
|
||||
}
|
||||
s->eid = -1; s->used = ++m->clock;
|
||||
pthread_mutex_unlock(&g_pilot_mx);
|
||||
|
||||
load_expert_merged(m, layer, eid, s);
|
||||
|
||||
pthread_mutex_lock(&g_pilot_mx);
|
||||
s->eid = eid;
|
||||
s->pinned = m->is_pinned[layer * c->n_experts + eid];
|
||||
s->used = ++m->clock;
|
||||
m->is_queued[layer * c->n_experts + eid] = 0;
|
||||
pthread_mutex_unlock(&g_pilot_mx);
|
||||
}
|
||||
|
||||
static void *pilot_worker(void *arg) {
|
||||
(void)arg;
|
||||
while (1) {
|
||||
unsigned r = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE);
|
||||
unsigned w = __atomic_load_n(&pilot_w, __ATOMIC_ACQUIRE);
|
||||
if (r == w) {
|
||||
sleep_ms(1);
|
||||
continue;
|
||||
}
|
||||
int layer = pilot_q[r & 4095].l;
|
||||
int eid = pilot_q[r & 4095].e;
|
||||
pilot_realload(pilot_m, layer, eid);
|
||||
__atomic_store_n(&pilot_r, r + 1, __ATOMIC_RELEASE);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void pilot_prefetch(Model *m, int lnext, const float *x, int S) {
|
||||
if (lnext < 0 || lnext >= m->c.n_layers) return;
|
||||
Cfg *c = &m->c; int D = c->hidden, E = c->n_experts;
|
||||
ensure_pilot_worker_started(m);
|
||||
float *logits = falloc((int64_t)S * E);
|
||||
Layer *l = &m->L[lnext];
|
||||
|
||||
// PREDICTION IMPROVEMENT B: Apply RMSNorm to x using destination layer's post_ln
|
||||
// This scales inputs to the distribution expected by l->gate.
|
||||
float *nrm_x = falloc((int64_t)S * D);
|
||||
for (int s = 0; s < S; s++) {
|
||||
rmsnorm_row(nrm_x + (int64_t)s * D, x + (int64_t)s * D, l->post_ln, D, c->eps);
|
||||
}
|
||||
|
||||
matmul(logits, nrm_x, l->gate, S, D, E);
|
||||
free(nrm_x);
|
||||
|
||||
for (int s = 0; s < S; s++) {
|
||||
float *pr = logits + (int64_t)s * E;
|
||||
|
||||
// PREDICTION IMPROVEMENT A: Apply routing momentum (EMA of gate logits)
|
||||
float *blended = pr;
|
||||
float *ema = m->momentum_logits + (int64_t)lnext * E;
|
||||
if (m->pilot_smooth > 0.f) {
|
||||
blended = falloc(E);
|
||||
int is_zero = 1;
|
||||
for (int e = 0; e < E; e++) { if (ema[e] != 0.f) { is_zero = 0; break; } }
|
||||
if (is_zero) {
|
||||
for (int e = 0; e < E; e++) {
|
||||
ema[e] = pr[e];
|
||||
blended[e] = pr[e];
|
||||
}
|
||||
} else {
|
||||
for (int e = 0; e < E; e++) {
|
||||
blended[e] = (1.f - m->pilot_smooth) * pr[e] + m->pilot_smooth * ema[e];
|
||||
ema[e] = blended[e]; // update EMA
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int cand = 0;
|
||||
int idx[128];
|
||||
|
||||
float max_logit = -1e30f;
|
||||
for (int e = 0; e < E; e++) { if (blended[e] > max_logit) max_logit = blended[e]; }
|
||||
float *exps = falloc(E);
|
||||
float sum_exps = 0.f;
|
||||
for (int e = 0; e < E; e++) {
|
||||
exps[e] = expf(blended[e] - max_logit);
|
||||
sum_exps += exps[e];
|
||||
}
|
||||
|
||||
float cum_sum = 0.f;
|
||||
int min_cand = c->topk;
|
||||
int max_cand = c->topk * g_wide;
|
||||
if (max_cand < min_cand) max_cand = min_cand;
|
||||
if (max_cand > 128) max_cand = 128; /* idx[] buffer bound */
|
||||
if (max_cand > E) max_cand = E;
|
||||
|
||||
for (int kk = 0; kk < max_cand; kk++) {
|
||||
int best = -1; float bv = -1.f;
|
||||
for (int e = 0; e < E; e++) {
|
||||
int taken = 0; for (int j = 0; j < kk; j++) if (idx[j] == e) { taken=1; break; }
|
||||
if (!taken && exps[e] > bv) { bv = exps[e]; best = e; }
|
||||
}
|
||||
if (best < 0) break;
|
||||
idx[kk] = best;
|
||||
cum_sum += bv;
|
||||
cand++;
|
||||
if (cum_sum >= m->pilot_conf_limit * sum_exps && cand >= min_cand) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
free(exps);
|
||||
|
||||
if (blended != pr) free(blended);
|
||||
|
||||
/* IMPROVEMENT 5: sort candidates by eid for sequential SSD read locality */
|
||||
for (int a = 0; a < cand-1; a++)
|
||||
for (int b = a+1; b < cand; b++)
|
||||
if (idx[b] >= 0 && (idx[a] < 0 || idx[a] > idx[b])) { int t = idx[a]; idx[a] = idx[b]; idx[b] = t; }
|
||||
|
||||
for (int kk = 0; kk < cand; kk++) {
|
||||
int eid = idx[kk];
|
||||
if (eid < 0) continue;
|
||||
int found = 0;
|
||||
pthread_mutex_lock(&g_pilot_mx);
|
||||
LCache *lc = &m->cache[lnext];
|
||||
for (int z = 0; z < lc->n; z++) {
|
||||
if (lc->slots[z].eid == eid) { found = 1; break; }
|
||||
}
|
||||
pthread_mutex_unlock(&g_pilot_mx);
|
||||
if (!found) {
|
||||
int gidx = lnext * E + eid;
|
||||
pthread_mutex_lock(&g_pilot_mx);
|
||||
int already_queued = m->is_queued[gidx];
|
||||
if (!already_queued) {
|
||||
m->is_queued[gidx] = 1;
|
||||
}
|
||||
pthread_mutex_unlock(&g_pilot_mx);
|
||||
|
||||
if (!already_queued) {
|
||||
unsigned w2 = __atomic_load_n(&pilot_w, __ATOMIC_RELAXED);
|
||||
unsigned r2 = __atomic_load_n(&pilot_r, __ATOMIC_ACQUIRE);
|
||||
if (w2 - r2 < 4096) {
|
||||
pilot_q[w2 & 4095].l = lnext;
|
||||
pilot_q[w2 & 4095].e = eid;
|
||||
__atomic_store_n(&pilot_w, w2 + 1, __ATOMIC_RELEASE);
|
||||
} else {
|
||||
pthread_mutex_lock(&g_pilot_mx);
|
||||
m->is_queued[gidx] = 0;
|
||||
pthread_mutex_unlock(&g_pilot_mx);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
free(logits);
|
||||
}
|
||||
|
||||
|
||||
/* generazione greedy. prompt[np] -> riempie out[np+n_new] */
|
||||
static void generate(Model *m, const int *prompt, int np, int n_new, int *out) {
|
||||
Cfg *c = &m->c;
|
||||
@@ -400,6 +861,37 @@ static void generate(Model *m, const int *prompt, int np, int n_new, int *out) {
|
||||
}
|
||||
}
|
||||
|
||||
/* teacher-forced NLL of full_ids[np..nfull): feed the REFERENCE token at each step
|
||||
* (never the argmax), accumulate -log softmax(logits)[next_ref]. A loss meter for
|
||||
* throughput experiments: same engine path as decode, so hit rate/speed stay
|
||||
* comparable, but quality is measured as perplexity instead of exact-match.
|
||||
* Cross-checked vs HF transformers bf16 on identical token ids: engine (int8
|
||||
* experts) 12.11 ppl vs reference 12.25 (#108). Enabled by PPL=1. */
|
||||
static int tf_nll(Model *m, const int *full, int nfull, int np, double *nll_out) {
|
||||
Cfg *c = &m->c;
|
||||
m->max_t = nfull;
|
||||
m->K = calloc(c->n_layers, sizeof(float*)); m->V = calloc(c->n_layers, sizeof(float*));
|
||||
for (int i = 0; i < c->n_layers; i++) {
|
||||
m->K[i] = falloc((int64_t)c->n_heads * m->max_t * c->head_dim);
|
||||
m->V[i] = falloc((int64_t)c->n_heads * m->max_t * c->head_dim);
|
||||
}
|
||||
double nll = 0; int scored = 0;
|
||||
float *logit = step(m, full, np, 0); /* prefill on the prompt */
|
||||
for (int i = np; i < nfull; i++) {
|
||||
/* log softmax(logit)[full[i]] without materializing the softmax */
|
||||
float mx = logit[0]; for (int v = 1; v < c->vocab; v++) if (logit[v] > mx) mx = logit[v];
|
||||
double Z = 0; for (int v = 0; v < c->vocab; v++) Z += exp((double)logit[v] - mx);
|
||||
nll += -((double)logit[full[i]] - mx - log(Z));
|
||||
scored++;
|
||||
free(logit); logit = NULL;
|
||||
if (i == nfull - 1) break;
|
||||
logit = step(m, &full[i], 1, i); /* teacher forcing */
|
||||
}
|
||||
if (logit) free(logit);
|
||||
*nll_out = nll / scored;
|
||||
return scored;
|
||||
}
|
||||
|
||||
/* ---------- lettura ref.json ---------- */
|
||||
static int *read_int_array(jval *o, const char *key, int *n_out) {
|
||||
jval *a = json_get(o, key);
|
||||
@@ -411,25 +903,48 @@ static int *read_int_array(jval *o, const char *key, int *n_out) {
|
||||
int main(int argc, char **argv) {
|
||||
const char *snap = getenv("SNAP");
|
||||
if (!snap) { fprintf(stderr, "set SNAP=<snapshot directory>\n"); return 1; }
|
||||
int cap = argc > 1 ? atoi(argv[1]) : 16;
|
||||
int bits = argc > 2 ? atoi(argv[2]) : 8;
|
||||
if (bits < 2 || bits > 8) { /* expert storage is int8_t: bits>8 truncates in quantize_rows (#134). f32 mode is not implemented here — int8 is already token-exact vs the oracle. */
|
||||
fprintf(stderr, "quant_bits must be 2..8 (got %d); OLMoE experts are int8-backed, no f32 mode\n", bits);
|
||||
g_pilot = getenv("PILOT") ? atoi(getenv("PILOT")) : 0;
|
||||
g_wide = getenv("WIDE") ? atoi(getenv("WIDE")) : 1;
|
||||
if (g_wide < 1) g_wide = 1;
|
||||
if (g_wide > 4) g_wide = 4;
|
||||
int hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0;
|
||||
int cap = argc > 1 ? atoi(argv[1]) : 16;
|
||||
int bits = argc > 2 ? atoi(argv[2]) : 8;
|
||||
if (bits < 2 || bits > 8) {
|
||||
fprintf(stderr, "quant_bits must be 2..8 (got %d)\n", bits);
|
||||
return 1;
|
||||
}
|
||||
const char *refpath = argc > 3 ? argv[3] : "ref.json";
|
||||
|
||||
FILE *f = fopen(refpath, "rb"); if(!f){perror(refpath);return 1;}
|
||||
float smooth = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f;
|
||||
float conf = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f;
|
||||
|
||||
printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d smooth=%.2f conf=%.2f ==\n",
|
||||
cap, bits, g_pilot, g_wide, hot_n, smooth, conf);
|
||||
|
||||
FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; }
|
||||
fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);
|
||||
char *buf=malloc(n+1); if(fread(buf,1,n,f)!=(size_t)n){} buf[n]=0; fclose(f);
|
||||
char *buf=malloc(n+1); if (fread(buf,1,n,f)!=(size_t)n) {} buf[n]=0; fclose(f);
|
||||
char *arena=NULL; jval *ref = json_parse(buf, &arena);
|
||||
int np, nfull; int *prompt = read_int_array(ref,"prompt_ids",&np); int *full = read_int_array(ref,"full_ids",&nfull);
|
||||
int n_new = nfull - np;
|
||||
|
||||
printf("== Streaming C engine, cache = %d experts/layer, experts @ %d-bit ==\n", cap, bits);
|
||||
Model m; model_init(&m, snap, cap, bits);
|
||||
printf("resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb());
|
||||
|
||||
if (getenv("PPL") && atoi(getenv("PPL")) == 1) { /* loss-meter mode: teacher-forced NLL */
|
||||
double nll; double t = now_s();
|
||||
int scored = tf_nll(&m, full, nfull, np, &nll);
|
||||
double dt = now_s() - t;
|
||||
double tot = m.hits + m.miss;
|
||||
printf("TF-NLL: %.4f nats/token over %d tokens | ppl = %.2f\n", nll, scored, exp(nll));
|
||||
printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0,
|
||||
(unsigned long long)m.hits, (unsigned long long)m.miss);
|
||||
printf("Speed: %.2f tok/s (%.1fs for %d tokens) | PEAK RSS: %.2f GB\n", scored/dt, dt, scored, rss_gb());
|
||||
free(buf); free(arena);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int *out = malloc((np + n_new) * sizeof(int));
|
||||
double t = now_s();
|
||||
generate(&m, prompt, np, n_new, out);
|
||||
@@ -443,6 +958,26 @@ int main(int argc, char **argv) {
|
||||
printf("\nPEAK RSS: %.2f GB\n", rss_gb());
|
||||
printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu)\n", tot?100.0*m.hits/tot:0.0,
|
||||
(unsigned long long)m.hits, (unsigned long long)m.miss);
|
||||
|
||||
|
||||
// Persistent Hot Pinning: save dynamic pinning if newly created
|
||||
if (m.hot_pinned) {
|
||||
char pinpath[512];
|
||||
snprintf(pinpath, sizeof(pinpath), "%s/hot_pinned.bin", snap);
|
||||
FILE *pinf_chk = fopen(pinpath, "rb");
|
||||
if (!pinf_chk) {
|
||||
FILE *pinf_save = fopen(pinpath, "wb");
|
||||
if (pinf_save) {
|
||||
size_t expected_size = (size_t)m.c.n_layers * m.c.n_experts;
|
||||
fwrite(m.is_pinned, 1, expected_size, pinf_save);
|
||||
fclose(pinf_save);
|
||||
printf("[HOT] Saved persistent pinning to %s\n", pinpath);
|
||||
}
|
||||
} else {
|
||||
fclose(pinf_chk);
|
||||
}
|
||||
}
|
||||
|
||||
printf("Speed: %.2f tok/s (%.1fs for %d tokens)\n", n_new/dt, dt, n_new);
|
||||
free(buf); free(arena);
|
||||
return 0;
|
||||
|
||||
+35
-1
@@ -27,6 +27,7 @@ HERE = Path(__file__).resolve().parent
|
||||
END = b"\x01\x01END\x01\x01\n"
|
||||
READY = b"\x01\x01READY\x01\x01\n"
|
||||
MAX_BODY = 4 << 20
|
||||
PROFILE_TURNS = 120 # rolling window of per-turn PROF snapshots kept for /profile
|
||||
DEFAULT_CORS_ORIGINS = (
|
||||
"http://127.0.0.1:8000",
|
||||
"http://localhost:8000",
|
||||
@@ -270,6 +271,14 @@ def parse_tool_calls(reply, tools=None):
|
||||
salvaged.append(name)
|
||||
calls.append({"id": "call_" + uuid.uuid4().hex[:24], "type": "function",
|
||||
"function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)}})
|
||||
if tools and not calls and re.search(r"</?tool_call>|</?arg_key>|</?arg_value>", reply):
|
||||
# Diagnosi per la #401: il client ha dichiarato i tools e il modello ha PROVATO la
|
||||
# sintassi, ma il parse rigoroso non ha agganciato nulla (tipico output int4 storpiato).
|
||||
# EN: #401 field diagnosis: tools were declared and the model attempted the syntax,
|
||||
# EN: but the strict parse matched nothing (typically quantization-mangled output).
|
||||
sys.stderr.write("[api] tools declared and tool-call markers present, but no call "
|
||||
"parsed -- output may be quantization-mangled; try COLI_TOOL_SALVAGE=1\n")
|
||||
sys.stderr.flush()
|
||||
text = _BOX_RE.sub("", reply)
|
||||
if THINK_CLOSE in text:
|
||||
text = text.split(THINK_CLOSE, 1)[1]
|
||||
@@ -426,7 +435,10 @@ def generation_options(body, limit):
|
||||
maximum = body.get("max_tokens")
|
||||
maximum_param = "max_tokens"
|
||||
if maximum is None:
|
||||
maximum = min(256, limit)
|
||||
# Client omitted max_tokens: honor the operator's configured budget (--max-tokens /
|
||||
# --ngen), not an arbitrary 256 — `coli serve --ngen 32768` must mean 32768 (#382).
|
||||
# Generation still ends at EOS, so this is a cap, not a target.
|
||||
maximum = limit
|
||||
temperature = body.get("temperature")
|
||||
top_p = body.get("top_p")
|
||||
temperature = 0.7 if temperature is None else temperature
|
||||
@@ -494,6 +506,8 @@ class Engine:
|
||||
self.emap = None
|
||||
self.hits = None
|
||||
self.hits_seq = 0 # latest "TIERS" snapshot from the engine
|
||||
self.profile = collections.deque(maxlen=PROFILE_TURNS) # per-turn phase timings
|
||||
self.profile_seq = 0
|
||||
read_engine_turn(self.process.stdout, READY, lambda _: None)
|
||||
self.dispatcher = threading.Thread(target=self._dispatch_stdout,
|
||||
name="colibri-stdout", daemon=True)
|
||||
@@ -571,6 +585,20 @@ class Engine:
|
||||
elif kind == "HITS" and len(fields) == 4:
|
||||
self.hits = fields[3]
|
||||
self.hits_seq += 1
|
||||
elif kind == "PROF" and len(fields) >= 10:
|
||||
# per-turn phase timings: where the engine spent this turn's wall time
|
||||
self.profile.append({
|
||||
"wall_s": float(fields[1]),
|
||||
"prompt_tokens": int(fields[2]),
|
||||
"completion_tokens": int(fields[3]),
|
||||
"expert_disk_s": float(fields[4]),
|
||||
"expert_wait_s": float(fields[5]),
|
||||
"expert_matmul_s": float(fields[6]),
|
||||
"attention_s": float(fields[7]),
|
||||
"lm_head_s": float(fields[8]),
|
||||
"forwards": int(fields[9]),
|
||||
})
|
||||
self.profile_seq += 1
|
||||
elif kind == "TIERS" and len(fields) >= 6:
|
||||
self.tiers = {"vram": int(fields[1]), "ram": int(fields[2]),
|
||||
"disk": int(fields[3]), "vram_gb": float(fields[4]),
|
||||
@@ -803,6 +831,12 @@ class APIHandler(BaseHTTPRequestHandler):
|
||||
payload["seq"] = eng.hits_seq
|
||||
self.send_json(200, payload, request_id)
|
||||
return
|
||||
if path == "/profile":
|
||||
eng = self.server.engine
|
||||
payload = {"seq": getattr(eng, "profile_seq", 0) if eng else 0,
|
||||
"turns": list(getattr(eng, "profile", ()) or ()) if eng else []}
|
||||
self.send_json(200, payload, request_id)
|
||||
return
|
||||
if self.serve_static(path):
|
||||
return
|
||||
self.require_auth()
|
||||
|
||||
@@ -0,0 +1,672 @@
|
||||
/* quant.h — quantized matmul kernels (header-only, all functions static).
|
||||
* Multi-architecture SIMD: AVX2 / AVX-512 / AVX-VNNI / ARM NEON / NEON-SDOT /
|
||||
* NEON-i8mm / POWER VSX. Pure compute — no Model or QT dependency. */
|
||||
#ifndef COLI_QUANT_H
|
||||
#define COLI_QUANT_H
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef _OPENMP
|
||||
#include <omp.h>
|
||||
#endif
|
||||
|
||||
/* ---- SIMD includes -------------------------------------------------------- */
|
||||
#ifdef __AVX2__
|
||||
#include <immintrin.h>
|
||||
static inline float hsum256(__m256 v){
|
||||
__m128 lo=_mm256_castps256_ps128(v), hi=_mm256_extractf128_ps(v,1);
|
||||
lo=_mm_add_ps(lo,hi); __m128 sh=_mm_movehl_ps(lo,lo); lo=_mm_add_ps(lo,sh);
|
||||
sh=_mm_shuffle_ps(lo,lo,1); lo=_mm_add_ss(lo,sh); return _mm_cvtss_f32(lo);
|
||||
}
|
||||
static inline int hsum256_i32(__m256i v){
|
||||
__m128i lo=_mm256_castsi256_si128(v), hi=_mm256_extracti128_si256(v,1);
|
||||
lo=_mm_add_epi32(lo,hi); lo=_mm_hadd_epi32(lo,lo); lo=_mm_hadd_epi32(lo,lo);
|
||||
return _mm_cvtsi128_si32(lo);
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVXVNNI__) && defined(__AVX2__)
|
||||
static inline int hsum128_i32(__m128i v){
|
||||
v=_mm_hadd_epi32(v,v); v=_mm_hadd_epi32(v,v); return _mm_cvtsi128_si32(v);
|
||||
}
|
||||
#endif
|
||||
#ifdef __ARM_NEON
|
||||
#include <arm_neon.h>
|
||||
#endif
|
||||
#ifdef __VSX__
|
||||
#include <altivec.h>
|
||||
#undef vector
|
||||
#undef pixel
|
||||
#undef bool
|
||||
#endif
|
||||
|
||||
/* ---- AVX-512 int4->float accumulator -------------------------------------- */
|
||||
#if defined(__AVX512F__) && defined(__AVX512BW__)
|
||||
static int g_i4_acc512=1;
|
||||
static inline float dot_i4f_avx512(const uint8_t *w,const float *x,int I){
|
||||
const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8);
|
||||
__m512 acc0=_mm512_setzero_ps(),acc1=_mm512_setzero_ps(); int i=0;
|
||||
for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1)));
|
||||
__m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
|
||||
__m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi);
|
||||
__m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8));
|
||||
__m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8));
|
||||
acc0=_mm512_fmadd_ps(_mm512_loadu_ps(x+i),w0,acc0);
|
||||
acc1=_mm512_fmadd_ps(_mm512_loadu_ps(x+i+16),w1,acc1);
|
||||
}
|
||||
return _mm512_reduce_add_ps(_mm512_add_ps(acc0,acc1));
|
||||
}
|
||||
static int i4_acc512_selftest(void){
|
||||
enum { N=224 }; uint8_t w[(N+1)/2]; float x[N];
|
||||
for(int i=0;i<N;i++){
|
||||
int q=((i*13+5)&15)-8;
|
||||
if(!(i&1)) w[i>>1]=(uint8_t)(q+8);
|
||||
else w[i>>1]|=(uint8_t)((q+8)<<4);
|
||||
x[i]=(float)(((i*29+7)%101)-50)/37.f;
|
||||
}
|
||||
for(int n=32;n<=N;n+=32){
|
||||
float ref=0; for(int i=0;i<n;i++) ref+=x[i]*(float)(((w[i>>1]>>((i&1)*4))&15)-8);
|
||||
float got=dot_i4f_avx512(w,x,n),tol=2e-5f*(1.f+fabsf(ref));
|
||||
if(fabsf(got-ref)>tol){ fprintf(stderr,"AVX512 i4 selftest n=%d: %.9g != %.9g\n",n,got,ref); return 0; }
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ---- y[S,O] = x[S,I] @ W^T, W[O,I] f32 ---------------------------------- */
|
||||
static void matmul(float *y, const float *x, const float *W, int S, int I, int O){
|
||||
#pragma omp parallel for schedule(static)
|
||||
for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;
|
||||
for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a+=xs[i]*w[i]; y[(int64_t)s*O+o]=a; } }
|
||||
}
|
||||
|
||||
/* ---- y[S,O] = x[S,I] @ W^T, W int8 per-row + scale[O] ------------------- */
|
||||
static void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int S, int I, int O){
|
||||
#pragma omp parallel for schedule(static)
|
||||
for (int o=0;o<O;o++){ const int8_t *w=q+(int64_t)o*I; float sc=scale[o];
|
||||
for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
|
||||
#ifdef __AVX2__
|
||||
__m256 acc=_mm256_setzero_ps();
|
||||
for(;i+8<=I;i+=8){ __m256i wi=_mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*)(w+i)));
|
||||
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), _mm256_cvtepi32_ps(wi), acc); }
|
||||
a=hsum256(acc);
|
||||
#elif defined(__ARM_NEON)
|
||||
float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
|
||||
for(;i+8<=I;i+=8){ int16x8_t w16=vmovl_s8(vld1_s8(w+i));
|
||||
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w16))));
|
||||
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w16)))); }
|
||||
a=vaddvq_f32(vaddq_f32(ac0,ac1));
|
||||
#endif
|
||||
for(;i<I;i++) a+=xs[i]*(float)w[i]; y[(int64_t)s*O+o]=a*sc; } }
|
||||
}
|
||||
|
||||
/* ---- y[S,O] = x[S,I] @ W^T, W int4 packed (2/byte) + scale[O] ------------ */
|
||||
static void matmul_i4(float *y, const float *x, const uint8_t *q4, const float *scale, int S, int I, int O){
|
||||
int rb=(I+1)/2;
|
||||
#pragma omp parallel for schedule(static)
|
||||
for (int o=0;o<O;o++){ const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
|
||||
for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
|
||||
#if defined(__AVX512F__) && defined(__AVX512BW__)
|
||||
if(g_i4_acc512){ a=dot_i4f_avx512(w,xs,I); i=I&~31; }
|
||||
else {
|
||||
#endif
|
||||
#ifdef __AVX2__
|
||||
const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
|
||||
__m256 acc=_mm256_setzero_ps();
|
||||
for(;i+16<=I;i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));
|
||||
__m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
|
||||
__m128i nib=_mm_unpacklo_epi8(lo,hi);
|
||||
__m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8));
|
||||
__m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8));
|
||||
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc);
|
||||
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); }
|
||||
a=hsum256(acc);
|
||||
#elif defined(__ARM_NEON)
|
||||
const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8);
|
||||
float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
|
||||
for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1));
|
||||
uint8x8x2_t z=vzip_u8(vand_u8(by,m4), vshr_n_u8(by,4));
|
||||
int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[0]),b8));
|
||||
int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[1]),b8));
|
||||
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
|
||||
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
|
||||
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
|
||||
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
|
||||
a=vaddvq_f32(vaddq_f32(ac0,ac1));
|
||||
#endif
|
||||
#if defined(__AVX512F__) && defined(__AVX512BW__)
|
||||
}
|
||||
#endif
|
||||
for(;i+1<I;i+=2){ uint8_t byte=w[i>>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8;
|
||||
a += xs[i]*(float)lo + xs[i+1]*(float)hi; }
|
||||
if(i<I){ uint8_t byte=w[i>>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; }
|
||||
y[(int64_t)s*O+o]=a*sc; } }
|
||||
}
|
||||
|
||||
/* ---- y[S,O] = x[S,I] @ W^T, W int4 packed + per-GROUP scales (fmt=4) ----- */
|
||||
static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const float *scale,
|
||||
int S, int I, int O, int gs){
|
||||
int rb=(I+1)/2; int ng=(I+gs-1)/gs;
|
||||
#pragma omp parallel for schedule(static)
|
||||
for(int o=0;o<O;o++){
|
||||
const uint8_t *w=q4+(int64_t)o*rb;
|
||||
const float *scl=scale+(int64_t)o*ng;
|
||||
for(int s=0;s<S;s++){
|
||||
const float *xs=x+(int64_t)s*I; float a=0;
|
||||
for(int g=0; g*gs<I; g++){
|
||||
int base=g*gs; int glen=gs; if(base+glen>I) glen=I-base;
|
||||
float sc=scl[g];
|
||||
int i=base;
|
||||
#ifdef __AVX2__
|
||||
const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
|
||||
__m256 acc=_mm256_setzero_ps();
|
||||
for(; i+16<=base+glen; i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));
|
||||
__m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
|
||||
__m128i nib=_mm_unpacklo_epi8(lo,hi);
|
||||
__m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8));
|
||||
__m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8));
|
||||
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc);
|
||||
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); }
|
||||
a+=hsum256(acc)*sc;
|
||||
#endif
|
||||
for(; i<base+glen; i+=2){
|
||||
if(i+1<base+glen){ uint8_t byte=w[i>>1];
|
||||
a+=(xs[i]*(float)((int)(byte&0xF)-8)+xs[i+1]*(float)((int)(byte>>4)-8))*sc; }
|
||||
else { uint8_t byte=w[i>>1]; a+=xs[i]*(float)((int)(byte&0xF)-8)*sc; }
|
||||
}
|
||||
}
|
||||
y[(int64_t)s*O+o]=a;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ---- fused gate+up: one OMP dispatch for both matrices -------------------- */
|
||||
static void matmul_i4_pair(float *yg, float *yu, const float *x,
|
||||
const uint8_t *qg, const float *sg,
|
||||
const uint8_t *qu, const float *su, int I, int O){
|
||||
int rb=(I+1)/2;
|
||||
#pragma omp parallel for schedule(static)
|
||||
for(int z=0;z<2*O;z++){
|
||||
int o=z<O?z:z-O; const uint8_t *w=(z<O?qg:qu)+(int64_t)o*rb;
|
||||
float a=0; int i=0;
|
||||
#if defined(__AVX512F__) && defined(__AVX512BW__)
|
||||
if(g_i4_acc512){ a=dot_i4f_avx512(w,x,I); i=I&~31; }
|
||||
else {
|
||||
#endif
|
||||
#ifdef __AVX2__
|
||||
const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
|
||||
__m256 acc=_mm256_setzero_ps();
|
||||
for(;i+16<=I;i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));
|
||||
__m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
|
||||
__m128i nib=_mm_unpacklo_epi8(lo,hi);
|
||||
__m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8));
|
||||
__m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8));
|
||||
acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc);
|
||||
acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); }
|
||||
a=hsum256(acc);
|
||||
#elif defined(__ARM_NEON)
|
||||
const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8);
|
||||
float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0);
|
||||
for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1));
|
||||
uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4));
|
||||
int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8));
|
||||
int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8));
|
||||
ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
|
||||
ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
|
||||
ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
|
||||
ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
|
||||
a=vaddvq_f32(vaddq_f32(ac0,ac1));
|
||||
#endif
|
||||
#if defined(__AVX512F__) && defined(__AVX512BW__)
|
||||
}
|
||||
#endif
|
||||
for(;i+1<I;i+=2){ uint8_t b=w[i>>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); }
|
||||
if(i<I) a+=x[i]*(float)((w[i>>1]&15)-8);
|
||||
(z<O?yg:yu)[o]=a*(z<O?sg:su)[o];
|
||||
}
|
||||
}
|
||||
|
||||
/* ---- y[S,O] = x[S,I] @ W^T, W int2 packed (4/byte) + scale[O] ------------ */
|
||||
static void matmul_i2(float *y, const float *x, const uint8_t *q2, const float *scale, int S, int I, int O){
|
||||
int rb=(I+3)/4;
|
||||
#pragma omp parallel for schedule(static)
|
||||
for (int o=0;o<O;o++){ const uint8_t *w=q2+(int64_t)o*rb; float sc=scale[o];
|
||||
for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
|
||||
#ifdef __AVX2__
|
||||
const __m128i m2=_mm_set1_epi8(0x03); const __m256i b2=_mm256_set1_epi32(2);
|
||||
__m256 acc=_mm256_setzero_ps();
|
||||
for(;i+16<=I;i+=16){ __m128i by=_mm_cvtsi32_si128(*(const int*)(w+(i>>2)));
|
||||
__m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2);
|
||||
__m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2);
|
||||
__m128i lo=_mm_unpacklo_epi8(p0,p1), hi=_mm_unpacklo_epi8(p2,p3);
|
||||
__m128i nib=_mm_unpacklo_epi16(lo,hi);
|
||||
__m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b2));
|
||||
__m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b2));
|
||||
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc);
|
||||
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); }
|
||||
a=hsum256(acc);
|
||||
#elif defined(__ARM_NEON)
|
||||
const uint8x8_t m2v=vdup_n_u8(3); const int8x8_t b2v=vdup_n_s8(2);
|
||||
float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
|
||||
for(;i+16<=I;i+=16){ uint32_t wd; memcpy(&wd, w+(i>>2), 4);
|
||||
uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd));
|
||||
uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v));
|
||||
uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6));
|
||||
uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0]));
|
||||
int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[0]),b2v));
|
||||
int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[1]),b2v));
|
||||
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
|
||||
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
|
||||
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
|
||||
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
|
||||
a=vaddvq_f32(vaddq_f32(ac0,ac1));
|
||||
#endif
|
||||
for(;i<I;i++){ uint8_t byte=w[i>>2]; int sh=(i&3)*2; a += xs[i]*(float)((int)((byte>>sh)&3)-2); }
|
||||
y[(int64_t)s*O+o]=a*sc; } }
|
||||
}
|
||||
|
||||
/* ---- IDOT: integer dot kernels (int8-quantized activations) --------------- */
|
||||
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
|
||||
#define IDOT_KERNEL "avx512-vnni"
|
||||
#elif defined(__AVXVNNI__) && defined(__AVX2__)
|
||||
#define IDOT_KERNEL "avx-vnni"
|
||||
#elif defined(__AVX2__)
|
||||
#define IDOT_KERNEL "avx2"
|
||||
#elif defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
|
||||
#define IDOT_KERNEL "neon-i8mm"
|
||||
#elif defined(__ARM_NEON)
|
||||
#define IDOT_KERNEL "neon"
|
||||
#elif defined(__VSX__)
|
||||
#define IDOT_KERNEL "vsx"
|
||||
#else
|
||||
#define IDOT_KERNEL "scalar"
|
||||
#endif
|
||||
static int g_idot=1;
|
||||
#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
|
||||
static int g_i4s=1;
|
||||
#elif defined(__VSX__)
|
||||
static int g_i4s=1;
|
||||
#else
|
||||
static int g_i4s=2;
|
||||
#endif
|
||||
|
||||
static inline float qrow_i8(const float *x, int8_t *q, int I){
|
||||
float amax=0; for(int i=0;i<I;i++){ float a=fabsf(x[i]); if(a>amax)amax=a; }
|
||||
float s=amax/127.f; if(s<1e-12f) s=1e-12f; float inv=1.f/s;
|
||||
for(int i=0;i<I;i++) q[i]=(int8_t)lrintf(x[i]*inv);
|
||||
return s;
|
||||
}
|
||||
|
||||
/* dot int8*int8 */
|
||||
static inline int32_t dot_i8i8(const int8_t *w, const int8_t *x, int I){
|
||||
int32_t sum=0; int i=0;
|
||||
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
|
||||
__m512i acc=_mm512_setzero_si512();
|
||||
for(;i+64<=I;i+=64){
|
||||
__m512i wv=_mm512_loadu_si512((const void*)(w+i));
|
||||
__m512i xv=_mm512_loadu_si512((const void*)(x+i));
|
||||
__mmask64 neg=_mm512_movepi8_mask(wv);
|
||||
__m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv);
|
||||
acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs);
|
||||
}
|
||||
sum=_mm512_reduce_add_epi32(acc);
|
||||
#elif defined(__AVXVNNI__) && defined(__AVX2__)
|
||||
__m128i acc=_mm_setzero_si128();
|
||||
for(;i+16<=I;i+=16){
|
||||
__m128i wv=_mm_loadu_si128((const __m128i*)(w+i));
|
||||
__m128i xv=_mm_loadu_si128((const __m128i*)(x+i));
|
||||
__m128i xs=_mm_sign_epi8(xv,wv);
|
||||
acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(wv),xs);
|
||||
}
|
||||
sum=hsum128_i32(acc);
|
||||
#elif defined(__AVX2__)
|
||||
__m256i acc=_mm256_setzero_si256(); const __m256i ones=_mm256_set1_epi16(1);
|
||||
for(;i+32<=I;i+=32){
|
||||
__m256i wv=_mm256_loadu_si256((const __m256i*)(w+i));
|
||||
__m256i xv=_mm256_loadu_si256((const __m256i*)(x+i));
|
||||
__m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv));
|
||||
acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones));
|
||||
}
|
||||
sum=hsum256_i32(acc);
|
||||
#elif defined(__ARM_NEON)
|
||||
#if defined(__ARM_FEATURE_DOTPROD)
|
||||
int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0);
|
||||
for(;i+64<=I;i+=64){
|
||||
a0=vdotq_s32(a0,vld1q_s8(w+i), vld1q_s8(x+i));
|
||||
a1=vdotq_s32(a1,vld1q_s8(w+i+16),vld1q_s8(x+i+16));
|
||||
a2=vdotq_s32(a2,vld1q_s8(w+i+32),vld1q_s8(x+i+32));
|
||||
a3=vdotq_s32(a3,vld1q_s8(w+i+48),vld1q_s8(x+i+48));
|
||||
}
|
||||
int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
|
||||
for(;i+16<=I;i+=16) acc=vdotq_s32(acc,vld1q_s8(w+i),vld1q_s8(x+i));
|
||||
sum=vaddvq_s32(acc);
|
||||
#else
|
||||
int32x4_t acc=vdupq_n_s32(0);
|
||||
for(;i+16<=I;i+=16){
|
||||
int8x16_t wv=vld1q_s8(w+i), xv=vld1q_s8(x+i);
|
||||
int16x8_t p=vmull_s8(vget_low_s8(wv),vget_low_s8(xv));
|
||||
p=vmlal_s8(p,vget_high_s8(wv),vget_high_s8(xv));
|
||||
acc=vpadalq_s16(acc,p);
|
||||
}
|
||||
sum=vaddvq_s32(acc);
|
||||
#endif
|
||||
#elif defined(__VSX__)
|
||||
__vector signed int acc=vec_splats(0);
|
||||
const __vector signed char vz=vec_splats((signed char)0);
|
||||
for(;i+16<=I;i+=16){
|
||||
__vector signed char wv=vec_xl(0,(const signed char*)(w+i));
|
||||
__vector signed char xv=vec_xl(0,(const signed char*)(x+i));
|
||||
__vector __bool char neg=vec_cmplt(wv,vz);
|
||||
__vector signed char xs=vec_sel(xv,vec_sub(vz,xv),neg);
|
||||
__vector unsigned char wa=(__vector unsigned char)vec_sel(wv,vec_sub(vz,wv),neg);
|
||||
acc=vec_msum(xs,wa,acc);
|
||||
}
|
||||
sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3);
|
||||
#endif
|
||||
for(;i<I;i++) sum+=(int32_t)w[i]*x[i];
|
||||
return sum;
|
||||
}
|
||||
|
||||
/* dot int4(packed)*int8 */
|
||||
static inline int32_t dot_i4i8(const uint8_t *w4, const int8_t *x, int I){
|
||||
int32_t sum=0; int i=0;
|
||||
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
|
||||
const __m256i m4v=_mm256_set1_epi8(0x0F);
|
||||
const __m512i b8v=_mm512_set1_epi8(8);
|
||||
const __m512i xidx=_mm512_setr_epi64(0,1,4,5,2,3,6,7);
|
||||
__m512i acc=_mm512_setzero_si512();
|
||||
for(;i+64<=I;i+=64){
|
||||
__m256i by=_mm256_loadu_si256((const __m256i*)(w4+(i>>1)));
|
||||
__m256i lo=_mm256_and_si256(by,m4v), hi=_mm256_and_si256(_mm256_srli_epi16(by,4),m4v);
|
||||
__m256i z0=_mm256_unpacklo_epi8(lo,hi), z1=_mm256_unpackhi_epi8(lo,hi);
|
||||
__m512i wv=_mm512_sub_epi8(_mm512_inserti64x4(_mm512_castsi256_si512(z0),z1,1),b8v);
|
||||
__m512i xv=_mm512_permutexvar_epi64(xidx,_mm512_loadu_si512((const void*)(x+i)));
|
||||
__mmask64 neg=_mm512_movepi8_mask(wv);
|
||||
__m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv);
|
||||
acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs);
|
||||
}
|
||||
sum=_mm512_reduce_add_epi32(acc);
|
||||
#elif defined(__AVXVNNI__) && defined(__AVX2__)
|
||||
const __m128i m4=_mm_set1_epi8(0x0F); const __m128i b8=_mm_set1_epi8(8);
|
||||
__m128i acc=_mm_setzero_si128();
|
||||
for(;i+32<=I;i+=32){
|
||||
__m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1)));
|
||||
__m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
|
||||
__m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi);
|
||||
__m128i w0=_mm_sub_epi8(n0,b8), w1=_mm_sub_epi8(n1,b8);
|
||||
__m128i x0=_mm_loadu_si128((const __m128i*)(x+i));
|
||||
__m128i x1=_mm_loadu_si128((const __m128i*)(x+i+16));
|
||||
acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w0),_mm_sign_epi8(x0,w0));
|
||||
acc=_mm_dpbusd_epi32(acc,_mm_abs_epi8(w1),_mm_sign_epi8(x1,w1));
|
||||
}
|
||||
sum=hsum128_i32(acc);
|
||||
#elif defined(__AVX2__)
|
||||
const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi8(8);
|
||||
const __m256i ones=_mm256_set1_epi16(1);
|
||||
__m256i acc=_mm256_setzero_si256();
|
||||
for(;i+32<=I;i+=32){
|
||||
__m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1)));
|
||||
__m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
|
||||
__m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi);
|
||||
__m256i wv=_mm256_sub_epi8(_mm256_set_m128i(n1,n0),b8);
|
||||
__m256i xv=_mm256_loadu_si256((const __m256i*)(x+i));
|
||||
__m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv));
|
||||
acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones));
|
||||
}
|
||||
sum=hsum256_i32(acc);
|
||||
#elif defined(__ARM_NEON)
|
||||
const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8);
|
||||
#if defined(__ARM_FEATURE_DOTPROD)
|
||||
int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0);
|
||||
for(;i+64<=I;i+=64){
|
||||
uint8x16_t byA=vld1q_u8(w4+(i>>1)), byB=vld1q_u8(w4+(i>>1)+16);
|
||||
uint8x16x2_t zA=vzipq_u8(vandq_u8(byA,m4q), vshrq_n_u8(byA,4));
|
||||
uint8x16x2_t zB=vzipq_u8(vandq_u8(byB,m4q), vshrq_n_u8(byB,4));
|
||||
a0=vdotq_s32(a0,vsubq_s8(vreinterpretq_s8_u8(zA.val[0]),b8q),vld1q_s8(x+i));
|
||||
a1=vdotq_s32(a1,vsubq_s8(vreinterpretq_s8_u8(zA.val[1]),b8q),vld1q_s8(x+i+16));
|
||||
a2=vdotq_s32(a2,vsubq_s8(vreinterpretq_s8_u8(zB.val[0]),b8q),vld1q_s8(x+i+32));
|
||||
a3=vdotq_s32(a3,vsubq_s8(vreinterpretq_s8_u8(zB.val[1]),b8q),vld1q_s8(x+i+48));
|
||||
}
|
||||
int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
|
||||
for(;i+32<=I;i+=32){
|
||||
uint8x16_t by=vld1q_u8(w4+(i>>1));
|
||||
uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4));
|
||||
acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q),vld1q_s8(x+i));
|
||||
acc=vdotq_s32(acc,vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q),vld1q_s8(x+i+16));
|
||||
}
|
||||
sum=vaddvq_s32(acc);
|
||||
#else
|
||||
int32x4_t acc=vdupq_n_s32(0);
|
||||
for(;i+32<=I;i+=32){
|
||||
uint8x16_t by=vld1q_u8(w4+(i>>1));
|
||||
uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4));
|
||||
int8x16_t w0=vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q);
|
||||
int8x16_t w1=vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q);
|
||||
int8x16_t x0=vld1q_s8(x+i), x1=vld1q_s8(x+i+16);
|
||||
int16x8_t p=vmull_s8(vget_low_s8(w0),vget_low_s8(x0));
|
||||
p=vmlal_s8(p,vget_high_s8(w0),vget_high_s8(x0));
|
||||
acc=vpadalq_s16(acc,p);
|
||||
p=vmull_s8(vget_low_s8(w1),vget_low_s8(x1));
|
||||
p=vmlal_s8(p,vget_high_s8(w1),vget_high_s8(x1));
|
||||
acc=vpadalq_s16(acc,p);
|
||||
}
|
||||
sum=vaddvq_s32(acc);
|
||||
#endif
|
||||
#elif defined(__VSX__)
|
||||
const __vector unsigned char m4v=vec_splats((unsigned char)0x0F);
|
||||
const __vector unsigned char sh4=vec_splats((unsigned char)4);
|
||||
const __vector signed char b8v=vec_splats((signed char)8);
|
||||
const __vector signed char vz=vec_splats((signed char)0);
|
||||
__vector signed int acc=vec_splats(0);
|
||||
for(;i+32<=I;i+=32){
|
||||
__vector unsigned char by=vec_xl(0,w4+(i>>1));
|
||||
__vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4);
|
||||
__vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v);
|
||||
__vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v);
|
||||
__vector signed char x0=vec_xl(0,(const signed char*)(x+i));
|
||||
__vector signed char x1=vec_xl(0,(const signed char*)(x+i+16));
|
||||
__vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz);
|
||||
acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0),
|
||||
(__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc);
|
||||
acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1),
|
||||
(__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc);
|
||||
}
|
||||
sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3);
|
||||
#endif
|
||||
for(;i+1<I;i+=2){ uint8_t b=w4[i>>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; }
|
||||
if(i<I){ uint8_t b=w4[i>>1]; sum+=((int)(b&0xF)-8)*x[i]; }
|
||||
return sum;
|
||||
}
|
||||
|
||||
/* ---- ARM i8mm SMMLA tiled kernels ---------------------------------------- */
|
||||
#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
|
||||
static inline int32x4_t mm_tile16(int32x4_t acc, int8x16_t wo, int8x16_t wo1,
|
||||
int8x16_t xs, int8x16_t xs1){
|
||||
acc=vmmlaq_s32(acc, vcombine_s8(vget_low_s8(wo), vget_low_s8(wo1)),
|
||||
vcombine_s8(vget_low_s8(xs), vget_low_s8(xs1)));
|
||||
return vmmlaq_s32(acc, vcombine_s8(vget_high_s8(wo), vget_high_s8(wo1)),
|
||||
vcombine_s8(vget_high_s8(xs), vget_high_s8(xs1)));
|
||||
}
|
||||
static void matmul_q_idot_mm(float *y, const int8_t *xq, const float *sx, const int8_t *q,
|
||||
const float *scale, int S, int I, int O){
|
||||
#pragma omp parallel for schedule(static)
|
||||
for(int o=0;o<(O&~1);o+=2){
|
||||
const int8_t *wo=q+(int64_t)o*I, *wo1=q+(int64_t)(o+1)*I;
|
||||
float sc0=scale[o], sc1=scale[o+1];
|
||||
for(int s=0;s<(S&~1);s+=2){
|
||||
const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I;
|
||||
int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0;
|
||||
for(;i+64<=I;i+=64){
|
||||
a0=mm_tile16(a0,vld1q_s8(wo+i), vld1q_s8(wo1+i), vld1q_s8(xs+i), vld1q_s8(xs1+i));
|
||||
a1=mm_tile16(a1,vld1q_s8(wo+i+16),vld1q_s8(wo1+i+16),vld1q_s8(xs+i+16),vld1q_s8(xs1+i+16));
|
||||
a2=mm_tile16(a2,vld1q_s8(wo+i+32),vld1q_s8(wo1+i+32),vld1q_s8(xs+i+32),vld1q_s8(xs1+i+32));
|
||||
a3=mm_tile16(a3,vld1q_s8(wo+i+48),vld1q_s8(wo1+i+48),vld1q_s8(xs+i+48),vld1q_s8(xs1+i+48));
|
||||
}
|
||||
for(;i+16<=I;i+=16)
|
||||
a0=mm_tile16(a0,vld1q_s8(wo+i),vld1q_s8(wo1+i),vld1q_s8(xs+i),vld1q_s8(xs1+i));
|
||||
int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
|
||||
int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1);
|
||||
int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3);
|
||||
for(;i<I;i++){ int a=wo[i],b=wo1[i],u=xs[i],v=xs1[i];
|
||||
d00+=a*u; d01+=a*v; d10+=b*u; d11+=b*v; }
|
||||
y[(int64_t)s*O+o] =(float)d00*sc0*sx[s];
|
||||
y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s];
|
||||
y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1];
|
||||
y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1];
|
||||
}
|
||||
if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I;
|
||||
y[(int64_t)s*O+o] =(float)dot_i8i8(wo, xs,I)*sc0*sx[s];
|
||||
y[(int64_t)s*O+(o+1)]=(float)dot_i8i8(wo1,xs,I)*sc1*sx[s]; }
|
||||
}
|
||||
if(O&1){ int o=O-1; const int8_t *w=q+(int64_t)o*I; float sc=scale[o];
|
||||
#pragma omp parallel for schedule(static)
|
||||
for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i8i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
|
||||
}
|
||||
static void matmul_i4_idot_mm(float *y, const int8_t *xq, const float *sx, const uint8_t *q4,
|
||||
const float *scale, int S, int I, int O){
|
||||
int rb=(I+1)/2;
|
||||
#pragma omp parallel for schedule(static)
|
||||
for(int o=0;o<(O&~1);o+=2){
|
||||
const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8);
|
||||
const uint8_t *wo=q4+(int64_t)o*rb, *wo1=q4+(int64_t)(o+1)*rb;
|
||||
float sc0=scale[o], sc1=scale[o+1];
|
||||
for(int s=0;s<(S&~1);s+=2){
|
||||
const int8_t *xs=xq+(int64_t)s*I, *xs1=xq+(int64_t)(s+1)*I;
|
||||
int32x4_t a0=vdupq_n_s32(0),a1=vdupq_n_s32(0),a2=vdupq_n_s32(0),a3=vdupq_n_s32(0); int i=0;
|
||||
for(;i+64<=I;i+=64){
|
||||
uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1));
|
||||
uint8x16_t cyo=vld1q_u8(wo+(i>>1)+16), cyo1=vld1q_u8(wo1+(i>>1)+16);
|
||||
uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4));
|
||||
uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4));
|
||||
uint8x16x2_t ko =vzipq_u8(vandq_u8(cyo, m4q), vshrq_n_u8(cyo, 4));
|
||||
uint8x16x2_t ko1=vzipq_u8(vandq_u8(cyo1,m4q), vshrq_n_u8(cyo1,4));
|
||||
a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q),
|
||||
vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q),
|
||||
vld1q_s8(xs+i), vld1q_s8(xs1+i));
|
||||
a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q),
|
||||
vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q),
|
||||
vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16));
|
||||
a2=mm_tile16(a2, vsubq_s8(vreinterpretq_s8_u8(ko.val[0]),b8q),
|
||||
vsubq_s8(vreinterpretq_s8_u8(ko1.val[0]),b8q),
|
||||
vld1q_s8(xs+i+32), vld1q_s8(xs1+i+32));
|
||||
a3=mm_tile16(a3, vsubq_s8(vreinterpretq_s8_u8(ko.val[1]),b8q),
|
||||
vsubq_s8(vreinterpretq_s8_u8(ko1.val[1]),b8q),
|
||||
vld1q_s8(xs+i+48), vld1q_s8(xs1+i+48));
|
||||
}
|
||||
for(;i+32<=I;i+=32){
|
||||
uint8x16_t byo=vld1q_u8(wo+(i>>1)), byo1=vld1q_u8(wo1+(i>>1));
|
||||
uint8x16x2_t zo =vzipq_u8(vandq_u8(byo, m4q), vshrq_n_u8(byo, 4));
|
||||
uint8x16x2_t zo1=vzipq_u8(vandq_u8(byo1,m4q), vshrq_n_u8(byo1,4));
|
||||
a0=mm_tile16(a0, vsubq_s8(vreinterpretq_s8_u8(zo.val[0]),b8q),
|
||||
vsubq_s8(vreinterpretq_s8_u8(zo1.val[0]),b8q),
|
||||
vld1q_s8(xs+i), vld1q_s8(xs1+i));
|
||||
a1=mm_tile16(a1, vsubq_s8(vreinterpretq_s8_u8(zo.val[1]),b8q),
|
||||
vsubq_s8(vreinterpretq_s8_u8(zo1.val[1]),b8q),
|
||||
vld1q_s8(xs+i+16), vld1q_s8(xs1+i+16));
|
||||
}
|
||||
int32x4_t acc=vaddq_s32(vaddq_s32(a0,a1),vaddq_s32(a2,a3));
|
||||
int32_t d00=vgetq_lane_s32(acc,0), d01=vgetq_lane_s32(acc,1);
|
||||
int32_t d10=vgetq_lane_s32(acc,2), d11=vgetq_lane_s32(acc,3);
|
||||
for(;i+1<I;i+=2){ uint8_t bo=wo[i>>1], bo1=wo1[i>>1];
|
||||
int a0=(int)(bo&0xF)-8, a1=(int)(bo>>4)-8, b0=(int)(bo1&0xF)-8, b1=(int)(bo1>>4)-8;
|
||||
int u0=xs[i],u1=xs[i+1],v0=xs1[i],v1=xs1[i+1];
|
||||
d00+=a0*u0+a1*u1; d01+=a0*v0+a1*v1; d10+=b0*u0+b1*u1; d11+=b0*v0+b1*v1; }
|
||||
if(i<I){ uint8_t bo=wo[i>>1], bo1=wo1[i>>1];
|
||||
int a0=(int)(bo&0xF)-8, b0=(int)(bo1&0xF)-8;
|
||||
d00+=a0*xs[i]; d01+=a0*xs1[i]; d10+=b0*xs[i]; d11+=b0*xs1[i]; }
|
||||
y[(int64_t)s*O+o] =(float)d00*sc0*sx[s];
|
||||
y[(int64_t)s*O+(o+1)] =(float)d10*sc1*sx[s];
|
||||
y[(int64_t)(s+1)*O+o] =(float)d01*sc0*sx[s+1];
|
||||
y[(int64_t)(s+1)*O+(o+1)]=(float)d11*sc1*sx[s+1];
|
||||
}
|
||||
if(S&1){ int s=S-1; const int8_t *xs=xq+(int64_t)s*I;
|
||||
y[(int64_t)s*O+o] =(float)dot_i4i8(wo, xs,I)*sc0*sx[s];
|
||||
y[(int64_t)s*O+(o+1)]=(float)dot_i4i8(wo1,xs,I)*sc1*sx[s]; }
|
||||
}
|
||||
if(O&1){ int o=O-1; const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
|
||||
#pragma omp parallel for schedule(static)
|
||||
for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i4i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ---- IDOT dispatch (int8-quantized activations) --------------------------- */
|
||||
static void matmul_q_idot(float *y, const int8_t *xq, const float *sx, const int8_t *q,
|
||||
const float *scale, int S, int I, int O){
|
||||
#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
|
||||
if(S>=2){ matmul_q_idot_mm(y,xq,sx,q,scale,S,I,O); return; }
|
||||
#endif
|
||||
#pragma omp parallel for schedule(static)
|
||||
for(int o=0;o<O;o++){ const int8_t *w=q+(int64_t)o*I; float sc=scale[o];
|
||||
for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i8i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
|
||||
}
|
||||
static void matmul_i4_idot(float *y, const int8_t *xq, const float *sx, const uint8_t *q4,
|
||||
const float *scale, int S, int I, int O){
|
||||
int rb=(I+1)/2;
|
||||
#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
|
||||
if(S>=2){ matmul_i4_idot_mm(y,xq,sx,q4,scale,S,I,O); return; }
|
||||
#endif
|
||||
#pragma omp parallel for schedule(static)
|
||||
for(int o=0;o<O;o++){ const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
|
||||
for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i4i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
|
||||
}
|
||||
|
||||
/* ---- per-thread quantization scratch -------------------------------------- */
|
||||
typedef struct { int8_t *xq; size_t xq_cap; float *sx; size_t sx_cap; } QScratch;
|
||||
static _Thread_local QScratch g_qscratch;
|
||||
static void quant_scratch(size_t xn, size_t sn, int8_t **xq, float **sx){
|
||||
if(xn>g_qscratch.xq_cap){
|
||||
int8_t *p=realloc(g_qscratch.xq,xn);
|
||||
if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); }
|
||||
g_qscratch.xq=p; g_qscratch.xq_cap=xn;
|
||||
}
|
||||
if(sn>g_qscratch.sx_cap){
|
||||
float *p=realloc(g_qscratch.sx,sn*sizeof(float));
|
||||
if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); }
|
||||
g_qscratch.sx=p; g_qscratch.sx_cap=sn;
|
||||
}
|
||||
*xq=g_qscratch.xq; *sx=g_qscratch.sx;
|
||||
}
|
||||
|
||||
/* ---- f32 -> quantized packing --------------------------------------------- */
|
||||
static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits){
|
||||
int qmax=(1<<(bits-1))-1;
|
||||
#pragma omp parallel for schedule(static)
|
||||
for(int o=0;o<O;o++){ const float *wr=w+(int64_t)o*I; float amax=0;
|
||||
for(int i=0;i<I;i++){ float a=fabsf(wr[i]); if(a>amax)amax=a; }
|
||||
float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s;
|
||||
int8_t *qr=q+(int64_t)o*I;
|
||||
for(int i=0;i<I;i++){ int v=(int)lrintf(wr[i]/s); if(v>qmax)v=qmax; if(v<-qmax-1)v=-qmax-1; qr[i]=(int8_t)v; }
|
||||
}
|
||||
}
|
||||
static void pack_int4(const float *w, uint8_t *q4, float *scale, int O, int I, int bits){
|
||||
int qmax=(1<<(bits-1))-1, rb=(I+1)/2;
|
||||
#pragma omp parallel for schedule(static)
|
||||
for(int o=0;o<O;o++){ const float *wr=w+(int64_t)o*I; float amax=0;
|
||||
for(int i=0;i<I;i++){ float a=fabsf(wr[i]); if(a>amax)amax=a; }
|
||||
float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s;
|
||||
uint8_t *qr=q4+(int64_t)o*rb;
|
||||
for(int i=0;i<I;i+=2){
|
||||
int v0=(int)lrintf(wr[i]/s); if(v0>qmax)v0=qmax; if(v0<-8)v0=-8;
|
||||
int v1=0; if(i+1<I){ v1=(int)lrintf(wr[i+1]/s); if(v1>qmax)v1=qmax; if(v1<-8)v1=-8; }
|
||||
qr[i>>1] = (uint8_t)((v0+8) | ((v1+8)<<4));
|
||||
}
|
||||
}
|
||||
}
|
||||
static void pack_int2(const float *w, uint8_t *q2, float *scale, int O, int I, int bits){
|
||||
int qmax=(1<<(bits-1))-1, rb=(I+3)/4;
|
||||
#pragma omp parallel for schedule(static)
|
||||
for(int o=0;o<O;o++){ const float *wr=w+(int64_t)o*I; float amax=0;
|
||||
for(int i=0;i<I;i++){ float a=fabsf(wr[i]); if(a>amax)amax=a; }
|
||||
float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s;
|
||||
uint8_t *qr=q2+(int64_t)o*rb;
|
||||
for(int i=0;i<I;i+=4){ uint8_t byte=0;
|
||||
for(int k=0;k<4 && i+k<I;k++){ int v=(int)lrintf(wr[i+k]/s); if(v>qmax)v=qmax; if(v<-2)v=-2; byte|=(uint8_t)((v+2)<<(k*2)); }
|
||||
qr[i>>2]=byte;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* COLI_QUANT_H */
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"prompt_ids": [
|
||||
510,
|
||||
5347,
|
||||
273,
|
||||
6181,
|
||||
310
|
||||
],
|
||||
"full_ids": [
|
||||
510,
|
||||
5347,
|
||||
273,
|
||||
6181,
|
||||
310,
|
||||
7785,
|
||||
15,
|
||||
187,
|
||||
187,
|
||||
510,
|
||||
3565,
|
||||
3448,
|
||||
273,
|
||||
6181,
|
||||
310,
|
||||
5112,
|
||||
15
|
||||
]
|
||||
}
|
||||
+107
-4
@@ -203,6 +203,70 @@ def physical_cpu_count():
|
||||
return os.cpu_count() or 1
|
||||
|
||||
|
||||
def cpu_socket_count():
|
||||
"""Return the number of physical CPU sockets visible to this process."""
|
||||
if not sys.platform.startswith("linux"):
|
||||
return 1
|
||||
try:
|
||||
result = subprocess.run(["lscpu", "-p=socket"], text=True,
|
||||
capture_output=True, check=True, timeout=5)
|
||||
sockets = {int(line) for line in result.stdout.splitlines()
|
||||
if line and not line.startswith("#")}
|
||||
if sockets:
|
||||
return len(sockets)
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
pass
|
||||
return 1
|
||||
|
||||
|
||||
def _auto_tune(bottleneck_class, projected_hit, gpus, cpu_sockets, plan_has_metal):
|
||||
"""Derive tuning knobs from the bottleneck classification."""
|
||||
tune = {}
|
||||
has_gpu = bool(gpus)
|
||||
n_gpu = len(gpus)
|
||||
|
||||
# MTP: costs more than it saves when compute-bound (#389 measured 42% loss)
|
||||
if bottleneck_class == "compute":
|
||||
tune["DRAFT"] = {"value": "0",
|
||||
"reason": "compute-bound: MTP batch overhead exceeds yield"}
|
||||
elif bottleneck_class == "disk" and projected_hit < 0.90:
|
||||
tune["DRAFT"] = {"value": "0",
|
||||
"reason": "low hit rate: MTP widens expert union, adds disk reads"}
|
||||
# otherwise leave DRAFT unset (engine default: auto)
|
||||
|
||||
# PIPE: resident pipeline mode depends on GPU count
|
||||
if has_gpu and n_gpu == 1:
|
||||
tune["COLI_CUDA_PIPE"] = {"value": "1",
|
||||
"reason": "single GPU: S=1 pipeline gate"}
|
||||
elif has_gpu and n_gpu > 1:
|
||||
tune["COLI_CUDA_PIPE"] = {"value": "2",
|
||||
"reason": "multi-GPU: residual stays on-device across layers"}
|
||||
elif not has_gpu and bottleneck_class == "disk":
|
||||
tune["PIPE"] = {"value": "1",
|
||||
"reason": "overlap disk reads with resident expert compute"}
|
||||
|
||||
# NUMA: selective interleave for GPU hosts, blanket hint for CPU-only
|
||||
if cpu_sockets > 1 and has_gpu:
|
||||
tune["COLI_NUMA"] = {"value": "1",
|
||||
"reason": "multi-socket + GPU: interleave expert slabs, protect DMA buffers"}
|
||||
elif cpu_sockets > 1 and not has_gpu:
|
||||
tune["COLI_NUMA"] = {"value": "1",
|
||||
"reason": "multi-socket CPU-only: interleave expert slabs across nodes"}
|
||||
tune["_numa_hint"] = "numactl --interleave=all may perform better on CPU-only hosts"
|
||||
|
||||
# OMP: kill hot-thread spin when GPU/Metal owns the power budget
|
||||
if plan_has_metal:
|
||||
tune["COLI_NO_OMP_TUNE"] = {"value": "1",
|
||||
"reason": "Metal: OMP spin-wait steals GPU power budget"}
|
||||
|
||||
# PIN: fully resident if RAM allows and no GPU tier competes
|
||||
if projected_hit >= 0.99 and not has_gpu:
|
||||
tune["PIN_GB"] = {"value": "all",
|
||||
"reason": "enough RAM for full expert residency"}
|
||||
|
||||
return tune
|
||||
|
||||
|
||||
POLICIES = {
|
||||
"quality": {"preserve_quantization": True, "preserve_router": True},
|
||||
"balanced": {"preserve_quantization": True, "preserve_router": True},
|
||||
@@ -212,11 +276,12 @@ POLICIES = {
|
||||
|
||||
def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
|
||||
available_memory=None, available_disk=None, gpus=None,
|
||||
policy="quality", physical_cpus=None):
|
||||
policy="quality", physical_cpus=None, cpu_sockets=None):
|
||||
if policy not in POLICIES:
|
||||
raise ValueError(f"unknown policy: {policy}")
|
||||
info = analyze_model(model)
|
||||
physical_cpus = physical_cpu_count() if physical_cpus is None else physical_cpus
|
||||
cpu_sockets = cpu_socket_count() if cpu_sockets is None else cpu_sockets
|
||||
cfg = info["config"]
|
||||
available_memory = memory_available() if available_memory is None else available_memory
|
||||
if available_disk is None:
|
||||
@@ -273,12 +338,28 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
|
||||
if cold_bytes:
|
||||
warnings.append("cold expert misses may reach disk; normal decode speed depends on hit rate")
|
||||
|
||||
total_expert = info["expert_bytes"]
|
||||
resident_expert = hot_bytes + warm_bytes
|
||||
projected_hit = resident_expert / total_expert if total_expert else 1.0
|
||||
|
||||
if cold_bytes:
|
||||
bottleneck = "disk expert misses"
|
||||
elif warm_bytes:
|
||||
bottleneck = "CPU expert compute and RAM bandwidth"
|
||||
bottleneck_class = "disk"
|
||||
elif warm_bytes and gpus:
|
||||
bottleneck = "CPU expert tail and GPU compute"
|
||||
bottleneck_class = "mixed"
|
||||
elif projected_hit >= 0.99:
|
||||
if gpus:
|
||||
bottleneck = "GPU compute and interconnect"
|
||||
else:
|
||||
bottleneck = "CPU expert compute (fully resident)"
|
||||
bottleneck_class = "compute"
|
||||
else:
|
||||
bottleneck = "GPU compute and interconnect"
|
||||
bottleneck = "CPU expert compute and RAM bandwidth"
|
||||
bottleneck_class = "memory"
|
||||
|
||||
tune = _auto_tune(bottleneck_class, projected_hit, gpus, cpu_sockets,
|
||||
plan_has_metal=False)
|
||||
|
||||
return {
|
||||
"version": 2,
|
||||
@@ -286,6 +367,7 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
|
||||
"quality_preserving": policy != "experimental-fast"},
|
||||
"model": {key: value for key, value in info.items() if key != "config"},
|
||||
"cpu": {"physical_cores": max(1, int(physical_cpus)),
|
||||
"sockets": max(1, int(cpu_sockets)),
|
||||
"thread_policy": "physical-cores"},
|
||||
"tiers": {
|
||||
"disk": {"role": "cold-backing", "model_bytes": info["model_bytes"],
|
||||
@@ -299,6 +381,9 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
|
||||
"expert_capacity": vram_experts, "requires_host_backing": False},
|
||||
},
|
||||
"expected_bottleneck": bottleneck,
|
||||
"bottleneck_class": bottleneck_class,
|
||||
"projected_hit_rate": round(projected_hit, 4),
|
||||
"tune": tune,
|
||||
"decisions": [
|
||||
{"target": "VRAM", "reason": "profile-ranked hot experts"},
|
||||
{"target": "RAM", "reason": "warm experts execute on CPU without quality loss"},
|
||||
@@ -318,6 +403,11 @@ def environment_for_plan(plan, env=None, cuda_enabled=True):
|
||||
# ("Affinity not supported on this configuration"): non impostarle li'.
|
||||
result.setdefault("OMP_PROC_BIND", "spread")
|
||||
result.setdefault("OMP_PLACES", "cores")
|
||||
tune = plan.get("tune", {})
|
||||
for key, entry in tune.items():
|
||||
if key.startswith("_"):
|
||||
continue
|
||||
result.setdefault(key, entry["value"])
|
||||
if plan["policy"]["name"] == "balanced":
|
||||
result.setdefault("REPIN", "64")
|
||||
ram = plan["tiers"]["ram"]
|
||||
@@ -364,5 +454,18 @@ def format_plan(plan):
|
||||
else:
|
||||
lines.append("VRAM no NVIDIA device detected · CPU path")
|
||||
lines.append(f"limit {plan['expected_bottleneck']}")
|
||||
hit = plan.get("projected_hit_rate", 0)
|
||||
lines.append(f"hit {hit:.0%} projected expert residency")
|
||||
tune = plan.get("tune", {})
|
||||
if tune:
|
||||
lines.append("")
|
||||
lines.append("auto-tune:")
|
||||
for key, entry in tune.items():
|
||||
if key.startswith("_"):
|
||||
continue
|
||||
lines.append(f" {key}={entry['value']:12s} {entry['reason']}")
|
||||
hint = tune.get("_numa_hint")
|
||||
if hint:
|
||||
lines.append(f" hint: {hint}")
|
||||
lines.extend(f"warn {warning}" for warning in plan["warnings"])
|
||||
return "\n".join(lines)
|
||||
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
/* sample.h — sampling (temperature + nucleus) and stop-set management.
|
||||
* Header-only: all functions are static — include from the main engine file. */
|
||||
#ifndef SAMPLE_H
|
||||
#define SAMPLE_H
|
||||
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "tok.h"
|
||||
|
||||
/* ---- RNG (xorshift64*) -------------------------------------------------- */
|
||||
static uint64_t g_rng = 0x9E3779B97F4A7C15ULL;
|
||||
static inline double rndu(void){
|
||||
g_rng ^= g_rng << 13; g_rng ^= g_rng >> 7; g_rng ^= g_rng << 17;
|
||||
return (double)(g_rng >> 11) * (1.0 / 9007199254740992.0);
|
||||
}
|
||||
|
||||
/* ---- argmax over a float vector ----------------------------------------- */
|
||||
static inline int argmax_v(const float *lo, int V){
|
||||
int b=-1; float bv=-INFINITY;
|
||||
for(int i=0;i<V;i++){ float x=lo[i]; if(x==x && x>bv){ bv=x; b=i; } }
|
||||
return b<0?0:b;
|
||||
}
|
||||
|
||||
/* ---- distribution buffers (reused, single-threaded decode) --------------- */
|
||||
static float *g_pbuf = NULL;
|
||||
static int *g_pidx = NULL;
|
||||
|
||||
/* sift-down on max-heap in h[0..n), key = g_pbuf[h[i]] (#335: partial top-p).
|
||||
* "hole" variant: carries the root value and deposits only at the end, so
|
||||
* heapify is O(V) and each pop is O(log n) without qsort on the full vocab. */
|
||||
static void topp_siftdown(int *h, int n, int i){
|
||||
int iv = h[i]; float kv = g_pbuf[iv];
|
||||
for (;;) {
|
||||
int l = 2*i + 1;
|
||||
if (l >= n) break;
|
||||
int b = l; if (l+1 < n && g_pbuf[h[l+1]] > g_pbuf[h[l]]) b = l+1;
|
||||
if (g_pbuf[h[b]] <= kv) break;
|
||||
h[i] = h[b]; i = b;
|
||||
}
|
||||
h[i] = iv;
|
||||
}
|
||||
|
||||
/* build the target distribution in g_pbuf: softmax(lo/temp) truncated to
|
||||
* top-p g_nuc. Invariant: g_pbuf stays indexed by token-id (never reordered);
|
||||
* the truncated tail is zeroed (dist_sample reads by id directly).
|
||||
* Requires: g_temp, g_nuc, falloc() — declared in the main engine file. */
|
||||
static void dist_build(const float *lo, int V){
|
||||
if (!g_pbuf) { g_pbuf = falloc(V); g_pidx = malloc(V * sizeof(int)); }
|
||||
int mxi = -1; float mx = 0;
|
||||
for (int i = 0; i < V; i++)
|
||||
if (isfinite(lo[i]) && (mxi < 0 || lo[i] > mx)) { mx = lo[i]; mxi = i; }
|
||||
double s = 0; float invt = 1.f / (g_temp > 1e-4f ? g_temp : 1e-4f);
|
||||
if (mxi >= 0) {
|
||||
for (int i = 0; i < V; i++) {
|
||||
g_pbuf[i] = isfinite(lo[i]) ? expf((lo[i] - mx) * invt) : 0.f;
|
||||
s += g_pbuf[i];
|
||||
}
|
||||
}
|
||||
if (mxi < 0 || !isfinite(s) || s <= 0.0) {
|
||||
static int warned = 0;
|
||||
if (!warned) { warned = 1; fprintf(stderr,
|
||||
"[SAMPLE] warning: non-finite logits (NaN/Inf) — falling back to argmax; "
|
||||
"output may be degraded. This usually means a numerical blow-up upstream.\n"); }
|
||||
int a = (mxi >= 0) ? mxi : 0;
|
||||
for (int i = 0; i < V; i++) g_pbuf[i] = 0.f;
|
||||
g_pbuf[a] = 1.f;
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < V; i++) g_pbuf[i] /= (float)s;
|
||||
if (g_nuc > 0 && g_nuc < 1.f) {
|
||||
for (int i = 0; i < V; i++) g_pidx[i] = i;
|
||||
for (int i = V/2-1; i >= 0; i--) topp_siftdown(g_pidx, V, i);
|
||||
double s2 = 0, cum = 0; int out = V;
|
||||
do {
|
||||
int root = g_pidx[0];
|
||||
g_pidx[0] = g_pidx[--out]; g_pidx[out] = root;
|
||||
s2 += g_pbuf[root]; cum += g_pbuf[root];
|
||||
if (out > 0) topp_siftdown(g_pidx, out, 0);
|
||||
} while (cum < g_nuc && out > 0);
|
||||
for (int i = 0; i < out; i++) g_pbuf[g_pidx[i]] = 0;
|
||||
float s2f = (float)s2;
|
||||
for (int i = out; i < V; i++) g_pbuf[g_pidx[i]] /= s2f;
|
||||
}
|
||||
}
|
||||
|
||||
/* sample from g_pbuf; ban>=0 excludes that token (renormalizing on the fly) */
|
||||
static int dist_sample(int V, int ban){
|
||||
double z = 1.0 - (ban >= 0 ? g_pbuf[ban] : 0.0);
|
||||
if (z <= 1e-12) z = 1e-12;
|
||||
double u = rndu() * z, cum = 0;
|
||||
for (int i = 0; i < V; i++) { if (i == ban) continue; cum += g_pbuf[i]; if (cum >= u) return i; }
|
||||
for (int i = V-1; i >= 0; i--) if (i != ban && g_pbuf[i] > 0) return i;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* next token from logits: greedy if g_temp<=0, sampling otherwise.
|
||||
* ban = token excluded because it was rejected by speculative verification. */
|
||||
static int pick_tok(const float *lo, int V, int ban){
|
||||
if (g_temp <= 0) return argmax_v(lo, V);
|
||||
dist_build(lo, V);
|
||||
return dist_sample(V, ban);
|
||||
}
|
||||
|
||||
/* ---- stop set ----------------------------------------------------------- */
|
||||
static int g_stop[64], g_nstop = 0;
|
||||
static inline int is_stop(int t){
|
||||
for (int i = 0; i < g_nstop; i++) if (t == g_stop[i]) return 1;
|
||||
return 0;
|
||||
}
|
||||
/* T=NULL -> config stops only (validation/oracle, where the tokenizer is not needed). */
|
||||
static void stops_arm_tok(const Cfg *c, int tok_eos, Tok *T){
|
||||
g_nstop = 0;
|
||||
for (int i = 0; i < c->n_stop && g_nstop < 64; i++) g_stop[g_nstop++] = c->stop_ids[i];
|
||||
if (tok_eos >= 0 && !is_stop(tok_eos) && g_nstop < 64) g_stop[g_nstop++] = tok_eos;
|
||||
int nsp = 0;
|
||||
if (T) for (int id = 0; id < T->n_ids && g_nstop < 64; id++)
|
||||
if (T->id_special[id] && !is_stop(id)) { g_stop[g_nstop++] = id; nsp++; }
|
||||
/* #401: in serve mode keep ONLY <|endoftext|>. Role markers <|user|>/<|observation|>
|
||||
* (config stops + tokenizer special set) are boundaries the Python server owns; as
|
||||
* hard stops they cut generation the moment the model opens a <tool_call> block,
|
||||
* because int4 argmax noise picks a stop-token ID over the correct '<' token. */
|
||||
if (getenv("SERVE") && tok_eos >= 0) {
|
||||
int kept = 0;
|
||||
for (int i = 0; i < g_nstop; i++) if (g_stop[i] == tok_eos) g_stop[kept++] = g_stop[i];
|
||||
if (kept < g_nstop) fprintf(stderr, "[stop] serve mode: filtered %d non-EOS stop tokens (tool-call safety, #401)\n", g_nstop - kept);
|
||||
g_nstop = kept; nsp = 0;
|
||||
}
|
||||
fprintf(stderr, "[stop] %d stop tokens:", g_nstop);
|
||||
for (int i = 0; i < g_nstop; i++) fprintf(stderr, " %d", g_stop[i]);
|
||||
if (nsp) fprintf(stderr, " (%d from the tokenizer's special set)", nsp);
|
||||
fprintf(stderr, "\n");
|
||||
}
|
||||
static void stops_arm(const Cfg *c, int tok_eos){ stops_arm_tok(c, tok_eos, NULL); }
|
||||
|
||||
/* ---- log-prob of a target token given the logit vector ------------------- */
|
||||
static double logprob_target(const float *lo, int V, int target, int *am){
|
||||
float mx = lo[0]; int best = 0;
|
||||
for (int i = 1; i < V; i++) if (lo[i] > mx) { mx = lo[i]; best = i; }
|
||||
double se = 0;
|
||||
for (int i = 0; i < V; i++) se += exp((double)lo[i] - mx);
|
||||
if (am) *am = (best == target);
|
||||
return (double)(lo[target] - mx) - log(se);
|
||||
}
|
||||
|
||||
/* "glm" in model_type, case-insensitive */
|
||||
static int mt_is_glm(const char *s){
|
||||
if (s) for (; *s; s++)
|
||||
if ((s[0]|32) == 'g' && (s[1]|32) == 'l' && (s[2]|32) == 'm') return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif /* SAMPLE_H */
|
||||
+2
-2
@@ -32,11 +32,11 @@ esac
|
||||
|
||||
# 2) build: nativa (veloce, per QUESTA macchina). Per un binario da distribuire: make portable
|
||||
echo " building (ARCH=${ARCH:-native})…"
|
||||
make -s glm ARCH="${ARCH:-native}"
|
||||
make -s colibri ARCH="${ARCH:-native}"
|
||||
|
||||
# 3) self-test sull'oracolo tiny, se presente
|
||||
if [ -d glm_tiny ] && [ -f ref_glm.json ]; then
|
||||
r=$(SNAP=./glm_tiny TF=1 ./glm 64 16 16 2>/dev/null | grep -oE "[0-9]+/[0-9]+ positions" || true)
|
||||
r=$(SNAP=./glm_tiny TF=1 ./colibri 64 16 16 2>/dev/null | grep -oE "[0-9]+/[0-9]+ positions" || true)
|
||||
echo " engine self-test: ${r:-?} (expected 32/32)"
|
||||
fi
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
#include <fcntl.h>
|
||||
#include <errno.h>
|
||||
#include <unistd.h>
|
||||
#include <dirent.h>
|
||||
#include <sys/stat.h>
|
||||
@@ -104,6 +105,38 @@ static int st_direct_fd(shards *S, int fd) {
|
||||
}
|
||||
|
||||
/* indicizza tutti i model-*.safetensors in snap_dir */
|
||||
/* pread completo: chunk-loop (una singola pread si ferma a ~2^31 byte su Linux
|
||||
* — i tensori bf16 grandi la superano), riprova su EINTR e riporta un errore
|
||||
* ONESTO: perror stampava "Success" su una short-read (errno resta 0), lo
|
||||
* stesso sintomo corretto in glm.c per #236. ST_PREAD_CHUNK e' sovrascrivibile
|
||||
* per i test. EN: full pread — chunk loop (one pread caps at ~2^31 bytes and
|
||||
* big bf16 tensors exceed it), EINTR retry, honest short-read errors.
|
||||
* Exits on failure, like every st.h reader. */
|
||||
#ifndef ST_PREAD_CHUNK
|
||||
#define ST_PREAD_CHUNK (1u << 30)
|
||||
#endif
|
||||
static void st_pread_full(int fd, void *buf, int64_t n, int64_t off, const char *tag) {
|
||||
char *p = (char *)buf;
|
||||
int64_t got = 0;
|
||||
while (got < n) {
|
||||
int64_t want = n - got;
|
||||
if (want > (int64_t)ST_PREAD_CHUNK) want = ST_PREAD_CHUNK;
|
||||
ssize_t r = pread(fd, p + got, (size_t)want, off + got);
|
||||
if (r < 0) {
|
||||
if (errno == EINTR) continue;
|
||||
fprintf(stderr, "%s: %s (off %lld, %lld/%lld bytes)\n", tag, strerror(errno),
|
||||
(long long)off, (long long)got, (long long)n);
|
||||
exit(1);
|
||||
}
|
||||
if (r == 0) {
|
||||
fprintf(stderr, "%s: short read at EOF (off %lld, %lld/%lld bytes) — truncated file?\n",
|
||||
tag, (long long)off, (long long)got, (long long)n);
|
||||
exit(1);
|
||||
}
|
||||
got += r;
|
||||
}
|
||||
}
|
||||
|
||||
static void st_init(shards *S, const char *snap_dir) {
|
||||
memset(S, 0, sizeof(*S));
|
||||
S->cap = 4096; S->t = calloc(S->cap, sizeof(st_tensor));
|
||||
@@ -128,7 +161,7 @@ static void st_init(shards *S, const char *snap_dir) {
|
||||
if (fstat(fd, &sst) != 0) { perror("fstat shard"); exit(1); }
|
||||
int64_t fsz = (int64_t)sst.st_size;
|
||||
uint64_t hlen;
|
||||
if (pread(fd, &hlen, 8, 0) != 8) { perror("pread hlen"); exit(1); }
|
||||
st_pread_full(fd, &hlen, 8, 0, "pread hlen");
|
||||
/* file malevolo/troncato: hlen deve stare nel file dopo gli 8 byte di
|
||||
* prefisso e sotto il tetto. Senza questo bound hlen+1 puo' andare in
|
||||
* overflow (malloc(0) e poi hdr[hlen]=0 fuori limiti) o forzare una
|
||||
@@ -138,7 +171,7 @@ static void st_init(shards *S, const char *snap_dir) {
|
||||
files[fi], (unsigned long long)hlen, (long long)fsz); exit(1); }
|
||||
char *hdr = malloc(hlen + 1);
|
||||
if (!hdr) { perror("malloc safetensors header"); exit(1); }
|
||||
if (pread(fd, hdr, hlen, 8) != (ssize_t)hlen) { perror("pread hdr"); exit(1); }
|
||||
st_pread_full(fd, hdr, (int64_t)hlen, 8, "pread hdr");
|
||||
hdr[hlen] = 0;
|
||||
int64_t data_start = 8 + (int64_t)hlen;
|
||||
char *arena = NULL;
|
||||
@@ -167,7 +200,19 @@ static void st_init(shards *S, const char *snap_dir) {
|
||||
if (a0 < 0 || b0 < a0 || data_start + b0 > fsz) {
|
||||
fprintf(stderr, "%s: tensor '%s' data_offsets [%lld,%lld] out of file bounds (%lld)\n",
|
||||
files[fi], name, (long long)a0, (long long)b0, (long long)fsz); exit(1); }
|
||||
int64_t numel = 1; for (int k = 0; k < shp->len; k++) numel *= (int64_t)shp->kids[k]->num;
|
||||
/* SEC: lo shape viene da un file non fidato (mirror). Senza il guard
|
||||
* di overflow, uno shape tipo [65535,65535,65535,...] fa avvolgere
|
||||
* numel a un valore piccolo/negativo che poi passerebbe il cross-check
|
||||
* numel*esz==nbytes in st_read_f32, riaprendo l'OOB. */
|
||||
int64_t numel = 1; int bad_shape = 0;
|
||||
for (int k = 0; k < shp->len; k++) {
|
||||
int64_t d = (int64_t)shp->kids[k]->num;
|
||||
if (d < 0 || (d != 0 && numel > INT64_MAX / d)) { bad_shape = 1; break; }
|
||||
numel *= d;
|
||||
}
|
||||
if (bad_shape) {
|
||||
fprintf(stderr, "%s: tensor '%s' shape overflows int64 — refusing (hostile or corrupt file)\n",
|
||||
files[fi], name); exit(1); }
|
||||
if (S->n == S->cap) { S->cap *= 2; S->t = realloc(S->t, S->cap*sizeof(st_tensor)); }
|
||||
st_tensor *t = &S->t[S->n++];
|
||||
t->name = strdup(name); t->fd = fd; t->off = data_start + a0;
|
||||
@@ -216,9 +261,18 @@ static void st_prefetch(shards *S, const char *name) {
|
||||
static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) {
|
||||
st_tensor *t = st_find(S, name);
|
||||
if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); }
|
||||
/* SEC: numel viene dallo shape, nbytes dagli offset — due campi indipendenti
|
||||
* del file. Se non concordano, la memcpy F32 (nbytes) o i loop BF16/F16
|
||||
* (numel elementi da un raw di soli nbytes) sforano il buffer del chiamante,
|
||||
* che e' dimensionato sul config, non sul file. Il chiamante che alloca su
|
||||
* st_numel resta coerente; questo blocca l'ingresso ostile a monte. */
|
||||
int esz = (t->dtype == 2) ? 4 : 2;
|
||||
if (t->numel < 0 || t->numel > t->nbytes / esz || t->numel * (int64_t)esz != t->nbytes) {
|
||||
fprintf(stderr, "%s: tensor '%s' shape/bytes mismatch (numel %lld, %lld bytes, dtype %d) — refusing (hostile or corrupt file)\n",
|
||||
name, name, (long long)t->numel, (long long)t->nbytes, t->dtype); exit(1); }
|
||||
void *raw = malloc(t->nbytes);
|
||||
if (!raw) { fprintf(stderr, "malloc %lld bytes for tensor %s failed\n", (long long)t->nbytes, name); exit(1); }
|
||||
if (pread(t->fd, raw, t->nbytes, t->off) != t->nbytes) { perror("pread data"); exit(1); }
|
||||
st_pread_full(t->fd, raw, t->nbytes, t->off, "pread data");
|
||||
if (t->dtype == 2) {
|
||||
memcpy(out, raw, t->nbytes);
|
||||
} else if (t->dtype == 0) {
|
||||
@@ -243,7 +297,7 @@ static int64_t st_nbytes(shards *S, const char *name) {
|
||||
static void st_read_raw(shards *S, const char *name, void *out, int drop) {
|
||||
st_tensor *t = st_find(S, name);
|
||||
if (!t) { fprintf(stderr, "missing tensor: %s\n", name); exit(1); }
|
||||
if (pread(t->fd, out, t->nbytes, t->off) != t->nbytes) { perror("pread raw"); exit(1); }
|
||||
st_pread_full(t->fd, out, t->nbytes, t->off, "pread raw");
|
||||
if (drop) posix_fadvise(t->fd, t->off, t->nbytes, POSIX_FADV_DONTNEED);
|
||||
}
|
||||
|
||||
@@ -256,7 +310,7 @@ static void st_read_slice_f32(shards *S, const char *name, int64_t elem_off, int
|
||||
int esz = (t->dtype == 2) ? 4 : 2;
|
||||
int64_t boff = t->off + elem_off * esz, nb = n_elems * esz;
|
||||
void *raw = malloc(nb);
|
||||
if (pread(t->fd, raw, nb, boff) != nb) { perror("pread slice"); exit(1); }
|
||||
st_pread_full(t->fd, raw, nb, boff, "pread slice");
|
||||
if (t->dtype == 2) memcpy(out, raw, nb);
|
||||
else if (t->dtype == 0) { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = bf16_to_f32(p[i]); }
|
||||
else { uint16_t *p = raw; for (int64_t i = 0; i < n_elems; i++) out[i] = f16_to_f32(p[i]); }
|
||||
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
/* telemetry.h — dashboard protocol lines, stats/usage persistence, hardware probe.
|
||||
* Include after Model/Cfg/QT/ESlot/shards and st.h are defined; requires
|
||||
* qt_bytes(), now_s(), rss_gb(), edisk_s(), and the g_cuda_* globals (ifdef). */
|
||||
#ifndef TELEMETRY_H
|
||||
#define TELEMETRY_H
|
||||
|
||||
static int64_t tbytes(int O,int I,int bits){
|
||||
if(bits>=16) return (int64_t)O*I*4;
|
||||
if(bits>=5) return (int64_t)O*I + (int64_t)O*4;
|
||||
return (int64_t)O*((I+1)/2) + (int64_t)O*4;
|
||||
}
|
||||
|
||||
static int64_t expert_bytes_probe(Model *m, int ebits){
|
||||
Cfg *c=&m->c; int64_t eb=0; char nm[256];
|
||||
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.gate_proj.weight",c->first_dense);
|
||||
if(st_nbytes(&m->S,nm)>0){
|
||||
const char *suf[3]={"gate_proj","up_proj","down_proj"};
|
||||
for(int k=0;k<3;k++){
|
||||
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight",c->first_dense,suf[k]);
|
||||
eb+=st_nbytes(&m->S,nm);
|
||||
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight.qs",c->first_dense,suf[k]);
|
||||
int64_t q=st_nbytes(&m->S,nm); if(q>0) eb+=q;
|
||||
}
|
||||
}
|
||||
if(eb<=0) eb = tbytes(c->moe_inter,c->hidden,ebits)*2 + tbytes(c->hidden,c->moe_inter,ebits);
|
||||
return eb;
|
||||
}
|
||||
|
||||
/* BRAIN MAP: per-turn expert hit bitmap for the dashboard. */
|
||||
static uint8_t **g_ehit;
|
||||
static void ehit_mark(Model *m, int layer, int eid){
|
||||
if(!g_ehit){ Cfg *c=&m->c;
|
||||
g_ehit=calloc(c->n_layers+1,sizeof(uint8_t*));
|
||||
for(int i=0;i<=c->n_layers;i++) g_ehit[i]=calloc(c->n_experts,1);
|
||||
}
|
||||
g_ehit[layer][eid]=1;
|
||||
}
|
||||
|
||||
/* CPU model + cores + RAM (GB); empty/zero where unavailable. */
|
||||
static void hw_probe(char *cpu, size_t cn, int *cores, double *ram_total, double *ram_avail){
|
||||
cpu[0]=0;
|
||||
#ifdef _WIN32
|
||||
#if defined(__x86_64__) || defined(__i386__)
|
||||
{ unsigned int r[12]={0}; unsigned int *w=r;
|
||||
for(unsigned int f=0x80000002u; f<=0x80000004u; f++,w+=4)
|
||||
__get_cpuid(f,&w[0],&w[1],&w[2],&w[3]);
|
||||
char *b=(char*)r; b[47]=0; while(*b==' ')b++;
|
||||
snprintf(cpu,cn,"%s",b); }
|
||||
#endif
|
||||
#else
|
||||
FILE *ci=fopen("/proc/cpuinfo","r");
|
||||
if(ci){ char ln[256];
|
||||
while(fgets(ln,sizeof(ln),ci)) if(!strncmp(ln,"model name",10)){
|
||||
char *p=strchr(ln,':'); if(p){ p++; while(*p==' ')p++;
|
||||
int n=(int)strlen(p); if(n>0&&p[n-1]=='\n')p[--n]=0;
|
||||
snprintf(cpu,cn,"%s",p); } break; }
|
||||
fclose(ci); }
|
||||
#endif
|
||||
*cores=0;
|
||||
#ifdef _WIN32
|
||||
{ SYSTEM_INFO si; GetSystemInfo(&si); *cores=(int)si.dwNumberOfProcessors; }
|
||||
#elif defined(_SC_NPROCESSORS_ONLN)
|
||||
*cores=(int)sysconf(_SC_NPROCESSORS_ONLN);
|
||||
#endif
|
||||
*ram_total=*ram_avail=0;
|
||||
#ifdef _WIN32
|
||||
compat_meminfo(ram_total,ram_avail);
|
||||
#else
|
||||
FILE *mi=fopen("/proc/meminfo","r");
|
||||
if(mi){ char ln[256]; double mt=0,ma=0;
|
||||
while(fgets(ln,sizeof(ln),mi)){
|
||||
if(sscanf(ln,"MemTotal: %lf",&mt)==1) *ram_total=mt/1e6;
|
||||
if(sscanf(ln,"MemAvailable: %lf",&ma)==1) *ram_avail=ma/1e6;
|
||||
} fclose(mi); }
|
||||
#endif
|
||||
}
|
||||
|
||||
static void hwinfo_emit(Model *m){
|
||||
Cfg *c=&m->c; (void)c;
|
||||
char cpu[256]; int cores; double ram_total,ram_avail;
|
||||
hw_probe(cpu,sizeof(cpu),&cores,&ram_total,&ram_avail);
|
||||
int ngpu=0; double vram_total=0;
|
||||
char gpu_name[128]="";
|
||||
#ifdef COLI_CUDA
|
||||
ngpu=g_cuda_ndev; vram_total=m->gpu_expert_bytes/1e9;
|
||||
for(int i=0;i<g_cuda_ndev;i++){
|
||||
size_t fr=0,to=0; coli_cuda_mem_info(g_cuda_devices[i],&fr,&to);
|
||||
if(!i) vram_total=(double)to*g_cuda_ndev/1e9;
|
||||
}
|
||||
if(g_cuda_ndev>0)
|
||||
snprintf(gpu_name,sizeof(gpu_name),"CUDA device x%d",g_cuda_ndev);
|
||||
#endif
|
||||
printf("HWINFO %d %.1f %.1f %d %.1f %s|%s\n",
|
||||
cores,ram_total,ram_avail,ngpu,vram_total,cpu,gpu_name);
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
static void tiers_emit(Model *m){
|
||||
Cfg *c=&m->c; int nsp=0;
|
||||
for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse) nsp++;
|
||||
int total=(nsp+(m->has_mtp?1:0))*c->n_experts;
|
||||
int pinned=0,lru=0;
|
||||
for(int i=0;i<=c->n_layers;i++){ pinned+=m->npin?m->npin[i]:0; lru+=m->ecn?m->ecn[i]:0; }
|
||||
int vram=0; double vram_gb=0;
|
||||
#ifdef COLI_CUDA
|
||||
vram=m->gpu_expert_count; vram_gb=m->gpu_expert_bytes/1e9;
|
||||
#endif
|
||||
int ram=pinned-vram+lru; if(ram<0) ram=0;
|
||||
int disk=total-vram-ram; if(disk<0) disk=0;
|
||||
double eb=(double)expert_bytes_probe(m,m->ebits);
|
||||
printf("TIERS %d %d %d %.2f %.2f\n",vram,ram,disk,vram_gb,ram*eb/1e9);
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
static void emap_emit(Model *m){
|
||||
Cfg *c=&m->c;
|
||||
int rows=0;
|
||||
for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse) rows++;
|
||||
int has_mtp = m->has_mtp && m->eusage[c->n_layers];
|
||||
if(has_mtp) rows++;
|
||||
int cols=c->n_experts;
|
||||
char *hex=malloc((size_t)rows*cols*2+1); int w=0;
|
||||
for(int i=0;i<=c->n_layers;i++){
|
||||
int is_row = (i<c->n_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp);
|
||||
if(!is_row) continue;
|
||||
for(int e=0;e<cols;e++){
|
||||
int tier=0;
|
||||
ESlot *P=m->pin[i];
|
||||
for(int z=0;z<m->npin[i];z++) if(P[z].eid==e){
|
||||
#ifdef COLI_CUDA
|
||||
tier = P[z].g.cuda?2:1;
|
||||
#else
|
||||
tier = 1;
|
||||
#endif
|
||||
break; }
|
||||
if(!tier && m->ecache && m->ecache[i])
|
||||
for(int z=0;z<m->ecn[i];z++) if(m->ecache[i][z].eid==e){ tier=1; break; }
|
||||
uint32_t u = m->eusage[i]?m->eusage[i][e]:0;
|
||||
int heat=0; while(u){ heat++; u>>=1; } if(heat>63) heat=63;
|
||||
int b=(tier<<6)|heat;
|
||||
hex[w++]="0123456789abcdef"[b>>4]; hex[w++]="0123456789abcdef"[b&15];
|
||||
}
|
||||
}
|
||||
hex[w]=0;
|
||||
printf("EMAP %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex);
|
||||
}
|
||||
|
||||
static void hits_emit(Model *m){
|
||||
Cfg *c=&m->c; if(!g_ehit) return;
|
||||
int rows=0;
|
||||
for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse) rows++;
|
||||
int has_mtp = m->has_mtp && m->eusage[c->n_layers];
|
||||
if(has_mtp) rows++;
|
||||
int cols=c->n_experts, nb=(rows*cols+7)/8;
|
||||
uint8_t *bm=calloc(nb,1); int bit=0;
|
||||
for(int i=0;i<=c->n_layers;i++){
|
||||
int is_row = (i<c->n_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp);
|
||||
if(!is_row) continue;
|
||||
for(int e=0;e<cols;e++,bit++)
|
||||
if(g_ehit[i][e]){ bm[bit>>3]|=1<<(bit&7); g_ehit[i][e]=0; }
|
||||
}
|
||||
char *hex=malloc((size_t)nb*2+1); int w=0;
|
||||
for(int b=0;b<nb;b++){ hex[w++]="0123456789abcdef"[bm[b]>>4]; hex[w++]="0123456789abcdef"[bm[b]&15]; }
|
||||
hex[w]=0;
|
||||
printf("HITS %d %d %s\n",rows,cols,hex); fflush(stdout); free(hex); free(bm);
|
||||
}
|
||||
|
||||
static void stats_dump_q(Model *m, const char *path, int quiet){
|
||||
char tmp[2100]; snprintf(tmp,sizeof(tmp),"%s.tmp",path);
|
||||
FILE *f=fopen(tmp,"w"); if(!f){ if(!quiet) perror(tmp); return; }
|
||||
Cfg *c=&m->c; int64_t tot=0, nz=0;
|
||||
for(int i=0;i<=c->n_layers;i++){ if(!m->eusage[i]) continue;
|
||||
for(int e=0;e<c->n_experts;e++) if(m->eusage[i][e]){ fprintf(f,"%d %d %u\n",i,e,m->eusage[i][e]); tot+=m->eusage[i][e]; nz++; } }
|
||||
fclose(f); rename(tmp,path);
|
||||
if(!quiet) fprintf(stderr,"[STATS] %lld selections across %lld distinct experts -> %s\n",(long long)tot,(long long)nz,path);
|
||||
}
|
||||
static void stats_dump(Model *m, const char *path){ stats_dump_q(m,path,0); }
|
||||
|
||||
static char g_usage_path[2100]="";
|
||||
static int64_t usage_load(Model *m, const char *path){
|
||||
FILE *f=fopen(path,"r"); if(!f) return 0;
|
||||
Cfg *c=&m->c; int l,e; uint32_t cnt; int64_t tot=0;
|
||||
while(fscanf(f,"%d %d %u",&l,&e,&cnt)==3)
|
||||
if(l>=0&&l<=c->n_layers&&e>=0&&e<c->n_experts&&m->eusage[l]){ m->eusage[l][e]+=cnt; tot+=cnt; }
|
||||
fclose(f); return tot;
|
||||
}
|
||||
static void usage_save(Model *m){ if(g_usage_path[0]) stats_dump_q(m,g_usage_path,1); }
|
||||
|
||||
#endif /* TELEMETRY_H */
|
||||
@@ -0,0 +1,165 @@
|
||||
/* Microbenchmark: old (full-qsort) vs new (quickselect partial-select) DSA top-keep.
|
||||
*
|
||||
* This is NOT a unit test -- test_dsa_select.c proves correctness. This measures the
|
||||
* headline claim of #356: that replacing the O(nk log nk) qsort over all nk context
|
||||
* scores with an O(nk) partial_select_desc is materially faster per call, which is
|
||||
* the win the issue was opened for -- and that the win GROWS with context length
|
||||
* (because quickselect is linear average, qsort is n-log-n).
|
||||
*
|
||||
* It re-implements the OLD top-keep inline (qsort + threshold + scans) on a private
|
||||
* buffer so the A/B runs in one process, same inputs, same warm caches -- a controlled
|
||||
* comparison. It calls the REAL (new) partial_select_desc via the include-glm.c
|
||||
* pattern, replicating the production threshold derivation + scans.
|
||||
*
|
||||
* Methodology (chosen to be honest, not to flatter the change):
|
||||
* - keep = 2048 (the real GLM-5.2 index_topk), nk swept across context lengths from
|
||||
* the 2049 activation boundary up to 65536 (a long conversation).
|
||||
* - Three score shapes: (a) realistic peaked -- a few hot keys, long tail, the shape
|
||||
* real DSA attention scores take; (b) uniform random -- no structure; (c) a plateau
|
||||
* of ties, to exercise the boundary-membership path.
|
||||
* - Each (shape, nk) is timed over N_REPEAT=2000 iterations, with the scores frozen
|
||||
* so both algorithms do IDENTICAL work. We report median ns/call and the new/old
|
||||
* ratio. A warmup pass primes caches before timing.
|
||||
*
|
||||
* Run: make tests/bench_dsa_select && ./tests/bench_dsa_select (not in TEST_BINS)
|
||||
*/
|
||||
#define main coli_glm_main_unused
|
||||
#include "../colibri.c"
|
||||
#undef main
|
||||
|
||||
#include <math.h>
|
||||
#include <stdint.h>
|
||||
|
||||
/* ---- the OLD algorithm, verbatim from dev before #356, on a private buffer ---- */
|
||||
static int cmp_pdesc_old(const void *a, const void *b){
|
||||
float x=*(const float*)a, y=*(const float*)b; return x<y?1:x>y?-1:0; }
|
||||
static void keep_old(const float *isc, int nk, int keep, int *dst, int *nd_out){
|
||||
float *tmp=malloc((size_t)nk*sizeof(float));
|
||||
memcpy(tmp,isc,(size_t)nk*sizeof(float));
|
||||
qsort(tmp,(size_t)nk,sizeof(float),cmp_pdesc_old);
|
||||
float thr=tmp[keep-1]; int nd=0;
|
||||
for(int t=0;t<nk && nd<keep;t++) if(isc[t]>thr) dst[nd++]=t;
|
||||
for(int t=0;t<nk && nd<keep;t++) if(isc[t]==thr) dst[nd++]=t;
|
||||
free(tmp); *nd_out=nd;
|
||||
}
|
||||
|
||||
/* ---- timing: median of N_REPEAT runs in ns/call, sorted ascending ---- */
|
||||
#define N_REPEAT 2000
|
||||
static double bench_ns(void (*fn)(const float*,int,int,int*,int*),
|
||||
const float *isc, int nk, int keep){
|
||||
static double ts[N_REPEAT]; int *dst=malloc((size_t)nk*sizeof(int)); int nd;
|
||||
for(int r=0;r<N_REPEAT;r++){
|
||||
double t0=now_s();
|
||||
fn(isc,nk,keep,dst,&nd);
|
||||
ts[r]=(now_s()-t0)*1e9;
|
||||
}
|
||||
for(int a=1;a<N_REPEAT;a++){ double k=ts[a]; int b=a-1;
|
||||
while(b>=0 && ts[b]>k){ ts[b+1]=ts[b]; b--; } ts[b+1]=k; }
|
||||
free(dst);
|
||||
return ts[N_REPEAT/2];
|
||||
}
|
||||
|
||||
/* Sort an array of doubles ascending (median-of-medians aggregation below). */
|
||||
static void dsort(double *a, int n){
|
||||
for(int s=1;s<n;s++){ double k=a[s]; int b=s-1;
|
||||
while(b>=0 && a[b]>k){ a[b+1]=a[b]; b--; } a[b+1]=k; }
|
||||
}
|
||||
|
||||
/* the NEW algorithm calls the real partial_select_desc + replicates the production
|
||||
* threshold derivation and position scans. */
|
||||
static void keep_new(const float *isc, int nk, int keep, int *dst, int *nd_out){
|
||||
float *tmp=malloc((size_t)nk*sizeof(float));
|
||||
memcpy(tmp,isc,(size_t)nk*sizeof(float));
|
||||
partial_select_desc(tmp,nk,keep);
|
||||
float thr=tmp[0]; for(int t=1;t<keep;t++) if(tmp[t]<thr) thr=tmp[t];
|
||||
int nd=0;
|
||||
for(int t=0;t<nk && nd<keep;t++) if(isc[t]>thr) dst[nd++]=t;
|
||||
for(int t=0;t<nk && nd<keep;t++) if(isc[t]==thr) dst[nd++]=t;
|
||||
free(tmp); *nd_out=nd;
|
||||
}
|
||||
|
||||
/* deterministic score fill for three shapes */
|
||||
static uint32_t brng = 0xA5A5A5A5u;
|
||||
static void brng_seed(uint32_t s){ brng = s; }
|
||||
static double brand(void){ brng ^= brng << 13; brng ^= brng >> 17; brng ^= brng << 5;
|
||||
return (double)(brng >> 8) * (1.0 / 16777216.0); }
|
||||
static void fill_realistic(float *isc, int nk){ /* few hot, long distinct tail */
|
||||
for(int i=0;i<nk;i++) isc[i]=(float)(-1.0 - brand()*4.0);
|
||||
isc[0]=3.f; isc[nk/50<nk?nk/50:nk-1]=1.f; isc[nk/200<nk?nk/200:nk-1]=0.5f;
|
||||
}
|
||||
static void fill_uniform(float *isc, int nk){ /* no structure */
|
||||
for(int i=0;i<nk;i++) isc[i]=(float)(brand()*1000.0);
|
||||
}
|
||||
static void fill_plateau(float *isc, int nk){ /* tie blocks -> boundary path */
|
||||
for(int i=0;i<nk;i++) isc[i]=(float)(-(double)(i/7));
|
||||
}
|
||||
|
||||
/* Multi-seed aggregation. Per @KingIcyCreamProjects (#357 thread): with a single frozen
|
||||
* input per cell, quickselect's deterministic median-of-three pivot means one lucky input
|
||||
* can spike a single nk row (an observed ~75x at nk=8192 on a 9950X3D that was really a
|
||||
* ~13-40x algorithm). Two bugs compounded it: (1) the old bench drew ONE input per cell;
|
||||
* (2) brng was never reset, so each cell's input depended on every prior cell's draws --
|
||||
* reordering nks[] silently shifted all later inputs. Both fixed here: brng is reseeded
|
||||
* per (shape, nk, seed), and we take the MEDIAN of N_SEEDS independent inputs, each itself
|
||||
* a median over N_REPEAT timing reps. A lucky pivot now moves one of the N_SEEDS samples,
|
||||
* not the reported number. */
|
||||
|
||||
int main(void){
|
||||
int keep = 2048; /* GLM-5.2 index_topk */
|
||||
int nks[] = {2049, 4096, 8192, 16384, 32768, 65536};
|
||||
const int N_SEEDS = 11; /* odd so the median is a real sample, not interpolated */
|
||||
float *isc = malloc((size_t)65536*sizeof(float));
|
||||
int *dst = malloc((size_t)65536*sizeof(int)); int nd;
|
||||
double *old_seeds = malloc((size_t)N_SEEDS*sizeof(double));
|
||||
double *new_seeds = malloc((size_t)N_SEEDS*sizeof(double));
|
||||
|
||||
struct { const char *name; void (*fill)(float*,int); } shapes[] = {
|
||||
{ "realistic", fill_realistic },
|
||||
{ "uniform", fill_uniform },
|
||||
{ "plateau", fill_plateau },
|
||||
};
|
||||
|
||||
printf("bench_dsa_select: DSA top-keep, old (qsort) vs new (partial-select) keep=%d (median of %d seeds x %d reps)\n",
|
||||
keep, N_SEEDS, N_REPEAT);
|
||||
printf("%-12s %7s %14s %14s %9s\n", "shape", "nk", "old ns/call", "new ns/call", "speedup");
|
||||
printf("------------------------------------------------------------------------\n");
|
||||
|
||||
for(size_t sh=0; sh<sizeof(shapes)/sizeof(shapes[0]); sh++){
|
||||
for(size_t ni=0; ni<sizeof(nks)/sizeof(nks[0]); ni++){
|
||||
int nk=nks[ni];
|
||||
int bad = 0;
|
||||
for(int sd=0; sd<N_SEEDS; sd++){
|
||||
/* reseed per (shape,nk,seed) so each cell's input is reproducible and
|
||||
* independent of cell ordering, and so lucky pivots are sampled, not fixed. */
|
||||
brng_seed(0xA5A5A5A5u + (uint32_t)(sd*0x9E3779B9u));
|
||||
shapes[sh].fill(isc,nk);
|
||||
/* warmup both paths so caches/branch predictors are primed */
|
||||
for(int w=0; w<50; w++){ keep_old(isc,nk,keep,dst,&nd); keep_new(isc,nk,keep,dst,&nd); }
|
||||
/* sanity: both must keep exactly `keep` (correctness is test_dsa_select's
|
||||
* job, but a count divergence here would make the timing meaningless) */
|
||||
keep_old(isc,nk,keep,dst,&nd); int na=nd;
|
||||
keep_new(isc,nk,keep,dst,&nd); int nb=nd;
|
||||
if(na!=keep || nb!=keep){ bad++; continue; }
|
||||
old_seeds[sd] = bench_ns(keep_old,isc,nk,keep);
|
||||
new_seeds[sd] = bench_ns(keep_new,isc,nk,keep);
|
||||
}
|
||||
if(bad == N_SEEDS){
|
||||
printf("%-12s %7d (BAD COUNTS on all %d seeds, skipped)\n",
|
||||
shapes[sh].name, nk, bad);
|
||||
continue;
|
||||
}
|
||||
/* report median-of-seed-medians: robust to a single lucky/unlucky pivot */
|
||||
dsort(old_seeds, N_SEEDS);
|
||||
dsort(new_seeds, N_SEEDS);
|
||||
double t_old = old_seeds[N_SEEDS/2];
|
||||
double t_new = new_seeds[N_SEEDS/2];
|
||||
printf("%-12s %7d %14.0f %14.0f %8.2fx\n",
|
||||
shapes[sh].name, nk, t_old, t_new, t_old/t_new);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
printf("bench_dsa_select: done (lower ns is better; speedup = old/new)\n");
|
||||
free(isc); free(dst); free(old_seeds); free(new_seeds);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
/* Microbenchmark: old (full-vocab qsort) vs new (heap partial-select) top-p truncation.
|
||||
*
|
||||
* This is NOT a unit test -- test_topp.c proves correctness. This measures the headline
|
||||
* claim of #335: that replacing the O(V log V) qsort over all 151936 vocab entries with
|
||||
* an O(V) heapify + k*log-V pops is materially faster per call, which is the win the issue
|
||||
* was opened for.
|
||||
*
|
||||
* It re-implements the OLD dist_build inline (qsort + scan) on a private buffer so the A/B
|
||||
* runs in one process, same inputs, same warm caches -- a controlled comparison. It calls
|
||||
* the REAL (new) dist_build via the include-glm.c pattern on the global g_pbuf.
|
||||
*
|
||||
* Methodology (chosen to be honest, not to flatter the change):
|
||||
* - V = 151936 (the actual GLM-5.2 vocab), g_nuc swept across the values that matter
|
||||
* for serving: 0.5 / 0.9 (serve default) / 0.95 / 0.99.
|
||||
* - Three logit shapes: (a) realistic peaked -- one hot token, long exponential tail,
|
||||
* the shape real language-model logits take; (b) uniform -- worst case for the heap,
|
||||
* maximum pop count; (c) a plateau of ties, to exercise the tie path.
|
||||
* - Each (shape, nuc) is timed over N_REPEAT=2000 iterations, with the RNG/logits frozen
|
||||
* so both algorithms do IDENTICAL work. We report median ns/call and the new/old ratio.
|
||||
* - A warmup pass primes caches before timing.
|
||||
*
|
||||
* Run: make tests/bench_topp && ./tests/bench_topp (not in TEST_BINS -- not a gate)
|
||||
*/
|
||||
#define main coli_glm_main_unused
|
||||
#include "../colibri.c"
|
||||
#undef main
|
||||
|
||||
#include <math.h>
|
||||
#include <stdint.h>
|
||||
|
||||
/* ---- the OLD algorithm, verbatim from dev before #354, on a private buffer ---- */
|
||||
static float *s_pbuf; static int *s_pidx; static double *s_ref;
|
||||
static int cmp_pdesc_old(const void *a, const void *b){
|
||||
double pa = s_ref[*(const int*)a], pb = s_ref[*(const int*)b];
|
||||
return pa < pb ? 1 : pa > pb ? -1 : 0; }
|
||||
static void dist_build_old(const float *lo, int V, double temp, double nuc){
|
||||
double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i];
|
||||
double s = 0, invt = 1.0 / (temp > 1e-4 ? temp : 1e-4);
|
||||
for (int i = 0; i < V; i++){ s_ref[i] = exp((lo[i]-mx)*invt); s += s_ref[i]; }
|
||||
for (int i = 0; i < V; i++) s_ref[i] /= s;
|
||||
if (nuc > 0 && nuc < 1.0){
|
||||
for (int i = 0; i < V; i++) s_pidx[i] = i;
|
||||
qsort(s_pidx, V, sizeof(int), cmp_pdesc_old);
|
||||
double cum = 0; int keep = V;
|
||||
for (int i = 0; i < V; i++){ cum += s_ref[s_pidx[i]]; if (cum >= nuc){ keep = i+1; break; } }
|
||||
double s2 = 0;
|
||||
for (int i = keep; i < V; i++) s_ref[s_pidx[i]] = 0;
|
||||
for (int i = 0; i < keep; i++) s2 += s_ref[s_pidx[i]];
|
||||
for (int i = 0; i < keep; i++) s_ref[s_pidx[i]] /= s2;
|
||||
}
|
||||
(void)s_pbuf;
|
||||
}
|
||||
|
||||
/* ---- timing: median of N_REPEAT runs in ns/call, sorted ascending ---- */
|
||||
#define N_REPEAT 2000
|
||||
static double bench_ns(void (*fn)(const float*,int,double,double),
|
||||
const float *lo, int V, double temp, double nuc){
|
||||
static double ts[N_REPEAT];
|
||||
for (int r = 0; r < N_REPEAT; r++){
|
||||
double t0 = now_s();
|
||||
fn(lo, V, temp, nuc);
|
||||
ts[r] = (now_s() - t0) * 1e9;
|
||||
}
|
||||
/* insertion sort the N_REPEAT samples (small), take median */
|
||||
for (int a = 1; a < N_REPEAT; a++){ double k = ts[a]; int b = a-1;
|
||||
while (b >= 0 && ts[b] > k){ ts[b+1] = ts[b]; b--; } ts[b+1] = k; }
|
||||
return ts[N_REPEAT/2];
|
||||
}
|
||||
|
||||
/* the NEW algorithm is the real dist_build, but it writes g_pbuf (not a private buf).
|
||||
* Wrap it so the bench signature matches, and set the globals it reads. */
|
||||
static void dist_build_new(const float *lo, int V, double temp, double nuc){
|
||||
g_temp = (float)temp; g_nuc = (float)nuc;
|
||||
dist_build(lo, V);
|
||||
}
|
||||
|
||||
/* deterministic logit fill for three shapes */
|
||||
static uint32_t brng = 0xA5A5A5A5u;
|
||||
static double brand(void){ brng ^= brng << 13; brng ^= brng >> 17; brng ^= brng << 5;
|
||||
return (double)(brng >> 8) * (1.0 / 16777216.0); }
|
||||
static void fill_realistic(float *lo, int V){ /* one hot, exponential tail -- like real logits */
|
||||
for (int i = 0; i < V; i++) lo[i] = (float)(-4.0 * brand() - (double)i * 0.0001);
|
||||
lo[0] = 6.f; lo[V/50] = 4.f; lo[V/200] = 3.f;
|
||||
}
|
||||
static void fill_uniform(float *lo, int V){ /* worst case for the heap: max pop count */
|
||||
for (int i = 0; i < V; i++) lo[i] = 0.f;
|
||||
}
|
||||
static void fill_plateau(float *lo, int V){ /* ties: blocks of equal value */
|
||||
for (int i = 0; i < V; i++) lo[i] = (float)(-(double)(i / 50));
|
||||
}
|
||||
|
||||
int main(void){
|
||||
int V = 151936;
|
||||
float *lo = malloc((size_t)V * sizeof(float));
|
||||
s_ref = malloc((size_t)V * sizeof(double));
|
||||
s_pidx = malloc((size_t)V * sizeof(int));
|
||||
/* force the new dist_build to allocate g_pbuf/g_pidx at full V once */
|
||||
g_temp = 0.7f; g_nuc = 0.9f; dist_build(lo, V);
|
||||
|
||||
double temp = 0.7;
|
||||
struct { const char *name; void (*fill)(float*,int); } shapes[] = {
|
||||
{ "realistic", fill_realistic },
|
||||
{ "uniform", fill_uniform },
|
||||
{ "plateau", fill_plateau },
|
||||
};
|
||||
double nucs[] = { 0.5, 0.9, 0.95, 0.99 };
|
||||
|
||||
printf("bench_topp: top-p truncation, old (qsort) vs new (heap) V=%d temp=%.2f\n", V, temp);
|
||||
printf("%-12s %6s %14s %14s %9s %9s\n", "shape", "nuc", "old ns/call", "new ns/call", "speedup", "keep");
|
||||
printf("-----------------------------------------------------------------------------\n");
|
||||
|
||||
for (size_t sh = 0; sh < sizeof(shapes)/sizeof(shapes[0]); sh++){
|
||||
shapes[sh].fill(lo, V);
|
||||
for (size_t ni = 0; ni < sizeof(nucs)/sizeof(nucs[0]); ni++){
|
||||
double nuc = nucs[ni];
|
||||
/* warmup both paths so caches/branch predictors are primed */
|
||||
for (int w = 0; w < 50; w++){ dist_build_old(lo, V, temp, nuc); dist_build_new(lo, V, temp, nuc); }
|
||||
double t_old = bench_ns(dist_build_old, lo, V, temp, nuc);
|
||||
double t_new = bench_ns(dist_build_new, lo, V, temp, nuc);
|
||||
/* keep count = non-zero entries the new path leaves (== old's keep) */
|
||||
int keep = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) keep++;
|
||||
printf("%-12s %6.2f %14.0f %14.0f %8.2fx %9d\n",
|
||||
shapes[sh].name, nuc, t_old, t_new, t_old / t_new, keep);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
printf("bench_topp: done (lower ns is better; speedup = old/new)\n");
|
||||
free(lo); free(s_ref); free(s_pidx);
|
||||
return 0;
|
||||
}
|
||||
@@ -1,11 +1,12 @@
|
||||
import unittest
|
||||
|
||||
from tools.benchmark_cuda_fixture import parse_output
|
||||
from tools.benchmark_cuda_fixture import parse_output, parse_p0
|
||||
|
||||
|
||||
SAMPLE = """
|
||||
REPLAY decode: 4 tokens | 12.34 tok/s
|
||||
PROFILE: expert-disk 1.25s | expert-matmul 2.50s | attention 0.75s | lm_head 0.10s | other -0.05s
|
||||
P0-EXEC: routed CPU 1.200s / 123.40 GB/s (456 row) | routed GPU critical 0.150s | router 0.200s | residual P2P 0.030s / 75 hop | orchestration 0.100s
|
||||
"""
|
||||
|
||||
|
||||
@@ -19,6 +20,9 @@ class ParseOutputTest(unittest.TestCase):
|
||||
with self.assertRaisesRegex(RuntimeError, "benchmark output missing"):
|
||||
parse_output("REPLAY decode: 4 tokens | 12.34 tok/s", "engine failed")
|
||||
|
||||
def test_extracts_p0_profile(self):
|
||||
self.assertEqual(parse_p0(SAMPLE), [1.2, 123.4, 456.0, 0.15, 0.2, 0.03, 75.0, 0.1])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
/* DSA top-keep partial-select: the quickselect rewrite (#356) must produce a
|
||||
* BIT-IDENTICAL kept-position set to the old full-vocab qsort, for every score
|
||||
* shape the attention indexer can see.
|
||||
*
|
||||
* Why this test exists (#356): attention_rows() selects the top-`keep` context
|
||||
* keys (index_topk=2048 on GLM-5.2) to attend to. It previously did this by
|
||||
* full-qsorting all `nk` scores (O(nk log nk)) and reading tmp[keep-1] as the
|
||||
* threshold. It now does a partial_select_desc (quickselect, O(nk) average) and
|
||||
* takes the threshold as the min of the selected top-keep block. The contract is
|
||||
* subtle but STRONGER than test_topp's:
|
||||
*
|
||||
* The two position-order scans that build dst[] --
|
||||
* for t: if isc[t] > thr -> keep (strictly above threshold)
|
||||
* for t: if isc[t] == thr -> keep (ties, in position order)
|
||||
* -- are UNCHANGED by the rewrite. So if the threshold value is identical,
|
||||
* the kept-position set is identical element-by-element (not just as a
|
||||
* multiset, which is all the unstable sampling heap in #335 could promise).
|
||||
*
|
||||
* Strategy: drive the REAL partial_select_desc (via the include-glm.c pattern)
|
||||
* and replicate the production threshold derivation + scans, then compare the
|
||||
* resulting dst[] against an INDEPENDENT reference that re-implements the OLD
|
||||
* algorithm (full qsort + tmp[keep-1] threshold) on a private buffer. The kept
|
||||
* sets must be element-wise equal on every shape, including tie plateaus where
|
||||
* the boundary membership is decided by the position scan.
|
||||
*
|
||||
* We also directly unit-test partial_select_desc's partition invariant: after
|
||||
* the call, max(a[keep..n)) <= min(a[0..keep)) -- i.e. the keep largest really
|
||||
* did land in the prefix. This catches a broken quickselect even before the
|
||||
* end-to-end comparison.
|
||||
*
|
||||
* In-memory only (no scratch files), so it builds clean on the Windows MinGW CI
|
||||
* job without the unmerged compat shim. */
|
||||
#define main coli_glm_main_unused
|
||||
#include "../colibri.c"
|
||||
#undef main
|
||||
|
||||
#include <math.h>
|
||||
|
||||
static int g_nfails = 0;
|
||||
|
||||
#define FAIL(fmt, ...) do { \
|
||||
fprintf(stderr, " FAIL [%s nk=%d keep=%d shape=%s]: " fmt "\n", \
|
||||
label, nk, keep, shape_name, ##__VA_ARGS__); \
|
||||
g_nfails++; \
|
||||
return; \
|
||||
} while (0)
|
||||
|
||||
/* ---- independent reference: the OLD algorithm (full qsort + tmp[keep-1]) ---- */
|
||||
/* qsort comparator matching the production cmp_fdesc exactly (desc, unstable). */
|
||||
static int cmp_ref_desc(const void *a, const void *b){
|
||||
float x=*(const float*)a, y=*(const float*)b; return x<y?1:x>y?-1:0; }
|
||||
|
||||
/* Reproduce the OLD glm.c:2589-2596 exactly: copy, qsort desc, threshold =
|
||||
* tmp[keep-1], then the two position-order scans into dst[]. Returns nd. */
|
||||
static int keep_old(const float *isc, int nk, int keep, int *dst){
|
||||
float *tmp=malloc((size_t)nk*sizeof(float));
|
||||
memcpy(tmp,isc,(size_t)nk*sizeof(float));
|
||||
qsort(tmp,(size_t)nk,sizeof(float),cmp_ref_desc);
|
||||
float thr=tmp[keep-1];
|
||||
int nd=0;
|
||||
for(int t=0;t<nk && nd<keep;t++) if(isc[t]>thr) dst[nd++]=t;
|
||||
for(int t=0;t<nk && nd<keep;t++) if(isc[t]==thr) dst[nd++]=t;
|
||||
free(tmp);
|
||||
return nd;
|
||||
}
|
||||
|
||||
/* Reproduce the NEW glm.c path: partial_select desc, threshold = min of the
|
||||
* selected block, same two scans. Uses the REAL partial_select_desc from glm.c. */
|
||||
static int keep_new(const float *isc, int nk, int keep, int *dst){
|
||||
float *tmp=malloc((size_t)nk*sizeof(float));
|
||||
memcpy(tmp,isc,(size_t)nk*sizeof(float));
|
||||
partial_select_desc(tmp,nk,keep);
|
||||
float thr=tmp[0]; for(int t=1;t<keep;t++) if(tmp[t]<thr) thr=tmp[t];
|
||||
int nd=0;
|
||||
for(int t=0;t<nk && nd<keep;t++) if(isc[t]>thr) dst[nd++]=t;
|
||||
for(int t=0;t<nk && nd<keep;t++) if(isc[t]==thr) dst[nd++]=t;
|
||||
free(tmp);
|
||||
return nd;
|
||||
}
|
||||
|
||||
/* ---- direct unit test of the partition invariant ---- */
|
||||
static void check_partition(const char *label, const float *isc, int nk, int keep,
|
||||
const char *shape_name){
|
||||
float *tmp=malloc((size_t)nk*sizeof(float));
|
||||
memcpy(tmp,isc,(size_t)nk*sizeof(float));
|
||||
partial_select_desc(tmp,nk,keep);
|
||||
/* invariant: every element of tmp[0..keep) is >= every element of tmp[keep..n).
|
||||
* (>=, not >: equal values may sit on either side of the partition boundary,
|
||||
* which is fine -- the threshold is the MIN of the prefix, and the position
|
||||
* scan handles ties.) */
|
||||
float top_min=INFINITY, tail_max=-INFINITY;
|
||||
for(int i=0;i<keep;i++) if(tmp[i]<top_min) top_min=tmp[i];
|
||||
for(int i=keep;i<nk;i++) if(tmp[i]>tail_max) tail_max=tmp[i];
|
||||
if(!(top_min >= tail_max))
|
||||
FAIL("partition invariant violated: top_min=%.9g < tail_max=%.9g", top_min, tail_max);
|
||||
free(tmp);
|
||||
}
|
||||
|
||||
/* ---- end-to-end: old vs new kept-set must be element-wise identical ---- */
|
||||
static void check_case(const char *label, int nk, int keep, const char *shape_name,
|
||||
const float *isc){
|
||||
int *da=malloc((size_t)nk*sizeof(int));
|
||||
int *db=malloc((size_t)nk*sizeof(int));
|
||||
int na=keep_old(isc,nk,keep,da);
|
||||
int nb=keep_new(isc,nk,keep,db);
|
||||
|
||||
/* 1. both keep exactly `keep` positions (the contract: keep the top-keep by
|
||||
* count). A count mismatch is a real bug, not a tie artifact. */
|
||||
if(na!=keep) FAIL("old kept %d, expected %d (old path is the reference)", na, keep);
|
||||
if(nb!=keep) FAIL("new kept %d, expected %d", nb, keep);
|
||||
if(na!=nb) FAIL("keep-count mismatch: old=%d new=%d", na, nb);
|
||||
|
||||
/* 2. element-wise identical dst[]. This is the strong contract: because the
|
||||
* threshold is derived identically and the position-order scans are byte-
|
||||
* for-byte the same, the kept SET and its ORDER must match exactly. (This
|
||||
* is what makes #356 cleaner than #335, which was multiset-only.) */
|
||||
int first_diff=-1;
|
||||
for(int i=0;i<na;i++){ if(da[i]!=db[i]){ first_diff=i; break; } }
|
||||
if(first_diff>=0)
|
||||
FAIL("kept-set differs at index %d: old dst[%d]=%d new dst[%d]=%d",
|
||||
first_diff, first_diff, da[first_diff], first_diff, db[first_diff]);
|
||||
|
||||
/* 3. also check the partition invariant directly (catches a subtly broken
|
||||
* quickselect even if the threshold happened to come out right). */
|
||||
check_partition(label,isc,nk,keep,shape_name);
|
||||
|
||||
free(da); free(db);
|
||||
printf(" ok [nk=%d keep=%d shape=%s]\n", nk, keep, shape_name);
|
||||
}
|
||||
#undef FAIL
|
||||
|
||||
/* deterministic xorshift32 RNG (matches the test_i4_grouped.c / test_topp.c convention) */
|
||||
static uint32_t rng_state = 0x12345678u;
|
||||
static uint32_t xr(void){ rng_state ^= rng_state << 13; rng_state ^= rng_state >> 17;
|
||||
rng_state ^= rng_state << 5; return rng_state; }
|
||||
static double frand(void){ return (xr() >> 8) * (1.0 / 16777216.0); } /* [0,1) */
|
||||
|
||||
/* fill scores for a given shape. Shapes stress the threshold boundary and the
|
||||
* quickselect's median-of-three pivot differently. */
|
||||
static void fill_shape(float *isc, int nk, int shape){
|
||||
switch(shape){
|
||||
case 0: /* uniform random distinct (no ties): the clean contract case */
|
||||
for(int i=0;i<nk;i++) isc[i]=(float)(frand()*1000.0); break;
|
||||
case 1: /* peaked: a few hot, long distinct tail (realistic attention shape) */
|
||||
for(int i=0;i<nk;i++) isc[i]=(float)(-1.0 - frand()*4.0);
|
||||
isc[0]=3.f; if(nk>3) isc[nk/3]=1.f; if(nk>2) isc[nk/2]=0.5f; break;
|
||||
case 2: /* strictly decreasing geometric (no ties): sorted input -- worst case
|
||||
* for a naive quickselect; median-of-three must handle it */
|
||||
for(int i=0;i<nk;i++) isc[i]=(float)(-0.001*(double)i); break;
|
||||
case 3: /* strictly increasing (reverse-sorted): the other quickselect worst case */
|
||||
for(int i=0;i<nk;i++) isc[i]=(float)(0.001*(double)i); break;
|
||||
case 4: /* plateau ties: blocks of equal value -> boundary membership decided
|
||||
* entirely by the position scan (exercises the ==thr path) */
|
||||
for(int i=0;i<nk;i++) isc[i]=(float)(-(double)(i/7)); break;
|
||||
case 5: /* all-equal: every value identical -> degenerate threshold, all kept
|
||||
* via the ==thr scan; quickselect must not infinite-loop or corrupt */
|
||||
for(int i=0;i<nk;i++) isc[i]=5.f; break;
|
||||
}
|
||||
}
|
||||
|
||||
int main(void){
|
||||
/* Sizes around the real index_topk=2048 boundary, plus small cases that
|
||||
* exercise the k>=n / k==1 / k==n edges. */
|
||||
int nks[] = {1, 2, 8, 64, 2049, 4097, 8193};
|
||||
int keeps[] = {1, 8, 256, 1024, 2048};
|
||||
int n_shapes = 6;
|
||||
|
||||
int cases = 0;
|
||||
for(size_t ni=0; ni<sizeof(nks)/sizeof(nks[0]); ni++){
|
||||
int nk=nks[ni];
|
||||
float *isc=malloc((size_t)nk*sizeof(float));
|
||||
for(int shape=0; shape<n_shapes; shape++){
|
||||
/* skip shapes that write out of bounds on tiny nk (fill_shape guards
|
||||
* the hot-spots with nk>n, but skip the plateau/geometric edge if nk<7) */
|
||||
fill_shape(isc,nk,shape);
|
||||
for(size_t ki=0; ki<sizeof(keeps)/sizeof(keeps[0]); ki++){
|
||||
int keep=keeps[ki];
|
||||
if(keep>nk) continue; /* keep<=nk invariant of the production code */
|
||||
if(keep<=0) continue;
|
||||
char label[40]; snprintf(label,sizeof(label),"nk[%zu]/keep[%zu]/shape[%d]",ni,ki,shape);
|
||||
const char *sn=(const char*[]){"random","peaked","decreasing","increasing","plateau","all-equal"}[shape];
|
||||
check_case(label,nk,keep,sn,isc);
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
free(isc);
|
||||
}
|
||||
|
||||
/* edge: keep == nk (nothing to partition; both paths keep everything) */
|
||||
{
|
||||
int nk=100, keep=100; float isc[100];
|
||||
for(int i=0;i<nk;i++) isc[i]=(float)frand();
|
||||
int *db=malloc(sizeof(int)*nk); int nb=keep_new(isc,nk,keep,db);
|
||||
if(nb!=nk){ fprintf(stderr," FAIL [keep==nk]: kept %d expected %d\n",nb,nk); g_nfails++; }
|
||||
else printf(" ok [keep==nk nk=%d]\n",nk);
|
||||
free(db); cases++;
|
||||
}
|
||||
/* edge: keep == 1 (threshold = the single max; quickselect must find it) */
|
||||
{
|
||||
int nk=500, keep=1; float isc[500];
|
||||
for(int i=0;i<nk;i++) isc[i]=(float)frand();
|
||||
int da[1],db[1]; int na=keep_old(isc,nk,keep,da), nb=keep_new(isc,nk,keep,db);
|
||||
if(na!=1||nb!=1||da[0]!=db[0]){
|
||||
fprintf(stderr," FAIL [keep==1]: old={%d (n=%d)} new={%d (n=%d)}\n",da[0],na,db[0],nb); g_nfails++; }
|
||||
else printf(" ok [keep==1 argmax=%d]\n",db[0]); cases++;
|
||||
}
|
||||
/* edge: all-equal scores, keep in the middle -> every kept slot is a tie;
|
||||
* the position scan must pick positions 0..keep-1 deterministically */
|
||||
{
|
||||
int nk=1000, keep=500; float isc[1000];
|
||||
for(int i=0;i<nk;i++) isc[i]=3.14f;
|
||||
int *db=malloc(sizeof(int)*nk); int nb=keep_new(isc,nk,keep,db);
|
||||
int bad=0; for(int i=0;i<nb;i++) if(db[i]!=i) bad=1;
|
||||
if(nb!=keep||bad){ fprintf(stderr," FAIL [all-equal keep=%d]: nb=%d bad=%d\n",keep,nb,bad); g_nfails++; }
|
||||
else printf(" ok [all-equal keep=%d -> positions 0..%d]\n",keep,keep-1); cases++;
|
||||
free(db);
|
||||
}
|
||||
|
||||
printf("\ntest_dsa_select: %d cases run, %d failure(s)\n", cases, g_nfails);
|
||||
if(g_nfails){ printf("test_dsa_select: FAIL\n"); return 1; }
|
||||
printf("test_dsa_select: ok\n");
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
/* Exactness test for the grouped-int4 kernel (fmt=4, one f32 scale per `gs`
|
||||
* elements along I) against a plain-C reference that dequantizes and multiplies
|
||||
* in double.
|
||||
*
|
||||
* Why this test exists: matmul_i4_grouped is the REFERENCE the CUDA fmt=4 path
|
||||
* (#298) is expected to reproduce, and it had no test of its own. Debugging a
|
||||
* backend against an unverified oracle means two moving targets. Anyone porting
|
||||
* fmt=4 to a new backend can now diff against a kernel that is known exact here.
|
||||
*
|
||||
* Covers: I a clean multiple of gs, I with a partial last group (the `glen`
|
||||
* clamp), odd I (the nibble tail), gs larger than I (single group), and the
|
||||
* nibble edges 0 and 15 (which decode to -8 and +7 — an offset encoding, NOT
|
||||
* two's complement; getting this backwards is silent and looks like noise).
|
||||
*
|
||||
* FP note: the kernel sums each group in f32 (AVX2 accumulator + scalar tail)
|
||||
* while the reference sums in double, so we compare against a relative epsilon
|
||||
* rather than bit-exactly. The tolerance is tight enough that a wrong scale
|
||||
* index, a wrong group boundary or a swapped nibble cannot hide under it —
|
||||
* those are O(1) relative errors, not O(1e-6). */
|
||||
#define main coli_glm_main_unused
|
||||
#include "../colibri.c"
|
||||
#undef main
|
||||
|
||||
static uint32_t rng_state=0xC0FFEEu;
|
||||
static uint32_t xr(void){ rng_state^=rng_state<<13; rng_state^=rng_state>>17; rng_state^=rng_state<<5; return rng_state; }
|
||||
static float frand(void){ return (float)((int)(xr()%2001)-1000)/1000.0f; }
|
||||
|
||||
/* Reference: dequantize nibble -> (v-8)*scale[group], accumulate in double.
|
||||
* Deliberately the dumbest possible expression of the format. */
|
||||
static void ref_grouped(double *y, double *mag, const float *x, const uint8_t *q4,
|
||||
const float *scale, int S, int I, int O, int gs){
|
||||
int rb=(I+1)/2, ng=(I+gs-1)/gs;
|
||||
for(int o=0;o<O;o++){
|
||||
const uint8_t *w=q4+(int64_t)o*rb;
|
||||
const float *scl=scale+(int64_t)o*ng;
|
||||
for(int s=0;s<S;s++){
|
||||
const float *xs=x+(int64_t)s*I; double a=0, m=0;
|
||||
for(int i=0;i<I;i++){
|
||||
uint8_t byte=w[i>>1];
|
||||
int nib=(i&1)?(int)(byte>>4):(int)(byte&0xF);
|
||||
double term=(double)xs[i] * (double)(nib-8) * (double)scl[i/gs];
|
||||
a += term; m += fabs(term);
|
||||
}
|
||||
y[(int64_t)s*O+o]=a;
|
||||
/* Sum of |terms|: the scale the f32 rounding error actually lives on.
|
||||
* Comparing against |result| instead would flag pure cancellation --
|
||||
* a dot product of signed terms can land near zero, and then a 1e-6
|
||||
* absolute error reads as a 1e-3 relative one. That is the accumulator's
|
||||
* precision, not a kernel defect. A wrong scale index or group boundary
|
||||
* shifts the result by a fraction OF THE TERMS, so it is caught here. */
|
||||
mag[(int64_t)s*O+o]=m;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static int check(const char *name, int S, int I, int O, int gs, int fill_edges){
|
||||
int rb=(I+1)/2, ng=(I+gs-1)/gs;
|
||||
uint8_t *q4=malloc((size_t)O*rb);
|
||||
float *scale=malloc((size_t)O*ng*sizeof(float));
|
||||
float *x=malloc((size_t)S*I*sizeof(float));
|
||||
float *y=malloc((size_t)S*O*sizeof(float));
|
||||
double *yr=malloc((size_t)S*O*sizeof(double));
|
||||
double *ym=malloc((size_t)S*O*sizeof(double));
|
||||
if(!q4||!scale||!x||!y||!yr||!ym){ fprintf(stderr,"%s: OOM\n",name); return 1; }
|
||||
|
||||
for(size_t i=0;i<(size_t)O*rb;i++) q4[i]=(uint8_t)(xr()&0xFF);
|
||||
if(fill_edges){
|
||||
/* nibble extremes: 0x0F -> +7, 0x00 -> -8. A two's-complement misread
|
||||
* turns 15 into -1 instead of +7 and the error is data-dependent noise. */
|
||||
for(size_t i=0;i<(size_t)O*rb && i<64;i++) q4[i]=(i&1)?0x00:0xFF;
|
||||
}
|
||||
/* scales span a few orders of magnitude: a wrong group index shows up big */
|
||||
for(int i=0;i<O*ng;i++) scale[i]=(0.001f+(float)(xr()%1000)/1000.0f)*((xr()&1)?1.f:-1.f);
|
||||
for(int i=0;i<S*I;i++) x[i]=frand();
|
||||
|
||||
matmul_i4_grouped(y,x,q4,scale,S,I,O,gs);
|
||||
ref_grouped(yr,ym,x,q4,scale,S,I,O,gs);
|
||||
|
||||
int bad=0; double worst=0;
|
||||
for(int i=0;i<S*O;i++){
|
||||
double d=fabs((double)y[i]-yr[i]);
|
||||
double rel = ym[i]>1e-30 ? d/ym[i] : d; /* error relative to the summed magnitude */
|
||||
if(rel>worst) worst=rel;
|
||||
if(rel>1e-6){
|
||||
if(bad<3) fprintf(stderr,"%s: [%d] got %.9g want %.9g (|terms| %.3g, rel %.3g)\n",
|
||||
name,i,(double)y[i],yr[i],ym[i],rel);
|
||||
bad++;
|
||||
}
|
||||
}
|
||||
free(q4);free(scale);free(x);free(y);free(yr);free(ym);
|
||||
if(bad){ fprintf(stderr,"%s: FAIL (%d/%d mismatched, worst rel %.3g)\n",name,bad,S*O,worst); return 1; }
|
||||
printf(" %-42s ok (S=%d I=%d O=%d gs=%d ng=%d, worst rel %.2g)\n",name,S,I,O,gs,ng,worst);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* matmul_i4_grouped_pair (fused gate+up, #298) reads x once instead of twice.
|
||||
* Checked two ways, because "identical" is only true where it can be:
|
||||
*
|
||||
* - Correctness, always: both outputs must match the double reference within
|
||||
* the same magnitude-relative epsilon as the unfused kernel.
|
||||
* - Bit-exactness, only when I % gs == 0: then every group is covered by the
|
||||
* AVX2 body, whose accumulation order is identical to the unfused kernel, so
|
||||
* the results agree to the last bit. This is the shape the real g64
|
||||
* checkpoints have (I = 2048 / 6144, gs = 64), i.e. the production path.
|
||||
*
|
||||
* With a PARTIAL last group the group tail falls to scalar code, and the
|
||||
* compiler is free to contract/reassociate the fused body differently from the
|
||||
* single-matrix one. The results then differ by ~1e-7 -- rounding, not logic
|
||||
* (which of gate/up "differs" is arbitrary, the tell that it is FP luck).
|
||||
* Demanding bit-exactness there would report a compiler artifact as a bug. */
|
||||
#ifdef COLI_HAVE_GROUPED_PAIR
|
||||
static int check_pair(const char *name, int S, int I, int O, int gs){
|
||||
int rb=(I+1)/2, ng=(I+gs-1)/gs;
|
||||
uint8_t *qg=malloc((size_t)O*rb), *qu=malloc((size_t)O*rb);
|
||||
float *sg=malloc((size_t)O*ng*sizeof(float)), *su=malloc((size_t)O*ng*sizeof(float));
|
||||
float *x=malloc((size_t)S*I*sizeof(float));
|
||||
float *yg=malloc((size_t)S*O*sizeof(float)), *yu=malloc((size_t)S*O*sizeof(float));
|
||||
float *rg=malloc((size_t)S*O*sizeof(float)), *ru=malloc((size_t)S*O*sizeof(float));
|
||||
double *dg=malloc((size_t)S*O*sizeof(double)), *du=malloc((size_t)S*O*sizeof(double));
|
||||
double *mg=malloc((size_t)S*O*sizeof(double)), *mu=malloc((size_t)S*O*sizeof(double));
|
||||
if(!qg||!qu||!sg||!su||!x||!yg||!yu||!rg||!ru||!dg||!du||!mg||!mu){ fprintf(stderr,"%s: OOM\n",name); return 1; }
|
||||
|
||||
for(size_t i=0;i<(size_t)O*rb;i++){ qg[i]=(uint8_t)(xr()&0xFF); qu[i]=(uint8_t)(xr()&0xFF); }
|
||||
for(int i=0;i<O*ng;i++){ sg[i]=frand(); su[i]=frand(); }
|
||||
for(int i=0;i<S*I;i++) x[i]=frand();
|
||||
|
||||
matmul_i4_grouped_pair(yg,yu,x,qg,sg,qu,su,S,I,O,gs);
|
||||
matmul_i4_grouped(rg,x,qg,sg,S,I,O,gs);
|
||||
matmul_i4_grouped(ru,x,qu,su,S,I,O,gs);
|
||||
ref_grouped(dg,mg,x,qg,sg,S,I,O,gs);
|
||||
ref_grouped(du,mu,x,qu,su,S,I,O,gs);
|
||||
|
||||
int bad=0, exact=1; double worst=0;
|
||||
for(int i=0;i<S*O;i++){
|
||||
double eg = mg[i]>1e-30 ? fabs((double)yg[i]-dg[i])/mg[i] : fabs((double)yg[i]-dg[i]);
|
||||
double eu = mu[i]>1e-30 ? fabs((double)yu[i]-du[i])/mu[i] : fabs((double)yu[i]-du[i]);
|
||||
if(eg>worst) worst=eg;
|
||||
if(eu>worst) worst=eu;
|
||||
if(eg>1e-6||eu>1e-6){
|
||||
if(bad<3) fprintf(stderr,"%s: [%d] gate %.9g/%.9g up %.9g/%.9g (rel %.3g/%.3g)\n",
|
||||
name,i,(double)yg[i],dg[i],(double)yu[i],du[i],eg,eu);
|
||||
bad++;
|
||||
}
|
||||
if(yg[i]!=rg[i]||yu[i]!=ru[i]) exact=0;
|
||||
}
|
||||
/* Aligned shapes run entirely through the AVX2 body: same order as unfused,
|
||||
* so bit-exactness is a real invariant there and worth asserting. */
|
||||
if(I%gs==0 && !exact){
|
||||
fprintf(stderr,"%s: FAIL fused != unfused bitwise on an ALIGNED shape "
|
||||
"(no scalar tail runs here; the orders must match)\n",name);
|
||||
bad++;
|
||||
}
|
||||
free(qg);free(qu);free(sg);free(su);free(x);free(yg);free(yu);free(rg);free(ru);
|
||||
free(dg);free(du);free(mg);free(mu);
|
||||
if(bad){ fprintf(stderr,"%s: FAIL (%d mismatched)\n",name,bad); return 1; }
|
||||
printf(" %-42s ok (S=%d I=%d O=%d gs=%d, worst rel %.2g%s)\n",name,S,I,O,gs,worst,
|
||||
I%gs==0?", bit-exact vs unfused":"");
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
int main(void){
|
||||
int fail=0;
|
||||
printf("test_i4_grouped: matmul_i4_grouped vs plain-C dequant reference\n");
|
||||
|
||||
/* the shape the g64 checkpoints actually use */
|
||||
fail|=check("gs=64, I multiple of gs", 2, 512, 8, 64, 0);
|
||||
fail|=check("gs=64, single row single token", 1, 128, 1, 64, 0);
|
||||
fail|=check("gs=64, nibble edges (0x00/0xFF)", 1, 256, 4, 64, 1);
|
||||
|
||||
/* partial last group: glen clamp, the classic off-by-one */
|
||||
fail|=check("gs=64, partial last group (I=200)", 2, 200, 4, 64, 0);
|
||||
fail|=check("gs=64, I just over a group (I=65)", 1, 65, 3, 64, 0);
|
||||
fail|=check("gs=64, I one under a group (I=63)", 1, 63, 3, 64, 0);
|
||||
|
||||
/* odd I: the scalar nibble tail (i+1 == I) */
|
||||
fail|=check("gs=64, odd I (I=201)", 2, 201, 4, 64, 0);
|
||||
fail|=check("gs=16, odd I (I=33)", 1, 33, 2, 16, 0);
|
||||
|
||||
/* gs > I: everything in one group */
|
||||
fail|=check("gs=128 > I=64 (single group)", 1, 64, 4, 128, 0);
|
||||
|
||||
/* the other documented group size */
|
||||
fail|=check("gs=128, I multiple of gs", 2, 512, 4, 128, 0);
|
||||
|
||||
/* batch: S>1 exercises the per-s inner loop against a shared scale row */
|
||||
fail|=check("gs=64, batch S=8", 8, 320, 6, 64, 0);
|
||||
|
||||
#ifdef COLI_HAVE_GROUPED_PAIR
|
||||
printf("test_i4_grouped: matmul_i4_grouped_pair (fused gate+up) vs two separate calls\n");
|
||||
fail|=check_pair("pair: gs=64, I multiple of gs", 2, 512, 8, 64);
|
||||
fail|=check_pair("pair: gs=64, partial last group",2, 200, 4, 64);
|
||||
fail|=check_pair("pair: gs=64, odd I (I=201)", 2, 201, 4, 64);
|
||||
fail|=check_pair("pair: gs=64, decode S=1", 1, 320, 6, 64);
|
||||
fail|=check_pair("pair: gs=128, I=512", 2, 512, 4, 128);
|
||||
#endif
|
||||
|
||||
if(fail){ printf("test_i4_grouped: FAIL\n"); return 1; }
|
||||
printf("test_i4_grouped: ok\n");
|
||||
return 0;
|
||||
}
|
||||
+1
-1
@@ -7,7 +7,7 @@
|
||||
* (sign-trick kernels must treat |−128| as 128 unsigned, not saturate to 127),
|
||||
* and random data at qrow_i8's contract (|x| <= 127, w full int8 range). */
|
||||
#define main coli_glm_main_unused
|
||||
#include "../glm.c"
|
||||
#include "../colibri.c"
|
||||
#undef main
|
||||
|
||||
static uint32_t rng_state=0x12345678u;
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
* arrays twice on the second call -> allocator abort. No model file needed:
|
||||
* the CPU path of kv_alloc only reads c->n_layers/kv_lora/qk_rope. */
|
||||
#define main coli_glm_main_unused
|
||||
#include "../glm.c"
|
||||
#include "../colibri.c"
|
||||
#undef main
|
||||
|
||||
int main(void){
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
/* Regression for non-finite-logit poisoning of sampling.
|
||||
*
|
||||
* A single NaN or +Inf in the logits (a bad streamed expert tile, or an fp
|
||||
* overflow in the matmul at a low-RAM eviction boundary) used to make softmax
|
||||
* produce an all-NaN g_pbuf; dist_sample then never satisfied cum>=u and fell
|
||||
* through to return token 0 — so the engine silently emitted an unbroken run of
|
||||
* token 0 with no error, on the DEFAULT serve path (TEMP>0, 0<NUCLEUS<1).
|
||||
*
|
||||
* Fix under test: argmax_v() skips NaN (picks the max finite/+Inf entry instead
|
||||
* of being pinned to index 0), and dist_build() detects a non-finite softmax sum
|
||||
* and collapses to a one-hot over the finite argmax (warning once) rather than
|
||||
* dividing every entry into NaN. Degrade + diagnose, never silently corrupt.
|
||||
*
|
||||
* No model file needed: exercises argmax_v / dist_build / dist_sample directly. */
|
||||
#include <assert.h>
|
||||
#include <math.h>
|
||||
#define main coli_glm_main_unused
|
||||
#include "../colibri.c"
|
||||
#undef main
|
||||
|
||||
static int approx1(double x){ return x > 0.999 && x < 1.001; }
|
||||
|
||||
int main(void){
|
||||
/* --- argmax_v must skip NaN (greedy decode + speculative-verify paths) --- */
|
||||
{ float lo[8]={NAN,1.f,5.f,2.f,NAN,-3.f,4.f,0.f};
|
||||
assert(argmax_v(lo,8)==2 && "pick max finite (idx2=5.0), not NaN-pinned idx0"); }
|
||||
{ float lo[8]={3.f,INFINITY,1.f,2.f,0.f,-1.f,2.5f,1.5f};
|
||||
assert(argmax_v(lo,8)==1 && "pick the +Inf position"); }
|
||||
{ float lo[8]; for(int i=0;i<8;i++) lo[i]=NAN;
|
||||
assert(argmax_v(lo,8)==0 && "all-NaN: no crash, defined fallback"); }
|
||||
|
||||
g_temp=0.7f; g_nuc=0.9f; /* the default serve/chat sampling path */
|
||||
|
||||
/* --- dist_build: a NaN logit must yield a finite one-hot, not all-NaN --- */
|
||||
{ float lo[8]={0.5f,1.f,NAN,8.f,0.2f,-1.f,0.f,0.3f}; /* max finite = idx3 (8.0) */
|
||||
dist_build(lo,8);
|
||||
double sum=0; int nan=0;
|
||||
for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; }
|
||||
assert(!nan && "g_pbuf must be finite after a NaN logit");
|
||||
assert(approx1(sum) && "g_pbuf must normalize to 1");
|
||||
assert(approx1(g_pbuf[3]) && "mass must land on the max finite logit (idx3)");
|
||||
assert(dist_sample(8,-1)==3 && "sampler emits the finite argmax, not token 0"); }
|
||||
|
||||
/* --- NaN at index 0: poisons the max scan itself (the old mx=lo[0] seed),
|
||||
* the failure mode that starts before the sum (review note on #369) --- */
|
||||
{ float lo[8]={NAN,1.f,0.5f,6.f,0.2f,-1.f,0.f,0.3f}; /* max finite = idx3 (6.0) */
|
||||
dist_build(lo,8);
|
||||
double sum=0; int nan=0;
|
||||
for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; }
|
||||
assert(!nan && "NaN at lo[0] must not poison via the mx seed");
|
||||
assert(approx1(sum) && "still normalizes to 1");
|
||||
assert(dist_sample(8,-1)==3 && "emits the max finite logit, not token 0"); }
|
||||
|
||||
/* --- all-NaN logits: worst case — must stay finite, no crash --- */
|
||||
{ float lo[8]; for(int i=0;i<8;i++) lo[i]=NAN;
|
||||
dist_build(lo,8);
|
||||
for(int i=0;i<8;i++) assert(g_pbuf[i]==g_pbuf[i] && "all-NaN: g_pbuf stays finite"); }
|
||||
|
||||
/* --- regression: clean logits still produce a valid distribution --- */
|
||||
{ float lo[8]={0.1f,0.2f,3.0f,0.4f,0.5f,0.6f,0.7f,0.8f}; /* peak = idx2 */
|
||||
dist_build(lo,8);
|
||||
double sum=0; int nan=0;
|
||||
for(int i=0;i<8;i++){ if(!(g_pbuf[i]==g_pbuf[i])) nan=1; sum+=g_pbuf[i]; }
|
||||
assert(!nan && "clean softmax stays finite");
|
||||
assert(approx1(sum) && "clean softmax must sum to 1");
|
||||
assert(g_pbuf[2]>=g_pbuf[0] && "peak token keeps the most mass"); }
|
||||
|
||||
printf("OK test_logit_nan: argmax_v NaN-skip + dist_build finite-collapse\n");
|
||||
return 0;
|
||||
}
|
||||
@@ -35,7 +35,7 @@ class MakefilePlatformTests(unittest.TestCase):
|
||||
env["PATH"] = ""
|
||||
|
||||
result = subprocess.run(
|
||||
[MAKE, "--no-print-directory", "-B", "-n", "glm"],
|
||||
[MAKE, "--no-print-directory", "-B", "-n", "colibri"],
|
||||
cwd=C_DIR,
|
||||
env=env,
|
||||
text=True,
|
||||
@@ -43,7 +43,7 @@ class MakefilePlatformTests(unittest.TestCase):
|
||||
check=True,
|
||||
)
|
||||
|
||||
self.assertIn("-o glm.exe", result.stdout)
|
||||
self.assertIn("-o colibri.exe", result.stdout)
|
||||
self.assertIn("-fopenmp", result.stdout)
|
||||
self.assertIn("-static", result.stdout)
|
||||
|
||||
|
||||
@@ -380,6 +380,25 @@ class DispatcherTest(unittest.TestCase):
|
||||
engine.close()
|
||||
self.assertEqual(chunks, ["é"])
|
||||
|
||||
def test_records_profile_snapshots_from_prof_lines(self):
|
||||
def respond(process, frame):
|
||||
request_id = frame.split()[1]
|
||||
process.stdout.feed(b"DATA " + request_id + b" 2\nok\n")
|
||||
process.stdout.feed(b"PROF 2.500 7 12 0.400 0.100 0.900 0.600 0.200 15\n")
|
||||
process.stdout.feed(b"DONE " + request_id + b" STAT 12 4.8 0 1.0 7 0\n")
|
||||
|
||||
process = FakeProcess(respond)
|
||||
with patch("openai_server.subprocess.Popen", return_value=process):
|
||||
engine = Engine("glm", "model")
|
||||
engine.generate("hello", 16, 0.7, 0.9, lambda _: None)
|
||||
engine.close()
|
||||
self.assertEqual(engine.profile_seq, 1)
|
||||
self.assertEqual(list(engine.profile), [{
|
||||
"wall_s": 2.5, "prompt_tokens": 7, "completion_tokens": 12,
|
||||
"expert_disk_s": 0.4, "expert_wait_s": 0.1, "expert_matmul_s": 0.9,
|
||||
"attention_s": 0.6, "lm_head_s": 0.2, "forwards": 15,
|
||||
}])
|
||||
|
||||
def test_cancels_generation_after_consumer_disconnects(self):
|
||||
request_id = None
|
||||
|
||||
@@ -443,6 +462,20 @@ class HTTPTest(unittest.TestCase):
|
||||
self.assertIn("queued", scheduler)
|
||||
self.assertEqual(health["kv_slots"], 2)
|
||||
|
||||
def test_profile_reports_recent_turns_without_auth(self):
|
||||
with urlopen(self.base + "/profile", timeout=2) as response:
|
||||
self.assertEqual(json.load(response), {"seq": 0, "turns": []})
|
||||
turn = {"wall_s": 2.5, "prompt_tokens": 7, "completion_tokens": 12,
|
||||
"expert_disk_s": 0.4, "expert_wait_s": 0.1, "expert_matmul_s": 0.9,
|
||||
"attention_s": 0.6, "lm_head_s": 0.2, "forwards": 15}
|
||||
self.engine.profile = [turn]
|
||||
self.engine.profile_seq = 1
|
||||
try:
|
||||
with urlopen(self.base + "/profile", timeout=2) as response:
|
||||
self.assertEqual(json.load(response), {"seq": 1, "turns": [turn]})
|
||||
finally:
|
||||
del self.engine.profile, self.engine.profile_seq
|
||||
|
||||
def test_browser_preflight(self):
|
||||
request = Request(self.base + "/v1/chat/completions", method="OPTIONS", headers={
|
||||
"Origin": "http://localhost:5173",
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
"""End-to-end tool-calling test for the OpenAI gateway (#401).
|
||||
|
||||
Unlike the unit tests in test_openai_server.py (which call parse_tool_calls /
|
||||
render_chat directly), this suite runs openai_server.py as a real subprocess
|
||||
against a mock engine that speaks the actual SERVE wire protocol
|
||||
(READY / SUBMIT / DATA / DONE), then talks to it over real HTTP. It pins down
|
||||
the full path a coding client exercises: tool declaration rendering, marker
|
||||
suppression in streamed deltas (across chunk boundaries), tool_calls in both
|
||||
response shapes, and the <|observation|><tool_response> round trip.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
SERVER = Path(__file__).resolve().parent.parent / "openai_server.py"
|
||||
MODEL_ID = "glm-5.2-colibri"
|
||||
|
||||
# Mock engine: replies are keyed on the prompt so one process covers every case.
|
||||
# Prompts received are appended to MOCK_LOG for assertions on the rendering.
|
||||
MOCK_ENGINE = r'''#!/usr/bin/env python3
|
||||
import sys, os
|
||||
out, inp = sys.stdout.buffer, sys.stdin.buffer
|
||||
out.write(b"\x01\x01READY\x01\x01\n" + b"STAT 0 0 0 0 0\n"); out.flush()
|
||||
|
||||
def reply(rid, text, chunks=1):
|
||||
data = text.encode("utf-8")
|
||||
n = max(1, len(data) // chunks)
|
||||
for i in range(0, len(data), n):
|
||||
part = data[i:i+n]
|
||||
out.write(("DATA %s %d\n" % (rid, len(part))).encode() + part + b"\n"); out.flush()
|
||||
out.write(("DONE %s STAT %d 1.0 50.0 10.0 42 0\n" % (rid, len(text.split()))).encode())
|
||||
out.flush()
|
||||
|
||||
while True:
|
||||
line = inp.readline()
|
||||
if not line: break
|
||||
f = line.decode().strip().split()
|
||||
if not f or f[0] != "SUBMIT": continue
|
||||
rid, plen = f[1], int(f[3])
|
||||
prompt = inp.read(plen).decode("utf-8", "replace"); inp.read(1)
|
||||
with open(os.environ["MOCK_LOG"], "a") as log:
|
||||
log.write(prompt + "\n\x00\n")
|
||||
if "<tool_response>" in prompt:
|
||||
reply(rid, "25 degrees and sunny in Rome.")
|
||||
elif "weather in Rome" in prompt:
|
||||
reply(rid, "<tool_call>get_weather<arg_key>city</arg_key>"
|
||||
"<arg_value>Rome</arg_value></tool_call>")
|
||||
elif "weather in Milan" in prompt:
|
||||
# split across many tiny DATA chunks: streamed marker suppression must
|
||||
# hold even when a marker straddles a chunk boundary
|
||||
reply(rid, "Checking. <tool_call>get_weather<arg_key>city</arg_key>"
|
||||
"<arg_value>Milan</arg_value></tool_call>", chunks=20)
|
||||
else:
|
||||
reply(rid, "Hello from the mock engine.")
|
||||
'''
|
||||
|
||||
TOOLS = [{"type": "function", "function": {
|
||||
"name": "get_weather",
|
||||
"description": "Current weather for a city",
|
||||
"parameters": {"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"]}}}]
|
||||
|
||||
|
||||
@unittest.skipUnless(os.name == "posix",
|
||||
"the mock engine is a shebang script the gateway execs directly; "
|
||||
"Windows CreateProcess cannot run it. The gateway logic under test "
|
||||
"is platform-independent and covered by the POSIX CI jobs.")
|
||||
class ToolCallingE2E(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tmp = tempfile.TemporaryDirectory()
|
||||
mock = Path(cls.tmp.name) / "mock_engine.py"
|
||||
mock.write_text(MOCK_ENGINE)
|
||||
mock.chmod(0o755)
|
||||
cls.mock_log = Path(cls.tmp.name) / "prompts.log"
|
||||
cls.mock_log.touch()
|
||||
with socket.socket() as probe: # free port, then hand it to the server
|
||||
probe.bind(("127.0.0.1", 0))
|
||||
cls.port = probe.getsockname()[1]
|
||||
env = dict(os.environ, MOCK_LOG=str(cls.mock_log))
|
||||
cls.server = subprocess.Popen(
|
||||
[sys.executable, str(SERVER), "--model", cls.tmp.name,
|
||||
"--engine", str(mock), "--port", str(cls.port)],
|
||||
env=env, stderr=subprocess.DEVNULL)
|
||||
cls.base = f"http://127.0.0.1:{cls.port}/v1"
|
||||
for _ in range(100):
|
||||
try:
|
||||
urllib.request.urlopen(cls.base + "/models", timeout=2)
|
||||
return
|
||||
except OSError:
|
||||
if cls.server.poll() is not None:
|
||||
raise RuntimeError("gateway exited during startup")
|
||||
import time
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("gateway did not come up")
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.server.terminate()
|
||||
cls.server.wait(timeout=5)
|
||||
cls.tmp.cleanup()
|
||||
|
||||
def post(self, body, stream=False):
|
||||
req = urllib.request.Request(
|
||||
self.base + "/chat/completions", json.dumps(body).encode(),
|
||||
{"Content-Type": "application/json"})
|
||||
resp = urllib.request.urlopen(req, timeout=30)
|
||||
if not stream:
|
||||
return json.loads(resp.read())
|
||||
events = []
|
||||
for raw in resp:
|
||||
line = raw.decode().strip()
|
||||
if line.startswith("data: ") and line != "data: [DONE]":
|
||||
events.append(json.loads(line[6:]))
|
||||
return events
|
||||
|
||||
def test_tool_call_non_stream(self):
|
||||
r = self.post({"model": MODEL_ID, "tools": TOOLS,
|
||||
"messages": [{"role": "user",
|
||||
"content": "What is the weather in Rome?"}]})
|
||||
choice = r["choices"][0]
|
||||
self.assertEqual(choice["finish_reason"], "tool_calls")
|
||||
calls = choice["message"]["tool_calls"]
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(calls[0]["function"]["name"], "get_weather")
|
||||
self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"city": "Rome"})
|
||||
self.assertNotIn("<tool_call>", choice["message"].get("content") or "")
|
||||
|
||||
def test_tool_call_streamed_markers_suppressed(self):
|
||||
events = self.post({"model": MODEL_ID, "tools": TOOLS, "stream": True,
|
||||
"messages": [{"role": "user",
|
||||
"content": "What is the weather in Milan?"}]},
|
||||
stream=True)
|
||||
deltas = [e["choices"][0]["delta"] for e in events if e["choices"]]
|
||||
text = "".join(d.get("content") or "" for d in deltas)
|
||||
self.assertNotIn("<tool_call>", text)
|
||||
self.assertNotIn("<arg_key>", text)
|
||||
calls = [d["tool_calls"] for d in deltas if d.get("tool_calls")]
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(calls[0][0]["function"]["name"], "get_weather")
|
||||
self.assertEqual(json.loads(calls[0][0]["function"]["arguments"]),
|
||||
{"city": "Milan"})
|
||||
finish = [e["choices"][0]["finish_reason"] for e in events
|
||||
if e["choices"] and e["choices"][0].get("finish_reason")]
|
||||
self.assertEqual(finish, ["tool_calls"])
|
||||
|
||||
def test_tool_result_round_trip(self):
|
||||
r = self.post({"model": MODEL_ID, "tools": TOOLS, "messages": [
|
||||
{"role": "user", "content": "What is the weather in Rome?"},
|
||||
{"role": "assistant", "content": None, "tool_calls": [
|
||||
{"id": "call_x", "type": "function",
|
||||
"function": {"name": "get_weather",
|
||||
"arguments": "{\"city\": \"Rome\"}"}}]},
|
||||
{"role": "tool", "tool_call_id": "call_x",
|
||||
"content": "25 degrees, sunny"}]})
|
||||
choice = r["choices"][0]
|
||||
self.assertEqual(choice["finish_reason"], "stop")
|
||||
self.assertFalse(choice["message"].get("tool_calls"))
|
||||
self.assertIn("25 degrees", choice["message"]["content"])
|
||||
rendered = self.mock_log.read_text().split("\x00")[-2]
|
||||
self.assertIn("<|observation|><tool_response>25 degrees, sunny</tool_response>",
|
||||
rendered)
|
||||
self.assertIn("# Tools", rendered)
|
||||
self.assertIn('"get_weather"', rendered)
|
||||
|
||||
def test_no_tools_plain_text(self):
|
||||
r = self.post({"model": MODEL_ID,
|
||||
"messages": [{"role": "user", "content": "Hi!"}]})
|
||||
choice = r["choices"][0]
|
||||
self.assertEqual(choice["finish_reason"], "stop")
|
||||
self.assertIn("mock engine", choice["message"]["content"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,39 @@
|
||||
#include "../backend_cuda.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <vector>
|
||||
|
||||
int main(){
|
||||
int dev=0;if(!coli_cuda_init(&dev,1))return 77;
|
||||
constexpr int S=3,H=2,Q=2,R=1,V=2,K=3,D=H*V,O=3,T=3;
|
||||
std::vector<float> w(H*(Q+V)*K),p(O*D),q(S*H*(Q+R));
|
||||
for(size_t i=0;i<w.size();i++)w[i]=((int)(i%11)-5)*.07f;
|
||||
for(size_t i=0;i<p.size();i++)p[i]=((int)(i%7)-3)*.09f;
|
||||
for(size_t i=0;i<q.size();i++)q[i]=((int)(i%13)-6)*.05f;
|
||||
ColiCudaTensor *tw=nullptr,*tp=nullptr;
|
||||
if(!coli_cuda_tensor_upload(&tw,w.data(),nullptr,0,K,H*(Q+V),dev)||
|
||||
!coli_cuda_tensor_upload(&tp,p.data(),nullptr,0,D,O,dev))return 1;
|
||||
int n[S]={1,2,3};std::vector<std::vector<float>> l(S),r(S);
|
||||
const float *lp[S],*rp[S];
|
||||
const void *keys[S];
|
||||
for(int s=0;s<S;s++){
|
||||
l[s].resize(n[s]*K);r[s].resize(n[s]*R);
|
||||
for(size_t i=0;i<l[s].size();i++)l[s][i]=((int)((i+s*3)%9)-4)*.08f;
|
||||
for(size_t i=0;i<r[s].size();i++)r[s][i]=((int)((i+s)%5)-2)*.06f;
|
||||
lp[s]=l[s].data();rp[s]=r[s].data();keys[s]=&l[s];
|
||||
}
|
||||
float got[S*O],ref[S*O],warm[S*O];
|
||||
int first[S]={1,1,1};
|
||||
if(!coli_cuda_attention_project_ragged(tw,tp,warm,q.data(),keys,lp,rp,first,
|
||||
S,H,Q,R,V,K,1,.2f))return 2;
|
||||
if(!coli_cuda_attention_project_ragged(tw,tp,got,q.data(),keys,lp,rp,n,S,H,Q,R,V,K,T,.2f))return 2;
|
||||
for(int s=0;s<S;s++)if(!coli_cuda_attention_project_batch(tw,tp,ref+s*O,
|
||||
q.data()+s*H*(Q+R),lp[s],rp[s],1,H,Q,R,V,K,n[s],.2f))return 3;
|
||||
double e=0,z=0;for(int i=0;i<S*O;i++){
|
||||
double d=got[i]-ref[i];e+=d*d;z+=(double)ref[i]*ref[i];
|
||||
}
|
||||
double rms=std::sqrt(e/(z+1e-30));std::printf("ragged_relative_rms=%.9g\n",rms);
|
||||
coli_cuda_tensor_free(tw);coli_cuda_tensor_free(tp);coli_cuda_shutdown();
|
||||
return rms<1e-6?0:4;
|
||||
}
|
||||
@@ -10,6 +10,7 @@ from resource_plan import (
|
||||
GB,
|
||||
analyze_model,
|
||||
build_plan,
|
||||
cpu_socket_count,
|
||||
environment_for_plan,
|
||||
format_plan,
|
||||
memory_available,
|
||||
@@ -66,15 +67,19 @@ class ResourcePlanTest(unittest.TestCase):
|
||||
# 0 slots/layer. The value must be a sane positive number of bytes.
|
||||
self.assertGreater(memory_available(), 0)
|
||||
|
||||
def test_cpu_socket_count_is_positive(self):
|
||||
self.assertGreaterEqual(cpu_socket_count(), 1)
|
||||
|
||||
def test_builds_bounded_three_tier_plan(self):
|
||||
gpus = [{"index": 0, "name": "test-gpu", "total_bytes": 12 * GB,
|
||||
"free_bytes": 10 * GB}]
|
||||
plan = build_plan(self.model, ram_gb=16, context=32, vram_gb=20,
|
||||
available_memory=32 * GB, available_disk=100 * GB, gpus=gpus,
|
||||
physical_cpus=24)
|
||||
physical_cpus=24, cpu_sockets=2)
|
||||
self.assertEqual(plan["version"], 2)
|
||||
self.assertEqual(plan["policy"]["name"], "quality")
|
||||
self.assertEqual(plan["cpu"]["physical_cores"], 24)
|
||||
self.assertEqual(plan["cpu"]["sockets"], 2)
|
||||
self.assertTrue(plan["policy"]["preserve_quantization"])
|
||||
self.assertFalse(plan["tiers"]["vram"]["requires_host_backing"])
|
||||
self.assertEqual(plan["tiers"]["ram"]["budget_bytes"], 16 * GB)
|
||||
@@ -105,7 +110,7 @@ class ResourcePlanTest(unittest.TestCase):
|
||||
{"index": 1, "name": "b", "total_bytes": 12 * GB, "free_bytes": 10 * GB},
|
||||
]
|
||||
plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB,
|
||||
available_disk=1, gpus=gpus)
|
||||
available_disk=1, gpus=gpus, cpu_sockets=2)
|
||||
env = environment_for_plan(plan, {"RAM_GB": "12", "PIN": "stats.txt",
|
||||
"COLI_GPUS": "1"})
|
||||
self.assertEqual(env["RAM_GB"], "12")
|
||||
@@ -125,6 +130,91 @@ class ResourcePlanTest(unittest.TestCase):
|
||||
"OMP_PROC_BIND": "close"})
|
||||
self.assertEqual(explicit_threads["OMP_NUM_THREADS"], "7")
|
||||
self.assertEqual(explicit_threads["OMP_PROC_BIND"], "close")
|
||||
|
||||
if sys.platform.startswith("linux"):
|
||||
self.assertEqual(env["COLI_NUMA"], "1")
|
||||
explicit_numa = environment_for_plan(plan, {"COLI_NUMA": "0"})
|
||||
self.assertEqual(explicit_numa["COLI_NUMA"], "0")
|
||||
|
||||
def test_single_socket_plan_does_not_enable_numa(self):
|
||||
plan = build_plan(self.model, available_memory=16 * GB, available_disk=1,
|
||||
gpus=[], physical_cpus=8, cpu_sockets=1)
|
||||
self.assertNotIn("COLI_NUMA", environment_for_plan(plan))
|
||||
|
||||
def test_auto_tune_mtp_off_when_compute_bound(self):
|
||||
# Tiny model with 64 GB RAM and no GPU: all experts fit in RAM with no
|
||||
# warm tier, so the plan classifies as compute-bound.
|
||||
plan = build_plan(self.model, ram_gb=64, available_memory=64 * GB,
|
||||
available_disk=100 * GB, gpus=[], physical_cpus=24,
|
||||
cpu_sockets=2)
|
||||
# With such a small model fully in RAM and no GPU, bottleneck is compute
|
||||
self.assertEqual(plan["bottleneck_class"], "compute")
|
||||
self.assertIn("DRAFT", plan["tune"])
|
||||
self.assertEqual(plan["tune"]["DRAFT"]["value"], "0")
|
||||
env = environment_for_plan(plan)
|
||||
self.assertEqual(env["DRAFT"], "0")
|
||||
explicit = environment_for_plan(plan, {"DRAFT": "3"})
|
||||
self.assertEqual(explicit["DRAFT"], "3")
|
||||
|
||||
def test_auto_tune_mtp_off_when_disk_low_hit(self):
|
||||
# Use a model large enough that 8 GB RAM can't hold all experts.
|
||||
big = tempfile.TemporaryDirectory()
|
||||
bigmodel = Path(big.name)
|
||||
(bigmodel / "config.json").write_text(json.dumps({
|
||||
"num_hidden_layers": 2, "n_routed_experts": 4,
|
||||
"kv_lora_rank": 4, "qk_rope_head_dim": 2,
|
||||
"qk_nope_head_dim": 3, "v_head_dim": 5, "num_attention_heads": 2,
|
||||
}))
|
||||
expert_size = 3 * GB # each expert 3 GB → 12 GB total, won't fit in 8 GB budget
|
||||
write_shard(bigmodel / "out-00000.safetensors", [
|
||||
("model.embed_tokens.weight", 100),
|
||||
("model.layers.0.self_attn.q_a_proj.weight", 200),
|
||||
])
|
||||
for i in range(4):
|
||||
write_shard(bigmodel / f"out-{i+1:05d}.safetensors", [
|
||||
(f"model.layers.1.mlp.experts.{i}.gate_proj.weight", expert_size),
|
||||
])
|
||||
plan = build_plan(bigmodel, ram_gb=0, available_memory=4 * GB,
|
||||
available_disk=100 * GB, gpus=[], physical_cpus=8,
|
||||
cpu_sockets=1)
|
||||
big.cleanup()
|
||||
self.assertEqual(plan["bottleneck_class"], "disk")
|
||||
self.assertLess(plan["projected_hit_rate"], 0.90)
|
||||
self.assertEqual(plan["tune"]["DRAFT"]["value"], "0")
|
||||
|
||||
def test_auto_tune_pipe_multi_gpu(self):
|
||||
gpus = [
|
||||
{"index": 0, "name": "a", "total_bytes": 32 * GB, "free_bytes": 30 * GB},
|
||||
{"index": 1, "name": "b", "total_bytes": 32 * GB, "free_bytes": 30 * GB},
|
||||
]
|
||||
plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB,
|
||||
available_disk=1, gpus=gpus, cpu_sockets=2)
|
||||
self.assertEqual(plan["tune"]["COLI_CUDA_PIPE"]["value"], "2")
|
||||
env = environment_for_plan(plan)
|
||||
self.assertEqual(env["COLI_CUDA_PIPE"], "2")
|
||||
|
||||
def test_auto_tune_pipe_single_gpu(self):
|
||||
gpus = [{"index": 0, "name": "a", "total_bytes": 12 * GB, "free_bytes": 10 * GB}]
|
||||
plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB,
|
||||
available_disk=1, gpus=gpus, cpu_sockets=1)
|
||||
self.assertEqual(plan["tune"]["COLI_CUDA_PIPE"]["value"], "1")
|
||||
|
||||
def test_auto_tune_numa_hint_for_cpu_only(self):
|
||||
plan = build_plan(self.model, ram_gb=64, available_memory=64 * GB,
|
||||
available_disk=1, gpus=[], physical_cpus=64, cpu_sockets=2)
|
||||
self.assertIn("_numa_hint", plan["tune"])
|
||||
self.assertIn("numactl", plan["tune"]["_numa_hint"])
|
||||
self.assertIn("auto-tune", format_plan(plan))
|
||||
|
||||
def test_format_plan_shows_tune_and_hit_rate(self):
|
||||
plan = build_plan(self.model, ram_gb=64, available_memory=64 * GB,
|
||||
available_disk=100 * GB, gpus=[], physical_cpus=24,
|
||||
cpu_sockets=1)
|
||||
text = format_plan(plan)
|
||||
self.assertIn("hit", text)
|
||||
self.assertIn("auto-tune", text)
|
||||
self.assertIn("DRAFT", text)
|
||||
|
||||
def test_cpu_binary_does_not_apply_gpu_tier(self):
|
||||
plan = build_plan(self.model, available_memory=16 * GB, available_disk=1,
|
||||
gpus=[{"index": 0, "name": "a", "total_bytes": 8 * GB,
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
/* Regressione #369: un solo logit NaN o +Inf faceva emettere a dist_build/dist_sample
|
||||
* il token 0 PER SEMPRE, in silenzio (il fallback `g_pbuf[i]>0` e' falso su NaN ovunque).
|
||||
* Ora la distribuzione degenere ripiega sull'argmax dei logit FINITI e avvisa una volta.
|
||||
*
|
||||
* Il test verifica: (a) con logit sani il campionamento resta corretto; (b) con NaN/+Inf
|
||||
* iniettato il token scelto e' l'argmax dei FINITI (mai 0 per default), su ogni posizione
|
||||
* del NaN inclusa lo[0]; (c) nessun NaN sopravvive in g_pbuf. */
|
||||
#define main coli_glm_main_unused
|
||||
#include "../colibri.c"
|
||||
#undef main
|
||||
#include <stdio.h>
|
||||
|
||||
static uint32_t rs=0x1234abcd; static uint32_t xr(){rs^=rs<<13;rs^=rs>>17;rs^=rs<<5;return rs;}
|
||||
|
||||
static int pbuf_has_nan(int V){ for(int i=0;i<V;i++) if(!isfinite(g_pbuf[i])) return 1; return 0; }
|
||||
|
||||
int main(void){
|
||||
int fail=0, V=2000;
|
||||
float *lo=malloc(V*sizeof(float));
|
||||
g_temp=0.7f; g_nuc=0.90f; /* il path serve di default */
|
||||
|
||||
/* (a) logit sani: il campionato deve avere prob > 0 e nessun NaN nel buffer */
|
||||
for(int i=0;i<V;i++) lo[i]=(float)((int)(xr()%2000)-1000)/100.f;
|
||||
int known=1337; lo[known]=50.f; /* picco netto */
|
||||
dist_build(lo,V);
|
||||
if(pbuf_has_nan(V)){ printf(" FAIL: NaN in g_pbuf con logit sani\n"); fail=1; }
|
||||
if(g_pbuf[known]<=0.f){ printf(" FAIL: il picco ha prob 0\n"); fail=1; }
|
||||
if(!fail) printf(" logit sani: distribuzione valida, picco vivo ok\n");
|
||||
|
||||
/* (b) NaN/+Inf iniettato in varie posizioni; il finito-argmax deve vincere */
|
||||
float bad[3]; bad[0]=NAN; bad[1]=INFINITY; bad[2]=-INFINITY;
|
||||
const char *bn[3]={"NaN","+Inf","-Inf"};
|
||||
for(int b=0;b<3;b++){
|
||||
for(int pos=0;pos<3;pos++){ /* lo[0], meta', ultimo */
|
||||
for(int i=0;i<V;i++) lo[i]=(float)((int)(xr()%400)-200)/100.f;
|
||||
int amax=777; lo[amax]=9.0f; /* massimo FINITO atteso */
|
||||
int at = pos==0?0 : pos==1?V/2 : V-1;
|
||||
if(at==amax) amax=amax+1; /* non sovrapporre */
|
||||
lo[amax]=9.0f;
|
||||
lo[at]=bad[b]; /* veleno */
|
||||
dist_build(lo,V);
|
||||
if(pbuf_has_nan(V)){ printf(" FAIL: NaN sopravvive (%s @ %d)\n",bn[b],at); fail=1; continue; }
|
||||
/* con -Inf il fallback puo' non scattare (max finito resta), ma il buffer
|
||||
* deve restare valido e sommare ~1: con +Inf/NaN scatta il delta su amax */
|
||||
int picked=-1; float pv=-1;
|
||||
for(int i=0;i<V;i++) if(g_pbuf[i]>pv){pv=g_pbuf[i];picked=i;}
|
||||
if(b<2 && picked!=amax){ /* NaN e +Inf: delta esatto su amax */
|
||||
printf(" FAIL: %s @ %d -> picked %d, atteso argmax finito %d\n",bn[b],at,picked,amax); fail=1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!fail) printf(" NaN/+Inf iniettato: argmax dei finiti vince, mai 0/NaN ok\n");
|
||||
|
||||
/* (c) caso estremo: TUTTI non finiti -> non deve crashare, buffer valido */
|
||||
for(int i=0;i<V;i++) lo[i]=NAN;
|
||||
dist_build(lo,V);
|
||||
if(pbuf_has_nan(V)){ printf(" FAIL: tutti-NaN lascia NaN nel buffer\n"); fail=1; }
|
||||
else printf(" tutti non-finiti: nessun crash, buffer valido ok\n");
|
||||
|
||||
printf(fail?"test_sample_nan: FAIL\n":"test_sample_nan: ok\n");
|
||||
return fail;
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
/* st_pread_full: chunk loop + honest truncation errors.
|
||||
* Built with -DST_PREAD_CHUNK=7 so a ~100-byte tensor takes many pread calls —
|
||||
* exercising the loop that production only needs past 2^31 bytes (one pread
|
||||
* caps there on Linux; big bf16 tensors exceed it). Also forks a child against
|
||||
* a truncated shard and requires exit(1) with a "short read" message instead
|
||||
* of the old perror("... : Success"). */
|
||||
#define _GNU_SOURCE
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#ifndef _WIN32
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#include "../st.h"
|
||||
|
||||
#define CHECK(condition) do { \
|
||||
if (!(condition)) { \
|
||||
fprintf(stderr, "%s:%d: check failed: %s\n", __FILE__, __LINE__, #condition); \
|
||||
return 1; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
static void write_snap(const char *dir, int truncate_bytes) {
|
||||
char path[512];
|
||||
snprintf(path, sizeof(path), "%s/model.safetensors", dir);
|
||||
unsigned char data[96];
|
||||
for (int i = 0; i < 96; i++) data[i] = (unsigned char)(i * 7 + 3);
|
||||
const char *hdr = "{\"t\":{\"dtype\":\"U8\",\"shape\":[96],\"data_offsets\":[0,96]}}";
|
||||
uint64_t hlen = strlen(hdr);
|
||||
FILE *f = fopen(path, "wb");
|
||||
fwrite(&hlen, 8, 1, f);
|
||||
fwrite(hdr, 1, hlen, f);
|
||||
fwrite(data, 1, (size_t)(96 - truncate_bytes), f);
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
/* relative to the CWD, per test_stops: MinGW .exe files resolve Windows
|
||||
* paths and "/tmp" is not one */
|
||||
char dir[] = "test_st_pread_XXXXXX";
|
||||
if (!mkdtemp(dir)) { perror("mkdtemp"); return 1; }
|
||||
|
||||
/* 1) chunk loop: 96-byte tensor read 7 bytes at a time, content exact */
|
||||
write_snap(dir, 0);
|
||||
shards S; st_init(&S, dir);
|
||||
unsigned char out[96] = {0};
|
||||
st_read_raw(&S, "t", out, 0);
|
||||
for (int i = 0; i < 96; i++) CHECK(out[i] == (unsigned char)(i * 7 + 3));
|
||||
|
||||
#ifndef _WIN32
|
||||
/* 2) shard truncated AFTER st_init (init validates static bounds, so the
|
||||
* pread path only fires when the file shrinks underneath a live handle):
|
||||
* child must exit(1) with an honest message, not perror's "Success" */
|
||||
char shard[512]; snprintf(shard, sizeof(shard), "%s/model.safetensors", dir);
|
||||
struct stat sb; CHECK(stat(shard, &sb) == 0);
|
||||
CHECK(truncate(shard, sb.st_size - 40) == 0);
|
||||
int pipefd[2]; CHECK(pipe(pipefd) == 0);
|
||||
pid_t pid = fork(); CHECK(pid >= 0);
|
||||
if (pid == 0) {
|
||||
dup2(pipefd[1], 2); close(pipefd[0]); close(pipefd[1]);
|
||||
unsigned char buf[96];
|
||||
st_read_raw(&S, "t", buf, 0); /* inherited handles; must exit(1) inside */
|
||||
_exit(42); /* reaching here = bug */
|
||||
}
|
||||
close(pipefd[1]);
|
||||
char err[512] = {0};
|
||||
ssize_t n = read(pipefd[0], err, sizeof(err)-1); (void)n;
|
||||
close(pipefd[0]);
|
||||
int status = 0; waitpid(pid, &status, 0);
|
||||
CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 1);
|
||||
CHECK(strstr(err, "short read") != NULL);
|
||||
CHECK(strstr(err, "Success") == NULL);
|
||||
#else
|
||||
/* fork/pipe/truncate are POSIX; Windows still runs the chunk-loop check */
|
||||
printf("test_st_pread: truncation subtest skipped on Windows\n");
|
||||
#endif
|
||||
|
||||
char cmd[600];
|
||||
#ifdef _WIN32
|
||||
snprintf(cmd, sizeof(cmd), "rmdir /s /q %s", dir);
|
||||
#else
|
||||
snprintf(cmd, sizeof(cmd), "rm -rf %s", dir);
|
||||
#endif
|
||||
if (system(cmd)) {}
|
||||
printf("test_st_pread: chunk loop + honest truncation error: ok\n");
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
/* Stop-token arming: the engine must not depend on a converted checkpoint's
|
||||
* config.json being complete.
|
||||
*
|
||||
* Why this test exists (#298, #307): GLM-5.2 declares THREE eos ids
|
||||
* (<|endoftext|>, <|user|>, <|observation|>). A conversion tool that rewrites
|
||||
* config.json with a reduced eos_token_id list leaves the engine stopping on
|
||||
* fewer tokens than the model actually emits -- and the ones it misses get
|
||||
* DETOKENIZED AND PRINTED INTO THE CHAT as literal text, with generation
|
||||
* continuing past the real end of the turn. @woolcoxm hit exactly this on a
|
||||
* g64 checkpoint and confirmed it by comparing token ids.
|
||||
*
|
||||
* Two independent defenses, one test each:
|
||||
* 1. eos_token_id is unioned from generation_config.json (HuggingFace's
|
||||
* authority for generation) as well as config.json.
|
||||
* 2. every added-token the TOKENIZER marks "special":true is a stop, whatever
|
||||
* the configs say. Those are control tokens (<|user|>, <sop>, ...) and are
|
||||
* never legitimate content. <think>/<tool_call> are "special":false and
|
||||
* must NOT be swept up -- they are real output.
|
||||
*
|
||||
* Defense 2 is what makes this robust against checkpoints we don't control:
|
||||
* even with BOTH configs mutilated, a control token cannot leak into a reply. */
|
||||
#define main coli_glm_main_unused
|
||||
#include "../colibri.c"
|
||||
#undef main
|
||||
|
||||
static const char *TOKJSON =
|
||||
"{\"model\":{\"vocab\":{\"a\":0,\"b\":1,\"c\":2},\"merges\":[[\"a\",\"b\"]]},"
|
||||
" \"added_tokens\":["
|
||||
" {\"id\":100,\"content\":\"<|endoftext|>\",\"special\":true},"
|
||||
" {\"id\":101,\"content\":\"<|user|>\",\"special\":true},"
|
||||
" {\"id\":102,\"content\":\"<|observation|>\",\"special\":true},"
|
||||
" {\"id\":103,\"content\":\"<|assistant|>\",\"special\":true},"
|
||||
" {\"id\":104,\"content\":\"<sop>\",\"special\":true},"
|
||||
" {\"id\":110,\"content\":\"<think>\",\"special\":false},"
|
||||
" {\"id\":111,\"content\":\"<tool_call>\",\"special\":false}"
|
||||
"]}";
|
||||
|
||||
/* minimal config.json that survives load_cfg's range validation */
|
||||
static void write_cfg(const char *dir, const char *fname, const char *eos_json){
|
||||
char p[512]; snprintf(p,sizeof(p),"%s/%s",dir,fname);
|
||||
FILE *f=fopen(p,"w"); if(!f){ perror(p); exit(1); }
|
||||
fprintf(f,"{\"hidden_size\":64,\"num_hidden_layers\":2,\"num_attention_heads\":4,"
|
||||
"\"n_routed_experts\":8,\"num_experts_per_tok\":2,\"moe_intermediate_size\":32,"
|
||||
"\"intermediate_size\":64,\"first_k_dense_replace\":1,\"q_lora_rank\":0,"
|
||||
"\"kv_lora_rank\":16,\"qk_nope_head_dim\":8,\"qk_rope_head_dim\":8,"
|
||||
"\"v_head_dim\":8,\"n_shared_experts\":1,\"vocab_size\":200,"
|
||||
"\"n_group\":1,\"topk_group\":1,\"rope_theta\":10000.0");
|
||||
if(eos_json) fprintf(f,",\"eos_token_id\":%s",eos_json);
|
||||
fprintf(f,"}\n"); fclose(f);
|
||||
}
|
||||
static void write_tok(const char *dir){
|
||||
char p[512]; snprintf(p,sizeof(p),"%s/tokenizer.json",dir);
|
||||
FILE *f=fopen(p,"w"); if(!f){ perror(p); exit(1); }
|
||||
fputs(TOKJSON,f); fclose(f);
|
||||
}
|
||||
static void rm_file(const char *dir, const char *fname){
|
||||
char p[512]; snprintf(p,sizeof(p),"%s/%s",dir,fname); remove(p);
|
||||
}
|
||||
|
||||
static int expect(const char *what, int id, int want){
|
||||
int got=is_stop(id);
|
||||
if(got!=want){ fprintf(stderr," FAIL %s: token %d stop=%d, expected %d\n",what,id,got,want); return 1; }
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(void){
|
||||
int fail=0;
|
||||
/* Relative to the CWD, like test_compat_direct's TMPF — NOT "/tmp/...".
|
||||
* These binaries are built by MinGW into native Windows .exe files, which
|
||||
* resolve Windows paths: "/tmp" is not one, so mkdtemp there fails ENOENT
|
||||
* and the whole `make check` goes red on the windows job (and only there). */
|
||||
char dir[]="test_stops_XXXXXX";
|
||||
if(!mkdtemp(dir)){ perror("mkdtemp"); return 1; }
|
||||
write_tok(dir);
|
||||
char tkp[512]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",dir);
|
||||
Tok T; tok_load(&T,tkp);
|
||||
printf("test_stops: stop arming vs incomplete checkpoint metadata\n");
|
||||
|
||||
/* 1. the tokenizer's "special" flag must survive loading, and must NOT
|
||||
* swallow <think>/<tool_call>, which are real content. */
|
||||
if(!T.id_special[101]){ fprintf(stderr," FAIL <|user|> not marked special\n"); fail=1; }
|
||||
if(!T.id_special[104]){ fprintf(stderr," FAIL <sop> not marked special\n"); fail=1; }
|
||||
if(T.id_special[110]){ fprintf(stderr," FAIL <think> wrongly marked special\n"); fail=1; }
|
||||
if(T.id_special[111]){ fprintf(stderr," FAIL <tool_call> wrongly marked special\n"); fail=1; }
|
||||
if(!fail) printf(" tokenizer: special flag parsed, <think>/<tool_call> excluded ok\n");
|
||||
|
||||
/* 2. config.json mutilated to one eos, generation_config.json intact:
|
||||
* the union must recover all three. This is @woolcoxm's case. */
|
||||
{ Cfg c; memset(&c,0,sizeof c);
|
||||
write_cfg(dir,"config.json","[100]");
|
||||
write_cfg(dir,"generation_config.json","[100,101,102]");
|
||||
load_cfg(&c,dir);
|
||||
if(c.n_stop!=3){ fprintf(stderr," FAIL union: expected 3 stops, got %d\n",c.n_stop); fail=1; }
|
||||
else printf(" config eos=[100] + generation_config=[100,101,102] -> 3 stops ok\n"); }
|
||||
|
||||
/* 3. no generation_config.json at all: must not crash, config alone wins */
|
||||
{ Cfg c; memset(&c,0,sizeof c);
|
||||
write_cfg(dir,"config.json","[100,101,102]");
|
||||
rm_file(dir,"generation_config.json");
|
||||
load_cfg(&c,dir);
|
||||
if(c.n_stop!=3){ fprintf(stderr," FAIL no-genconfig: expected 3, got %d\n",c.n_stop); fail=1; }
|
||||
else printf(" generation_config.json absent -> config alone, no crash ok\n"); }
|
||||
|
||||
/* 4. BOTH configs mutilated: the tokenizer's special set must still stop the
|
||||
* turn on every control token, and must still leave <think> alone. */
|
||||
{ Cfg c; memset(&c,0,sizeof c);
|
||||
write_cfg(dir,"config.json","[100]");
|
||||
write_cfg(dir,"generation_config.json","[100]");
|
||||
load_cfg(&c,dir);
|
||||
stops_arm_tok(&c,-1,&T);
|
||||
fail|=expect("endoftext (config)", 100,1);
|
||||
fail|=expect("<|user|> (tokenizer)", 101,1);
|
||||
fail|=expect("<|observation|>", 102,1);
|
||||
fail|=expect("<|assistant|>", 103,1);
|
||||
fail|=expect("<sop>", 104,1);
|
||||
fail|=expect("<think> must NOT stop",110,0);
|
||||
fail|=expect("<tool_call> must NOT stop",111,0);
|
||||
fail|=expect("plain token must NOT stop",2,0);
|
||||
if(!fail) printf(" both configs mutilated -> tokenizer still stops all 5 control tokens ok\n"); }
|
||||
|
||||
/* 5. T=NULL (validation/oracle path): config stops only, unchanged behaviour */
|
||||
{ Cfg c; memset(&c,0,sizeof c);
|
||||
write_cfg(dir,"config.json","[100]");
|
||||
rm_file(dir,"generation_config.json");
|
||||
load_cfg(&c,dir);
|
||||
stops_arm_tok(&c,-1,NULL);
|
||||
fail|=expect("T=NULL: config stop",100,1);
|
||||
fail|=expect("T=NULL: no tokenizer sweep",101,0);
|
||||
if(!fail) printf(" T=NULL -> config stops only (validation path untouched) ok\n"); }
|
||||
|
||||
rm_file(dir,"config.json"); rm_file(dir,"generation_config.json"); rm_file(dir,"tokenizer.json");
|
||||
rmdir(dir);
|
||||
if(fail){ printf("test_stops: FAIL\n"); return 1; }
|
||||
printf("test_stops: ok\n");
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
/* Top-p (nucleus) truncation in dist_build: the partial-select rewrite (#335) must be
|
||||
* indistinguishable from the old full-vocab qsort for every shape dist_sample can see.
|
||||
*
|
||||
* Why this test exists (#335): dist_build() previously qsort-ed the entire 151936-entry
|
||||
* vocab on every sampled token to find the few-hundred-token head whose cumulative mass
|
||||
* reaches g_nuc. It now heapifies (O(V)) and pops only the head (k * O(log V)). The win
|
||||
* is structural; the risk is a silent sampling-distribution change, because the contract
|
||||
* is subtle:
|
||||
*
|
||||
* dist_sample() iterates g_pbuf[0..V-1] BY TOKEN ID and sums probabilities directly.
|
||||
* So dist_build MUST leave g_pbuf indexed by id (never reordered) AND must zero every
|
||||
* truncated tail entry -- merely excluding the tail from the head would leave mass on
|
||||
* it and the sampled distribution would drift with no crash and no error.
|
||||
*
|
||||
* Strategy: drive the REAL dist_build (via the test_stops.c include-glm.c pattern) on a
|
||||
* sweep of distributions and g_nuc values, and compare against an INDEPENDENT reference
|
||||
* that re-implements the OLD algorithm (full qsort + zero-tail + renorm) in double on a
|
||||
* private buffer. On shapes with no ties the renormalized head must be BIT-IDENTICAL to
|
||||
* the reference (the issue's stated invariant: s2 accumulates in the same descending
|
||||
* order). On tie shapes, where the unstable qsort already left ordering unspecified, we
|
||||
* check multiset equality instead. Every shape also checks: exact-zero tails, head sums
|
||||
* to 1.0, and a sane keep-count.
|
||||
*
|
||||
* No scratch files: the test runs entirely in memory (no mkdtemp), so it builds clean on
|
||||
* the Windows MinGW CI job without the unmerged compat shim (#352). */
|
||||
#define main coli_glm_main_unused
|
||||
#include "../colibri.c"
|
||||
#undef main
|
||||
|
||||
#include <math.h>
|
||||
|
||||
static int g_nfails = 0;
|
||||
|
||||
/* pointer set by ref_build so cmp_ref_desc can read the current reference buffer
|
||||
* (the qsort comparator gets no user-data argument in C). */
|
||||
static const double *g_ref_p = NULL;
|
||||
|
||||
#define FAIL(fmt, ...) do { \
|
||||
fprintf(stderr, " FAIL [%s V=%d nuc=%.3f shape=%s]: " fmt "\n", \
|
||||
label, V, nuc, shape_name, ##__VA_ARGS__); \
|
||||
g_nfails++; \
|
||||
return; \
|
||||
} while (0)
|
||||
|
||||
/* ---- independent reference: the OLD algorithm, in double, on a private buffer ------- */
|
||||
/* Stable qsort by descending probability (ties broken by ascending index, which makes
|
||||
* the reference deterministic regardless of the production comparator). */
|
||||
static int cmp_ref_desc(const void *a, const void *b){
|
||||
double pa = ((const double *)g_ref_p)[*(const int*)a];
|
||||
double pb = ((const double *)g_ref_p)[*(const int*)b];
|
||||
if (pa < pb) return 1;
|
||||
if (pa > pb) return -1;
|
||||
/* tie -> lower index first (stable, unlike the production comparator) */
|
||||
return *(const int*)a - *(const int*)b;
|
||||
}
|
||||
|
||||
/* Build the reference distribution into out[0..V-1] (indexed by token id), mirroring the
|
||||
* old dist_build: softmax(lo/temp) truncated to top-p nuc, tail zeroed, head renormalized.
|
||||
* Returns the keep-count through *keep_out. */
|
||||
static void ref_build(const float *lo, int V, double temp, double nuc,
|
||||
double *out, int *pidx, int *keep_out){
|
||||
double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i];
|
||||
double s = 0, invt = 1.0 / (temp > 1e-4 ? temp : 1e-4);
|
||||
for (int i = 0; i < V; i++){ out[i] = exp((lo[i]-mx)*invt); s += out[i]; }
|
||||
for (int i = 0; i < V; i++) out[i] /= s;
|
||||
|
||||
if (nuc > 0 && nuc < 1.0){
|
||||
for (int i = 0; i < V; i++) pidx[i] = i;
|
||||
qsort(pidx, V, sizeof(int), cmp_ref_desc);
|
||||
double cum = 0; int keep = V;
|
||||
for (int i = 0; i < V; i++){ cum += out[pidx[i]]; if (cum >= nuc){ keep = i+1; break; } }
|
||||
double s2 = 0;
|
||||
for (int i = keep; i < V; i++) out[pidx[i]] = 0;
|
||||
for (int i = 0; i < keep; i++) s2 += out[pidx[i]];
|
||||
for (int i = 0; i < keep; i++) out[pidx[i]] /= s2;
|
||||
*keep_out = keep;
|
||||
} else {
|
||||
*keep_out = V;
|
||||
}
|
||||
}
|
||||
|
||||
/* count how many production g_pbuf entries are non-zero == the head size */
|
||||
static int head_count(int V){
|
||||
int n = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) n++; return n;
|
||||
}
|
||||
|
||||
/* Run one case: load logits into g_pbuf via the real dist_build, compare to reference.
|
||||
* shape_name is for diagnostics only. */
|
||||
static void check_case(const char *label, int V, double nuc, const char *shape_name,
|
||||
const float *lo){
|
||||
/* reference on a private buffer */
|
||||
double *ref = malloc((size_t)V * sizeof(double));
|
||||
int *ridx = malloc((size_t)V * sizeof(int));
|
||||
int ref_keep = 0;
|
||||
g_ref_p = ref; /* cmp_ref_desc reads this */
|
||||
ref_build(lo, V, g_temp, nuc, ref, ridx, &ref_keep);
|
||||
|
||||
/* production: drive the real dist_build (writes the global g_pbuf) */
|
||||
g_nuc = (float)nuc;
|
||||
dist_build(lo, V);
|
||||
|
||||
int got_keep = head_count(V);
|
||||
|
||||
/* 1. keep-count must match the reference exactly. The partial select and the old
|
||||
* qsort keep the same NUMBER of tokens by construction (same cumulative-mass rule);
|
||||
* a count divergence is a real bug, not a tie artifact. */
|
||||
if (got_keep != ref_keep)
|
||||
FAIL("keep-count mismatch: got %d, ref %d", got_keep, ref_keep);
|
||||
|
||||
/* 2. Detect ties across the WHOLE pre-truncation distribution, not just the kept set.
|
||||
* A tie at the head/tail boundary makes which-side-a-token-lands-on interchangeable:
|
||||
* both algorithms keep the right count but may keep different MEMBERS. So any input
|
||||
* with a duplicated softmax value needs the relaxed multiset comparison below. We
|
||||
* detect this on the reference softmax (pre-truncation) by sorting all V values. */
|
||||
int has_ties = 0;
|
||||
{
|
||||
double *all = malloc((size_t)V * sizeof(double));
|
||||
/* reconstruct the pre-truncation softmax the same way ref_build does */
|
||||
double mx = lo[0]; for (int i = 1; i < V; i++) if (lo[i] > mx) mx = lo[i];
|
||||
double s = 0, invt = 1.0 / (g_temp > 1e-4 ? g_temp : 1e-4);
|
||||
for (int i = 0; i < V; i++){ all[i] = exp((lo[i]-mx)*invt); s += all[i]; }
|
||||
for (int i = 0; i < V; i++) all[i] /= s;
|
||||
for (int a = 1; a < V; a++){ double k = all[a]; int b = a-1;
|
||||
while (b >= 0 && all[b] > k){ all[b+1] = all[b]; b--; } all[b+1] = k; }
|
||||
for (int a = 1; a < V; a++) if (all[a] == all[a-1]){ has_ties = 1; break; }
|
||||
free(all);
|
||||
}
|
||||
|
||||
if (has_ties){
|
||||
/* Multiset equality of the non-zero (head) values. Ties make membership
|
||||
* interchangeable, so we compare sorted value-multisets, not id-aligned values.
|
||||
* Tolerance is 1e-6 relative -- the engine uses float arithmetic, the reference
|
||||
* double, so sub-ULP noise is expected (matches test_i4_grouped.c's convention). */
|
||||
double *got = malloc((size_t)ref_keep * sizeof(double));
|
||||
int gm = 0;
|
||||
for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) got[gm++] = (double)g_pbuf[i];
|
||||
if (gm != ref_keep)
|
||||
FAIL("tie-shape head size mismatch: got %d non-zero, ref %d", gm, ref_keep);
|
||||
for (int a = 1; a < gm; a++){ double k = got[a]; int b = a-1;
|
||||
while (b >= 0 && got[b] > k){ got[b+1] = got[b]; b--; } got[b+1] = k; }
|
||||
double *rsort = malloc((size_t)ref_keep * sizeof(double));
|
||||
int rm = 0;
|
||||
for (int i = 0; i < V; i++) if (ref[i] != 0.0) rsort[rm++] = ref[i];
|
||||
for (int a = 1; a < rm; a++){ double k = rsort[a]; int b = a-1;
|
||||
while (b >= 0 && rsort[b] > k){ rsort[b+1] = rsort[b]; b--; } rsort[b+1] = k; }
|
||||
int mm = 0; double worst = 0;
|
||||
for (int i = 0; i < gm; i++){
|
||||
double d = fabs(got[i] - rsort[i]);
|
||||
double rel = rsort[i] > 1e-30 ? d / rsort[i] : d;
|
||||
if (rel > worst) worst = rel;
|
||||
if (rel > 1e-6) mm++;
|
||||
}
|
||||
free(got); free(rsort);
|
||||
if (mm) FAIL("tie-shape multiset mismatch: %d/%d head values differ beyond 1e-6 rel (worst %.3g)",
|
||||
mm, ref_keep, worst);
|
||||
} else {
|
||||
/* No ties anywhere: membership is forced, so compare id-aligned head values. The
|
||||
* engine computes in float (g_pbuf /= (float)s2) while the reference uses double,
|
||||
* so the comparison is relative-tolerance (1e-6), not bit-exact -- the partial
|
||||
* select and qsort accumulate s2 in the same descending order, so any difference
|
||||
* is pure float-rounding noise, not an ordering bug. */
|
||||
int bad = 0; int first_id = -1; float gv = 0, rv = 0; double worst = 0;
|
||||
for (int i = 0; i < V; i++){
|
||||
if (ref[i] == 0.0) continue; /* tail */
|
||||
float want = (float)ref[i];
|
||||
double d = fabs((double)g_pbuf[i] - (double)want);
|
||||
double rel = fabs((double)want) > 1e-30 ? d / fabs((double)want) : d;
|
||||
if (rel > worst) worst = rel;
|
||||
if (rel > 1e-6){
|
||||
bad++; if (first_id < 0){ first_id = i; gv = g_pbuf[i]; rv = want; }
|
||||
if (bad > 3) break;
|
||||
}
|
||||
}
|
||||
if (bad)
|
||||
FAIL("head not within 1e-6 rel of reference: %d entries differ (first id %d: got %.9g want %.9g, worst %.3g)",
|
||||
bad, first_id, (double)gv, (double)rv, worst);
|
||||
}
|
||||
|
||||
/* 3. head must renormalize to 1.0 (within float epsilon) */
|
||||
double sum = 0; for (int i = 0; i < V; i++) sum += g_pbuf[i];
|
||||
if (fabs(sum - 1.0) > 1e-5)
|
||||
FAIL("head does not sum to 1.0: sum=%.12g (keep=%d)", sum, got_keep);
|
||||
|
||||
free(ref); free(ridx);
|
||||
printf(" ok [V=%d nuc=%.3f shape=%s keep=%d%s sum=%.10f]\n",
|
||||
V, nuc, shape_name, got_keep, has_ties ? " (ties)" : "", sum);
|
||||
}
|
||||
#undef FAIL
|
||||
|
||||
/* deterministic xorshift32 RNG (matches the test_i4_grouped.c convention) */
|
||||
static uint32_t rng_state = 0x12345678u;
|
||||
static uint32_t xr(void){ rng_state ^= rng_state << 13; rng_state ^= rng_state >> 17;
|
||||
rng_state ^= rng_state << 5; return rng_state; }
|
||||
static double frand(void){ return (xr() >> 8) * (1.0 / 16777216.0); } /* [0,1) */
|
||||
|
||||
/* fill logits for a given shape. Shapes chosen to stress the comparator and the head/tail
|
||||
* boundary differently. */
|
||||
static void fill_shape(float *lo, int V, int shape){
|
||||
switch (shape){
|
||||
case 0: /* uniform -> every token equal probability -> massive tie plateau */
|
||||
for (int i = 0; i < V; i++) lo[i] = 0.f; break;
|
||||
case 1: /* peaked: one dominant token, rest small and distinct (no ties).
|
||||
* The fixed hot-spots are clamped to V-1 so small V (incl. V=1) doesn't
|
||||
* write out of bounds and corrupt heap metadata on the later free(lo). */
|
||||
for (int i = 0; i < V; i++) lo[i] = (float)(-1.0 - frand()*4.0);
|
||||
lo[0] = 3.f; lo[V/3<V?V/3:V-1] = 1.f; lo[V/2<V?V/2:V-1] = 0.5f; break;
|
||||
case 2: /* all-equal distinct decay (no ties): geometric, strictly decreasing */
|
||||
for (int i = 0; i < V; i++) lo[i] = (float)(-0.001 * (double)i); break;
|
||||
case 3: /* plateau ties: blocks of equal value -> comparator tie handling */
|
||||
for (int i = 0; i < V; i++) lo[i] = (float)(-(double)(i / 7)); /* 7-wide plateaus */
|
||||
break;
|
||||
case 4: /* sharp-tail: a few hot, then a long flat floor (small tie at the floor).
|
||||
* Hot count is min(12,V) so V<12 (incl. V=1) stays in bounds. */
|
||||
for (int i = 0; i < V; i++) lo[i] = -8.f;
|
||||
{ int hot = V<12 ? V : 12; for (int i = 0; i < hot; i++) lo[i] = (float)(2.0 - frand()); } break;
|
||||
}
|
||||
}
|
||||
|
||||
int main(void){
|
||||
/* sizes: small for exhaustive tie detection up to near-production scale */
|
||||
int sizes[] = {1, 2, 8, 64, 257, 1519}; /* 1519 ~= V/100 of GLM-5.2 */
|
||||
double nucs[] = {0.001, 0.5, 0.9, 0.999}; /* tight -> almost-everything */
|
||||
int n_shapes = 5;
|
||||
|
||||
/* temperature used by dist_build: pick a normal serving value */
|
||||
g_temp = 0.7f;
|
||||
|
||||
int cases = 0;
|
||||
for (size_t si = 0; si < sizeof(sizes)/sizeof(sizes[0]); si++){
|
||||
int V = sizes[si];
|
||||
/* dist_build allocates g_pbuf/g_pidx ONCE and reuses them (single-V invariant in
|
||||
* real serving, where V is the constant model vocab). This sweep varies V, so free
|
||||
* and force a reallocation per size -- otherwise a later, larger V would overflow
|
||||
* the buffer sized for the first (smallest) V. */
|
||||
free(g_pbuf); g_pbuf = NULL; free(g_pidx); g_pidx = NULL;
|
||||
float *lo = malloc((size_t)V * sizeof(float));
|
||||
for (int shape = 0; shape < n_shapes; shape++){
|
||||
fill_shape(lo, V, shape);
|
||||
for (size_t ni = 0; ni < sizeof(nucs)/sizeof(nucs[0]); ni++){
|
||||
char label[32]; snprintf(label, sizeof(label), "size[%zu]/shape[%d]", si, shape);
|
||||
const char *sn = (const char*[]){"uniform","peaked","geometric","plateau","sharptail"}[shape];
|
||||
check_case(label, V, nucs[ni], sn, lo);
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
free(lo);
|
||||
}
|
||||
|
||||
/* guard-off path: g_nuc >= 1 must skip truncation entirely (full softmax kept) */
|
||||
{
|
||||
int V = 256; float lo[256];
|
||||
for (int i = 0; i < V; i++) lo[i] = (float)(frand()*4 - 2);
|
||||
g_nuc = 1.0f; dist_build(lo, V);
|
||||
int nz = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) nz++;
|
||||
if (nz != V){ fprintf(stderr, " FAIL [guard-off nuc=1.0]: %d/%d entries kept, expected all\n", nz, V); g_nfails++; }
|
||||
else printf(" ok [guard-off nuc=1.0 keep=%d]\n", nz);
|
||||
cases++;
|
||||
|
||||
g_nuc = 0.0f; dist_build(lo, V);
|
||||
nz = 0; for (int i = 0; i < V; i++) if (g_pbuf[i] != 0.f) nz++;
|
||||
if (nz != V){ fprintf(stderr, " FAIL [guard-off nuc=0.0]: %d/%d entries kept, expected all\n", nz, V); g_nfails++; }
|
||||
else printf(" ok [guard-off nuc=0.0 keep=%d]\n", nz);
|
||||
cases++;
|
||||
}
|
||||
|
||||
/* extreme tie edge case: V=1, single token -> keep=1 regardless of nuc */
|
||||
{
|
||||
float lo[1] = {5.f};
|
||||
g_nuc = 0.5f; dist_build(lo, 1);
|
||||
if (g_pbuf[0] == 0.f || !(fabs((double)g_pbuf[0] - 1.0) < 1e-6)){
|
||||
fprintf(stderr, " FAIL [V=1]: g_pbuf[0]=%.9g, expected 1.0\n", (double)g_pbuf[0]); g_nfails++;
|
||||
} else printf(" ok [V=1 keep=1]\n");
|
||||
cases++;
|
||||
}
|
||||
|
||||
printf("\ntest_topp: %d cases run, %d failure(s)\n", cases, g_nfails);
|
||||
if (g_nfails){ printf("test_topp: FAIL\n"); return 1; }
|
||||
printf("test_topp: ok\n");
|
||||
return 0;
|
||||
}
|
||||
@@ -6,7 +6,7 @@
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#define main coli_glm_main_unused
|
||||
#include "../glm.c"
|
||||
#include "../colibri.c"
|
||||
#undef main
|
||||
|
||||
static int fail(const char *s){ fprintf(stderr,"FAIL: %s\n",s); return 1; }
|
||||
|
||||
@@ -42,6 +42,11 @@ typedef struct {
|
||||
hmap vocab; /* stringa byte-level -> id */
|
||||
hmap merges; /* "left\0right" -> rank */
|
||||
char **id2str; int *id_added; int n_ids; /* id -> stringa; id_added=1 se added-token (output letterale) */
|
||||
int *id_special; /* 1 = added-token con "special":true nel tokenizer:
|
||||
* token di CONTROLLO (<|user|>, <|assistant|>, <sop>, ...),
|
||||
* mai contenuto legittimo di una risposta. Distinto da
|
||||
* id_added, che copre anche <think>/<tool_call> ("special"
|
||||
* false), i quali sono testo vero e vanno renderizzati. */
|
||||
Special *sp; int nsp; /* added tokens, ordinati per lunghezza decrescente */
|
||||
uint32_t byte2cp[256]; int byte2cp_len[256]; char byte2str[256][3];
|
||||
int16_t cp2byte[1024];
|
||||
@@ -106,6 +111,7 @@ static void tok_load(Tok *T, const char *path){
|
||||
T->n_ids=maxid+1;
|
||||
T->id2str=calloc(T->n_ids,sizeof(char*));
|
||||
T->id_added=calloc(T->n_ids,sizeof(int));
|
||||
T->id_special=calloc(T->n_ids,sizeof(int));
|
||||
|
||||
/* vocab: stringa -> id (capacita' potenza di 2, ~2-3x) */
|
||||
int vc=1; while(vc < vocab->len*2) vc<<=1;
|
||||
@@ -133,6 +139,8 @@ static void tok_load(Tok *T, const char *path){
|
||||
char *content=json_get(a,"content")->str; int id=(int)json_get(a,"id")->num;
|
||||
T->sp[i].str=content; T->sp[i].len=(int)strlen(content); T->sp[i].id=id;
|
||||
T->id2str[id]=content; T->id_added[id]=1;
|
||||
jval *sf=json_get(a,"special"); /* "special": true/false */
|
||||
if(sf && sf->t==J_BOOL && sf->boolean) T->id_special[id]=1;
|
||||
}
|
||||
qsort(T->sp,T->nsp,sizeof(Special),cmp_sp_len); /* match piu' lungo per primo */
|
||||
}
|
||||
|
||||
@@ -17,6 +17,12 @@ PROFILE_RE = re.compile(
|
||||
r"\| attention ([0-9.]+)s .* lm_head ([0-9.]+)s \| other ([0-9.-]+)s"
|
||||
)
|
||||
PROFILE_KEYS = ("disk", "expert_matmul", "attention", "lm_head", "other")
|
||||
P0_RE = re.compile(
|
||||
r"P0-EXEC: routed CPU ([0-9.]+)s / ([0-9.]+) GB/s \(([0-9]+) row\) \| routed GPU critical ([0-9.]+)s \| "
|
||||
r"router ([0-9.]+)s \| residual P2P ([0-9.]+)s / ([0-9]+) hop \| orchestration ([0-9.]+)s"
|
||||
)
|
||||
P0_KEYS = ("routed_cpu", "routed_cpu_gb_s", "routed_cpu_rows", "routed_gpu_critical",
|
||||
"router", "p2p", "p2p_hops", "orchestration")
|
||||
|
||||
|
||||
def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]:
|
||||
@@ -30,11 +36,20 @@ def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]:
|
||||
return float(speed.group(1)), [disk] + [float(value) for value in rest]
|
||||
|
||||
|
||||
def execute(engine: str, env: dict[str, str]) -> tuple[float, list[float]]:
|
||||
def parse_p0(stdout: str) -> list[float]:
|
||||
"""Extract the optional PROF=1 execution-layer breakdown."""
|
||||
row = P0_RE.search(stdout)
|
||||
if not row:
|
||||
raise RuntimeError("benchmark output missing P0-EXEC profile")
|
||||
return [float(value) for value in row.groups()]
|
||||
|
||||
|
||||
def execute(engine: str, env: dict[str, str]) -> tuple[float, list[float], list[float]]:
|
||||
run = subprocess.run(
|
||||
[engine, "4", "4", "4"], env=env, text=True, capture_output=True, check=True
|
||||
)
|
||||
return parse_output(run.stdout, run.stderr)
|
||||
speed, profile = parse_output(run.stdout, run.stderr)
|
||||
return speed, profile, parse_p0(run.stdout)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
@@ -63,6 +78,8 @@ def main() -> None:
|
||||
OMP_NUM_THREADS=str(args.threads),
|
||||
OMP_PROC_BIND="spread",
|
||||
OMP_PLACES="cores",
|
||||
DRAFT="0",
|
||||
PROF="1",
|
||||
)
|
||||
|
||||
execute(args.engine, base | {"STATS": str(stats)})
|
||||
@@ -86,13 +103,15 @@ def main() -> None:
|
||||
execute(args.engine, base | extra) # warm-up
|
||||
speeds = {name: [] for name in modes}
|
||||
profiles = {name: [] for name in modes}
|
||||
p0_profiles = {name: [] for name in modes}
|
||||
names = list(modes)
|
||||
for run_index in range(args.runs):
|
||||
order = names[run_index % len(names):] + names[:run_index % len(names)]
|
||||
for name in order:
|
||||
speed, profile = execute(args.engine, base | modes[name])
|
||||
speed, profile, p0 = execute(args.engine, base | modes[name])
|
||||
speeds[name].append(speed)
|
||||
profiles[name].append(profile)
|
||||
p0_profiles[name].append(p0)
|
||||
|
||||
result = {}
|
||||
for name in names:
|
||||
@@ -103,6 +122,10 @@ def main() -> None:
|
||||
key: statistics.median(row[index] for row in profiles[name])
|
||||
for index, key in enumerate(PROFILE_KEYS)
|
||||
},
|
||||
"median_p0": {
|
||||
key: statistics.median(row[index] for row in p0_profiles[name])
|
||||
for index, key in enumerate(P0_KEYS)
|
||||
},
|
||||
}
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
@@ -286,6 +286,17 @@ def main():
|
||||
# testa MTP a int4 = acceptance ~0-4% (misurato, issue #8): il draft sbaglia sempre
|
||||
# e la speculazione non parte mai. A int8: 39-59%, 2.2-2.8 token/forward.
|
||||
a.ebits = 8 if (a.mtp or a.indexer) else 4
|
||||
if a.mtp and a.ebits < 8 and a.group_size <= 0:
|
||||
# Non solo lossy: eh_proj ha ~20-30x di asimmetria di scala fra le due meta' di
|
||||
# colonna, quindi l'int4 per-riga (UNA scala per riga) arrotonda a ZERO l'intera
|
||||
# meta' embedding -> il draft non vede il token -> acceptance ~0% (issue #8).
|
||||
# EN: not merely lossy: eh_proj has ~20-30x column-scale asymmetry, so per-row
|
||||
# EN: int4 rounds its ENTIRE embedding half to exact zeros -> the draft cannot
|
||||
# EN: see the input token -> ~0% acceptance (issue #8). A container converted
|
||||
# EN: this way is repairable in place with tools/repair_mtp_int8.py.
|
||||
print(f"WARNING: --mtp with --ebits {a.ebits} and per-row scales ZEROES eh_proj's "
|
||||
"embedding half -> MTP acceptance ~0% (issue #8). Use the default --ebits 8, "
|
||||
"or add --group-size 128 for group-scaled int4.")
|
||||
if a.xbits is None: a.xbits = a.ebits
|
||||
|
||||
# Build per-type bits map. If a type-specific arg is set, use it; otherwise the
|
||||
@@ -299,6 +310,16 @@ def main():
|
||||
if bits_map:
|
||||
print(f"[MIXED] precision map: " + ", ".join(f"{k}={v}bit" for k,v in sorted(bits_map.items())))
|
||||
|
||||
# Il PIANO risolto, PRIMA di toccare qualunque cosa (#383): --mtp/--indexer cambiano il
|
||||
# default di ebits a 8 (testa int4 = acceptance ~0%, issue #8) e il ramo grouped e'
|
||||
# gated su bits<=4 — combinazioni sorprendenti devono mostrarsi al secondo 1 di un job
|
||||
# da ore, non nel size-check dopo. EN: print the RESOLVED plan before doing anything.
|
||||
mode = "MTP head only" if a.mtp else "DSA indexer only" if a.indexer else "main model"
|
||||
grp = f"grouped gs={a.group_size} (fmt=4)" if (a.group_size and a.ebits <= 4) else \
|
||||
(f"PER-ROW (grouped branch needs bits<=4; ebits={a.ebits} disables it)" if a.group_size else "per-row")
|
||||
print(f"[PLAN] mode: {mode} | source: {'local ' + a.indir if a.indir else 'download ' + a.repo} | "
|
||||
f"experts {a.ebits}-bit, embed/lm_head {a.io_bits}-bit, x {a.xbits}-bit | {grp}")
|
||||
|
||||
if a.selftest_nvfp4:
|
||||
import torch
|
||||
# 1) LUT e2m1: i 16 codici devono decodificare esattamente ai valori attesi.
|
||||
@@ -379,14 +400,99 @@ def main():
|
||||
if a.indir: # conversione locale (test)
|
||||
shards = sorted(glob.glob(os.path.join(a.indir, "*.safetensors")))
|
||||
from safetensors.numpy import save_file
|
||||
# #383: se l'indice c'e', i passaggi --mtp/--indexer convertono SOLO gli shard
|
||||
# che contengono i tensori richiesti (3 invece di scandire tutti i 141 — ogni
|
||||
# scansione a vuoto apre comunque uno shard da 5 GB). Senza indice: scansione
|
||||
# completa come prima.
|
||||
# EN: #383: when the index is present, the --mtp/--indexer passes convert ONLY
|
||||
# the shards that hold the requested tensors (3 instead of scanning all 141 —
|
||||
# every empty scan still opens a 5 GB shard). Without the index: full scan as
|
||||
# before.
|
||||
if a.mtp or a.indexer:
|
||||
idxp = os.path.join(a.indir, "model.safetensors.index.json")
|
||||
if os.path.exists(idxp):
|
||||
wmap = json.load(open(idxp))["weight_map"]
|
||||
if a.mtp:
|
||||
want = {v for k, v in wmap.items() if k.startswith(f"model.layers.{a.n_layers}.")}
|
||||
else:
|
||||
want = {v for k, v in wmap.items() if "indexer" in k and 0 <= layer_idx(k) < a.n_layers}
|
||||
keep = [sp for sp in shards if os.path.basename(sp) in want]
|
||||
print(f"[PLAN] index: {len(keep)}/{len(shards)} local shard(s) hold the requested tensors")
|
||||
shards = keep
|
||||
# BUG #355: questo ramo ignorava --mtp/--indexer. Con --mtp scriveva
|
||||
# out-NNNNN (gli STESSI nomi di una conversione normale) in ebits=8 e
|
||||
# keep_mtp=False -> il "secondo passaggio MTP" nella stessa outdir
|
||||
# SOVRASCRIVEVA il container gia' finito con una riconversione int8
|
||||
# completa, in silenzio (137/141 shard distrutti prima di accorgersene).
|
||||
# Ora il ramo locale rispecchia il download path: prefisso corretto,
|
||||
# flag passate, shard vuoti saltati.
|
||||
prefix = "out-mtp-" if a.mtp else "out-idx-" if a.indexer else "out-"
|
||||
# RIPRESA (#383): i nomi out-NNNNN contano gli shard EMESSI, non l'indice di
|
||||
# input (gli shard senza tensori rilevanti non producono file), quindi "il
|
||||
# file esiste" non basta per saltare il lavoro gia' fatto. Un manifest
|
||||
# sidecar ricorda input -> output (o "vuoto") e con quali parametri: la
|
||||
# ripresa salta solo cio' che combacia, e parametri diversi sulla stessa
|
||||
# outdir vengono rifiutati invece di mescolare container (il modo #355).
|
||||
# EN: RESUME (#383): out-NNNNN names count EMITTED shards, not the input
|
||||
# EN: index (shards with no relevant tensors emit no file), so "the file
|
||||
# EN: exists" is not enough to skip completed work. A sidecar manifest
|
||||
# EN: records input -> output (or "empty") plus the conversion parameters:
|
||||
# EN: resume skips only what matches, and different parameters on the same
|
||||
# EN: outdir are refused instead of mixing containers (the #355 failure mode).
|
||||
params = {"ebits": a.ebits, "io_bits": a.io_bits, "xbits": a.xbits,
|
||||
"group_size": a.group_size, "n_layers": a.n_layers, "bits_map": bits_map}
|
||||
prog_path = os.path.join(a.outdir, f".{prefix}progress.json")
|
||||
prog = {}
|
||||
if os.path.exists(prog_path):
|
||||
try: prog = json.loads(open(prog_path).read())
|
||||
except (OSError, ValueError): prog = {}
|
||||
if prog and prog.get("params") != params:
|
||||
print(f"ERROR: {prog_path} records a conversion with {prog.get('params')};\n"
|
||||
f" this run uses {params}. Refusing to mix conversions in the same "
|
||||
f"outdir — use a fresh --outdir (or delete the manifest and the "
|
||||
f"{prefix}*.safetensors shards to redo).")
|
||||
return
|
||||
done = prog.setdefault("shards", {}); prog["params"] = params
|
||||
n = 0; fresh = 0; skipped = 0
|
||||
for i, sp in enumerate(shards):
|
||||
out = {}; convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits, group_size=a.group_size, bits_map=bits_map)
|
||||
save_file(out, os.path.join(a.outdir, f"out-{i:05d}.safetensors"))
|
||||
# copia config + tokenizer
|
||||
for fn in ["config.json"]:
|
||||
src = os.path.join(a.indir, fn)
|
||||
if os.path.exists(src): shutil.copy(src, a.outdir)
|
||||
print(f"converted {len(shards)} shards -> {a.outdir}")
|
||||
key = os.path.basename(sp)
|
||||
prev = done.get(key) # None = mai visto; "" = visto, vuoto; nome = emesso
|
||||
if prev is not None and (prev == "" or os.path.exists(os.path.join(a.outdir, prev))):
|
||||
if prev: n += 1
|
||||
skipped += 1
|
||||
continue
|
||||
out = {}
|
||||
convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits,
|
||||
keep_mtp=a.mtp, keep_idx=a.indexer,
|
||||
group_size=a.group_size, bits_map=bits_map)
|
||||
if not out: # shard senza MTP/idx: niente file (come il download path)
|
||||
done[key] = ""
|
||||
else:
|
||||
name = f"{prefix}{n:05d}.safetensors"
|
||||
save_file(out, os.path.join(a.outdir, name))
|
||||
done[key] = name; n += 1; fresh += 1
|
||||
tmp_prog = prog_path + ".tmp" # scrittura atomica: una ripresa non vede mai un manifest mezzo scritto
|
||||
with open(tmp_prog, "w") as f: json.dump(prog, f, indent=1) # EN: atomic write: a resume never sees a half-written manifest
|
||||
os.replace(tmp_prog, prog_path)
|
||||
if skipped: print(f"[RESUME] {skipped} shard(s) already done in {a.outdir}, skipped")
|
||||
# metadati per la conversione principale: gli stessi quattro file del download
|
||||
# path — senza tokenizer.json chat/serve non partono. I passaggi mtp/idx vanno
|
||||
# nella stessa outdir di un container gia' completo di metadati.
|
||||
# EN: metadata for the main pass: the same four files as the download path —
|
||||
# EN: chat/serve won't start without tokenizer.json. The mtp/idx passes target
|
||||
# EN: an outdir whose container already has its metadata.
|
||||
if not a.mtp and not a.indexer:
|
||||
copied, missing = [], []
|
||||
for fn in ["config.json", "tokenizer.json", "tokenizer_config.json", "generation_config.json"]:
|
||||
src = os.path.join(a.indir, fn)
|
||||
if os.path.exists(src): shutil.copy(src, a.outdir); copied.append(fn)
|
||||
else: missing.append(fn)
|
||||
print(f"[META] copied from {a.indir}: {', '.join(copied) if copied else 'nothing'}")
|
||||
if missing:
|
||||
print(f"[META] WARNING: not found in {a.indir}: {', '.join(missing)}"
|
||||
+ (" — chat/serve need tokenizer.json" if "tokenizer.json" in missing else ""))
|
||||
tag = "MTP" if a.mtp else "indexer" if a.indexer else "main"
|
||||
print(f"converted {fresh} {tag} shard(s), {n} in container -> {a.outdir} ({prefix}NNNNN)")
|
||||
return
|
||||
|
||||
# reale: scarica shard per shard, converte, cancella
|
||||
|
||||
@@ -3,7 +3,10 @@
|
||||
|
||||
Downloads or converts a local OLMoE checkpoint (e.g., allenai/OLMoE-1B-7B-0125-Instruct).
|
||||
Dense weights stay as-is (engine reads BF16/F16 → F32 on load).
|
||||
Expert weights get row-wise int8 quantization with float32 scales.
|
||||
Expert weights get row-wise symmetric quantization to --ebits bits (default 4)
|
||||
with float32 scales. Storage stays one value per int8 byte regardless of bits,
|
||||
matching the engine's expert layout (olmoe.c quantize_rows) — for 4 bits the
|
||||
values are simply confined to [-8, 7] with scales computed against qmax=7.
|
||||
|
||||
Usage:
|
||||
python tools/convert_olmoe.py --repo allenai/OLMoE-1B-7B-0125-Instruct --out ./olmoe_i4
|
||||
@@ -29,12 +32,21 @@ except ImportError as exc:
|
||||
EXPERT_KEY_RE = r"model\.layers\.\d+\.mlp\.experts\.\d+\.(gate_proj|up_proj|down_proj)\.weight"
|
||||
|
||||
|
||||
def quantize_row(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Row-wise int8 quantization. Returns (int8_weights, float32_scales)."""
|
||||
def quantize_row(w: torch.Tensor, bits: int = 8) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Row-wise symmetric quantization to `bits` (2..8).
|
||||
|
||||
Returns (int8_weights, float32_scales). Storage is one value per int8 byte
|
||||
for every bit width — the engine dequantizes as q*scale and never assumes
|
||||
the full int8 range — mirroring olmoe.c quantize_rows():
|
||||
qmax = 2**(bits-1) - 1 (8 -> 127, 4 -> 7, 2 -> 1)
|
||||
scale = amax(|w|, row) / qmax
|
||||
q = clamp(round(w / scale), -qmax-1, qmax)
|
||||
"""
|
||||
qmax = (1 << (bits - 1)) - 1
|
||||
w_f32 = w.float()
|
||||
row_max = w_f32.abs().amax(dim=1, keepdim=True).clamp(min=1e-12)
|
||||
scales = row_max / 127.0
|
||||
q = (w_f32 / scales).round().clamp(-128, 127).to(torch.int8)
|
||||
scales = row_max / qmax
|
||||
q = (w_f32 / scales).round().clamp(-qmax - 1, qmax).to(torch.int8)
|
||||
return q, scales.squeeze(1)
|
||||
|
||||
|
||||
@@ -50,9 +62,12 @@ def main():
|
||||
src.add_argument("--model", help="Local HF checkpoint directory")
|
||||
ap.add_argument("--out", required=True, help="Output directory for int4 model")
|
||||
ap.add_argument("--ebits", type=int, default=4,
|
||||
help="Expert quant bits (4 or 8, default 4)")
|
||||
help="Expert quant bits (2..8, default 4)")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not 2 <= args.ebits <= 8: # storage is int8_t; engine rejects the same range (olmoe.c)
|
||||
sys.exit(f"--ebits must be 2..8 (got {args.ebits})")
|
||||
|
||||
if args.repo:
|
||||
from huggingface_hub import snapshot_download
|
||||
from huggingface_hub.errors import LocalEntryNotFoundError
|
||||
@@ -96,7 +111,7 @@ def main():
|
||||
for name, tensor in tensors.items():
|
||||
if is_expert_weight(name):
|
||||
expert_count += 1
|
||||
q, scales = quantize_row(tensor)
|
||||
q, scales = quantize_row(tensor, args.ebits)
|
||||
total_expert_f32 += tensor.numel() * tensor.element_size()
|
||||
total_expert_q += q.numel() * 1 + scales.numel() * 4
|
||||
out_tensors[name] = q
|
||||
@@ -109,7 +124,7 @@ def main():
|
||||
ratio = total_expert_q / max(total_expert_f32, 1) * 100
|
||||
print(f"ok")
|
||||
|
||||
print(f"\nDone. {expert_count} expert tensors quantized.")
|
||||
print(f"\nDone. {expert_count} expert tensors quantized to int{args.ebits}.")
|
||||
print(f"Expert storage: {total_expert_f32/1e9:.1f} GB -> {total_expert_q/1e9:.1f} GB ({ratio:.0f}%)")
|
||||
print(f"Model ready at: {out}")
|
||||
print(f"\nRun: SNAP={out} ./olmoe.exe 32 4 16")
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Convert OLMoE HuggingFace checkpoint to colibri merged int8 format.
|
||||
|
||||
Consolidates gate_proj, up_proj, and down_proj into a single merged tensor per expert.
|
||||
This allows olmoe.c to load an expert in a single disk read call instead of 3.
|
||||
|
||||
Usage:
|
||||
python tools/convert_olmoe_merged.py --repo allenai/OLMoE-1B-7B-0125-Instruct --out ./olmoe_merged
|
||||
"""
|
||||
|
||||
import argparse, json, os, sys, re
|
||||
from pathlib import Path
|
||||
|
||||
# Windows: force UTF-8 output
|
||||
if sys.platform == "win32":
|
||||
for s in (sys.stdout, sys.stderr):
|
||||
try: s.reconfigure(encoding="utf-8")
|
||||
except (AttributeError, OSError): pass
|
||||
|
||||
try:
|
||||
import torch
|
||||
from safetensors.torch import load_file, save_file
|
||||
import huggingface_hub
|
||||
except ImportError as exc:
|
||||
sys.exit(f"Missing dependencies: {exc}. Install: pip install torch safetensors huggingface_hub")
|
||||
|
||||
EXPERT_KEY_RE = r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight"
|
||||
|
||||
def quantize_row(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Row-wise int8 quantization. Returns (int8_weights, float32_scales)."""
|
||||
w_f32 = w.float()
|
||||
row_max = w_f32.abs().amax(dim=1, keepdim=True).clamp(min=1e-12)
|
||||
scales = row_max / 127.0
|
||||
q = (w_f32 / scales).round().clamp(-128, 127).to(torch.int8)
|
||||
return q, scales.squeeze(1)
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Convert OLMoE HF checkpoint -> colibri merged int8")
|
||||
src = ap.add_mutually_exclusive_group(required=True)
|
||||
src.add_argument("--repo", help="HuggingFace repo ID")
|
||||
src.add_argument("--model", help="Local HF checkpoint directory")
|
||||
ap.add_argument("--out", required=True, help="Output directory for merged model")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.repo:
|
||||
from huggingface_hub import snapshot_download
|
||||
from huggingface_hub.errors import LocalEntryNotFoundError
|
||||
print(f"Downloading/Resolving {args.repo}...")
|
||||
try:
|
||||
src_dir = snapshot_download(args.repo, local_files_only=True, max_workers=4)
|
||||
except LocalEntryNotFoundError:
|
||||
src_dir = None
|
||||
if src_dir is None or not any(Path(src_dir).glob("*.safetensors")):
|
||||
print("Downloading safetensors...")
|
||||
src_dir = snapshot_download(args.repo, max_workers=4)
|
||||
else:
|
||||
src_dir = args.model
|
||||
|
||||
src = Path(src_dir)
|
||||
if not src.is_dir():
|
||||
sys.exit(f"Model directory not found: {src}")
|
||||
if not (src / "config.json").is_file():
|
||||
sys.exit(f"config.json missing in {src}")
|
||||
|
||||
out = Path(args.out)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Copy config.json
|
||||
import shutil
|
||||
shutil.copy2(src / "config.json", out / "config.json")
|
||||
print(f"config.json -> {out}")
|
||||
|
||||
# Process safetensors
|
||||
shards = sorted(src.glob("*.safetensors"))
|
||||
if not shards:
|
||||
sys.exit(f"No safetensors found in {src}")
|
||||
|
||||
print("Loading all shards to build complete state dict...")
|
||||
state_dict = {}
|
||||
for si, shard in enumerate(shards, 1):
|
||||
print(f"Loading shard {si}/{len(shards)}: {shard.name}...")
|
||||
tensors = load_file(str(shard))
|
||||
state_dict.update(tensors)
|
||||
|
||||
# Gather experts
|
||||
experts = {}
|
||||
for name in list(state_dict.keys()):
|
||||
m = re.match(EXPERT_KEY_RE, name)
|
||||
if m:
|
||||
layer_idx, expert_idx, proj = m.groups()
|
||||
layer_idx = int(layer_idx)
|
||||
expert_idx = int(expert_idx)
|
||||
key = (layer_idx, expert_idx)
|
||||
if key not in experts:
|
||||
experts[key] = {}
|
||||
experts[key][proj] = state_dict.pop(name)
|
||||
|
||||
print(f"Found {len(experts)} experts to merge.")
|
||||
|
||||
# Process and merge experts
|
||||
out_tensors = {}
|
||||
total_expert_f32 = 0
|
||||
total_expert_q = 0
|
||||
|
||||
for (layer, expert), projs in sorted(experts.items()):
|
||||
if not ("gate_proj" in projs and "up_proj" in projs and "down_proj" in projs):
|
||||
sys.exit(f"Missing projection for layer {layer} expert {expert}!")
|
||||
|
||||
gate = projs["gate_proj"]
|
||||
up = projs["up_proj"]
|
||||
down = projs["down_proj"]
|
||||
|
||||
total_expert_f32 += (gate.numel() + up.numel() + down.numel()) * gate.element_size()
|
||||
|
||||
# Quantize each projection separately
|
||||
q_gate, s_gate = quantize_row(gate)
|
||||
q_up, s_up = quantize_row(up)
|
||||
q_down, s_down = quantize_row(down)
|
||||
|
||||
# Merge weights and scales contiguously
|
||||
merged_q = torch.cat([q_gate.flatten(), q_up.flatten(), q_down.flatten()])
|
||||
merged_scales = torch.cat([s_gate, s_up, s_down])
|
||||
|
||||
total_expert_q += merged_q.numel() * 1 + merged_scales.numel() * 4
|
||||
|
||||
# Save to output
|
||||
out_tensors[f"model.layers.{layer}.mlp.experts.{expert}.merged_weight"] = merged_q
|
||||
out_tensors[f"model.layers.{layer}.mlp.experts.{expert}.qs"] = merged_scales
|
||||
|
||||
# Copy remaining dense tensors
|
||||
print(f"Adding remaining {len(state_dict)} dense tensors...")
|
||||
out_tensors.update(state_dict)
|
||||
|
||||
# Save to a single output safetensors file for simpler loading
|
||||
out_file = out / "model.safetensors"
|
||||
print(f"Saving merged safetensors model to {out_file}...")
|
||||
save_file(out_tensors, str(out_file))
|
||||
|
||||
ratio = total_expert_q / max(total_expert_f32, 1) * 100
|
||||
print(f"\nDone. {len(experts)} experts successfully merged and saved.")
|
||||
print(f"Expert storage: {total_expert_f32/1e9:.1f} GB -> {total_expert_q/1e9:.1f} GB ({ratio:.0f}%)")
|
||||
print(f"Model ready at: {out}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -17,6 +17,45 @@ EN: computed AFTER the FP8 round-trip, so the reference matches exactly what the
|
||||
EN: ingests. Default: bf16 (original oracle unchanged)."""
|
||||
import json, sys, argparse
|
||||
from pathlib import Path
|
||||
|
||||
# --- Version gate (must run BEFORE the heavy `from transformers import ...`,
|
||||
# which triggers transformers' lazy-loading and can in turn reset the in-memory
|
||||
# __version__ attribute; importlib.metadata reads the installed package version
|
||||
# directly and is immune to that). ---
|
||||
#
|
||||
# GLM-5.2's MLA attention uses interleaved (DeepSeek-style) RoPE, which is what
|
||||
# the C engine implements. transformers < 5.11.0 applied split-half (Llama-style)
|
||||
# RoPE in GlmMoeDsa* instead; an oracle built on those versions drifts and the
|
||||
# engine then scores 25/32 instead of the documented 32/32 (issue #281). Weights
|
||||
# come out identical across versions — only the forward pass differs — so there
|
||||
# is no safe "partial" run: a too-old transformers silently produces an invalid
|
||||
# ref_glm.json. Hard-fail rather than warn. EN: same.
|
||||
import transformers
|
||||
from importlib.metadata import version as _pkg_version, PackageNotFoundError
|
||||
|
||||
_MIN_TRANSFORMERS = (5, 11)
|
||||
def _tf_version_tuple():
|
||||
try:
|
||||
v = _pkg_version("transformers") # authoritative: installed dist metadata
|
||||
except PackageNotFoundError:
|
||||
v = getattr(transformers, "__version__", "0") # fallback (editable/src installs)
|
||||
out = []
|
||||
for part in v.split(".")[:2]: # major.minor only
|
||||
out.append(int("".join(c for c in part if c.isdigit()) or "0"))
|
||||
while len(out) < 2:
|
||||
out.append(0)
|
||||
return tuple(out[:2])
|
||||
|
||||
_tf_ver = _tf_version_tuple()
|
||||
if _tf_ver < _MIN_TRANSFORMERS:
|
||||
sys.exit(
|
||||
f"\nERROR: make_glm_oracle.py requires transformers >= "
|
||||
f"{'.'.join(map(str, _MIN_TRANSFORMERS))}.0 (found {_tf_ver[0]}.{_tf_ver[1]}). "
|
||||
f"GLM-5.2 MLA uses interleaved RoPE; older versions apply split-half RoPE "
|
||||
f"and silently produce an oracle the engine scores 25/32 against (issue #281). "
|
||||
f"Upgrade: pip install -U 'transformers>=5.11'\n"
|
||||
)
|
||||
|
||||
import torch
|
||||
from transformers import GlmMoeDsaConfig, GlmMoeDsaForCausalLM
|
||||
|
||||
@@ -115,6 +154,7 @@ print("tf_pred:", tf_pred)
|
||||
sd = model.state_dict()
|
||||
unfuse_experts(sd)
|
||||
|
||||
Path("glm_tiny").mkdir(parents=True, exist_ok=True) # safetensors/json won't create the dir themselves
|
||||
if args.fp8:
|
||||
n_fp8, n_tot = save_fp8_safetensors(sd, "glm_tiny/model.safetensors")
|
||||
print(f"\nsaved FP8: {n_fp8} e4m3 tensors (+{n_tot - n_fp8} scale_inv sidecars / f32) "
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Generate reference token IDs for the real OLMoE-1B-7B model.
|
||||
|
||||
Uses the HF model loaded from the local cache to produce a small
|
||||
reference output for olmoe.exe validation. Saves to ref_olmoe_real.json.
|
||||
|
||||
Usage: python tools/make_olmoe_real_oracle.py
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if sys.platform == "win32":
|
||||
for s in (sys.stdout, sys.stderr):
|
||||
try:
|
||||
s.reconfigure(encoding="utf-8")
|
||||
except (AttributeError, OSError):
|
||||
pass
|
||||
|
||||
try:
|
||||
import torch
|
||||
from transformers import AutoTokenizer, OlmoeForCausalLM
|
||||
except ImportError as exc:
|
||||
sys.exit(f"Missing deps: {exc}. Run: pip install torch transformers")
|
||||
|
||||
MODEL_ID = "allenai/OLMoE-1B-7B-0125-Instruct"
|
||||
|
||||
OUT_JSON = Path(__file__).resolve().parent.parent / "ref_olmoe_real.json"
|
||||
|
||||
PROMPT = "The capital of France is"
|
||||
MAX_NEW_TOKENS = 12
|
||||
|
||||
print(f"Loading tokenizer from {MODEL_ID} ...")
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
|
||||
|
||||
print("Encoding prompt ...")
|
||||
enc = tokenizer(PROMPT, return_tensors="pt")
|
||||
prompt_ids = enc["input_ids"][0].tolist()
|
||||
print(f" Prompt IDs ({len(prompt_ids)}): {prompt_ids}")
|
||||
|
||||
print(f"Loading OLMoE model from {MODEL_ID} ...")
|
||||
print(" (this will use ~14 GB RAM — please be patient)")
|
||||
model = OlmoeForCausalLM.from_pretrained(
|
||||
MODEL_ID,
|
||||
torch_dtype=torch.bfloat16,
|
||||
device_map="cpu",
|
||||
low_cpu_mem_usage=True,
|
||||
)
|
||||
model.eval()
|
||||
print(" Model loaded!")
|
||||
|
||||
print(f"Generating {MAX_NEW_TOKENS} tokens ...")
|
||||
with torch.no_grad():
|
||||
out = model.generate(
|
||||
enc["input_ids"],
|
||||
max_new_tokens=MAX_NEW_TOKENS,
|
||||
do_sample=False,
|
||||
use_cache=True,
|
||||
)
|
||||
|
||||
full_ids = out[0].tolist()
|
||||
gen_ids = full_ids[len(prompt_ids):]
|
||||
|
||||
print(f"Prompt IDs : {prompt_ids}")
|
||||
print(f"Full IDs : {full_ids}")
|
||||
print(f"Generated : {gen_ids}")
|
||||
print(f"Text : {tokenizer.decode(gen_ids, skip_special_tokens=True)!r}")
|
||||
|
||||
payload = {"prompt_ids": prompt_ids, "full_ids": full_ids}
|
||||
OUT_JSON.write_text(json.dumps(payload, indent=2))
|
||||
print(f"\nSaved reference to {OUT_JSON}")
|
||||
@@ -118,7 +118,7 @@ def quantize_param(w, bits, group, rot=False, e8=""):
|
||||
|
||||
def _grid_or_e8(x, bits, group, e8):
|
||||
if e8:
|
||||
return _quant_e8(x.float(), group, ball=(e8 == "-e8"))
|
||||
return _quant_e8(x.float(), group, bits, ball=(e8 == "-e8"))
|
||||
return _quant_last_dim(x, bits, group)
|
||||
|
||||
|
||||
@@ -169,8 +169,15 @@ def _e8_ball(y, r2=10.0):
|
||||
return p
|
||||
|
||||
|
||||
def _quant_e8(x, group, ball):
|
||||
"""Blocks of 8 along the input dim; per-group scale by MSE search over RMS multiples."""
|
||||
_E8_R2_REPORTED = set()
|
||||
def _e8_radius(bits):
|
||||
# E8 lattice: points within |p|^2<=r2 grow ~r2^4, so +1 bit (x256 codebook) needs r2 x4.
|
||||
# Anchor: r2=10 is the ~2^16 E8P ball (2 bits over 8 dims). Scale from there.
|
||||
return 10.0 * (4.0 ** (bits - 2))
|
||||
|
||||
def _quant_e8(x, group, bits, ball):
|
||||
"""Blocks of 8 along the input dim; per-group scale by MSE search over RMS multiples.
|
||||
ball=True clamps to the rate-scaled E8 ball for `bits`; ball=False is the unbounded ideal."""
|
||||
if x.shape[-1] % 8:
|
||||
raise SystemExit(f"-e8 needs input dim divisible by 8 (got {x.shape[-1]})")
|
||||
g = group or x.shape[-1]
|
||||
@@ -183,7 +190,11 @@ def _quant_e8(x, group, ball):
|
||||
for k in (0.5, 0.7, 0.9, 1.1, 1.4, 1.8, 2.4):
|
||||
s = rms * k
|
||||
yb = (xg / s).reshape(-1, g // 8, 8)
|
||||
p = _e8_ball(yb) if ball else _e8_nearest(yb)
|
||||
p = _e8_ball(yb, _e8_radius(bits)) if ball else _e8_nearest(yb)
|
||||
if ball and bits not in _E8_R2_REPORTED:
|
||||
_E8_R2_REPORTED.add(bits)
|
||||
import sys as _sys
|
||||
_sys.stderr.write(f"[e8] bits={bits}: ball r2={_e8_radius(bits):.1f}\n")
|
||||
out = (p.reshape(-1, g) * s)
|
||||
err = (out - xg).pow(2).sum(-1, keepdim=True)
|
||||
if best_err is None:
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Repair an existing colibri int4 container whose MTP head was quantized at int4.
|
||||
|
||||
WHY THIS EXISTS
|
||||
`model.layers.<N>.eh_proj.weight` [D, 2D] multiplies the MTP concat
|
||||
[embedding_norm ; hidden_norm], and its two column halves differ in scale by
|
||||
~20-30x per row (embedding-half absmax ~0.05, hidden-half ~1.5 on GLM-5.2).
|
||||
Per-row int4 uses ONE scale (= absmax/7) per row, so every embedding-half
|
||||
weight lands below half a quantization step and np.rint rounds the ENTIRE
|
||||
embedding half to exact zeros (packed bytes 0x88). The MTP head then drafts
|
||||
garbage: acceptance ~0% (issue #8 measured 0-4% at int4; 39-59% at int8 —
|
||||
which is why `convert_fp8_to_int4.py --mtp` defaults to --ebits 8).
|
||||
|
||||
A container converted (or downloaded) with an int4 MTP head does not need a
|
||||
full re-conversion: this script re-downloads ONLY the affected dense tensors
|
||||
(~355 MB of HTTP range reads against the FP8 source repo), requantizes them
|
||||
at int8 with the converter's exact math, and patches the local shards in
|
||||
place. Originals are kept beside as *.bak-int4.
|
||||
|
||||
WHAT IT TOUCHES
|
||||
The MTP layer's dense tensors only (eh_proj, q_a/q_b/kv_a/kv_b/o_proj,
|
||||
shared_experts.*): the ones that stream into RAM once and stay resident.
|
||||
Routed experts (model.layers.<N>.mlp.experts.*) are NOT touched — they are
|
||||
statistically like the main layers' experts and int4 is acceptable there.
|
||||
The engine auto-detects int8 vs int4 by blob size (qt_from_disk), so no
|
||||
engine or config change is needed. Cost: ~+133 MB on disk / resident RAM.
|
||||
|
||||
USAGE
|
||||
python3 tools/repair_mtp_int8.py --snap /path/to/glm52_i4 # repair
|
||||
python3 tools/repair_mtp_int8.py --snap /path/to/glm52_i4 --dry-run # inspect only
|
||||
|
||||
--source-repo defaults to zai-org/GLM-5.2-FP8 (the checkpoint the public
|
||||
int4 containers were converted from). Requires numpy and network access;
|
||||
no torch, no HF token (public repo, anonymous range reads).
|
||||
"""
|
||||
import argparse, glob, json, os, ssl, struct, sys, urllib.request
|
||||
import numpy as np
|
||||
|
||||
# macOS python.org builds ship no CA bundle: use certifi when available (Linux
|
||||
# system Pythons generally have working system certs and skip this).
|
||||
try:
|
||||
import certifi
|
||||
_SSL_CTX = ssl.create_default_context(cafile=certifi.where())
|
||||
except ImportError:
|
||||
_SSL_CTX = ssl.create_default_context()
|
||||
|
||||
|
||||
# ---------- HTTP range reads against the source repo ----------
|
||||
def http_range(url, start, length, tries=5):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "colibri-mtp-repair",
|
||||
"Range": f"bytes={start}-{start+length-1}"})
|
||||
for attempt in range(tries):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30, context=_SSL_CTX) as r:
|
||||
data = r.read()
|
||||
if len(data) == length:
|
||||
return data
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except Exception as ex:
|
||||
if attempt == tries - 1:
|
||||
raise RuntimeError(f"range read failed for {url}: {ex}")
|
||||
raise RuntimeError(f"short range read for {url}")
|
||||
|
||||
|
||||
class SourceRepo:
|
||||
def __init__(self, repo, revision="main"):
|
||||
self.base = f"https://huggingface.co/{repo}/resolve/{revision}/"
|
||||
with urllib.request.urlopen(self.base + "model.safetensors.index.json", timeout=30, context=_SSL_CTX) as r:
|
||||
self.wmap = json.loads(r.read())["weight_map"]
|
||||
self._hdr = {}
|
||||
|
||||
def _shard_header(self, shard):
|
||||
if shard not in self._hdr:
|
||||
n = struct.unpack("<Q", http_range(self.base + shard, 0, 8))[0]
|
||||
self._hdr[shard] = (json.loads(http_range(self.base + shard, 8, n)), 8 + n)
|
||||
return self._hdr[shard]
|
||||
|
||||
def meta(self, name):
|
||||
shard = self.wmap.get(name)
|
||||
if not shard:
|
||||
return None
|
||||
hdr, _ = self._shard_header(shard)
|
||||
return hdr[name]
|
||||
|
||||
def fetch_f32(self, name):
|
||||
"""Download one tensor and dequantize to f32 [O, I] (BF16 or FP8+block scales)."""
|
||||
shard = self.wmap[name]
|
||||
hdr, base = self._shard_header(shard)
|
||||
m = hdr[name]
|
||||
o0, o1 = m["data_offsets"]
|
||||
raw = http_range(self.base + shard, base + o0, o1 - o0)
|
||||
if m["dtype"] == "BF16":
|
||||
u = np.frombuffer(raw, dtype=np.uint16).astype(np.uint32) << 16
|
||||
return u.view(np.float32).reshape(m["shape"]).astype(np.float32)
|
||||
if m["dtype"] == "F8_E4M3":
|
||||
b = np.frombuffer(raw, dtype=np.uint8).astype(np.uint16)
|
||||
sign = np.where(b & 0x80, -1.0, 1.0)
|
||||
e = (b >> 3) & 0xF
|
||||
mant = (b & 7).astype(np.float64)
|
||||
v = (sign * np.where(e > 0, (1 + mant / 8) * np.exp2(e.astype(np.float64) - 7),
|
||||
mant / 8 * np.exp2(-6.0))).reshape(m["shape"])
|
||||
sn = name + "_scale_inv"
|
||||
sshard = self.wmap[sn]
|
||||
shdr, sbase = self._shard_header(sshard)
|
||||
sm = shdr[sn]
|
||||
so0, so1 = sm["data_offsets"]
|
||||
sc = np.frombuffer(http_range(self.base + sshard, sbase + so0, so1 - so0),
|
||||
dtype=np.float32).reshape(sm["shape"])
|
||||
O, I = m["shape"]
|
||||
scf = np.repeat(np.repeat(sc, 128, axis=0)[:O], 128, axis=1)[:, :I]
|
||||
return (v * scf).astype(np.float32)
|
||||
raise ValueError(f"{name}: unsupported source dtype {m['dtype']}")
|
||||
|
||||
|
||||
# ---------- quantization: identical to convert_fp8_to_int4.quant_int8 ----------
|
||||
def quant_int8(w):
|
||||
amax = np.abs(w).max(axis=1, keepdims=True)
|
||||
s = np.maximum(amax / 127, 1e-8)
|
||||
q = np.clip(np.rint(w / s), -128, 127).astype(np.int8)
|
||||
return q.reshape(-1).view(np.uint8).copy(), s[:, 0].astype(np.float32)
|
||||
|
||||
|
||||
# ---------- local safetensors IO (no deps; preserves byte-identity of untouched tensors) ----------
|
||||
def read_shard(path):
|
||||
with open(path, "rb") as fh:
|
||||
n = struct.unpack("<Q", fh.read(8))[0]
|
||||
hdr = json.loads(fh.read(n))
|
||||
base = 8 + n
|
||||
order = sorted(((k, v) for k, v in hdr.items() if k != "__metadata__"),
|
||||
key=lambda kv: kv[1]["data_offsets"][0])
|
||||
out = {}
|
||||
for k, v in order:
|
||||
fh.seek(base + v["data_offsets"][0])
|
||||
out[k] = (v["dtype"], v["shape"], fh.read(v["data_offsets"][1] - v["data_offsets"][0]))
|
||||
return out, hdr.get("__metadata__")
|
||||
|
||||
|
||||
def write_shard(path, tensors, meta):
|
||||
hdr = {}
|
||||
off = 0
|
||||
for k, (dt, shape, raw) in tensors.items():
|
||||
hdr[k] = {"dtype": dt, "shape": shape, "data_offsets": [off, off + len(raw)]}
|
||||
off += len(raw)
|
||||
if meta:
|
||||
hdr["__metadata__"] = meta
|
||||
hj = json.dumps(hdr).encode()
|
||||
hj += b" " * ((8 - len(hj) % 8) % 8)
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(struct.pack("<Q", len(hj)))
|
||||
fh.write(hj)
|
||||
for _, (_, _, raw) in tensors.items():
|
||||
fh.write(raw)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--snap", required=True, help="local colibri int4 container directory")
|
||||
ap.add_argument("--source-repo", default="zai-org/GLM-5.2-FP8")
|
||||
ap.add_argument("--revision", default="main")
|
||||
ap.add_argument("--dry-run", action="store_true", help="report what would change, touch nothing")
|
||||
a = ap.parse_args()
|
||||
|
||||
cfg = json.load(open(os.path.join(a.snap, "config.json")))
|
||||
L = cfg["num_hidden_layers"]
|
||||
pref = f"model.layers.{L}."
|
||||
src = SourceRepo(a.source_repo, a.revision)
|
||||
|
||||
# find the MTP layer's dense int4 tensors across the local shards
|
||||
plan = {} # shard path -> [tensor names to repair]
|
||||
already_ok, skipped_experts = [], 0
|
||||
for f in sorted(glob.glob(os.path.join(a.snap, "*.safetensors"))):
|
||||
with open(f, "rb") as fh:
|
||||
n = struct.unpack("<Q", fh.read(8))[0]
|
||||
hdr = json.loads(fh.read(n))
|
||||
for name, v in hdr.items():
|
||||
if not name.startswith(pref) or name.endswith(".qs") or v.get("dtype") != "U8":
|
||||
continue
|
||||
if ".mlp.experts." in name:
|
||||
skipped_experts += 1
|
||||
continue
|
||||
m = src.meta(name)
|
||||
if m is None:
|
||||
print(f" ?? {name}: not in source repo, skipping")
|
||||
continue
|
||||
O, I = m["shape"]
|
||||
nb = v["data_offsets"][1] - v["data_offsets"][0]
|
||||
if nb == O * I:
|
||||
already_ok.append(name)
|
||||
elif nb == O * ((I + 1) // 2):
|
||||
plan.setdefault(f, []).append((name, O, I))
|
||||
else:
|
||||
print(f" ?? {name}: unexpected blob size {nb} for shape {O}x{I}, skipping")
|
||||
|
||||
n_fix = sum(len(v) for v in plan.values())
|
||||
print(f"MTP layer {L}: {n_fix} dense tensor(s) at per-row int4 to repair, "
|
||||
f"{len(already_ok)} already int8, {skipped_experts} routed-expert tensors left as-is")
|
||||
if not n_fix:
|
||||
print("nothing to do."); return
|
||||
if a.dry_run:
|
||||
for f, names in plan.items():
|
||||
for nm, O, I in names:
|
||||
print(f" would repair {nm} [{O},{I}] in {os.path.basename(f)}")
|
||||
return
|
||||
|
||||
for f, names in plan.items():
|
||||
print(f"patching {os.path.basename(f)} ({len(names)} tensors)")
|
||||
tensors, meta = read_shard(f)
|
||||
for nm, O, I in names:
|
||||
print(f" {nm}: fetching source + requantizing at int8...", flush=True)
|
||||
w = src.fetch_f32(nm)
|
||||
assert w.shape == (O, I), f"{nm}: source shape {w.shape} != container {O}x{I}"
|
||||
q, s = quant_int8(w)
|
||||
tensors[nm] = ("U8", [len(q)], q.tobytes())
|
||||
tensors[nm + ".qs"] = ("F32", [O], s.tobytes())
|
||||
# verify first row against the source
|
||||
loc = q[:I].view(np.int8).astype(np.float64) * s[0]
|
||||
ref = w[0].astype(np.float64)
|
||||
cos = float(ref @ loc / (np.linalg.norm(ref) * np.linalg.norm(loc) + 1e-30))
|
||||
print(f" row-0 cosine vs source: {cos:.5f}")
|
||||
bak = f + ".bak-int4"
|
||||
write_shard(f + ".new", tensors, meta)
|
||||
os.replace(f, bak)
|
||||
os.replace(f + ".new", f)
|
||||
print(f" saved; original kept as {os.path.basename(bak)}")
|
||||
print("done. Re-run with --dry-run to confirm (should report 'already int8').")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Bootstrap ref_olmoe_real.json by running olmoe.exe once and capturing output.
|
||||
|
||||
Step 1: Creates a temp ref with only prompt_ids (no full_ids).
|
||||
Step 2: Runs olmoe.exe, parses the generated IDs from stdout.
|
||||
Step 3: Saves {prompt_ids, full_ids} as ref_olmoe_real.json.
|
||||
Step 4: Runs olmoe.exe again against the saved ref to verify determinism.
|
||||
|
||||
No RAM loading of the full model -- the engine streams from SSD as designed.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if sys.platform == "win32":
|
||||
for s in (sys.stdout, sys.stderr):
|
||||
try:
|
||||
s.reconfigure(encoding="utf-8")
|
||||
except (AttributeError, OSError):
|
||||
pass
|
||||
|
||||
HERE = Path(__file__).resolve().parent.parent
|
||||
ext = ".exe" if sys.platform == "win32" else ""
|
||||
ENGINE = HERE / f"olmoe{ext}"
|
||||
SNAP = os.getenv("SNAP", str(HERE.parent / "olmoe_merged"))
|
||||
REF_OUT = HERE / "ref_olmoe_real.json"
|
||||
BOOTSTRAP_REF = HERE / "ref_olmoe_bootstrap.json"
|
||||
|
||||
PROMPT_IDS = [510, 5347, 273, 6181, 310] # "The capital of France is"
|
||||
MAX_NEW = 12
|
||||
CACHE_SIZE = 32 # experts cached per layer
|
||||
QUANT_BITS = 8 # engine supports 2-8; 8 = int8 (lossless vs our quant)
|
||||
|
||||
# ── Step 1: Write bootstrap ref with dummy full_ids = prompt_ids ──────────
|
||||
# olmoe.exe needs full_ids to know how many tokens to generate (nfull - np).
|
||||
# We extend with MAX_NEW zeros so the engine generates MAX_NEW tokens.
|
||||
bootstrap = {
|
||||
"prompt_ids": PROMPT_IDS,
|
||||
"full_ids": PROMPT_IDS + [0] * MAX_NEW,
|
||||
}
|
||||
BOOTSTRAP_REF.write_text(json.dumps(bootstrap))
|
||||
print(f"Bootstrap ref written to {BOOTSTRAP_REF}")
|
||||
|
||||
env = {**os.environ, "SNAP": str(SNAP)}
|
||||
|
||||
# ── Step 2: Run engine once to capture generated IDs ─────────────────────
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Run 1/2 — capturing engine output (cache={CACHE_SIZE}, bits={QUANT_BITS}) ...")
|
||||
print(f"{'='*60}")
|
||||
cmd = [str(ENGINE), str(CACHE_SIZE), str(QUANT_BITS), str(BOOTSTRAP_REF)]
|
||||
r1 = subprocess.run(cmd, env=env, capture_output=True, text=True, cwd=str(HERE))
|
||||
print(r1.stdout)
|
||||
if r1.returncode != 0:
|
||||
print("STDERR:", r1.stderr, file=sys.stderr)
|
||||
sys.exit(r1.returncode)
|
||||
|
||||
# Parse "C engine : <id> <id> ..." line
|
||||
m = re.search(r"C engine\s*:\s*([\d ]+)", r1.stdout)
|
||||
if not m:
|
||||
sys.exit("Could not parse 'C engine :' line from output")
|
||||
gen_ids = [int(x) for x in m.group(1).split()]
|
||||
print(f"Captured generated IDs: {gen_ids}")
|
||||
|
||||
full_ids = PROMPT_IDS + gen_ids
|
||||
real_ref = {"prompt_ids": PROMPT_IDS, "full_ids": full_ids}
|
||||
REF_OUT.write_text(json.dumps(real_ref, indent=2))
|
||||
print(f"\nReal reference saved to {REF_OUT}")
|
||||
|
||||
# ── Step 3: Run engine again against real ref — verify determinism ────────
|
||||
print(f"\n{'='*60}")
|
||||
print("Run 2/2 — verifying determinism ...")
|
||||
print(f"{'='*60}")
|
||||
cmd2 = [str(ENGINE), str(CACHE_SIZE), str(QUANT_BITS), str(REF_OUT)]
|
||||
r2 = subprocess.run(cmd2, env=env, capture_output=True, text=True, cwd=str(HERE))
|
||||
print(r2.stdout)
|
||||
if r2.returncode != 0:
|
||||
print("STDERR:", r2.stderr, file=sys.stderr)
|
||||
sys.exit(r2.returncode)
|
||||
|
||||
if "Matching tokens: 12/12" in r2.stdout or f"Matching tokens: {MAX_NEW}/{MAX_NEW}" in r2.stdout:
|
||||
print("✓ Engine is DETERMINISTIC — same output on both runs!")
|
||||
else:
|
||||
m2 = re.search(r"Matching tokens: (\d+)/(\d+)", r2.stdout)
|
||||
if m2:
|
||||
print(f"⚠ Partial match: {m2.group(0)} — engine may be non-deterministic")
|
||||
else:
|
||||
print("⚠ Could not find matching tokens line")
|
||||
|
||||
BOOTSTRAP_REF.unlink(missing_ok=True)
|
||||
@@ -0,0 +1,3 @@
|
||||
"""Single source of truth for the colibrì version number."""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
@@ -0,0 +1,5 @@
|
||||
"""colibrì — tiny engine, immense model."""
|
||||
|
||||
from colibri._version import __version__
|
||||
|
||||
__all__ = ["__version__"]
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Version accessor for the pip package.
|
||||
|
||||
The single source of truth is c/version.py (#394): coli --version and the
|
||||
GitHub Release workflow read it, so the pip metadata must read the SAME file
|
||||
instead of carrying a second literal that would drift on the first bump.
|
||||
|
||||
From a checkout (the supported install: `pip install -e .`) the file is read
|
||||
directly. From an installed wheel c/ is not on disk, so fall back to the
|
||||
package metadata that setuptools baked at build time from that same file.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
_ns = {}
|
||||
exec((Path(__file__).resolve().parent.parent / "c" / "version.py").read_text(), _ns)
|
||||
__version__ = _ns["__version__"]
|
||||
except OSError:
|
||||
from importlib.metadata import PackageNotFoundError, version
|
||||
|
||||
try:
|
||||
__version__ = version("colibri-engine")
|
||||
except PackageNotFoundError:
|
||||
__version__ = "0.0.0+unknown"
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Entry point for `coli` when installed via pip.
|
||||
|
||||
Delegates to the original c/coli script which handles all subcommands.
|
||||
This wrapper exists so `pip install colibri-engine` creates a `coli` console
|
||||
script that works without the user having to add c/ to PATH manually.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import runpy
|
||||
|
||||
|
||||
def main():
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
engine_dir = os.path.join(os.path.dirname(here), "c")
|
||||
coli_script = os.path.join(engine_dir, "coli")
|
||||
|
||||
if not os.path.exists(coli_script):
|
||||
sys.exit(
|
||||
"colibri engine directory not found.\n"
|
||||
"Install from source: git clone + pip install -e ."
|
||||
)
|
||||
|
||||
sys.path.insert(0, engine_dir)
|
||||
sys.argv[0] = coli_script
|
||||
runpy.run_path(coli_script, run_name="__main__")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+37
-3
@@ -2,7 +2,7 @@
|
||||
|
||||
Reference for the environment variables read by the colibrì engine.
|
||||
|
||||
**Generated from `upstream/dev @ 6d3ed7e`** by scanning every `getenv()` site in `c/glm.c`. Defaults and behavior are taken from the source; see [MAINTAINING-DOCS.md](MAINTAINING-DOCS.md) to regenerate this after the code changes.
|
||||
**Generated from `dev @ d5327e2`** by scanning every `getenv()` site in `c/glm.c` and the other C sources (`c/olmoe.c`, `c/backend_cuda.cu`, `c/backend_metal.mm`). Defaults and behavior are taken from the source; see [MAINTAINING-DOCS.md](MAINTAINING-DOCS.md) to regenerate this after the code changes.
|
||||
|
||||
## Which program reads these?
|
||||
|
||||
@@ -43,6 +43,7 @@ Format: `VAR` — default — effect.
|
||||
| `URING` | `0` (off) | Linux-only queued expert I/O. `URING=1` implies `PIPE=1`, forces cold reads through io-wq (`IOSQE_ASYNC`), replaces blocking loader pthreads and spin waits with batched SQEs/CQEs, and batches `PILOT_REAL` loads on a separate ring. Use `DIRECT=1` for cold NVMe to avoid page-cache copy/readahead limits. Fails clearly if the kernel denies io_uring; incompatible with `COLI_MMAP=1`. |
|
||||
| `DIRECT` | `0` (off) | Use `O_DIRECT`/unbuffered reads for expert slabs. Helps sustained NVMe; keeps the zero-copy GPU path. |
|
||||
| `COLI_NO_OMP_TUNE` | off | **Kill-switch** for the OpenMP hot-thread tuning (`OMP_WAIT_POLICY=active` spin + proc-bind). Set `=1` when the CPU is mostly waiting on the GPU (Metal) so spin doesn't steal the shared power budget. |
|
||||
| `COLI_NUMA` | auto in generated plans on multi-socket Linux; otherwise off | `COLI_NUMA=1` selectively interleaves large expert and dense slabs across NUMA nodes via `mbind` (raw syscall, no libnuma). Helps multi-socket hosts (+7–40% expert matmul); silent no-op on single-node or non-Linux. Explicit `COLI_NUMA=0` overrides the generated plan. |
|
||||
| `MLOCK` | `-1` (auto: on for macOS) | Wire the streamed expert cache into physical RAM (`mlock`) to dodge the memory compressor. `0` off, `1` force. |
|
||||
| `CAP_RAISE` | `1` (on) | Let the engine raise the expert-cache cap above `topk` when RAM allows (bigger batches). `0` fixes the cap. |
|
||||
| `PREFETCH` | `0` | Prefetch depth for streamed experts. |
|
||||
@@ -54,15 +55,26 @@ Format: `VAR` — default — effect.
|
||||
| `PILOT` | `0` (off) | Router-piloted cross-layer expert prefetch. |
|
||||
| `PILOT_REAL` | `0` (off) | Value-preserving real cross-layer prefetch loads (`PILOT_REAL=1` opts in). |
|
||||
| `PILOT_K` | `6` if `PILOT_REAL` else `8` | Number of experts the pilot prefetches per step. |
|
||||
| `PILOT_TWO` | `0` (off) | Two-step shared-expert-corrected router prediction for the pilot. |
|
||||
| `COUPLE` | unset | Path to a coupling-score file driving cross-layer expert prefetch (#176). When set, `couple_load` reads it. |
|
||||
| `COUPLE_K` | `8` | Top-K coupled experts per layer when `COUPLE` is set. |
|
||||
| `COUPLE_D` | `1` | Coupling lookahead depth (`1` or `2`) when `COUPLE` is set. |
|
||||
| `CACHE_ROUTE` | `0` (off) | Opt-in max-rank cache-aware MoE routing (pin∪LRU prefer within top-M). See [CACHE_ROUTE.md](CACHE_ROUTE.md). |
|
||||
| `ROUTE_J` | `2` | Sacred top ranks always taken when `CACHE_ROUTE=1`. |
|
||||
| `ROUTE_M` | `12` | Max-rank window for resident preference when `CACHE_ROUTE=1`. |
|
||||
| `ROUTE_P` | `0` | Cumulative mass window for CACHE_ROUTE (`0` = fixed M). |
|
||||
| `ROUTE_ALPHA` | `1` | Scale gate mass of substituted experts before renorm (`1` = off). |
|
||||
| `ROUTE_AGREE` | auto | Overlap% + KL vs true top-K; auto-on when `CACHE_ROUTE=1`. |
|
||||
| `ROUTE_TRACE` | unset | If set to a path, logs every routing decision there (testing/analysis). |
|
||||
| `ABSORB` | `-1` (auto: absorbed for S≤4) | MLA attention absorption mode. |
|
||||
| `IDOT` | `1` | Integer dot-product kernel. `IDOT=0` uses exact f32 kernels (for A/B numerical checks). |
|
||||
| `COLI_POLICY` | `quality` | Resource policy: `quality`, `balanced`, or `experimental-fast`. |
|
||||
| `PROF` | `0` (off) | Performance profile: a startup header (machine + effective config), then per run — or per turn in serve mode, on stderr — forward-latency percentiles (p50/p90/p99/max), expert-I/O totals and cache-tier fill, phase shares of wall time, and a verdict naming the knob most likely to help on this machine. Output is additive; `PROF` unset changes nothing. |
|
||||
| `COLI_NO_FUSED_PAIR` | `0` (off) | `=1` disables the fused-pair matmul kernel. |
|
||||
| `DISK_SPLIT` | `0` (off) | `=1` splits the reported disk-load time across the draft/absorb/forward phases in stats. |
|
||||
| `I4S` | unset | Engage the int4 `IDOT` kernel only for batch `S>=<n>` (testing). |
|
||||
| `SPEC_PIN` | `1` (on) | Speculation gate mode. `0` reverts to the legacy S-dependent speculation gates (#163). |
|
||||
| `COLI_RAM_OVERCOMMIT` | off | `=1` overrides the "projected peak > MemAvailable → exit(2)" guard so a run that risks kernel OOM-kill is allowed to proceed. |
|
||||
|
||||
---
|
||||
|
||||
@@ -76,7 +88,22 @@ Format: `VAR` — default — effect.
|
||||
| `CUDA_EXPERT_GB` | `0` | VRAM budget (GB) for caching experts on the GPU. |
|
||||
| `CUDA_RELEASE_HOST` | auto (`1` if >1 device) | Release host-side copies after upload. |
|
||||
| `COLI_CUDA_ATTN` | off | Run S≤4 attention on the GPU. |
|
||||
| `COLI_CUDA_ATTN_SHARD` | off | `=1` splits KV-b heads across devices during attention load (multi-GPU). |
|
||||
| `COLI_CUDA_PROFILE` | off | Emit CUDA timing. |
|
||||
| `COLI_CUDA_PIPE` | `0` (off) | `1` engages the multi-step attention pipeline; `2` enables the pipe2 path. |
|
||||
| `COLI_CUDA_PIPE_SHARD` | off | `=1` runs the multi-device P2P head-shard attention path (opt-in for NVLink topologies; serializes ~95 MB/layer over a star PCIe topology). |
|
||||
| `COLI_CUDA_PIPE_S_MIN` | `1` single-GPU, `8` multi-GPU | Minimum prefill batch S to engage the pipe2 CUDA path. |
|
||||
| `COLI_CUDA_MTP` | `0` (off) | `=1` opts into MTP speculation under CUDA (off by default: cold streaming experts run on CPU where the fused-pair/IDOT kernels diverge in FP order, collapsing draft acceptance, #163/#292). |
|
||||
| `COLI_CUDA_ASYNC` | on | `=0` forces synchronous `cudaMemcpy` instead of async + pinned host staging. |
|
||||
| `COLI_CUDA_DUAL_PROJ` | on | `=0` issues gate+up as two separate launches instead of one fused `grouped_hidden_w4_dual`. |
|
||||
| `COLI_CUDA_W4_PACKED` | on | `=0` disables the grouped packed-int4 path. |
|
||||
| `COLI_CUDA_TC_INT4` | off | `=1` uses the W4A4 WMMA Tensor Core path (when all expert tensors are int4 and dims divide). |
|
||||
| `COLI_CUDA_TC_MIN_ROWS` | `8` | Min rows-per-expert to engage the W4A4 Tensor Core path. |
|
||||
| `COLI_CUDA_TC_W4A16` | off | `=1` uses the lossless W4A16 Tensor Core path (compute capability ≥7). |
|
||||
| `COLI_CUDA_TC_W4A16_MIN` | `16` | Per-expert row threshold above which W4A16 TC tiles dispatch (smaller batches fall back to the naive kernel). |
|
||||
| `COLI_CUDA_SHARED_W4A16` | off | `=1` uploads shared-expert weights and runs the shared-MLP W4A16 Tensor Core kernel. |
|
||||
| `COLI_CUDA_SHARED_W4A16_MIN_ROWS` | `32` | Min row count to engage the shared-MLP W4A16 kernel. |
|
||||
| `COLI_METAL_UNTRACKED` | off (Metal only) | `=1` sets `MTLResourceHazardTrackingModeUntracked` on Metal buffers (reduces hazard-tracking overhead). |
|
||||
|
||||
---
|
||||
|
||||
@@ -88,8 +115,11 @@ These are for testing, benchmarking, or internal use — not part of the everyda
|
||||
|---|---|---|
|
||||
| `SPEC` | `1` | Speculative decoding on/off. |
|
||||
| `DRAFT` | `-1` (auto: 3 with MTP, else 0) | Number of speculative draft tokens per step. |
|
||||
| `GRAMMAR` | unset | Path to a GBNF grammar file to constrain generation. |
|
||||
| `GRAMMAR` | unset | Path to a GBNF grammar file to constrain generation. Takes precedence over `SCHEMA`. |
|
||||
| `SCHEMA` | unset | Path to a JSON-Schema file compiled to GBNF to constrain generation (consulted only when `GRAMMAR` is empty). |
|
||||
| `GRAMMAR_DRAFT` | unset | Max grammar-forced draft span length. |
|
||||
| `EXPERT_BUDGET` | `0` (off) | Cap experts loaded per layer (MoE-Spec). **Quarantined:** silently forced to `0` unless `EXPERT_BUDGET_EXPERIMENTAL` is set — every tested value is either no faster or incoherent (issue #303). |
|
||||
| `EXPERT_BUDGET_EXPERIMENTAL` | unset | Setting it (any value) allows `EXPERT_BUDGET>0` to actually take effect (expect garbage, #294). |
|
||||
| `DSA` | on | Dynamic Sparse Attention indexer. `DSA=0` disables. |
|
||||
| `DSA_FORCE` | `0` | Force the DSA path on. |
|
||||
| `DSA_TOPK` | model value | Override the DSA index top-k (testing). |
|
||||
@@ -100,11 +130,15 @@ These are for testing, benchmarking, or internal use — not part of the everyda
|
||||
| `PIN_FILL` | `0` | Fill the pinned store even without usage data. |
|
||||
| `MTP_DEBUG` / `MTP_PRENORM` / `MTP_SWAP` | off | MTP head debugging / ablations. |
|
||||
| `STATS` | unset | Write an expert-usage histogram to `STATS=<file>` at end of run. |
|
||||
| `TOKENS` | unset | If set, dumps generated token ids to stderr for A/B comparison. |
|
||||
| `SCORE` | unset | Scoring/eval mode over `SCORE=<file>`. |
|
||||
| `SCORE_PREFIX` | on | If unset or `≠0`, prepends `[gMASK]<sop>` to scoring contexts (GLM-family only). |
|
||||
| `REPIN_VERBOSE` | off | If set, prints per-swap `[REPIN]` diagnostics during VRAM repin. |
|
||||
| `REF` / `REF_FORCE` | `ref_glm.json` | Reference-output comparison mode. |
|
||||
| `REPLAY` | unset | Replay mode. |
|
||||
| `TF` | unset | Teacher-forcing mode. |
|
||||
| `CHAT_TEMPLATE` | `1` | Apply the GLM chat template (`0` = raw prompt). |
|
||||
| `PPL` | off (`olmoe.c` only) | `PPL=1` enters teacher-forced NLL/perplexity meter mode in the OLMoE sister engine. |
|
||||
|
||||
---
|
||||
|
||||
@@ -135,7 +169,7 @@ These are read by the Python programs (not the `glm` engine), so they don't appe
|
||||
|
||||
- `SNAP` — model snapshot directory (required by `glm`; set from `--model`).
|
||||
- `SERVE`, `SERVE_BATCH` — select serve / batched-serve mode.
|
||||
- `PROMPT` — one-shot text mode.
|
||||
- `PROMPT` — one-shot text mode (the engine also honors `COLI_PROMPT`, preferred cross-platform; `PROMPT` is ignored on Windows if it contains cmd.exe `$`-metacharacters).
|
||||
- `COLI_OMP_TUNED` — internal sentinel guarding the OMP re-exec (see `COLI_NO_OMP_TUNE`); not user-facing.
|
||||
|
||||
---
|
||||
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
# OpenAI-compatible API, KV contexts & web UI
|
||||
|
||||
## `coli serve`
|
||||
|
||||
`coli serve` keeps one model process loaded and exposes a text-only
|
||||
OpenAI-compatible HTTP API. The gateway uses only the Python standard library;
|
||||
inference still runs in the same dependency-free C engine.
|
||||
|
||||
```bash
|
||||
cd c
|
||||
COLI_MODEL=/nvme/glm52_i4 COLI_API_KEY=local-secret ./coli serve \
|
||||
--host 127.0.0.1 --port 8000 --model-id glm-5.2-colibri
|
||||
|
||||
curl http://127.0.0.1:8000/v1/chat/completions \
|
||||
-H 'Authorization: Bearer local-secret' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model": "glm-5.2-colibri",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"stream": true
|
||||
}'
|
||||
```
|
||||
|
||||
Implemented endpoints are `GET /v1/models`, `GET /v1/models/{model}`,
|
||||
`POST /v1/chat/completions`, and legacy `POST /v1/completions`. Chat and
|
||||
completion requests support JSON responses, SSE streaming, usage counts,
|
||||
`max_tokens`/`max_completion_tokens`, `temperature`, and `top_p`. The extension
|
||||
`enable_thinking: true` enables GLM-5.2's reasoning block; the standard
|
||||
`reasoning_effort` field also enables it unless set to `none`.
|
||||
|
||||
The server is deliberately text-only and serves one generation at a time: the
|
||||
744B model stays in one persistent process, so concurrent HTTP requests queue
|
||||
instead of loading duplicate model copies. Tools, image/audio input, custom
|
||||
stop sequences, log probabilities, and token penalties return an explicit error
|
||||
rather than being silently ignored. The default bind address is localhost; set
|
||||
`COLI_API_KEY` before exposing the server beyond the machine.
|
||||
|
||||
Browser access from the Vite development server and Tauri local origins is
|
||||
enabled by default. Repeat `--cors-origin https://your-ui.example` to allow
|
||||
another exact origin, or use `--cors-origin '*'` only on a trusted local
|
||||
network.
|
||||
|
||||
The engine owns its KV contexts, so HTTP generation uses a bounded FIFO
|
||||
admission queue instead of pretending to run unsafe parallel sequences.
|
||||
Configure it with `--max-queue N` (default 8) and `--queue-timeout SECONDS`
|
||||
(default 300), or the `COLI_MAX_QUEUE` / `COLI_QUEUE_TIMEOUT` environment
|
||||
variables. Saturated and timed-out requests receive OpenAI-shaped HTTP 429
|
||||
errors before streaming headers are sent. `GET /health` exposes
|
||||
active/queued/completed/rejected counters, and successful generation responses
|
||||
include `x-colibri-queue-wait-ms`.
|
||||
|
||||
## Connect a coding CLI or editor
|
||||
|
||||
The API is OpenAI-compatible, so most coding CLIs and editor extensions work by
|
||||
pointing them at Colibri as an *OpenAI-compatible* provider. Three settings:
|
||||
|
||||
- **Base URL** — `http://localhost:8000/v1`
|
||||
- **Model** — `glm-5.2-colibri` (or whatever you pass to `--model-id`)
|
||||
- **API key** — any non-empty string, e.g. `local`
|
||||
|
||||
Colibri needs **no** API key by default, but many clients refuse to start without
|
||||
one — give them any dummy value. The key is only enforced if you set `COLI_API_KEY`.
|
||||
|
||||
Smoke-test the endpoint first (no key needed unless you set one):
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8000/v1/chat/completions \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"model":"glm-5.2-colibri","messages":[{"role":"user","content":"hi"}]}'
|
||||
```
|
||||
|
||||
**aider**
|
||||
|
||||
```bash
|
||||
export OPENAI_API_BASE=http://localhost:8000/v1
|
||||
export OPENAI_API_KEY=local
|
||||
aider --model openai/glm-5.2-colibri # the openai/ prefix routes to OPENAI_API_BASE
|
||||
```
|
||||
|
||||
**crush** — add a provider to `crush.json` (`~/.config/crush/crush.json`, or
|
||||
`%USERPROFILE%\AppData\Local\crush\crush.json` on Windows):
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "https://charm.land/crush.json",
|
||||
"providers": {
|
||||
"colibri": {
|
||||
"name": "Colibri",
|
||||
"type": "openai-compat",
|
||||
"base_url": "http://localhost:8000/v1/",
|
||||
"api_key": "local",
|
||||
"models": [
|
||||
{ "name": "GLM-5.2 (Colibri)", "id": "glm-5.2-colibri",
|
||||
"context_window": 131072, "default_max_tokens": 1024 }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The `"api_key": "local"` dummy is what satisfies clients that demand a key.
|
||||
`context_window` is only the client's budget display — set it to whatever your
|
||||
KV configuration actually allows.
|
||||
|
||||
**Continue, Cline / Roo, `llm`, the OpenAI SDKs, …** — set the provider's base
|
||||
URL to `http://localhost:8000/v1`, the model to `glm-5.2-colibri`, and any dummy
|
||||
key (`OPENAI_API_KEY` / `OPENAI_BASE_URL` for env-based tools).
|
||||
|
||||
> **Set your expectations before connecting an agentic CLI.** Two costs dominate,
|
||||
> and the first one is invisible until you know it's there:
|
||||
>
|
||||
> 1. **Prefill.** Coding agents (crush, aider in repo-map mode, Cline, …) send a
|
||||
> large system prompt plus tool definitions — often 10–20k tokens — *before
|
||||
> your first word*. Prefill on the CPU-streaming path runs at a few tokens per
|
||||
> second (it is attention-bound, see #153), so a 15k-token agent preamble is
|
||||
> **an hour of silent "thinking" before the first output token**. The client
|
||||
> looks hung; it isn't. Smoke-test with the tiny `curl` above first — if that
|
||||
> answers in about a minute, the pipeline works and what you're paying for is
|
||||
> prompt size.
|
||||
> 2. **Decode.** Roughly 1 tok/s for a large model, so multi-turn agent loops
|
||||
> (which re-pay the growing context every turn) compound the cost.
|
||||
>
|
||||
> Practical guidance: single surgical asks with a short context work; iterative
|
||||
> agent sessions against a disk-streaming 744B model do not resemble a hosted
|
||||
> API and mostly won't be worth the wait. If your client lets you trim or disable
|
||||
> its system preamble and tool catalog, do it.
|
||||
|
||||
## Isolated KV contexts
|
||||
|
||||
`coli serve --kv-slots N` allocates up to 16 independent sequence contexts.
|
||||
Requests select one with the optional integer `cache_slot` field; ordinary
|
||||
OpenAI clients omit it and keep the original slot 0 behavior.
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "glm-5.2-colibri",
|
||||
"messages": [{"role": "user", "content": "Continue this conversation"}],
|
||||
"cache_slot": 1
|
||||
}
|
||||
```
|
||||
|
||||
Each slot owns its token history, compressed MLA/DSA KV memory, MTP window, and
|
||||
crash-safe persistence file (`.coli_kv`, `.coli_kv.1`, ...). The engine matches
|
||||
each request's tokenized prompt against the slot's history and reuses the common
|
||||
KV prefix, so stateless HTTP turns keep their cache across requests and even
|
||||
across engine restarts. Use `COLI_KV_SLOTS=N` as the environment equivalent.
|
||||
Start small: at the default 4096-token context, every slot costs hundreds of MB.
|
||||
|
||||
## Web dashboard
|
||||
|
||||
One command serves the OpenAI-compatible API **and** the web console on the
|
||||
same port, then opens your browser when the engine is ready:
|
||||
|
||||
```bash
|
||||
cd web && npm install && npm run build # once
|
||||
./coli web --model <model-dir>
|
||||
```
|
||||
|
||||
What you get:
|
||||
|
||||
- **Chat** with live metrics: a flashing token counter while generating, then
|
||||
tok/s, time-to-first-token, prompt→completion counts and queue wait;
|
||||
- **Runtime panel**: your hardware (CPU, GPUs + VRAM, RAM, cores), the
|
||||
scheduler, and the live expert-tier bar — how many of the 19,456 experts sit
|
||||
in VRAM / RAM / disk right now;
|
||||
- **Brain**: the whole model as a 76×256 cortex, one cell per expert. Colour =
|
||||
tier, brightness = routing heat, and the experts routed in each turn flash
|
||||
white and decay — you watch the model think. Hover any cell for its tier,
|
||||
heat and [measured topic affinity](https://github.com/JustVugg/colibri/issues/175);
|
||||
- **Atlas**: the measured expert atlas as a 3-D galaxy (publish `experts.json`
|
||||
from `tools/expert_atlas/analyze.py --web`).
|
||||
|
||||
The dashboard talks to the engine over a small line protocol and plain JSON
|
||||
endpoints — nothing heavier than the engine itself. `web/` is a pure OpenAI-API
|
||||
client (React + TypeScript) and also works against any other compatible
|
||||
endpoint; the terminal `coli chat` remains the first-class interface.
|
||||
|
||||
The layout is responsive down to phone widths, and the sidebar carries the full
|
||||
telemetry stack — hardware, scheduler, tier bar, per-turn time breakdown, tok/s
|
||||
trend and per-GPU expert counts:
|
||||
|
||||
<p align="center">
|
||||
<img src="media/colibri-mobile.png" width="270" alt="the dashboard on a phone-sized viewport">
|
||||
|
||||
<img src="media/colibri-metrics.png" width="300" alt="the telemetry sidebar">
|
||||
</p>
|
||||
@@ -0,0 +1,144 @@
|
||||
# Benchmarks & measured numbers
|
||||
|
||||
Everything on this page is a measurement, not a promise. If you run colibrì on
|
||||
hardware not listed here, **please open an issue with your numbers** — real
|
||||
datapoints are what move this project.
|
||||
|
||||
## Reference numbers (the original dev box: WSL2, 12 cores, 25 GB RAM, NVMe via VHDX)
|
||||
|
||||
Detailed GPU experiment: [GLM-5.2 on 6× RTX 5090](experiments/glm52-6x5090-2026-07-12.md) —
|
||||
full expert residency across VRAM+RAM reaches **6.84 tok/s** single-request decode.
|
||||
|
||||
| metric | value |
|
||||
|---|---|
|
||||
| model on disk (int4 container) | ~370 GB |
|
||||
| resident RAM (dense, int4) | 9.9 GB |
|
||||
| load time | ~30 s |
|
||||
| peak RSS during chat | ~20 GB (auto-capped) |
|
||||
| cold decode cost | ~11 GB disk reads/token (75 layers × 8 experts) |
|
||||
| disk ceiling (this dev box's drive) | ~1 GB/s → ~0.05–0.1 tok/s cold |
|
||||
| MTP speculation (int8 head) | 2.2–2.8 tok/forward measured ([#8](https://github.com/JustVugg/colibri/issues/8)) |
|
||||
|
||||
This is not fast. It is a 744B frontier-class model **answering correctly on a
|
||||
machine that costs less than one H100 fan**. Warm cache, pinned hot experts and
|
||||
MTP push the useful-response latency down considerably; the physics of the disk
|
||||
does the rest.
|
||||
|
||||
### SSD note
|
||||
|
||||
Cold starts are heavy on random reads (~11 GB/token), but reads don't
|
||||
meaningfully wear an SSD — colibrì's streaming is read-only. The real concerns
|
||||
under heavy use are (1) **swap traffic** if the system runs out of RAM (writes
|
||||
do wear the drive — keep a sane `--ram` budget; colibrì's auto-budget is designed
|
||||
to stay clear of swap) and (2) **sustained thermals**: hours at full read duty
|
||||
cycle will heat cheaper drives. Monitor drive temperature and health.
|
||||
|
||||
## Test your machine, in order
|
||||
|
||||
```bash
|
||||
cd c && ./setup.sh # build + architecture self-test (expects 32/32)
|
||||
|
||||
# 1) measure YOUR disk the way the engine uses it (parallel 19 MB random reads):
|
||||
gcc -O2 -fopenmp iobench.c -o iobench
|
||||
./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 0 # buffered, 8 threads
|
||||
./iobench /path/to/glm52_i4/out-00069.safetensors 19 64 8 1 # O_DIRECT (bypass cache)
|
||||
# Caveat (#86): iobench reads a bounded ~1 GB shard, so buffered reads on a big-RAM box
|
||||
# report the PAGE CACHE, not the disk. Use the O_DIRECT run (arg 1) for a true number, and
|
||||
# run it on a shard you haven't touched this session (a prior buffered run caches its pages).
|
||||
# On macOS there is no O_DIRECT — iobench uses F_NOCACHE, which stops *new* caching but can't
|
||||
# evict pages a prior buffered run already resident-mapped, so a macOS "O_DIRECT" figure right
|
||||
# after a buffered run still reads cache. Reboot or use a fresh shard for a real cold read.
|
||||
|
||||
# 2) chat; watch the per-turn stats line (tok/s, expert hit-rate, RSS):
|
||||
COLI_MODEL=/path/to/glm52_i4 ./coli chat
|
||||
|
||||
# 3) record expert usage, then pin the hottest experts in your spare RAM:
|
||||
STATS=stats.txt ./coli chat
|
||||
PIN=stats.txt PIN_GB=20 ./coli chat # scale PIN_GB to your free RAM
|
||||
|
||||
# 4) quality benchmarks (MMLU/HellaSwag/ARC):
|
||||
./coli bench
|
||||
```
|
||||
|
||||
## Back-of-envelope predictions
|
||||
|
||||
Decode is disk-bound: a cold token costs ~11.4 GB of expert reads; MTP
|
||||
speculation roughly halves the effective cost *once the cache is warm*; RAM
|
||||
turns cold reads into free cache hits.
|
||||
|
||||
| machine | expected |
|
||||
|---|---|
|
||||
| the dev box (WSL2 VHDX, ~1 GB/s, 25 GB RAM) | ~0.05–0.1 tok/s cold — proven baseline |
|
||||
| native Linux, PCIe4 NVMe (~3–5 GB/s random), 32 GB | ~0.5–1 tok/s |
|
||||
| PCIe5 NVMe or 2×NVMe RAID0 (~8–12 GB/s), 64 GB (PIN ~40 GB of hot experts) | ~2–4 tok/s |
|
||||
| 128–256 GB RAM, 12 cores (hot experts cached) | ~2–4 tok/s — matmul-bound: ~80 GFLOP/token vs ~250 GFLOP/s of our AVX2 kernels |
|
||||
| same RAM + 24–32 cores, or AVX-512/VNNI kernels | ~5–15 tok/s — interactive; kernel work is the multiplier |
|
||||
|
||||
These are estimates, not measurements.
|
||||
|
||||
## Community benchmarks (measured)
|
||||
|
||||
Real numbers from real machines, stock build (`setup.sh`, gcc 13), greedy decoding, `--ngen 32`, MTP active:
|
||||
|
||||
| machine | disk (iobench, 19 MB × 64, 8 threads) | config | measured |
|
||||
|---|---|---|---|
|
||||
| Intel Core Ultra 7 270K Plus (24 threads) · WSL2 · 24 GB RAM · NVMe VHDX ([#2](https://github.com/JustVugg/colibri/issues/2)) | 1.96 GB/s buffered · 2.74 GB/s O_DIRECT | default | 0.07 tok/s · expert hit 3–4% · RSS 14.1 GB |
|
||||
| 〃 | 〃 | `--topp 0.7` | **0.11 tok/s** · expert hit 11% · RSS 14.7 GB |
|
||||
| Apple M5 Max (18 cores) · macOS · 128 GB unified · internal SSD ([#4](https://github.com/JustVugg/colibri/issues/4), [#5](https://github.com/JustVugg/colibri/issues/5)) | ~4 GB/s cold (the 14.2 GB/s reading was cache-influenced — see note) | default, MTP off | **1.06 tok/s** · expert hit 23% · RSS 21.8 GB |
|
||||
| Apple M5 Max · macOS · 128 GB unified · 2 TB SSD · **Metal backend** ([#72](https://github.com/JustVugg/colibri/pull/72), [#87](https://github.com/JustVugg/colibri/issues/87)) | (macOS O_DIRECT figure unreliable — see note) | Metal on · `--ram 96` · 39.7 GB warm pin · MTP off | **1.83 tok/s** · expert hit 66% · warmed 1.11 → 1.83 over the run |
|
||||
| 〃 · 46.9 GB pin (2.94M-selection history) · `--ram 110`, 1024-token run ([#103](https://github.com/JustVugg/colibri/issues/103)) | 〃 | Metal on (experts + attention) · MTP off | **2.06 tok/s** · hit 72.5% · coherent output |
|
||||
| Mac Mini M4 Pro · macOS · **48 GB** unified · **Metal backend** ([#107](https://github.com/JustVugg/colibri/issues/107)) | 6.59 GB/s F_NOCACHE (fresh shard) | Metal on · `--ram 38` | **0.30 tok/s** (vs 0.18 CPU-only) |
|
||||
| Epyc 9654 ES · Linux · 4x16GB DDR5-4800-rdimm · Samsung PCIe Gen3 x4 NVME SSD | — | `MTP=1 DIRECT=1` | 0.31 tok/s · expert hit 35% · RSS 21.52 GB |
|
||||
| Ryzen AI 9 HX 370 (Framework 13) · Arch Linux · 128 GB · WD SN850X, BTRFS zstd ([#12](https://github.com/JustVugg/colibri/issues/12)) | — | int8 MTP head · `--cap 32` · 46.7 GB auto-learned PIN | **0.37 tok/s** · expert hit 66% · MTP acceptance 52% (2.59 tok/fw) · RSS 105 GB |
|
||||
| Ryzen 9 9950X (32 threads) · Linux · 123 GB · Crucial P3 QLC Gen3 ([#31](https://github.com/JustVugg/colibri/issues/31)) | 1.51 GB/s buffered | default, 2 runs from cold | 0.10 tok/s · hit 53% · profile 66% disk |
|
||||
| 〃 same machine, model moved to a Samsung 9100 PRO PCIe 5.0 ([#31](https://github.com/JustVugg/colibri/issues/31)) | **8.81 GB/s** O_DIRECT | 〃 (usage history retained) | **0.28 tok/s** · hit 57% · profile flips: 32% disk / **57% matmul** |
|
||||
| Ryzen AI Max+ 395 (Framework Desktop) · Ubuntu · 128 GB LPDDR5x · Intel Optane 905p PCIe 3.0 ([#39](https://github.com/JustVugg/colibri/issues/39)) | 3.27 GB/s buffered | int8 MTP head · fresh history (pure LRU, auto-raised cap 65) | 0.16 tok/s · hit 57% · profile 49% disk / 47% matmul |
|
||||
| 〃 five runs later — learned pin 47.6 GB ([#39](https://github.com/JustVugg/colibri/issues/39)) | 〃 | `--temp 0.7 --topp 0.7` | **0.40 tok/s** · hit 71% |
|
||||
| Ryzen 7 9800X3D (16T) · WSL2 · 70 GB RAM · Samsung 9100 PRO PCIe 5.0 · RTX 5090 ([#101](https://github.com/JustVugg/colibri/issues/101)) | **10.51 GB/s** O_DIRECT | MTP off · learned pin 24 GB · hit 54% · OMP hot-team on | **0.41 tok/s** · disk-bound (36.5 s disk vs 24.0 s matmul) · **CUDA expert tier ≈ 0%** (AVX-512 CPU matches the 5090) · `--topp 0.7` → **0.52 tok/s** |
|
||||
| EPYC 7443 (24C/48T, Zen3 AVX2) · Linux · **430 GB RAM** · NVMe RAID-Z1 via TrueNAS VM ([#104](https://github.com/JustVugg/colibri/issues/104)) | ~1 GB/s (VM overhead) | 77.5 GB pin · cap auto-raised to 194/layer · MTP off | **1.00 tok/s** · **hit 98%** · disk eliminated → **RAM-bandwidth + matmul bound** |
|
||||
| Intel i5-12600K (10C/16T, AVX2) · **native Windows 11, no WSL** · 32 GB · MinGW GCC 16.1 ([#113](https://github.com/JustVugg/colibri/issues/113)) | buffered (no O_DIRECT on MinGW) | int8 MTP head · cold, small-RAM (cap ~2/layer) | **0.08 tok/s** · hit 3.7% · **MTP 57% acceptance** — first native-Windows datapoint |
|
||||
| Ryzen 9 9950X3D2 (16C/32T, avx512-vnni) · native Linux · 121 GB · Samsung 9100 PRO **PCIe Gen5** · RTX 5090 (28 GB expert tier, 1475 pinned) ([#120](https://github.com/JustVugg/colibri/issues/120)) | **11.48 GB/s** O_DIRECT | `MTP=0 DIRECT=1 PIPE_WORKERS=16 PREFETCH=1` | **1.23 tok/s** |
|
||||
| Ryzen AI Max+ 395 (Strix Halo, 16C/32T Zen5, avx512-vnni) · Arch Linux · 128 GB unified LPDDR5x · SK hynix P41 PCIe 4.0 ([#124](https://github.com/JustVugg/colibri/issues/124)) | — | `DIRECT=1 PIPE=1 --topp 0.7` · auto-pin | 0.06 cold → **1.10 tok/s** sustained · later **1.83 tok/s** on current dev with `DIRECT=1 PIPE=1 PILOT_REAL=1 PILOT_TWO=1` ([#200](https://github.com/JustVugg/colibri/issues/200)) |
|
||||
| Intel Core Ultra 9 185H (16C/22T, avx-vnni) · **native Windows 11, no WSL** · 32 GB · Crucial P3 QLC NTFS · RTX 5070 Ti ([#128](https://github.com/JustVugg/colibri/issues/128), [#273](https://github.com/JustVugg/colibri/issues/273)) | — | int8 MTP head · warm cache · GPU-resident pipeline at decode | 0.03 cold → 0.5 warm CPU → **1.07 tok/s** with the pipe2 decode gate (#274) |
|
||||
| Dell Pro Max GB10 (DGX Spark: Grace, **aarch64 i8mm/sve2**) · Linux · 121 GB unified LPDDR5x · GB10 sm_121 ([#136](https://github.com/JustVugg/colibri/issues/136), [#161](https://github.com/JustVugg/colibri/issues/161)) | **5.58 GB/s** O_DIRECT | int8 MTP head · warm cache | 0.50 tok/s warm · **2.4 tok/s full-k8**, **3.33 tok/s** with `CACHE_ROUTE` (#199) |
|
||||
| **6 × RTX 5090 · dual Xeon Silver 4510 · 251 GB** (author's rig, [experiment log](experiments/glm52-6x5090-2026-07-12.md)) | NVMe | `CUDA_EXPERT_GB=auto PIN_GB=all` full residency · `COLI_CUDA_PIPE=2 TC_W4A16` · DRAFT=0 | **5.8–6.8 tok/s** decode · TTFT ~13 s · hit 89–100% |
|
||||
|
||||
### Takeaways
|
||||
|
||||
With 24 GB of RAM the engine auto-caps the expert cache to 2 slots/layer, so
|
||||
decode stays cold even on a fast disk — **on small-RAM machines the RAM cap, not
|
||||
the disk, is the binding constraint**; `--topp 0.7` alone bought a clean 1.6×
|
||||
end-to-end speedup. The 9950X pair is the cleanest bottleneck experiment: same
|
||||
machine, same history, only the disk swapped — ×5.8 disk bandwidth bought ×2.9
|
||||
tokens, and the profile **flipped from 66% disk to 57% matmul**. But the
|
||||
crossover depends on the CPU kernel: with OMP hot-team tuning on, an AVX-512 CPU
|
||||
can match an RTX 5090 on expert matmul ([#101](https://github.com/JustVugg/colibri/issues/101)),
|
||||
so **the GPU tier earns its VRAM only when the CPU is the weak link**. On
|
||||
multi-socket hosts, NUMA placement is a further lever: interleaving the resident
|
||||
weights across nodes measured **+13% (2-socket) and +40% (4-socket CPU-only)**
|
||||
([#82](https://github.com/JustVugg/colibri/issues/82)). On a 2-socket Xeon Silver
|
||||
4510 host with 6× RTX 5090, selective `COLI_NUMA=1` raised effective CPU-expert
|
||||
bandwidth from **42.42 to 58.26/65.89 GB/s** and greedy decode from **7.66 to
|
||||
9.02/9.17 tok/s** (64 tokens, `TEMP=0 DRAFT=0`, byte-identical output). Do not
|
||||
blanket-interleave a GPU host: it also spreads DMA staging pages and has measured
|
||||
up to a 10× regression; generated plans enable only the selective slab policy.
|
||||
|
||||
## Quality benchmark
|
||||
|
||||
**Measured** ([#108](https://github.com/JustVugg/colibri/issues/108)): the int4
|
||||
container scored **62.5% mean acc_norm** on hellaswag/arc/mmlu (0-shot
|
||||
log-likelihood, n=40) — but 0-shot MC scoring underserves a reasoning model, and
|
||||
the OLMoE fp16-vs-int4 A/B under the same harness measured the pure quantization
|
||||
cost at **-8.2pp**, concentrated on the hardest task (per-row int4 scales erode
|
||||
the small logit margins hard questions depend on — grouped scales recover ~63%
|
||||
of that loss, see [#225](https://github.com/JustVugg/colibri/issues/225)). The
|
||||
scale-granularity/rotation/lattice ablation lives in
|
||||
`tools/quant_ablation.py` ([#81](https://github.com/JustVugg/colibri/issues/81)).
|
||||
|
||||
```bash
|
||||
cd c
|
||||
pip install tokenizers datasets
|
||||
./coli bench # hellaswag, arc_challenge, mmlu — 40 questions each
|
||||
./coli bench hellaswag --limit 200 # one task, more questions
|
||||
./coli bench mmlu arc_challenge --ram 100 # pick tasks, set a RAM budget
|
||||
```
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
# CUDA backend (Linux)
|
||||
|
||||
colibrì includes an opt-in CUDA backend for model-resident tensors. Streaming
|
||||
experts deliberately remain on the original CPU path: copying an expert from
|
||||
NVMe to the GPU on every use would only replace the disk bottleneck with a PCIe
|
||||
bottleneck. Resident quantized tensors are uploaded lazily once and reused.
|
||||
|
||||
```bash
|
||||
cd c
|
||||
make cuda-test CUDA=1 # q8/q4/q2/f32 kernel correctness
|
||||
make CUDA=1
|
||||
# optional dense-path experiment (hot experts are configured below)
|
||||
COLI_CUDA=1 COLI_GPU=0 CUDA_DENSE=1 SNAP=/nvme/glm52_i4 ./glm 64 4 4
|
||||
```
|
||||
|
||||
Requirements: Linux, an NVIDIA driver, and a CUDA Toolkit under
|
||||
`/usr/local/cuda` (override with `CUDA_HOME=/path/to/cuda`).
|
||||
`CUDA_ARCH=native` builds for the GPU in the current machine. Requesting CUDA
|
||||
with a CPU-only binary, an invalid device, or an unavailable runtime fails at
|
||||
startup instead of silently falling back. For Windows, see
|
||||
[windows.md](windows.md) (runtime DLL path).
|
||||
|
||||
## The VRAM expert tier
|
||||
|
||||
A measured `PIN` profile promotes its hottest experts into a persistent VRAM
|
||||
tier while keeping the rest in RAM:
|
||||
|
||||
```bash
|
||||
STATS=stats.txt SNAP=/nvme/glm52_i4 ./glm 64 4 4 # collect routing frequencies first
|
||||
COLI_CUDA=1 COLI_GPU=0 CUDA_EXPERT_GB=16 \
|
||||
PIN=stats.txt PIN_GB=160 SNAP=/nvme/glm52_i4 ./glm 64 4 4
|
||||
|
||||
# multi-GPU expert tier, 150 GB total budget across six 32 GB devices
|
||||
COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=150 \
|
||||
CUDA_DENSE=1 PIN=stats.txt PIN_GB=300 RAM_GB=226 \
|
||||
SNAP=/nvme/glm52_i4 ./glm 64 4 4
|
||||
|
||||
# large-RAM host: fill safe VRAM, then keep every remaining expert in RAM
|
||||
COLI_CUDA=1 COLI_GPUS=0,1,2,3,4,5 CUDA_EXPERT_GB=auto \
|
||||
CUDA_DENSE=1 COLI_CUDA_ATTN=1 PIN=stats.txt PIN_GB=all RAM_GB=auto \
|
||||
SNAP=/nvme/glm52_i4 ./glm 64 4 4
|
||||
```
|
||||
|
||||
Selected experts are uploaded during startup, so capacity failures occur before
|
||||
inference. The budget is clamped against free VRAM after reserving the projected
|
||||
dense resident set and 2 GB of runtime headroom per device. With `COLI_GPUS`,
|
||||
`CUDA_EXPERT_GB` is a total budget across the device set; experts are assigned
|
||||
whole to the least-loaded device that can hold them. Multi-GPU runs default to
|
||||
`PIN_FILL=1` (measured hot set first, then unused VRAM filled with zero-heat
|
||||
experts) and `CUDA_RELEASE_HOST=1` (RAM copy released after upload, reloaded
|
||||
from disk only if CUDA later fails).
|
||||
|
||||
`CUDA_EXPERT_GB=auto` fills each device up to measured free memory minus
|
||||
projected dense tensors and headroom. `PIN_GB=all` then loads the remaining
|
||||
routed experts into RAM **up to the `--ram` budget** (it clamps — [#229](https://github.com/JustVugg/colibri/issues/229)),
|
||||
eliminating decode-time disk misses when capacity permits. This mode is intended
|
||||
for dedicated high-memory inference hosts.
|
||||
|
||||
### Full-residency reference result (6× RTX 5090, 251 GiB host)
|
||||
|
||||
`CUDA_EXPERT_GB=auto PIN_GB=all` selected a 176.7 GB VRAM tier + 191.3 GB RAM
|
||||
tier (all 19,456 experts resident), adapting the VRAM tier every 16 tokens.
|
||||
With the GPU-resident pipeline (`COLI_CUDA_PIPE=2`) and Tensor-Core W4A16
|
||||
dispatch (`COLI_CUDA_TC_W4A16=1`), 96-token greedy decode measured
|
||||
**5.8–6.8 tok/s** (TTFT ~13 s; 1571-token prefill ~122 s then 4.2 tok/s).
|
||||
Full experiment log: [experiments/glm52-6x5090-2026-07-12.md](experiments/glm52-6x5090-2026-07-12.md).
|
||||
These are host-specific capacity results, not portable defaults.
|
||||
|
||||
## The GPU-resident pipeline (`COLI_CUDA_PIPE`)
|
||||
|
||||
`COLI_CUDA_PIPE=2` keeps the residual stream on-device across layers: rmsnorms,
|
||||
residual adds, router GEMMs and the shared expert run on the GPU while the CPU
|
||||
expert loop runs uninterrupted, with batched attention and grouped expert
|
||||
uploads at prefill. On a single-GPU host this also pays at decode (S=1):
|
||||
**+49%** measured on a 5070 Ti ([#273](https://github.com/JustVugg/colibri/issues/273)/#274);
|
||||
on multi-GPU hosts the per-layer P2P hops cancel the gain, so the decode gate is
|
||||
device-count aware. `COLI_CUDA_TC_W4A16=1` enables Tensor-Core int4×fp16 mixed
|
||||
dispatch for batched rows (pays at ≥16 rows).
|
||||
|
||||
## Notes and limitations
|
||||
|
||||
- Text-mode timing reports prefill separately from decode.
|
||||
- MTP speculation defaults off on CUDA (cold draft routes increase expert
|
||||
traffic); explicit `DRAFT=n` overrides. Since #294, `SPEC_PIN=1` keeps
|
||||
draft/verify kernels consistent when speculation is on.
|
||||
- Devices use independent contexts; a single expert is not sharded. Kernels are
|
||||
correctness-first custom kernels.
|
||||
- Profile quality matters more than raw VRAM capacity: the same 150 GB tier
|
||||
measured 0.94–1.64 tok/s hot-first vs 0.29 tok/s filled without routing heat.
|
||||
- The GPU tier earns its VRAM only when the CPU is the weak link — a tuned
|
||||
AVX-512 CPU can match a 5090 on expert matmul
|
||||
([#101](https://github.com/JustVugg/colibri/issues/101)).
|
||||
|
||||
## Reproducible backend A/B without the full checkpoint
|
||||
|
||||
```bash
|
||||
cd c
|
||||
python tools/make_glm_bench_model.py --output /nvme/colibri-bench-medium --device cuda
|
||||
python tools/benchmark_cuda_fixture.py --model /nvme/colibri-bench-medium --gpu 0
|
||||
```
|
||||
|
||||
The 313M-parameter fixture has random weights and is not a language model. It
|
||||
preserves the real MLA/MoE/streaming shapes to compare CPU streaming, dense-only
|
||||
CUDA, CPU hot-store, and CUDA hot-expert execution with identical replay tokens.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 875 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 421 KiB After Width: | Height: | Size: 428 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 545 KiB After Width: | Height: | Size: 588 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 117 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 151 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 115 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 62 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 139 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 117 KiB |
@@ -0,0 +1,30 @@
|
||||
# Metal backend (Apple Silicon, experimental)
|
||||
|
||||
On Apple Silicon the decode profile is matmul-bound, and unified memory removes
|
||||
the PCIe copy tax that keeps CUDA's streaming experts on the CPU — so colibrì
|
||||
has an opt-in Metal backend that runs the **routed-expert SwiGLU (batched,
|
||||
zero-copy from the RAM slabs)**, the **fused decode attention** (full MLA layer
|
||||
in one command buffer, S≤4), and **prefill's large GEMMs** on the GPU.
|
||||
Token-exact vs the CPU path.
|
||||
|
||||
```bash
|
||||
cd c
|
||||
make glm METAL=1 # macOS only; no Xcode needed (shader compiles at runtime)
|
||||
make metal-test # standalone kernel/attention correctness vs CPU reference
|
||||
COLI_METAL=1 COLI_MODEL=/path/glm52_i4 ./coli chat --ram 96
|
||||
```
|
||||
|
||||
Measured on an M4 Max (128 GB, warm cache, MTP on): CPU 0.30 → Metal
|
||||
**0.42 tok/s (~1.4×)** (best config adds `DIRECT=1`; ~3× vs this machine's
|
||||
first cold run). An M5 Max with a 46.9 GB learned pin reached **2.06 tok/s**
|
||||
([#103](https://github.com/JustVugg/colibri/issues/103); see also the
|
||||
[M5 Max performance report](METAL-M5MAX-PERF-REPORT.md)).
|
||||
|
||||
Key design points: Metal's ~5 ms submit latency makes per-matmul dispatch a
|
||||
loss — everything is batched into few command buffers per layer, and the
|
||||
resident experts' GPU work is submitted *before* the missed experts' disk reads
|
||||
so I/O and compute overlap. `COLI_METAL_GEMM_MIN` tunes the prefill GEMM row
|
||||
threshold (default 16). Streaming, cache, MTP, DSA and the persistence formats
|
||||
are unchanged; every GPU path falls back to the CPU per-block on any fault.
|
||||
Numerics are dequant→f32-MAC (same as the CUDA tier); greedy outputs are
|
||||
byte-identical to the CPU engine.
|
||||
@@ -0,0 +1,164 @@
|
||||
# Quick Start — from zero to a running model
|
||||
|
||||
A step-by-step guide for first-time users on **Linux**, **Windows**, and **macOS**.
|
||||
No prior experience with C, CUDA, or model conversion is assumed. If you get
|
||||
stuck, `./coli doctor` (below) tells you exactly what's missing.
|
||||
|
||||
> **What you're setting up:** colibrì runs a very large Mixture-of-Experts model
|
||||
> (e.g. GLM-5.2, 744B parameters) on a normal machine by streaming the model's
|
||||
> experts from disk instead of needing them all in RAM. The engine is a single
|
||||
> C program; Python is only used once, to prepare the model files.
|
||||
|
||||
---
|
||||
|
||||
## 0. What you need first (prerequisites)
|
||||
|
||||
| | Minimum | Recommended |
|
||||
|---|---|---|
|
||||
| **RAM** | ~16 GB | 24 GB+ |
|
||||
| **Free disk** | ~380 GB for the int4 model | a fast NVMe SSD (streaming speed = your token speed) |
|
||||
| **OS** | Linux, Windows 10/11, or macOS | any |
|
||||
| **Tools** | a C compiler + `make` + `git` + `python3` | — |
|
||||
|
||||
You do **not** need a GPU. A GPU only helps if you have one; the engine runs
|
||||
CPU-only by default.
|
||||
|
||||
---
|
||||
|
||||
## 1. Install the build tools
|
||||
|
||||
### Linux (Ubuntu / Debian)
|
||||
|
||||
```bash
|
||||
sudo apt update
|
||||
sudo apt install -y build-essential git python3
|
||||
```
|
||||
|
||||
`build-essential` gives you `gcc`, `make`, and OpenMP (libgomp) — everything the
|
||||
engine needs.
|
||||
|
||||
### Windows
|
||||
|
||||
You have two options.
|
||||
|
||||
**Option A — download a prebuilt binary (no compiler needed).**
|
||||
Grab the latest `colibri-<version>-windows-x86_64.zip` from the
|
||||
[Releases page](https://github.com/JustVugg/colibri/releases), unzip it, and
|
||||
skip to [step 3](#3-get-the-model). Python 3 (from
|
||||
[python.org](https://www.python.org/downloads/)) is still needed if you want to
|
||||
convert a model yourself.
|
||||
|
||||
**Option B — build from source with MSYS2.**
|
||||
Install [MSYS2](https://www.msys2.org/), open the **UCRT64** shell, and run:
|
||||
|
||||
```bash
|
||||
pacman -S --needed mingw-w64-ucrt-x86_64-gcc make git python
|
||||
```
|
||||
|
||||
### macOS
|
||||
|
||||
```bash
|
||||
xcode-select --install # C compiler (clang)
|
||||
brew install libomp git python # OpenMP for multithreading
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. Get the code and build the engine
|
||||
|
||||
```bash
|
||||
git clone https://github.com/JustVugg/colibri.git
|
||||
cd colibri/c
|
||||
./setup.sh
|
||||
```
|
||||
|
||||
`setup.sh` checks your compiler and OpenMP, builds the engine, and runs a tiny
|
||||
self-test. When it prints:
|
||||
|
||||
```
|
||||
engine self-test: 32/32 (expected 32/32)
|
||||
```
|
||||
|
||||
the engine is working correctly. (On Windows Option A you already have the
|
||||
binary — you can skip this step.)
|
||||
|
||||
---
|
||||
|
||||
## 3. Get the model
|
||||
|
||||
You have two paths.
|
||||
|
||||
### Easiest — download a ready-made int4 container
|
||||
|
||||
A pre-converted **GLM-5.2 int4** model is on Hugging Face. **Use the version
|
||||
with the int8 MTP heads** (the plain int4 heads disable speculative decoding —
|
||||
see [#8](https://github.com/JustVugg/colibri/issues/8)):
|
||||
|
||||
**https://huggingface.co/mateogrgic/GLM-5.2-colibri-int4-with-int8-mtp**
|
||||
|
||||
Download it into a folder on a fast disk, e.g. `/nvme/glm52_i4` (Linux/macOS) or
|
||||
`D:\glm52_i4` (Windows). It is about **372 GB**, so make sure you have the space.
|
||||
|
||||
### Or convert it yourself from the FP8 source
|
||||
|
||||
One resumable command downloads and converts the model shard by shard, so it
|
||||
never needs the full ~756 GB on disk at once:
|
||||
|
||||
```bash
|
||||
./coli convert --model /nvme/glm52_i4
|
||||
```
|
||||
|
||||
This step uses Python and runs only once. Safe to interrupt and re-run — it
|
||||
resumes where it left off.
|
||||
|
||||
---
|
||||
|
||||
## 4. Run it
|
||||
|
||||
Point `COLI_MODEL` at the folder from step 3 and start chatting:
|
||||
|
||||
```bash
|
||||
# Linux / macOS
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli chat
|
||||
|
||||
# Windows (UCRT64 shell)
|
||||
COLI_MODEL=/d/glm52_i4 ./coli chat
|
||||
```
|
||||
|
||||
Useful first commands:
|
||||
|
||||
```bash
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli doctor # read-only check: is everything ready?
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli plan # shows where the model will live (RAM/disk/GPU)
|
||||
COLI_MODEL=/nvme/glm52_i4 ./coli chat --topp 0.85 # faster: reads less from disk, same quality
|
||||
```
|
||||
|
||||
> **Tip:** `--topp 0.85` is worth adding on a disk-bound machine — it reads
|
||||
> fewer expert bytes per token with no quality loss, which directly means more
|
||||
> tokens per second.
|
||||
|
||||
---
|
||||
|
||||
## 5. What to expect
|
||||
|
||||
- **First launch loads the resident weights** (~10 GB) — this takes a moment.
|
||||
- **Speed depends on your disk.** The experts stream from storage, so a fast
|
||||
NVMe SSD is the single biggest factor in tokens/second. On a slow or shared
|
||||
disk, generation can be well under 1 token/second — that's expected, and it's
|
||||
the honest cost of running a 744B model on a small machine.
|
||||
- **It's still the full model.** Placement only changes speed, never the model's
|
||||
answers or precision.
|
||||
|
||||
If something doesn't work, run `./coli doctor` — it reports exactly what's
|
||||
missing (compiler, model files, permissions) and how to fix it.
|
||||
|
||||
---
|
||||
|
||||
## Where to go next
|
||||
|
||||
| Topic | Doc |
|
||||
|---|---|
|
||||
| Windows native build (and CUDA DLL) | [docs/windows.md](windows.md) |
|
||||
| Tuning: cache, prefetch, speculation | [docs/tuning.md](tuning.md) |
|
||||
| OpenAI-compatible API + web dashboard | [docs/api.md](api.md) |
|
||||
| Every environment variable | [docs/ENVIRONMENT.md](ENVIRONMENT.md) |
|
||||
@@ -0,0 +1,107 @@
|
||||
# The serve protocols — engine ⇄ server wire format
|
||||
|
||||
The engine speaks two line-oriented protocols over stdin/stdout. Both are plain text
|
||||
plus byte-counted payload frames; every outbound line is written with a trailing
|
||||
`fflush`, one line per write. On Windows both ends of the pipe are switched to binary
|
||||
mode at startup — the CRT's CRLF translation otherwise corrupts the sentinels and
|
||||
stalls byte-counted reads (#195).
|
||||
|
||||
| protocol | entry | selected by | used by |
|
||||
|---|---|---|---|
|
||||
| **mux** (continuous batching, up to 16 KV slots) | `run_serve_mux` | `SERVE_BATCH=1` | `openai_server.py`, `coli web` |
|
||||
| **legacy** (single slot, interactive) | `run_serve` | `SERVE=1` (without `SERVE_BATCH`) | `coli chat` |
|
||||
|
||||
This document is the reference for the **mux** protocol; the legacy protocol is
|
||||
summarized at the end. Line formats below are quoted from the emitting `printf`s in
|
||||
`glm.c` — if this document and the code disagree, the code wins and this file needs a PR.
|
||||
|
||||
## Startup handshake (engine → server)
|
||||
|
||||
```
|
||||
\x01\x01READY\x01\x01
|
||||
STAT 0 0.00 0.0 <rss_gb>
|
||||
HWINFO <cores> <ram_total_gb> <ram_avail_gb> <ngpu> <vram_total_gb> <cpu_name>|<gpu_name>
|
||||
TIERS <vram_experts> <ram_experts> <disk_experts> <vram_gb> <ram_gb>
|
||||
EMAP <rows> <cols> <hex>
|
||||
```
|
||||
|
||||
The server must not send requests before `READY`. `HWINFO`/`TIERS`/`EMAP` are
|
||||
telemetry (see below) and may grow — **servers must ignore line kinds they do not
|
||||
recognize**; that is the protocol's forward-compatibility rule.
|
||||
|
||||
## Requests (server → engine)
|
||||
|
||||
```
|
||||
SUBMIT <id> <slot> <bytes> <max_tokens> <temperature> <top_p>\n<payload>\n
|
||||
CANCEL <id>\n
|
||||
```
|
||||
|
||||
- `id` — non-zero u64, unique among in-flight requests.
|
||||
- `slot` — KV slot index, `0 … KV_SLOTS-1` (`KV_SLOTS` env, 1–16, default 1). A slot
|
||||
holds one conversation's KV; the engine matches the tokenized payload against the
|
||||
slot's history and reuses the common prefix (truncate-and-extend), so stateless
|
||||
HTTP turns keep their cache.
|
||||
- `bytes` — exact byte length of `payload` (UTF-8, may contain newlines). The engine
|
||||
reads exactly that many bytes after the header line, then one trailing `\n`.
|
||||
- `payload` — the fully rendered prompt (the server owns the chat template).
|
||||
- EOF on stdin = graceful shutdown: in-flight requests finish first.
|
||||
|
||||
Prefill is serial; decode is continuously batched — every active slot contributes
|
||||
one row per forward.
|
||||
|
||||
## Responses (engine → server)
|
||||
|
||||
Per request, in order:
|
||||
|
||||
```
|
||||
DATA <id> <n>\n<n bytes of UTF-8>\n # a decoded token's text; repeated
|
||||
TOPK <id> 5 <logprob> <hextext> ... ×5 # candidates for the sampled token (SERVE_TOPK=1)
|
||||
HITS <rows> <cols> <hex> # ~every 6 tokens: routed-expert bitmap since last HITS
|
||||
REPIN <layer> <eid> <old_tier> <gpu> # live re-pin swap events, as they happen
|
||||
...
|
||||
DONE <id> STAT <emitted> <tok_s> <hit_pct> <rss_gb> <prompt_tokens> <length_limited>
|
||||
```
|
||||
|
||||
Errors replace the stream: `ERROR <id> <CODE>` with codes `BAD_FRAME`, `BAD_REQUEST`,
|
||||
`SLOT_BUSY`, `DUPLICATE_ID`, `EMPTY_PROMPT`, `NOT_FOUND` (CANCEL of unknown id),
|
||||
`CANCELLED`. A `CANCEL` is acknowledged by `ERROR <id> CANCELLED` after the slot's KV
|
||||
is persisted.
|
||||
|
||||
Immediately before each `DONE` the engine emits a telemetry block for the finished
|
||||
turn: `HWINFO`, `PERF`, `ENTROPY`, `GPUS`, `TIERS`, `EMAP`, `HITS` (formats below).
|
||||
`.coli_usage` is persisted at every turn end, not only at exit.
|
||||
|
||||
## Telemetry lines
|
||||
|
||||
| line | format | meaning |
|
||||
|---|---|---|
|
||||
| `TIERS` | `TIERS <vram> <ram> <disk> <vram_gb> <ram_gb>` | expert count per tier + resident bytes |
|
||||
| `HWINFO` | `HWINFO <cores> <ram_total> <ram_avail> <ngpu> <vram_total> <cpu>\|<gpu>` | host snapshot (GBs are floats) |
|
||||
| `EMAP` | `EMAP <rows> <cols> <hex>` | one byte per expert, row-major over `rows×cols` (sparse layers +MTP × experts): `byte = (tier<<6) \| heat` — 2-bit tier (0 disk / 1 RAM / 2 VRAM), 6-bit log₂-bucketed usage heat |
|
||||
| `HITS` | `HITS <rows> <cols> <hex>` | 1 bit per expert, experts routed since the previous `HITS` |
|
||||
| `PERF` | `PERF <id> <dt> <t_edisk> <t_ewait> <t_emm> <t_attn> <t_kvb> <t_head>` | this turn's PROFILO deltas, seconds |
|
||||
| `ENTROPY` | `ENTROPY <h0> <h1> …` | per-sparse-layer routing entropy of the turn, bits |
|
||||
| `GPUS` | `GPUS <n> (<used_gb> <total_gb> <experts>)×n` | per-device VRAM + resident expert count (CUDA builds) |
|
||||
| `TOPK` | `TOPK <id> 5 (<logprob> <hextext>)×5` | token text hex-encoded so the line stays line-shaped |
|
||||
| `REPIN` | `REPIN <layer> <eid> <old_tier> <gpu>` | one line per hot-store swap (`REPIN=n` mode) |
|
||||
|
||||
All telemetry is advisory: servers render what they know and skip the rest.
|
||||
|
||||
## HTTP surface (`openai_server.py`)
|
||||
|
||||
- `POST /v1/chat/completions` — OpenAI-compatible; streaming responses emit one extra
|
||||
SSE frame `data: {"colibri": {stats, perf, topk, entropy, gpus, repin}}` immediately
|
||||
before `data: [DONE]`; non-streaming responses attach the same object as a
|
||||
`"colibri"` field.
|
||||
- `GET /experts` — the latest `EMAP`/`HITS` state: `{rows, cols, map, hits, seq,
|
||||
gpus, entropy, repin}`.
|
||||
- `GET /*` — static hosting of `web/dist` (SPA fallback, path-traversal-safe), plus
|
||||
`experts.json` if published there (the measured expert atlas, #175/#218).
|
||||
|
||||
## Legacy protocol (`run_serve`, `coli chat`)
|
||||
|
||||
Interactive lines are prompts; control frames: `\x02RESET` (clear history),
|
||||
`\x02MORE` (continue an NGEN-truncated answer), and
|
||||
`\x02PROMPT <bytes> <max_tokens> <temperature> <top_p> [kv_slot]\n<prompt>\n`
|
||||
(the pre-mux API mode). Responses are raw text terminated by `\x01\x01END\x01\x01`
|
||||
plus a `STAT` line. New integrations should use the mux protocol.
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
# Tuning & runtime knobs
|
||||
|
||||
Everything here is opt-in; the defaults are chosen so a plain `./coli chat`
|
||||
is safe on any machine. See also [SETTINGS.md](SETTINGS.md) and
|
||||
[ENVIRONMENT.md](ENVIRONMENT.md) for the full variable inventory.
|
||||
|
||||
## The knobs that matter most
|
||||
|
||||
| knob | what it does |
|
||||
|---|---|
|
||||
| `--temp T` | token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy) |
|
||||
| `--topp 0.7` | adaptive expert top-p (30–40% less disk; lossy — prints a warning) |
|
||||
| `--ngen N` | max tokens per answer (`:more` in chat continues a truncated one) |
|
||||
| `--repin N` | adapt RAM/VRAM hot experts every N emitted tokens |
|
||||
| `RAM_GB=<n>` | claim more RAM for the expert cache than the conservative auto-detect |
|
||||
| `PIN=stats PIN_GB=g` | pin the hottest experts from a measured usage profile |
|
||||
| `DRAFT=n` | MTP draft depth (0 disables speculation) |
|
||||
| `GRAMMAR=g.gbnf` | grammar-forced drafts for constrained JSON/NDJSON output ([docs](grammar-draft.md)) |
|
||||
| `THINK=1` | enable GLM-5.2's reasoning block |
|
||||
| `PILOT=1` | router-lookahead disk prefetch (see below) |
|
||||
| `URING=1` | Linux-only batched expert I/O (implies `PIPE=1`) |
|
||||
| `PIPE=0` | disable the async expert-load pool (default ON — overlaps `pread` with matmul, −18% disk service) |
|
||||
| `DIRECT=1` | O_DIRECT expert reads (measured **+65%** alone on a Strix Halo, [#200](https://github.com/JustVugg/colibri/issues/200)) |
|
||||
| `COLI_NUMA=1` | interleave resident weights across NUMA nodes on multi-socket hosts ([#82](https://github.com/JustVugg/colibri/issues/82)) |
|
||||
| `CACHE_ROUTE=1` | cache-aware max-rank routing (opt-in, [#199](https://github.com/JustVugg/colibri/issues/199)) |
|
||||
| `AUTOPIN=0` | disable the learning cache's auto-pin |
|
||||
| `CAP_RAISE=0` | don't auto-grow the expert cache |
|
||||
| `KVSAVE=0` | disable KV-cache persistence |
|
||||
| `TF=1` | teacher-forcing validation |
|
||||
|
||||
## Resource policy
|
||||
|
||||
`coli plan` reports the planned hot (VRAM), warm (RAM), and cold backing (disk)
|
||||
tiers, the reason for each placement, and the expected bottleneck. The default
|
||||
`--policy quality` and `--policy balanced` modes preserve checkpoint quantization
|
||||
and router decisions unless `--topk` or `--topp` is passed; those explicit lossy
|
||||
overrides print a warning and proceed.
|
||||
|
||||
Auto-tier plans size OpenMP from physical cores and bind workers across cores.
|
||||
Memory-bound quantized kernels can regress sharply when SMT siblings compete for
|
||||
limited memory channels; explicit `OMP_*` settings always take precedence.
|
||||
|
||||
```bash
|
||||
coli plan --model /models/glm52_i4 --policy quality
|
||||
coli run --auto-tier --policy quality "Explain MoE offloading"
|
||||
# Explicit research-only router reduction:
|
||||
coli run --policy experimental-fast --topk 4 "Benchmark prompt"
|
||||
```
|
||||
|
||||
Disk is an immutable recovery source, not a normal decode target. If the plan
|
||||
leaves cold expert bytes on disk, speed depends on cache hit rate; output quality
|
||||
does not.
|
||||
|
||||
Cold expert reads can use a deferred pipeline: resident RAM/VRAM experts execute
|
||||
while missing experts are loaded in a bounded background I/O pool, then the cold
|
||||
results join before the layer completes. The pool engages only under `PIPE=1`;
|
||||
`PIPE_WORKERS=n` sets its worker count (default 8). Profiling reports both disk
|
||||
service time and the smaller foreground-visible wait time so overlap is explicit.
|
||||
|
||||
`--policy balanced` enables lossless live placement (`REPIN=64`). At safe request
|
||||
boundaries, a per-layer LFRU score combines decaying session frequency with recent
|
||||
access and replaces at most four sufficiently colder pinned experts. `--policy
|
||||
quality` leaves live replacement off by default; `REPIN=0` always disables it.
|
||||
|
||||
## The learning cache
|
||||
|
||||
The engine records which experts your usage actually routes to (`.coli_usage`
|
||||
next to the model, updated every turn) and at startup automatically pins the
|
||||
hottest ones in spare RAM — colibrì literally gets faster the more you use it.
|
||||
`PIN=auto` seeds the pin directly from the live usage history
|
||||
([#301](https://github.com/JustVugg/colibri/pull/301)).
|
||||
|
||||
**The expert cache auto-sizes to your RAM** (since 2026-07-10): the engine
|
||||
*raises* the LRU cap to fill your `--ram` budget instead of only lowering it.
|
||||
If you benchmarked colibrì before that date, rerun — your numbers were capped.
|
||||
|
||||
**Live tier adaptation** (`--repin N`, opt-in): at safe turn boundaries, a
|
||||
decaying session heat map replaces cold pinned experts with hotter streamed
|
||||
experts. A 25% hysteresis and a four-swap limit prevent tier thrashing.
|
||||
Persistent `.coli_usage` remains the long-term signal and is not decayed.
|
||||
|
||||
## Router-lookahead prefetch (`PILOT=1`, experimental)
|
||||
|
||||
GLM-5.2's expert routing is measurably predictable *ahead of time* — applying
|
||||
layer L+1's router to layer L's post-attention state recalls **71.6%** of the
|
||||
true top-8 (vs 41.3% for "same experts as last token"). `PILOT=1` issues
|
||||
next-layer expert readahead from a dedicated I/O thread while the current layer
|
||||
computes. `PILOT_REAL=1` moves the prefetched loads off the critical path
|
||||
(measured +11pp hit rate on a big-cache host), and `PILOT_TWO=1` folds the
|
||||
computed shared-expert into the prediction (+3% recall,
|
||||
[#200](https://github.com/JustVugg/colibri/issues/200)). On disk-saturated
|
||||
hosts hint-only PILOT can be net negative — measure on yours.
|
||||
|
||||
## Speculation and reproducibility
|
||||
|
||||
Speculative decoding requires that the draft and verify paths compute the same
|
||||
function — `SPEC_PIN=1` (default since [#294](https://github.com/JustVugg/colibri/pull/294))
|
||||
pins every forward issued while drafts are live to the platform's S=1 kernel
|
||||
family. For byte-exact reproducibility across runs: `DRAFT=0`, plus `IDOT=0
|
||||
COLI_CUDA=0` if you also want kernel-family/GPU independence. Acceptance
|
||||
percentages are not comparable across engine versions under `--topp`
|
||||
([#163](https://github.com/JustVugg/colibri/issues/163) has the full story).
|
||||
|
||||
## Conversations reopen warm
|
||||
|
||||
`coli chat` persists the compressed MLA KV-cache to disk after every turn
|
||||
(`.coli_kv`, ~182 KB/token, appended incrementally, crash-safe). Close the chat,
|
||||
reopen it tomorrow — the model still remembers the whole conversation and **zero
|
||||
re-prefill happens**: validated byte-identical to an uninterrupted session.
|
||||
`:reset` clears it, `KVSAVE=0` disables it.
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
# Windows 11 native install — a complete walkthrough (no WSL)
|
||||
|
||||
A start-to-finish, reproducible path from a fresh Windows 11 machine to GLM-5.2 generating tokens, with the GPU tier. Every step and every failure mode below was hit and verified on real hardware: Core Ultra 9 285K (AVX-VNNI) / RTX 5080 (sm_120) / 128 GB RAM / Windows 11 24H2 (issue #306). Steps are ordered so the long downloads run while you build.
|
||||
|
||||
## 0. What you need
|
||||
|
||||
| Piece | Why | Get it |
|
||||
|---|---|---|
|
||||
| git, Python 3 | clone + `coli` launcher | winget / python.org |
|
||||
| MinGW-w64 gcc + make | builds the engine (MSVC can't) | `scoop install mingw-winlibs`, MSYS2, or portable **w64devkit** (no admin, unzip and go) |
|
||||
| CUDA Toolkit ≥ 12.8 | GPU tier; ≥12.8 required for Blackwell/sm_120 | `winget install Nvidia.CUDA` |
|
||||
| MSVC Build Tools (C++ workload) | nvcc's host compiler for the CUDA DLL | `winget install Microsoft.VisualStudio.2022.BuildTools` + "Desktop development with C++" |
|
||||
| ~400 GB free on a local NVMe | the int4 model (~370–384 GB) | NTFS is fine; **never** a network mount |
|
||||
|
||||
RAM: 16 GB minimum, more = bigger expert cache = faster. The build itself needs none of the CUDA/MSVC pieces — do the CPU build first, add the GPU tier later.
|
||||
|
||||
## 1. Start the model download first (it's the long pole)
|
||||
|
||||
```powershell
|
||||
python -m pip install -U "huggingface_hub[hf_transfer]"
|
||||
$env:HF_HUB_ENABLE_HF_TRANSFER = "1"
|
||||
hf download <model-repo> --local-dir D:\glm52_i4
|
||||
```
|
||||
|
||||
Use the container recommended in the README (with **int8 MTP heads** — int4 heads silently give 0% draft acceptance). The download is resumable: if it stops, rerun the same command. Expect hours; everything below fits inside them.
|
||||
|
||||
## 2. Build the engine (CPU)
|
||||
|
||||
From a normal PowerShell, in the repo's `c\` directory:
|
||||
|
||||
```powershell
|
||||
make glm.exe ARCH=native # ARCH=native unlocks AVX-VNNI on Alder Lake+/Arrow Lake
|
||||
make iobench.exe # disk benchmark, useful before committing to the download
|
||||
```
|
||||
|
||||
Warnings about `#pragma comment` and unused variables are normal (MSVC-isms gcc ignores). The engine banner should print `idot: avx-vnni` on VNNI-capable CPUs — if it says avx2, you built without `ARCH=native`.
|
||||
|
||||
### ⚠️ Smart App Control will block your fresh binary
|
||||
|
||||
On Windows 11 machines with **Smart App Control** enforced (`VerifiedAndReputablePolicyState = 1`), running your self-compiled `glm.exe` fails with:
|
||||
|
||||
```
|
||||
Program 'glm.exe' failed to run: An Application Control policy has blocked this file
|
||||
```
|
||||
|
||||
This is not Defender and not Mark-of-the-Web — SAC blocks *all* unsigned, unknown binaries, which includes anything you compile yourself. **Fix:** Windows Security → App & browser control → Smart App Control settings → **Off**, then **reboot** (the policy only reloads on restart). Note SAC is one-way: re-enabling later requires resetting Windows. If the settings page is missing, the registry equivalent is setting `HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy\VerifiedAndReputablePolicyState` to `0` (admin PowerShell), then rebooting. Check your current state before touching anything:
|
||||
|
||||
```powershell
|
||||
(Get-ItemProperty "HKLM:\SYSTEM\CurrentControlSet\Control\CI\Policy").VerifiedAndReputablePolicyState
|
||||
# 0 = off, 1 = enforced, 2 = evaluation
|
||||
```
|
||||
|
||||
## 3. Build the CUDA DLL (GPU tier)
|
||||
|
||||
nvcc needs MSVC as host compiler, so this one step must run from a shell with the MSVC environment: open **"x64 Native Tools Command Prompt for VS 2022"** from the Start menu (plain PowerShell will fail the `cl` check). Then:
|
||||
|
||||
```cmd
|
||||
make cuda-dll CUDA_ARCH=sm_120 # match your GPU: sm_120 Blackwell, sm_89 Ada, ...
|
||||
make glm.exe CUDA_DLL=1 ARCH=native # relink host with the runtime loader
|
||||
```
|
||||
|
||||
Two pitfalls, both fixed on current `dev` (#314) but worth knowing on older checkouts:
|
||||
|
||||
- **Spaces in `CUDA_HOME`** (`C:\Program Files\...`) used to break the recipe → fixed; nvcc now comes from PATH and `"$(NVCC)"` is quoted.
|
||||
- **`make glm.exe CUDA_DLL=1` after a CPU-only build** used to report `up to date` and silently keep the CPU-only binary (GPU tier never engages, no error). Current `dev` has a build-config stamp that forces the relink. On older trees: delete `glm.exe` first.
|
||||
|
||||
Sanity check: first GPU run should print `[CUDA] device 0: <your GPU>, ... sm_XX` and `[CUDA] mode: routed experts + resident dense tensors`.
|
||||
|
||||
## 4. First run
|
||||
|
||||
```powershell
|
||||
cd <repo>\c
|
||||
$env:OMP_NUM_THREADS = "<physical cores>"
|
||||
python coli run "Explain what a mixture-of-experts model is." --model D:\glm52_i4 --ngen 48
|
||||
```
|
||||
|
||||
The first run is cold — expect the profile to be dominated by `expert-disk` while the cache warms; hit rate climbs run over run. GPU tier on top:
|
||||
|
||||
```powershell
|
||||
$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_DENSE="1"; $env:CUDA_EXPERT_GB="4"
|
||||
python coli run "..." --model D:\glm52_i4 --ngen 64
|
||||
```
|
||||
|
||||
Size `CUDA_EXPERT_GB` so dense (~10 GB) + experts + working set stays under your VRAM. Note MTP speculation is off by default under CUDA (#293, float-accumulation divergence between draft and verify) — `COLI_CUDA_MTP=1` opts back in.
|
||||
|
||||
## 5. Reference numbers from this walkthrough's hardware
|
||||
|
||||
285K / RTX 5080 / 128 GB / NVMe at 5.85 GB/s random-read (19 MB blocks, `iobench`): 0.26 tok/s cold CPU → 0.30 warm CPU (MTP 2.2–2.3 tok/forward) → 0.42 tok/s GPU tier + auto-pin, expert hit 66%, ~65% of wall time in expert-disk. Disk-bound is the expected shape at ~25% expert residency — a faster disk and more RAM move the floor, the GPU moves the compute.
|
||||
|
||||
## Quick failure index
|
||||
|
||||
| Symptom | Cause | Fix |
|
||||
|---|---|---|
|
||||
| `An Application Control policy has blocked this file` | Smart App Control | §2 — turn SAC off + **reboot** |
|
||||
| `cuda-dll ... Error 1` immediately | old tree: spaced CUDA_HOME / MSVC rejects `-Wextra` | update to current `dev` (#314) |
|
||||
| `glm.exe is up to date` but GPU never engages | old tree: stale CPU-only binary | update to `dev`, or delete `glm.exe` and rebuild |
|
||||
| `cl.exe (MSVC) not in PATH` | built from plain PowerShell | use the x64 Native Tools prompt |
|
||||
| `nvcc fatal: unsupported gpu architecture 'sm_120'` | CUDA < 12.8 | install CUDA 12.8+ |
|
||||
| MTP `0% (0/0)` on CPU path | int4 MTP heads in the container | use the int8-MTP container |
|
||||
| MTP `draft=0` under CUDA | intended default since #293 | `COLI_CUDA_MTP=1` to opt in |
|
||||
|
||||
---
|
||||
|
||||
## Reference: build flags & warmup
|
||||
|
||||
# AVX-VNNI: Intel Alder Lake+ (and Meteor Lake+) CPUs have a 128-bit int8
|
||||
# dot-product instruction (VPDPBUSD) the engine can use for ~1.3x faster
|
||||
# quantized matmul. The x86-64-v3 default (portable AVX2) compiles it out;
|
||||
# build for THIS machine to enable it:
|
||||
make glm.exe ARCH=native # banner prints "idot: avx-vnni"
|
||||
|
||||
# Verify (tiny model, 2.4 MB):
|
||||
pip install torch transformers safetensors huggingface_hub
|
||||
python tools/make_glm_oracle.py # generate tiny oracle
|
||||
SNAP=./glm_tiny TF=1 ./glm.exe 64 16 16 # expect "32/32 positions"
|
||||
|
||||
# Run with real model:
|
||||
SNAP=D:\glm52_i4 ./glm.exe 64 4 16 # batch inference
|
||||
python coli chat --model D:\glm52_i4 # interactive chat
|
||||
python coli serve --model D:\glm52_i4 # OpenAI-compatible API
|
||||
```
|
||||
|
||||
> Windows Store's `python` alias stub is the single most common native-Windows
|
||||
> trap: install real Python (python.org or `winget install Python.Python.3.12`)
|
||||
> or disable the alias under *Settings → Apps → App execution aliases*.
|
||||
|
||||
## Warmup (overnight cache priming)
|
||||
|
||||
The engine's expert cache learns from your workload. The included `warmup.ps1`
|
||||
script runs `coli run` in a loop with diverse prompts to build the
|
||||
`.coli_usage` histogram unattended, so the next real session starts with a
|
||||
large, accurate hot-expert pin. Each run saves usage atomically on clean
|
||||
completion.
|
||||
|
||||
```powershell
|
||||
.\warmup.ps1 -Rounds 1 -Ngen 32 # ~60-90 min, durable progress
|
||||
```
|
||||
|
||||
## NVIDIA GPU (optional, via runtime DLL)
|
||||
|
||||
On Windows the engine is built with MinGW gcc but CUDA kernels require MSVC +
|
||||
nvcc. The split is clean: build the CUDA backend into a standalone
|
||||
`coli_cuda.dll` (nvcc + MSVC), then the host `glm.exe` loads it at runtime via
|
||||
`LoadLibrary` (`c/backend_loader.c`). The host never links cudart directly; if
|
||||
the DLL is absent the engine falls back to CPU without error.
|
||||
|
||||
```powershell
|
||||
# Prerequisites: CUDA Toolkit + MSVC Build Tools (cl.exe) + nvcc on PATH.
|
||||
# Build the DLL from a shell with the MSVC environment set (vcvars64.bat or
|
||||
# "x64 Native Tools Command Prompt for VS"):
|
||||
make cuda-dll CUDA_HOME="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" CUDA_ARCH=sm_120
|
||||
|
||||
# Build the host with the runtime loader (CUDA_DLL=1 adds -DCOLI_CUDA and
|
||||
# links backend_loader.o instead of cudart):
|
||||
make glm.exe CUDA_DLL=1 ARCH=native
|
||||
|
||||
# Run with the GPU expert tier (8 GB VRAM budget here; scale to your free VRAM):
|
||||
$env:COLI_CUDA="1"; $env:COLI_GPU="0"; $env:CUDA_EXPERT_GB="8"
|
||||
python coli chat --model D:\glm52_i4 --topp 0.7
|
||||
```
|
||||
|
||||
The DLL exports the full `extern "C"` surface (including the #111 pipeline ABI);
|
||||
`backend_loader.c` resolves symbols via `GetProcAddress` on first use.
|
||||
`ColiCudaTensor*` is opaque to the host (stored, never dereferenced), so the
|
||||
MSVC-allocated struct is safe across the ABI boundary. `CUDA_ARCH` must match
|
||||
your GPU's compute capability (e.g. `sm_120` for Blackwell / RTX 50-series,
|
||||
`sm_89` for Ada / RTX 40-series). A one-shot `build_cuda.bat` wrapper is also
|
||||
available.
|
||||
|
||||
**Measured on a single RTX 5070 Ti + Core Ultra 9 (32 GB RAM):** CPU-only 0.63
|
||||
→ CUDA attention+dense 0.72 → **1.07 tok/s** with the GPU-resident pipeline at
|
||||
decode ([#273](https://github.com/JustVugg/colibri/issues/273), merged in #274).
|
||||
Generated
+61
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"nodes": {
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1731533236,
|
||||
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1784160687,
|
||||
"narHash": "sha256-iYL/bixrb6FlHFu/gIuBYzq6c6lM5AAXsXNSWXtIgQc=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "4382ed2b7a6839d4280a9b386db49cbc5907414d",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-26.05",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
@@ -26,7 +26,9 @@
|
||||
version = "1.0";
|
||||
src = ./.;
|
||||
|
||||
nativeBuildInputs = [ pkgs.makeWrapper ];
|
||||
# python3 is needed by checkPhase: `make test-c` shells out to
|
||||
# `python3 tools/run_tests.py` (see c/Makefile, PYTHON ?= python3).
|
||||
nativeBuildInputs = [ pkgs.makeWrapper pkgs.python3 ];
|
||||
|
||||
buildInputs = [
|
||||
pkgs.gcc
|
||||
@@ -44,18 +46,29 @@
|
||||
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
mkdir -p $out/bin
|
||||
cp c/glm $out/bin/glm
|
||||
|
||||
# Wrap coli (the Python CLI) so it finds the right python and the engine
|
||||
mkdir -p $out/share/colibri
|
||||
cp c/coli $out/share/colibri/coli
|
||||
chmod +x $out/share/colibri/coli
|
||||
cp -r c/tools $out/share/colibri/tools
|
||||
# Self-contained layout under $out/lib/colibri that mirrors the
|
||||
# source tree `coli` runs in (see the path-resolution logic at the
|
||||
# top of c/coli): the engine, the coli CLI script, the support
|
||||
# modules it imports (openai_server.py, resource_plan.py,
|
||||
# doctor.py), and tools/ all sit next to each other.
|
||||
mkdir -p $out/lib/colibri/tools $out/bin
|
||||
cp c/glm $out/lib/colibri/glm
|
||||
cp c/coli $out/lib/colibri/coli
|
||||
chmod +x $out/lib/colibri/coli
|
||||
cp c/openai_server.py c/resource_plan.py c/doctor.py $out/lib/colibri/
|
||||
cp -r c/tools/* $out/lib/colibri/tools/
|
||||
|
||||
# $out/bin holds the user-facing entry points.
|
||||
ln -s ../lib/colibri/glm $out/bin/glm
|
||||
|
||||
# Wrap coli: point it at the bundled engine (COLI_ENGINE) so it is
|
||||
# found by default, and at the module dir (PYTHONPATH) so
|
||||
# `import openai_server` / `resource_plan` / `doctor` resolve.
|
||||
makeWrapper ${pythonEnv}/bin/python $out/bin/coli \
|
||||
--add-flags "$out/share/colibri/coli" \
|
||||
--set PYTHONPATH "${pythonEnv}/${pkgs.python3.sitePackages}"
|
||||
--add-flags "$out/lib/colibri/coli" \
|
||||
--set-default COLI_ENGINE "$out/lib/colibri/glm" \
|
||||
--set PYTHONPATH "$out/lib/colibri:${pythonEnv}/${pkgs.python3.sitePackages}"
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=68.0"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "colibri-engine"
|
||||
dynamic = ["version"]
|
||||
description = "Tiny engine, immense model — run GLM-5.2 (744B MoE) locally"
|
||||
readme = "README.md"
|
||||
license = "Apache-2.0"
|
||||
requires-python = ">=3.10"
|
||||
authors = [
|
||||
{name = "JustVugg"},
|
||||
]
|
||||
classifiers = [
|
||||
"Development Status :: 4 - Beta",
|
||||
"Environment :: Console",
|
||||
"Intended Audience :: Science/Research",
|
||||
"Operating System :: POSIX :: Linux",
|
||||
"Operating System :: MacOS",
|
||||
"Operating System :: Microsoft :: Windows",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
convert = [
|
||||
"numpy",
|
||||
"huggingface_hub",
|
||||
]
|
||||
oracle = [
|
||||
"torch>=2.0",
|
||||
"transformers>=4.40",
|
||||
"safetensors",
|
||||
]
|
||||
bench = [
|
||||
"tokenizers",
|
||||
"datasets",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
coli = "colibri.cli:main"
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/JustVugg/colibri"
|
||||
Issues = "https://github.com/JustVugg/colibri/issues"
|
||||
|
||||
[tool.setuptools.dynamic]
|
||||
version = {attr = "colibri._version.__version__"}
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["colibri*"]
|
||||
+6
-1
@@ -17,9 +17,14 @@ npm test
|
||||
npm run build
|
||||
```
|
||||
|
||||
Besides Chat and Brain, the **Profiling** tab charts where the engine spent
|
||||
each turn's wall time (I/O wait, expert matmul, attention, LM head) from the
|
||||
server's `/profile` endpoint — a rolling window of per-turn `PROF` snapshots
|
||||
emitted by the engine.
|
||||
|
||||
The test suite stays browser-light: API requests use a mocked `fetch`, while
|
||||
runtime capability and storage behavior are covered through pure helpers. It
|
||||
checks that `/health` is resolved next to (not below) the OpenAI `/v1` prefix,
|
||||
checks that `/health` and `/profile` are resolved next to (not below) the OpenAI `/v1` prefix,
|
||||
supports both boolean and numeric `scheduler.active` responses, and sends the
|
||||
colibrì-specific `cache_slot` field only when KV-slot support was advertised.
|
||||
|
||||
|
||||
+68
-58
@@ -9,6 +9,7 @@ import {
|
||||
Database,
|
||||
Feather,
|
||||
Gauge,
|
||||
Globe,
|
||||
HardDrive,
|
||||
KeyRound,
|
||||
Layers,
|
||||
@@ -31,8 +32,10 @@ import { Textarea } from "@/components/ui/textarea"
|
||||
import { getHealth, listModels, streamChat, type ChatMessage, type HealthResponse, type StreamChatResult } from "@/lib/api"
|
||||
import { activeRequests, supportsCacheSlots } from "@/lib/runtime"
|
||||
import { Brain } from "./Brain"
|
||||
import { Profiling } from "./Profiling"
|
||||
import { persistPublicSettings, stored } from "@/lib/storage"
|
||||
import { cn } from "@/lib/utils"
|
||||
import { useLocale } from "./i18n"
|
||||
|
||||
const message = (role: ChatMessage["role"], content: string): ChatMessage => {
|
||||
let id: string
|
||||
@@ -41,15 +44,12 @@ const message = (role: ChatMessage["role"], content: string): ChatMessage => {
|
||||
}
|
||||
|
||||
export default function App() {
|
||||
// When the page is served by the engine itself (coli web), same-origin is the
|
||||
// right default: no CORS, no manual endpoint editing. The Vite dev server
|
||||
// (port 5173) keeps the classic default.
|
||||
const { t, locale, setLocale, locales } = useLocale()
|
||||
|
||||
const servedByEngine = typeof window !== "undefined" && window.location.port !== "5173" && window.location.protocol.startsWith("http")
|
||||
const defaultBase = servedByEngine ? `${window.location.origin}/v1` : "http://127.0.0.1:8000/v1"
|
||||
const [baseUrl, setBaseUrl] = useState(() => {
|
||||
const saved = stored(localStorage, "colibri.baseUrl", defaultBase)
|
||||
// migrate: a stored FACTORY default pointing at another origin would trip CORS
|
||||
// when the page is engine-served — upgrade it to same-origin once.
|
||||
if (servedByEngine && saved === "http://127.0.0.1:8000/v1" && defaultBase !== saved) return defaultBase
|
||||
return saved
|
||||
})
|
||||
@@ -73,7 +73,7 @@ export default function App() {
|
||||
const [totalTokens, setTotalTokens] = useState({ prompt: 0, completion: 0 })
|
||||
const [connecting, setConnecting] = useState(false)
|
||||
const [connected, setConnected] = useState(false)
|
||||
const [view, setView] = useState<"chat" | "brain">("chat")
|
||||
const [view, setView] = useState<"chat" | "brain" | "profiling">("chat")
|
||||
const [error, setError] = useState("")
|
||||
const autoConnected = useRef(false)
|
||||
const abortRef = useRef<AbortController | null>(null)
|
||||
@@ -119,7 +119,7 @@ export default function App() {
|
||||
const result = await getHealth(baseUrl, apiKey)
|
||||
if (!disposed) { setHealth(result); setHealthError("") }
|
||||
} catch (cause) {
|
||||
if (!disposed) setHealthError(cause instanceof Error ? cause.message : "Runtime metrics unavailable")
|
||||
if (!disposed) setHealthError(cause instanceof Error ? cause.message : "status.runtimeUnavailable")
|
||||
}
|
||||
}
|
||||
const timer = window.setInterval(() => void poll(), 5000)
|
||||
@@ -154,13 +154,13 @@ export default function App() {
|
||||
} catch (cause) {
|
||||
if (!controller.signal.aborted) {
|
||||
setHealth(null)
|
||||
setHealthError(cause instanceof Error ? cause.message : "Runtime metrics unavailable")
|
||||
setHealthError(cause instanceof Error ? cause.message : "status.runtimeUnavailable")
|
||||
}
|
||||
}
|
||||
} catch (cause) {
|
||||
if (controller.signal.aborted) return
|
||||
setConnected(false)
|
||||
setError(cause instanceof Error ? cause.message : "Could not reach the server.")
|
||||
setError(cause instanceof Error ? cause.message : "status.serverError")
|
||||
} finally {
|
||||
if (probeRef.current === controller) { probeRef.current = null; setConnecting(false) }
|
||||
}
|
||||
@@ -226,7 +226,7 @@ export default function App() {
|
||||
if (controller.signal.aborted) {
|
||||
updateMessages((current) => current.filter((item) => item.id !== assistant.id || item.content))
|
||||
} else {
|
||||
setError(cause instanceof Error ? cause.message : "Generation failed.")
|
||||
setError(cause instanceof Error ? cause.message : "status.generationFailed")
|
||||
updateMessages((current) => current.filter((item) => item.id !== assistant.id || item.content))
|
||||
}
|
||||
} finally {
|
||||
@@ -240,22 +240,22 @@ export default function App() {
|
||||
<aside className="sidebar">
|
||||
<div className="brand-row">
|
||||
<div className="brand-mark"><Feather className="size-5" /></div>
|
||||
<div><h1>colibrì</h1><p>local giant, tiny footprint</p></div>
|
||||
<div><h1>colibrì</h1><p>{t("brand.tagline")}</p></div>
|
||||
</div>
|
||||
|
||||
<section className="side-section">
|
||||
<div className="section-title"><Link2 className="size-3.5" /> Connection</div>
|
||||
<label>API endpoint<Input value={baseUrl} onChange={(event) => setBaseUrl(event.target.value)} /></label>
|
||||
<label>API key<div className="relative"><KeyRound className="field-icon" /><Input className="pl-9" type="password" value={apiKey} placeholder="optional" onChange={(event) => setApiKey(event.target.value)} /></div><span className="field-help">Kept in memory only · sent to this endpoint</span></label>
|
||||
<div className="section-title"><Link2 className="size-3.5" /> {t("sidebar.connection")}</div>
|
||||
<label>{t("sidebar.endpoint")}<Input value={baseUrl} onChange={(event) => setBaseUrl(event.target.value)} /></label>
|
||||
<label>{t("sidebar.apiKey")}<div className="relative"><KeyRound className="field-icon" /><Input className="pl-9" type="password" value={apiKey} placeholder={t("sidebar.apiKeyPlaceholder")} onChange={(event) => setApiKey(event.target.value)} /></div><span className="field-help">{t("sidebar.apiKeyHelp")}</span></label>
|
||||
<Button type="button" variant="secondary" onClick={connect} disabled={connecting}>
|
||||
{connecting ? <LoaderCircle className="size-4 animate-spin" /> : <RefreshCw className="size-4" />}
|
||||
Probe server
|
||||
{t("sidebar.probe")}
|
||||
</Button>
|
||||
<div className={cn("connection-state", connected && "connected")} aria-live="polite"><span />{connected ? "Engine reachable" : "Not connected"}</div>
|
||||
<div className={cn("connection-state", connected && "connected")} aria-live="polite"><span />{connected ? t("status.connected") : t("status.notConnected")}</div>
|
||||
</section>
|
||||
|
||||
<section className="side-section runtime-section" aria-live="polite">
|
||||
<div className="section-title"><Activity className="size-3.5" /> Runtime</div>
|
||||
<div className="section-title"><Activity className="size-3.5" /> {t("sidebar.runtime")}</div>
|
||||
{health?.hwinfo ? <div className="hw-panel">
|
||||
{health.hwinfo.cpu ? <div className="hw-row"><Cpu className="size-3.5" /><span>{health.hwinfo.cpu}</span></div> : null}
|
||||
{health.hwinfo.gpus > 0 ? <div className="hw-row"><MonitorDot className="size-3.5" /><span>{health.hwinfo.gpus}× GPU<small>{health.hwinfo.vram_total_gb.toFixed(0)} GB VRAM</small></span></div> : null}
|
||||
@@ -264,79 +264,89 @@ export default function App() {
|
||||
</div> : null}
|
||||
{health?.scheduler ? <>
|
||||
<div className="runtime-grid">
|
||||
<div><span>Active</span><strong>{active}<small> / {capacity}</small></strong></div>
|
||||
<div><span>Queued</span><strong>{health.scheduler.queued}<small> / {health.scheduler.max_queue}</small></strong></div>
|
||||
<div><span>Completed</span><strong>{health.scheduler.completed}</strong></div>
|
||||
<div><span>Failures</span><strong>{failures}</strong></div>
|
||||
<div><span>{t("dashboard.active")}</span><strong>{active}<small> / {capacity}</small></strong></div>
|
||||
<div><span>{t("dashboard.queued")}</span><strong>{health.scheduler.queued}<small> / {health.scheduler.max_queue}</small></strong></div>
|
||||
<div><span>{t("dashboard.completed")}</span><strong>{health.scheduler.completed}</strong></div>
|
||||
<div><span>{t("dashboard.failures")}</span><strong>{failures}</strong></div>
|
||||
</div>
|
||||
{health.tiers ? (() => {
|
||||
const t = health.tiers
|
||||
const total = Math.max(t.vram + t.ram + t.disk, 1)
|
||||
const ti = health.tiers
|
||||
const total = Math.max(ti.vram + ti.ram + ti.disk, 1)
|
||||
return <div className="tier-panel">
|
||||
<div className="tier-bar" role="img" aria-label={`Experts: ${t.vram} VRAM, ${t.ram} RAM, ${t.disk} disk`}>
|
||||
<span className="tier-vram" style={{ width: `${(100 * t.vram) / total}%` }} />
|
||||
<span className="tier-ram" style={{ width: `${(100 * t.ram) / total}%` }} />
|
||||
<span className="tier-disk" style={{ width: `${(100 * t.disk) / total}%` }} />
|
||||
<div className="tier-bar" role="img" aria-label={t("tier.ariaLabel", { vram: ti.vram, ram: ti.ram, disk: ti.disk })}>
|
||||
<span className="tier-vram" style={{ width: `${(100 * ti.vram) / total}%` }} />
|
||||
<span className="tier-ram" style={{ width: `${(100 * ti.ram) / total}%` }} />
|
||||
<span className="tier-disk" style={{ width: `${(100 * ti.disk) / total}%` }} />
|
||||
</div>
|
||||
<div className="tier-legend">
|
||||
<span><i className="tier-vram" />VRAM <strong>{t.vram.toLocaleString()}</strong><small>{t.vram_gb.toFixed(1)} GB</small></span>
|
||||
<span><i className="tier-ram" />RAM <strong>{t.ram.toLocaleString()}</strong><small>{t.ram_gb.toFixed(1)} GB</small></span>
|
||||
<span><i className="tier-disk" />Disk <strong>{t.disk.toLocaleString()}</strong></span>
|
||||
<span><i className="tier-vram" />{t("tier.vram")} <strong>{ti.vram.toLocaleString()}</strong><small>{ti.vram_gb.toFixed(1)} GB</small></span>
|
||||
<span><i className="tier-ram" />{t("tier.ram")} <strong>{ti.ram.toLocaleString()}</strong><small>{ti.ram_gb.toFixed(1)} GB</small></span>
|
||||
<span><i className="tier-disk" />{t("tier.disk")} <strong>{ti.disk.toLocaleString()}</strong></span>
|
||||
</div>
|
||||
</div>
|
||||
})() : null}
|
||||
{totalTokens.prompt + totalTokens.completion > 0 ? <div className="session-stats">
|
||||
<span><Database className="size-3" /> Session: <strong>{totalTokens.prompt.toLocaleString()}</strong> prompt + <strong>{totalTokens.completion.toLocaleString()}</strong> completion</span>
|
||||
<span><Database className="size-3" /> {t("dashboard.session")} <strong>{totalTokens.prompt.toLocaleString()}</strong> {t("dashboard.prompt")} + <strong>{totalTokens.completion.toLocaleString()}</strong> {t("dashboard.completion")}</span>
|
||||
</div> : null}
|
||||
<div className="runtime-foot"><span className="runtime-dot" /> Scheduler online <code>{kvSlots} KV</code></div>
|
||||
</> : <p className="runtime-unavailable">{connected ? (healthError || "Runtime metrics unavailable") : "Probe the server to inspect runtime state."}</p>}
|
||||
<div className="runtime-foot"><span className="runtime-dot" /> {t("sidebar.schedulerOnline")} <code>{kvSlots} KV</code></div>
|
||||
</> : <p className="runtime-unavailable">{connected ? (healthError ? t(healthError) : t("status.runtimeUnavailable")) : t("sidebar.runtimeProbe")}</p>}
|
||||
</section>
|
||||
|
||||
<section className="side-section">
|
||||
<div className="section-title"><SlidersHorizontal className="size-3.5" /> Inference</div>
|
||||
<label>Model<select value={model} onChange={(event) => setModel(event.target.value)}>{models.length ? models.map((id) => <option key={id}>{id}</option>) : <option>{model}</option>}</select></label>
|
||||
{health?.kv_slots && health.kv_slots > 1 ? <label>KV session<select value={cacheSlot} onChange={(event) => setCacheSlot(Number(event.target.value))} disabled={loading}>
|
||||
{Array.from({ length: kvSlots }, (_, slot) => <option key={slot} value={slot}>Session {slot + 1}</option>)}
|
||||
</select><span className="field-help">Isolated context · conversation follows the selected slot</span></label> : null}
|
||||
<label><span className="label-line"><span>Temperature</span><code>{temperature.toFixed(1)}</code></span><input className="range" type="range" min="0" max="2" step="0.1" value={temperature} onChange={(event) => setTemperature(Number(event.target.value))} /></label>
|
||||
<label>Max output tokens<Input type="number" min={1} max={4096} value={maxTokens} onChange={(event) => { const value = Number(event.target.value); if (Number.isFinite(value)) setMaxTokens(Math.min(4096, Math.max(1, Math.round(value)))) }} /></label>
|
||||
<div className="section-title"><SlidersHorizontal className="size-3.5" /> {t("sidebar.inference")}</div>
|
||||
<label>{t("sidebar.model")}<select value={model} onChange={(event) => setModel(event.target.value)}>{models.length ? models.map((id) => <option key={id}>{id}</option>) : <option>{model}</option>}</select></label>
|
||||
{health?.kv_slots && health.kv_slots > 1 ? <label>{t("sidebar.kvSession")}<select value={cacheSlot} onChange={(event) => setCacheSlot(Number(event.target.value))} disabled={loading}>
|
||||
{Array.from({ length: kvSlots }, (_, slot) => <option key={slot} value={slot}>{t("sidebar.sessionLabel", { slot: slot + 1 })}</option>)}
|
||||
</select><span className="field-help">{t("sidebar.kvSessionHelp")}</span></label> : null}
|
||||
<label><span className="label-line"><span>{t("sidebar.temperature")}</span><code>{temperature.toFixed(1)}</code></span><input className="range" type="range" min="0" max="2" step="0.1" value={temperature} onChange={(event) => setTemperature(Number(event.target.value))} /></label>
|
||||
<label>{t("sidebar.maxTokens")}<Input type="number" min={1} max={4096} value={maxTokens} onChange={(event) => { const value = Number(event.target.value); if (Number.isFinite(value)) setMaxTokens(Math.min(4096, Math.max(1, Math.round(value)))) }} /></label>
|
||||
<button type="button" className={cn("toggle-row", thinking && "active")} aria-pressed={thinking} onClick={() => setThinking((value) => !value)}>
|
||||
<span><BrainCircuit className="size-4" /> Reasoning</span><i><b /></i>
|
||||
<span><BrainCircuit className="size-4" /> {t("sidebar.reasoning")}</span><i><b /></i>
|
||||
</button>
|
||||
</section>
|
||||
|
||||
<div className="sidebar-foot"><Cpu className="size-3.5" /><span>OpenAI-compatible transport</span></div>
|
||||
<div className="sidebar-foot">
|
||||
<div><Cpu className="size-3.5" /><span>{t("sidebar.transport")}</span></div>
|
||||
<div className="locale-switcher">
|
||||
<Globe className="size-3.5" />
|
||||
<select value={locale} onChange={(e) => setLocale(e.target.value)}>
|
||||
{locales.map((l) => <option key={l.code} value={l.code}>{l.label}</option>)}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<main className="chat-panel">
|
||||
<header className="topbar">
|
||||
<div><span className="eyebrow">ACTIVE MODEL</span><strong>{model}</strong></div>
|
||||
<div><span className="eyebrow">{t("topbar.activeModel")}</span><strong>{model}</strong></div>
|
||||
<div className="view-tabs">
|
||||
<button className={view === "chat" ? "active" : ""} onClick={() => setView("chat")}><MessageSquareText className="size-3.5" /> Chat</button>
|
||||
<button className={view === "brain" ? "active" : ""} onClick={() => setView("brain")}><BrainCircuit className="size-3.5" /> Brain</button>
|
||||
<button className={view === "chat" ? "active" : ""} onClick={() => setView("chat")}><MessageSquareText className="size-3.5" /> {t("nav.chat")}</button>
|
||||
<button className={view === "brain" ? "active" : ""} onClick={() => setView("brain")}><BrainCircuit className="size-3.5" /> {t("nav.brain")}</button>
|
||||
<button className={view === "profiling" ? "active" : ""} onClick={() => setView("profiling")}><Gauge className="size-3.5" /> {t("nav.profiling")}</button>
|
||||
</div>
|
||||
<div className="top-actions">
|
||||
{loading && tokenCount > 0 ? <Badge className="badge-live"><Zap className="size-3 flash" /> {tokenCount} tokens</Badge> : null}
|
||||
{!loading && tokPerSec != null ? <Badge className="badge-speed"><Gauge className="size-3" /> {tokPerSec.toFixed(1)} tok/s</Badge> : null}
|
||||
{loading && tokenCount > 0 ? <Badge className="badge-live"><Zap className="size-3 flash" /> {t("topbar.tokens", { n: tokenCount })}</Badge> : null}
|
||||
{!loading && tokPerSec != null ? <Badge className="badge-speed"><Gauge className="size-3" /> {t("topbar.tokPerSec", { n: tokPerSec.toFixed(1) })}</Badge> : null}
|
||||
{!loading && ttft != null ? <Badge><Timer className="size-3" /> TTFT {(ttft/1000).toFixed(1)}s</Badge> : null}
|
||||
{!loading && lastRun?.usage ? <Badge><Layers className="size-3" /> {lastRun.usage.prompt_tokens}→{lastRun.usage.completion_tokens}</Badge> : null}
|
||||
{lastRun?.queueWaitMs != null ? <Badge><Clock className="size-3" /> queue {Math.round(lastRun.queueWaitMs)}ms</Badge> : null}
|
||||
<Badge><MonitorDot className="size-3" /> slot {cacheSlot + 1}</Badge>
|
||||
<Button variant="ghost" size="sm" onClick={() => { updateMessages([]); setTokPerSec(null); setTtft(null); setTokenCount(0); setTotalTokens({prompt:0,completion:0}) }} disabled={!messages.length || loading}><Trash2 className="size-3.5" /> Clear</Button>
|
||||
<Badge><MonitorDot className="size-3" /> {t("topbar.slot", { n: cacheSlot + 1 })}</Badge>
|
||||
<Button variant="ghost" size="sm" onClick={() => { updateMessages([]); setTokPerSec(null); setTtft(null); setTokenCount(0); setTotalTokens({prompt:0,completion:0}) }} disabled={!messages.length || loading}><Trash2 className="size-3.5" /> {t("topbar.clear")}</Button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{view === "brain" ? <Brain baseUrl={baseUrl} apiKey={apiKey} connected={connected} /> : <>
|
||||
{view === "brain" ? <Brain baseUrl={baseUrl} apiKey={apiKey} connected={connected} />
|
||||
: view === "profiling" ? <Profiling baseUrl={baseUrl} apiKey={apiKey} connected={connected} /> : <>
|
||||
|
||||
<div className="conversation">
|
||||
{!messages.length ? (
|
||||
<div className="empty-state">
|
||||
<div className="orb"><Feather /></div>
|
||||
<span className="eyebrow">COLIBRÌ ENGINE</span>
|
||||
<h2>Ask the giant.<br /><em>Keep the machine yours.</em></h2>
|
||||
<p>Connect to a local colibrì server and stream responses directly from your hardware. Nothing leaves the endpoint you choose.</p>
|
||||
<span className="eyebrow">{t("hero.title")}</span>
|
||||
<h2>{t("hero.subtitle")}<br /><em>{t("hero.tagline")}</em></h2>
|
||||
<p>{t("hero.description")}</p>
|
||||
<div className="suggestions">
|
||||
{["Explain how expert routing works", "Write a small C benchmark", "Compare RAM and VRAM caching"].map((item) => <button key={item} onClick={() => setDraft(item)}>{item}<ArrowUp className="size-3.5 rotate-45" /></button>)}
|
||||
{[t("prompts.routing"), t("prompts.benchmark"), t("prompts.caching")].map((item) => <button key={item} onClick={() => setDraft(item)}>{item}<ArrowUp className="size-3.5 rotate-45" /></button>)}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
@@ -344,7 +354,7 @@ export default function App() {
|
||||
{messages.map((item) => (
|
||||
<article key={item.id} className={cn("message", item.role)}>
|
||||
<div className="avatar">{item.role === "user" ? "Y" : <Feather className="size-4" />}</div>
|
||||
<div><div className="message-meta">{item.role === "user" ? "You" : "colibrì"}</div><div className="message-body">{item.content || <span className="typing" aria-label="Generating"><i /><i /><i /></span>}</div></div>
|
||||
<div><div className="message-meta">{item.role === "user" ? t("chat.you") : t("chat.colibri")}</div><div className="message-body">{item.content || <span className="typing" aria-label="Generating"><i /><i /><i /></span>}</div></div>
|
||||
</article>
|
||||
))}
|
||||
<div ref={bottomRef} />
|
||||
@@ -353,10 +363,10 @@ export default function App() {
|
||||
</div>
|
||||
|
||||
<div className="composer-wrap">
|
||||
{error && <div className="error-banner" role="alert">{error}</div>}
|
||||
{error && <div className="error-banner" role="alert">{t(error)}</div>}
|
||||
<div className="composer">
|
||||
<Textarea value={draft} onChange={(event) => setDraft(event.target.value)} placeholder="Message colibrì…" onKeyDown={(event) => { if (event.key === "Enter" && !event.shiftKey && !event.nativeEvent.isComposing) { event.preventDefault(); void send() } }} />
|
||||
<div className="composer-foot"><span><MessageSquareText className="size-3.5" /> Enter to send · Shift+Enter for newline</span>{loading ? <Button variant="destructive" size="icon" aria-label="Stop generation" onClick={() => abortRef.current?.abort()}><CircleStop className="size-4" /></Button> : <Button size="icon" aria-label="Send message" disabled={!canSend} onClick={() => void send()}><ArrowUp className="size-4" /></Button>}</div>
|
||||
<Textarea value={draft} onChange={(event) => setDraft(event.target.value)} placeholder={t("chat.placeholder")} onKeyDown={(event) => { if (event.key === "Enter" && !event.shiftKey && !event.nativeEvent.isComposing) { event.preventDefault(); void send() } }} />
|
||||
<div className="composer-foot"><span><MessageSquareText className="size-3.5" /> {t("chat.inputHint")}</span>{loading ? <Button variant="destructive" size="icon" aria-label={t("chat.stop")} onClick={() => abortRef.current?.abort()}><CircleStop className="size-4" /></Button> : <Button size="icon" aria-label={t("chat.send")} disabled={!canSend} onClick={() => void send()}><ArrowUp className="size-4" /></Button>}</div>
|
||||
</div>
|
||||
</div>
|
||||
</>}
|
||||
|
||||
+21
-22
@@ -2,27 +2,26 @@ import { useEffect, useRef, useState } from "react"
|
||||
import { BrainCircuit, Flame, Layers } from "lucide-react"
|
||||
|
||||
import { endpoint } from "@/lib/api"
|
||||
import { useLocale } from "./i18n"
|
||||
|
||||
interface ExpertMap { rows: number; cols: number; map: string; hits: string; seq: number }
|
||||
interface AtlasEntry { affinity: Record<string, number>; entropy: number; top: string; label: string }
|
||||
|
||||
const TIER_NAME = ["Disk", "RAM", "VRAM"]
|
||||
const TIER_KEYS = ["tier.disk", "tier.ram", "tier.vram"] as const
|
||||
const TIER_RGB: [number, number, number][] = [[58, 71, 80], [90, 155, 216], [78, 214, 165]]
|
||||
|
||||
/* Layer-depth heuristic: what this region of the network tends to specialise in.
|
||||
* Honest framing — these are the depth roles observed across MoE interpretability
|
||||
* work, not per-expert ground truth (that needs co-activation analysis, #119). */
|
||||
function depthRole(row: number, rows: number, isMtp: boolean): string {
|
||||
if (isMtp) return "MTP head — drafts the next token for speculative decoding"
|
||||
function depthRoleKey(row: number, rows: number, isMtp: boolean): string {
|
||||
if (isMtp) return "brain.mtp"
|
||||
const f = row / Math.max(rows - 1, 1)
|
||||
if (f < 0.2) return "early layers — surface features: tokens, spelling, local syntax"
|
||||
if (f < 0.45) return "lower-middle — phrase structure, word relations, simple facts"
|
||||
if (f < 0.7) return "upper-middle — semantics, long-range context, reasoning steps"
|
||||
if (f < 0.9) return "late layers — planning the answer, style, coherence"
|
||||
return "final layers — output shaping: picks the actual next-token distribution"
|
||||
if (f < 0.2) return "brain.early"
|
||||
if (f < 0.45) return "brain.lowerMiddle"
|
||||
if (f < 0.7) return "brain.upperMiddle"
|
||||
if (f < 0.9) return "brain.late"
|
||||
return "brain.final"
|
||||
}
|
||||
|
||||
export function Brain({ baseUrl, apiKey, connected }: { baseUrl: string; apiKey: string; connected: boolean }) {
|
||||
const { t } = useLocale()
|
||||
const canvasRef = useRef<HTMLCanvasElement>(null)
|
||||
const wrapRef = useRef<HTMLDivElement>(null)
|
||||
const [wrapSize, setWrapSize] = useState({ w: 1200, h: 700 })
|
||||
@@ -142,18 +141,18 @@ export function Brain({ baseUrl, apiKey, connected }: { baseUrl: string; apiKey:
|
||||
return (
|
||||
<div className="brain-page">
|
||||
<div className="brain-head">
|
||||
<div className="section-title"><BrainCircuit className="size-4" /> Expert Cortex — {data ? `${data.rows} layers × ${data.cols} experts` : "waiting for engine"}</div>
|
||||
<div className="section-title"><BrainCircuit className="size-4" /> {t("brain.title")} — {data ? t("brain.layers", { rows: data.rows, cols: data.cols }) : t("brain.waiting")}</div>
|
||||
<div className="brain-legend">
|
||||
<span><i style={{ background: "#4ed6a5" }} /> VRAM {totals[2].toLocaleString()}</span>
|
||||
<span><i style={{ background: "#5a9bd8" }} /> RAM {totals[1].toLocaleString()}</span>
|
||||
<span><i style={{ background: "#3a4750" }} /> Disk {totals[0].toLocaleString()}</span>
|
||||
<span><Flame className="size-3" /> brightness = routing heat</span>
|
||||
<span className="brain-pulse-hint">⚡ white flash = routed this turn</span>
|
||||
<span><i style={{ background: "#4ed6a5" }} /> {t("tier.vram")} {totals[2].toLocaleString()}</span>
|
||||
<span><i style={{ background: "#5a9bd8" }} /> {t("tier.ram")} {totals[1].toLocaleString()}</span>
|
||||
<span><i style={{ background: "#3a4750" }} /> {t("tier.disk")} {totals[0].toLocaleString()}</span>
|
||||
<span><Flame className="size-3" /> {t("brain.brightnessHint")}</span>
|
||||
<span className="brain-pulse-hint">{t("brain.flashHint")}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div className="brain-canvas-wrap" ref={wrapRef}>
|
||||
<canvas ref={canvasRef} onMouseMove={onMove} onMouseLeave={() => setTip(null)} />
|
||||
{!connected && <p className="runtime-unavailable">Connect to the engine to see the cortex.</p>}
|
||||
{!connected && <p className="runtime-unavailable">{t("brain.connectHint")}</p>}
|
||||
</div>
|
||||
{tip && data && (() => {
|
||||
const isMtp = tip.row === data.rows - 1
|
||||
@@ -162,16 +161,16 @@ export function Brain({ baseUrl, apiKey, connected }: { baseUrl: string; apiKey:
|
||||
return (
|
||||
<div className="brain-tip" style={{ left: tip.x + 14, top: tip.y + 14 }}>
|
||||
<div className="brain-tip-title"><Layers className="size-3" /> Layer {realLayer}{isMtp ? " (MTP)" : ""} · Expert {tip.col}</div>
|
||||
<div>Tier: <strong style={{ color: ["#8b9aa3", "#5a9bd8", "#4ed6a5"][tip.tier] }}>{TIER_NAME[tip.tier]}</strong></div>
|
||||
<div>Heat: <strong>{tip.heat === 0 ? "never routed" : `~2^${tip.heat} selections`}</strong></div>
|
||||
<div>Tier: <strong style={{ color: ["#8b9aa3", "#5a9bd8", "#4ed6a5"][tip.tier] }}>{t(TIER_KEYS[tip.tier])}</strong></div>
|
||||
<div>Heat: <strong>{tip.heat === 0 ? t("brain.neverRouted") : t("brain.selections", { heat: tip.heat })}</strong></div>
|
||||
{entry ? <>
|
||||
<div className={entry.label.startsWith("specialist") ? "brain-tip-spec" : undefined}>
|
||||
{entry.label.startsWith("specialist") ? `⭐ Specialist: ${entry.top}` : "Generalist"}
|
||||
{entry.label.startsWith("specialist") ? t("brain.specialist", { top: entry.top }) : t("brain.generalist")}
|
||||
<small> (entropy {entry.entropy})</small>
|
||||
</div>
|
||||
<div className="brain-tip-aff">{Object.entries(entry.affinity).sort((a, b) => b[1] - a[1]).slice(0, 3)
|
||||
.map(([c, p]) => `${c} ${Math.round(p * 100)}%`).join(" · ")}</div>
|
||||
</> : <div className="brain-tip-role">{depthRole(tip.row, data.rows, isMtp)}</div>}
|
||||
</> : <div className="brain-tip-role">{t(depthRoleKey(tip.row, data.rows, isMtp))}</div>}
|
||||
</div>
|
||||
)
|
||||
})()}
|
||||
|
||||
@@ -1,4 +1,18 @@
|
||||
import { Component, type ReactNode } from "react"
|
||||
import { useLocale } from "./i18n"
|
||||
|
||||
function ErrorFallback({ error, onRetry }: { error: Error; onRetry: () => void }) {
|
||||
const { t } = useLocale()
|
||||
return (
|
||||
<div style={{ padding: "2rem", fontFamily: "ui-monospace, monospace", color: "#e5e7eb", background: "#0b0f10", minHeight: "100vh" }}>
|
||||
<h2 style={{ color: "#4ed6a5" }}>{t("error.title")}</h2>
|
||||
<p style={{ color: "#9ca3af" }}>{t("error.hint")}</p>
|
||||
<pre style={{ whiteSpace: "pre-wrap", color: "#f87171" }}>{String(error)}</pre>
|
||||
<button onClick={onRetry} style={{ marginTop: "1rem", padding: "0.5rem 1rem", background: "#1f2937", color: "#e5e7eb", border: "1px solid #374151", borderRadius: 8, cursor: "pointer" }}>{t("error.retry")}</button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
interface State { error: Error | null; stack: string }
|
||||
export class ErrorBoundary extends Component<{ children: ReactNode }, State> {
|
||||
state: State = { error: null, stack: "" }
|
||||
@@ -9,11 +23,6 @@ export class ErrorBoundary extends Component<{ children: ReactNode }, State> {
|
||||
}
|
||||
render() {
|
||||
if (!this.state.error) return this.props.children
|
||||
return <div style={{ padding: "2rem", fontFamily: "ui-monospace, monospace", color: "#e5e7eb", background: "#0b0f10", minHeight: "100vh" }}>
|
||||
<h2 style={{ color: "#4ed6a5" }}>colibrì UI hit an error</h2>
|
||||
<p style={{ color: "#9ca3af" }}>The engine is unaffected. Try refreshing.</p>
|
||||
<pre style={{ whiteSpace: "pre-wrap", color: "#f87171" }}>{String(this.state.error)}</pre>
|
||||
<button onClick={() => this.setState({ error: null, stack: "" })} style={{ marginTop: "1rem", padding: "0.5rem 1rem", background: "#1f2937", color: "#e5e7eb", border: "1px solid #374151", borderRadius: 8, cursor: "pointer" }}>Retry</button>
|
||||
</div>
|
||||
return <ErrorFallback error={this.state.error} onRetry={() => this.setState({ error: null, stack: "" })} />
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
import { useEffect, useState } from "react"
|
||||
import { Activity, Gauge, HardDrive, Timer } from "lucide-react"
|
||||
|
||||
import { getProfile, type ProfileTurn } from "@/lib/api"
|
||||
import { useLocale } from "./i18n"
|
||||
|
||||
const PHASES = [
|
||||
{ key: "expert_wait_s", i18n: "profile.ioWait", color: "#3987e5" },
|
||||
{ key: "expert_matmul_s", i18n: "profile.expertMatmul", color: "#199e70" },
|
||||
{ key: "attention_s", i18n: "profile.attention", color: "#c98500" },
|
||||
{ key: "lm_head_s", i18n: "profile.lmHead", color: "#008300" },
|
||||
{ key: "other_s", i18n: "profile.other", color: "#9085e9" },
|
||||
] as const
|
||||
|
||||
interface Turn extends ProfileTurn { other_s: number; toks: number }
|
||||
|
||||
const derive = (turn: ProfileTurn): Turn => ({
|
||||
...turn,
|
||||
other_s: Math.max(0, turn.wall_s - turn.expert_wait_s - turn.expert_matmul_s - turn.attention_s - turn.lm_head_s),
|
||||
toks: turn.wall_s > 0 ? turn.completion_tokens / turn.wall_s : 0,
|
||||
})
|
||||
|
||||
const seconds = (value: number) => (value >= 10 ? value.toFixed(1) : value.toFixed(2)) + "s"
|
||||
|
||||
function ShareBar({ label, turns }: { label: string; turns: Turn[] }) {
|
||||
const { t } = useLocale()
|
||||
const total = turns.reduce((sum, turn) => sum + turn.wall_s, 0)
|
||||
const parts = PHASES.map((phase) => ({ ...phase, name: t(phase.i18n), value: turns.reduce((sum, turn) => sum + turn[phase.key], 0) }))
|
||||
return (
|
||||
<div className="prof-share">
|
||||
<div className="prof-share-head"><span>{label}</span><code>{seconds(total)}</code></div>
|
||||
<div className="prof-share-bar" role="img" aria-label={parts.map((part) => `${part.name} ${seconds(part.value)}`).join(", ")}>
|
||||
{parts.map((part) => {
|
||||
const share = total > 0 ? part.value / total : 0
|
||||
return share > 0.001 ? (
|
||||
<span key={part.key} style={{ width: `${100 * share}%`, background: part.color }} title={`${part.name} — ${seconds(part.value)} (${(100 * share).toFixed(1)}%)`}>
|
||||
{share >= 0.09 ? `${Math.round(100 * share)}%` : ""}
|
||||
</span>
|
||||
) : null
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function TurnColumns({ turns, stacked, height, format, footLabel, footLabelOne }: { turns: Turn[]; stacked: boolean; height: number; format: (turn: Turn) => string; footLabel: string; footLabelOne: string }) {
|
||||
const [hover, setHover] = useState<number | null>(null)
|
||||
const peak = Math.max(...turns.map((turn) => (stacked ? turn.wall_s : turn.toks)), 1e-9)
|
||||
const gap = 2
|
||||
const width = Math.max(1, (100 - gap * (turns.length - 1)) / turns.length)
|
||||
return (
|
||||
<div className="prof-plot" onMouseLeave={() => setHover(null)}>
|
||||
<svg viewBox={`0 0 100 ${height}`} preserveAspectRatio="none" aria-hidden="true">
|
||||
{[0.25, 0.5, 0.75].map((line) => <line key={line} x1="0" x2="100" y1={height * line} y2={height * line} className="prof-grid" />)}
|
||||
{turns.map((turn, index) => {
|
||||
const x = index * (width + gap)
|
||||
if (!stacked) {
|
||||
const h = (height * turn.toks) / peak
|
||||
return <rect key={index} x={x} y={height - h} width={width} height={h} rx="1" fill="var(--primary)" opacity={hover === null || hover === index ? 1 : 0.45} />
|
||||
}
|
||||
let y = height
|
||||
return PHASES.map((phase) => {
|
||||
const h = (height * turn[phase.key]) / peak
|
||||
y -= h
|
||||
return h > 0.1 ? <rect key={`${index}-${phase.key}`} x={x} y={y + 0.35} width={width} height={Math.max(h - 0.7, 0.35)} fill={phase.color} opacity={hover === null || hover === index ? 1 : 0.45} /> : null
|
||||
})
|
||||
})}
|
||||
{turns.map((_, index) => <rect key={index} x={index * (width + gap) - gap / 2} y="0" width={width + gap} height={height} fill="transparent" onMouseEnter={() => setHover(index)} />)}
|
||||
</svg>
|
||||
<div className="prof-plot-foot">
|
||||
<span>{turns.length > 1 ? footLabel : footLabelOne}</span>
|
||||
<code>{hover !== null && turns[hover] ? format(turns[hover]) : `peak ${stacked ? seconds(peak) : peak.toFixed(1) + " tok/s"}`}</code>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export function Profiling({ baseUrl, apiKey, connected }: { baseUrl: string; apiKey: string; connected: boolean }) {
|
||||
const { t } = useLocale()
|
||||
const [turns, setTurns] = useState<Turn[]>([])
|
||||
|
||||
useEffect(() => {
|
||||
if (!connected) return
|
||||
let disposed = false
|
||||
const poll = async () => {
|
||||
if (document.visibilityState === "hidden") return
|
||||
try {
|
||||
const result = await getProfile(baseUrl, apiKey)
|
||||
if (!disposed) setTurns(result.turns.map(derive))
|
||||
} catch { /* engine busy or restarting — keep the last snapshot */ }
|
||||
}
|
||||
void poll()
|
||||
const timer = window.setInterval(() => void poll(), 2000)
|
||||
return () => { disposed = true; window.clearInterval(timer) }
|
||||
}, [baseUrl, apiKey, connected])
|
||||
|
||||
const latest = turns[turns.length - 1]
|
||||
const recent = turns.slice(-40)
|
||||
const diskService = turns.reduce((sum, turn) => sum + turn.expert_disk_s, 0)
|
||||
|
||||
return (
|
||||
<div className="prof-page">
|
||||
<div className="prof-head">
|
||||
<div className="section-title"><Gauge className="size-4" /> {t("profile.title")}</div>
|
||||
<div className="prof-legend">
|
||||
{PHASES.map((phase) => <span key={phase.key}><i style={{ background: phase.color }} />{t(phase.i18n)}</span>)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!latest ? (
|
||||
<p className="runtime-unavailable">{connected ? t("profile.empty") : t("profile.connectHint")}</p>
|
||||
) : (
|
||||
<>
|
||||
<div className="prof-tiles">
|
||||
<div><span><Gauge className="size-3" /> {t("profile.lastTurn")}</span><strong>{latest.toks.toFixed(1)}</strong><small>tok/s</small></div>
|
||||
<div><span><Timer className="size-3" /> {t("profile.wallTime")}</span><strong>{seconds(latest.wall_s)}</strong><small>{latest.prompt_tokens} → {latest.completion_tokens} tokens</small></div>
|
||||
<div><span><Activity className="size-3" /> {t("profile.batching")}</span><strong>{latest.forwards > 0 ? (latest.completion_tokens / latest.forwards).toFixed(2) : "—"}</strong><small>{t("profile.tokensPerForward")}</small></div>
|
||||
<div><span><HardDrive className="size-3" /> {t("profile.diskService")}</span><strong>{seconds(latest.expert_disk_s)}</strong><small>{t("profile.overlapped")}</small></div>
|
||||
</div>
|
||||
|
||||
<div className="prof-shares">
|
||||
<ShareBar label={t("profile.lastTurn")} turns={[latest]} />
|
||||
{turns.length > 1 ? <ShareBar label={t("profile.window", { n: turns.length })} turns={turns} /> : null}
|
||||
</div>
|
||||
|
||||
<div className="prof-charts">
|
||||
<div className="prof-chart">
|
||||
<div className="prof-chart-title">{t("profile.throughputTitle")}</div>
|
||||
<TurnColumns turns={recent} stacked={false} height={36} footLabel={t("profile.turnsLabel", { n: recent.length })} footLabelOne={t("profile.oneTurn")} format={(turn) => `${turn.toks.toFixed(1)} tok/s · ${turn.completion_tokens} tokens`} />
|
||||
</div>
|
||||
<div className="prof-chart">
|
||||
<div className="prof-chart-title">{t("profile.phaseTitle")}</div>
|
||||
<TurnColumns turns={recent} stacked height={36} footLabel={t("profile.turnsLabel", { n: recent.length })} footLabelOne={t("profile.oneTurn")} format={(turn) => `${seconds(turn.wall_s)} · ${PHASES.map((phase) => `${t(phase.i18n)} ${seconds(turn[phase.key])}`).join(" · ")}`} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="prof-table-wrap">
|
||||
<table className="prof-table">
|
||||
<thead><tr><th>{t("profile.turnCol")}</th><th>{t("profile.tokensCol")}</th><th>tok/s</th><th>{t("profile.wallCol")}</th>{PHASES.map((phase) => <th key={phase.key}><i style={{ background: phase.color }} />{t(phase.i18n)}</th>)}<th>{t("profile.diskService")}</th></tr></thead>
|
||||
<tbody>
|
||||
{recent.slice().reverse().map((turn, index) => (
|
||||
<tr key={turns.length - index}>
|
||||
<td>{turns.length - index}</td>
|
||||
<td>{turn.prompt_tokens} → {turn.completion_tokens}</td>
|
||||
<td>{turn.toks.toFixed(1)}</td>
|
||||
<td>{seconds(turn.wall_s)}</td>
|
||||
{PHASES.map((phase) => <td key={phase.key}>{seconds(turn[phase.key])}</td>)}
|
||||
<td>{seconds(turn.expert_disk_s)}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{diskService > 0 ? <p className="prof-note">{t("profile.diskNote")}</p> : null}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
const en: Record<string, string> = {
|
||||
// nav
|
||||
"nav.chat": "Chat",
|
||||
"nav.brain": "Brain",
|
||||
"nav.profiling": "Profiling",
|
||||
|
||||
// brand
|
||||
"brand.tagline": "local giant, tiny footprint",
|
||||
|
||||
// sidebar — connection
|
||||
"sidebar.connection": "Connection",
|
||||
"sidebar.endpoint": "API endpoint",
|
||||
"sidebar.apiKey": "API key",
|
||||
"sidebar.apiKeyPlaceholder": "optional",
|
||||
"sidebar.apiKeyHelp": "Kept in memory only · sent to this endpoint",
|
||||
"sidebar.probe": "Probe server",
|
||||
"status.connected": "Engine reachable",
|
||||
"status.notConnected": "Not connected",
|
||||
"status.runtimeUnavailable": "Runtime metrics unavailable",
|
||||
"status.serverError": "Could not reach the server.",
|
||||
"status.generationFailed": "Generation failed.",
|
||||
|
||||
// sidebar — runtime
|
||||
"sidebar.runtime": "Runtime",
|
||||
"sidebar.runtimeProbe": "Probe the server to inspect runtime state.",
|
||||
"sidebar.schedulerOnline": "Scheduler online",
|
||||
"dashboard.active": "Active",
|
||||
"dashboard.queued": "Queued",
|
||||
"dashboard.completed": "Completed",
|
||||
"dashboard.failures": "Failures",
|
||||
"dashboard.session": "Session:",
|
||||
"dashboard.prompt": "prompt",
|
||||
"dashboard.completion": "completion",
|
||||
|
||||
// sidebar — tiers
|
||||
"tier.vram": "VRAM",
|
||||
"tier.ram": "RAM",
|
||||
"tier.disk": "Disk",
|
||||
"tier.ariaLabel": "Experts: {{vram}} VRAM, {{ram}} RAM, {{disk}} disk",
|
||||
|
||||
// sidebar — inference
|
||||
"sidebar.inference": "Inference",
|
||||
"sidebar.model": "Model",
|
||||
"sidebar.kvSession": "KV session",
|
||||
"sidebar.kvSessionHelp": "Isolated context · conversation follows the selected slot",
|
||||
"sidebar.sessionLabel": "Session {{slot}}",
|
||||
"sidebar.temperature": "Temperature",
|
||||
"sidebar.maxTokens": "Max output tokens",
|
||||
"sidebar.reasoning": "Reasoning",
|
||||
"sidebar.transport": "OpenAI-compatible transport",
|
||||
|
||||
// top bar
|
||||
"topbar.activeModel": "ACTIVE MODEL",
|
||||
"topbar.tokens": "{{n}} tokens",
|
||||
"topbar.tokPerSec": "{{n}} tok/s",
|
||||
"topbar.slot": "slot {{n}}",
|
||||
"topbar.clear": "Clear",
|
||||
|
||||
// hero / empty state
|
||||
"hero.title": "COLIBRÌ ENGINE",
|
||||
"hero.subtitle": "Ask the giant.",
|
||||
"hero.tagline": "Keep the machine yours.",
|
||||
"hero.description": "Connect to a local colibrì server and stream responses directly from your hardware. Nothing leaves the endpoint you choose.",
|
||||
"prompts.routing": "Explain how expert routing works",
|
||||
"prompts.benchmark": "Write a small C benchmark",
|
||||
"prompts.caching": "Compare RAM and VRAM caching",
|
||||
|
||||
// chat
|
||||
"chat.you": "You",
|
||||
"chat.colibri": "colibrì",
|
||||
"chat.placeholder": "Message colibrì…",
|
||||
"chat.inputHint": "Enter to send · Shift+Enter for newline",
|
||||
"chat.stop": "Stop generation",
|
||||
"chat.send": "Send message",
|
||||
|
||||
// brain
|
||||
"brain.title": "Expert Cortex",
|
||||
"brain.waiting": "waiting for engine",
|
||||
"brain.layers": "{{rows}} layers × {{cols}} experts",
|
||||
"brain.brightnessHint": "brightness = routing heat",
|
||||
"brain.flashHint": "⚡ white flash = routed this turn",
|
||||
"brain.connectHint": "Connect to the engine to see the cortex.",
|
||||
"brain.neverRouted": "never routed",
|
||||
"brain.selections": "~2^{{heat}} selections",
|
||||
"brain.specialist": "⭐ Specialist: {{top}}",
|
||||
"brain.generalist": "Generalist",
|
||||
"brain.mtp": "MTP head — drafts the next token for speculative decoding",
|
||||
"brain.early": "early layers — surface features: tokens, spelling, local syntax",
|
||||
"brain.lowerMiddle": "lower-middle — phrase structure, word relations, simple facts",
|
||||
"brain.upperMiddle": "upper-middle — semantics, long-range context, reasoning steps",
|
||||
"brain.late": "late layers — planning the answer, style, coherence",
|
||||
"brain.final": "final layers — output shaping: picks the actual next-token distribution",
|
||||
|
||||
// profiling
|
||||
"profile.title": "Profiling — where the engine spends each turn",
|
||||
"profile.ioWait": "I/O wait",
|
||||
"profile.expertMatmul": "Expert matmul",
|
||||
"profile.attention": "Attention",
|
||||
"profile.lmHead": "LM head",
|
||||
"profile.other": "Other",
|
||||
"profile.empty": "No profiled turns yet — send a chat message and the breakdown appears here.",
|
||||
"profile.connectHint": "Connect to the engine to collect per-turn timings.",
|
||||
"profile.lastTurn": "Last turn",
|
||||
"profile.wallTime": "Wall time",
|
||||
"profile.batching": "Batching",
|
||||
"profile.tokensPerForward": "tokens / forward",
|
||||
"profile.diskService": "Disk service",
|
||||
"profile.overlapped": "overlapped with compute",
|
||||
"profile.window": "Window · last {{n}} turns",
|
||||
"profile.throughputTitle": "Throughput per turn (tok/s)",
|
||||
"profile.phaseTitle": "Turn wall time by phase (s)",
|
||||
"profile.turnCol": "Turn",
|
||||
"profile.tokensCol": "Tokens",
|
||||
"profile.wallCol": "Wall",
|
||||
"profile.turnsLabel": "{{n}} turns · oldest → newest",
|
||||
"profile.oneTurn": "1 turn",
|
||||
"profile.diskNote": "Disk service is time spent reading experts on I/O threads; it overlaps with compute, so only the I/O wait the compute thread felt counts inside the wall-time stack. With multiple KV sessions the shares describe the whole engine over the turn's window.",
|
||||
|
||||
// error boundary
|
||||
"error.title": "colibrì UI hit an error",
|
||||
"error.hint": "The engine is unaffected. Try refreshing.",
|
||||
"error.retry": "Retry",
|
||||
}
|
||||
|
||||
export default en
|
||||
@@ -0,0 +1,78 @@
|
||||
import { createContext, useContext, useState, useCallback, useMemo, type ReactNode } from "react"
|
||||
import { createElement } from "react"
|
||||
import en from "./en"
|
||||
import zhCN from "./zh-CN"
|
||||
import zhTW from "./zh-TW"
|
||||
import it from "./it"
|
||||
|
||||
const LOCALES = [
|
||||
{ code: "en", label: "English" },
|
||||
{ code: "zh-CN", label: "简体中文" },
|
||||
{ code: "zh-TW", label: "繁體中文" },
|
||||
{ code: "it", label: "Italiano" },
|
||||
] as const
|
||||
|
||||
const DICTS: Record<string, Record<string, string>> = {
|
||||
"en": en,
|
||||
"zh-CN": zhCN,
|
||||
"zh-TW": zhTW,
|
||||
"it": it,
|
||||
}
|
||||
|
||||
const STORAGE_KEY = "colibri-locale"
|
||||
|
||||
function detectLocale(): string {
|
||||
try {
|
||||
const saved = localStorage.getItem(STORAGE_KEY)
|
||||
if (saved && DICTS[saved]) return saved
|
||||
} catch {}
|
||||
const nav = navigator.language || ""
|
||||
if (DICTS[nav]) return nav
|
||||
const prefix = nav.split("-")[0]
|
||||
if (prefix === "zh") return nav.includes("TW") || nav.includes("Hant") ? "zh-TW" : "zh-CN"
|
||||
for (const { code } of LOCALES) if (code.startsWith(prefix)) return code
|
||||
return "en"
|
||||
}
|
||||
|
||||
function interpolate(template: string, vars?: Record<string, string | number>): string {
|
||||
if (!vars) return template
|
||||
return template.replace(/\{\{(\w+)\}\}/g, (_, key) => String(vars[key] ?? `{{${key}}}`))
|
||||
}
|
||||
|
||||
interface LocaleContext {
|
||||
locale: string
|
||||
setLocale: (code: string) => void
|
||||
t: (key: string, vars?: Record<string, string | number>) => string
|
||||
locales: readonly { code: string; label: string }[]
|
||||
}
|
||||
|
||||
const Ctx = createContext<LocaleContext>({
|
||||
locale: "en",
|
||||
setLocale: () => {},
|
||||
t: (key) => key,
|
||||
locales: LOCALES,
|
||||
})
|
||||
|
||||
export function LocaleProvider({ children }: { children: ReactNode }) {
|
||||
const [locale, setLocaleState] = useState(detectLocale)
|
||||
|
||||
const setLocale = useCallback((code: string) => {
|
||||
if (!DICTS[code]) return
|
||||
setLocaleState(code)
|
||||
try { localStorage.setItem(STORAGE_KEY, code) } catch {}
|
||||
}, [])
|
||||
|
||||
const t = useCallback((key: string, vars?: Record<string, string | number>) => {
|
||||
const dict = DICTS[locale] || en
|
||||
const template = dict[key] ?? en[key] ?? key
|
||||
return interpolate(template, vars)
|
||||
}, [locale])
|
||||
|
||||
const value = useMemo(() => ({ locale, setLocale, t, locales: LOCALES }), [locale, setLocale, t])
|
||||
|
||||
return createElement(Ctx.Provider, { value }, children)
|
||||
}
|
||||
|
||||
export function useLocale() {
|
||||
return useContext(Ctx)
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
const it: Record<string, string> = {
|
||||
"nav.chat": "Chat",
|
||||
"nav.brain": "Cervello",
|
||||
"nav.profiling": "Profiling",
|
||||
|
||||
"brand.tagline": "gigante locale, impronta minima",
|
||||
|
||||
"sidebar.connection": "Connessione",
|
||||
"sidebar.endpoint": "Endpoint API",
|
||||
"sidebar.apiKey": "Chiave API",
|
||||
"sidebar.apiKeyPlaceholder": "opzionale",
|
||||
"sidebar.apiKeyHelp": "Conservata solo in memoria · inviata a questo endpoint",
|
||||
"sidebar.probe": "Sonda il server",
|
||||
"status.connected": "Motore raggiungibile",
|
||||
"status.notConnected": "Non connesso",
|
||||
"status.runtimeUnavailable": "Metriche runtime non disponibili",
|
||||
"status.serverError": "Impossibile raggiungere il server.",
|
||||
"status.generationFailed": "Generazione fallita.",
|
||||
|
||||
"sidebar.runtime": "Runtime",
|
||||
"sidebar.runtimeProbe": "Sonda il server per ispezionare lo stato runtime.",
|
||||
"sidebar.schedulerOnline": "Scheduler online",
|
||||
"dashboard.active": "Attive",
|
||||
"dashboard.queued": "In coda",
|
||||
"dashboard.completed": "Completate",
|
||||
"dashboard.failures": "Fallite",
|
||||
"dashboard.session": "Sessione:",
|
||||
"dashboard.prompt": "prompt",
|
||||
"dashboard.completion": "completion",
|
||||
|
||||
"tier.vram": "VRAM",
|
||||
"tier.ram": "RAM",
|
||||
"tier.disk": "Disco",
|
||||
"tier.ariaLabel": "Expert: {{vram}} VRAM, {{ram}} RAM, {{disk}} disco",
|
||||
|
||||
"sidebar.inference": "Inferenza",
|
||||
"sidebar.model": "Modello",
|
||||
"sidebar.kvSession": "Sessione KV",
|
||||
"sidebar.kvSessionHelp": "Contesto isolato · la conversazione segue lo slot selezionato",
|
||||
"sidebar.sessionLabel": "Sessione {{slot}}",
|
||||
"sidebar.temperature": "Temperatura",
|
||||
"sidebar.maxTokens": "Token di output massimi",
|
||||
"sidebar.reasoning": "Ragionamento",
|
||||
"sidebar.transport": "Trasporto compatibile OpenAI",
|
||||
|
||||
"topbar.activeModel": "MODELLO ATTIVO",
|
||||
"topbar.tokens": "{{n}} token",
|
||||
"topbar.tokPerSec": "{{n}} tok/s",
|
||||
"topbar.slot": "slot {{n}}",
|
||||
"topbar.clear": "Pulisci",
|
||||
|
||||
"hero.title": "MOTORE COLIBRÌ",
|
||||
"hero.subtitle": "Interroga il gigante.",
|
||||
"hero.tagline": "La macchina resta tua.",
|
||||
"hero.description": "Connettiti a un server colibrì locale e ricevi le risposte in streaming direttamente dal tuo hardware. Nulla lascia l'endpoint che scegli.",
|
||||
"prompts.routing": "Spiega come funziona il routing degli expert",
|
||||
"prompts.benchmark": "Scrivi un piccolo benchmark in C",
|
||||
"prompts.caching": "Confronta il caching RAM e VRAM",
|
||||
|
||||
"chat.you": "Tu",
|
||||
"chat.colibri": "colibrì",
|
||||
"chat.placeholder": "Scrivi a colibrì…",
|
||||
"chat.inputHint": "Invio per inviare · Shift+Invio per andare a capo",
|
||||
"chat.stop": "Ferma la generazione",
|
||||
"chat.send": "Invia messaggio",
|
||||
|
||||
"brain.title": "Corteccia degli expert",
|
||||
"brain.waiting": "in attesa del motore",
|
||||
"brain.layers": "{{rows}} layer × {{cols}} expert",
|
||||
"brain.brightnessHint": "luminosità = calore di routing",
|
||||
"brain.flashHint": "⚡ flash bianco = instradato in questo turno",
|
||||
"brain.connectHint": "Connettiti al motore per vedere la corteccia.",
|
||||
"brain.neverRouted": "mai instradato",
|
||||
"brain.selections": "~2^{{heat}} selezioni",
|
||||
"brain.specialist": "⭐ Specialista: {{top}}",
|
||||
"brain.generalist": "Generalista",
|
||||
"brain.mtp": "Testa MTP — prepara il prossimo token per la decodifica speculativa",
|
||||
"brain.early": "layer iniziali — caratteristiche superficiali: token, ortografia, sintassi locale",
|
||||
"brain.lowerMiddle": "layer medio-bassi — struttura frasale, relazioni tra parole, fatti semplici",
|
||||
"brain.upperMiddle": "layer medio-alti — semantica, contesto a lungo raggio, passi di ragionamento",
|
||||
"brain.late": "layer avanzati — pianificazione della risposta, stile, coerenza",
|
||||
"brain.final": "layer finali — formazione dell'output: scelta della distribuzione next-token",
|
||||
|
||||
"profile.title": "Profiling — dove il motore spende ogni turno",
|
||||
"profile.ioWait": "Attesa I/O",
|
||||
"profile.expertMatmul": "Matmul expert",
|
||||
"profile.attention": "Attenzione",
|
||||
"profile.lmHead": "LM head",
|
||||
"profile.other": "Altro",
|
||||
"profile.empty": "Nessun turno profilato — invia un messaggio e i dettagli appariranno qui.",
|
||||
"profile.connectHint": "Connettiti al motore per raccogliere i tempi per turno.",
|
||||
"profile.lastTurn": "Ultimo turno",
|
||||
"profile.wallTime": "Tempo totale",
|
||||
"profile.batching": "Batching",
|
||||
"profile.tokensPerForward": "token / forward",
|
||||
"profile.diskService": "Servizio disco",
|
||||
"profile.overlapped": "sovrapposto al calcolo",
|
||||
"profile.window": "Finestra · ultimi {{n}} turni",
|
||||
"profile.throughputTitle": "Throughput per turno (tok/s)",
|
||||
"profile.phaseTitle": "Tempo per turno per fase (s)",
|
||||
"profile.turnCol": "Turno",
|
||||
"profile.tokensCol": "Token",
|
||||
"profile.wallCol": "Totale",
|
||||
"profile.turnsLabel": "{{n}} turni · dal meno al più recente",
|
||||
"profile.oneTurn": "1 turno",
|
||||
"profile.diskNote": "Il servizio disco è il tempo speso a leggere gli expert sui thread I/O; si sovrappone al calcolo, quindi solo l'attesa I/O effettivamente percepita dal thread di calcolo conta nella ripartizione del tempo totale. Con più sessioni KV, le quote descrivono l'intero motore nella finestra del turno.",
|
||||
|
||||
"error.title": "L'interfaccia colibrì ha riscontrato un errore",
|
||||
"error.hint": "Il motore non è stato coinvolto. Prova a ricaricare la pagina.",
|
||||
"error.retry": "Riprova",
|
||||
}
|
||||
|
||||
export default it
|
||||
@@ -0,0 +1,113 @@
|
||||
const zhCN: Record<string, string> = {
|
||||
"nav.chat": "对话",
|
||||
"nav.brain": "大脑",
|
||||
"nav.profiling": "性能分析",
|
||||
|
||||
"brand.tagline": "本地巨人，极小足迹",
|
||||
|
||||
"sidebar.connection": "连接",
|
||||
"sidebar.endpoint": "API 端点",
|
||||
"sidebar.apiKey": "API 密钥",
|
||||
"sidebar.apiKeyPlaceholder": "可选",
|
||||
"sidebar.apiKeyHelp": "仅保存在内存中 · 发送到此端点",
|
||||
"sidebar.probe": "探测服务器",
|
||||
"status.connected": "引擎已连接",
|
||||
"status.notConnected": "未连接",
|
||||
"status.runtimeUnavailable": "运行时指标不可用",
|
||||
"status.serverError": "无法连接到服务器。",
|
||||
"status.generationFailed": "生成失败。",
|
||||
|
||||
"sidebar.runtime": "运行时",
|
||||
"sidebar.runtimeProbe": "探测服务器以查看运行时状态。",
|
||||
"sidebar.schedulerOnline": "调度器在线",
|
||||
"dashboard.active": "活跃",
|
||||
"dashboard.queued": "排队",
|
||||
"dashboard.completed": "已完成",
|
||||
"dashboard.failures": "失败",
|
||||
"dashboard.session": "会话：",
|
||||
"dashboard.prompt": "提示词",
|
||||
"dashboard.completion": "补全",
|
||||
|
||||
"tier.vram": "VRAM",
|
||||
"tier.ram": "RAM",
|
||||
"tier.disk": "磁盘",
|
||||
"tier.ariaLabel": "专家分布：{{vram}} VRAM、{{ram}} RAM、{{disk}} 磁盘",
|
||||
|
||||
"sidebar.inference": "推理",
|
||||
"sidebar.model": "模型",
|
||||
"sidebar.kvSession": "KV 会话",
|
||||
"sidebar.kvSessionHelp": "独立上下文 · 对话跟随所选槽位",
|
||||
"sidebar.sessionLabel": "会话 {{slot}}",
|
||||
"sidebar.temperature": "温度",
|
||||
"sidebar.maxTokens": "最大输出 token 数",
|
||||
"sidebar.reasoning": "推理模式",
|
||||
"sidebar.transport": "OpenAI 兼容协议",
|
||||
|
||||
"topbar.activeModel": "当前模型",
|
||||
"topbar.tokens": "{{n}} tokens",
|
||||
"topbar.tokPerSec": "{{n}} tok/s",
|
||||
"topbar.slot": "槽位 {{n}}",
|
||||
"topbar.clear": "清空",
|
||||
|
||||
"hero.title": "COLIBRÌ 引擎",
|
||||
"hero.subtitle": "向巨人提问。",
|
||||
"hero.tagline": "让机器属于你。",
|
||||
"hero.description": "连接到本地 colibrì 服务器，直接从你的硬件流式获取响应。所有数据都留在你选择的端点内。",
|
||||
"prompts.routing": "解释专家路由是如何工作的",
|
||||
"prompts.benchmark": "写一个简单的 C 基准测试",
|
||||
"prompts.caching": "比较 RAM 和 VRAM 缓存",
|
||||
|
||||
"chat.you": "你",
|
||||
"chat.colibri": "colibrì",
|
||||
"chat.placeholder": "给 colibrì 发消息…",
|
||||
"chat.inputHint": "回车发送 · Shift+回车换行",
|
||||
"chat.stop": "停止生成",
|
||||
"chat.send": "发送消息",
|
||||
|
||||
"brain.title": "专家皮层",
|
||||
"brain.waiting": "等待引擎连接",
|
||||
"brain.layers": "{{rows}} 层 × {{cols}} 专家",
|
||||
"brain.brightnessHint": "亮度 = 路由热度",
|
||||
"brain.flashHint": "⚡ 白色闪烁 = 本轮被路由",
|
||||
"brain.connectHint": "连接引擎以查看皮层。",
|
||||
"brain.neverRouted": "从未被路由",
|
||||
"brain.selections": "约 2^{{heat}} 次选择",
|
||||
"brain.specialist": "⭐ 专精：{{top}}",
|
||||
"brain.generalist": "通用型",
|
||||
"brain.mtp": "MTP 头 — 为投机解码起草下一个 token",
|
||||
"brain.early": "早期层 — 表面特征：token、拼写、局部语法",
|
||||
"brain.lowerMiddle": "中低层 — 短语结构、词语关系、简单事实",
|
||||
"brain.upperMiddle": "中高层 — 语义、长距离上下文、推理步骤",
|
||||
"brain.late": "后期层 — 规划答案、风格、连贯性",
|
||||
"brain.final": "末尾层 — 输出成型：选择实际的 next-token 分布",
|
||||
|
||||
"profile.title": "性能分析 — 引擎每轮的时间花在哪里",
|
||||
"profile.ioWait": "I/O 等待",
|
||||
"profile.expertMatmul": "专家矩阵乘",
|
||||
"profile.attention": "注意力",
|
||||
"profile.lmHead": "LM head",
|
||||
"profile.other": "其他",
|
||||
"profile.empty": "暂无性能数据 — 发送一条消息，分析结果将显示在这里。",
|
||||
"profile.connectHint": "连接引擎以采集每轮耗时。",
|
||||
"profile.lastTurn": "最近一轮",
|
||||
"profile.wallTime": "总耗时",
|
||||
"profile.batching": "批处理",
|
||||
"profile.tokensPerForward": "tokens / 前向",
|
||||
"profile.diskService": "磁盘服务",
|
||||
"profile.overlapped": "与计算重叠",
|
||||
"profile.window": "窗口 · 最近 {{n}} 轮",
|
||||
"profile.throughputTitle": "每轮吞吐量 (tok/s)",
|
||||
"profile.phaseTitle": "每轮各阶段耗时 (s)",
|
||||
"profile.turnCol": "轮次",
|
||||
"profile.tokensCol": "Tokens",
|
||||
"profile.wallCol": "总耗时",
|
||||
"profile.turnsLabel": "{{n}} 轮 · 从旧到新",
|
||||
"profile.oneTurn": "1 轮",
|
||||
"profile.diskNote": "磁盘服务是在 I/O 线程上读取专家的时间；它与计算重叠，因此只有计算线程实际感受到的 I/O 等待 才计入总耗时分解。多 KV 会话时，份额描述的是整个引擎在该轮窗口内的表现。",
|
||||
|
||||
"error.title": "colibrì UI 遇到错误",
|
||||
"error.hint": "引擎不受影响。请尝试刷新页面。",
|
||||
"error.retry": "重试",
|
||||
}
|
||||
|
||||
export default zhCN
|
||||
@@ -0,0 +1,113 @@
|
||||
const zhTW: Record<string, string> = {
|
||||
"nav.chat": "對話",
|
||||
"nav.brain": "大腦",
|
||||
"nav.profiling": "效能分析",
|
||||
|
||||
"brand.tagline": "本地巨人，極小足跡",
|
||||
|
||||
"sidebar.connection": "連線",
|
||||
"sidebar.endpoint": "API 端點",
|
||||
"sidebar.apiKey": "API 金鑰",
|
||||
"sidebar.apiKeyPlaceholder": "選填",
|
||||
"sidebar.apiKeyHelp": "僅保存在記憶體中 · 傳送到此端點",
|
||||
"sidebar.probe": "探測伺服器",
|
||||
"status.connected": "引擎已連線",
|
||||
"status.notConnected": "未連線",
|
||||
"status.runtimeUnavailable": "執行階段指標不可用",
|
||||
"status.serverError": "無法連線到伺服器。",
|
||||
"status.generationFailed": "生成失敗。",
|
||||
|
||||
"sidebar.runtime": "執行階段",
|
||||
"sidebar.runtimeProbe": "探測伺服器以檢視執行階段狀態。",
|
||||
"sidebar.schedulerOnline": "排程器上線",
|
||||
"dashboard.active": "進行中",
|
||||
"dashboard.queued": "排隊中",
|
||||
"dashboard.completed": "已完成",
|
||||
"dashboard.failures": "失敗",
|
||||
"dashboard.session": "工作階段：",
|
||||
"dashboard.prompt": "提示詞",
|
||||
"dashboard.completion": "補全",
|
||||
|
||||
"tier.vram": "VRAM",
|
||||
"tier.ram": "RAM",
|
||||
"tier.disk": "磁碟",
|
||||
"tier.ariaLabel": "專家分佈：{{vram}} VRAM、{{ram}} RAM、{{disk}} 磁碟",
|
||||
|
||||
"sidebar.inference": "推論",
|
||||
"sidebar.model": "模型",
|
||||
"sidebar.kvSession": "KV 工作階段",
|
||||
"sidebar.kvSessionHelp": "獨立上下文 · 對話跟隨所選插槽",
|
||||
"sidebar.sessionLabel": "工作階段 {{slot}}",
|
||||
"sidebar.temperature": "溫度",
|
||||
"sidebar.maxTokens": "最大輸出 token 數",
|
||||
"sidebar.reasoning": "推理模式",
|
||||
"sidebar.transport": "OpenAI 相容協定",
|
||||
|
||||
"topbar.activeModel": "目前模型",
|
||||
"topbar.tokens": "{{n}} tokens",
|
||||
"topbar.tokPerSec": "{{n}} tok/s",
|
||||
"topbar.slot": "插槽 {{n}}",
|
||||
"topbar.clear": "清除",
|
||||
|
||||
"hero.title": "COLIBRÌ 引擎",
|
||||
"hero.subtitle": "向巨人提問。",
|
||||
"hero.tagline": "讓機器屬於你。",
|
||||
"hero.description": "連線到本地 colibrì 伺服器，直接從你的硬體串流取得回應。所有資料都留在你選擇的端點內。",
|
||||
"prompts.routing": "解釋專家路由如何運作",
|
||||
"prompts.benchmark": "撰寫一個簡單的 C 基準測試",
|
||||
"prompts.caching": "比較 RAM 與 VRAM 快取",
|
||||
|
||||
"chat.you": "你",
|
||||
"chat.colibri": "colibrì",
|
||||
"chat.placeholder": "傳送訊息給 colibrì…",
|
||||
"chat.inputHint": "Enter 傳送 · Shift+Enter 換行",
|
||||
"chat.stop": "停止生成",
|
||||
"chat.send": "傳送訊息",
|
||||
|
||||
"brain.title": "專家皮層",
|
||||
"brain.waiting": "等待引擎連線",
|
||||
"brain.layers": "{{rows}} 層 × {{cols}} 專家",
|
||||
"brain.brightnessHint": "亮度 = 路由熱度",
|
||||
"brain.flashHint": "⚡ 白色閃爍 = 本輪被路由",
|
||||
"brain.connectHint": "連線引擎以檢視皮層。",
|
||||
"brain.neverRouted": "從未被路由",
|
||||
"brain.selections": "約 2^{{heat}} 次選擇",
|
||||
"brain.specialist": "⭐ 專精：{{top}}",
|
||||
"brain.generalist": "通用型",
|
||||
"brain.mtp": "MTP 頭 — 為推測式解碼起草下一個 token",
|
||||
"brain.early": "早期層 — 表面特徵：token、拼寫、局部語法",
|
||||
"brain.lowerMiddle": "中低層 — 片語結構、詞語關係、簡單事實",
|
||||
"brain.upperMiddle": "中高層 — 語意、長距離上下文、推理步驟",
|
||||
"brain.late": "後期層 — 規劃答案、風格、連貫性",
|
||||
"brain.final": "末尾層 — 輸出成型：選擇實際的 next-token 分佈",
|
||||
|
||||
"profile.title": "效能分析 — 引擎每輪的時間花在哪裡",
|
||||
"profile.ioWait": "I/O 等待",
|
||||
"profile.expertMatmul": "專家矩陣乘",
|
||||
"profile.attention": "注意力",
|
||||
"profile.lmHead": "LM head",
|
||||
"profile.other": "其他",
|
||||
"profile.empty": "尚無效能數據 — 傳送一則訊息，分析結果將顯示在這裡。",
|
||||
"profile.connectHint": "連線引擎以採集每輪耗時。",
|
||||
"profile.lastTurn": "最近一輪",
|
||||
"profile.wallTime": "總耗時",
|
||||
"profile.batching": "批次處理",
|
||||
"profile.tokensPerForward": "tokens / 前向",
|
||||
"profile.diskService": "磁碟服務",
|
||||
"profile.overlapped": "與運算重疊",
|
||||
"profile.window": "視窗 · 最近 {{n}} 輪",
|
||||
"profile.throughputTitle": "每輪吞吐量 (tok/s)",
|
||||
"profile.phaseTitle": "每輪各階段耗時 (s)",
|
||||
"profile.turnCol": "輪次",
|
||||
"profile.tokensCol": "Tokens",
|
||||
"profile.wallCol": "總耗時",
|
||||
"profile.turnsLabel": "{{n}} 輪 · 從舊到新",
|
||||
"profile.oneTurn": "1 輪",
|
||||
"profile.diskNote": "磁碟服務是在 I/O 執行緒上讀取專家的時間；它與運算重疊，因此只有運算執行緒實際感受到的 I/O 等待 才計入總耗時分解。多 KV 工作階段時，份額描述的是整個引擎在該輪視窗內的表現。",
|
||||
|
||||
"error.title": "colibrì UI 遇到錯誤",
|
||||
"error.hint": "引擎不受影響。請嘗試重新整理頁面。",
|
||||
"error.retry": "重試",
|
||||
}
|
||||
|
||||
export default zhTW
|
||||
+39
-1
@@ -64,7 +64,9 @@ button:focus-visible, input:focus-visible, textarea:focus-visible, select:focus-
|
||||
.toggle-row { display: flex; align-items: center; justify-content: space-between; height: 42px; padding: 0 11px; border: 1px solid var(--border); border-radius: 9px; color: #a9b4b8; background: var(--input); }
|
||||
.toggle-row > span { display: flex; align-items: center; gap: 8px; font-size: 11px; font-weight: 600; }.toggle-row.active { border-color: rgba(78,214,165,.35); color: var(--foreground); }
|
||||
.toggle-row i { width: 30px; height: 17px; padding: 2px; border-radius: 20px; background: #293136; transition: .2s; }.toggle-row i b { display: block; width: 13px; height: 13px; border-radius: 50%; background: #78858a; transition: .2s; }.toggle-row.active i { background: rgba(78,214,165,.28); }.toggle-row.active i b { transform: translateX(13px); background: var(--primary); }
|
||||
.sidebar-foot { margin-top: auto; display: flex; align-items: center; gap: 7px; color: #59666b; font-size: 10px; }
|
||||
.sidebar-foot { margin-top: auto; display: flex; flex-direction: column; gap: 6px; color: #59666b; font-size: 10px; }
|
||||
.sidebar-foot > div { display: flex; align-items: center; gap: 7px; }
|
||||
.locale-switcher select { background: transparent; border: 1px solid var(--border); border-radius: 4px; color: inherit; font-size: 10px; padding: 2px 4px; cursor: pointer; }
|
||||
|
||||
.chat-panel { min-width: 0; height: 100vh; display: grid; grid-template-rows: 72px minmax(0, 1fr) auto; }
|
||||
.topbar { display: flex; align-items: center; justify-content: space-between; padding: 0 32px; border-bottom: 1px solid var(--border); }
|
||||
@@ -151,3 +153,39 @@ button:focus-visible, input:focus-visible, textarea:focus-visible, select:focus-
|
||||
.brain-tip-spec { color: var(--primary); font-weight: 700; }
|
||||
.brain-tip-spec small, .brain-tip-aff { color: #8b9aa3; font-weight: 400; }
|
||||
.brain-tip-aff { font-size: 10px; }
|
||||
|
||||
/* ---- Profiling page ---- */
|
||||
.prof-page { flex: 1; display: flex; flex-direction: column; gap: 16px; padding: 18px 22px; overflow: auto; }
|
||||
.prof-head { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 10px; }
|
||||
.prof-legend { display: flex; flex-wrap: wrap; gap: 12px; font-size: 11px; color: #a5b0b4; align-items: center; }
|
||||
.prof-legend i { width: 9px; height: 9px; border-radius: 2px; display: inline-block; margin-right: 5px; vertical-align: -1px; }
|
||||
.prof-tiles { display: grid; grid-template-columns: repeat(4, 1fr); gap: 10px; }
|
||||
.prof-tiles > div { display: grid; gap: 4px; padding: 12px 14px; border: 1px solid var(--border); border-radius: 11px; background: var(--card); }
|
||||
.prof-tiles span { display: flex; align-items: center; gap: 5px; color: #7f8d92; font-size: 9px; font-weight: 700; letter-spacing: .08em; text-transform: uppercase; }
|
||||
.prof-tiles strong { font: 600 22px ui-monospace, SFMono-Regular, monospace; font-variant-numeric: tabular-nums; }
|
||||
.prof-tiles small { color: var(--muted-foreground); font-size: 10px; }
|
||||
.prof-shares { display: grid; gap: 10px; padding: 14px; border: 1px solid var(--border); border-radius: 11px; background: var(--card); }
|
||||
.prof-share { display: grid; gap: 6px; }
|
||||
.prof-share-head { display: flex; justify-content: space-between; color: #a5b0b4; font-size: 11px; font-weight: 600; }
|
||||
.prof-share-head code { color: var(--foreground); font-size: 11px; font-variant-numeric: tabular-nums; }
|
||||
.prof-share-bar { display: flex; gap: 2px; height: 22px; border-radius: 6px; overflow: hidden; }
|
||||
.prof-share-bar span { display: grid; place-items: center; min-width: 0; overflow: hidden; color: #06090b; font-size: 10px; font-weight: 700; font-variant-numeric: tabular-nums; transition: width .5s ease; }
|
||||
.prof-charts { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; }
|
||||
.prof-chart { display: grid; gap: 8px; padding: 12px 14px; border: 1px solid var(--border); border-radius: 11px; background: var(--card); }
|
||||
.prof-chart-title { color: #a5b0b4; font-size: 11px; font-weight: 600; }
|
||||
.prof-plot svg { display: block; width: 100%; height: 120px; }
|
||||
.prof-grid { stroke: var(--border); stroke-width: .3; }
|
||||
.prof-plot-foot { display: flex; justify-content: space-between; gap: 8px; margin-top: 6px; color: #7f8d92; font-size: 10px; }
|
||||
.prof-plot-foot code { color: var(--foreground); font-variant-numeric: tabular-nums; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
||||
.prof-table-wrap { overflow-x: auto; border: 1px solid var(--border); border-radius: 11px; background: var(--card); }
|
||||
.prof-table { width: 100%; border-collapse: collapse; font-size: 11px; font-variant-numeric: tabular-nums; }
|
||||
.prof-table th { padding: 9px 12px; border-bottom: 1px solid var(--border); color: #7f8d92; font-size: 9px; font-weight: 700; letter-spacing: .07em; text-transform: uppercase; text-align: left; white-space: nowrap; }
|
||||
.prof-table th i { width: 8px; height: 8px; border-radius: 2px; display: inline-block; margin-right: 5px; vertical-align: -1px; }
|
||||
.prof-table td { padding: 7px 12px; border-bottom: 1px solid var(--border); color: #c3ccd0; white-space: nowrap; }
|
||||
.prof-table tbody tr:last-child td { border-bottom: 0; }
|
||||
.prof-note { margin: 0; padding: 10px 12px; color: var(--muted-foreground); font-size: 10px; line-height: 1.5; border-top: 1px solid var(--border); }
|
||||
@media (max-width: 900px) {
|
||||
.prof-page { padding: 10px; }
|
||||
.prof-tiles { grid-template-columns: 1fr 1fr; }
|
||||
.prof-charts { grid-template-columns: 1fr; }
|
||||
}
|
||||
|
||||
+14
-1
@@ -1,6 +1,6 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest"
|
||||
|
||||
import { extractSSE, getHealth, serverEndpoint, streamChat } from "./api"
|
||||
import { extractSSE, getHealth, getProfile, serverEndpoint, streamChat } from "./api"
|
||||
|
||||
afterEach(() => vi.unstubAllGlobals())
|
||||
|
||||
@@ -36,6 +36,19 @@ describe("runtime API", () => {
|
||||
headers: expect.objectContaining({ Authorization: "Bearer secret" }),
|
||||
}))
|
||||
})
|
||||
|
||||
it("requests the profiling history next to the OpenAI v1 prefix", async () => {
|
||||
const turn = {
|
||||
wall_s: 2.5, prompt_tokens: 7, completion_tokens: 12,
|
||||
expert_disk_s: 0.4, expert_wait_s: 0.1, expert_matmul_s: 0.9,
|
||||
attention_s: 0.6, lm_head_s: 0.2, forwards: 15,
|
||||
}
|
||||
const fetchMock = vi.fn().mockResolvedValue(new Response(JSON.stringify({ seq: 1, turns: [turn] })))
|
||||
vi.stubGlobal("fetch", fetchMock)
|
||||
|
||||
await expect(getProfile("http://localhost:8000/v1/")).resolves.toEqual({ seq: 1, turns: [turn] })
|
||||
expect(fetchMock).toHaveBeenCalledWith("http://localhost:8000/profile", expect.anything())
|
||||
})
|
||||
})
|
||||
|
||||
describe("chat request extensions", () => {
|
||||
|
||||
@@ -49,6 +49,23 @@ export interface HealthResponse {
|
||||
hwinfo?: HwinfoHealth
|
||||
}
|
||||
|
||||
export interface ProfileTurn {
|
||||
wall_s: number
|
||||
prompt_tokens: number
|
||||
completion_tokens: number
|
||||
expert_disk_s: number
|
||||
expert_wait_s: number
|
||||
expert_matmul_s: number
|
||||
attention_s: number
|
||||
lm_head_s: number
|
||||
forwards: number
|
||||
}
|
||||
|
||||
export interface ProfileResponse {
|
||||
seq: number
|
||||
turns: ProfileTurn[]
|
||||
}
|
||||
|
||||
export interface TokenUsage {
|
||||
prompt_tokens: number
|
||||
completion_tokens: number
|
||||
@@ -100,6 +117,12 @@ export async function getHealth(baseUrl: string, apiKey = "", signal?: AbortSign
|
||||
return (await response.json()) as HealthResponse
|
||||
}
|
||||
|
||||
export async function getProfile(baseUrl: string, apiKey = "", signal?: AbortSignal): Promise<ProfileResponse> {
|
||||
const response = await fetch(serverEndpoint(baseUrl, "profile"), { headers: headers(apiKey), signal })
|
||||
if (!response.ok) throw new Error(await responseError(response))
|
||||
return (await response.json()) as ProfileResponse
|
||||
}
|
||||
|
||||
export function extractSSE(buffer: string) {
|
||||
const frames = buffer.split(/\r?\n\r?\n/)
|
||||
const rest = frames.pop() || ""
|
||||
|
||||
+4
-1
@@ -2,10 +2,13 @@ import { createRoot } from "react-dom/client"
|
||||
|
||||
import App from "./App"
|
||||
import { ErrorBoundary } from "./ErrorBoundary"
|
||||
import { LocaleProvider } from "./i18n"
|
||||
import "./index.css"
|
||||
|
||||
createRoot(document.getElementById("root")!).render(
|
||||
<ErrorBoundary>
|
||||
<App />
|
||||
<LocaleProvider>
|
||||
<App />
|
||||
</LocaleProvider>
|
||||
</ErrorBoundary>,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user