url/colibri
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tok: o200k pre-tokenizer support, auto-detected from tokenizer.json

Browse Source 
Inkling ships an o200k-family tokenizer (case-aware Split regex, GPT-4o
lineage) rather than cl100k. tok_load now detects the family from the
pattern itself (\p{Lu} appears only in the o200k regex) so GLM behavior
is untouched, and encode dispatches to a new pretok_chunk_o200k that
replays the regex engine's backtracking order exactly: greedy optional
prefix, maximally-greedy uppercase run given back until the lowercase
run can match, contractions attached to letter runs, \p{N}{1,3}, and
the [\r\n/]* punctuation tail.

tok_unicode_o200k.h adds the two range tables the new classes need
(Lu+Lt and Lm+Lo+M), generated from Python unicodedata.

Validated against HF tokenizers on 357 adversarial strings (case
transitions, contractions, CJK, combining marks, emoji + modifiers,
zero-width chars, 300 mixed-charset fuzz cases): 357/357 identical.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Nicholas Beerbower
2026-07-15 23:10:10 -04:00
parent d45cfa268a
commit 364b9741e2
2 changed files with 343 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c/tok.h
+115 -1
View File
@@ -19,6 +19,7 @@
#include <limits.h>
#include "json.h"
#include "tok_unicode.h"
#include "tok_unicode_o200k.h"
/* ---------- hash map (chiavi binarie con lunghezza) ---------- */
typedef struct { const char *k; int klen; int v; int used; } ment;
@@ -50,6 +51,7 @@ typedef struct {
Special *sp; int nsp; /* added tokens, ordinati per lunghezza decrescente */
uint32_t byte2cp[256]; int byte2cp_len[256]; char byte2str[256][3];
int16_t cp2byte[1024];
int o200k; /* pre_tokenizer regex family: 0 = cl100k (GLM), 1 = o200k (Inkling) */
} Tok;
/* ---------- UTF-8 ---------- */
@@ -144,6 +146,17 @@ static void tok_load(Tok *T, const char *path){
}
qsort(T->sp,T->nsp,sizeof(Special),cmp_sp_len); /* match piu' lungo per primo */
}
/* pre_tokenizer family: the o200k Split regex is recognizable by its
* case-category classes (\p{Lu}...) which cl100k does not use */
jval *pt=json_get(root,"pre_tokenizer");
if(pt){
jval *ps=json_get(pt,"pretokenizers");
if(ps&&ps->t==J_ARR) for(int i=0;i<ps->len;i++){
jval *pat=json_get(ps->kids[i],"pattern");
jval *rx=pat?json_get(pat,"Regex"):NULL;
if(rx&&rx->t==J_STR&&strstr(rx->str,"\\p{Lu}")) T->o200k=1;
}
}
/* arena/buf restano allocati: le stringhe (j_dup) sono malloc indipendenti e ci servono vive */
(void)arena;
}
@@ -241,6 +254,104 @@ static void pretok_chunk(Tok *T, const unsigned char *p, int a, int b, int *out,
free(cp); free(off);
}
/* ---------- pre-tokenizer o200k (Inkling / GPT-4o family) ----------
* Split regex:
* A: [^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?
* B: [^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?
* C: \p{N}{1,3} D: ' ?[^\s\p{L}\p{N}]+[\r\n/]*' E: \s*[\r\n]+ F: \s+(?!\S) G: \s+
* S1 = Lu|Lt|Lm|Lo|M, S2 = Ll|Lm|Lo|M. The letter matcher below replays the
* regex engine's backtracking order exactly: A with greedy optional prefix and
* maximally-greedy S1* given back until S2+ can take >=1 char, then B. */
#define O2_S1(c) (is_U(c)||is_X(c))
#define O2_S2(c) (is_X(c)||(is_L(c)&&!is_U(c)))
static uint32_t o2_low(uint32_t c){ return (c>='A'&&c<='Z')?c+32:c; }
static int o2_contraction(const uint32_t *cp, int n, int k){
if(k<n && cp[k]=='\'' && k+1<n){
uint32_t d=o2_low(cp[k+1]);
if(k+2<n){ uint32_t e=o2_low(cp[k+2]);
if((d=='r'&&e=='e')||(d=='v'&&e=='e')||(d=='l'&&e=='l')) return k+3; }
if(d=='s'||d=='t'||d=='m'||d=='d') return k+2;
}
return k;
}
/* end (cp index) of branch A|B match at i, or -1 */
static int o2_letters(const uint32_t *cp, int n, int i){
/* branch A, prefix greedy (taken first), then without prefix */
for(int pfx=1; pfx>=0; pfx--){
int j0=i;
if(pfx){
uint32_t c=cp[i];
if(c=='\r'||c=='\n'||is_L(c)||is_N(c)||i+1>=n) continue;
j0=i+1;
}
int m1=j0; while(m1<n && O2_S1(cp[m1])) m1++;
for(int s=m1; s>=j0; s--){
if(s<n && O2_S2(cp[s])){
int k=s+1; while(k<n && O2_S2(cp[k])) k++;
return o2_contraction(cp,n,k);
}
}
}
/* branch B */
for(int pfx=1; pfx>=0; pfx--){
int j0=i;
if(pfx){
uint32_t c=cp[i];
if(c=='\r'||c=='\n'||is_L(c)||is_N(c)||i+1>=n) continue;
j0=i+1;
}
int m1=j0; while(m1<n && O2_S1(cp[m1])) m1++;
if(m1>j0){
int k=m1; while(k<n && O2_S2(cp[k])) k++;
return o2_contraction(cp,n,k);
}
}
return -1;
}
static void pretok_chunk_o200k(Tok *T, const unsigned char *p, int a, int b, int *out, int *no, int max){
int nb=b-a; if(nb<=0) return;
uint32_t *cp=malloc((nb+1)*sizeof(uint32_t)); int *off=malloc((nb+2)*sizeof(int)); int n=0;
for(int i=a;i<b;){ uint32_t c; int k=u8_next(p,b,i,&c); off[n]=i; cp[n]=c; n++; i+=k; }
off[n]=b;
#define ISNL(c) ((c)=='\r'||(c)=='\n')
int i=0;
while(i<n){
int start=i; uint32_t c=cp[i];
/* A|B: letter runs with case-aware split + optional contraction */
{
int e=o2_letters(cp,n,i);
if(e>i){ i=e; bpe_piece(T,p,off[start],off[i],out,no,max); continue; }
}
/* C: \p{N}{1,3} */
if(is_N(c)){ int j=i,k=0; while(j<n && is_N(cp[j]) && k<3){ j++; k++; } i=j; bpe_piece(T,p,off[start],off[i],out,no,max); continue; }
/* D: ' ?[^\s\p{L}\p{N}]+[\r\n/]*' */
{
int j=i;
if(c==' ' && j+1<n && !is_S(cp[j+1]) && !is_L(cp[j+1]) && !is_N(cp[j+1])) j++;
if(j<n && !is_S(cp[j]) && !is_L(cp[j]) && !is_N(cp[j])){
while(j<n && !is_S(cp[j]) && !is_L(cp[j]) && !is_N(cp[j])) j++;
while(j<n && (ISNL(cp[j]) || cp[j]=='/')) j++;
i=j; bpe_piece(T,p,off[start],off[i],out,no,max); continue;
}
}
/* E: \s*[\r\n]+ F: \s+(?!\S) G: \s+ (same as cl100k) */
{
int r=i; while(r<n && is_S(cp[r])) r++;
if(r>i){ int last=-1; for(int j=i;j<r;j++) if(ISNL(cp[j])) last=j;
if(last>=0){ i=last+1; bpe_piece(T,p,off[start],off[i],out,no,max); continue; }
int end = (r<n) ? r-1 : r;
if(end<=i) end=i+1;
i=end; bpe_piece(T,p,off[start],off[i],out,no,max); continue;
}
}
i++;
bpe_piece(T,p,off[start],off[i],out,no,max);
}
#undef ISNL
free(cp); free(off);
}
/* ---------- encode: testo -> id (split sugli added token, poi pretok+BPE) ---------- */
static int tok_encode(Tok *T, const char *text, int len, int *out, int max){
const unsigned char *p=(const unsigned char*)text; int no=0; int i=0;
@@ -254,7 +365,10 @@ static int tok_encode(Tok *T, const char *text, int len, int *out, int max){
}
}
int chunk_end = (hitpos<0) ? len : hitpos;
if(chunk_end>i) pretok_chunk(T,p,i,chunk_end,out,&no,max);
if(chunk_end>i){
if(T->o200k) pretok_chunk_o200k(T,p,i,chunk_end,out,&no,max);
else pretok_chunk(T,p,i,chunk_end,out,&no,max);
}
if(hitpos<0) break;
if(no<max) out[no++]=hitid;
i=hitpos+hitlen;