Nicholas Beerbower
9e2d41567c
tests/tok_o200k_tiny.json is a synthetic byte-level BPE (274 vocab, a
few KB) whose Split regex is the o200k pattern; expected ids in
tok_o200k_cases.txt were generated by HF tokenizers on that same file.
test_tok_o200k (in TEST_BINS) scores 40/40 encode + 40/40 round-trip:
case-transition splits, contractions, digit groups, the [\r\n/]* tail,
whitespace branches, CJK/Greek/Cyrillic, added-token atomicity.
The cl100k path is untouched by construction — dispatch requires
\p{Lu} in the tokenizer's own Split pattern, which cl100k lacks — and
stays covered by the GLM oracle (verified on this branch: 32/32).
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
62 lines
2.9 KiB
C
62 lines
2.9 KiB
C
/* o200k pre-tokenizer validation against HF-tokenizers-generated expectations.
|
|
* Self-contained for the test-c harness: loads tests/tok_o200k_tiny.json (a
|
|
* synthetic byte-level BPE whose Split regex is the o200k pattern — a few KB,
|
|
* no model download) and scores tests/tok_o200k_cases.txt, whose expected ids
|
|
* were produced by HF `tokenizers` on the same file. Guards the case-aware
|
|
* letter matcher, contractions, digit groups, the [\r\n/]* punctuation tail,
|
|
* whitespace branches, and added-token atomicity; round-trips every case.
|
|
* The cl100k path is untouched by construction (dispatch requires \p{Lu} in
|
|
* the tokenizer's own Split pattern) and stays covered by the GLM oracle. */
|
|
#define _GNU_SOURCE
|
|
#include "../tok.h"
|
|
|
|
int main(void) {
|
|
Tok T;
|
|
tok_load(&T, "tests/tok_o200k_tiny.json");
|
|
if (!T.o200k) { fprintf(stderr, "test_tok_o200k: o200k pattern not detected\n"); return 1; }
|
|
FILE *f = fopen("tests/tok_o200k_cases.txt", "rb");
|
|
if (!f) { perror("tests/tok_o200k_cases.txt"); return 1; }
|
|
char *line = NULL; size_t cap = 0; ssize_t nr;
|
|
int pass = 0, tot = 0, dpass = 0;
|
|
while ((nr = getline(&line, &cap, f)) >= 0) {
|
|
if (nr > 0 && line[nr-1] == '\n') line[--nr] = 0;
|
|
if (nr == 0) continue;
|
|
char *tab = strchr(line, '\t'); if (!tab) continue;
|
|
*tab = 0;
|
|
const char *text = line, *idstr = tab + 1;
|
|
char tbuf[4096]; int tn = 0;
|
|
for (const char *q = text; *q && tn < 4095; q++) {
|
|
if (q[0]=='\\' && q[1]=='n') { tbuf[tn++]='\n'; q++; }
|
|
else if (q[0]=='\\' && q[1]=='t') { tbuf[tn++]='\t'; q++; }
|
|
else if (q[0]=='\\' && q[1]=='r') { tbuf[tn++]='\r'; q++; }
|
|
else if (q[0]=='\\' && q[1]=='\\') { tbuf[tn++]='\\'; q++; }
|
|
else tbuf[tn++] = *q;
|
|
}
|
|
tbuf[tn] = 0;
|
|
int exp[512], ne = 0;
|
|
for (const char *q = idstr; *q; ) {
|
|
while (*q == ',' || *q == ' ') q++;
|
|
if (!*q) break;
|
|
exp[ne++] = atoi(q);
|
|
while (*q && *q != ',') q++;
|
|
}
|
|
int got[512]; int ng = tok_encode(&T, tbuf, tn, got, 512);
|
|
int ok = (ng == ne);
|
|
for (int i = 0; i < ng && ok; i++) ok = (got[i] == exp[i]);
|
|
tot++; if (ok) pass++;
|
|
char dec[8192]; int dn = tok_decode(&T, got, ng, dec, 8191);
|
|
int drt = (dn == tn) && !memcmp(dec, tbuf, tn);
|
|
if (drt) dpass++;
|
|
if (!ok || !drt) {
|
|
fprintf(stderr, "MISMATCH text=%s\n exp(%d):", text, ne);
|
|
for (int i = 0; i < ne; i++) fprintf(stderr, " %d", exp[i]);
|
|
fprintf(stderr, "\n got(%d):", ng);
|
|
for (int i = 0; i < ng; i++) fprintf(stderr, " %d", got[i]);
|
|
fprintf(stderr, "\n decode_ok=%d\n", drt);
|
|
}
|
|
}
|
|
fclose(f);
|
|
printf("test_tok_o200k: ENCODE %d/%d DECODE %d/%d\n", pass, tot, dpass, tot);
|
|
return (pass == tot && dpass == tot) ? 0 : 2;
|
|
}
|