Nicholas Beerbower
5ad4d540ab
MinGW's UCRT has no getline; fixed-buffer fgets with CRLF trimming reads the same case file everywhere. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
65 lines
3.0 KiB
C
65 lines
3.0 KiB
C
/* o200k pre-tokenizer validation against HF-tokenizers-generated expectations.
|
|
* Self-contained for the test-c harness: loads tests/tok_o200k_tiny.json (a
|
|
* synthetic byte-level BPE whose Split regex is the o200k pattern — a few KB,
|
|
* no model download) and scores tests/tok_o200k_cases.txt, whose expected ids
|
|
* were produced by HF `tokenizers` on the same file. Guards the case-aware
|
|
* letter matcher, contractions, digit groups, the [\r\n/]* punctuation tail,
|
|
* whitespace branches, and added-token atomicity; round-trips every case.
|
|
* The cl100k path is untouched by construction (dispatch requires \p{Lu} in
|
|
* the tokenizer's own Split pattern) and stays covered by the GLM oracle. */
|
|
#define _GNU_SOURCE
|
|
#include "../tok.h"
|
|
|
|
int main(void) {
|
|
Tok T;
|
|
tok_load(&T, "tests/tok_o200k_tiny.json");
|
|
if (!T.o200k) { fprintf(stderr, "test_tok_o200k: o200k pattern not detected\n"); return 1; }
|
|
FILE *f = fopen("tests/tok_o200k_cases.txt", "rb");
|
|
if (!f) { perror("tests/tok_o200k_cases.txt"); return 1; }
|
|
/* fgets, not getline: MinGW's UCRT lacks getline and this must run on
|
|
* the windows job. Case lines are short; 8 KB is generous. */
|
|
char line[8192];
|
|
int pass = 0, tot = 0, dpass = 0;
|
|
while (fgets(line, sizeof(line), f)) {
|
|
size_t nr = strlen(line);
|
|
while (nr > 0 && (line[nr-1] == '\n' || line[nr-1] == '\r')) line[--nr] = 0;
|
|
if (nr == 0) continue;
|
|
char *tab = strchr(line, '\t'); if (!tab) continue;
|
|
*tab = 0;
|
|
const char *text = line, *idstr = tab + 1;
|
|
char tbuf[4096]; int tn = 0;
|
|
for (const char *q = text; *q && tn < 4095; q++) {
|
|
if (q[0]=='\\' && q[1]=='n') { tbuf[tn++]='\n'; q++; }
|
|
else if (q[0]=='\\' && q[1]=='t') { tbuf[tn++]='\t'; q++; }
|
|
else if (q[0]=='\\' && q[1]=='r') { tbuf[tn++]='\r'; q++; }
|
|
else if (q[0]=='\\' && q[1]=='\\') { tbuf[tn++]='\\'; q++; }
|
|
else tbuf[tn++] = *q;
|
|
}
|
|
tbuf[tn] = 0;
|
|
int exp[512], ne = 0;
|
|
for (const char *q = idstr; *q; ) {
|
|
while (*q == ',' || *q == ' ') q++;
|
|
if (!*q) break;
|
|
exp[ne++] = atoi(q);
|
|
while (*q && *q != ',') q++;
|
|
}
|
|
int got[512]; int ng = tok_encode(&T, tbuf, tn, got, 512);
|
|
int ok = (ng == ne);
|
|
for (int i = 0; i < ng && ok; i++) ok = (got[i] == exp[i]);
|
|
tot++; if (ok) pass++;
|
|
char dec[8192]; int dn = tok_decode(&T, got, ng, dec, 8191);
|
|
int drt = (dn == tn) && !memcmp(dec, tbuf, tn);
|
|
if (drt) dpass++;
|
|
if (!ok || !drt) {
|
|
fprintf(stderr, "MISMATCH text=%s\n exp(%d):", text, ne);
|
|
for (int i = 0; i < ne; i++) fprintf(stderr, " %d", exp[i]);
|
|
fprintf(stderr, "\n got(%d):", ng);
|
|
for (int i = 0; i < ng; i++) fprintf(stderr, " %d", got[i]);
|
|
fprintf(stderr, "\n decode_ok=%d\n", drt);
|
|
}
|
|
}
|
|
fclose(f);
|
|
printf("test_tok_o200k: ENCODE %d/%d DECODE %d/%d\n", pass, tot, dpass, tot);
|
|
return (pass == tot && dpass == tot) ? 0 : 2;
|
|
}
|