check single-len tokens as well

marinegor · marinegor · commit a10cce2bcf8a · 2026-02-26T23:30:11.000+01:00
diff --git a/crates/bpe/src/byte_pair_encoding.rs b/crates/bpe/src/byte_pair_encoding.rs
@@ -604,7 +604,7 @@ impl BytePairEncoding {
                     best = (m.value(), 1);
                     break;
                 } else if (last_token[m.start() - 1].1 + 1 < best.1)
-                    & !(forbidden_tokens_set.contains(&m.value()))
+                    & (!(forbidden_tokens_set.contains(&m.value())) | ((m.end() - m.start()) == 1))
                 {
                     best = (m.value(), last_token[m.start() - 1].1 + 1);
                 }

Original file line number	Diff line number	Diff line change
`@@ -604,7 +604,7 @@ impl BytePairEncoding {`
`604`	`604`	`best = (m.value(), 1);`
`605`	`605`	`break;`
`606`	`606`	`} else if (last_token[m.start() - 1].1 + 1 < best.1)`
`607`		`- & !(forbidden_tokens_set.contains(&m.value()))`
	`607`	`+ & (!(forbidden_tokens_set.contains(&m.value())) \| ((m.end() - m.start()) == 1))`
`608`	`608`	`{`
`609`	`609`	`best = (m.value(), last_token[m.start() - 1].1 + 1);`
`610`	`610`	`}`