You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
let all_tokens = tokens.into_iter().collect_vec();
174
174
letmut rnd = rand::rng();
@@ -573,7 +573,7 @@ impl BytePairEncoding {
573
573
// and hence may include previously discarded token later down the byte stream. At the sentence level though we don't expect it to make much difference.
574
574
// Also, this implementation of BPE constructs merges on the fly from the set of tokens, hence might come up with a different set of merges with the same dictionary.
0 commit comments