remove duplicated code

Author: marinegor

crates/bpe/src/byte_pair_encoding.rs

@@ -555,7 +555,12 @@ impl BytePairEncoding {
     /// Result of the encoding will be non-deterministic unless `seed` is provided.
     /// Implementation loosely follows original BPE dropout paper: https://arxiv.org/abs/1910.13267
     #[cfg(feature = "rand")]
-    pub fn encode_minimal_dropout<R: rand::Rng>(&self, text: &[u8], dropout: f32, mut rng: R) -> Vec<u32> {
+    pub fn encode_minimal_dropout<R: rand::Rng>(
+        &self,
+        text: &[u8],
+        dropout: f32,
+        mut rng: R,
+    ) -> Vec<u32> {
         assert!(0.0 <= dropout);
         assert!(dropout <= 1.0);
 
@@ -587,76 +592,6 @@ impl BytePairEncoding {
         }
         encoded
     }
-
-    /// This function computes the encoding while randomly rejecting some merges.
-    /// Result of the encoding will be non-deterministic unless `seed` is provided.
-    /// Implementation loosely follows original BPE dropout paper: https://arxiv.org/abs/1910.13267
-    #[cfg(feature = "rand")]
-    pub fn encode_minimal_dropout(&self, text: &[u8], dropout: f32, seed: Option<u64>) -> Vec<u32> {
-        use rand::rngs::StdRng;
-        use rand::seq::IndexedRandom;
-        use rand::Rng;
-        use rand::SeedableRng;
-        use std::collections::HashSet;
-
-        fn get_rng(seed: Option<u64>) -> StdRng {
-            match seed {
-                Some(num) => {
-                    // Expand the u64 seed to 32 bytes
-                    let mut seed_bytes = [0u8; 32];
-                    seed_bytes[..8].copy_from_slice(&num.to_le_bytes());
-                    StdRng::from_seed(seed_bytes)
-                }
-                None => {
-                    // Seed StdRng with a random 32-byte array from ThreadRng
-                    let mut thread_rng = rand::rng();
-                    let mut seed_bytes = [0u8; 32];
-                    thread_rng.fill(&mut seed_bytes);
-                    StdRng::from_seed(seed_bytes)
-                }
-            }
-        }
-
-        let mut rng = get_rng(seed);
-
-        assert!(0.0 <= dropout);
-        assert!(dropout <= 1.0);
-
-        let mut last_token: Vec<(u32, u32)> = Vec::with_capacity(text.len());
-
-        let allowed_tokens: Vec<u32> = self.pair_lookup.values().cloned().collect();
-        let tokens_after_dropout = (allowed_tokens.len() as f32) * dropout;
-        let forbidden_tokens_set: HashSet<&u32> = HashSet::from_iter(
-            allowed_tokens.choose_multiple(&mut rng, tokens_after_dropout.floor() as usize),
-        );
-
-        let mut state = self.overlapping_searcher.start_state();
-        for (pos, c) in text.iter().enumerate() {
-            let (s, iter) = self.overlapping_searcher.consume(state, pos + 1, *c);
-            state = s;
-            let mut best = (0, u32::MAX);
-            for m in iter {
-                if m.start() == 0 {
-                    best = (m.value(), 1);
-                    break;
-                } else if (last_token[m.start() - 1].1 + 1 < best.1)
-                    & (!(forbidden_tokens_set.contains(&m.value())) | ((m.end() - m.start()) == 1))
-                {
-                    best = (m.value(), last_token[m.start() - 1].1 + 1);
-                }
-            }
-            last_token.push(best);
-        }
-        let mut encoded = Vec::with_capacity(last_token.last().map(|l| l.1 as usize).unwrap_or(0));
-        let mut pos = text.len();
-        while pos > 0 {
-            let token = last_token[pos - 1].0;
-            encoded.push(token);
-            pos -= self.token_len(token);
-        }
-        encoded.reverse();
-        encoded
-    }
 }
 
 /// Create a random test string for the given [`BytePairEncoding`]. The string will be at least [`min_bytes`] long.