fn do_pre_tokenize<P: Into<PreTokenizedString>>(
        &self,
        pretokenized: P,
    ) -> Result<PreTokenizedString> {
        let mut pretokenized: PreTokenizedString = pretokenized.into();

        if let Some(ref pretok) = self.pre_tokenizer {
            pretok.pre_tokenize(&mut pretokenized)?;
        }

        Ok(pretokenized)
    }
/// PreTokenization logic, handling the case where there is no PreTokenizer set
PreTokenization logic, handling the case where there is no PreTokenizer set
PreTokenization logic handling the case where there is no PreTokenizer set
fn do_pre_tokenize<P: Into<PreTokenizedString>>(
        &self,
        pretokenized: P,
    ) -> Result<PreTokenizedString> {
        let mut pretokenized: PreTokenizedString = pretokenized.into();
        if let Some(ref pretok) = self.pre_tokenizer {
            pretok.pre_tokenize(&mut pretokenized)?;
        }
        Ok(pretokenized)
    }
fn do_pre_tokenize < P : Into < PreTokenizedString > > ( & self , pretokenized : P , ) -> Result < PreTokenizedString > { let mut pretokenized : PreTokenizedString = pretokenized . into ( ) ; if let Some ( ref pretok ) = self . pre_tokenizer { pretok . pre_tokenize ( & mut pretokenized ) ? ; } Ok ( pretokenized ) }
PreTokenization logic, handling the case where there is no PreTokenizer set
PreTokenization logic handling the case where there is no PreTokenizer set
&self
pretokenized
P