fn do_tokenize<P: Into<PreTokenizedString>>(
        &self,
        pretokenized: P,
        type_id: u32,
        word_idx: Option<u32>,
        offsets_type: OffsetType,
    ) -> Result<Encoding> {
        let mut pretokenized: PreTokenizedString = pretokenized.into();
        pretokenized.tokenize(|normalized| self.model.tokenize(normalized.get()))?;
        pretokenized.into_encoding(word_idx, type_id, offsets_type)
    }
/// Tokenization logic, makes the bridge between the pre-tokenization phase and the real
/// tokenization phase, and converting offsets back to the original referential.
Tokenization logic, makes the bridge between the pre-tokenization phase and the real
tokenization phase, and converting offsets back to the original referential.
Tokenization logic makes the bridge between the pre - tokenization phase and the real tokenization phase and converting offsets back to the original referential .
fn do_tokenize<P: Into<PreTokenizedString>>(
        &self,
        pretokenized: P,
        type_id: u32,
        word_idx: Option<u32>,
        offsets_type: OffsetType,
    ) -> Result<Encoding> {
        let mut pretokenized: PreTokenizedString = pretokenized.into();
        pretokenized.tokenize(|normalized| self.model.tokenize(normalized.get()))?;
        pretokenized.into_encoding(word_idx, type_id, offsets_type)
    }
fn do_tokenize < P : Into < PreTokenizedString > > ( & self , pretokenized : P , type_id : u32 , word_idx : Option < u32 > , offsets_type : OffsetType , ) -> Result < Encoding > { let mut pretokenized : PreTokenizedString = pretokenized . into ( ) ; pretokenized . tokenize ( | normalized | self . model . tokenize ( normalized . get ( ) ) ) ? ; pretokenized . into_encoding ( word_idx , type_id , offsets_type ) }
Tokenization logic, makes the bridge between the pre-tokenization phase and the real
tokenization phase, and converting offsets back to the original referential.
Tokenization logic makes the bridge between the pre - tokenization phase and the real tokenization phase and converting offsets back to the original referential .
&self
pretokenized
P
type_id
u32
word_idx
Option<u32>
offsets_type
OffsetType