pub fn new(model: M) -> Self {
        TokenizerImpl {
            normalizer: None,
            pre_tokenizer: None,
            model,
            post_processor: None,
            decoder: None,

            added_vocabulary: AddedVocabulary::new(),

            truncation: None,
            padding: None,
        }
    }
/// Instantiate a new Tokenizer, with the given Model
Instantiate a new Tokenizer, with the given Model
Instantiate a new Tokenizer with the given Model
pub fn new(model: M) -> Self {
        TokenizerImpl {
            normalizer: None,
            pre_tokenizer: None,
            model,
            post_processor: None,
            decoder: None,
            added_vocabulary: AddedVocabulary::new(),
            truncation: None,
            padding: None,
        }
    }
pub fn new ( model : M ) -> Self { TokenizerImpl { normalizer : None , pre_tokenizer : None , model , post_processor : None , decoder : None , added_vocabulary : AddedVocabulary :: new ( ) , truncation : None , padding : None , } }
Instantiate a new Tokenizer, with the given Model
Instantiate a new Tokenizer with the given Model
model
M