pub fn from_pretrained<S: AsRef<str>>(
        identifier: S,
        params: Option<FromPretrainedParameters>,
    ) -> Result<Self> {
        let tokenizer_file = from_pretrained(identifier, params)?;
        TokenizerImpl::from_file(tokenizer_file)
    }
/// Instantiate a new Tokenizer from a file hosted on the Hugging Face Hub.
/// It expects the `identifier` of a model that includes a `tokenizer.json` file.
Instantiate a new Tokenizer from a file hosted on the Hugging Face Hub.
It expects the `identifier` of a model that includes a `tokenizer.json` file.
Instantiate a new Tokenizer from a file hosted on the Hugging Face Hub . It expects the ` identifier ` of a model that includes a ` tokenizer . json ` file .
pub fn from_pretrained<S: AsRef<str>>(
        identifier: S,
        params: Option<FromPretrainedParameters>,
    ) -> Result<Self> {
        let tokenizer_file = from_pretrained(identifier, params)?;
        TokenizerImpl::from_file(tokenizer_file)
    }
pub fn from_pretrained < S : AsRef < str > > ( identifier : S , params : Option < FromPretrainedParameters > , ) -> Result < Self > { let tokenizer_file = from_pretrained ( identifier , params ) ? ; TokenizerImpl :: from_file ( tokenizer_file ) }
Instantiate a new Tokenizer from a file hosted on the Hugging Face Hub.
Instantiate a new Tokenizer from a file hosted on the Hugging Face Hub .
identifier
S
params
Option<FromPretrainedParameters>