pub fn from_pretrained<S: AsRef<str>>(
    identifier: S,
    params: Option<FromPretrainedParameters>,
) -> Result<PathBuf> {
    let params = params.unwrap_or_else(FromPretrainedParameters::default);
    let cache_dir = ensure_cache_dir()?;

    // Build a custom HTTP Client using our user-agent and custom headers
    let mut headers = header::HeaderMap::new();
    if let Some(ref token) = params.auth_token {
        headers.insert(
            "Authorization",
            header::HeaderValue::from_str(&format!("Bearer {}", token))?,
        );
    }
    let client_builder = Client::builder()
        .user_agent(user_agent(params.user_agent))
        .default_headers(headers);

    // Create a cache object
    let cache = CacheBuilder::with_client_builder(client_builder)
        .dir(cache_dir)
        .build()?;

    let url_to_download = format!(
        "https://huggingface.co/{}/resolve/{}/tokenizer.json",
        identifier.as_ref(),
        params.revision,
    );

    match cache.cached_path(&url_to_download) {
        Err(_) => Err(format!(
            "Model \"{}\" on the Hub doesn't have a tokenizer",
            identifier.as_ref()
        )
        .into()),
        Ok(path) => Ok(path),
    }
}
/// Downloads and cache the identified tokenizer if it exists on
/// the Hugging Face Hub, and returns a local path to the file
Downloads and cache the identified tokenizer if it exists on
the Hugging Face Hub, and returns a local path to the file
Downloads and cache the identified tokenizer if it exists on the Hugging Face Hub and returns a local path to the file
pub fn from_pretrained<S: AsRef<str>>(
    identifier: S,
    params: Option<FromPretrainedParameters>,
) -> Result<PathBuf> {
    let params = params.unwrap_or_else(FromPretrainedParameters::default);
    let cache_dir = ensure_cache_dir()?;
    // Build a custom HTTP Client using our user-agent and custom headers
    let mut headers = header::HeaderMap::new();
    if let Some(ref token) = params.auth_token {
        headers.insert(
            "Authorization",
            header::HeaderValue::from_str(&format!("Bearer {}", token))?,
        );
    }
    let client_builder = Client::builder()
        .user_agent(user_agent(params.user_agent))
        .default_headers(headers);
    // Create a cache object
    let cache = CacheBuilder::with_client_builder(client_builder)
        .dir(cache_dir)
        .build()?;
    let url_to_download = format!(
        "https://huggingface.co/{}/resolve/{}/tokenizer.json",
        identifier.as_ref(),
        params.revision,
    );
    match cache.cached_path(&url_to_download) {
        Err(_) => Err(format!(
            "Model \"{}\" on the Hub doesn't have a tokenizer",
            identifier.as_ref()
        )
        .into()),
        Ok(path) => Ok(path),
    }
}
pub fn from_pretrained < S : AsRef < str > > ( identifier : S , params : Option < FromPretrainedParameters > , ) -> Result < PathBuf > { let params = params . unwrap_or_else ( FromPretrainedParameters :: default ) ; let cache_dir = ensure_cache_dir ( ) ? ; // Build a custom HTTP Client using our user-agent and custom headers let mut headers = header :: HeaderMap :: new ( ) ; if let Some ( ref token ) = params . auth_token { headers . insert ( " " , header :: HeaderValue :: from_str ( & format ! ( " " token ) ) ? , ) ; } let client_builder = Client :: builder ( ) . user_agent ( user_agent ( params . user_agent ) ) . default_headers ( headers ) ; // Create a cache object let cache = CacheBuilder :: with_client_builder ( client_builder ) . dir ( cache_dir ) . build ( ) ? ; let url_to_download = format ! ( " " identifier as_ref ( ) params revision ) ; match cache . cached_path ( & url_to_download ) { Err ( _ ) => Err ( format ! ( " \" \" " identifier as_ref ( ) ) . into ( ) ) , Ok ( path ) => Ok ( path ) , } }
Downloads and cache the identified tokenizer if it exists on
the Hugging Face Hub, and returns a local path to the file
Downloads and cache the identified tokenizer if it exists on the Hugging Face Hub and returns a local path to the file
identifier
S
params
Option<FromPretrainedParameters>