pub fn from_file(
        model_type: ModelType,
        vocab_path: &str,
        merges_path: Option<&str>,
        lower_case: bool,
        strip_accents: Option<bool>,
        add_prefix_space: Option<bool>,
    ) -> Result<Self, RustBertError> {
        let tokenizer = match model_type {
            ModelType::Bert | ModelType::DistilBert | ModelType::Electra => {
                if add_prefix_space.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(
                        format!("Optional input `add_prefix_space` set to value {} but cannot be used by {:?}",
                                add_prefix_space.unwrap(),
                                model_type)));
                }
                TokenizerOption::Bert(BertTokenizer::from_file(
                    vocab_path,
                    lower_case,
                    strip_accents.unwrap_or(lower_case),
                )?)
            }
            ModelType::Roberta | ModelType::Bart => {
                if strip_accents.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(format!(
                        "Optional input `strip_accents` set to value {} but cannot be used by {:?}",
                        strip_accents.unwrap(),
                        model_type
                    )));
                }
                TokenizerOption::Roberta(RobertaTokenizer::from_file(
                    vocab_path,
                    merges_path.expect("No merges specified!"),
                    lower_case,
                    add_prefix_space.unwrap_or(false),
                )?)
            }
            ModelType::Marian => {
                if strip_accents.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(format!(
                        "Optional input `strip_accents` set to value {} but cannot be used by {:?}",
                        strip_accents.unwrap(),
                        model_type
                    )));
                }
                if add_prefix_space.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(
                        format!("Optional input `add_prefix_space` set to value {} but cannot be used by {:?}",
                                add_prefix_space.unwrap(),
                                model_type)));
                }
                TokenizerOption::Marian(MarianTokenizer::from_files(
                    vocab_path,
                    merges_path.expect("No merges specified!"),
                    lower_case,
                )?)
            }
            ModelType::T5 => {
                if strip_accents.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(format!(
                        "Optional input `strip_accents` set to value {} but cannot be used by {:?}",
                        strip_accents.unwrap(),
                        model_type
                    )));
                }
                if add_prefix_space.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(
                        format!("Optional input `add_prefix_space` set to value {} but cannot be used by {:?}",
                                add_prefix_space.unwrap(),
                                model_type)));
                }
                TokenizerOption::T5(T5Tokenizer::from_file(vocab_path, lower_case)?)
            }
            ModelType::XLMRoberta => {
                if strip_accents.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(format!(
                        "Optional input `strip_accents` set to value {} but cannot be used by {:?}",
                        strip_accents.unwrap(),
                        model_type
                    )));
                }
                if add_prefix_space.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(
                        format!("Optional input `add_prefix_space` set to value {} but cannot be used by {:?}",
                                add_prefix_space.unwrap(),
                                model_type)));
                }
                TokenizerOption::XLMRoberta(XLMRobertaTokenizer::from_file(vocab_path, lower_case)?)
            }
            ModelType::Albert => {
                if strip_accents.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(format!(
                        "Optional input `strip_accents` set to value {} but cannot be used by {:?}",
                        strip_accents.unwrap(),
                        model_type
                    )));
                }
                TokenizerOption::Albert(AlbertTokenizer::from_file(
                    vocab_path,
                    lower_case,
                    strip_accents.unwrap_or(lower_case),
                )?)
            }
        };
        Ok(tokenizer)
    }
/// Interface method to load a tokenizer from file
Interface method to load a tokenizer from file
Interface method to load a tokenizer from file
pub fn from_file(
        model_type: ModelType,
        vocab_path: &str,
        merges_path: Option<&str>,
        lower_case: bool,
        strip_accents: Option<bool>,
        add_prefix_space: Option<bool>,
    ) -> Result<Self, RustBertError> {
        let tokenizer = match model_type {
            ModelType::Bert | ModelType::DistilBert | ModelType::Electra => {
                if add_prefix_space.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(
                        format!("Optional input `add_prefix_space` set to value {} but cannot be used by {:?}",
                                add_prefix_space.unwrap(),
                                model_type)));
                }
                TokenizerOption::Bert(BertTokenizer::from_file(
                    vocab_path,
                    lower_case,
                    strip_accents.unwrap_or(lower_case),
                )?)
            }
            ModelType::Roberta | ModelType::Bart => {
                if strip_accents.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(format!(
                        "Optional input `strip_accents` set to value {} but cannot be used by {:?}",
                        strip_accents.unwrap(),
                        model_type
                    )));
                }
                TokenizerOption::Roberta(RobertaTokenizer::from_file(
                    vocab_path,
                    merges_path.expect("No merges specified!"),
                    lower_case,
                    add_prefix_space.unwrap_or(false),
                )?)
            }
            ModelType::Marian => {
                if strip_accents.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(format!(
                        "Optional input `strip_accents` set to value {} but cannot be used by {:?}",
                        strip_accents.unwrap(),
                        model_type
                    )));
                }
                if add_prefix_space.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(
                        format!("Optional input `add_prefix_space` set to value {} but cannot be used by {:?}",
                                add_prefix_space.unwrap(),
                                model_type)));
                }
                TokenizerOption::Marian(MarianTokenizer::from_files(
                    vocab_path,
                    merges_path.expect("No merges specified!"),
                    lower_case,
                )?)
            }
            ModelType::T5 => {
                if strip_accents.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(format!(
                        "Optional input `strip_accents` set to value {} but cannot be used by {:?}",
                        strip_accents.unwrap(),
                        model_type
                    )));
                }
                if add_prefix_space.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(
                        format!("Optional input `add_prefix_space` set to value {} but cannot be used by {:?}",
                                add_prefix_space.unwrap(),
                                model_type)));
                }
                TokenizerOption::T5(T5Tokenizer::from_file(vocab_path, lower_case)?)
            }
            ModelType::XLMRoberta => {
                if strip_accents.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(format!(
                        "Optional input `strip_accents` set to value {} but cannot be used by {:?}",
                        strip_accents.unwrap(),
                        model_type
                    )));
                }
                if add_prefix_space.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(
                        format!("Optional input `add_prefix_space` set to value {} but cannot be used by {:?}",
                                add_prefix_space.unwrap(),
                                model_type)));
                }
                TokenizerOption::XLMRoberta(XLMRobertaTokenizer::from_file(vocab_path, lower_case)?)
            }
            ModelType::Albert => {
                if strip_accents.is_some() {
                    return Err(RustBertError::InvalidConfigurationError(format!(
                        "Optional input `strip_accents` set to value {} but cannot be used by {:?}",
                        strip_accents.unwrap(),
                        model_type
                    )));
                }
                TokenizerOption::Albert(AlbertTokenizer::from_file(
                    vocab_path,
                    lower_case,
                    strip_accents.unwrap_or(lower_case),
                )?)
            }
        };
        Ok(tokenizer)
    }
pub fn from_file ( model_type : ModelType , vocab_path : & str , merges_path : Option < & str > , lower_case : bool , strip_accents : Option < bool > , add_prefix_space : Option < bool > , ) -> Result < Self , RustBertError > { let tokenizer = match model_type { ModelType :: Bert | ModelType :: DistilBert | ModelType :: Electra => { if add_prefix_space . is_some ( ) { return Err ( RustBertError :: InvalidConfigurationError ( format ! ( " " add_prefix_space unwrap ( ) model_type ) ) ) ; } TokenizerOption :: Bert ( BertTokenizer :: from_file ( vocab_path , lower_case , strip_accents . unwrap_or ( lower_case ) , ) ? ) } ModelType :: Roberta | ModelType :: Bart => { if strip_accents . is_some ( ) { return Err ( RustBertError :: InvalidConfigurationError ( format ! ( " " strip_accents unwrap ( ) model_type ) ) ) ; } TokenizerOption :: Roberta ( RobertaTokenizer :: from_file ( vocab_path , merges_path . expect ( " " ) , lower_case , add_prefix_space . unwrap_or ( false ) , ) ? ) } ModelType :: Marian => { if strip_accents . is_some ( ) { return Err ( RustBertError :: InvalidConfigurationError ( format ! ( " " strip_accents unwrap ( ) model_type ) ) ) ; } if add_prefix_space . is_some ( ) { return Err ( RustBertError :: InvalidConfigurationError ( format ! ( " " add_prefix_space unwrap ( ) model_type ) ) ) ; } TokenizerOption :: Marian ( MarianTokenizer :: from_files ( vocab_path , merges_path . expect ( " " ) , lower_case , ) ? ) } ModelType :: T5 => { if strip_accents . is_some ( ) { return Err ( RustBertError :: InvalidConfigurationError ( format ! ( " " strip_accents unwrap ( ) model_type ) ) ) ; } if add_prefix_space . is_some ( ) { return Err ( RustBertError :: InvalidConfigurationError ( format ! ( " " add_prefix_space unwrap ( ) model_type ) ) ) ; } TokenizerOption :: T5 ( T5Tokenizer :: from_file ( vocab_path , lower_case ) ? ) } ModelType :: XLMRoberta => { if strip_accents . is_some ( ) { return Err ( RustBertError :: InvalidConfigurationError ( format ! ( " " strip_accents unwrap ( ) model_type ) ) ) ; } if add_prefix_space . is_some ( ) { return Err ( RustBertError :: InvalidConfigurationError ( format ! ( " " add_prefix_space unwrap ( ) model_type ) ) ) ; } TokenizerOption :: XLMRoberta ( XLMRobertaTokenizer :: from_file ( vocab_path , lower_case ) ? ) } ModelType :: Albert => { if strip_accents . is_some ( ) { return Err ( RustBertError :: InvalidConfigurationError ( format ! ( " " strip_accents unwrap ( ) model_type ) ) ) ; } TokenizerOption :: Albert ( AlbertTokenizer :: from_file ( vocab_path , lower_case , strip_accents . unwrap_or ( lower_case ) , ) ? ) } } ; Ok ( tokenizer ) }
Interface method to load a tokenizer from file
Interface method to load a tokenizer from file
model_type
ModelType
vocab_path
&str
merges_path
Option<&str>
lower_case
bool
strip_accents
Option<bool>
add_prefix_space
Option<bool>
model_type
ModelType
vocab_path
&str
merges_path
Option<&str>
lower_case
bool
strip_accents
Option<bool>
add_prefix_space
Option<bool>