pub fn with_config(config: TokenizerConfig) -> LinderaResult<Tokenizer> {
        let dict = if let Some(ref path) = config.dict_path {
            lindera_dictionary::prefix_dict(path)?
        } else {
            lindera_ipadic::prefix_dict()
        };

        let cost_matrix = if let Some(path) = config.dict_path {
            lindera_dictionary::connection(path)?
        } else {
            lindera_ipadic::connection()
        };

        let char_definitions = if let Some(path) = config.dict_path {
            lindera_dictionary::char_def(&path)?
        } else {
            lindera_ipadic::char_def()?
        };

        let unknown_dictionary = if let Some(path) = config.dict_path {
            lindera_dictionary::unknown_dict(path)?
        } else {
            lindera_ipadic::unknown_dict()?
        };

        let words_idx_data = if let Some(path) = config.dict_path {
            lindera_dictionary::words_idx_data(path)?
        } else {
            lindera_ipadic::words_idx_data()
        };

        let words_data = if let Some(path) = config.dict_path {
            lindera_dictionary::words_data(path)?
        } else {
            lindera_ipadic::words_data()
        };

        let (user_dict, user_dict_words_idx_data, user_dict_words_data) =
            if let Some(path) = config.user_dict_path {
                let builder = IpadicBuilder::new();
                let user_dict = builder.build_user_dict(path)?;
                (
                    Some(user_dict.dict),
                    Some(user_dict.words_idx_data),
                    Some(user_dict.words_data),
                )
            } else if let Some(path) = config.user_dict_bin_path {
                let user_dict_bin_data = read_file(path)?;
                let user_dict = UserDictionary::load(&user_dict_bin_data)?;
                (
                    Some(user_dict.dict),
                    Some(user_dict.words_idx_data),
                    Some(user_dict.words_data),
                )
            } else {
                (None, None, None)
            };

        let tokenizer = Tokenizer {
            dict,
            cost_matrix,
            lattice: Lattice::default(),
            char_definitions,
            unknown_dictionary,
            words_idx_data,
            words_data,
            user_dict,
            user_dict_words_idx_data,
            user_dict_words_data,
            mode: config.mode,
        };

        Ok(tokenizer)
    }
/// Creates a new instance with the config
///
/// # Arguments
///
/// * `config`: settings of Tokenizer
///
/// returns: Result<Tokenizer, LinderaError>
///
Creates a new instance with the config
Creates a new instance with the config
pub fn with_config(config: TokenizerConfig) -> LinderaResult<Tokenizer> {
        let dict = if let Some(ref path) = config.dict_path {
            lindera_dictionary::prefix_dict(path)?
        } else {
            lindera_ipadic::prefix_dict()
        };
        let cost_matrix = if let Some(path) = config.dict_path {
            lindera_dictionary::connection(path)?
        } else {
            lindera_ipadic::connection()
        };
        let char_definitions = if let Some(path) = config.dict_path {
            lindera_dictionary::char_def(&path)?
        } else {
            lindera_ipadic::char_def()?
        };
        let unknown_dictionary = if let Some(path) = config.dict_path {
            lindera_dictionary::unknown_dict(path)?
        } else {
            lindera_ipadic::unknown_dict()?
        };
        let words_idx_data = if let Some(path) = config.dict_path {
            lindera_dictionary::words_idx_data(path)?
        } else {
            lindera_ipadic::words_idx_data()
        };
        let words_data = if let Some(path) = config.dict_path {
            lindera_dictionary::words_data(path)?
        } else {
            lindera_ipadic::words_data()
        };
        let (user_dict, user_dict_words_idx_data, user_dict_words_data) =
            if let Some(path) = config.user_dict_path {
                let builder = IpadicBuilder::new();
                let user_dict = builder.build_user_dict(path)?;
                (
                    Some(user_dict.dict),
                    Some(user_dict.words_idx_data),
                    Some(user_dict.words_data),
                )
            } else if let Some(path) = config.user_dict_bin_path {
                let user_dict_bin_data = read_file(path)?;
                let user_dict = UserDictionary::load(&user_dict_bin_data)?;
                (
                    Some(user_dict.dict),
                    Some(user_dict.words_idx_data),
                    Some(user_dict.words_data),
                )
            } else {
                (None, None, None)
            };
        let tokenizer = Tokenizer {
            dict,
            cost_matrix,
            lattice: Lattice::default(),
            char_definitions,
            unknown_dictionary,
            words_idx_data,
            words_data,
            user_dict,
            user_dict_words_idx_data,
            user_dict_words_data,
            mode: config.mode,
        };
        Ok(tokenizer)
    }
pub fn with_config ( config : TokenizerConfig ) -> LinderaResult < Tokenizer > { let dict = if let Some ( ref path ) = config . dict_path { lindera_dictionary :: prefix_dict ( path ) ? } else { lindera_ipadic :: prefix_dict ( ) } ; let cost_matrix = if let Some ( path ) = config . dict_path { lindera_dictionary :: connection ( path ) ? } else { lindera_ipadic :: connection ( ) } ; let char_definitions = if let Some ( path ) = config . dict_path { lindera_dictionary :: char_def ( & path ) ? } else { lindera_ipadic :: char_def ( ) ? } ; let unknown_dictionary = if let Some ( path ) = config . dict_path { lindera_dictionary :: unknown_dict ( path ) ? } else { lindera_ipadic :: unknown_dict ( ) ? } ; let words_idx_data = if let Some ( path ) = config . dict_path { lindera_dictionary :: words_idx_data ( path ) ? } else { lindera_ipadic :: words_idx_data ( ) } ; let words_data = if let Some ( path ) = config . dict_path { lindera_dictionary :: words_data ( path ) ? } else { lindera_ipadic :: words_data ( ) } ; let ( user_dict , user_dict_words_idx_data , user_dict_words_data ) = if let Some ( path ) = config . user_dict_path { let builder = IpadicBuilder :: new ( ) ; let user_dict = builder . build_user_dict ( path ) ? ; ( Some ( user_dict . dict ) , Some ( user_dict . words_idx_data ) , Some ( user_dict . words_data ) , ) } else if let Some ( path ) = config . user_dict_bin_path { let user_dict_bin_data = read_file ( path ) ? ; let user_dict = UserDictionary :: load ( & user_dict_bin_data ) ? ; ( Some ( user_dict . dict ) , Some ( user_dict . words_idx_data ) , Some ( user_dict . words_data ) , ) } else { ( None , None , None ) } ; let tokenizer = Tokenizer { dict , cost_matrix , lattice : Lattice :: default ( ) , char_definitions , unknown_dictionary , words_idx_data , words_data , user_dict , user_dict_words_idx_data , user_dict_words_data , mode : config . mode , } ; Ok ( tokenizer ) }
Creates a new instance with the config
Creates a new instance with the config
config
TokenizerConfig
config
TokenizerConfig