pub fn new(dict_path: &str) -> Self {
        NewmmTokenizer {
            dict: Box::from(
                create_dict_trie(DictSource::FilePath(PathBuf::from(dict_path))).unwrap(),
            ),
        }
    }
/// Create a new tokenizer using a dictionary from a text file
Create a new tokenizer using a dictionary from a text file
Create a new tokenizer using a dictionary from a text file
pub fn new(dict_path: &str) -> Self {
        NewmmTokenizer {
            dict: Box::from(
                create_dict_trie(DictSource::FilePath(PathBuf::from(dict_path))).unwrap(),
            ),
        }
    }
pub fn new ( dict_path : & str ) -> Self { NewmmTokenizer { dict : Box :: from ( create_dict_trie ( DictSource :: FilePath ( PathBuf :: from ( dict_path ) ) ) . unwrap ( ) , ) , } }
Create a new tokenizer using a dictionary from a text file
Create a new tokenizer using a dictionary from a text file
dict_path
&str