class_docstr: Tokenizes at word boundary
docstr: Args:
      vocab_file: path to file with vocabulary which consists
          of characters separated by 
  
      mask_token: mask token 
      bos_token: the beginning of sequence token
      eos_token: the end of sequence token. Usually equal to sep_token
      pad_token: token to use for padding
      sep_token: token used for separating sequences
      cls_token: class token. Usually equal to bos_token
      unk_token: token to use for unknown tokens
docstr_tok: ["Args",":","vocab_file",":","path","to","file","with","vocabulary","which","consists","of","characters","separated","by","mask_token",":","mask","token","bos_token",":","the","beginning","of","sequence","token","eos_token",":","the","end","of","sequence","token",".","Usually","equal","to","sep_token","pad_token",":","token","to","use","for","padding","sep_token",":","token","used","for","separating","sequences","cls_token",":","class","token",".","Usually","equal","to","bos_token","unk_token",":","token","to","use","for","unknown","tokens"]
code: def __init__(
      self,
      vocab_file: str,
      mask_token: Optional[str] = None,
      bos_token: Optional[str] = None,
      eos_token: Optional[str] = None,
      pad_token: Optional[str] = None,
      sep_token: Optional[str] = None,
      cls_token: Optional[str] = None,
      unk_token: Optional[str] = None,
  ):
      """
      Args:
          vocab_file: path to file with vocabulary which consists
              of characters separated by \n
          mask_token: mask token
          bos_token: the beginning of sequence token
          eos_token: the end of sequence token. Usually equal to sep_token
          pad_token: token to use for padding
          sep_token: token used for separating sequences
          cls_token: class token. Usually equal to bos_token
          unk_token: token to use for unknown tokens
      """
  
      super().__init__(
          vocab_file=vocab_file,
          mask_token=mask_token,
          bos_token=bos_token,
          eos_token=eos_token,
          pad_token=pad_token,
          unk_token=unk_token,
          sep_token=sep_token,
          cls_token=cls_token,
      )
  
imports: from typing import Optional
  from nemo.collections.common.tokenizers.char_tokenizer import CharTokenizer
label_desc: Use image node_id 1 to create a new WordTokenizer object from inherited base classes: CharTokenizer with example: obj = WordTokenizer(vocab_file, mask_token, bos_token, eos_token, pad_token, sep_token, cls_token, unk_token)