class_docstr: A regular expression-based tokenizer at word boundary.
  This tokenizer default to support MegaMolBART.
  <https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara/models/megamolbart>
docstr: Args:
      regex: regular expression that defined tokenization rules
      mask_token: mask token
      bos_token: the beginning of sequence token
      eos_token: the end of sequence token. Usually equal to sep_token
      pad_token: token to use for padding
      sep_token: token used for separating sequences
      cls_token: class token. Usually equal to bos_token
      unk_token: token to use for unknown tokens
docstr_tok: ["Args",":","regex",":","regular","expression","that","defined","tokenization","rules","mask_token",":","mask","token","bos_token",":","the","beginning","of","sequence","token","eos_token",":","the","end","of","sequence","token",".","Usually","equal","to","sep_token","pad_token",":","token","to","use","for","padding","sep_token",":","token","used","for","separating","sequences","cls_token",":","class","token",".","Usually","equal","to","bos_token","unk_token",":","token","to","use","for","unknown","tokens"]
code: def __init__(
      self,
      regex: Optional[str] = "",
      mask_token: Optional[str] = DEFAULT_MASK_TOKEN,
      bos_token: Optional[str] = DEFAULT_BOS_TOKEN,
      eos_token: Optional[str] = DEFAULT_EOS_TOKEN,
      pad_token: Optional[str] = DEFAULT_PAD_TOKEN,
      sep_token: Optional[str] = DEFAULT_SEP_TOKEN,
      unk_token: Optional[str] = DEFAULT_UNK_TOKEN,
  ):
      """
      Args:
          regex: regular expression that defined tokenization rules
          mask_token: mask token
          bos_token: the beginning of sequence token
          eos_token: the end of sequence token. Usually equal to sep_token
          pad_token: token to use for padding
          sep_token: token used for separating sequences
          cls_token: class token. Usually equal to bos_token
          unk_token: token to use for unknown tokens
      """
      self.regex = regex
      self.mask_token = mask_token
      self.bos_token = bos_token
      self.eos_token = eos_token
      self.pad_token = pad_token
      self.sep_token = sep_token
      self.unk_token = unk_token
  
      # holds names of .model/.vocab files
      self.regex_file = None
      self.vocab_file = None
  
      # initialize with default vocab
      self.vocab = {
          self.pad_token: 0,  # pad_token
          self.unk_token: 1,  # unk_token
          self.bos_token: 2,  # begin_token
          self.eos_token: 3,  # end_token
          self.mask_token: 4,  # mask_token
          self.sep_token: 5,  # sep_token
      }
      self._update_cache()
  
      # Computed attributes
      self._compile_regex()
  
imports: import os
  import re
  from typing import Optional
  import pandas
  from nemo.collections.common.tokenizers.char_tokenizer import TokenizerSpec
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new RegExTokenizer object from inherited base classes: TokenizerSpec with example: obj = RegExTokenizer(regex, mask_token, bos_token, eos_token, pad_token, sep_token, unk_token)