class_docstr: Base class for tokenizers.
docstr: Construct a new tokenizer.
docstr_tok: ["Construct","a","new","tokenizer","."]
code: def __init__(self, config: Dict[Text, Any]) -> None:
      """Construct a new tokenizer."""
      self._config = config
      # flag to check whether to split intents
      self.intent_tokenization_flag = config["intent_tokenization_flag"]
      # split symbol for intents
      self.intent_split_symbol = config["intent_split_symbol"]
      # token pattern to further split tokens
      token_pattern = config.get("token_pattern")
      self.token_pattern_regex = None
      if token_pattern:
          self.token_pattern_regex = re.compile(token_pattern)
      # split intent to prefix and suffix greedily, None means don't split
      self.prefix_separator_symbol = config.get(
          "prefix_separator_symbol"
      )
  
imports: import abc
  import logging
  import re
  from typing import Text, List, Dict, Any, Optional
  from rasa.engine.graph import ExecutionContext, GraphComponent
  from rasa.engine.storage.resource import Resource
  from rasa.engine.storage.storage import ModelStorage
  from rasa.shared.nlu.training_data.training_data import TrainingData
  from rasa.shared.nlu.training_data.message import Message
  from rasa.nlu.constants import TOKENS_NAMES, MESSAGE_ATTRIBUTES
  from rasa.shared.nlu.constants import INTENT, INTENT_RESPONSE_KEY, RESPONSE_IDENTIFIER_DELIMITER, ACTION_NAME
  import rasa.shared.utils.io
label_desc: Use image node_id 1 to create a new Tokenizer object from inherited base classes: GraphComponent, abc with example: obj = Tokenizer(config)