class_docstr: Base class for tokenizers.
docstr: Apply the token pattern to the given tokens.
  
  Args:
      tokens: list of tokens to split
  
  Returns:
      List of tokens.
docstr_tok: ["Apply","the","token","pattern","to","the","given","tokens",".","Args",":","tokens",":","list","of","tokens","to","split","Returns",":","List","of","tokens","."]
code: def _apply_token_pattern(self, tokens: List[Token]) -> List[Token]:
      """Apply the token pattern to the given tokens.
  
      Args:
          tokens: list of tokens to split
  
      Returns:
          List of tokens.
      """
      if not self.token_pattern_regex:
          return tokens
  
      final_tokens = []
      for token in tokens:
          new_tokens = self.token_pattern_regex.findall(token.text)
          new_tokens = [t for t in new_tokens if t]
  
          if not new_tokens:
              final_tokens.append(token)
  
          running_offset = 0
          for new_token in new_tokens:
              word_offset = token.text.index(new_token, running_offset)
              word_len = len(new_token)
              running_offset = word_offset + word_len
              final_tokens.append(
                  Token(
                      new_token,
                      token.start + word_offset,
                      data=token.data,
                      lemma=token.lemma,
                  )
              )
  
      return final_tokens
  
imports: import abc
  import logging
  import re
  from typing import Text, List, Dict, Any, Optional
  from rasa.engine.graph import ExecutionContext, GraphComponent
  from rasa.engine.storage.resource import Resource
  from rasa.engine.storage.storage import ModelStorage
  from rasa.shared.nlu.training_data.training_data import TrainingData
  from rasa.shared.nlu.training_data.message import Message
  from rasa.nlu.constants import TOKENS_NAMES, MESSAGE_ATTRIBUTES
  from rasa.shared.nlu.constants import INTENT, INTENT_RESPONSE_KEY, RESPONSE_IDENTIFIER_DELIMITER, ACTION_NAME
  import rasa.shared.utils.io
label_desc: Use image node_id 8 for calling the Tokenizer obj's underlying member method code with example usage: obj._apply_token_pattern(tokens) and returns: final_tokens, tokens