code: def _apply_tokenizer_to_states(
      tokenizer: Tokenizer, states: List[State]
  ) -> None:
      """Split each user text into tokens and concatenate them again.
  
      Args:
          tokenizer: A tokenizer to tokenize the user messages.
          states: The states to be tokenized.
      """
      for state in states:
          if USER in state and TEXT in state[USER]:
              state[USER][TEXT] = " ".join(
                  token.text
                  for token in tokenizer.tokenize(
                      Message({TEXT: state[USER][TEXT]}), TEXT
                  )
              )
  
imports: from collections import defaultdict
  import logging
  import json
  from typing import DefaultDict, Dict, Generator, List, NamedTuple, Optional, Text, Tuple
  from rasa.core.featurizers.tracker_featurizers import MaxHistoryTrackerFeaturizer
  from rasa.shared.core.constants import ACTION_LISTEN_NAME, PREVIOUS_ACTION, ACTION_UNLIKELY_INTENT_NAME, USER
  from rasa.shared.core.domain import Domain, State
  from rasa.shared.core.events import ActionExecuted, Event
  from rasa.shared.core.generator import TrackerWithCachedStates
  from rasa.nlu.tokenizers.tokenizer import Tokenizer
  from rasa.shared.nlu.constants import TEXT
  from rasa.shared.nlu.training_data.message import Message
label_desc: Use image node_id 6 for calling a global function with example usage: _apply_tokenizer_to_states(tokenizer, states) without return types