code: def tokenize(self, text):
      if self._lowercase_characters:
          text = text.lower()
      if self._byte_encoding is not None:
          # We add 1 here so that we can still use 0 for masking, no matter what bytes we get out
          # of this.
          tokens = [
              Token(text_id=c + 1)
              for c in text.encode(self._byte_encoding)
          ]
      else:
          tokens = [Token(t) for t in list(text)]
      for start_token in self._start_tokens:
          if isinstance(start_token, int):
              token = Token(text_id=start_token, idx=0)
          else:
              token = Token(text=start_token, idx=0)
          tokens.insert(0, token)
      for end_token in self._end_tokens:
          if isinstance(end_token, int):
              token = Token(text_id=end_token, idx=0)
          else:
              token = Token(text=end_token, idx=0)
          tokens.append(token)
      return tokens
  
file_path: magnitude/pymagnitude/third_party/allennlp/data/tokenizers/character_tokenizer.py
imports: from __future__ import absolute_import
  from allennlp.data.tokenizers.token import Token
  from allennlp.data.tokenizers.tokenizer import Tokenizer
label_desc: Use image node_id 3 for calling the CharacterTokenizer obj's underlying member method code with example usage: obj.tokenize(text) and returns: tokens