class_docstr: This :class:`TokenIndexer` represents tokens as lists of character indices.
  
  Parameters
  ----------
  namespace : ``str``, optional (default=``token_characters``)
      We will use this namespace in the :class:`Vocabulary` to map the characters in each token
      to indices.
  character_tokenizer : ``CharacterTokenizer``, optional (default=``CharacterTokenizer()``)
      We use a :class:`CharacterTokenizer` to handle splitting tokens into characters, as it has
      options for byte encoding and other things.  The default here is to instantiate a
      ``CharacterTokenizer`` with its default parameters, which uses unicode characters and
      retains casing.
class_docstr_tok: ["This",":","class",":","`","TokenIndexer","`","represents","tokens","as","lists","of","character","indices",".","Parameters","--","--","--","--","--","namespace",":","``","str","``",",","optional","(","default=","``","token_characters","``",")","We","will","use","this","namespace","in","the",":","class",":","`","Vocabulary","`","to","map","the","characters","in","each","token","to","indices",".","character_tokenizer",":","``","CharacterTokenizer","``",",","optional","(","default=","``","CharacterTokenizer","(",")","``",")","We","use","a",":","class",":","`","CharacterTokenizer","`","to","handle","splitting","tokens","into","characters",",","as","it","has","options","for","byte","encoding","and","other","things",".","The","default","here","is","to","instantiate","a","``","CharacterTokenizer","``","with","its","default","parameters",",","which","uses","unicode","characters","and","retains","casing","."]
code: def tokens_to_indices(self, tokens, vocabulary, index_name):
      indices = []
      for token in tokens:
          token_indices = []
          if token.text is None:
              raise ConfigurationError(
                  "TokenCharactersIndexer needs a tokenizer that retains text"
              )
          for character in self._character_tokenizer.tokenize(
              token.text
          ):
              if getattr(character, "text_id", None) is not None:
                  # `text_id` being set on the token means that we aren't using the vocab, we just
                  # use this id instead.
                  index = character.text_id
              else:
                  index = vocabulary.get_token_index(
                      character.text, self._namespace
                  )
              token_indices.append(index)
          indices.append(token_indices)
      return {index_name: indices}
  
file_path: magnitude/pymagnitude/third_party/allennlp/data/token_indexers/token_characters_indexer.py
imports: from __future__ import absolute_import
  import itertools
  from allennlp.common.checks import ConfigurationError
  from allennlp.common.util import pad_sequence_to_length
  from allennlp.data.tokenizers.token import Token
  from allennlp.data.token_indexers.token_indexer import TokenIndexer
  from allennlp.data.vocabulary import Vocabulary
  from allennlp.data.tokenizers.character_tokenizer import CharacterTokenizer
label_desc: Use image node_id 3 for calling the TokenCharactersIndexer obj's underlying member method code with example usage: obj.tokens_to_indices(tokens, vocabulary, index_name) and returns: dict