class_docstr: This :class:`TokenIndexer` represents tokens as lists of character indices.
  
  Parameters
  ----------
  namespace : ``str``, optional (default=``token_characters``)
      We will use this namespace in the :class:`Vocabulary` to map the characters in each token
      to indices.
  character_tokenizer : ``CharacterTokenizer``, optional (default=``CharacterTokenizer()``)
      We use a :class:`CharacterTokenizer` to handle splitting tokens into characters, as it has
      options for byte encoding and other things.  The default here is to instantiate a
      ``CharacterTokenizer`` with its default parameters, which uses unicode characters and
      retains casing.
class_docstr_tok: ["This",":","class",":","`","TokenIndexer","`","represents","tokens","as","lists","of","character","indices",".","Parameters","--","--","--","--","--","namespace",":","``","str","``",",","optional","(","default=","``","token_characters","``",")","We","will","use","this","namespace","in","the",":","class",":","`","Vocabulary","`","to","map","the","characters","in","each","token","to","indices",".","character_tokenizer",":","``","CharacterTokenizer","``",",","optional","(","default=","``","CharacterTokenizer","(",")","``",")","We","use","a",":","class",":","`","CharacterTokenizer","`","to","handle","splitting","tokens","into","characters",",","as","it","has","options","for","byte","encoding","and","other","things",".","The","default","here","is","to","instantiate","a","``","CharacterTokenizer","``","with","its","default","parameters",",","which","uses","unicode","characters","and","retains","casing","."]
code: def pad_token_sequence(
      self, tokens, desired_num_tokens, padding_lengths
  ):
      # Pad the tokens.
      # tokens has only one key...
      key = list(tokens.keys())[0]
  
      padded_tokens = pad_sequence_to_length(
          tokens[key],
          desired_num_tokens[key],
          default_value=self.get_padding_token,
      )
  
      # Pad the characters within the tokens.
      desired_token_length = padding_lengths["num_token_characters"]
      longest_token = max(tokens[key], key=len, default=[])
      padding_value = 0
      if desired_token_length > len(longest_token):
          # Since we want to pad to greater than the longest token, we add a
          # "dummy token" so we can take advantage of the fast implementation of itertools.zip_longest.
          padded_tokens.append([padding_value] * desired_token_length)
      # pad the list of lists to the longest sublist, appending 0's
      padded_tokens = list(
          izip(
              *itertools.zip_longest(
                  *padded_tokens, fillvalue=padding_value
              )
          )
      )
      if desired_token_length > len(longest_token):
          # Removes the "dummy token".
          padded_tokens.pop()
      # Truncates all the tokens to the desired length, and return the result.
      return {
          key: [
              list(token[:desired_token_length])
              for token in padded_tokens
          ]
      }
  
file_path: magnitude/pymagnitude/third_party/allennlp/data/token_indexers/token_characters_indexer.py
imports: from __future__ import absolute_import
  import itertools
  from allennlp.common.checks import ConfigurationError
  from allennlp.common.util import pad_sequence_to_length
  from allennlp.data.tokenizers.token import Token
  from allennlp.data.token_indexers.token_indexer import TokenIndexer
  from allennlp.data.vocabulary import Vocabulary
  from allennlp.data.tokenizers.character_tokenizer import CharacterTokenizer
label_desc: Use image node_id 6 for calling the TokenCharactersIndexer obj's underlying member method code with example usage: obj.pad_token_sequence(tokens, desired_num_tokens, padding_lengths) and returns: dict