class_docstr: This `TokenIndexer` assumes that Tokens already have their indexes in them (see `text_id` field).
  We still require `model_name` because we want to form allennlp vocabulary from pretrained one.
  This `Indexer` is only really appropriate to use if you've also used a
  corresponding :class:`PretrainedTransformerTokenizer` to tokenize your input.  Otherwise you'll
  have a mismatch between your tokens and your vocabulary, and you'll get a lot of UNK tokens.
  
  Registered as a `TokenIndexer` with name "pretrained_transformer".
  
  # Parameters
  
  model_name : `str`
      The name of the `transformers` model to use.
  namespace : `str`, optional (default=`tags`)
      We will add the tokens in the pytorch_transformer vocabulary to this vocabulary namespace.
      We use a somewhat confusing default value of `tags` so that we do not add padding or UNK
      tokens to this namespace, which would break on loading because we wouldn't find our default
      OOV token.
  max_length : `int`, optional (default = `None`)
      If not None, split the document into segments of this many tokens (including special tokens)
      before feeding into the embedder. The embedder embeds these segments independently and
      concatenate the results to get the original document representation. Should be set to
      the same value as the `max_length` option on the `PretrainedTransformerEmbedder`.
  tokenizer_kwargs : `Dict[str, Any]`, optional (default = `None`)
      Dictionary with
      [additional arguments](https://github.com/huggingface/transformers/blob/155c782a2ccd103cf63ad48a2becd7c76a7d2115/transformers/tokenization_utils.py#L691)
      for `AutoTokenizer.from_pretrained`.
class_docstr_tok: ["This","`","TokenIndexer","`","assumes","that","Tokens","already","have","their","indexes","in","them","(","see","`","text_id","`","field",")",".","We","still","require","`","model_name","`","because","we","want","to","form","allennlp","vocabulary","from","pretrained","one",".","This","`","Indexer","`","is","only","really","appropriate","to","use","if","you","'ve","also","used","a","corresponding",":","class",":","`","PretrainedTransformerTokenizer","`","to","tokenize","your","input",".","Otherwise","you'll","have","a","mismatch","between","your","tokens","and","your","vocabulary",",","and","you","'ll","get","a","lot","of","UNK","tokens",".","Registered","as","a","`","TokenIndexer","`","with","name","``","pretrained_transformer","''",".","#","Parameters","model_name",":","`","str","`","The","name","of","the","`","transformers","`","model","to","use",".","namespace",":","`","str","`",",","optional","(","default=","`","tags","`",")","We","will","add","the","tokens","in","the","pytorch_transformer","vocabulary","to","this","vocabulary","namespace",".","We","use","a","somewhat","confusing","default","value","of","`","tags","`","so","that","we","do","not","add","padding","or","UNK","tokens","to","this","namespace",",","which","would","break","on","loading","because","we","would","n't","find","our","default","OOV","token",".","max_length",":","`","int","`",",","optional","(","default","=","`","None","`",")","If","not","None",",","split","the","document","into","segments","of","this","many","tokens","(","including","special","tokens",")","before","feeding","into","the","embedder",".","The","embedder","embeds","these","segments","independently","and","concatenate","the","results","to","get","the","original","document","representation",".","Should","be","set","to","the","same","value","as","the","`","max_length","`","option","on","the","`","PretrainedTransformerEmbedder","`",".","tokenizer_kwargs",":","`","Dict","[","str",",","Any","]","`",",","optional","(","default","=","`","None","`",")","Dictionary","with","[","additional","arguments","]","(","https",":","\/\/github.com\/huggingface\/transformers\/blob\/155c782a2ccd103cf63ad48a2becd7c76a7d2115\/transformers\/tokenization_utils.py","#","L691",")","for","`","AutoTokenizer.from_pretrained","`","."]
docstr: Takes an IndexedTokenList about to be returned by `tokens_to_indices()` and adds any
  necessary postprocessing, e.g. long sequence splitting.
  
  The input should have a `"token_ids"` key corresponding to the token indices. They should
  have special tokens already inserted.
docstr_tok: ["Takes","an","IndexedTokenList","about","to","be","returned","by","`","tokens_to_indices","(",")","`","and","adds","any","necessary","postprocessing",",","e.g",".","long","sequence","splitting",".","The","input","should","have","a","`","``","token_ids","''","`","key","corresponding","to","the","token","indices",".","They","should","have","special","tokens","already","inserted","."]
code: def _postprocess_output(
      self, output: IndexedTokenList
  ) -> IndexedTokenList:
      """
      Takes an IndexedTokenList about to be returned by `tokens_to_indices()` and adds any
      necessary postprocessing, e.g. long sequence splitting.
  
      The input should have a `"token_ids"` key corresponding to the token indices. They should
      have special tokens already inserted.
      """
      if self._max_length is not None:
          # We prepare long indices by converting them to (assuming max_length == 5)
          # [CLS] A B C [SEP] [CLS] D E F [SEP] ...
          # Embedder is responsible for folding this 1-d sequence to 2-d and feed to the
          # transformer model.
          # TODO(zhaofengw): we aren't respecting word boundaries when segmenting wordpieces.
  
          indices = output["token_ids"]
          type_ids = output.get("type_ids", [0] * len(indices))
  
          # Strips original special tokens
          indices = indices[
              self._num_added_start_tokens : len(indices)
              - self._num_added_end_tokens
          ]
          type_ids = type_ids[
              self._num_added_start_tokens : len(type_ids)
              - self._num_added_end_tokens
          ]
  
          # Folds indices
          folded_indices = [
              indices[i : i + self._effective_max_length]
              for i in range(
                  0, len(indices), self._effective_max_length
              )
          ]
          folded_type_ids = [
              type_ids[i : i + self._effective_max_length]
              for i in range(
                  0, len(type_ids), self._effective_max_length
              )
          ]
  
          # Adds special tokens to each segment
          folded_indices = [
              self._tokenizer.build_inputs_with_special_tokens(segment)
              for segment in folded_indices
          ]
          single_sequence_start_type_ids = [
              t.type_id
              for t in self._allennlp_tokenizer.single_sequence_start_tokens
          ]
          single_sequence_end_type_ids = [
              t.type_id
              for t in self._allennlp_tokenizer.single_sequence_end_tokens
          ]
          folded_type_ids = [
              single_sequence_start_type_ids
              + segment
              + single_sequence_end_type_ids
              for segment in folded_type_ids
          ]
          assert all(
              len(segment_indices) == len(segment_type_ids)
              for segment_indices, segment_type_ids in zip(
                  folded_indices, folded_type_ids
              )
          )
  
          # Flattens
          indices = [i for segment in folded_indices for i in segment]
          type_ids = [i for segment in folded_type_ids for i in segment]
  
          output["token_ids"] = indices
          output["type_ids"] = type_ids
          output["segment_concat_mask"] = [True] * len(indices)
  
      return output
  
file_path: allennlp/allennlp/data/token_indexers/pretrained_transformer_indexer.py
imports: from typing import Dict, List, Optional, Tuple, Any
  import logging
  import torch
  from allennlp.common.util import pad_sequence_to_length
  from allennlp.data.vocabulary import Vocabulary
  from allennlp.data.tokenizers import Token, PretrainedTransformerTokenizer
  from allennlp.data.token_indexers.token_indexer import TokenIndexer, IndexedTokenList
label_desc: Use image node_id 7 for calling the PretrainedTransformerIndexer obj's underlying member method code with example usage: obj._postprocess_output(output) and returns: output