code: def token_generator(
      tree_path, source_token_vocab, target_token_vocab, eos=None
  ):
      """Generator for parsing as a sequence-to-sequence task that uses tokens.
  
      This generator assumes the files at source_path and target_path have
      the same number of lines and yields dictionaries of "inputs" and "targets"
      where inputs and targets are token ids from source and target lines
      converted to integers using the token_map.
  
      Args:
        tree_path: path to the file with WSJ format trees, one per line.
        source_token_vocab: GenericVocabulary object for source vocabulary.
        target_token_vocab: GenericVocabulary object for target vocabulary.
        eos: integer to append at the end of each sequence (default: None).
  
      Yields:
        A dictionary {"inputs": source-line, "targets": target-line} where
        the lines are integer lists converted from tokens in the file lines.
      """
      eos_list = [] if eos is None else [eos]
      with tf.gfile.GFile(tree_path, mode="r") as tree_file:
          tree_line = tree_file.readline()
          while tree_line:
              source, target = words_and_tags_from_wsj_tree(tree_line)
              source_ints = (
                  source_token_vocab.encode(source.strip()) + eos_list
              )
              target_ints = (
                  target_token_vocab.encode(target.strip()) + eos_list
              )
              yield {"inputs": source_ints, "targets": target_ints}
              tree_line = tree_file.readline()
  
file_path: tensor2tensor/tensor2tensor/data_generators/wsj_parsing.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import os
  from absl import flags
  from tensor2tensor.data_generators import problem
  from tensor2tensor.data_generators import text_problems
  from tensor2tensor.utils import registry
  import tensorflow.compat.v1
label_desc: Use image node_id 2 for calling a global function with example usage: token_generator(tree_path, source_token_vocab, target_token_vocab, eos) without return types