code: def clean_en_xx_pairs(en_xx_pairs):
      """Generates a cleaned-up stream of (English, other) translation pairs.
  
      Cleaning includes both filtering and simplistic sentence splitting, with
      minimal assumptions on the non-English pair member: (1) All filtering is
      done based on the English member of the pair, and (2) sentence splitting
      assumes only that sentences can end with one of '.!?' and begin with an
      ASCII uppercase letter. Input pairs that would get split into different
      numbers of sentences (e.g., three English sentences vs. two German ones) are
      discarded.
  
      Args:
        en_xx_pairs: A stream (iterable) of Unicode string pairs. Each item in the
            stream should be a (sentence_en, sentence_xx) pair.
      Yields:
        Cleaned-up (sentence_en, sentence_xx) pairs.
      """
      for s1, s2 in en_xx_pairs:
          if _regex_filter(s1):
              continue
          s1_list, s2_list = _split_sentences(s1, s2)
          if len(s1_list) != len(s2_list):
              continue  # discard this pair
          elif len(s1_list) == 1:
              yield s1, s2
          else:
              for s1_subsentence, s2_subsentence in itertools.izip(
                  s1_list, s2_list
              ):
                  if _regex_filter(s1_subsentence):
                      continue
                  yield s1_subsentence, s2_subsentence
  
file_path: tensor2tensor/tensor2tensor/data_generators/cleaner_en_xx.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from __future__ import unicode_literals
  import itertools
  import re
  from tensor2tensor.data_generators import text_encoder
  import tensorflow.compat.v1
label_desc: Use image node_id 3 for calling a global function with example usage: clean_en_xx_pairs(en_xx_pairs) without return types