code: def remove_duplicate_text(
      pages, min_num_sentences=_MIN_NUM_SENTENCES, line_delimiter="\n"
  ):
      """Utility to remove duplicate lines across text documents."""
      beam = tfds.core.lazy_imports.apache_beam
  
      # Select a single URL for each line in the input pages.
      # Hash before comparison to avoid biasing by domain.
      # line, [url]
      line_to_selected_url = (
          pages
          | beam.FlatMap(
              _emit_url_to_lines, line_delimiter=line_delimiter
          )
          | beam.combiners.Top.PerKey(1, key=_hash_text, reverse=True)
      )
      # url, line
      lines_to_keep = line_to_selected_url | beam.Map(
          lambda x: (x[1][0], x[0])
      )
  
      # Output: url, text
      final_docs = (
          {
              "pages": pages | beam.Map(lambda p: (p.url, p)),
              "lines": lines_to_keep,
          }
          | "group_features_and_lines_by_url" >> beam.CoGroupByKey()
          | beam.FlatMap(
              _remove_lines_from_text,
              counter_inc_fn=get_counter_inc_fn("dedupe-lines"),
              min_num_sentences=min_num_sentences,
              line_delimiter=line_delimiter,
          )
      )
  
      return final_docs
  
imports: import dataclasses
  import functools
  import gzip
  import hashlib
  import heapq
  import io
  import re
  import threading
  from typing import Collection, Iterable, Mapping, Optional, Sequence, Tuple
  from absl import logging
  from tensorflow_datasets.core.utils.lazy_imports_utils import tensorflow
  import tensorflow_datasets.public_api
label_desc: Use image node_id 12 for calling a global function with example usage: remove_duplicate_text(pages, min_num_sentences, line_delimiter) and returns: final_docs