code: def _deduplicate(data):
      """Remove duplicated records."""
      cnt = collections.Counter(row["id"] for row in data)
      nonuniq_ids = set(id for id, count in cnt.items() if count > 1)
      nonuniq_data = [row for row in data if row["id"] in nonuniq_ids]
  
      unique_data = [
          row for row in data if row["id"] not in nonuniq_ids
      ]
      # Make sure same id records are next to each other for itertools.groupby
      nonuniq_data = sorted(nonuniq_data, key=lambda row: row["id"])
      for _, same_id_data in itertools.groupby(
          nonuniq_data, lambda row: row["id"]
      ):
          same_id_data = list(same_id_data)
          if all(same_id_data[0] == x for x in same_id_data):
              unique_data.append(same_id_data[0])
          else:
              non_deleted_same_id_data = [
                  row
                  for row in same_id_data
                  if row["author"] != "[deleted]"
              ]
              if len(non_deleted_same_id_data) != 1:
                  raise ValueError(
                      "Found several message with id {} in the original data".format(
                          non_deleted_same_id_data[0]["id"]
                      )
                  )
              unique_data.append(non_deleted_same_id_data[0])
  
      return sorted(
          unique_data,
          key=lambda row: (row["link_id"], row["created_utc"]),
      )
  
file_path: datasets/tensorflow_datasets/datasets/reddit_disentanglement/reddit_disentanglement_dataset_builder.py
imports: import collections
  import csv
  import itertools
  import os
  from etils import epath
  import tensorflow_datasets.public_api
label_desc: Use image node_id 2 for calling a global function with example usage: _deduplicate(data) and returns: sorted