code: def get_unique_dups(
      duplicates,
      image_dir,
      dataset,
      thumb_to_sample_id,
      sample_id_to_thumb,
  ):
      selected_dups = set()
      all_dups = set()
      dups_map = defaultdict(list)
      dups_keep_lookup = {}
  
      for path, dups in duplicates.items():
          rel_path = str(Path(path).relative_to(image_dir))
          if rel_path not in selected_dups:
              for dup in dups:
                  dup_path, score = dup
                  rel_dup_path = str(
                      Path(dup_path).relative_to(image_dir)
                  )
                  all_dups.add(rel_dup_path)
                  dups_map[rel_path].append((rel_dup_path, score))
              thumb_paths = [rel_path] + [
                  str(Path(dup[0]).relative_to(image_dir))
                  for dup in dups
                  if dup not in selected_dups
              ]
              samples = [
                  dataset[thumb_to_sample_id[p]] for p in thumb_paths
              ]
              keep, remove_dups = select_to_keep(samples)
              remove_dups_thumbs = [
                  sample_id_to_thumb[s.id] for s in remove_dups
              ]
              selected_dups.update(remove_dups_thumbs)
              for dup in remove_dups_thumbs:
                  dups_keep_lookup[dup] = sample_id_to_thumb[keep.id]
      return all_dups, dups_map, selected_dups
  
file_path: gluon-cv/gluoncv/torch/data/gluoncv_motion_dataset/ingestion/duplicate_remover.py
imports: import logging
  from pathlib import Path, PosixPath
  from collections import defaultdict
  import tqdm
  from ..dataset import GluonCVMotionDataset, get_vis_thumb_location
  from ..utils.ingestion_utils import get_chunked_id_map
label_desc: Use image node_id 6 for calling a global function with example usage: get_unique_dups(duplicates, image_dir, dataset, thumb_to_sample_id, sample_id_to_thumb) and returns: all_dups,  dups_map,  selected_dups