code: def main(anno_path, out_anno=None, orig_anno_file=None):
      dataset = GluonCVMotionDataset(anno_path)
      if out_anno is None:
          anno_path = Path(anno_path)
          out_anno = anno_path.with_name(
              "{}_dedupe{}".format(anno_path.stem, anno_path.suffix)
          )
      out_dataset = GluonCVMotionDataset(
          out_anno, dataset.root_path, load_anno=False
      )
      out_dataset.metadata = dataset.metadata
      out_dataset.description += "; deduped"
      samples = dataset.samples
  
      if orig_anno_file:
          orig_dataset = GluonCVMotionDataset(
              orig_anno_file, root_path=dataset.root_path
          )
          orig_id_map = get_chunked_id_map(
              dataset,
              orig_dataset,
              chunk_subdirs=["vid_chunks_2m_1p"],
              exclude_unchunked=True,
          )
      else:
          orig_dataset = None
          orig_id_map = {}
  
      image_dir = Path(dataset.cache_root_path) / "thumbnails"
  
      thumb_paths = []
      thumb_to_sample_id = {}
      sample_id_to_thumb = {}
      for sample_id, sample in tqdm.tqdm(samples, mininterval=1):
          thumb_path = get_vis_thumb_location(sample)
          if not Path(thumb_path).exists():
              _log.warning("Missing thumbnail: {}".format(thumb_path))
              continue
          thumb_paths.append(thumb_path)
          thumb_rel_path = str(Path(thumb_path).relative_to(image_dir))
          thumb_to_sample_id[thumb_rel_path] = sample_id
          sample_id_to_thumb[sample_id] = thumb_rel_path
  
      def im_filter(path):
          if path.suffix.lower() not in [".jpeg", ".jpg", ".png"]:
              return False
          return True
  
      duplicates, _, _ = get_duplicates(
          thumb_paths, method="cnn", min_similarity_threshold=0.83
      )
  
      dup_set = set_from_dict(duplicates)
      addn_args = [
          image_dir,
          dataset,
          thumb_to_sample_id,
          sample_id_to_thumb,
      ]
      all_dups, dups_map, selected_dups = get_unique_dups(
          duplicates, *addn_args
      )
  
      print(len(selected_dups))
      dup_sample_ids = [
          thumb_to_sample_id[thumb_path]
          for thumb_path in selected_dups
          if "videos_MEVA" not in thumb_path
          and "Offline_CCTV_Footage" not in thumb_path
      ]
      print(len(dup_sample_ids))
      for sample_id, sample in tqdm.tqdm(samples):
          if sample_id not in dup_sample_ids:
              out_dataset.add_sample(sample)
  
      out_dataset.dump()
  
file_path: gluon-cv/gluoncv/torch/data/gluoncv_motion_dataset/ingestion/duplicate_remover.py
imports: import logging
  from pathlib import Path, PosixPath
  from collections import defaultdict
  import tqdm
  from ..dataset import GluonCVMotionDataset, get_vis_thumb_location
  from ..utils.ingestion_utils import get_chunked_id_map
label_desc: Use image node_id 10 for calling a global function with example usage: main(anno_path, out_anno, orig_anno_file) and returns: True, False