code: def make_duplicate_clusters(
      dataset_iterator: Type[Dataset], jaccard_threshold: float
  ):
      """Find duplicate clusters in the dataset in two steps:
      1. Compute MinHash for each code snippet. MinHash is a tool for fast jaccard similarity estimation.
      This step is computed using an asynchronous multiprocessing pool, minhash_iter
      2. Find duplicate clusters. The computed MinHash is added sequentially to the DuplicationIndex.
      This step cannot be parallelized. So using asynchronous thread in the previous step helps to speed up the process.
      """
      di = DuplicationIndex(
          duplication_jaccard_threshold=jaccard_threshold
      )
  
      for filename, min_hash in tqdm(
          ThreadedIterator(
              minhash_iter(enumerate(dataset_iterator)),
              max_queue_size=100,
          )
      ):
          di.add(filename, min_hash)
  
      # Returns a List[Cluster] where Cluster is List[str] with the filenames.
      return di.get_duplicate_clusters()
  
imports: import json
  import multiprocessing
  import re
  from collections import defaultdict
  from functools import partial
  from typing import Dict, List, Optional, Set, Tuple, Type
  from datasets import Dataset
  from tqdm import tqdm
  from datasketch import MinHash, MinHashLSH
  from dpu_utils.utils.iterators import ThreadedIterator
label_desc: Use image node_id 5 for calling a global function with example usage: make_duplicate_clusters(dataset_iterator, jaccard_threshold) and returns: di