code: def deduplicate_dataset(
      dataset: Type[Dataset], jaccard_threshold: float = 0.85
  ) -> Tuple[Type[Dataset], List[List[Dict]]]:
      """Deduplicate the dataset using minhash and jaccard similarity.
      This function first generate duplicate clusters, then each cluster
      is reduced to the extremes that are similar to the other elements in the cluster.
      Codes are called similar if their Jaccard similarity is greater than jaccard_threshold (0.85 default).
  
      Args:
          dataset (Type[Dataset]):
              The dataset to deduplicate.
          jaccard_threshold (float, default=0.85):
              jaccard threshold to determine if two codes are similar
  
      Returns:
          ds_dedup (Type[Dataset]):
              The deduplicated dataset.
          duplicate_clusters (List[List[Dict]]):
              The list of duplicate clusters.
              Each cluster is a list of dicts with the following keys:
              - base_index : int
                  The index of the code in the original dataset.
              - repo_name : str
              - path : str
              - copies : int
                  The number of copies of the code in the cluster. (find_cluster_extremes)
              - is_extreme : bool
                  Whether the code is an extreme in the cluster.
              All the codes in the cluster are removed from the dataset except the extremes.
  
      Example:
          >>> from datasets import load_dataset
          >>> from minhash_deduplication import deduplicate_dataset
          >>> ds = load_dataset("lvwerra/codeparrot-clean", split="train")
          >>> ds_dedup, duplicate_clusters = deduplicate_dataset(ds, jaccard_threshold=0.85)
      """
      duplicate_clusters = make_duplicate_clusters(
          dataset, jaccard_threshold
      )
      duplicate_indices = {
          x["base_index"]
          for cluster in duplicate_clusters
          for x in cluster
      }
      extreme_dict = {}
      extremes_clusters = find_extremes(
          duplicate_clusters, dataset, jaccard_threshold
      )
      for extremes in extremes_clusters:
          for element in extremes:
              extreme_dict[element["base_index"]] = element
      remove_indices = duplicate_indices - set(extreme_dict.keys())
      ds_filter = dataset.filter(
          lambda x, idx: idx not in remove_indices, with_indices=True
      )
  
      # update duplicate_clusters
      for cluster in duplicate_clusters:
          for element in cluster:
              element["is_extreme"] = (
                  element["base_index"] in extreme_dict
              )
              if element["is_extreme"]:
                  element["copies"] = extreme_dict[
                      element["base_index"]
                  ]["copies"]
  
      print(f"Original dataset size: {len(dataset)}")
      print(f"Number of duplicate clusters: {len(duplicate_clusters)}")
      print(f"Files in duplicate cluster: {len(duplicate_indices)}")
      print(f"Unique files in duplicate cluster: {len(extreme_dict)}")
      print(f"Filtered dataset size: {len(ds_filter)}")
  
      return ds_filter, duplicate_clusters
  
imports: import json
  import multiprocessing
  import re
  from collections import defaultdict
  from functools import partial
  from typing import Dict, List, Optional, Set, Tuple, Type
  from datasets import Dataset
  from datasketch import MinHash, MinHashLSH
  from dpu_utils.utils.iterators import ThreadedIterator
  from tqdm import tqdm
label_desc: Use image node_id 9 for calling a global function with example usage: deduplicate_dataset(dataset, jaccard_threshold) and returns: ds_filter,  duplicate_clusters