


import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__a	 =      logging.get_logger(__name__)


__a	 =      {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all BART models at https://huggingface.co/models?filter=bart
__a	 =      {
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
}

__a	 =      {
    """facebook/bart-base""": 1024,
    """facebook/bart-large""": 1024,
    """facebook/bart-large-mnli""": 1024,
    """facebook/bart-large-cnn""": 1024,
    """facebook/bart-large-xsum""": 1024,
    """yjernite/bart_eli5""": 1024,
}
@lru_cache()
def 	_UpperCamelCase					(    )    ->Dict:
   UpperCAmelCase      					=  (
       list(range(ord("""!""" )	,    ord("""~""" ) + 1 ) ) + list(range(ord("""¡""" )	,    ord("""¬""" ) + 1 ) ) + list(range(ord("""®""" )	,    ord("""ÿ""" ) + 1 ) )
   )
   UpperCAmelCase      					=  bs[:]
   UpperCAmelCase      					=  0
   for b in range(2**8 ):
      if b not in bs:
         bs.append(lowerCAmelCase_ )
         cs.append(2**8 + n )
         n += 1
   UpperCAmelCase      					=  [chr(lowerCAmelCase_ ) for n in cs]
   return dict(zip(lowerCAmelCase_	,    lowerCAmelCase_ ) )
def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->Dict:
   UpperCAmelCase      					=  set()
   UpperCAmelCase      					=  word[0]
   for char in word[1:]:
      pairs.add((prev_char, char) )
      UpperCAmelCase      					=  char
   return pairs







class 	__lowercase					(	__snake_case	):
      UpperCamelCase										=							VOCAB_FILES_NAMES
      UpperCamelCase										=							PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase										=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase										=							['''input_ids''', '''attention_mask''']





      def __init__(		self    :       Optional[int]       ,							__lowerCamelCase    :       Union[str, Any]       ,							__lowerCamelCase    :       Union[str, Any]       ,							__lowerCamelCase    :       Optional[Any]="replace"       ,							__lowerCamelCase    :       Optional[Any]="<s>"       ,							__lowerCamelCase    :       int="</s>"       ,							__lowerCamelCase    :       List[Any]="</s>"       ,							__lowerCamelCase    :       Optional[int]="<s>"       ,							__lowerCamelCase    :       Tuple="<unk>"       ,							__lowerCamelCase    :       Union[str, Any]="<pad>"       ,							__lowerCamelCase    :       Any="<mask>"       ,							__lowerCamelCase    :       List[Any]=False       ,							**__lowerCamelCase    :       Union[str, Any]       ,							)     -> Any:






         """simple docstring"""







         UpperCAmelCase      					=  AddedToken(__lowerCamelCase       ,							lstrip=__lowerCamelCase       ,							rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase       ,							__lowerCamelCase							) else bos_token
         UpperCAmelCase      					=  AddedToken(__lowerCamelCase       ,							lstrip=__lowerCamelCase       ,							rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase       ,							__lowerCamelCase							) else eos_token
         UpperCAmelCase      					=  AddedToken(__lowerCamelCase       ,							lstrip=__lowerCamelCase       ,							rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase       ,							__lowerCamelCase							) else sep_token
         UpperCAmelCase      					=  AddedToken(__lowerCamelCase       ,							lstrip=__lowerCamelCase       ,							rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase       ,							__lowerCamelCase							) else cls_token
         UpperCAmelCase      					=  AddedToken(__lowerCamelCase       ,							lstrip=__lowerCamelCase       ,							rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase       ,							__lowerCamelCase							) else unk_token
         UpperCAmelCase      					=  AddedToken(__lowerCamelCase       ,							lstrip=__lowerCamelCase       ,							rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase       ,							__lowerCamelCase							) else pad_token

         # Mask token behave like a normal word, i.e. include the space before it
         UpperCAmelCase      					=  AddedToken(__lowerCamelCase       ,							lstrip=__lowerCamelCase       ,							rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase       ,							__lowerCamelCase							) else mask_token

         super().__init__(
             errors=__lowerCamelCase       ,							bos_token=__lowerCamelCase       ,							eos_token=__lowerCamelCase       ,							unk_token=__lowerCamelCase       ,							sep_token=__lowerCamelCase       ,							cls_token=__lowerCamelCase       ,							pad_token=__lowerCamelCase       ,							mask_token=__lowerCamelCase       ,							add_prefix_space=__lowerCamelCase       ,							**__lowerCamelCase       ,							)

         with open(__lowerCamelCase       ,							encoding="""utf-8"""							) as vocab_handle:
            UpperCAmelCase      					=  json.load(__lowerCamelCase							)
         UpperCAmelCase      					=  {v: k for k, v in self.encoder.items()}
         UpperCAmelCase      					=  errors  # how to handle errors in decoding
         UpperCAmelCase      					=  bytes_to_unicode()
         UpperCAmelCase      					=  {v: k for k, v in self.byte_encoder.items()}
         with open(__lowerCamelCase       ,							encoding="""utf-8"""							) as merges_handle:
            UpperCAmelCase      					=  merges_handle.read().split("""\n"""							)[1:-1]
         UpperCAmelCase      					=  [tuple(merge.split()							) for merge in bpe_merges]
         UpperCAmelCase      					=  dict(zip(__lowerCamelCase       ,							range(len(__lowerCamelCase							)							)							)							)
         UpperCAmelCase      					=  {}
         UpperCAmelCase      					=  add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         UpperCAmelCase      					=  re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""							)





      @property
      def 							_lowercase					(		self    :       int							)     -> Union[str, Any]:






         """simple docstring"""







         return len(self.encoder							)





      def 							_lowercase					(		self    :       Optional[Any]							)     -> List[Any]:






         """simple docstring"""







         return dict(self.encoder       ,							**self.added_tokens_encoder							)





      def 							_lowercase					(		self    :       str       ,							__lowerCamelCase    :       str							)     -> Tuple:






         """simple docstring"""







         if token in self.cache:
            return self.cache[token]
         UpperCAmelCase      					=  tuple(__lowerCamelCase							)
         UpperCAmelCase      					=  get_pairs(__lowerCamelCase							)

         if not pairs:
            return token

         while True:
            UpperCAmelCase      					=  min(__lowerCamelCase       ,							key=lambda __lowerCamelCase							: self.bpe_ranks.get(__lowerCamelCase       ,							float("""inf"""							)							)							)
            if bigram not in self.bpe_ranks:
               break
            UpperCAmelCase					,	UpperCAmelCase      					=  bigram
            UpperCAmelCase      					=  []
            UpperCAmelCase      					=  0
            while i < len(__lowerCamelCase							):
               try:
                  UpperCAmelCase      					=  word.index(__lowerCamelCase       ,							__lowerCamelCase							)
               except ValueError:
                  new_word.extend(word[i:]							)
                  break
               else:
                  new_word.extend(word[i:j]							)
                  UpperCAmelCase      					=  j

               if word[i] == first and i < len(__lowerCamelCase							) - 1 and word[i + 1] == second:
                  new_word.append(first + second							)
                  i += 2
               else:
                  new_word.append(word[i]							)
                  i += 1
            UpperCAmelCase      					=  tuple(__lowerCamelCase							)
            UpperCAmelCase      					=  new_word
            if len(__lowerCamelCase							) == 1:
               break
            else:
               UpperCAmelCase      					=  get_pairs(__lowerCamelCase							)
         UpperCAmelCase      					=  """ """.join(__lowerCamelCase							)
         UpperCAmelCase      					=  word
         return word





      def 							_lowercase					(		self    :       List[Any]       ,							__lowerCamelCase    :       Tuple							)     -> Any:






         """simple docstring"""







         UpperCAmelCase      					=  []
         for token in re.findall(self.pat       ,							__lowerCamelCase							):
            UpperCAmelCase      					=  """""".join(
                self.byte_encoder[b] for b in token.encode("""utf-8"""							)							)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCamelCase							).split(""" """							)							)
         return bpe_tokens





      def 							_lowercase					(		self    :       int       ,							__lowerCamelCase    :       List[Any]							)     -> str:






         """simple docstring"""







         return self.encoder.get(__lowerCamelCase       ,							self.encoder.get(self.unk_token							)							)





      def 							_lowercase					(		self    :       Dict       ,							__lowerCamelCase    :       Dict							)     -> Optional[Any]:






         """simple docstring"""







         return self.decoder.get(__lowerCamelCase							)





      def 							_lowercase					(		self    :       Optional[Any]       ,							__lowerCamelCase    :       Union[str, Any]							)     -> List[Any]:






         """simple docstring"""







         UpperCAmelCase      					=  """""".join(__lowerCamelCase							)
         UpperCAmelCase      					=  bytearray([self.byte_decoder[c] for c in text]							).decode("""utf-8"""       ,							errors=self.errors							)
         return text





      def 							_lowercase					(		self    :       List[Any]       ,							__lowerCamelCase    :       str       ,							__lowerCamelCase    :       Optional[str] = None							)     -> Tuple[str]:






         """simple docstring"""







         if not os.path.isdir(__lowerCamelCase							):
            logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""							)
            return
         UpperCAmelCase      					=  os.path.join(
             __lowerCamelCase       ,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)
         UpperCAmelCase      					=  os.path.join(
             __lowerCamelCase       ,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]							)

         with open(__lowerCamelCase       ,							"""w"""       ,							encoding="""utf-8"""							) as f:
            f.write(json.dumps(self.encoder       ,							indent=2       ,							sort_keys=__lowerCamelCase       ,							ensure_ascii=__lowerCamelCase							) + """\n"""							)

         UpperCAmelCase      					=  0
         with open(__lowerCamelCase       ,							"""w"""       ,							encoding="""utf-8"""							) as writer:
            writer.write("""#version: 0.2\n"""							)
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,							key=lambda __lowerCamelCase							: kv[1]							):
               if index != token_index:
                  logger.warning(
                      F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                      """ Please check that the tokenizer is not corrupted!"""							)
                  UpperCAmelCase      					=  token_index
               writer.write(""" """.join(__lowerCamelCase							) + """\n"""							)
               index += 1

         return vocab_file, merge_file





      def 							_lowercase					(		self    :       Any       ,							__lowerCamelCase    :       List[int]       ,							__lowerCamelCase    :       Optional[List[int]] = None							)     -> List[int]:






         """simple docstring"""







         if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         UpperCAmelCase      					=  [self.cls_token_id]
         UpperCAmelCase      					=  [self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep





      def 							_lowercase					(		self    :       Dict       ,							__lowerCamelCase    :       List[int]       ,							__lowerCamelCase    :       Optional[List[int]] = None       ,							__lowerCamelCase    :       bool = False							)     -> List[int]:






         """simple docstring"""







         if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=__lowerCamelCase       ,							token_ids_a=__lowerCamelCase       ,							already_has_special_tokens=__lowerCamelCase							)

         if token_ids_a is None:
            return [1] + ([0] * len(__lowerCamelCase							)) + [1]
         return [1] + ([0] * len(__lowerCamelCase							)) + [1, 1] + ([0] * len(__lowerCamelCase							)) + [1]





      def 							_lowercase					(		self    :       Union[str, Any]       ,							__lowerCamelCase    :       List[int]       ,							__lowerCamelCase    :       Optional[List[int]] = None							)     -> List[int]:






         """simple docstring"""







         UpperCAmelCase      					=  [self.sep_token_id]
         UpperCAmelCase      					=  [self.cls_token_id]

         if token_ids_a is None:
            return len(cls + token_ids_a + sep							) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]





      def 							_lowercase					(		self    :       str       ,							__lowerCamelCase    :       List[str]       ,							__lowerCamelCase    :       Tuple=False       ,							**__lowerCamelCase    :       List[Any]							)     -> Any:






         """simple docstring"""







         UpperCAmelCase      					=  kwargs.pop("""add_prefix_space"""       ,							self.add_prefix_space							)
         if (is_split_into_words or add_prefix_space) and (len(__lowerCamelCase							) > 0 and not text[0].isspace()):
            UpperCAmelCase      					=  """ """ + text
         return (text, kwargs)







import json
import multiprocessing as mp
import re
from collections import defaultdict
from functools import partial
from typing import Dict, List, Optional, Set, Tuple, Type

from datasets import Dataset
from datasketch import MinHash, MinHashLSH
from dpu_utils.utils.iterators import ThreadedIterator
from tqdm import tqdm


__a	 =      re.compile("""[^A-Za-z_0-9]""")
# parameters used in DuplicationIndex
__a	 =      10
__a	 =      256
def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->Optional[MinHash]:
   if len(lowerCAmelCase_ ) < MIN_NUM_TOKENS:
      return None
   UpperCAmelCase      					=  MinHash(num_perm=lowerCAmelCase_ )
   for token in set(lowerCAmelCase_ ):
      min_hash.update(token.encode() )
   return min_hash
def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->Set[str]:
   return {t for t in NON_ALPHA.split(lowerCAmelCase_ ) if len(t.strip() ) > 0}






class 	__lowercase					:





      def __init__(		self    :       List[str]       ,							*,
          __lowerCamelCase    :       float = 0.85       ,							)     -> Any:






         """simple docstring"""







         UpperCAmelCase      					=  duplication_jaccard_threshold
         UpperCAmelCase      					=  NUM_PERM
         UpperCAmelCase      					=  MinHashLSH(threshold=self._duplication_jaccard_threshold       ,							num_perm=self._num_perm							)

         UpperCAmelCase      					=  defaultdict(__lowerCamelCase							)





      def 							_lowercase					(		self    :       Tuple       ,							__lowerCamelCase    :       Tuple       ,							__lowerCamelCase    :       MinHash							)     -> None:






         """simple docstring"""







         UpperCAmelCase      					=  self._index.query(__lowerCamelCase							)
         if code_key in self._index.keys:
            print(F"""Duplicate key {code_key}"""							)
            return

         self._index.insert(__lowerCamelCase       ,							__lowerCamelCase							)
         if len(__lowerCamelCase							) > 0:
            for base_duplicate in close_duplicates:
               if base_duplicate in self._duplicate_clusters:
                  self._duplicate_clusters[base_duplicate].add(__lowerCamelCase							)
                  break
            else:
               self._duplicate_clusters[close_duplicates[0]].add(__lowerCamelCase							)





      def 							_lowercase					(		self    :       Union[str, Any]							)     -> List[List[Dict]]:






         """simple docstring"""







         UpperCAmelCase      					=  []
         for base, duplicates in self._duplicate_clusters.items():
            UpperCAmelCase      					=  [base] + list(__lowerCamelCase							)
            # reformat the cluster to be a list of dict
            UpperCAmelCase      					=  [{"""base_index""": el[0], """repo_name""": el[1], """path""": el[2]} for el in cluster]
            duplicate_clusters.append(__lowerCamelCase							)
         return duplicate_clusters





      def 							_lowercase					(		self    :       Tuple       ,							__lowerCamelCase    :       Optional[int]							)     -> None:






         """simple docstring"""







         UpperCAmelCase      					=  self.get_duplicate_clusters()
         with open(__lowerCamelCase       ,							"""w"""							) as f:
            json.dump(__lowerCamelCase       ,							__lowerCamelCase							)

def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->Tuple:
   UpperCAmelCase					,	UpperCAmelCase      					=  element
   UpperCAmelCase      					=  get_min_hash([t for t in NON_ALPHA.split(data["""content"""] ) if len(t.strip() ) > 0] )
   if min_hash is not None:
      return (index, data["repo_name"], data["path"]), min_hash
def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->int:
   with mp.Pool() as pool:
      for data in pool.imap_unordered(
          _compute_min_hash	,    ThreadedIterator(lowerCAmelCase_	,    max_queue_size=1_0_0_0_0 )	,    chunksize=1_0_0	,    ):
         if data is not None:
            yield data
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_ )    ->Any:
   UpperCAmelCase      					=  DuplicationIndex(duplication_jaccard_threshold=lowerCAmelCase_ )

   for filename, min_hash in tqdm(ThreadedIterator(minhash_iter(enumerate(lowerCAmelCase_ ) )	,    max_queue_size=1_0_0 ) ):
      di.add(lowerCAmelCase_	,    lowerCAmelCase_ )

   # Returns a List[Cluster] where Cluster is List[str] with the filenames.
   return di.get_duplicate_clusters()
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_ )    ->float:
   UpperCAmelCase      					=  get_tokens(lowerCAmelCase_ )
   UpperCAmelCase      					=  get_tokens(lowerCAmelCase_ )
   return len(tokensa & tokensa ) / len(tokensa | tokensa )


__a	 =      None
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_ )    ->Dict:
   UpperCAmelCase      					=  []
   for elementa in cluster:
      UpperCAmelCase      					=  _shared_dataset[elementa["""base_index"""]]["""content"""]
      for elementa in extremes:
         UpperCAmelCase      					=  _shared_dataset[elementa["""base_index"""]]["""content"""]
         if jaccard_similarity(lowerCAmelCase_	,    lowerCAmelCase_ ) >= jaccard_threshold:
            elementa["copies"] += 1
            break
      else:
         UpperCAmelCase      					=  1
         extremes.append(lowerCAmelCase_ )
   return extremes
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_	,    lowerCAmelCase_ )    ->Optional[int]:
   global _shared_dataset
   UpperCAmelCase      					=  dataset
   UpperCAmelCase      					=  []
   UpperCAmelCase      					=  partial(_find_cluster_extremes_shared	,    jaccard_threshold=lowerCAmelCase_ )
   with mp.Pool() as pool:
      for extremes in tqdm(
          pool.imap_unordered(
              lowerCAmelCase_	,    lowerCAmelCase_	,    )	,    total=len(lowerCAmelCase_ )	,    ):
         extremes_list.append(lowerCAmelCase_ )
   return extremes_list







def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_ = 0.85 )    ->Tuple[Type[Dataset], List[List[Dict]]]:
   UpperCAmelCase      					=  make_duplicate_clusters(lowerCAmelCase_	,    lowerCAmelCase_ )
   UpperCAmelCase      					=  {x["""base_index"""] for cluster in duplicate_clusters for x in cluster}
   UpperCAmelCase      					=  {}
   UpperCAmelCase      					=  find_extremes(lowerCAmelCase_	,    lowerCAmelCase_	,    lowerCAmelCase_ )
   for extremes in extremes_clusters:
      for element in extremes:
         UpperCAmelCase      					=  element
   UpperCAmelCase      					=  duplicate_indices - set(extreme_dict.keys() )
   UpperCAmelCase      					=  dataset.filter(lambda lowerCAmelCase_	,    lowerCAmelCase_ : idx not in remove_indices	,    with_indices=lowerCAmelCase_ )

   # update duplicate_clusters
   for cluster in duplicate_clusters:
      for element in cluster:
         UpperCAmelCase      					=  element["""base_index"""] in extreme_dict
         if element["is_extreme"]:
            UpperCAmelCase      					=  extreme_dict[element["""base_index"""]]["""copies"""]

   print(F"""Original dataset size: {len(lowerCAmelCase_ )}""" )
   print(F"""Number of duplicate clusters: {len(lowerCAmelCase_ )}""" )
   print(F"""Files in duplicate cluster: {len(lowerCAmelCase_ )}""" )
   print(F"""Unique files in duplicate cluster: {len(lowerCAmelCase_ )}""" )
   print(F"""Filtered dataset size: {len(lowerCAmelCase_ )}""" )

   return ds_filter, duplicate_clusters



