


import logging
import os
from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

from filelock import FileLock

from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available


__a	 =      logging.getLogger(__name__)






@dataclass
class 	__lowercase					:
      UpperCamelCase										=							42
      UpperCamelCase										=							42
      UpperCamelCase										=							42






@dataclass
class 	__lowercase					:
      UpperCamelCase										=							42
      UpperCamelCase										=							42
      UpperCamelCase										=							None
      UpperCamelCase										=							None






class 	__lowercase					(	__snake_case	):
      UpperCamelCase										=							'''train'''
      UpperCamelCase										=							'''dev'''
      UpperCamelCase										=							'''test'''






class 	__lowercase					:





      @staticmethod
      def 							_lowercase					(		__lowerCamelCase    :       Tuple       ,							__lowerCamelCase    :       Union[Split, str]							)     -> List[InputExample]:






         """simple docstring"""







         raise NotImplementedError





      @staticmethod
      def 							_lowercase					(		__lowerCamelCase    :       str							)     -> List[str]:






         """simple docstring"""







         raise NotImplementedError





      @staticmethod
      def 							_lowercase					(		__lowerCamelCase    :       List[InputExample]       ,							__lowerCamelCase    :       List[str]       ,							__lowerCamelCase    :       int       ,							__lowerCamelCase    :       PreTrainedTokenizer       ,							__lowerCamelCase    :       Tuple=False       ,							__lowerCamelCase    :       List[str]="[CLS]"       ,							__lowerCamelCase    :       Any=1       ,							__lowerCamelCase    :       Any="[SEP]"       ,							__lowerCamelCase    :       Union[str, Any]=False       ,							__lowerCamelCase    :       str=False       ,							__lowerCamelCase    :       List[str]=0       ,							__lowerCamelCase    :       str=0       ,							__lowerCamelCase    :       Optional[int]=-1_0_0       ,							__lowerCamelCase    :       Tuple=0       ,							__lowerCamelCase    :       Any=True       ,							)     -> List[InputFeatures]:






         """simple docstring"""







         UpperCAmelCase      					=  {label: i for i, label in enumerate(__lowerCamelCase							)}

         UpperCAmelCase      					=  []
         for ex_index, example in enumerate(__lowerCamelCase							):
            if ex_index % 1_0_0_0_0 == 0:
               logger.info("""Writing example %d of %d"""       ,							__lowerCamelCase       ,							len(__lowerCamelCase							)							)

            UpperCAmelCase      					=  []
            UpperCAmelCase      					=  []
            for word, label in zip(example.words       ,							example.labels							):
               UpperCAmelCase      					=  tokenizer.tokenize(__lowerCamelCase							)

               # bert-base-multilingual-cased sometimes output "nothing ([]) when calling tokenize with just a space.
               if len(__lowerCamelCase							) > 0:
                  tokens.extend(__lowerCamelCase							)
                  # Use the real label id for the first token of the word, and padding ids for the remaining tokens
                  label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(__lowerCamelCase							) - 1)							)

            # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.
            UpperCAmelCase      					=  tokenizer.num_special_tokens_to_add()
            if len(__lowerCamelCase							) > max_seq_length - special_tokens_count:
               UpperCAmelCase      					=  tokens[: (max_seq_length - special_tokens_count)]
               UpperCAmelCase      					=  label_ids[: (max_seq_length - special_tokens_count)]

            # The convention in BERT is:
            # (a) For sequence pairs:
            #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
            #  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1
            # (b) For single sequences:
            #  tokens:   [CLS] the dog is hairy . [SEP]
            #  type_ids:   0   0   0   0  0     0   0
            #
            # Where "type_ids" are used to indicate whether this is the first
            # sequence or the second sequence. The embedding vectors for `type=0` and
            # `type=1` were learned during pre-training and are added to the wordpiece
            # embedding vector (and position vector). This is not *strictly* necessary
            # since the [SEP] token unambiguously separates the sequences, but it makes
            # it easier for the model to learn the concept of sequences.
            #
            # For classification tasks, the first vector (corresponding to [CLS]) is
            # used as the "sentence vector". Note that this only makes sense because
            # the entire model is fine-tuned.
            tokens += [sep_token]
            label_ids += [pad_token_label_id]
            if sep_token_extra:
               # roberta uses an extra separator b/w pairs of sentences
               tokens += [sep_token]
               label_ids += [pad_token_label_id]
            UpperCAmelCase      					=  [sequence_a_segment_id] * len(__lowerCamelCase							)

            if cls_token_at_end:
               tokens += [cls_token]
               label_ids += [pad_token_label_id]
               segment_ids += [cls_token_segment_id]
            else:
               UpperCAmelCase      					=  [cls_token] + tokens
               UpperCAmelCase      					=  [pad_token_label_id] + label_ids
               UpperCAmelCase      					=  [cls_token_segment_id] + segment_ids

            UpperCAmelCase      					=  tokenizer.convert_tokens_to_ids(__lowerCamelCase							)

            # The mask has 1 for real tokens and 0 for padding tokens. Only real
            # tokens are attended to.
            UpperCAmelCase      					=  [1 if mask_padding_with_zero else 0] * len(__lowerCamelCase							)

            # Zero-pad up to the sequence length.
            UpperCAmelCase      					=  max_seq_length - len(__lowerCamelCase							)
            if pad_on_left:
               UpperCAmelCase      					=  ([pad_token] * padding_length) + input_ids
               UpperCAmelCase      					=  ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask
               UpperCAmelCase      					=  ([pad_token_segment_id] * padding_length) + segment_ids
               UpperCAmelCase      					=  ([pad_token_label_id] * padding_length) + label_ids
            else:
               input_ids += [pad_token] * padding_length
               input_mask += [0 if mask_padding_with_zero else 1] * padding_length
               segment_ids += [pad_token_segment_id] * padding_length
               label_ids += [pad_token_label_id] * padding_length

            assert len(__lowerCamelCase							) == max_seq_length
            assert len(__lowerCamelCase							) == max_seq_length
            assert len(__lowerCamelCase							) == max_seq_length
            assert len(__lowerCamelCase							) == max_seq_length

            if ex_index < 5:
               logger.info("""*** Example ***"""							)
               logger.info("""guid: %s"""       ,							example.guid							)
               logger.info("""tokens: %s"""       ,							""" """.join([str(__lowerCamelCase							) for x in tokens]							)							)
               logger.info("""input_ids: %s"""       ,							""" """.join([str(__lowerCamelCase							) for x in input_ids]							)							)
               logger.info("""input_mask: %s"""       ,							""" """.join([str(__lowerCamelCase							) for x in input_mask]							)							)
               logger.info("""segment_ids: %s"""       ,							""" """.join([str(__lowerCamelCase							) for x in segment_ids]							)							)
               logger.info("""label_ids: %s"""       ,							""" """.join([str(__lowerCamelCase							) for x in label_ids]							)							)

            if "token_type_ids" not in tokenizer.model_input_names:
               UpperCAmelCase      					=  None

            features.append(
                InputFeatures(
                    input_ids=__lowerCamelCase       ,							attention_mask=__lowerCamelCase       ,							token_type_ids=__lowerCamelCase       ,							label_ids=__lowerCamelCase							)							)
         return features


if is_torch_available():
 import torch
 from torch import nn
 from torch.utils.data import Dataset






 class 	__lowercase					(	__snake_case	):
       UpperCamelCase										=							42
       UpperCamelCase										=							nn.CrossEntropyLoss().ignore_index





       def __init__(		self    :       List[Any]       ,							__lowerCamelCase    :       TokenClassificationTask       ,							__lowerCamelCase    :       str       ,							__lowerCamelCase    :       PreTrainedTokenizer       ,							__lowerCamelCase    :       List[str]       ,							__lowerCamelCase    :       str       ,							__lowerCamelCase    :       Optional[int] = None       ,							__lowerCamelCase    :       Tuple=False       ,							__lowerCamelCase    :       Split = Split.train       ,							)     -> Union[str, Any]:






          """simple docstring"""







          UpperCAmelCase      					=  os.path.join(
              __lowerCamelCase       ,							"""cached_{}_{}_{}""".format(mode.value       ,							tokenizer.__class__.__name__       ,							str(__lowerCamelCase							)							)       ,							)

          # Make sure only the first process in distributed training processes the dataset,
          # and the others will use the cache.
          UpperCAmelCase      					=  cached_features_file + """.lock"""
          with FileLock(__lowerCamelCase							):
             if os.path.exists(__lowerCamelCase							) and not overwrite_cache:
                logger.info(F"""Loading features from cached file {cached_features_file}"""							)
                UpperCAmelCase      					=  torch.load(__lowerCamelCase							)
             else:
                logger.info(F"""Creating features from dataset file at {data_dir}"""							)
                UpperCAmelCase      					=  token_classification_task.read_examples_from_file(__lowerCamelCase       ,							__lowerCamelCase							)
                # TODO clean up all this to leverage built-in features of tokenizers
                UpperCAmelCase      					=  token_classification_task.convert_examples_to_features(
                    __lowerCamelCase       ,							__lowerCamelCase       ,							__lowerCamelCase       ,							__lowerCamelCase       ,							cls_token_at_end=bool(model_type in ["""xlnet"""]							)       ,							cls_token=tokenizer.cls_token       ,							cls_token_segment_id=2 if model_type in ["""xlnet"""] else 0       ,							sep_token=tokenizer.sep_token       ,							sep_token_extra=__lowerCamelCase       ,							pad_on_left=bool(tokenizer.padding_side == """left"""							)       ,							pad_token=tokenizer.pad_token_id       ,							pad_token_segment_id=tokenizer.pad_token_type_id       ,							pad_token_label_id=self.pad_token_label_id       ,							)
                logger.info(F"""Saving features into cached file {cached_features_file}"""							)
                torch.save(self.features       ,							__lowerCamelCase							)





       def __len__(		self    :       List[str]							)     -> List[str]:






          """simple docstring"""







          return len(self.features							)





       def __getitem__(		self    :       str       ,							__lowerCamelCase    :       str							)     -> InputFeatures:






          """simple docstring"""







          return self.features[i]


if is_tf_available():
 import tensorflow as tf






 class 	__lowercase					:
       UpperCamelCase										=							42
       UpperCamelCase										=							-1_00





       def __init__(		self    :       str       ,							__lowerCamelCase    :       TokenClassificationTask       ,							__lowerCamelCase    :       str       ,							__lowerCamelCase    :       PreTrainedTokenizer       ,							__lowerCamelCase    :       List[str]       ,							__lowerCamelCase    :       str       ,							__lowerCamelCase    :       Optional[int] = None       ,							__lowerCamelCase    :       Any=False       ,							__lowerCamelCase    :       Split = Split.train       ,							)     -> str:






          """simple docstring"""







          UpperCAmelCase      					=  token_classification_task.read_examples_from_file(__lowerCamelCase       ,							__lowerCamelCase							)
          # TODO clean up all this to leverage built-in features of tokenizers
          UpperCAmelCase      					=  token_classification_task.convert_examples_to_features(
              __lowerCamelCase       ,							__lowerCamelCase       ,							__lowerCamelCase       ,							__lowerCamelCase       ,							cls_token_at_end=bool(model_type in ["""xlnet"""]							)       ,							cls_token=tokenizer.cls_token       ,							cls_token_segment_id=2 if model_type in ["""xlnet"""] else 0       ,							sep_token=tokenizer.sep_token       ,							sep_token_extra=__lowerCamelCase       ,							pad_on_left=bool(tokenizer.padding_side == """left"""							)       ,							pad_token=tokenizer.pad_token_id       ,							pad_token_segment_id=tokenizer.pad_token_type_id       ,							pad_token_label_id=self.pad_token_label_id       ,							)

          def gen():
             for ex in self.features:
                if ex.token_type_ids is None:
                   yield (
                       {"input_ids": ex.input_ids, "attention_mask": ex.attention_mask},
                       ex.label_ids,
                   )
                else:
                   yield (
                       {
                           "input_ids": ex.input_ids,
                           "attention_mask": ex.attention_mask,
                           "token_type_ids": ex.token_type_ids,
                       },
                       ex.label_ids,
                   )

          if "token_type_ids" not in tokenizer.model_input_names:
             UpperCAmelCase      					=  tf.data.Dataset.from_generator(
                 __lowerCamelCase       ,							({"""input_ids""": tf.intaa, """attention_mask""": tf.intaa}, tf.intaa)       ,							(
                     {"""input_ids""": tf.TensorShape([None]							), """attention_mask""": tf.TensorShape([None]							)},
                     tf.TensorShape([None]							),
                 )       ,							)
          else:
             UpperCAmelCase      					=  tf.data.Dataset.from_generator(
                 __lowerCamelCase       ,							({"""input_ids""": tf.intaa, """attention_mask""": tf.intaa, """token_type_ids""": tf.intaa}, tf.intaa)       ,							(
                     {
                         """input_ids""": tf.TensorShape([None]							),
                         """attention_mask""": tf.TensorShape([None]							),
                         """token_type_ids""": tf.TensorShape([None]							),
                     },
                     tf.TensorShape([None]							),
                 )       ,							)





       def 							_lowercase					(		self    :       List[Any]							)     -> int:






          """simple docstring"""







          UpperCAmelCase      					=  self.dataset.apply(tf.data.experimental.assert_cardinality(len(self.features							)							)							)

          return self.dataset





       def __len__(		self    :       int							)     -> List[Any]:






          """simple docstring"""







          return len(self.features							)





       def __getitem__(		self    :       List[Any]       ,							__lowerCamelCase    :       Optional[int]							)     -> InputFeatures:






          """simple docstring"""







          return self.features[i]







import json
import multiprocessing as mp
import re
from collections import defaultdict
from functools import partial
from typing import Dict, List, Optional, Set, Tuple, Type

from datasets import Dataset
from datasketch import MinHash, MinHashLSH
from dpu_utils.utils.iterators import ThreadedIterator
from tqdm import tqdm


__a	 =      re.compile("""[^A-Za-z_0-9]""")
# parameters used in DuplicationIndex
__a	 =      10
__a	 =      256
def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->Optional[MinHash]:
   if len(lowerCAmelCase_ ) < MIN_NUM_TOKENS:
      return None
   UpperCAmelCase      					=  MinHash(num_perm=lowerCAmelCase_ )
   for token in set(lowerCAmelCase_ ):
      min_hash.update(token.encode() )
   return min_hash
def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->Set[str]:
   return {t for t in NON_ALPHA.split(lowerCAmelCase_ ) if len(t.strip() ) > 0}






class 	__lowercase					:





      def __init__(		self    :       List[str]       ,							*,
          __lowerCamelCase    :       float = 0.85       ,							)     -> Any:






         """simple docstring"""







         UpperCAmelCase      					=  duplication_jaccard_threshold
         UpperCAmelCase      					=  NUM_PERM
         UpperCAmelCase      					=  MinHashLSH(threshold=self._duplication_jaccard_threshold       ,							num_perm=self._num_perm							)

         UpperCAmelCase      					=  defaultdict(__lowerCamelCase							)





      def 							_lowercase					(		self    :       Tuple       ,							__lowerCamelCase    :       Tuple       ,							__lowerCamelCase    :       MinHash							)     -> None:






         """simple docstring"""







         UpperCAmelCase      					=  self._index.query(__lowerCamelCase							)
         if code_key in self._index.keys:
            print(F"""Duplicate key {code_key}"""							)
            return

         self._index.insert(__lowerCamelCase       ,							__lowerCamelCase							)
         if len(__lowerCamelCase							) > 0:
            for base_duplicate in close_duplicates:
               if base_duplicate in self._duplicate_clusters:
                  self._duplicate_clusters[base_duplicate].add(__lowerCamelCase							)
                  break
            else:
               self._duplicate_clusters[close_duplicates[0]].add(__lowerCamelCase							)





      def 							_lowercase					(		self    :       Union[str, Any]							)     -> List[List[Dict]]:






         """simple docstring"""







         UpperCAmelCase      					=  []
         for base, duplicates in self._duplicate_clusters.items():
            UpperCAmelCase      					=  [base] + list(__lowerCamelCase							)
            # reformat the cluster to be a list of dict
            UpperCAmelCase      					=  [{"""base_index""": el[0], """repo_name""": el[1], """path""": el[2]} for el in cluster]
            duplicate_clusters.append(__lowerCamelCase							)
         return duplicate_clusters





      def 							_lowercase					(		self    :       Tuple       ,							__lowerCamelCase    :       Optional[int]							)     -> None:






         """simple docstring"""







         UpperCAmelCase      					=  self.get_duplicate_clusters()
         with open(__lowerCamelCase       ,							"""w"""							) as f:
            json.dump(__lowerCamelCase       ,							__lowerCamelCase							)

def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->Tuple:
   UpperCAmelCase					,	UpperCAmelCase      					=  element
   UpperCAmelCase      					=  get_min_hash([t for t in NON_ALPHA.split(data["""content"""] ) if len(t.strip() ) > 0] )
   if min_hash is not None:
      return (index, data["repo_name"], data["path"]), min_hash
def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->int:
   with mp.Pool() as pool:
      for data in pool.imap_unordered(
          _compute_min_hash	,    ThreadedIterator(lowerCAmelCase_	,    max_queue_size=1_0_0_0_0 )	,    chunksize=1_0_0	,    ):
         if data is not None:
            yield data
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_ )    ->Any:
   UpperCAmelCase      					=  DuplicationIndex(duplication_jaccard_threshold=lowerCAmelCase_ )

   for filename, min_hash in tqdm(ThreadedIterator(minhash_iter(enumerate(lowerCAmelCase_ ) )	,    max_queue_size=1_0_0 ) ):
      di.add(lowerCAmelCase_	,    lowerCAmelCase_ )

   # Returns a List[Cluster] where Cluster is List[str] with the filenames.
   return di.get_duplicate_clusters()
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_ )    ->float:
   UpperCAmelCase      					=  get_tokens(lowerCAmelCase_ )
   UpperCAmelCase      					=  get_tokens(lowerCAmelCase_ )
   return len(tokensa & tokensa ) / len(tokensa | tokensa )


__a	 =      None
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_ )    ->Dict:
   UpperCAmelCase      					=  []
   for elementa in cluster:
      UpperCAmelCase      					=  _shared_dataset[elementa["""base_index"""]]["""content"""]
      for elementa in extremes:
         UpperCAmelCase      					=  _shared_dataset[elementa["""base_index"""]]["""content"""]
         if jaccard_similarity(lowerCAmelCase_	,    lowerCAmelCase_ ) >= jaccard_threshold:
            elementa["copies"] += 1
            break
      else:
         UpperCAmelCase      					=  1
         extremes.append(lowerCAmelCase_ )
   return extremes
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_	,    lowerCAmelCase_ )    ->Optional[int]:
   global _shared_dataset
   UpperCAmelCase      					=  dataset
   UpperCAmelCase      					=  []
   UpperCAmelCase      					=  partial(_find_cluster_extremes_shared	,    jaccard_threshold=lowerCAmelCase_ )
   with mp.Pool() as pool:
      for extremes in tqdm(
          pool.imap_unordered(
              lowerCAmelCase_	,    lowerCAmelCase_	,    )	,    total=len(lowerCAmelCase_ )	,    ):
         extremes_list.append(lowerCAmelCase_ )
   return extremes_list







def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_ = 0.85 )    ->Tuple[Type[Dataset], List[List[Dict]]]:
   UpperCAmelCase      					=  make_duplicate_clusters(lowerCAmelCase_	,    lowerCAmelCase_ )
   UpperCAmelCase      					=  {x["""base_index"""] for cluster in duplicate_clusters for x in cluster}
   UpperCAmelCase      					=  {}
   UpperCAmelCase      					=  find_extremes(lowerCAmelCase_	,    lowerCAmelCase_	,    lowerCAmelCase_ )
   for extremes in extremes_clusters:
      for element in extremes:
         UpperCAmelCase      					=  element
   UpperCAmelCase      					=  duplicate_indices - set(extreme_dict.keys() )
   UpperCAmelCase      					=  dataset.filter(lambda lowerCAmelCase_	,    lowerCAmelCase_ : idx not in remove_indices	,    with_indices=lowerCAmelCase_ )

   # update duplicate_clusters
   for cluster in duplicate_clusters:
      for element in cluster:
         UpperCAmelCase      					=  element["""base_index"""] in extreme_dict
         if element["is_extreme"]:
            UpperCAmelCase      					=  extreme_dict[element["""base_index"""]]["""copies"""]

   print(F"""Original dataset size: {len(lowerCAmelCase_ )}""" )
   print(F"""Number of duplicate clusters: {len(lowerCAmelCase_ )}""" )
   print(F"""Files in duplicate cluster: {len(lowerCAmelCase_ )}""" )
   print(F"""Unique files in duplicate cluster: {len(lowerCAmelCase_ )}""" )
   print(F"""Filtered dataset size: {len(lowerCAmelCase_ )}""" )

   return ds_filter, duplicate_clusters



