


'''simple docstring'''
import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


UpperCAmelCase_			:  Union[str, Any]										=   re.compile(R'\s+')


def       _lowercase							(					UpperCamelCase__							:       List[str]      ):
 return {"hash": hashlib.mda(re.sub(UpperCamelCase__,					'',					example['content']      ).encode('utf-8'      )      ).hexdigest()}


def       _lowercase							(					UpperCamelCase__							:       Dict      ):
 __A	:		Optional[Any]	  =      [len(UpperCamelCase__      ) for line in example['content'].splitlines()]
 return {"line_mean": np.mean(UpperCamelCase__      ), "line_max": max(UpperCamelCase__      )}


def       _lowercase							(					UpperCamelCase__							:       Optional[Any]      ):
 __A	:		Dict	  =      np.mean([c.isalnum() for c in example['content']]      )
 return {"alpha_frac": alpha_frac}


def       _lowercase							(					UpperCamelCase__							:       int,					UpperCamelCase__							:       Optional[Any]      ):
 if example["hash"] in uniques:
  uniques.remove(example['hash']      )
  return True
 else:
  return False


def       _lowercase							(					UpperCamelCase__							:       str,					UpperCamelCase__							:       int=5      ):
 __A	:		int	  =      ['auto-generated', 'autogenerated', 'automatically generated']
 __A	:		Union[str, Any]	  =      example['content'].splitlines()
 for _, line in zip(range(UpperCamelCase__      ),					UpperCamelCase__      ):
  for keyword in keywords:
   if keyword in line.lower():
    return {"autogenerated": True}
 else:
  return {"autogenerated": False}


def       _lowercase							(					UpperCamelCase__							:       Union[str, Any],					UpperCamelCase__							:       Optional[Any]=5,					UpperCamelCase__							:       Union[str, Any]=0.05      ):
 __A	:		List[str]	  =      ['unit tests', 'test file', 'configuration file']
 __A	:		Any	  =      example['content'].splitlines()
 __A	:		Optional[Any]	  =      0
 __A	:		Tuple	  =      0
 # first test
 for _, line in zip(range(UpperCamelCase__      ),					UpperCamelCase__      ):
  for keyword in keywords:
   if keyword in line.lower():
    return {"config_or_test": True}
    # second test
 __A	:		Dict	  =      example['content'].count('\n'      )
 __A	:		str	  =      int(coeff * nlines      )
 for line in lines:
  count_config += line.lower().count('config'      )
  count_test += line.lower().count('test'      )
  if count_config > threshold or count_test > threshold:
   return {"config_or_test": True}
 return {"config_or_test": False}


def       _lowercase							(					UpperCamelCase__							:       List[str]      ):
 __A	:		str	  =      ['def ', 'class ', 'for ', 'while ']
 __A	:		List[Any]	  =      example['content'].splitlines()
 for line in lines:
  for keyword in keywords:
   if keyword in line.lower():
    return {"has_no_keywords": False}
 return {"has_no_keywords": True}


def       _lowercase							(					UpperCamelCase__							:       int,					UpperCamelCase__							:       int=4      ):
 __A	:		Tuple	  =      example['content'].splitlines()
 __A	:		List[str]	  =      0
 for line in lines:
  counter += line.lower().count('='      )
  if counter > minimum:
   return {"has_few_assignments": False}
 return {"has_few_assignments": True}


def       _lowercase							(					UpperCamelCase__							:       str      ):
 __A	:		List[Any]	  =      tokenizer(example['content'],					truncation=UpperCamelCase__      )['input_ids']
 __A	:		List[str]	  =      len(example['content']      ) / len(UpperCamelCase__      )
 return {"ratio": ratio}


def       _lowercase							(					UpperCamelCase__							:       Optional[int]      ):
 __A	:		str	  =      {}
 results.update(get_hash(UpperCamelCase__      )      )
 results.update(line_stats(UpperCamelCase__      )      )
 results.update(alpha_stats(UpperCamelCase__      )      )
 results.update(char_token_ratio(UpperCamelCase__      )      )
 results.update(is_autogenerated(UpperCamelCase__      )      )
 results.update(is_config_or_test(UpperCamelCase__      )      )
 results.update(has_no_keywords(UpperCamelCase__      )      )
 results.update(has_few_assignments(UpperCamelCase__      )      )
 return results


def       _lowercase							(					UpperCamelCase__							:       Optional[Any],					UpperCamelCase__							:       Optional[Any],					UpperCamelCase__							:       List[str]      ):
 if not check_uniques(UpperCamelCase__,					UpperCamelCase__      ):
  return False
 elif example["autogenerated"]:
  return False
 elif example["line_max"] > args.line_max:
  return False
 elif example["line_mean"] > args.line_mean:
  return False
 elif example["alpha_frac"] < args.alpha_frac:
  return False
 elif example["ratio"] < args.min_token_ratio:
  return False
 elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
  return False
 elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
  return False
 elif example["has_few_assignments"]:
  return False
 else:
  return True


def       _lowercase							(					UpperCamelCase__							:       str      ):
 with open(UpperCamelCase__,					'rb'      ) as f_in:
  with gzip.open(str(UpperCamelCase__      ) + '.gz',					'wb',					compresslevel=6      ) as f_out:
   shutil.copyfileobj(UpperCamelCase__,					UpperCamelCase__      )
 os.unlink(UpperCamelCase__      )


# Settings
UpperCAmelCase_			:  str										=   HfArgumentParser(PreprocessingArguments)
UpperCAmelCase_			:  Any										=   parser.parse_args()
if args.num_workers is None:
 UpperCAmelCase_			:  List[str]										=   multiprocessing.cpu_count()
UpperCAmelCase_			:  int										=   AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
UpperCAmelCase_			:  List[str]										=   time.time()
UpperCAmelCase_			:  Union[str, Any]										=   load_dataset(args.dataset_name, split='train')
print(f'''Time to load dataset: {time.time()-t_start:.2f}''')

# Run preprocessing
UpperCAmelCase_			:  str										=   time.time()
UpperCAmelCase_			:  Optional[Any]										=   ds.map(preprocess, num_proc=args.num_workers)
print(f'''Time to preprocess dataset: {time.time()-t_start:.2f}''')

# Deduplicate hashes
UpperCAmelCase_			:  Any										=   set(ds.unique('hash'))
UpperCAmelCase_			:  List[Any]										=   len(uniques) / len(ds)
print(f'''Fraction of duplicates: {1-frac:.2%}''')

# Deduplicate data and apply heuristics
UpperCAmelCase_			:  Tuple										=   time.time()
UpperCAmelCase_			:  Union[str, Any]										=   ds.filter(filter, fn_kwargs={'uniques': uniques, 'args': args})
print(f'''Time to filter dataset: {time.time()-t_start:.2f}''')
print(f'''Size of filtered dataset: {len(ds_filter)}''')

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
 UpperCAmelCase_			:  str										=   time.time()
 UpperCAmelCase_				,      UpperCAmelCase_			:  Any										=   deduplicate_dataset(ds_filter, args.jaccard_threshold)
 print(f'''Time to deduplicate dataset: {time.time()-t_start:.2f}''')
 print(f'''Size of deduplicate dataset: {len(ds_filter)}''')

# Save data in batches of samples_per_file
UpperCAmelCase_			:  List[Any]										=   Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
 with open(output_dir / 'duplicate_clusters.json', 'w') as f:
  json.dump(duplicate_clusters, f)

UpperCAmelCase_			:  List[str]										=   output_dir / 'data'
data_dir.mkdir(exist_ok=True)

UpperCAmelCase_			:  Tuple										=   time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
 UpperCAmelCase_			:  int										=   str(data_dir / f'''file-{file_number+1:012}.json''')
 UpperCAmelCase_			:  List[str]										=   min(len(ds_filter), index + args.samples_per_file)
 ds_filter.select(list(range(index, end_index))).to_json(file_path)
 compress_file(file_path)


print(f'''Time to save dataset: {time.time()-t_start:.2f}''')










'''simple docstring'''
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_			:  Any										=   logging.get_logger(__name__)

UpperCAmelCase_			:  Tuple										=   {'vocab_file': 'spm_char.model'}

UpperCAmelCase_			:  int										=   {
    'vocab_file': {
        'microsoft/speecht5_asr': 'https://huggingface.co/microsoft/speecht5_asr/resolve/main/spm_char.model',
        'microsoft/speecht5_tts': 'https://huggingface.co/microsoft/speecht5_tts/resolve/main/spm_char.model',
        'microsoft/speecht5_vc': 'https://huggingface.co/microsoft/speecht5_vc/resolve/main/spm_char.model',
    }
}

UpperCAmelCase_			:  List[Any]										=   {
    'microsoft/speecht5_asr': 1_0_2_4,
    'microsoft/speecht5_tts': 1_0_2_4,
    'microsoft/speecht5_vc': 1_0_2_4,
}

class     _lowerCamelCase       (     snake_case_					):




 '''simple docstring'''
 __lowercase					:		Optional[Any]     				=    VOCAB_FILES_NAMES
 __lowercase					:		int     				=    PRETRAINED_VOCAB_FILES_MAP
 __lowercase					:		int     				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowercase					:		Optional[int]     				=    ['''input_ids''', '''attention_mask''']


 def __init__(   self   ,   __lowercase   ,   __lowercase="<s>"   ,   __lowercase="</s>"   ,   __lowercase="<unk>"   ,   __lowercase="<pad>"   ,   __lowercase = None   ,   **__lowercase   ,   ):

  """simple docstring"""







  __A	:		Dict	  =      {} if sp_model_kwargs is None else sp_model_kwargs

  super().__init__(
      bos_token=__lowercase   ,   eos_token=__lowercase   ,   unk_token=__lowercase   ,   pad_token=__lowercase   ,   sp_model_kwargs=self.sp_model_kwargs   ,   **__lowercase   ,   )

  __A	:		Optional[Any]	  =      vocab_file

  __A	:		Any	  =      spm.SentencePieceProcessor(**self.sp_model_kwargs    )
  self.sp_model.Load(__lowercase    )


 @property
 def 			snake_case__ (   self    ):

  """simple docstring"""







  return self.sp_model.get_piece_size()


 def 			snake_case__ (   self    ):

  """simple docstring"""







  __A	:		Tuple	  =      {self.convert_ids_to_tokens(__lowercase    ): i for i in range(self.vocab_size    )}
  vocab.update(self.added_tokens_encoder    )
  return vocab


 def __getstate__(   self    ):

  """simple docstring"""







  __A	:		List[Any]	  =      self.__dict__.copy()
  __A	:		str	  =      None
  return state


 def __setstate__(   self   ,   __lowercase    ):

  """simple docstring"""







  __A	:		List[str]	  =      d

  # for backward compatibility
  if not hasattr(self   ,   'sp_model_kwargs'    ):
   __A	:		Any	  =      {}

  __A	:		List[str]	  =      spm.SentencePieceProcessor(**self.sp_model_kwargs    )
  self.sp_model.Load(self.vocab_file    )


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  return self.sp_model.encode(__lowercase   ,   out_type=__lowercase    )


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  return self.sp_model.piece_to_id(__lowercase    )


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  __A	:		Optional[Any]	  =      self.sp_model.IdToPiece(__lowercase    )
  return token


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  __A	:		str	  =      []
  __A	:		str	  =      ''
  for token in tokens:
   # make sure that special tokens are not decoded using sentencepiece model
   if token in self.all_special_tokens:
    out_string += self.sp_model.decode(__lowercase    ) + token
    __A	:		Any	  =      []
   else:
    current_sub_tokens.append(__lowercase    )
  out_string += self.sp_model.decode(__lowercase    )
  return out_string.strip()


 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase=None    ):

  """simple docstring"""







  if token_ids_a is None:
   return token_ids_a + [self.eos_token_id]
  # We don't expect to process pairs, but leave the pair logic for API consistency
  return token_ids_a + token_ids_a + [self.eos_token_id]


 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase = None   ,   __lowercase = False    ):

  """simple docstring"""







  if already_has_special_tokens:
   return super().get_special_tokens_mask(
       token_ids_a=__lowercase   ,   token_ids_a=__lowercase   ,   already_has_special_tokens=__lowercase    )

  __A	:		Dict	  =      [1]
  if token_ids_a is None:
   return ([0] * len(__lowercase    )) + suffix_ones
  return ([0] * len(__lowercase    )) + ([0] * len(__lowercase    )) + suffix_ones







 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase = None    ):

  """simple docstring"""







  if not os.path.isdir(__lowercase    ):
   logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
   return
  __A	:		Optional[int]	  =      os.path.join(
      __lowercase   ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )

  if os.path.abspath(self.vocab_file    ) != os.path.abspath(__lowercase    ) and os.path.isfile(self.vocab_file    ):
   copyfile(self.vocab_file   ,   __lowercase    )
  elif not os.path.isfile(self.vocab_file    ):
   with open(__lowercase   ,   'wb'    ) as fi:
    __A	:		Tuple	  =      self.sp_model.serialized_model_proto()
    fi.write(__lowercase    )

  return (out_vocab_file,)






