






'''simple docstring'''





import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


__A							:							int  							=		re.compile(R"""\s+""")






def    lowerCamelCase_     (    lowercase__):
	return {"hash": hashlib.mda(re.sub(lowercase__				,     ""				,     example["content"]).encode("utf-8")).hexdigest()}






def    lowerCamelCase_     (    lowercase__):
	lowerCamelCase__  =     [len(lowercase__) for line in example["content"].splitlines()]
	return {"line_mean": np.mean(lowercase__), "line_max": max(lowercase__)}






def    lowerCamelCase_     (    lowercase__):
	lowerCamelCase__  =     np.mean([c.isalnum() for c in example["content"]])
	return {"alpha_frac": alpha_frac}






def    lowerCamelCase_     (    lowercase__				,     lowercase__):
	if example["hash"] in uniques:
		uniques.remove(example["hash"])
		return True
	else:
		return False






def    lowerCamelCase_     (    lowercase__				,     lowercase__=5):
	lowerCamelCase__  =     ["auto-generated", "autogenerated", "automatically generated"]
	lowerCamelCase__  =     example["content"].splitlines()
	for _, line in zip(range(lowercase__)				,     lowercase__):
		for keyword in keywords:
			if keyword in line.lower():
				return {"autogenerated": True}
	else:
		return {"autogenerated": False}






def    lowerCamelCase_     (    lowercase__				,     lowercase__=5				,     lowercase__=0.05):
	lowerCamelCase__  =     ["unit tests", "test file", "configuration file"]
	lowerCamelCase__  =     example["content"].splitlines()
	lowerCamelCase__  =     0
	lowerCamelCase__  =     0
	# first test
	for _, line in zip(range(lowercase__)				,     lowercase__):
		for keyword in keywords:
			if keyword in line.lower():
				return {"config_or_test": True}
				# second test
	lowerCamelCase__  =     example["content"].count("\n")
	lowerCamelCase__  =     int(coeff * nlines)
	for line in lines:
		count_config += line.lower().count("config")
		count_test += line.lower().count("test")
		if count_config > threshold or count_test > threshold:
			return {"config_or_test": True}
	return {"config_or_test": False}






def    lowerCamelCase_     (    lowercase__):
	lowerCamelCase__  =     ["def ", "class ", "for ", "while "]
	lowerCamelCase__  =     example["content"].splitlines()
	for line in lines:
		for keyword in keywords:
			if keyword in line.lower():
				return {"has_no_keywords": False}
	return {"has_no_keywords": True}






def    lowerCamelCase_     (    lowercase__				,     lowercase__=4):
	lowerCamelCase__  =     example["content"].splitlines()
	lowerCamelCase__  =     0
	for line in lines:
		counter += line.lower().count("=")
		if counter > minimum:
			return {"has_few_assignments": False}
	return {"has_few_assignments": True}






def    lowerCamelCase_     (    lowercase__):
	lowerCamelCase__  =     tokenizer(example["content"]				,     truncation=lowercase__)["input_ids"]
	lowerCamelCase__  =     len(example["content"]) / len(lowercase__)
	return {"ratio": ratio}






def    lowerCamelCase_     (    lowercase__):
	lowerCamelCase__  =     {}
	results.update(get_hash(lowercase__))
	results.update(line_stats(lowercase__))
	results.update(alpha_stats(lowercase__))
	results.update(char_token_ratio(lowercase__))
	results.update(is_autogenerated(lowercase__))
	results.update(is_config_or_test(lowercase__))
	results.update(has_no_keywords(lowercase__))
	results.update(has_few_assignments(lowercase__))
	return results






def    lowerCamelCase_     (    lowercase__				,     lowercase__				,     lowercase__):
	if not check_uniques(lowercase__				,     lowercase__):
		return False
	elif example["autogenerated"]:
		return False
	elif example["line_max"] > args.line_max:
		return False
	elif example["line_mean"] > args.line_mean:
		return False
	elif example["alpha_frac"] < args.alpha_frac:
		return False
	elif example["ratio"] < args.min_token_ratio:
		return False
	elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
		return False
	elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
		return False
	elif example["has_few_assignments"]:
		return False
	else:
		return True






def    lowerCamelCase_     (    lowercase__):
	with open(lowercase__				,     "rb") as f_in:
		with gzip.open(str(lowercase__) + ".gz"				,     "wb"				,     compresslevel=6) as f_out:
			shutil.copyfileobj(lowercase__				,     lowercase__)
	os.unlink(lowercase__)


# Settings
__A							:							List[Any]  							=		HfArgumentParser(PreprocessingArguments)
__A							:							Dict  							=		parser.parse_args()
if args.num_workers is None:
	__A							:							List[str]  							=		multiprocessing.cpu_count()
__A							:							List[str]  							=		AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
__A							:							List[str]  							=		time.time()
__A							:							Union[str, Any]  							=		load_dataset(args.dataset_name, split="""train""")
print(F"""Time to load dataset: {time.time()-t_start:.2f}""")

# Run preprocessing
__A							:							Any  							=		time.time()
__A							:							List[Any]  							=		ds.map(preprocess, num_proc=args.num_workers)
print(F"""Time to preprocess dataset: {time.time()-t_start:.2f}""")

# Deduplicate hashes
__A							:							Union[str, Any]  							=		set(ds.unique("""hash"""))
__A							:							Optional[Any]  							=		len(uniques) / len(ds)
print(F"""Fraction of duplicates: {1-frac:.2%}""")

# Deduplicate data and apply heuristics
__A							:							Any  							=		time.time()
__A							:							Tuple  							=		ds.filter(filter, fn_kwargs={"""uniques""": uniques, """args""": args})
print(F"""Time to filter dataset: {time.time()-t_start:.2f}""")
print(F"""Size of filtered dataset: {len(ds_filter)}""")

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
	__A							:							int  							=		time.time()
	__A							:							str  							=		deduplicate_dataset(ds_filter, args.jaccard_threshold)
	print(F"""Time to deduplicate dataset: {time.time()-t_start:.2f}""")
	print(F"""Size of deduplicate dataset: {len(ds_filter)}""")

# Save data in batches of samples_per_file
__A							:							Optional[int]  							=		Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
	with open(output_dir / """duplicate_clusters.json""", """w""") as f:
		json.dump(duplicate_clusters, f)

__A							:							Dict  							=		output_dir / """data"""
data_dir.mkdir(exist_ok=True)

__A							:							Dict  							=		time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
	__A							:							Optional[Any]  							=		str(data_dir / F"""file-{file_number+1:012}.json""")
	__A							:							int  							=		min(len(ds_filter), index + args.samples_per_file)
	ds_filter.select(list(range(index, end_index))).to_json(file_path)
	compress_file(file_path)
print(F"""Time to save dataset: {time.time()-t_start:.2f}""")









'''simple docstring'''





from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast
from ...utils import logging


__A							:							Union[str, Any]  							=		logging.get_logger(__name__)

__A							:							Tuple  							=		{
    """EleutherAI/gpt-neo-1.3B""": """https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/config.json""",
    # See all GPTNeo models at https://huggingface.co/models?filter=gpt_neo
}
class 				lowercase       (							_lowerCamelCase      ):


    '''simple docstring'''

    lowerCAmelCase__      				=							"gpt_neo"
    lowerCAmelCase__      				=							["past_key_values"]
    lowerCAmelCase__      				=							{"num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}

    def __init__(					self     :     str				,     __lowerCamelCase     :     Optional[int]=50257				,     __lowerCamelCase     :     Optional[int]=2048				,     __lowerCamelCase     :     Optional[int]=2048				,     __lowerCamelCase     :     Any=24				,     __lowerCamelCase     :     List[Any]=[[["global", "local"], 12]]				,     __lowerCamelCase     :     Any=16				,     __lowerCamelCase     :     Any=None				,     __lowerCamelCase     :     Optional[Any]=256				,     __lowerCamelCase     :     List[Any]="gelu_new"				,     __lowerCamelCase     :     Any=0.0				,     __lowerCamelCase     :     List[Any]=0.0				,     __lowerCamelCase     :     Union[str, Any]=0.0				,     __lowerCamelCase     :     Any=0.1				,     __lowerCamelCase     :     Dict=1E-5				,     __lowerCamelCase     :     List[Any]=0.0_2				,     __lowerCamelCase     :     List[Any]=True				,     __lowerCamelCase     :     Optional[int]=50256				,     __lowerCamelCase     :     Tuple=50256				,     **__lowerCamelCase     :     Optional[int]				,     )							->							str:

     '''simple docstring'''




     lowerCamelCase__  =     vocab_size
     lowerCamelCase__  =     max_position_embeddings
     lowerCamelCase__  =     hidden_size
     lowerCamelCase__  =     num_layers
     lowerCamelCase__  =     num_heads
     lowerCamelCase__  =     intermediate_size
     lowerCamelCase__  =     window_size
     lowerCamelCase__  =     activation_function
     lowerCamelCase__  =     resid_dropout
     lowerCamelCase__  =     embed_dropout
     lowerCamelCase__  =     attention_dropout
     lowerCamelCase__  =     classifier_dropout
     lowerCamelCase__  =     layer_norm_epsilon
     lowerCamelCase__  =     initializer_range
     lowerCamelCase__  =     use_cache

     lowerCamelCase__  =     bos_token_id
     lowerCamelCase__  =     eos_token_id

     lowerCamelCase__  =     attention_types
     lowerCamelCase__  =     self.expand_attention_types_params(__lowerCamelCase     )

     if len(self.attention_layers     ) != self.num_layers:
      raise ValueError(
          "Configuration for convolutional module is incorrect. "
          "It is required that `len(config.attention_layers)` == `config.num_layers` "
          f'''but is `len(config.attention_layers) = {len(self.attention_layers     )}`, '''
          f'''`config.num_layers = {self.num_layers}`. '''
          "`config.attention_layers` is prepared using `config.attention_types`. "
          "Please verify the value of `config.attention_types` argument."     )

     super().__init__(bos_token_id=__lowerCamelCase				,     eos_token_id=__lowerCamelCase				,     **__lowerCamelCase     )




    @staticmethod
    def    a__				(					__lowerCamelCase     :     int     )							->							str:

     '''simple docstring'''




     lowerCamelCase__  =     []
     for item in attention_types:
      for _ in range(item[1]     ):
       attentions.extend(item[0]     )
     return attentions






def    lowerCamelCase_     (    lowercase__				,     lowercase__				,     lowercase__				,     lowercase__):
 import torch

 lowerCamelCase__  =     input.size()
 lowerCamelCase__  =     len(lowercase__)
 lowerCamelCase__  =     shape[dimension]

 lowerCamelCase__  =     torch.arange(0				,     lowercase__				,     lowercase__)
 lowerCamelCase__  =     torch.div(sizedim - size				,     lowercase__				,     rounding_mode="floor") + 1
 lowerCamelCase__  =     torch.arange(lowercase__) + low_indices[:min_length][:, None]

 lowerCamelCase__  =     [slice(lowercase__)] * rank
 lowerCamelCase__  =     indices
 lowerCamelCase__  =     input[s]

 lowerCamelCase__  =     list(range(0				,     rank + 1))
 perm.append(perm.pop(dimension + 1))

 return sliced.permute(lowercase__)






def    lowerCamelCase_     (    lowercase__				,     lowercase__):
 import torch

 lowerCamelCase__  =     torch.arange(1				,     lowercase__)
 lowerCamelCase__  =     torch.remainder(lowercase__				,     lowercase__)
 lowerCamelCase__  =     remainders == 0
 lowerCamelCase__  =     candidates[divisor_indices]
 lowerCamelCase__  =     torch.max(lowercase__)
 return largest_divisor, torch.div(lowercase__				,     lowercase__				,     rounding_mode="floor")
class 				lowercase       (							_lowerCamelCase      ):


    '''simple docstring'''

    @property
    def    a__				(					self     :     Any     )							->							Mapping[str, Mapping[int, str]]:

     '''simple docstring'''




     lowerCamelCase__  =     OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}     )
     if self.use_past:
      self.fill_with_past_key_values_(__lowerCamelCase				,     direction="inputs"     )
      lowerCamelCase__  =     {0: "batch", 1: "past_sequence + sequence"}
     else:
      lowerCamelCase__  =     {0: "batch", 1: "sequence"}

     return common_inputs

    @property
    def    a__				(					self     :     Any     )							->							int:

     '''simple docstring'''




     return self._config.num_heads

    def    a__				(					self     :     Optional[Any]				,     __lowerCamelCase     :     PreTrainedTokenizer				,     __lowerCamelCase     :     int = -1				,     __lowerCamelCase     :     int = -1				,     __lowerCamelCase     :     bool = False				,     __lowerCamelCase     :     Optional[TensorType] = None				,     )							->							Mapping[str, Any]:

     '''simple docstring'''




     lowerCamelCase__  =     super(__lowerCamelCase				,     self     ).generate_dummy_inputs(
         __lowerCamelCase				,     batch_size=__lowerCamelCase				,     seq_length=__lowerCamelCase				,     is_pair=__lowerCamelCase				,     framework=__lowerCamelCase     )

     # We need to order the input in the way they appears in the forward()
     lowerCamelCase__  =     OrderedDict({"input_ids": common_inputs["input_ids"]}     )

     # Need to add the past_keys
     if self.use_past:
      if not is_torch_available():
       raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."     )
      else:
       import torch

       lowerCamelCase__  ,	lowerCamelCase__  =     common_inputs["input_ids"].shape
       # Not using the same length for past_key_values
       lowerCamelCase__  =     seqlen + 2
       lowerCamelCase__  =     (
           batch,
           self.num_attention_heads,
           past_key_values_length,
           self._config.hidden_size // self.num_attention_heads,
       )
       lowerCamelCase__  =     [
           (torch.zeros(__lowerCamelCase     ), torch.zeros(__lowerCamelCase     )) for _ in range(self.num_layers     )
       ]

     lowerCamelCase__  =     common_inputs["attention_mask"]
     if self.use_past:
      lowerCamelCase__  =     ordered_inputs["attention_mask"].dtype
      lowerCamelCase__  =     torch.cat(
          [ordered_inputs["attention_mask"], torch.ones(__lowerCamelCase				,     __lowerCamelCase				,     dtype=__lowerCamelCase     )]				,     dim=1     )

     return ordered_inputs




    @property
    def    a__				(					self     :     Optional[Any]     )							->							int:

     '''simple docstring'''




     return 13

