


import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


_SCREAMING_SNAKE_CASE     :			Any        =							re.compile(R'\s+')


def 							__lowerCAmelCase      (			__magic_name__      ):
 return {"hash": hashlib.mda(re.sub(lowerCAmelCase__ ,   "" ,   example["content"]      ).encode("utf-8"      )      ).hexdigest()}


def 							__lowerCAmelCase      (			__magic_name__      ):
 _lowercase:       str						=				[len(lowerCAmelCase__      ) for line in example["content"].splitlines()]
 return {"line_mean": np.mean(lowerCAmelCase__      ), "line_max": max(lowerCAmelCase__      )}


def 							__lowerCAmelCase      (			__magic_name__      ):
 _lowercase:       Union[str, Any]						=				np.mean([c.isalnum() for c in example["content"]]      )
 return {"alpha_frac": alpha_frac}


def 							__lowerCAmelCase      (			__magic_name__ ,   __magic_name__      ):
 if example["hash"] in uniques:
  uniques.remove(example["hash"]      )
  return True
 else:
  return False


def 							__lowerCAmelCase      (			__magic_name__ ,   __magic_name__=5      ):
 _lowercase:       Optional[Any]						=				["auto-generated", "autogenerated", "automatically generated"]
 _lowercase:       int						=				example["content"].splitlines()
 for _, line in zip(range(lowerCAmelCase__      ) ,   lowerCAmelCase__      ):
  for keyword in keywords:
   if keyword in line.lower():
    return {"autogenerated": True}
 else:
  return {"autogenerated": False}


def 							__lowerCAmelCase      (			__magic_name__ ,   __magic_name__=5 ,   __magic_name__=0.05      ):
 _lowercase:       Any						=				["unit tests", "test file", "configuration file"]
 _lowercase:       List[Any]						=				example["content"].splitlines()
 _lowercase:       int						=				0
 _lowercase:       List[Any]						=				0
 # first test
 for _, line in zip(range(lowerCAmelCase__      ) ,   lowerCAmelCase__      ):
  for keyword in keywords:
   if keyword in line.lower():
    return {"config_or_test": True}
    # second test
 _lowercase:       Tuple						=				example["content"].count("\n"      )
 _lowercase:       List[Any]						=				int(coeff * nlines      )
 for line in lines:
  count_config += line.lower().count("config"      )
  count_test += line.lower().count("test"      )
  if count_config > threshold or count_test > threshold:
   return {"config_or_test": True}
 return {"config_or_test": False}


def 							__lowerCAmelCase      (			__magic_name__      ):
 _lowercase:       Optional[Any]						=				["def ", "class ", "for ", "while "]
 _lowercase:       Any						=				example["content"].splitlines()
 for line in lines:
  for keyword in keywords:
   if keyword in line.lower():
    return {"has_no_keywords": False}
 return {"has_no_keywords": True}


def 							__lowerCAmelCase      (			__magic_name__ ,   __magic_name__=4      ):
 _lowercase:       Union[str, Any]						=				example["content"].splitlines()
 _lowercase:       int						=				0
 for line in lines:
  counter += line.lower().count("="      )
  if counter > minimum:
   return {"has_few_assignments": False}
 return {"has_few_assignments": True}


def 							__lowerCAmelCase      (			__magic_name__      ):
 _lowercase:       List[str]						=				tokenizer(example["content"] ,   truncation=lowerCAmelCase__      )["input_ids"]
 _lowercase:       int						=				len(example["content"]      ) / len(lowerCAmelCase__      )
 return {"ratio": ratio}


def 							__lowerCAmelCase      (			__magic_name__      ):
 _lowercase:       List[str]						=				{}
 results.update(get_hash(lowerCAmelCase__      )      )
 results.update(line_stats(lowerCAmelCase__      )      )
 results.update(alpha_stats(lowerCAmelCase__      )      )
 results.update(char_token_ratio(lowerCAmelCase__      )      )
 results.update(is_autogenerated(lowerCAmelCase__      )      )
 results.update(is_config_or_test(lowerCAmelCase__      )      )
 results.update(has_no_keywords(lowerCAmelCase__      )      )
 results.update(has_few_assignments(lowerCAmelCase__      )      )
 return results


def 							__lowerCAmelCase      (			__magic_name__ ,   __magic_name__ ,   __magic_name__      ):
 if not check_uniques(lowerCAmelCase__ ,   lowerCAmelCase__      ):
  return False
 elif example["autogenerated"]:
  return False
 elif example["line_max"] > args.line_max:
  return False
 elif example["line_mean"] > args.line_mean:
  return False
 elif example["alpha_frac"] < args.alpha_frac:
  return False
 elif example["ratio"] < args.min_token_ratio:
  return False
 elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
  return False
 elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
  return False
 elif example["has_few_assignments"]:
  return False
 else:
  return True


def 							__lowerCAmelCase      (			__magic_name__      ):
 with open(lowerCAmelCase__ ,   "rb"      ) as f_in:
  with gzip.open(str(lowerCAmelCase__      ) + ".gz" ,   "wb" ,   compresslevel=6      ) as f_out:
   shutil.copyfileobj(lowerCAmelCase__ ,   lowerCAmelCase__      )
 os.unlink(lowerCAmelCase__      )


# Settings
_SCREAMING_SNAKE_CASE     :			Any        =							HfArgumentParser(PreprocessingArguments)
_SCREAMING_SNAKE_CASE     :			Dict        =							parser.parse_args()
if args.num_workers is None:
       _SCREAMING_SNAKE_CASE     :			str        =							multiprocessing.cpu_count()
_SCREAMING_SNAKE_CASE     :			Any        =							AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
_SCREAMING_SNAKE_CASE     :			Tuple        =							time.time()
_SCREAMING_SNAKE_CASE     :			str        =							load_dataset(args.dataset_name, split='train')
print(f'''Time to load dataset: {time.time()-t_start:.2f}''')

# Run preprocessing
_SCREAMING_SNAKE_CASE     :			Union[str, Any]        =							time.time()
_SCREAMING_SNAKE_CASE     :			Union[str, Any]        =							ds.map(preprocess, num_proc=args.num_workers)
print(f'''Time to preprocess dataset: {time.time()-t_start:.2f}''')

# Deduplicate hashes
_SCREAMING_SNAKE_CASE     :			Tuple        =							set(ds.unique('hash'))
_SCREAMING_SNAKE_CASE     :			str        =							len(uniques) / len(ds)
print(f'''Fraction of duplicates: {1-frac:.2%}''')

# Deduplicate data and apply heuristics
_SCREAMING_SNAKE_CASE     :			Union[str, Any]        =							time.time()
_SCREAMING_SNAKE_CASE     :			List[Any]        =							ds.filter(filter, fn_kwargs={'uniques': uniques, 'args': args})
print(f'''Time to filter dataset: {time.time()-t_start:.2f}''')
print(f'''Size of filtered dataset: {len(ds_filter)}''')

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
       _SCREAMING_SNAKE_CASE     :			Union[str, Any]        =							time.time()
       _SCREAMING_SNAKE_CASE ,      _SCREAMING_SNAKE_CASE     :			Any        =							deduplicate_dataset(ds_filter, args.jaccard_threshold)
       print(f'''Time to deduplicate dataset: {time.time()-t_start:.2f}''')
       print(f'''Size of deduplicate dataset: {len(ds_filter)}''')

# Save data in batches of samples_per_file
_SCREAMING_SNAKE_CASE     :			Dict        =							Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
       with open(output_dir / 'duplicate_clusters.json', 'w') as f:
              json.dump(duplicate_clusters, f)

_SCREAMING_SNAKE_CASE     :			Tuple        =							output_dir / 'data'
data_dir.mkdir(exist_ok=True)

_SCREAMING_SNAKE_CASE     :			Tuple        =							time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
       _SCREAMING_SNAKE_CASE     :			List[Any]        =							str(data_dir / f'''file-{file_number+1:012}.json''')
       _SCREAMING_SNAKE_CASE     :			Optional[int]        =							min(len(ds_filter), index + args.samples_per_file)
       ds_filter.select(list(range(index, end_index))).to_json(file_path)
       compress_file(file_path)





print(f'''Time to save dataset: {time.time()-t_start:.2f}''')



"""simple docstring"""


import copy
import tempfile
import unittest

from huggingface_hub import HfFolder, delete_repo
from parameterized import parameterized
from requests.exceptions import HTTPError

from transformers import AutoConfig, GenerationConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test




class  __lowerCamelCase  ( unittest.TestCase			):




				@parameterized.expand([(None,), ('''foo.json''',)]   )
				def        UpperCAmelCase__						(				self							,						UpperCAmelCase   ):
							lowerCamelCase_					=  GenerationConfig(
							    do_sample=UpperCAmelCase							,						temperature=0.7							,						length_penalty=1.0							,						bad_words_ids=[[1, 2, 3], [4, 5]]							,						)
							with tempfile.TemporaryDirectory() as tmp_dir:
										config.save_pretrained(UpperCAmelCase							,						config_name=UpperCAmelCase   )
										lowerCamelCase_					=  GenerationConfig.from_pretrained(UpperCAmelCase							,						config_name=UpperCAmelCase   )

							# Checks parameters that were specified
							self.assertEqual(loaded_config.do_sample							,						UpperCAmelCase   )
							self.assertEqual(loaded_config.temperature							,						0.7   )
							self.assertEqual(loaded_config.length_penalty							,						1.0   )
							self.assertEqual(loaded_config.bad_words_ids							,						[[1, 2, 3], [4, 5]]   )

							# Checks parameters that were not specified (defaults)
							self.assertEqual(loaded_config.top_k							,						50   )
							self.assertEqual(loaded_config.max_length							,						20   )
							self.assertEqual(loaded_config.max_time							,						UpperCAmelCase   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  AutoConfig.from_pretrained('''gpt2'''   )
							lowerCamelCase_					=  GenerationConfig.from_model_config(UpperCAmelCase   )
							lowerCamelCase_					=  GenerationConfig()

							# The generation config has loaded a few non-default parameters from the model config
							self.assertNotEqual(UpperCAmelCase							,						UpperCAmelCase   )

							# One of those parameters is eos_token_id -- check if it matches
							self.assertNotEqual(generation_config_from_model.eos_token_id							,						default_generation_config.eos_token_id   )
							self.assertEqual(generation_config_from_model.eos_token_id							,						model_config.eos_token_id   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  GenerationConfig()
							lowerCamelCase_					=  {
							    '''max_new_tokens''': 1024,
							    '''foo''': '''bar''',
							}
							lowerCamelCase_					=  copy.deepcopy(UpperCAmelCase   )
							lowerCamelCase_					=  generation_config.update(**UpperCAmelCase   )

							# update_kwargs was not modified (no side effects)
							self.assertEqual(UpperCAmelCase							,						UpperCAmelCase   )

							# update_kwargs was used to update the config on valid attributes
							self.assertEqual(generation_config.max_new_tokens							,						1024   )

							# `.update()` returns a dictionary of unused kwargs
							self.assertEqual(UpperCAmelCase							,						{'''foo''': '''bar'''}   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  GenerationConfig()
							lowerCamelCase_					=  '''bar'''

							with tempfile.TemporaryDirectory('''test-generation-config'''   ) as tmp_dir:
										generation_config.save_pretrained(UpperCAmelCase   )

										lowerCamelCase_					=  GenerationConfig.from_pretrained(UpperCAmelCase   )
							# update_kwargs was used to update the config on valid attributes
							self.assertEqual(new_config.foo							,						'''bar'''   )

							lowerCamelCase_					=  GenerationConfig.from_model_config(UpperCAmelCase   )
							assert not hasattr(UpperCAmelCase							,						'''foo'''   )  # no new kwargs should be initialized if from config





				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  GenerationConfig()
							self.assertEqual(default_config.temperature							,						1.0   )
							self.assertEqual(default_config.do_sample							,						UpperCAmelCase   )
							self.assertEqual(default_config.num_beams							,						1   )

							lowerCamelCase_					=  GenerationConfig(
							    do_sample=UpperCAmelCase							,						temperature=0.7							,						length_penalty=1.0							,						bad_words_ids=[[1, 2, 3], [4, 5]]							,						)
							self.assertEqual(config.temperature							,						0.7   )
							self.assertEqual(config.do_sample							,						UpperCAmelCase   )
							self.assertEqual(config.num_beams							,						1   )

							with tempfile.TemporaryDirectory() as tmp_dir:
										config.save_pretrained(UpperCAmelCase   )
										lowerCamelCase_					=  GenerationConfig.from_pretrained(UpperCAmelCase							,						temperature=1.0   )

							self.assertEqual(loaded_config.temperature							,						1.0   )
							self.assertEqual(loaded_config.do_sample							,						UpperCAmelCase   )
							self.assertEqual(loaded_config.num_beams							,						1   )  # default value




@is_staging_test
class  __lowerCamelCase  ( unittest.TestCase			):




				@classmethod
				def        UpperCAmelCase__						(				cls   ):
							lowerCamelCase_					=  TOKEN
							HfFolder.save_token(UpperCAmelCase   )




				@classmethod
				def        UpperCAmelCase__						(				cls   ):
							try:
										delete_repo(token=cls._token							,						repo_id='''test-generation-config'''   )
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token							,						repo_id='''valid_org/test-generation-config-org'''   )
							except HTTPError:
										pass




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  GenerationConfig(
							    do_sample=UpperCAmelCase							,						temperature=0.7							,						length_penalty=1.0							,						)
							config.push_to_hub('''test-generation-config'''							,						use_auth_token=self._token   )

							lowerCamelCase_					=  GenerationConfig.from_pretrained(f"{USER}/test-generation-config"   )
							for k, v in config.to_dict().items():
										if k != "transformers_version":
													self.assertEqual(UpperCAmelCase							,						getattr(UpperCAmelCase							,						UpperCAmelCase   )   )

        # Reset repo
							delete_repo(token=self._token							,						repo_id='''test-generation-config'''   )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
										config.save_pretrained(
										    UpperCAmelCase							,						repo_id='''test-generation-config'''							,						push_to_hub=UpperCAmelCase							,						use_auth_token=self._token   )

							lowerCamelCase_					=  GenerationConfig.from_pretrained(f"{USER}/test-generation-config"   )
							for k, v in config.to_dict().items():
										if k != "transformers_version":
													self.assertEqual(UpperCAmelCase							,						getattr(UpperCAmelCase							,						UpperCAmelCase   )   )





				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  GenerationConfig(
							    do_sample=UpperCAmelCase							,						temperature=0.7							,						length_penalty=1.0							,						)
							config.push_to_hub('''valid_org/test-generation-config-org'''							,						use_auth_token=self._token   )

							lowerCamelCase_					=  GenerationConfig.from_pretrained('''valid_org/test-generation-config-org'''   )
							for k, v in config.to_dict().items():
										if k != "transformers_version":
													self.assertEqual(UpperCAmelCase							,						getattr(UpperCAmelCase							,						UpperCAmelCase   )   )

        # Reset repo
							delete_repo(token=self._token							,						repo_id='''valid_org/test-generation-config-org'''   )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
										config.save_pretrained(
										    UpperCAmelCase							,						repo_id='''valid_org/test-generation-config-org'''							,						push_to_hub=UpperCAmelCase							,						use_auth_token=self._token   )

							lowerCamelCase_					=  GenerationConfig.from_pretrained('''valid_org/test-generation-config-org'''   )
							for k, v in config.to_dict().items():
										if k != "transformers_version":
													self.assertEqual(UpperCAmelCase							,						getattr(UpperCAmelCase							,						UpperCAmelCase   )   )



