
"""simple docstring"""





from typing import List, Optional, Union

import numpy as np
import torch
import torchaudio.compliance.kaldi as ta_kaldi

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


_a								=						logging.get_logger(__name__)






class        _UpperCAmelCase(       __snake_case      ):
		lowercase__         = ["""input_features""", """attention_mask"""]



		def __init__(   self							,						__a=80							,						__a=1_60_00							,						__a=80							,						__a=0.0							,						__a=True							,						__a=True							,						__a=True							,						**__a							,						)			->			Any:






							'''simple docstring'''
							super().__init__(feature_size=__SCREAMING_SNAKE_CASE							,						sampling_rate=__SCREAMING_SNAKE_CASE							,						padding_value=__SCREAMING_SNAKE_CASE							,						**__SCREAMING_SNAKE_CASE)
							_UpperCamelCase									=    num_mel_bins
							_UpperCamelCase									=    do_ceptral_normalize
							_UpperCamelCase									=    normalize_means
							_UpperCamelCase									=    normalize_vars
							_UpperCamelCase									=    True



		def 		UpperCAmelCase				(   self							,						__a							,						)			->			np.ndarray:






							'''simple docstring'''
							_UpperCamelCase									=    waveform * (2**15)  # Kaldi compliance: 16-bit signed integers
							_UpperCamelCase									=    torch.from_numpy(__SCREAMING_SNAKE_CASE).unsqueeze(0)
							_UpperCamelCase									=    ta_kaldi.fbank(__SCREAMING_SNAKE_CASE							,						num_mel_bins=self.num_mel_bins							,						sample_frequency=self.sampling_rate)
							return features.numpy()



		@staticmethod
		def 		UpperCAmelCase				(   __a							,						__a							,						__a = True							,						__a = True							,						__a = 0.0							,						)			->			np.ndarray:






							'''simple docstring'''
							# make sure we normalize float32 arrays
							if normalize_means:
												_UpperCamelCase									=    x[:input_length].mean(axis=0)
												_UpperCamelCase									=    np.subtract(__SCREAMING_SNAKE_CASE							,						__SCREAMING_SNAKE_CASE)
							if normalize_vars:
												_UpperCamelCase									=    x[:input_length].std(axis=0)
												_UpperCamelCase									=    np.divide(__SCREAMING_SNAKE_CASE							,						__SCREAMING_SNAKE_CASE)

							if input_length < x.shape[0]:
												_UpperCamelCase									=    padding_value

							# make sure array is in float32
							_UpperCamelCase									=    x.astype(np.floataa)

							return x



		def 		UpperCAmelCase				(   self							,						__a							,						__a = None)			->			List[np.ndarray]:






							'''simple docstring'''
							_UpperCamelCase									=    attention_mask.sum(-1) if attention_mask is not None else [x.shape[0] for x in input_features]
							return [
							    self.utterance_cmvn(__SCREAMING_SNAKE_CASE							,						__SCREAMING_SNAKE_CASE							,						self.normalize_means							,						self.normalize_vars							,						self.padding_value)
							    for x, n in zip(__SCREAMING_SNAKE_CASE							,						__SCREAMING_SNAKE_CASE)
							]




		def __call__(   self							,						__a							,						__a = False							,						__a = None							,						__a = False							,						__a = None							,						__a = None							,						__a = None							,						__a = None							,						**__a							,						)			->			BatchFeature:






							'''simple docstring'''

							if sampling_rate is not None:
												if sampling_rate != self.sampling_rate:
																	raise ValueError(
																	    F'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
																	    F''' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'''
																	    F''' {self.sampling_rate} and not {sampling_rate}.''')
							else:
												logger.warning(
												    '''It is strongly recommended to pass the `sampling_rate` argument to this function. '''
												    '''Failing to do so can result in silent errors that might be hard to debug.''')

							_UpperCamelCase									=    isinstance(__SCREAMING_SNAKE_CASE							,						np.ndarray) and len(raw_speech.shape) > 1
							if is_batched_numpy and len(raw_speech.shape) > 2:
												raise ValueError(F'''Only mono-channel audio is supported for input to {self}''')
							_UpperCamelCase									=    is_batched_numpy or (
							    isinstance(__SCREAMING_SNAKE_CASE							,						(list, tuple)) and (isinstance(raw_speech[0]							,						(np.ndarray, tuple, list)))
							)

							if is_batched:
												_UpperCamelCase									=    [np.asarray(__SCREAMING_SNAKE_CASE							,						dtype=np.floataa) for speech in raw_speech]
							elif not is_batched and not isinstance(__SCREAMING_SNAKE_CASE							,						np.ndarray):
												_UpperCamelCase									=    np.asarray(__SCREAMING_SNAKE_CASE							,						dtype=np.floataa)
							elif isinstance(__SCREAMING_SNAKE_CASE							,						np.ndarray) and raw_speech.dtype is np.dtype(np.floataa):
												_UpperCamelCase									=    raw_speech.astype(np.floataa)

							# always return batch
							if not is_batched:
												_UpperCamelCase									=    [raw_speech]

							# extract fbank features
							_UpperCamelCase									=    [self._extract_fbank_features(__SCREAMING_SNAKE_CASE) for waveform in raw_speech]

							# convert into correct format for padding
							_UpperCamelCase									=    BatchFeature({'''input_features''': features})

							_UpperCamelCase									=    self.pad(
							    __SCREAMING_SNAKE_CASE							,						padding=__SCREAMING_SNAKE_CASE							,						max_length=__SCREAMING_SNAKE_CASE							,						truncation=__SCREAMING_SNAKE_CASE							,						pad_to_multiple_of=__SCREAMING_SNAKE_CASE							,						return_attention_mask=__SCREAMING_SNAKE_CASE							,						**__SCREAMING_SNAKE_CASE							,						)

							# make sure list is in array format
							_UpperCamelCase									=    padded_inputs.get('''input_features''')
							if isinstance(input_features[0]							,						__SCREAMING_SNAKE_CASE):
												_UpperCamelCase									=    [np.asarray(__SCREAMING_SNAKE_CASE							,						dtype=np.floataa) for feature in input_features]

							_UpperCamelCase									=    padded_inputs.get('''attention_mask''')
							if attention_mask is not None:
												_UpperCamelCase									=    [np.asarray(__SCREAMING_SNAKE_CASE							,						dtype=np.intaa) for array in attention_mask]

							# Utterance-level cepstral mean and variance normalization
							if self.do_ceptral_normalize:
												_UpperCamelCase									=    (
												    np.array(__SCREAMING_SNAKE_CASE							,						dtype=np.intaa)
												    if self._get_padding_strategies(__SCREAMING_SNAKE_CASE							,						max_length=__SCREAMING_SNAKE_CASE) is not PaddingStrategy.DO_NOT_PAD
												    else None
												)
												_UpperCamelCase									=    self.normalize(
												    padded_inputs['''input_features''']							,						attention_mask=__SCREAMING_SNAKE_CASE)

							if return_tensors is not None:
												_UpperCamelCase									=    padded_inputs.convert_to_tensors(__SCREAMING_SNAKE_CASE)

							return padded_inputs














"""simple docstring"""





import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


_SCREAMING_SNAKE_CASE   :							int          =		re.compile(R'''\s+''')







def 	lowerCamelCase__			(				_lowerCamelCase			:						Dict					) ->							List[str]:
  return {"hash": hashlib.mda(re.sub(_lowerCamelCase						,							''						,							example['content']					).encode('utf-8'					)					).hexdigest()}







def 	lowerCamelCase__			(				_lowerCamelCase			:						Tuple					) ->							int:
  lowerCamelCase_       	=		[len(_lowerCamelCase					) for line in example['content'].splitlines()]
  return {"line_mean": np.mean(_lowerCamelCase					), "line_max": max(_lowerCamelCase					)}







def 	lowerCamelCase__			(				_lowerCamelCase			:						List[str]					) ->							int:
  lowerCamelCase_       	=		np.mean([c.isalnum() for c in example['content']]					)
  return {"alpha_frac": alpha_frac}







def 	lowerCamelCase__			(				_lowerCamelCase			:						int						,							_lowerCamelCase			:						Optional[int]					) ->							Optional[Any]:
  if example["hash"] in uniques:
    uniques.remove(example['hash']					)
    return True
  else:
    return False







def 	lowerCamelCase__			(				_lowerCamelCase			:						Any						,							_lowerCamelCase			:						Optional[Any]=5					) ->							int:
  lowerCamelCase_       	=		['auto-generated', 'autogenerated', 'automatically generated']
  lowerCamelCase_       	=		example['content'].splitlines()
  for _, line in zip(range(_lowerCamelCase					)						,							_lowerCamelCase					):
    for keyword in keywords:
      if keyword in line.lower():
        return {"autogenerated": True}
  else:
    return {"autogenerated": False}







def 	lowerCamelCase__			(				_lowerCamelCase			:						Tuple						,							_lowerCamelCase			:						Dict=5						,							_lowerCamelCase			:						List[str]=0.05					) ->							Tuple:
  lowerCamelCase_       	=		['unit tests', 'test file', 'configuration file']
  lowerCamelCase_       	=		example['content'].splitlines()
  lowerCamelCase_       	=		0
  lowerCamelCase_       	=		0
  # first test
  for _, line in zip(range(_lowerCamelCase					)						,							_lowerCamelCase					):
    for keyword in keywords:
      if keyword in line.lower():
        return {"config_or_test": True}
    # second test
  lowerCamelCase_       	=		example['content'].count('\n'					)
  lowerCamelCase_       	=		int(coeff * nlines					)
  for line in lines:
    count_config += line.lower().count('config'					)
    count_test += line.lower().count('test'					)
    if count_config > threshold or count_test > threshold:
      return {"config_or_test": True}
  return {"config_or_test": False}







def 	lowerCamelCase__			(				_lowerCamelCase			:						Any					) ->							List[str]:
  lowerCamelCase_       	=		['def ', 'class ', 'for ', 'while ']
  lowerCamelCase_       	=		example['content'].splitlines()
  for line in lines:
    for keyword in keywords:
      if keyword in line.lower():
        return {"has_no_keywords": False}
  return {"has_no_keywords": True}







def 	lowerCamelCase__			(				_lowerCamelCase			:						Dict						,							_lowerCamelCase			:						Dict=4					) ->							Optional[Any]:
  lowerCamelCase_       	=		example['content'].splitlines()
  lowerCamelCase_       	=		0
  for line in lines:
    counter += line.lower().count('='					)
    if counter > minimum:
      return {"has_few_assignments": False}
  return {"has_few_assignments": True}







def 	lowerCamelCase__			(				_lowerCamelCase			:						Dict					) ->							List[str]:
  lowerCamelCase_       	=		tokenizer(example['content']						,							truncation=_lowerCamelCase					)['input_ids']
  lowerCamelCase_       	=		len(example['content']					) / len(_lowerCamelCase					)
  return {"ratio": ratio}







def 	lowerCamelCase__			(				_lowerCamelCase			:						Optional[Any]					) ->							List[Any]:
  lowerCamelCase_       	=		{}
  results.update(get_hash(_lowerCamelCase					)					)
  results.update(line_stats(_lowerCamelCase					)					)
  results.update(alpha_stats(_lowerCamelCase					)					)
  results.update(char_token_ratio(_lowerCamelCase					)					)
  results.update(is_autogenerated(_lowerCamelCase					)					)
  results.update(is_config_or_test(_lowerCamelCase					)					)
  results.update(has_no_keywords(_lowerCamelCase					)					)
  results.update(has_few_assignments(_lowerCamelCase					)					)
  return results







def 	lowerCamelCase__			(				_lowerCamelCase			:						Any						,							_lowerCamelCase			:						Dict						,							_lowerCamelCase			:						Optional[Any]					) ->							Any:
  if not check_uniques(_lowerCamelCase						,							_lowerCamelCase					):
    return False
  elif example["autogenerated"]:
    return False
  elif example["line_max"] > args.line_max:
    return False
  elif example["line_mean"] > args.line_mean:
    return False
  elif example["alpha_frac"] < args.alpha_frac:
    return False
  elif example["ratio"] < args.min_token_ratio:
    return False
  elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
    return False
  elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
    return False
  elif example["has_few_assignments"]:
    return False
  else:
    return True







def 	lowerCamelCase__			(				_lowerCamelCase			:						str					) ->							int:
  with open(_lowerCamelCase						,							'rb'					) as f_in:
    with gzip.open(str(_lowerCamelCase					) + '.gz'						,							'wb'						,							compresslevel=6					) as f_out:
      shutil.copyfileobj(_lowerCamelCase						,							_lowerCamelCase					)
  os.unlink(_lowerCamelCase					)


# Settings
_SCREAMING_SNAKE_CASE   :							Optional[Any]          =		HfArgumentParser(PreprocessingArguments)
_SCREAMING_SNAKE_CASE   :							Any          =		parser.parse_args()
if args.num_workers is None:
  _SCREAMING_SNAKE_CASE   :							List[str]          =		multiprocessing.cpu_count()
_SCREAMING_SNAKE_CASE   :							str          =		AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
_SCREAMING_SNAKE_CASE   :							Optional[Any]          =		time.time()
_SCREAMING_SNAKE_CASE   :							int          =		load_dataset(args.dataset_name, split='''train''')
print(F'''Time to load dataset: {time.time()-t_start:.2f}''')

# Run preprocessing
_SCREAMING_SNAKE_CASE   :							Optional[Any]          =		time.time()
_SCREAMING_SNAKE_CASE   :							Tuple          =		ds.map(preprocess, num_proc=args.num_workers)
print(F'''Time to preprocess dataset: {time.time()-t_start:.2f}''')

# Deduplicate hashes
_SCREAMING_SNAKE_CASE   :							List[Any]          =		set(ds.unique('''hash'''))
_SCREAMING_SNAKE_CASE   :							Optional[int]          =		len(uniques) / len(ds)
print(F'''Fraction of duplicates: {1-frac:.2%}''')

# Deduplicate data and apply heuristics
_SCREAMING_SNAKE_CASE   :							Dict          =		time.time()
_SCREAMING_SNAKE_CASE   :							Tuple          =		ds.filter(filter, fn_kwargs={'''uniques''': uniques, '''args''': args})
print(F'''Time to filter dataset: {time.time()-t_start:.2f}''')
print(F'''Size of filtered dataset: {len(ds_filter)}''')

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
  _SCREAMING_SNAKE_CASE   :							Optional[Any]          =		time.time()
  _SCREAMING_SNAKE_CASE       ,				_SCREAMING_SNAKE_CASE   :							Union[str, Any]          =		deduplicate_dataset(ds_filter, args.jaccard_threshold)
  print(F'''Time to deduplicate dataset: {time.time()-t_start:.2f}''')
  print(F'''Size of deduplicate dataset: {len(ds_filter)}''')

# Save data in batches of samples_per_file
_SCREAMING_SNAKE_CASE   :							Dict          =		Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
  with open(output_dir / '''duplicate_clusters.json''', '''w''') as f:
    json.dump(duplicate_clusters, f)

_SCREAMING_SNAKE_CASE   :							Optional[Any]          =		output_dir / '''data'''
data_dir.mkdir(exist_ok=True)

_SCREAMING_SNAKE_CASE   :							Optional[Any]          =		time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
  _SCREAMING_SNAKE_CASE   :							Dict          =		str(data_dir / F'''file-{file_number+1:012}.json''')
  _SCREAMING_SNAKE_CASE   :							Any          =		min(len(ds_filter), index + args.samples_per_file)
  ds_filter.select(list(range(index, end_index))).to_json(file_path)
  compress_file(file_path)






print(F'''Time to save dataset: {time.time()-t_start:.2f}''')




