



'''simple docstring'''
import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


snake_case_					:      List[str]    			=    re.compile(r'\s+')






def A__       (	UpperCAmelCase_				):
						return {"hash": hashlib.mda(re.sub(UpperCAmelCase_    ,    ''    ,    example['content']				).encode('utf-8'				)				).hexdigest()}






def A__       (	UpperCAmelCase_				):
						_UpperCamelCase   :						Tuple												=  [len(UpperCAmelCase_				) for line in example['content'].splitlines()]
						return {"line_mean": np.mean(UpperCAmelCase_				), "line_max": max(UpperCAmelCase_				)}






def A__       (	UpperCAmelCase_				):
						_UpperCamelCase   :						Optional[Any]												=  np.mean([c.isalnum() for c in example['content']]				)
						return {"alpha_frac": alpha_frac}






def A__       (	UpperCAmelCase_    ,    UpperCAmelCase_				):
						if example["hash"] in uniques:
												uniques.remove(example['hash']				)
												return True
						else:
												return False






def A__       (	UpperCAmelCase_    ,    UpperCAmelCase_=5				):
						_UpperCamelCase   :						Optional[Any]												=  ['auto-generated', 'autogenerated', 'automatically generated']
						_UpperCamelCase   :						Optional[int]												=  example['content'].splitlines()
						for _, line in zip(range(UpperCAmelCase_				)    ,    UpperCAmelCase_				):
												for keyword in keywords:
																		if keyword in line.lower():
																								return {"autogenerated": True}
						else:
												return {"autogenerated": False}






def A__       (	UpperCAmelCase_    ,    UpperCAmelCase_=5    ,    UpperCAmelCase_=0.05				):
						_UpperCamelCase   :						List[str]												=  ['unit tests', 'test file', 'configuration file']
						_UpperCamelCase   :						str												=  example['content'].splitlines()
						_UpperCamelCase   :						Union[str, Any]												=  0
						_UpperCamelCase   :						Any												=  0
						# first test
						for _, line in zip(range(UpperCAmelCase_				)    ,    UpperCAmelCase_				):
												for keyword in keywords:
																		if keyword in line.lower():
																								return {"config_or_test": True}
    # second test
						_UpperCamelCase   :						Optional[Any]												=  example['content'].count('\n'				)
						_UpperCamelCase   :						Optional[Any]												=  int(coeff * nlines				)
						for line in lines:
												count_config += line.lower().count('config'				)
												count_test += line.lower().count('test'				)
												if count_config > threshold or count_test > threshold:
																		return {"config_or_test": True}
						return {"config_or_test": False}






def A__       (	UpperCAmelCase_				):
						_UpperCamelCase   :						List[str]												=  ['def ', 'class ', 'for ', 'while ']
						_UpperCamelCase   :						Union[str, Any]												=  example['content'].splitlines()
						for line in lines:
												for keyword in keywords:
																		if keyword in line.lower():
																								return {"has_no_keywords": False}
						return {"has_no_keywords": True}






def A__       (	UpperCAmelCase_    ,    UpperCAmelCase_=4				):
						_UpperCamelCase   :						List[Any]												=  example['content'].splitlines()
						_UpperCamelCase   :						Dict												=  0
						for line in lines:
												counter += line.lower().count('='				)
												if counter > minimum:
																		return {"has_few_assignments": False}
						return {"has_few_assignments": True}






def A__       (	UpperCAmelCase_				):
						_UpperCamelCase   :						List[str]												=  tokenizer(example['content']    ,    truncation=UpperCAmelCase_				)['input_ids']
						_UpperCamelCase   :						List[str]												=  len(example['content']				) / len(UpperCAmelCase_				)
						return {"ratio": ratio}






def A__       (	UpperCAmelCase_				):
						_UpperCamelCase   :						Optional[int]												=  {}
						results.update(get_hash(UpperCAmelCase_				)				)
						results.update(line_stats(UpperCAmelCase_				)				)
						results.update(alpha_stats(UpperCAmelCase_				)				)
						results.update(char_token_ratio(UpperCAmelCase_				)				)
						results.update(is_autogenerated(UpperCAmelCase_				)				)
						results.update(is_config_or_test(UpperCAmelCase_				)				)
						results.update(has_no_keywords(UpperCAmelCase_				)				)
						results.update(has_few_assignments(UpperCAmelCase_				)				)
						return results






def A__       (	UpperCAmelCase_    ,    UpperCAmelCase_    ,    UpperCAmelCase_				):
						if not check_uniques(UpperCAmelCase_    ,    UpperCAmelCase_				):
												return False
						elif example["autogenerated"]:
												return False
						elif example["line_max"] > args.line_max:
												return False
						elif example["line_mean"] > args.line_mean:
												return False
						elif example["alpha_frac"] < args.alpha_frac:
												return False
						elif example["ratio"] < args.min_token_ratio:
												return False
						elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
												return False
						elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
												return False
						elif example["has_few_assignments"]:
												return False
						else:
												return True






def A__       (	UpperCAmelCase_				):
						with open(UpperCAmelCase_    ,    'rb'				) as f_in:
												with gzip.open(str(UpperCAmelCase_				) + '.gz'    ,    'wb'    ,    compresslevel=6				) as f_out:
																		shutil.copyfileobj(UpperCAmelCase_    ,    UpperCAmelCase_				)
						os.unlink(UpperCAmelCase_				)


# Settings
snake_case_					:      Tuple    			=    HfArgumentParser(PreprocessingArguments)
snake_case_					:      str    			=    parser.parse_args()
if args.num_workers is None:
					snake_case_					:      Optional[int]    			=    multiprocessing.cpu_count()
snake_case_					:      Optional[Any]    			=    AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
snake_case_					:      Dict    			=    time.time()
snake_case_					:      Any    			=    load_dataset(args.dataset_name, split='train')
print(F"""Time to load dataset: {time.time()-t_start:.2f}""")

# Run preprocessing
snake_case_					:      List[str]    			=    time.time()
snake_case_					:      Dict    			=    ds.map(preprocess, num_proc=args.num_workers)
print(F"""Time to preprocess dataset: {time.time()-t_start:.2f}""")

# Deduplicate hashes
snake_case_					:      str    			=    set(ds.unique('hash'))
snake_case_					:      List[str]    			=    len(uniques) / len(ds)
print(F"""Fraction of duplicates: {1-frac:.2%}""")

# Deduplicate data and apply heuristics
snake_case_					:      List[str]    			=    time.time()
snake_case_					:      Union[str, Any]    			=    ds.filter(filter, fn_kwargs={'uniques': uniques, 'args': args})
print(F"""Time to filter dataset: {time.time()-t_start:.2f}""")
print(F"""Size of filtered dataset: {len(ds_filter)}""")

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
					snake_case_					:      Any    			=    time.time()
					snake_case_			,				snake_case_					:      Any    			=    deduplicate_dataset(ds_filter, args.jaccard_threshold)
					print(F"""Time to deduplicate dataset: {time.time()-t_start:.2f}""")
					print(F"""Size of deduplicate dataset: {len(ds_filter)}""")

# Save data in batches of samples_per_file
snake_case_					:      List[str]    			=    Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
					with open(output_dir / 'duplicate_clusters.json', 'w') as f:
										json.dump(duplicate_clusters, f)

snake_case_					:      Union[str, Any]    			=    output_dir / 'data'
data_dir.mkdir(exist_ok=True)

snake_case_					:      Union[str, Any]    			=    time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
					snake_case_					:      Any    			=    str(data_dir / F"""file-{file_number+1:012}.json""")
					snake_case_					:      Tuple    			=    min(len(ds_filter), index + args.samples_per_file)
					ds_filter.select(list(range(index, end_index))).to_json(file_path)
					compress_file(file_path)

print(F"""Time to save dataset: {time.time()-t_start:.2f}""")







'''simple docstring'''
from collections import UserDict
from typing import Union

import numpy as np
import requests

from ..utils import (
    add_end_docstrings,
    logging,
)
from .audio_classification import ffmpeg_read
from .base import PIPELINE_INIT_ARGS, Pipeline


snake_case_					:      List[Any]    			=    logging.get_logger(__name__)






@add_end_docstrings(lowercase  )
class 				lowercase__      (				lowercase  ):





	def __init__(					self   :							Tuple					,**lowerCamelCase__   :							Union[str, Any]					):







							'''simple docstring'''
							super().__init__(**lowerCamelCase__					)

							if self.framework != "pt":
													raise ValueError(F'The {self.__class__} is only available in PyTorch.'					)
							# No specific FOR_XXX available yet





	def __call__(					self   :							Dict					,lowerCamelCase__   :							Union[np.ndarray, bytes, str]					,**lowerCamelCase__   :							Any					):







							'''simple docstring'''
							return super().__call__(lowerCamelCase__					,**lowerCamelCase__					)





	def       UpperCamelCase_    (					self   :							int					,**lowerCamelCase__   :							Optional[int]					):







							'''simple docstring'''
							_UpperCamelCase   :						List[Any]												=  {}
							if "candidate_labels" in kwargs:
													_UpperCamelCase   :						Any												=  kwargs['candidate_labels']
							if "hypothesis_template" in kwargs:
													_UpperCamelCase   :						Any												=  kwargs['hypothesis_template']

							return preprocess_params, {}, {}





	def       UpperCamelCase_    (					self   :							Any					,lowerCamelCase__   :							List[str]					,lowerCamelCase__   :							str=None					,lowerCamelCase__   :							List[Any]="This is a sound of {}."					):







							'''simple docstring'''
							if isinstance(lowerCamelCase__					,lowerCamelCase__					):
													if audio.startswith('http://'					) or audio.startswith('https://'					):
																			# We need to actually check for a real protocol, otherwise it's impossible to use a local file
																			# like http_huggingface_co.png
																			_UpperCamelCase   :						Optional[Any]												=  requests.get(lowerCamelCase__					).content
													else:
																			with open(lowerCamelCase__					,'rb'					) as f:
																									_UpperCamelCase   :						List[str]												=  f.read()

							if isinstance(lowerCamelCase__					,lowerCamelCase__					):
													_UpperCamelCase   :						int												=  ffmpeg_read(lowerCamelCase__					,self.feature_extractor.sampling_rate					)

							if not isinstance(lowerCamelCase__					,np.ndarray					):
													raise ValueError('We expect a numpy ndarray as input'					)
							if len(audio.shape					) != 1:
													raise ValueError('We expect a single channel audio input for ZeroShotAudioClassificationPipeline'					)

							_UpperCamelCase   :						Dict												=  self.feature_extractor(
							    [audio]					,sampling_rate=self.feature_extractor.sampling_rate					,return_tensors='pt'					)
							_UpperCamelCase   :						List[Any]												=  candidate_labels
							_UpperCamelCase   :						Tuple												=  [hypothesis_template.format(lowerCamelCase__					) for x in candidate_labels]
							_UpperCamelCase   :						Dict												=  self.tokenizer(lowerCamelCase__					,return_tensors=self.framework					,padding=lowerCamelCase__					)
							_UpperCamelCase   :						Optional[Any]												=  [text_inputs]
							return inputs





	def       UpperCamelCase_    (					self   :							str					,lowerCamelCase__   :							Tuple					):







							'''simple docstring'''
							_UpperCamelCase   :						Any												=  model_inputs.pop('candidate_labels'					)
							_UpperCamelCase   :						str												=  model_inputs.pop('text_inputs'					)
							if isinstance(text_inputs[0]					,lowerCamelCase__					):
													_UpperCamelCase   :						Union[str, Any]												=  text_inputs[0]
							else:
													# Batching case.
													_UpperCamelCase   :						int												=  text_inputs[0][0]

							_UpperCamelCase   :						List[str]												=  self.model(**lowerCamelCase__					,**lowerCamelCase__					)

							_UpperCamelCase   :						Optional[Any]												=  {
							    'candidate_labels': candidate_labels,
							    'logits': outputs.logits_per_audio,
							}
							return model_outputs





	def       UpperCamelCase_    (					self   :							int					,lowerCamelCase__   :							Optional[Any]					):







							'''simple docstring'''
							_UpperCamelCase   :						Optional[Any]												=  model_outputs.pop('candidate_labels'					)
							_UpperCamelCase   :						int												=  model_outputs['logits'][0]

							if self.framework == "pt":
													_UpperCamelCase   :						Tuple												=  logits.softmax(dim=0					)
													_UpperCamelCase   :						str												=  probs.tolist()
							else:
													raise ValueError('`tf` framework not supported.'					)

							_UpperCamelCase   :						Union[str, Any]												=  [
							    {'score': score, 'label': candidate_label}
							    for score, candidate_label in sorted(zip(lowerCamelCase__					,lowerCamelCase__					)					,key=lambda lowerCamelCase__					: -x[0]					)
							]
							return result


