
import json
import os
import tempfile

import transformers

import datasets
from utils import generate_example_dataset, get_duration


lowercase   =	5_0_0_0_0_0

lowercase ,    lowercase   =	os.path.split(__file__)
lowercase   =	os.path.join(RESULTS_BASEPATH, '''results''', RESULTS_FILENAME.replace('''.py''', '''.json'''))



@get_duration
def   __lowerCAmelCase    (				UpperCAmelCase__					:   datasets.Dataset      ,					**UpperCAmelCase__					:   int							)	->     str:
							lowerCamelCase_					=       dataset.map(**UpperCAmelCase__							)



@get_duration
def   __lowerCAmelCase    (				UpperCAmelCase__					:   datasets.Dataset      ,					**UpperCAmelCase__					:   Union[str, Any]							)	->     Optional[int]:
							lowerCamelCase_					=       dataset.filter(**UpperCAmelCase__							)



def   __lowerCAmelCase    (				)	->     List[Any]:
							lowerCamelCase_					=       {"""num examples""": SPEED_TEST_N_EXAMPLES}
							with tempfile.TemporaryDirectory() as tmp_dir:
														lowerCamelCase_					=       datasets.Features({"""text""": datasets.Value("""string"""							), """numbers""": datasets.Value("""float32"""							)}							)
														lowerCamelCase_					=       generate_example_dataset(
														    os.path.join(UpperCAmelCase__      ,					"""dataset.arrow"""							)      ,					UpperCAmelCase__      ,					num_examples=UpperCAmelCase__							)

														lowerCamelCase_					=       transformers.AutoTokenizer.from_pretrained("""bert-base-cased"""      ,					use_fast=UpperCAmelCase__							)

														def tokenize(UpperCAmelCase__					:   Optional[Any]							):
																					return tokenizer(examples["""text"""]							)

														lowerCamelCase_					=       map(UpperCAmelCase__							)

														lowerCamelCase_					=       map(UpperCAmelCase__      ,					batched=UpperCAmelCase__							)

														lowerCamelCase_					=       map(UpperCAmelCase__      ,					function=lambda UpperCAmelCase__							: None      ,					batched=UpperCAmelCase__							)

														with dataset.formatted_as(type="""numpy"""							):
																					lowerCamelCase_					=       map(UpperCAmelCase__      ,					function=lambda UpperCAmelCase__							: None      ,					batched=UpperCAmelCase__							)

														with dataset.formatted_as(type="""pandas"""							):
																					lowerCamelCase_					=       map(UpperCAmelCase__      ,					function=lambda UpperCAmelCase__							: None      ,					batched=UpperCAmelCase__							)

														with dataset.formatted_as(type="""torch"""      ,					columns="""numbers"""							):
																					lowerCamelCase_					=       map(UpperCAmelCase__      ,					function=lambda UpperCAmelCase__							: None      ,					batched=UpperCAmelCase__							)

														with dataset.formatted_as(type="""tensorflow"""      ,					columns="""numbers"""							):
																					lowerCamelCase_					=       map(UpperCAmelCase__      ,					function=lambda UpperCAmelCase__							: None      ,					batched=UpperCAmelCase__							)

														lowerCamelCase_					=       map(UpperCAmelCase__      ,					function=UpperCAmelCase__      ,					batched=UpperCAmelCase__							)

														lowerCamelCase_					=       filter(UpperCAmelCase__							)

														# Activate later when tokenizer support batched inputs
														# with dataset.formatted_as(type='numpy'):
														#     times[func.__name__ + " fast-tokenizer batched numpy"] = func(dataset, function=tokenize, batched=True)

							with open(UpperCAmelCase__      ,					"""wb"""							) as f:
														f.write(json.dumps(UpperCAmelCase__							).encode("""utf-8"""							)							)


if __name__ == "__main__":  # useful to run the profiler
		benchmark_map_filter()







import os
import sys

lowercase   =	os.path.join(os.path.dirname(__file__), '''src''')
sys.path.append(SRC_DIR)


from transformers import (
    AutoConfig,
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForMaskedLM,
    AutoModelForQuestionAnswering,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    add_start_docstrings,
)


lowercase   =	[
    '''torch''',
    '''numpy''',
    '''tokenizers''',
    '''filelock''',
    '''requests''',
    '''tqdm''',
    '''regex''',
    '''sentencepiece''',
    '''sacremoses''',
    '''importlib_metadata''',
    '''huggingface_hub''',
]



@add_start_docstrings(AutoConfig.__doc__							)
def   __lowerCAmelCase    (				*UpperCAmelCase__					:   List[Any]      ,					**UpperCAmelCase__					:   List[str]							)	->     List[str]:
							return AutoConfig.from_pretrained(*UpperCAmelCase__      ,					**UpperCAmelCase__							)



@add_start_docstrings(AutoTokenizer.__doc__							)
def   __lowerCAmelCase    (				*UpperCAmelCase__					:   List[str]      ,					**UpperCAmelCase__					:   List[Any]							)	->     Any:
							return AutoTokenizer.from_pretrained(*UpperCAmelCase__      ,					**UpperCAmelCase__							)



@add_start_docstrings(AutoModel.__doc__							)
def   __lowerCAmelCase    (				*UpperCAmelCase__					:   Union[str, Any]      ,					**UpperCAmelCase__					:   Dict							)	->     Union[str, Any]:
							return AutoModel.from_pretrained(*UpperCAmelCase__      ,					**UpperCAmelCase__							)



@add_start_docstrings(AutoModelForCausalLM.__doc__							)
def   __lowerCAmelCase    (				*UpperCAmelCase__					:   List[str]      ,					**UpperCAmelCase__					:   List[Any]							)	->     Optional[Any]:
							return AutoModelForCausalLM.from_pretrained(*UpperCAmelCase__      ,					**UpperCAmelCase__							)



@add_start_docstrings(AutoModelForMaskedLM.__doc__							)
def   __lowerCAmelCase    (				*UpperCAmelCase__					:   Union[str, Any]      ,					**UpperCAmelCase__					:   Dict							)	->     int:
							return AutoModelForMaskedLM.from_pretrained(*UpperCAmelCase__      ,					**UpperCAmelCase__							)



@add_start_docstrings(AutoModelForSequenceClassification.__doc__							)
def   __lowerCAmelCase    (				*UpperCAmelCase__					:   List[Any]      ,					**UpperCAmelCase__					:   Dict							)	->     Dict:
							return AutoModelForSequenceClassification.from_pretrained(*UpperCAmelCase__      ,					**UpperCAmelCase__							)



@add_start_docstrings(AutoModelForQuestionAnswering.__doc__							)
def   __lowerCAmelCase    (				*UpperCAmelCase__					:   Any      ,					**UpperCAmelCase__					:   List[str]							)	->     Union[str, Any]:
							return AutoModelForQuestionAnswering.from_pretrained(*UpperCAmelCase__      ,					**UpperCAmelCase__							)





