




"""simple docstring"""

import json
import os
import tempfile

import transformers

import datasets
from utils import generate_example_dataset, get_duration


__magic_name__				     =     50_00_00

__magic_name__     ,							__magic_name__				     =     os.path.split(__file__)
__magic_name__				     =     os.path.join(RESULTS_BASEPATH, """results""", RESULTS_FILENAME.replace(""".py""", """.json"""))




@get_duration
def     _A					( __lowercase							,     **__lowercase    ):





       """simple docstring"""



       lowerCamelCase__          =						dataset.map(**__UpperCamelCase    )




@get_duration
def     _A					( __lowercase							,     **__lowercase    ):





       """simple docstring"""



       lowerCamelCase__          =						dataset.filter(**__UpperCamelCase    )




def     _A					( ):





       """simple docstring"""



       lowerCamelCase__          =						{"""num examples""": SPEED_TEST_N_EXAMPLES}
       with tempfile.TemporaryDirectory() as tmp_dir:
              lowerCamelCase__          =						datasets.Features({"""text""": datasets.Value("""string"""    ), """numbers""": datasets.Value("""float32"""    )}    )
              lowerCamelCase__          =						generate_example_dataset(
                  os.path.join(__UpperCamelCase							,     """dataset.arrow"""    )							,     __UpperCamelCase							,     num_examples=__UpperCamelCase    )

              lowerCamelCase__          =						transformers.AutoTokenizer.from_pretrained("""bert-base-cased"""							,     use_fast=__UpperCamelCase    )

              def tokenize(__lowercase    ):
                     return tokenizer(examples["""text"""]    )

              lowerCamelCase__          =						map(__UpperCamelCase    )

              lowerCamelCase__          =						map(__UpperCamelCase							,     batched=__UpperCamelCase    )

              lowerCamelCase__          =						map(__UpperCamelCase							,     function=lambda __lowercase    : None							,     batched=__UpperCamelCase    )

              with dataset.formatted_as(type="""numpy"""    ):
                     lowerCamelCase__          =						map(__UpperCamelCase							,     function=lambda __lowercase    : None							,     batched=__UpperCamelCase    )

              with dataset.formatted_as(type="""pandas"""    ):
                     lowerCamelCase__          =						map(__UpperCamelCase							,     function=lambda __lowercase    : None							,     batched=__UpperCamelCase    )

              with dataset.formatted_as(type="""torch"""							,     columns="""numbers"""    ):
                     lowerCamelCase__          =						map(__UpperCamelCase							,     function=lambda __lowercase    : None							,     batched=__UpperCamelCase    )

              with dataset.formatted_as(type="""tensorflow"""							,     columns="""numbers"""    ):
                     lowerCamelCase__          =						map(__UpperCamelCase							,     function=lambda __lowercase    : None							,     batched=__UpperCamelCase    )

              lowerCamelCase__          =						map(__UpperCamelCase							,     function=__UpperCamelCase							,     batched=__UpperCamelCase    )

              lowerCamelCase__          =						filter(__UpperCamelCase    )

              # Activate later when tokenizer support batched inputs
              # with dataset.formatted_as(type='numpy'):
              #     times[func.__name__ + " fast-tokenizer batched numpy"] = func(dataset, function=tokenize, batched=True)

       with open(__UpperCamelCase							,     """wb"""    ) as f:
              f.write(json.dumps(__UpperCamelCase    ).encode("""utf-8"""    )    )


if __name__ == "__main__":  # useful to run the profiler
  benchmark_map_filter()











SCREAMING_SNAKE_CASE__   			=   '''
# Transformers installation
! pip install transformers datasets
# To install from source instead of the last release, comment the command above and uncomment the following one.
# ! pip install git+https://github.com/huggingface/transformers.git
'''

SCREAMING_SNAKE_CASE__   			=   [{'''type''': '''code''', '''content''': INSTALL_CONTENT}]
SCREAMING_SNAKE_CASE__   			=   {
    '''{processor_class}''': '''FakeProcessorClass''',
    '''{model_class}''': '''FakeModelClass''',
    '''{object_class}''': '''FakeObjectClass''',
}




