






import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__       		=  get_tests_dir("""fixtures/test_sentencepiece.model""")
lowerCamelCase__       		=  get_tests_dir("""fixtures/test_sentencepiece_bpe.model""")

lowerCamelCase__       		=  """pt""" if is_torch_available() else """tf"""




@require_sentencepiece
@require_tokenizers
class       snake_case__ (				lowercase_						,     unittest.TestCase):

       '''simple docstring'''
       lowerCamelCase				:					str								=			CamembertTokenizer
       lowerCamelCase				:					int								=			CamembertTokenizerFast
       lowerCamelCase				:					Tuple								=			True
       lowerCamelCase				:					Any								=			True



       def        __lowercase    (				self     )			->   List[str]:






            '''simple docstring'''





            super().setUp()

            # We have a SentencePiece fixture for testing
            __snake_case      :List[Any] 							=    CamembertTokenizer(a__     )
            tokenizer.save_pretrained(self.tmpdirname     )



       def        __lowercase    (				self     )			->   List[str]:






            '''simple docstring'''





            __snake_case      :Tuple 							=    """<pad>"""
            __snake_case      :Optional[int] 							=    1

            self.assertEqual(self.get_tokenizer()._convert_token_to_id(a__     )      ,    a__     )
            self.assertEqual(self.get_tokenizer()._convert_id_to_token(a__     )      ,    a__     )



       def        __lowercase    (				self     )			->   int:






            '''simple docstring'''





            __snake_case      :int 							=    list(self.get_tokenizer().get_vocab().keys()     )

            self.assertEqual(vocab_keys[0]      ,    """<s>NOTUSED"""     )
            self.assertEqual(vocab_keys[1]      ,    """<pad>"""     )
            self.assertEqual(vocab_keys[-1]      ,    """<mask>"""     )
            self.assertEqual(len(a__     )      ,    10_04     )



       def        __lowercase    (				self     )			->   str:






            '''simple docstring'''





            self.assertEqual(self.get_tokenizer().vocab_size      ,    10_05     )



       def        __lowercase    (				self     )			->   List[Any]:






            '''simple docstring'''





            __snake_case      :List[Any] 							=    CamembertTokenizer(a__     )
            tokenizer.save_pretrained(self.tmpdirname     )
            __snake_case      :Union[str, Any] 							=    CamembertTokenizerFast.from_pretrained(self.tmpdirname     )

            __snake_case      :Any 							=    """I was born in 92000, and this is falsé."""

            __snake_case      :Optional[int] 							=    tokenizer.encode(a__     )
            __snake_case      :List[Any] 							=    rust_tokenizer.encode(a__     )
            self.assertListEqual(a__      ,    a__     )

            __snake_case      :Tuple 							=    tokenizer.encode(a__      ,    add_special_tokens=a__     )
            __snake_case      :str 							=    rust_tokenizer.encode(a__      ,    add_special_tokens=a__     )
            self.assertListEqual(a__      ,    a__     )

            # <unk> tokens are not the same for `rust` than for `slow`.
            # Because spm gives back raw token instead of `unk` in EncodeAsPieces
            # tokens = tokenizer.tokenize(sequence)
            __snake_case      :Optional[Any] 							=    tokenizer.convert_ids_to_tokens(a__     )
            __snake_case      :Optional[int] 							=    rust_tokenizer.tokenize(a__     )
            self.assertListEqual(a__      ,    a__     )



       def        __lowercase    (				self     )			->   int:






            '''simple docstring'''





            if not self.test_rust_tokenizer:
                 return

            __snake_case      :Optional[Any] 							=    self.get_tokenizer()
            __snake_case      :Optional[int] 							=    self.get_rust_tokenizer()

            __snake_case      :Union[str, Any] 							=    """I was born in 92000, and this is falsé."""

            __snake_case      :Dict 							=    tokenizer.tokenize(a__     )
            __snake_case      :int 							=    rust_tokenizer.tokenize(a__     )
            self.assertListEqual(a__      ,    a__     )

            __snake_case      :Tuple 							=    tokenizer.encode(a__      ,    add_special_tokens=a__     )
            __snake_case      :List[str] 							=    rust_tokenizer.encode(a__      ,    add_special_tokens=a__     )
            self.assertListEqual(a__      ,    a__     )

            __snake_case      :Optional[Any] 							=    self.get_rust_tokenizer()
            __snake_case      :Tuple 							=    tokenizer.encode(a__     )
            __snake_case      :Tuple 							=    rust_tokenizer.encode(a__     )
            self.assertListEqual(a__      ,    a__     )



       @slow
       def        __lowercase    (				self     )			->   int:






            '''simple docstring'''





            __snake_case      :Any 							=    {"""input_ids""": [[5, 54, 71_96, 2_97, 30, 23, 7_76, 18, 11, 32_15, 37_05, 82_52, 22, 31_64, 11_81, 21_16, 29, 16, 8_13, 25, 7_91, 33_14, 20, 34_46, 38, 2_75_75, 1_20, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 4_68, 17, 11, 90_88, 20, 15_17, 8, 2_28_04, 1_88_18, 10, 38, 6_29, 6_07, 6_07, 1_42, 19, 71_96, 8_67, 56, 1_03_26, 24, 22_67, 20, 4_16, 50_72, 1_56_12, 2_33, 7_34, 7, 23_99, 27, 16, 30_15, 16_49, 7, 24, 20, 43_38, 23_99, 27, 13, 34_00, 14, 13, 61_89, 8, 9_30, 9, 6]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
            # fmt: on

            # camembert is a french model. So we also use french texts.
            __snake_case      :str 							=    [
                """Le transformeur est un modèle d'apprentissage profond introduit en 2017, """
                """utilisé principalement dans le domaine du traitement automatique des langues (TAL).""",
                """À l'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus """
                """pour gérer des données séquentielles, telles que le langage naturel, pour des tâches """
                """telles que la traduction et la synthèse de texte.""",
            ]

            self.tokenizer_integration_test_util(
                expected_encoding=a__      ,    model_name="""camembert-base"""      ,    revision="""3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf"""      ,    sequences=a__      ,    )










import json
import os
import tempfile

import transformers

import datasets
from utils import generate_example_dataset, get_duration


lowerCamelCase__       		=  50_0000

lowerCamelCase__		,      lowerCamelCase__       		=  os.path.split(__file__)
lowerCamelCase__       		=  os.path.join(RESULTS_BASEPATH, """results""", RESULTS_FILENAME.replace(""".py""", """.json"""))






@get_duration
def 	UpperCamelCase	(							snake_case__  : datasets.Dataset						,**snake_case__  : Any	):




     '''simple docstring'''






     __snake_case      :List[str] 							=    dataset.map(**snake_case__	)






@get_duration
def 	UpperCamelCase	(							snake_case__  : datasets.Dataset						,**snake_case__  : Dict	):




     '''simple docstring'''






     __snake_case      :int 							=    dataset.filter(**snake_case__	)






def 	UpperCamelCase	(							):




     '''simple docstring'''






     __snake_case      :str 							=    {"""num examples""": SPEED_TEST_N_EXAMPLES}
     with tempfile.TemporaryDirectory() as tmp_dir:
          __snake_case      :Dict 							=    datasets.Features({"""text""": datasets.Value("""string"""	), """numbers""": datasets.Value("""float32"""	)}	)
          __snake_case      :Optional[int] 							=    generate_example_dataset(
              os.path.join(snake_case__						,"""dataset.arrow"""	)						,snake_case__						,num_examples=snake_case__	)

          __snake_case      :str 							=    transformers.AutoTokenizer.from_pretrained("""bert-base-cased"""						,use_fast=snake_case__	)

          def tokenize(snake_case__  : Optional[int]	):
               return tokenizer(examples["""text"""]	)

          __snake_case      :Optional[Any] 							=    map(snake_case__	)

          __snake_case      :Optional[Any] 							=    map(snake_case__						,batched=snake_case__	)

          __snake_case      :Optional[Any] 							=    map(snake_case__						,function=lambda snake_case__	: None						,batched=snake_case__	)

          with dataset.formatted_as(type="""numpy"""	):
               __snake_case      :int 							=    map(snake_case__						,function=lambda snake_case__	: None						,batched=snake_case__	)

          with dataset.formatted_as(type="""pandas"""	):
               __snake_case      :List[Any] 							=    map(snake_case__						,function=lambda snake_case__	: None						,batched=snake_case__	)

          with dataset.formatted_as(type="""torch"""						,columns="""numbers"""	):
               __snake_case      :str 							=    map(snake_case__						,function=lambda snake_case__	: None						,batched=snake_case__	)

          with dataset.formatted_as(type="""tensorflow"""						,columns="""numbers"""	):
               __snake_case      :Optional[int] 							=    map(snake_case__						,function=lambda snake_case__	: None						,batched=snake_case__	)

          __snake_case      :Dict 							=    map(snake_case__						,function=snake_case__						,batched=snake_case__	)

          __snake_case      :Tuple 							=    filter(snake_case__	)

          # Activate later when tokenizer support batched inputs
          # with dataset.formatted_as(type='numpy'):
          #     times[func.__name__ + " fast-tokenizer batched numpy"] = func(dataset, function=tokenize, batched=True)

     with open(snake_case__						,"""wb"""	) as f:
          f.write(json.dumps(snake_case__	).encode("""utf-8"""	)	)


if __name__ == "__main__":  # useful to run the profiler
     benchmark_map_filter()


