






"""simple docstring"""
import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


snake_case : Any     		=		get_tests_dir("""fixtures/test_sentencepiece.model""")
snake_case : Tuple     		=		get_tests_dir("""fixtures/test_sentencepiece_bpe.model""")

snake_case : Tuple     		=		"""pt""" if is_torch_available() else """tf"""



@require_sentencepiece
@require_tokenizers
class        UpperCamelCase__ (				a_							,	unittest.TestCase):







    """simple docstring"""
    __UpperCAmelCase           = CamembertTokenizer
    __UpperCAmelCase           = CamembertTokenizerFast
    __UpperCAmelCase           = True
    __UpperCAmelCase           = True







    def 				a__			(		self :	Union[str, Any] ):





           '''simple docstring'''

           super().setUp()

           # We have a SentencePiece fixture for testing
           __magic_name__      				=    CamembertTokenizer(UpperCamelCase_ )
           tokenizer.save_pretrained(self.tmpdirname )







    def 				a__			(		self :	Optional[int] ):





           '''simple docstring'''

           __magic_name__      				=    '<pad>'
           __magic_name__      				=    1

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase_ )				,    UpperCamelCase_ )
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase_ )				,    UpperCamelCase_ )







    def 				a__			(		self :	Optional[int] ):





           '''simple docstring'''

           __magic_name__      				=    list(self.get_tokenizer().get_vocab().keys() )

           self.assertEqual(vocab_keys[0]				,    '<s>NOTUSED' )
           self.assertEqual(vocab_keys[1]				,    '<pad>' )
           self.assertEqual(vocab_keys[-1]				,    '<mask>' )
           self.assertEqual(len(UpperCamelCase_ )				,    1_0_0_4 )







    def 				a__			(		self :	Optional[int] ):





           '''simple docstring'''

           self.assertEqual(self.get_tokenizer().vocab_size				,    1_0_0_5 )







    def 				a__			(		self :	List[str] ):





           '''simple docstring'''

           __magic_name__      				=    CamembertTokenizer(UpperCamelCase_ )
           tokenizer.save_pretrained(self.tmpdirname )
           __magic_name__      				=    CamembertTokenizerFast.from_pretrained(self.tmpdirname )

           __magic_name__      				=    'I was born in 92000, and this is falsé.'

           __magic_name__      				=    tokenizer.encode(UpperCamelCase_ )
           __magic_name__      				=    rust_tokenizer.encode(UpperCamelCase_ )
           self.assertListEqual(UpperCamelCase_				,    UpperCamelCase_ )

           __magic_name__      				=    tokenizer.encode(UpperCamelCase_				,    add_special_tokens=UpperCamelCase_ )
           __magic_name__      				=    rust_tokenizer.encode(UpperCamelCase_				,    add_special_tokens=UpperCamelCase_ )
           self.assertListEqual(UpperCamelCase_				,    UpperCamelCase_ )

           # <unk> tokens are not the same for `rust` than for `slow`.
           # Because spm gives back raw token instead of `unk` in EncodeAsPieces
           # tokens = tokenizer.tokenize(sequence)
           __magic_name__      				=    tokenizer.convert_ids_to_tokens(UpperCamelCase_ )
           __magic_name__      				=    rust_tokenizer.tokenize(UpperCamelCase_ )
           self.assertListEqual(UpperCamelCase_				,    UpperCamelCase_ )







    def 				a__			(		self :	Union[str, Any] ):





           '''simple docstring'''

           if not self.test_rust_tokenizer:
                  return

           __magic_name__      				=    self.get_tokenizer()
           __magic_name__      				=    self.get_rust_tokenizer()

           __magic_name__      				=    'I was born in 92000, and this is falsé.'

           __magic_name__      				=    tokenizer.tokenize(UpperCamelCase_ )
           __magic_name__      				=    rust_tokenizer.tokenize(UpperCamelCase_ )
           self.assertListEqual(UpperCamelCase_				,    UpperCamelCase_ )

           __magic_name__      				=    tokenizer.encode(UpperCamelCase_				,    add_special_tokens=UpperCamelCase_ )
           __magic_name__      				=    rust_tokenizer.encode(UpperCamelCase_				,    add_special_tokens=UpperCamelCase_ )
           self.assertListEqual(UpperCamelCase_				,    UpperCamelCase_ )

           __magic_name__      				=    self.get_rust_tokenizer()
           __magic_name__      				=    tokenizer.encode(UpperCamelCase_ )
           __magic_name__      				=    rust_tokenizer.encode(UpperCamelCase_ )
           self.assertListEqual(UpperCamelCase_				,    UpperCamelCase_ )







    @slow
    def 				a__			(		self :	Union[str, Any] ):





           '''simple docstring'''

           __magic_name__      				=    {'input_ids': [[5, 5_4, 7_1_9_6, 2_9_7, 3_0, 2_3, 7_7_6, 1_8, 1_1, 3_2_1_5, 3_7_0_5, 8_2_5_2, 2_2, 3_1_6_4, 1_1_8_1, 2_1_1_6, 2_9, 1_6, 8_1_3, 2_5, 7_9_1, 3_3_1_4, 2_0, 3_4_4_6, 3_8, 2_7_5_7_5, 1_2_0, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 4_6_8, 1_7, 1_1, 9_0_8_8, 2_0, 1_5_1_7, 8, 2_2_8_0_4, 1_8_8_1_8, 1_0, 3_8, 6_2_9, 6_0_7, 6_0_7, 1_4_2, 1_9, 7_1_9_6, 8_6_7, 5_6, 1_0_3_2_6, 2_4, 2_2_6_7, 2_0, 4_1_6, 5_0_7_2, 1_5_6_1_2, 2_3_3, 7_3_4, 7, 2_3_9_9, 2_7, 1_6, 3_0_1_5, 1_6_4_9, 7, 2_4, 2_0, 4_3_3_8, 2_3_9_9, 2_7, 1_3, 3_4_0_0, 1_4, 1_3, 6_1_8_9, 8, 9_3_0, 9, 6]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
           # fmt: on

           # camembert is a french model. So we also use french texts.
           __magic_name__      				=    [
               'Le transformeur est un modèle d\'apprentissage profond introduit en 2017, '
               'utilisé principalement dans le domaine du traitement automatique des langues (TAL).',
               'À l\'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus '
               'pour gérer des données séquentielles, telles que le langage naturel, pour des tâches '
               'telles que la traduction et la synthèse de texte.',
           ]

           self.tokenizer_integration_test_util(
               expected_encoding=UpperCamelCase_				,    model_name='camembert-base'				,    revision='3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf'				,    sequences=UpperCamelCase_				,    )







"""simple docstring"""
from ..utils import DummyObject, requires_backends



class        UpperCamelCase__ (				metaclass=a_):







    """simple docstring"""
    __UpperCAmelCase           = ["""flax""", """transformers"""]







    def __init__(		self :	List[Any]				,    *UpperCamelCase_ :	str				,    **UpperCamelCase_ :	Any ):





           '''simple docstring'''

           requires_backends(self				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	Optional[int]				,    *UpperCamelCase_ :	Tuple				,    **UpperCamelCase_ :	List[Any] ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	str				,    *UpperCamelCase_ :	Optional[Any]				,    **UpperCamelCase_ :	Dict ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )



class        UpperCamelCase__ (				metaclass=a_):







    """simple docstring"""
    __UpperCAmelCase           = ["""flax""", """transformers"""]







    def __init__(		self :	List[Any]				,    *UpperCamelCase_ :	str				,    **UpperCamelCase_ :	str ):





           '''simple docstring'''

           requires_backends(self				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	Optional[int]				,    *UpperCamelCase_ :	Optional[int]				,    **UpperCamelCase_ :	List[str] ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	Tuple				,    *UpperCamelCase_ :	List[str]				,    **UpperCamelCase_ :	Optional[Any] ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )



class        UpperCamelCase__ (				metaclass=a_):







    """simple docstring"""
    __UpperCAmelCase           = ["""flax""", """transformers"""]







    def __init__(		self :	Optional[int]				,    *UpperCamelCase_ :	List[Any]				,    **UpperCamelCase_ :	List[str] ):





           '''simple docstring'''

           requires_backends(self				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	str				,    *UpperCamelCase_ :	List[str]				,    **UpperCamelCase_ :	Tuple ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	Tuple				,    *UpperCamelCase_ :	Any				,    **UpperCamelCase_ :	List[str] ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )



class        UpperCamelCase__ (				metaclass=a_):







    """simple docstring"""
    __UpperCAmelCase           = ["""flax""", """transformers"""]







    def __init__(		self :	Optional[Any]				,    *UpperCamelCase_ :	Optional[Any]				,    **UpperCamelCase_ :	Dict ):





           '''simple docstring'''

           requires_backends(self				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	str				,    *UpperCamelCase_ :	Dict				,    **UpperCamelCase_ :	List[str] ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )







    @classmethod
    def 				a__			(		cls :	str				,    *UpperCamelCase_ :	Optional[Any]				,    **UpperCamelCase_ :	Union[str, Any] ):





           '''simple docstring'''

           requires_backends(cls				,    ['flax', 'transformers'] )