




"""simple docstring"""

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowerCamelCase  :			Optional[Any]        =       logging.get_logger(__name__)

_lowerCamelCase  :			int        =       {
    'microsoft/trocr-base-handwritten': (
        'https://huggingface.co/microsoft/trocr-base-handwritten/resolve/main/config.json'
    ),
    # See all TrOCR models at https://huggingface.co/models?filter=trocr
}

class        lowercase      (    __UpperCAmelCase		):
				__lowerCAmelCase    :    str         =   """trocr"""
				__lowerCAmelCase    :    Union[str, Any]         =   ["""past_key_values"""]
				__lowerCAmelCase    :    int         =   {
				    """num_attention_heads""": """decoder_attention_heads""",
				    """hidden_size""": """d_model""",
				    """num_hidden_layers""": """decoder_layers""",
				}







				def __init__(							self   :					Optional[int]      ,     _lowerCamelCase   :					Optional[Any]=5_02_65      ,     _lowerCamelCase   :					Union[str, Any]=10_24      ,     _lowerCamelCase   :					str=12      ,     _lowerCamelCase   :					str=16      ,     _lowerCamelCase   :					Tuple=40_96      ,     _lowerCamelCase   :					Optional[Any]="gelu"      ,     _lowerCamelCase   :					List[Any]=5_12      ,     _lowerCamelCase   :					Dict=0.1      ,     _lowerCamelCase   :					List[Any]=0.0      ,     _lowerCamelCase   :					Dict=0.0      ,     _lowerCamelCase   :					List[Any]=2      ,     _lowerCamelCase   :					List[Any]=0.02      ,     _lowerCamelCase   :					Dict=0.0      ,     _lowerCamelCase   :					Any=True      ,     _lowerCamelCase   :					List[str]=False      ,     _lowerCamelCase   :					Optional[Any]=True      ,     _lowerCamelCase   :					List[Any]=True      ,     _lowerCamelCase   :					Union[str, Any]=1      ,     _lowerCamelCase   :					Optional[int]=0      ,     _lowerCamelCase   :					Tuple=2      ,     **_lowerCamelCase   :					List[Any]      ,     ):



					"""simple docstring"""

					A_  :				Optional[Any]	   =		vocab_size
					A_  :				Union[str, Any]	   =		d_model
					A_  :				Optional[int]	   =		decoder_layers
					A_  :				Optional[Any]	   =		decoder_attention_heads
					A_  :				Tuple	   =		decoder_ffn_dim
					A_  :				List[Any]	   =		activation_function
					A_  :				List[Any]	   =		max_position_embeddings
					A_  :				List[Any]	   =		dropout
					A_  :				Dict	   =		attention_dropout
					A_  :				List[Any]	   =		activation_dropout
					A_  :				Union[str, Any]	   =		init_std
					A_  :				str	   =		decoder_layerdrop
					A_  :				Tuple	   =		use_cache
					A_  :				List[Any]	   =		scale_embedding
					A_  :				Optional[Any]	   =		use_learned_position_embeddings
					A_  :				Tuple	   =		layernorm_embedding

					super().__init__(
					    pad_token_id=_lowerCamelCase      ,     bos_token_id=_lowerCamelCase      ,     eos_token_id=_lowerCamelCase      ,     decoder_start_token_id=_lowerCamelCase      ,     **_lowerCamelCase      ,     )











"""simple docstring"""

import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCamelCase  :			Tuple        =       get_tests_dir('fixtures/test_sentencepiece.model')

@require_sentencepiece
@require_tokenizers
class        lowercase      (    __UpperCAmelCase     ,	unittest.TestCase):
				__lowerCAmelCase    :    Tuple         =   XLNetTokenizer
				__lowerCAmelCase    :    Optional[int]         =   XLNetTokenizerFast
				__lowerCAmelCase    :    Union[str, Any]         =   True
				__lowerCAmelCase    :    Any         =   True







				def 					a_  (							self   :					List[str]       ):



					"""simple docstring"""

					super().setUp()

					# We have a SentencePiece fixture for testing
					A_  :				Optional[Any]	   =		XLNetTokenizer(_lowerCamelCase      ,     keep_accents=_lowerCamelCase       )
					tokenizer.sanitize_special_tokens()
					tokenizer.save_pretrained(self.tmpdirname       )







				def 					a_  (							self   :					List[str]       ):



					"""simple docstring"""

					A_  :				Optional[int]	   =		'''<s>'''
					A_  :				int	   =		1

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCamelCase       )      ,     _lowerCamelCase       )
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCamelCase       )      ,     _lowerCamelCase       )







				def 					a_  (							self   :					str       ):



					"""simple docstring"""

					A_  :				Any	   =		list(self.get_tokenizer().get_vocab().keys()       )

					self.assertEqual(vocab_keys[0]      ,     '''<unk>'''       )
					self.assertEqual(vocab_keys[1]      ,     '''<s>'''       )
					self.assertEqual(vocab_keys[-1]      ,     '''<eod>'''       )
					self.assertEqual(len(_lowerCamelCase       )      ,     10_06       )







				def 					a_  (							self   :					Optional[int]       ):



					"""simple docstring"""

					self.assertEqual(self.get_tokenizer().vocab_size      ,     10_00       )







				def 					a_  (							self   :					Optional[Any]       ):



					"""simple docstring"""

					A_  :				Optional[int]	   =		XLNetTokenizer(_lowerCamelCase      ,     keep_accents=_lowerCamelCase       )

					A_  :				Union[str, Any]	   =		tokenizer.tokenize('''This is a test'''       )
					self.assertListEqual(_lowerCamelCase      ,     ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']       )

					self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase       )      ,     [2_85, 46, 10, 1_70, 3_82]       )

					A_  :				List[str]	   =		tokenizer.tokenize('''I was born in 92000, and this is falsé.'''       )
					self.assertListEqual(
					    _lowerCamelCase      ,     [
					        SPIECE_UNDERLINE + '''I''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''9''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''s''',
					        '''é''',
					        '''.''',
					    ]      ,     )
					A_  :				int	   =		tokenizer.convert_tokens_to_ids(_lowerCamelCase       )
					self.assertListEqual(_lowerCamelCase      ,     [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]       )

					A_  :				Union[str, Any]	   =		tokenizer.convert_ids_to_tokens(_lowerCamelCase       )
					self.assertListEqual(
					    _lowerCamelCase      ,     [
					        SPIECE_UNDERLINE + '''I''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''<unk>''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''s''',
					        '''<unk>''',
					        '''.''',
					    ]      ,     )







				def 					a_  (							self   :					List[str]       ):



					"""simple docstring"""

					A_  :				Optional[Any]	   =		XLNetTokenizer(_lowerCamelCase      ,     do_lower_case=_lowerCamelCase       )
					A_  :				str	   =		tokenizer.tokenize('''I was born in 92000, and this is falsé.'''       )
					self.assertListEqual(
					    _lowerCamelCase      ,     [
					        SPIECE_UNDERLINE + '''''',
					        '''i''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''9''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''se''',
					        '''.''',
					    ]      ,     )
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )      ,     ['''▁he''', '''ll''', '''o''']       )







				def 					a_  (							self   :					Optional[Any]       ):



					"""simple docstring"""

					A_  :				Optional[int]	   =		XLNetTokenizer(_lowerCamelCase      ,     do_lower_case=_lowerCamelCase       )
					A_  :				Tuple	   =		tokenizer.tokenize('''I was born in 92000, and this is falsé.'''       )
					self.assertListEqual(
					    _lowerCamelCase      ,     [
					        SPIECE_UNDERLINE + '''I''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''9''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''se''',
					        '''.''',
					    ]      ,     )







				@slow
				def 					a_  (							self   :					Dict       ):



					"""simple docstring"""

					A_  :				int	   =		XLNetTokenizer.from_pretrained('''xlnet-base-cased'''       )

					A_  :				Optional[int]	   =		tokenizer.encode('''sequence builders'''      ,     add_special_tokens=_lowerCamelCase       )
					A_  :				Optional[int]	   =		tokenizer.encode('''multi-sequence build'''      ,     add_special_tokens=_lowerCamelCase       )

					A_  :				Tuple	   =		tokenizer.build_inputs_with_special_tokens(_lowerCamelCase       )
					A_  :				Union[str, Any]	   =		tokenizer.build_inputs_with_special_tokens(_lowerCamelCase      ,     _lowerCamelCase       )

					assert encoded_sentence == text + [4, 3]
					assert encoded_pair == text + [4] + text_a + [4, 3]







				@slow
				def 					a_  (							self   :					List[str]       ):



					"""simple docstring"""

					A_  :				Dict	   =		{'''input_ids''': [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
					# fmt: on

					self.tokenizer_integration_test_util(
					    expected_encoding=_lowerCamelCase      ,     model_name='''xlnet-base-cased'''      ,     revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''      ,     )





