
'''simple docstring'''







import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_lowercase			:  Dict   							=	get_tests_dir("fixtures/test_sentencepiece.model")






@require_sentencepiece
class 			lowerCAmelCase__       (  lowerCamelCase_   ,						unittest.TestCase					):
					lowerCAmelCase_					       =					XLMProphetNetTokenizer
					lowerCAmelCase_					       =					False
					lowerCAmelCase_					       =					True






					def      _snake_case   (       self		):


											"""simple docstring"""





											super().setUp()

											# We have a SentencePiece fixture for testing
											lowercase_     :					Optional[int]													=	XLMProphetNetTokenizer(__SCREAMING_SNAKE_CASE    ,							keep_accents=__SCREAMING_SNAKE_CASE		)
											tokenizer.save_pretrained(self.tmpdirname		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					str													=	'''[PAD]'''
											lowercase_     :					Tuple													=	0

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(__SCREAMING_SNAKE_CASE		)    ,							__SCREAMING_SNAKE_CASE		)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(__SCREAMING_SNAKE_CASE		)    ,							__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Optional[int]													=	list(self.get_tokenizer().get_vocab().keys()		)

											self.assertEqual(vocab_keys[0]    ,							'''[PAD]'''		)
											self.assertEqual(vocab_keys[1]    ,							'''[CLS]'''		)
											self.assertEqual(vocab_keys[-1]    ,							'''j'''		)
											self.assertEqual(len(__SCREAMING_SNAKE_CASE		)    ,							10_12		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											self.assertEqual(self.get_tokenizer().vocab_size    ,							10_12		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	XLMProphetNetTokenizer(__SCREAMING_SNAKE_CASE    ,							keep_accents=__SCREAMING_SNAKE_CASE		)

											lowercase_     :					Optional[int]													=	tokenizer.tokenize('''This is a test'''		)
											self.assertListEqual(__SCREAMING_SNAKE_CASE    ,							['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']		)

											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE		)    ,							[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]    ,							)

											lowercase_     :					str													=	tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
											self.assertListEqual(
											    __SCREAMING_SNAKE_CASE    ,							[
											        SPIECE_UNDERLINE + '''I''',
											        SPIECE_UNDERLINE + '''was''',
											        SPIECE_UNDERLINE + '''b''',
											        '''or''',
											        '''n''',
											        SPIECE_UNDERLINE + '''in''',
											        SPIECE_UNDERLINE + '''''',
											        '''9''',
											        '''2''',
											        '''0''',
											        '''0''',
											        '''0''',
											        ''',''',
											        SPIECE_UNDERLINE + '''and''',
											        SPIECE_UNDERLINE + '''this''',
											        SPIECE_UNDERLINE + '''is''',
											        SPIECE_UNDERLINE + '''f''',
											        '''al''',
											        '''s''',
											        '''é''',
											        '''.''',
											    ]    ,							)
											lowercase_     :					Optional[int]													=	tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE		)
											self.assertListEqual(
											    __SCREAMING_SNAKE_CASE    ,							[
											        value + tokenizer.fairseq_offset
											        for value in [8, 21, 84, 55, 24, 19, 7, -9, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, -9, 4]
											    ]    ,							)

											lowercase_     :					int													=	tokenizer.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE		)
											self.assertListEqual(
											    __SCREAMING_SNAKE_CASE    ,							[
											        SPIECE_UNDERLINE + '''I''',
											        SPIECE_UNDERLINE + '''was''',
											        SPIECE_UNDERLINE + '''b''',
											        '''or''',
											        '''n''',
											        SPIECE_UNDERLINE + '''in''',
											        SPIECE_UNDERLINE + '''''',
											        '''[UNK]''',
											        '''2''',
											        '''0''',
											        '''0''',
											        '''0''',
											        ''',''',
											        SPIECE_UNDERLINE + '''and''',
											        SPIECE_UNDERLINE + '''this''',
											        SPIECE_UNDERLINE + '''is''',
											        SPIECE_UNDERLINE + '''f''',
											        '''al''',
											        '''s''',
											        '''[UNK]''',
											        '''.''',
											    ]    ,							)






					@cached_property
					def      _snake_case   (       self		):


											"""simple docstring"""





											return XLMProphetNetTokenizer.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased'''		)






					@slow
					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Dict													=	'''Hello World!'''
											lowercase_     :					str													=	[3_53_89, 66_72, 49, 2]
											self.assertListEqual(__SCREAMING_SNAKE_CASE    ,							self.big_tokenizer.encode(__SCREAMING_SNAKE_CASE		)		)






					@slow
					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Union[str, Any]													=	{'''input_ids''': [[1_10_73, 8_27_83, 18, 26, 8_27_83, 5_49, 5_15_40, 2_48, 1_72_09, 13_01, 2_17, 20, 21_51_86, 13_25, 1_47, 1_72_09, 13_01, 2_17, 20, 5_63_70, 53, 12_20_20, 20, 1_64_77, 27, 8_73_55, 45_48, 20, 47_28, 7_83_92, 17, 15_99_69, 18, 26, 2_44_91, 6_29, 15, 5_38, 2_27_04, 54_39, 15, 27_88, 2_44_91, 98_85, 15, 4_35_34, 6_05, 15, 8_14, 1_84_03, 3_32_00, 29, 15, 4_35_34, 2_44_58, 1_24_10, 1_11, 2_49_66, 8_36_69, 96_37, 14_40_68, 26, 8_50, 2_23_46, 27, 1_47, 2_49_66, 8_36_69, 8_34_90, 26, 3_91_13, 7_35, 27, 6_89, 6_56, 28_00, 13_39, 46_00, 53, 12_20_20, 11_57_85, 34, 8_16, 13_39, 4_68_87, 18, 1_47, 5_39_05, 19_51, 4_22_38, 4_11_70, 1_77_32, 8_34, 4_36, 15, 2_75_23, 9_87_33, 2_17, 1_47, 55_42, 49_81, 9_30, 1_73_47, 16, 2], [2_00_91, 6_29, 94, 8_27_86, 58, 4_90, 20, 15_28, 84, 5_39_05, 3_44, 8_05_92, 11_01_28, 1_88_22, 52_67, 13_06, 62, 15_25_37, 3_08, 79_97, 4_01, 12_44_27, 5_49, 3_54_42, 2_25, 1_09, 1_50_55, 2_57_48, 1_47, 71_19, 4_37_12, 34, 7_67, 13_53_66, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_92, 6_37_84, 11_94_66, 17, 14_78_08, 8_82_14, 18, 6_56, 81, 32, 32_96, 1_02_80, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=__SCREAMING_SNAKE_CASE    ,							model_name='''microsoft/xprophetnet-large-wiki100-cased'''    ,							revision='''1acad1643ddd54a44df6a1b797ada8373685d90e'''    ,							)





'''simple docstring'''







from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_lowercase			:  List[Any]   							=	logging.get_logger(__name__)

_lowercase			:  Optional[Any]   							=	{
    "google/vit-base-patch16-224": "https://huggingface.co/vit-base-patch16-224/resolve/main/config.json",
    # See all ViT models at https://huggingface.co/models?filter=vit
}






class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					'''vit'''






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE=7_68    ,							__SCREAMING_SNAKE_CASE=12    ,							__SCREAMING_SNAKE_CASE=12    ,							__SCREAMING_SNAKE_CASE=30_72    ,							__SCREAMING_SNAKE_CASE="gelu"    ,							__SCREAMING_SNAKE_CASE=0.0    ,							__SCREAMING_SNAKE_CASE=0.0    ,							__SCREAMING_SNAKE_CASE=0.02    ,							__SCREAMING_SNAKE_CASE=1E-1_2    ,							__SCREAMING_SNAKE_CASE=2_24    ,							__SCREAMING_SNAKE_CASE=16    ,							__SCREAMING_SNAKE_CASE=3    ,							__SCREAMING_SNAKE_CASE=True    ,							__SCREAMING_SNAKE_CASE=16    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											super().__init__(**__SCREAMING_SNAKE_CASE		)

											lowercase_     :					Union[str, Any]													=	hidden_size
											lowercase_     :					Dict													=	num_hidden_layers
											lowercase_     :					List[Any]													=	num_attention_heads
											lowercase_     :					Any													=	intermediate_size
											lowercase_     :					Union[str, Any]													=	hidden_act
											lowercase_     :					Dict													=	hidden_dropout_prob
											lowercase_     :					List[str]													=	attention_probs_dropout_prob
											lowercase_     :					Any													=	initializer_range
											lowercase_     :					Tuple													=	layer_norm_eps
											lowercase_     :					Union[str, Any]													=	image_size
											lowercase_     :					Tuple													=	patch_size
											lowercase_     :					Tuple													=	num_channels
											lowercase_     :					Union[str, Any]													=	qkv_bias
											lowercase_     :					List[Any]													=	encoder_stride







class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					version.parse('''1.11'''					)






					@property
					def      _snake_case   (       self		):


											"""simple docstring"""





											return OrderedDict(
											    [
											        ('''pixel_values''', {0: '''batch''', 1: '''num_channels''', 2: '''height''', 3: '''width'''}),
											    ]		)






					@property
					def      _snake_case   (       self		):


											"""simple docstring"""





											return 1E-4



