

'''simple docstring'''





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_							=					get_tests_dir("""fixtures/test_sentencepiece.model""")





@require_sentencepiece
@require_tokenizers
class 		__SCREAMING_SNAKE_CASE       ( lowercase__	,       unittest.TestCase			):
				lowerCamelCase_							=						XLNetTokenizer
				lowerCamelCase_							=						XLNetTokenizerFast
				lowerCamelCase_							=						True
				lowerCamelCase_							=						True







				def    lowerCamelCase_  (					self    :		Optional[int]      ):
										'''simple docstring'''
										super().setUp()

										# We have a SentencePiece fixture for testing
										lowercase   :       int								=XLNetTokenizer(UpperCAmelCase__		,				keep_accents=UpperCAmelCase__      )
										tokenizer.sanitize_special_tokens()
										tokenizer.save_pretrained(self.tmpdirname      )







				def    lowerCamelCase_  (					self    :		Tuple      ):
										'''simple docstring'''
										lowercase   :       str								='''<s>'''
										lowercase   :       List[str]								=1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase__      )		,				UpperCAmelCase__      )
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase__      )		,				UpperCAmelCase__      )







				def    lowerCamelCase_  (					self    :		Tuple      ):
										'''simple docstring'''
										lowercase   :       Tuple								=list(self.get_tokenizer().get_vocab().keys()      )

										self.assertEqual(vocab_keys[0]		,				'''<unk>'''      )
										self.assertEqual(vocab_keys[1]		,				'''<s>'''      )
										self.assertEqual(vocab_keys[-1]		,				'''<eod>'''      )
										self.assertEqual(len(UpperCAmelCase__      )		,				1006      )







				def    lowerCamelCase_  (					self    :		int      ):
										'''simple docstring'''
										self.assertEqual(self.get_tokenizer().vocab_size		,				1000      )







				def    lowerCamelCase_  (					self    :		List[Any]      ):
										'''simple docstring'''
										lowercase   :       Union[str, Any]								=XLNetTokenizer(UpperCAmelCase__		,				keep_accents=UpperCAmelCase__      )

										lowercase   :       Optional[int]								=tokenizer.tokenize('''This is a test'''      )
										self.assertListEqual(UpperCAmelCase__		,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

										self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__      )		,				[285, 46, 10, 170, 382]      )

										lowercase   :       Union[str, Any]								=tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
										self.assertListEqual(
										    UpperCAmelCase__		,				[
										        SPIECE_UNDERLINE + '''I''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''9''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''s''',
										        '''é''',
										        '''.''',
										    ]		,				)
										lowercase   :       List[str]								=tokenizer.convert_tokens_to_ids(UpperCAmelCase__      )
										self.assertListEqual(UpperCAmelCase__		,				[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]      )

										lowercase   :       Dict								=tokenizer.convert_ids_to_tokens(UpperCAmelCase__      )
										self.assertListEqual(
										    UpperCAmelCase__		,				[
										        SPIECE_UNDERLINE + '''I''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''<unk>''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''s''',
										        '''<unk>''',
										        '''.''',
										    ]		,				)







				def    lowerCamelCase_  (					self    :		Any      ):
										'''simple docstring'''
										lowercase   :       int								=XLNetTokenizer(UpperCAmelCase__		,				do_lower_case=UpperCAmelCase__      )
										lowercase   :       Any								=tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
										self.assertListEqual(
										    UpperCAmelCase__		,				[
										        SPIECE_UNDERLINE + '''''',
										        '''i''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''9''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''se''',
										        '''.''',
										    ]		,				)
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )		,				['''▁he''', '''ll''', '''o''']      )







				def    lowerCamelCase_  (					self    :		int      ):
										'''simple docstring'''
										lowercase   :       Union[str, Any]								=XLNetTokenizer(UpperCAmelCase__		,				do_lower_case=UpperCAmelCase__      )
										lowercase   :       Tuple								=tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
										self.assertListEqual(
										    UpperCAmelCase__		,				[
										        SPIECE_UNDERLINE + '''I''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''9''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''se''',
										        '''.''',
										    ]		,				)







				@slow
				def    lowerCamelCase_  (					self    :		Union[str, Any]      ):
										'''simple docstring'''
										lowercase   :       Dict								=XLNetTokenizer.from_pretrained('''xlnet-base-cased'''      )

										lowercase   :       List[Any]								=tokenizer.encode('''sequence builders'''		,				add_special_tokens=UpperCAmelCase__      )
										lowercase   :       Optional[int]								=tokenizer.encode('''multi-sequence build'''		,				add_special_tokens=UpperCAmelCase__      )

										lowercase   :       List[Any]								=tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__      )
										lowercase   :       Optional[Any]								=tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__		,				UpperCAmelCase__      )

										assert encoded_sentence == text + [4, 3]
										assert encoded_pair == text + [4] + text_a + [4, 3]







				@slow
				def    lowerCamelCase_  (					self    :		Dict      ):
										'''simple docstring'''
										lowercase   :       List[str]								={'''input_ids''': [[17, 21442, 270, 17, 10, 14645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 22018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 14431, 13, 5500, 11, 1176, 580, 13, 16819, 4797, 23, 17, 10, 17135, 658, 19, 457, 7932, 13, 184, 19, 3154, 17135, 6468, 19, 1404, 12269, 19, 4229, 5356, 16264, 46, 19, 17, 20545, 10395, 9, 9, 9, 11, 28, 6421, 9531, 20729, 17, 10, 353, 17022, 11, 21, 6421, 9531, 16949, 17, 10, 11509, 753, 11, 33, 95, 2421, 7385, 956, 14431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 24738, 19, 13203, 658, 218, 787, 21, 430, 18482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22178, 27, 1064, 22, 956, 13, 11101, 1429, 5854, 24313, 18953, 40, 422, 24366, 68, 1758, 37, 10483, 14257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 13894, 3380, 23, 95, 18, 17634, 2288, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=UpperCAmelCase__		,				model_name='''xlnet-base-cased'''		,				revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''		,				)








'''simple docstring'''





import unittest

import numpy as np

from transformers import RobertaConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
							from transformers.models.roberta.modeling_flax_roberta import (
							    FlaxRobertaForCausalLM,
							    FlaxRobertaForMaskedLM,
							    FlaxRobertaForMultipleChoice,
							    FlaxRobertaForQuestionAnswering,
							    FlaxRobertaForSequenceClassification,
							    FlaxRobertaForTokenClassification,
							    FlaxRobertaModel,
							)





class 		__SCREAMING_SNAKE_CASE       ( unittest.TestCase			):







				def __init__(					self    :		Optional[Any]		,				UpperCAmelCase__    :		Tuple		,				UpperCAmelCase__    :		str=13		,				UpperCAmelCase__    :		Optional[Any]=7		,				UpperCAmelCase__    :		Dict=True		,				UpperCAmelCase__    :		int=True		,				UpperCAmelCase__    :		int=True		,				UpperCAmelCase__    :		Dict=True		,				UpperCAmelCase__    :		Dict=99		,				UpperCAmelCase__    :		str=32		,				UpperCAmelCase__    :		Optional[Any]=5		,				UpperCAmelCase__    :		Any=4		,				UpperCAmelCase__    :		List[Any]=37		,				UpperCAmelCase__    :		str="gelu"		,				UpperCAmelCase__    :		Dict=0.1		,				UpperCAmelCase__    :		List[Any]=0.1		,				UpperCAmelCase__    :		Optional[Any]=512		,				UpperCAmelCase__    :		Any=16		,				UpperCAmelCase__    :		str=2		,				UpperCAmelCase__    :		Tuple=0.02		,				UpperCAmelCase__    :		int=4		,				):
										'''simple docstring'''
										lowercase   :       int								=parent
										lowercase   :       List[str]								=batch_size
										lowercase   :       str								=seq_length
										lowercase   :       Optional[Any]								=is_training
										lowercase   :       Union[str, Any]								=use_attention_mask
										lowercase   :       Optional[Any]								=use_token_type_ids
										lowercase   :       Tuple								=use_labels
										lowercase   :       List[str]								=vocab_size
										lowercase   :       List[str]								=hidden_size
										lowercase   :       Tuple								=num_hidden_layers
										lowercase   :       Any								=num_attention_heads
										lowercase   :       List[str]								=intermediate_size
										lowercase   :       Optional[Any]								=hidden_act
										lowercase   :       Dict								=hidden_dropout_prob
										lowercase   :       List[Any]								=attention_probs_dropout_prob
										lowercase   :       Optional[Any]								=max_position_embeddings
										lowercase   :       Tuple								=type_vocab_size
										lowercase   :       Optional[int]								=type_sequence_label_size
										lowercase   :       Optional[Any]								=initializer_range
										lowercase   :       Optional[int]								=num_choices







				def    lowerCamelCase_  (					self    :		str      ):
										'''simple docstring'''
										lowercase   :       Optional[Any]								=ids_tensor([self.batch_size, self.seq_length]		,				self.vocab_size      )

										lowercase   :       Union[str, Any]								=None
										if self.use_attention_mask:
																lowercase   :       List[str]								=random_attention_mask([self.batch_size, self.seq_length]      )

										lowercase   :       Tuple								=None
										if self.use_token_type_ids:
																lowercase   :       str								=ids_tensor([self.batch_size, self.seq_length]		,				self.type_vocab_size      )

										lowercase   :       int								=RobertaConfig(
										    vocab_size=self.vocab_size		,				hidden_size=self.hidden_size		,				num_hidden_layers=self.num_hidden_layers		,				num_attention_heads=self.num_attention_heads		,				intermediate_size=self.intermediate_size		,				hidden_act=self.hidden_act		,				hidden_dropout_prob=self.hidden_dropout_prob		,				attention_probs_dropout_prob=self.attention_probs_dropout_prob		,				max_position_embeddings=self.max_position_embeddings		,				type_vocab_size=self.type_vocab_size		,				is_decoder=UpperCAmelCase__		,				initializer_range=self.initializer_range		,				)

										return config, input_ids, token_type_ids, attention_mask







				def    lowerCamelCase_  (					self    :		Dict      ):
										'''simple docstring'''
										lowercase   :       List[Any]								=self.prepare_config_and_inputs()
										lowercase		,  lowercase		,  lowercase		,  lowercase   :       str								=config_and_inputs
										lowercase   :       Optional[Any]								={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': attention_mask}
										return config, inputs_dict







				def    lowerCamelCase_  (					self    :		Optional[Any]      ):
										'''simple docstring'''
										lowercase   :       List[str]								=self.prepare_config_and_inputs()
										lowercase		,  lowercase		,  lowercase		,  lowercase   :       Any								=config_and_inputs

										lowercase   :       List[str]								=True
										lowercase   :       Optional[Any]								=floats_tensor([self.batch_size, self.seq_length, self.hidden_size]      )
										lowercase   :       str								=ids_tensor([self.batch_size, self.seq_length]		,				vocab_size=2      )

										return (
										    config,
										    input_ids,
										    token_type_ids,
										    encoder_hidden_states,
										    encoder_attention_mask,
										)





@require_flax
class 		__SCREAMING_SNAKE_CASE       ( lowercase__	,       unittest.TestCase			):
				lowerCamelCase_							=						True

				lowerCamelCase_							=						(
				    (
				        FlaxRobertaModel,
				        FlaxRobertaForCausalLM,
				        FlaxRobertaForMaskedLM,
				        FlaxRobertaForSequenceClassification,
				        FlaxRobertaForTokenClassification,
				        FlaxRobertaForMultipleChoice,
				        FlaxRobertaForQuestionAnswering,
				    )
				    if is_flax_available()
				    else ()
				)







				def    lowerCamelCase_  (					self    :		List[Any]      ):
										'''simple docstring'''
										lowercase   :       str								=FlaxRobertaModelTester(self      )







				@slow
				def    lowerCamelCase_  (					self    :		Optional[int]      ):
										'''simple docstring'''
										for model_class_name in self.all_model_classes:
																lowercase   :       Optional[int]								=model_class_name.from_pretrained('''roberta-base'''		,				from_pt=UpperCAmelCase__      )
																lowercase   :       List[Any]								=model(np.ones((1, 1)      )      )
																self.assertIsNotNone(UpperCAmelCase__      )





