



import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCamelCase    :		Union[str, Any]		      =    get_tests_dir("""fixtures/test_sentencepiece.model""")


@require_sentencepiece
@require_tokenizers
class 						lowercase    (							SCREAMING_SNAKE_CASE_      ,    unittest.TestCase):
						'''simple docstring'''


						UpperCAmelCase			:		Union[str, Any]		=		XLNetTokenizer
						UpperCAmelCase			:		Any		=		XLNetTokenizerFast
						UpperCAmelCase			:		List[str]		=		True
						UpperCAmelCase			:		int		=		True







						def       lowerCamelCase_				(  self    :				str	):



											'''simple docstring'''


											super().setUp()

											# We have a SentencePiece fixture for testing
											SCREAMING_SNAKE_CASE							:     List[Any]						=				XLNetTokenizer(snake_case       ,	keep_accents=snake_case	)
											tokenizer.sanitize_special_tokens()
											tokenizer.save_pretrained(self.tmpdirname	)







						def       lowerCamelCase_				(  self    :				Dict	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				'<s>'
											SCREAMING_SNAKE_CASE							:     Tuple						=				1

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case	)       ,	snake_case	)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case	)       ,	snake_case	)







						def       lowerCamelCase_				(  self    :				str	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[Any]						=				list(self.get_tokenizer().get_vocab().keys()	)

											self.assertEqual(vocab_keys[0]       ,	'<unk>'	)
											self.assertEqual(vocab_keys[1]       ,	'<s>'	)
											self.assertEqual(vocab_keys[-1]       ,	'<eod>'	)
											self.assertEqual(len(snake_case	)       ,	1006	)







						def       lowerCamelCase_				(  self    :				Tuple	):



											'''simple docstring'''


											self.assertEqual(self.get_tokenizer().vocab_size       ,	1000	)







						def       lowerCamelCase_				(  self    :				Dict	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     str						=				XLNetTokenizer(snake_case       ,	keep_accents=snake_case	)

											SCREAMING_SNAKE_CASE							:     List[str]						=				tokenizer.tokenize('This is a test'	)
											self.assertListEqual(snake_case       ,	['▁This', '▁is', '▁a', '▁t', 'est']	)

											self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case	)       ,	[285, 46, 10, 170, 382]	)

											SCREAMING_SNAKE_CASE							:     List[str]						=				tokenizer.tokenize('I was born in 92000, and this is falsé.'	)
											self.assertListEqual(
											    snake_case       ,	[
											        SPIECE_UNDERLINE + 'I',
											        SPIECE_UNDERLINE + 'was',
											        SPIECE_UNDERLINE + 'b',
											        'or',
											        'n',
											        SPIECE_UNDERLINE + 'in',
											        SPIECE_UNDERLINE + '',
											        '9',
											        '2',
											        '0',
											        '0',
											        '0',
											        ',',
											        SPIECE_UNDERLINE + 'and',
											        SPIECE_UNDERLINE + 'this',
											        SPIECE_UNDERLINE + 'is',
											        SPIECE_UNDERLINE + 'f',
											        'al',
											        's',
											        'é',
											        '.',
											    ]       ,	)
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				tokenizer.convert_tokens_to_ids(snake_case	)
											self.assertListEqual(snake_case       ,	[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]	)

											SCREAMING_SNAKE_CASE							:     int						=				tokenizer.convert_ids_to_tokens(snake_case	)
											self.assertListEqual(
											    snake_case       ,	[
											        SPIECE_UNDERLINE + 'I',
											        SPIECE_UNDERLINE + 'was',
											        SPIECE_UNDERLINE + 'b',
											        'or',
											        'n',
											        SPIECE_UNDERLINE + 'in',
											        SPIECE_UNDERLINE + '',
											        '<unk>',
											        '2',
											        '0',
											        '0',
											        '0',
											        ',',
											        SPIECE_UNDERLINE + 'and',
											        SPIECE_UNDERLINE + 'this',
											        SPIECE_UNDERLINE + 'is',
											        SPIECE_UNDERLINE + 'f',
											        'al',
											        's',
											        '<unk>',
											        '.',
											    ]       ,	)







						def       lowerCamelCase_				(  self    :				str	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				XLNetTokenizer(snake_case       ,	do_lower_case=snake_case	)
											SCREAMING_SNAKE_CASE							:     Dict						=				tokenizer.tokenize('I was born in 92000, and this is falsé.'	)
											self.assertListEqual(
											    snake_case       ,	[
											        SPIECE_UNDERLINE + '',
											        'i',
											        SPIECE_UNDERLINE + 'was',
											        SPIECE_UNDERLINE + 'b',
											        'or',
											        'n',
											        SPIECE_UNDERLINE + 'in',
											        SPIECE_UNDERLINE + '',
											        '9',
											        '2',
											        '0',
											        '0',
											        '0',
											        ',',
											        SPIECE_UNDERLINE + 'and',
											        SPIECE_UNDERLINE + 'this',
											        SPIECE_UNDERLINE + 'is',
											        SPIECE_UNDERLINE + 'f',
											        'al',
											        'se',
											        '.',
											    ]       ,	)
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)       ,	['▁he', 'll', 'o']	)







						def       lowerCamelCase_				(  self    :				List[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				XLNetTokenizer(snake_case       ,	do_lower_case=snake_case	)
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				tokenizer.tokenize('I was born in 92000, and this is falsé.'	)
											self.assertListEqual(
											    snake_case       ,	[
											        SPIECE_UNDERLINE + 'I',
											        SPIECE_UNDERLINE + 'was',
											        SPIECE_UNDERLINE + 'b',
											        'or',
											        'n',
											        SPIECE_UNDERLINE + 'in',
											        SPIECE_UNDERLINE + '',
											        '9',
											        '2',
											        '0',
											        '0',
											        '0',
											        ',',
											        SPIECE_UNDERLINE + 'and',
											        SPIECE_UNDERLINE + 'this',
											        SPIECE_UNDERLINE + 'is',
											        SPIECE_UNDERLINE + 'f',
											        'al',
											        'se',
											        '.',
											    ]       ,	)







						@slow
						def       lowerCamelCase_				(  self    :				Dict	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				XLNetTokenizer.from_pretrained('xlnet-base-cased'	)

											SCREAMING_SNAKE_CASE							:     Any						=				tokenizer.encode('sequence builders'       ,	add_special_tokens=snake_case	)
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				tokenizer.encode('multi-sequence build'       ,	add_special_tokens=snake_case	)

											SCREAMING_SNAKE_CASE							:     Any						=				tokenizer.build_inputs_with_special_tokens(snake_case	)
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				tokenizer.build_inputs_with_special_tokens(snake_case       ,	snake_case	)

											assert encoded_sentence == text + [4, 3]
											assert encoded_pair == text + [4] + text_a + [4, 3]







						@slow
						def       lowerCamelCase_				(  self    :				str	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				{'input_ids': [[17, 21442, 270, 17, 10, 14645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 22018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 14431, 13, 5500, 11, 1176, 580, 13, 16819, 4797, 23, 17, 10, 17135, 658, 19, 457, 7932, 13, 184, 19, 3154, 17135, 6468, 19, 1404, 12269, 19, 4229, 5356, 16264, 46, 19, 17, 20545, 10395, 9, 9, 9, 11, 28, 6421, 9531, 20729, 17, 10, 353, 17022, 11, 21, 6421, 9531, 16949, 17, 10, 11509, 753, 11, 33, 95, 2421, 7385, 956, 14431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 24738, 19, 13203, 658, 218, 787, 21, 430, 18482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22178, 27, 1064, 22, 956, 13, 11101, 1429, 5854, 24313, 18953, 40, 422, 24366, 68, 1758, 37, 10483, 14257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 13894, 3380, 23, 95, 18, 17634, 2288, 9, 4, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=snake_case       ,	model_name='xlnet-base-cased'       ,	revision='c841166438c31ec7ca9a106dee7bb312b73ae511'       ,	)




import unittest

import numpy as np

from transformers import DistilBertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
			import jax.numpy as jnp

			from transformers.models.distilbert.modeling_flax_distilbert import (
			    FlaxDistilBertForMaskedLM,
			    FlaxDistilBertForMultipleChoice,
			    FlaxDistilBertForQuestionAnswering,
			    FlaxDistilBertForSequenceClassification,
			    FlaxDistilBertForTokenClassification,
			    FlaxDistilBertModel,
			)


class 						lowercase    (							unittest.TestCase):
						'''simple docstring'''


						def __init__(  self    :				int       ,	snake_case    :				Optional[Any]       ,	snake_case    :				Optional[Any]=13       ,	snake_case    :				str=7       ,	snake_case    :				Optional[Any]=True       ,	snake_case    :				int=True       ,	snake_case    :				str=True       ,	snake_case    :				List[str]=True       ,	snake_case    :				Optional[Any]=99       ,	snake_case    :				Optional[int]=32       ,	snake_case    :				Optional[int]=5       ,	snake_case    :				Optional[Any]=4       ,	snake_case    :				Optional[Any]=37       ,	snake_case    :				str="gelu"       ,	snake_case    :				List[str]=0.1       ,	snake_case    :				List[str]=0.1       ,	snake_case    :				int=512       ,	snake_case    :				Union[str, Any]=16       ,	snake_case    :				Optional[Any]=2       ,	snake_case    :				int=0.02       ,	snake_case    :				str=4       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				parent
											SCREAMING_SNAKE_CASE							:     str						=				batch_size
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				seq_length
											SCREAMING_SNAKE_CASE							:     int						=				is_training
											SCREAMING_SNAKE_CASE							:     List[str]						=				use_attention_mask
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				use_token_type_ids
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				use_labels
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				vocab_size
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				hidden_size
											SCREAMING_SNAKE_CASE							:     Any						=				num_hidden_layers
											SCREAMING_SNAKE_CASE							:     Tuple						=				num_attention_heads
											SCREAMING_SNAKE_CASE							:     Any						=				intermediate_size
											SCREAMING_SNAKE_CASE							:     Tuple						=				hidden_act
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				hidden_dropout_prob
											SCREAMING_SNAKE_CASE							:     str						=				attention_probs_dropout_prob
											SCREAMING_SNAKE_CASE							:     Tuple						=				max_position_embeddings
											SCREAMING_SNAKE_CASE							:     Any						=				type_vocab_size
											SCREAMING_SNAKE_CASE							:     List[Any]						=				type_sequence_label_size
											SCREAMING_SNAKE_CASE							:     Tuple						=				initializer_range
											SCREAMING_SNAKE_CASE							:     List[str]						=				num_choices







						def       lowerCamelCase_				(  self    :				List[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				ids_tensor([self.batch_size, self.seq_length]       ,	self.vocab_size	)

											SCREAMING_SNAKE_CASE							:     Optional[int]						=				None
											if self.use_attention_mask:
																SCREAMING_SNAKE_CASE							:     Any						=				random_attention_mask([self.batch_size, self.seq_length]	)

											SCREAMING_SNAKE_CASE							:     Tuple						=				DistilBertConfig(
											    vocab_size=self.vocab_size       ,	dim=self.hidden_size       ,	n_layers=self.num_hidden_layers       ,	n_heads=self.num_attention_heads       ,	hidden_dim=self.intermediate_size       ,	hidden_act=self.hidden_act       ,	dropout=self.hidden_dropout_prob       ,	attention_dropout=self.attention_probs_dropout_prob       ,	max_position_embeddings=self.max_position_embeddings       ,	initializer_range=self.initializer_range       ,	tie_weights_=snake_case       ,	)

											return config, input_ids, attention_mask







						def       lowerCamelCase_				(  self    :				int	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				self.prepare_config_and_inputs()
											SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE							:     Optional[int]						=				config_and_inputs
											SCREAMING_SNAKE_CASE							:     int						=				{'input_ids': input_ids, 'attention_mask': attention_mask}
											return config, inputs_dict


@require_flax
class 						lowercase    (							SCREAMING_SNAKE_CASE_      ,    unittest.TestCase):
						'''simple docstring'''


						UpperCAmelCase			:		int		=		(
						    (
						        FlaxDistilBertModel,
						        FlaxDistilBertForMaskedLM,
						        FlaxDistilBertForMultipleChoice,
						        FlaxDistilBertForQuestionAnswering,
						        FlaxDistilBertForSequenceClassification,
						        FlaxDistilBertForTokenClassification,
						        FlaxDistilBertForQuestionAnswering,
						    )
						    if is_flax_available()
						    else ()
						)







						def       lowerCamelCase_				(  self    :				Optional[int]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     str						=				FlaxDistilBertModelTester(self	)







						@slow
						def       lowerCamelCase_				(  self    :				Any	):



											'''simple docstring'''


											for model_class_name in self.all_model_classes:
																SCREAMING_SNAKE_CASE							:     Dict						=				model_class_name.from_pretrained('distilbert-base-uncased'	)
																SCREAMING_SNAKE_CASE							:     Dict						=				model(np.ones((1, 1)	)	)
																self.assertIsNotNone(snake_case	)


@require_flax
class 						lowercase    (							unittest.TestCase):
						'''simple docstring'''


						@slow
						def       lowerCamelCase_				(  self    :				Tuple	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[Any]						=				FlaxDistilBertModel.from_pretrained('distilbert-base-uncased'	)
											SCREAMING_SNAKE_CASE							:     List[str]						=				np.array([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]	)
											SCREAMING_SNAKE_CASE							:     Dict						=				np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]	)
											SCREAMING_SNAKE_CASE							:     str						=				model(snake_case       ,	attention_mask=snake_case	)[0]
											SCREAMING_SNAKE_CASE							:     int						=				(1, 11, 768)
											self.assertEqual(output.shape       ,	snake_case	)
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				np.array([[[-0.1639, 0.3299, 0.1648], [-0.1746, 0.3289, 0.1710], [-0.1884, 0.3357, 0.1810]]]	)

											self.assertTrue(jnp.allclose(output[:, 1:4, 1:4]       ,	snake_case       ,	atol=1E-4	)	)