'''simple docstring'''




import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


snake_case	=      get_tests_dir('''fixtures/test_sentencepiece.model''')





@require_sentencepiece
@require_tokenizers
class 							SCREAMING_SNAKE_CASE (					__a						,    unittest.TestCase			):
  """simple docstring"""







  __A  =		XLNetTokenizer
  __A  =		XLNetTokenizerFast
  __A  =		True
  __A  =		True



  def 	a (	self :       int       ):


        """simple docstring"""



        super().setUp()

        # We have a SentencePiece fixture for testing
        _lowerCAmelCase		 = XLNetTokenizer(__lowerCAmelCase       ,       keep_accents=__lowerCAmelCase       )
        tokenizer.sanitize_special_tokens()
        tokenizer.save_pretrained(self.tmpdirname       )



  def 	a (	self :       int       ):


        """simple docstring"""



        _lowerCAmelCase		 = '<s>'
        _lowerCAmelCase		 = 1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCAmelCase       )       ,       __lowerCAmelCase       )
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCAmelCase       )       ,       __lowerCAmelCase       )



  def 	a (	self :       Optional[Any]       ):


        """simple docstring"""



        _lowerCAmelCase		 = list(self.get_tokenizer().get_vocab().keys()       )

        self.assertEqual(vocab_keys[0]       ,       '<unk>'       )
        self.assertEqual(vocab_keys[1]       ,       '<s>'       )
        self.assertEqual(vocab_keys[-1]       ,       '<eod>'       )
        self.assertEqual(len(__lowerCAmelCase       )       ,       1006       )



  def 	a (	self :       List[str]       ):


        """simple docstring"""



        self.assertEqual(self.get_tokenizer().vocab_size       ,       1000       )



  def 	a (	self :       Union[str, Any]       ):


        """simple docstring"""



        _lowerCAmelCase		 = XLNetTokenizer(__lowerCAmelCase       ,       keep_accents=__lowerCAmelCase       )

        _lowerCAmelCase		 = tokenizer.tokenize('This is a test'       )
        self.assertListEqual(__lowerCAmelCase       ,       ['▁This', '▁is', '▁a', '▁t', 'est']       )

        self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase       )       ,       [285, 46, 10, 170, 382]       )

        _lowerCAmelCase		 = tokenizer.tokenize('I was born in 92000, and this is falsé.'       )
        self.assertListEqual(
            __lowerCAmelCase       ,       [
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '9',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                'é',
                '.',
            ]       ,       )
        _lowerCAmelCase		 = tokenizer.convert_tokens_to_ids(__lowerCAmelCase       )
        self.assertListEqual(__lowerCAmelCase       ,       [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]       )

        _lowerCAmelCase		 = tokenizer.convert_ids_to_tokens(__lowerCAmelCase       )
        self.assertListEqual(
            __lowerCAmelCase       ,       [
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '<unk>',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                '<unk>',
                '.',
            ]       ,       )



  def 	a (	self :       str       ):


        """simple docstring"""



        _lowerCAmelCase		 = XLNetTokenizer(__lowerCAmelCase       ,       do_lower_case=__lowerCAmelCase       )
        _lowerCAmelCase		 = tokenizer.tokenize('I was born in 92000, and this is falsé.'       )
        self.assertListEqual(
            __lowerCAmelCase       ,       [
                SPIECE_UNDERLINE + '',
                'i',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '9',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                'se',
                '.',
            ]       ,       )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )       ,       ['▁he', 'll', 'o']       )



  def 	a (	self :       str       ):


        """simple docstring"""



        _lowerCAmelCase		 = XLNetTokenizer(__lowerCAmelCase       ,       do_lower_case=__lowerCAmelCase       )
        _lowerCAmelCase		 = tokenizer.tokenize('I was born in 92000, and this is falsé.'       )
        self.assertListEqual(
            __lowerCAmelCase       ,       [
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '9',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                'se',
                '.',
            ]       ,       )



  @slow
  def 	a (	self :       Union[str, Any]       ):


        """simple docstring"""



        _lowerCAmelCase		 = XLNetTokenizer.from_pretrained('xlnet-base-cased'       )

        _lowerCAmelCase		 = tokenizer.encode('sequence builders'       ,       add_special_tokens=__lowerCAmelCase       )
        _lowerCAmelCase		 = tokenizer.encode('multi-sequence build'       ,       add_special_tokens=__lowerCAmelCase       )

        _lowerCAmelCase		 = tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase       )
        _lowerCAmelCase		 = tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase       ,       __lowerCAmelCase       )

        assert encoded_sentence == text + [4, 3]
        assert encoded_pair == text + [4] + text_a + [4, 3]



  @slow
  def 	a (	self :       Dict       ):


        """simple docstring"""



        _lowerCAmelCase		 = {'input_ids': [[17, 2_1442, 270, 17, 10, 1_4645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 2_2018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 1_4431, 13, 5500, 11, 1176, 580, 13, 1_6819, 4797, 23, 17, 10, 1_7135, 658, 19, 457, 7932, 13, 184, 19, 3154, 1_7135, 6468, 19, 1404, 1_2269, 19, 4229, 5356, 1_6264, 46, 19, 17, 2_0545, 1_0395, 9, 9, 9, 11, 28, 6421, 9531, 2_0729, 17, 10, 353, 1_7022, 11, 21, 6421, 9531, 1_6949, 17, 10, 1_1509, 753, 11, 33, 95, 2421, 7385, 956, 1_4431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 2_4738, 19, 1_3203, 658, 218, 787, 21, 430, 1_8482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 2_2178, 27, 1064, 22, 956, 13, 1_1101, 1429, 5854, 2_4313, 1_8953, 40, 422, 2_4366, 68, 1758, 37, 1_0483, 1_4257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 1_3894, 3380, 23, 95, 18, 1_7634, 2288, 9, 4, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=__lowerCAmelCase       ,       model_name='xlnet-base-cased'       ,       revision='c841166438c31ec7ca9a106dee7bb312b73ae511'       ,       )













"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


__SCREAMING_SNAKE_CASE   =   {
    'configuration_transfo_xl': ['TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP', 'TransfoXLConfig'],
    'tokenization_transfo_xl': ['TransfoXLCorpus', 'TransfoXLTokenizer'],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__SCREAMING_SNAKE_CASE   =   [
		    'TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'AdaptiveEmbedding',
		    'TransfoXLForSequenceClassification',
		    'TransfoXLLMHeadModel',
		    'TransfoXLModel',
		    'TransfoXLPreTrainedModel',
		    'load_tf_weights_in_transfo_xl',
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__SCREAMING_SNAKE_CASE   =   [
		    'TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'TFAdaptiveEmbedding',
		    'TFTransfoXLForSequenceClassification',
		    'TFTransfoXLLMHeadModel',
		    'TFTransfoXLMainLayer',
		    'TFTransfoXLModel',
		    'TFTransfoXLPreTrainedModel',
		]


if TYPE_CHECKING:
		from .configuration_transfo_xl import TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig
		from .tokenization_transfo_xl import TransfoXLCorpus, TransfoXLTokenizer

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_transfo_xl import (
				    TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
				    AdaptiveEmbedding,
				    TransfoXLForSequenceClassification,
				    TransfoXLLMHeadModel,
				    TransfoXLModel,
				    TransfoXLPreTrainedModel,
				    load_tf_weights_in_transfo_xl,
				)

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_transfo_xl import (
				    TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFAdaptiveEmbedding,
				    TFTransfoXLForSequenceClassification,
				    TFTransfoXLLMHeadModel,
				    TFTransfoXLMainLayer,
				    TFTransfoXLModel,
				    TFTransfoXLPreTrainedModel,
				)

else:
		import sys

		__SCREAMING_SNAKE_CASE   =   _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)