import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowercase_     :   List[Any] 				=				get_tests_dir('fixtures/test_sentencepiece.model')






@require_sentencepiece
@require_tokenizers
class 		_lowerCamelCase		(   UpperCamelCase_    ,     unittest.TestCase    ):
				__a  =      XLNetTokenizer
				__a  =      XLNetTokenizerFast
				__a  =      True
				__a  =      True
				def 						UpperCamelCase_				(  self   )       ->					Any:
					super().setUp()

					# We have a SentencePiece fixture for testing
					SCREAMING_SNAKE_CASE__:   int=      XLNetTokenizer(_lowerCAmelCase , keep_accents=_lowerCAmelCase   )
					tokenizer.sanitize_special_tokens()
					tokenizer.save_pretrained(self.tmpdirname   )
				def 						UpperCamelCase_				(  self   )       ->					Tuple:
					SCREAMING_SNAKE_CASE__:   Tuple=      '''<s>'''
					SCREAMING_SNAKE_CASE__:   List[str]=      1

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCAmelCase   ) , _lowerCAmelCase   )
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCAmelCase   ) , _lowerCAmelCase   )
				def 						UpperCamelCase_				(  self   )       ->					List[str]:
					SCREAMING_SNAKE_CASE__:   int=      list(self.get_tokenizer().get_vocab().keys()   )

					self.assertEqual(vocab_keys[0] , '''<unk>'''   )
					self.assertEqual(vocab_keys[1] , '''<s>'''   )
					self.assertEqual(vocab_keys[-1] , '''<eod>'''   )
					self.assertEqual(len(_lowerCAmelCase   ) , 1006   )
				def 						UpperCamelCase_				(  self   )       ->					Tuple:
					self.assertEqual(self.get_tokenizer().vocab_size , 1000   )
				def 						UpperCamelCase_				(  self   )       ->					List[Any]:
					SCREAMING_SNAKE_CASE__:   Dict=      XLNetTokenizer(_lowerCAmelCase , keep_accents=_lowerCAmelCase   )

					SCREAMING_SNAKE_CASE__:   str=      tokenizer.tokenize('''This is a test'''   )
					self.assertListEqual(_lowerCAmelCase , ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']   )

					self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase   ) , [285, 46, 10, 170, 382]   )

					SCREAMING_SNAKE_CASE__:   Optional[Any]=      tokenizer.tokenize('''I was born in 92000, and this is falsé.'''   )
					self.assertListEqual(
					    _lowerCAmelCase , [
					        SPIECE_UNDERLINE + '''I''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''9''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''s''',
					        '''é''',
					        '''.''',
					    ] , )
					SCREAMING_SNAKE_CASE__:   List[Any]=      tokenizer.convert_tokens_to_ids(_lowerCAmelCase   )
					self.assertListEqual(_lowerCAmelCase , [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]   )

					SCREAMING_SNAKE_CASE__:   List[Any]=      tokenizer.convert_ids_to_tokens(_lowerCAmelCase   )
					self.assertListEqual(
					    _lowerCAmelCase , [
					        SPIECE_UNDERLINE + '''I''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''<unk>''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''s''',
					        '''<unk>''',
					        '''.''',
					    ] , )
				def 						UpperCamelCase_				(  self   )       ->					List[Any]:
					SCREAMING_SNAKE_CASE__:   List[Any]=      XLNetTokenizer(_lowerCAmelCase , do_lower_case=_lowerCAmelCase   )
					SCREAMING_SNAKE_CASE__:   int=      tokenizer.tokenize('''I was born in 92000, and this is falsé.'''   )
					self.assertListEqual(
					    _lowerCAmelCase , [
					        SPIECE_UNDERLINE + '''''',
					        '''i''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''9''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''se''',
					        '''.''',
					    ] , )
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   ) , ['''▁he''', '''ll''', '''o''']   )
				def 						UpperCamelCase_				(  self   )       ->					int:
					SCREAMING_SNAKE_CASE__:   Union[str, Any]=      XLNetTokenizer(_lowerCAmelCase , do_lower_case=_lowerCAmelCase   )
					SCREAMING_SNAKE_CASE__:   List[str]=      tokenizer.tokenize('''I was born in 92000, and this is falsé.'''   )
					self.assertListEqual(
					    _lowerCAmelCase , [
					        SPIECE_UNDERLINE + '''I''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''9''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''se''',
					        '''.''',
					    ] , )
				@slow
				def 						UpperCamelCase_				(  self   )       ->					Tuple:
					SCREAMING_SNAKE_CASE__:   List[Any]=      XLNetTokenizer.from_pretrained('''xlnet-base-cased'''   )

					SCREAMING_SNAKE_CASE__:   List[str]=      tokenizer.encode('''sequence builders''' , add_special_tokens=_lowerCAmelCase   )
					SCREAMING_SNAKE_CASE__:   Dict=      tokenizer.encode('''multi-sequence build''' , add_special_tokens=_lowerCAmelCase   )

					SCREAMING_SNAKE_CASE__:   int=      tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase   )
					SCREAMING_SNAKE_CASE__:   Tuple=      tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase , _lowerCAmelCase   )

					assert encoded_sentence == text + [4, 3]
					assert encoded_pair == text + [4] + text_a + [4, 3]



				@slow
				def 						UpperCamelCase_				(  self   )       ->					List[str]:
					# fmt: off
					SCREAMING_SNAKE_CASE__:   Dict=      {'''input_ids''': [[17, 21442, 270, 17, 10, 14645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 22018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 14431, 13, 5500, 11, 1176, 580, 13, 16819, 4797, 23, 17, 10, 17135, 658, 19, 457, 7932, 13, 184, 19, 3154, 17135, 6468, 19, 1404, 12269, 19, 4229, 5356, 16264, 46, 19, 17, 20545, 10395, 9, 9, 9, 11, 28, 6421, 9531, 20729, 17, 10, 353, 17022, 11, 21, 6421, 9531, 16949, 17, 10, 11509, 753, 11, 33, 95, 2421, 7385, 956, 14431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 24738, 19, 13203, 658, 218, 787, 21, 430, 18482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22178, 27, 1064, 22, 956, 13, 11101, 1429, 5854, 24313, 18953, 40, 422, 24366, 68, 1758, 37, 10483, 14257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 13894, 3380, 23, 95, 18, 17634, 2288, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
					# fmt: on

					self.tokenizer_integration_test_util(
					    expected_encoding=_lowerCAmelCase , model_name='''xlnet-base-cased''' , revision='''c841166438c31ec7ca9a106dee7bb312b73ae511''' , )


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


lowercase_     :   List[str] 				=				{
    'configuration_maskformer': ['MASKFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MaskFormerConfig'],
    'configuration_maskformer_swin': ['MaskFormerSwinConfig'],
}

try:
      if not is_vision_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      lowercase_     :   Optional[Any] 				=				['MaskFormerFeatureExtractor']
      lowercase_     :   Dict 				=				['MaskFormerImageProcessor']


try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      lowercase_     :   str 				=				[
          'MASKFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
          'MaskFormerForInstanceSegmentation',
          'MaskFormerModel',
          'MaskFormerPreTrainedModel',
      ]
      lowercase_     :   Optional[Any] 				=				[
          'MaskFormerSwinBackbone',
          'MaskFormerSwinModel',
          'MaskFormerSwinPreTrainedModel',
      ]

if TYPE_CHECKING:
      from .configuration_maskformer import MASKFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, MaskFormerConfig
      from .configuration_maskformer_swin import MaskFormerSwinConfig

      try:
            if not is_vision_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .feature_extraction_maskformer import MaskFormerFeatureExtractor
            from .image_processing_maskformer import MaskFormerImageProcessor
      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_maskformer import (
                MASKFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
                MaskFormerForInstanceSegmentation,
                MaskFormerModel,
                MaskFormerPreTrainedModel,
            )
            from .modeling_maskformer_swin import (
                MaskFormerSwinBackbone,
                MaskFormerSwinModel,
                MaskFormerSwinPreTrainedModel,
            )


else:
      import sys

      lowercase_     :   Any 				=				_LazyModule(__name__, globals()['__file__'], _import_structure)

