





import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


a__: str							   =						get_tests_dir('fixtures/test_sentencepiece.model')




@require_sentencepiece
@require_tokenizers
class        SCREAMING_SNAKE_CASE__					(		UpperCamelCase__							,   unittest.TestCase ):
 __SCREAMING_SNAKE_CASE							=      ReformerTokenizer
 __SCREAMING_SNAKE_CASE							=      ReformerTokenizerFast
 __SCREAMING_SNAKE_CASE							=      True
 __SCREAMING_SNAKE_CASE							=      False
 __SCREAMING_SNAKE_CASE							=      True



 def 							UpperCamelCase     (				self					):
        super().setUp()

        A__		   =		ReformerTokenizer(__lowerCamelCase,keep_accents=__lowerCamelCase					)
        tokenizer.save_pretrained(self.tmpdirname					)



 def 							UpperCamelCase     (				self					):
        A__		   =		'''<s>'''
        A__		   =		1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase					),__lowerCamelCase					)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase					),__lowerCamelCase					)



 def 							UpperCamelCase     (				self					):
        A__		   =		list(self.get_tokenizer().get_vocab().keys()					)

        self.assertEqual(vocab_keys[0],'''<unk>'''					)
        self.assertEqual(vocab_keys[1],'''<s>'''					)
        self.assertEqual(vocab_keys[-1],'''j'''					)
        self.assertEqual(len(__lowerCamelCase					),1000					)



 def 							UpperCamelCase     (				self					):
        self.assertEqual(self.get_tokenizer().vocab_size,1000					)



 def 							UpperCamelCase     (				self					):
        if not self.test_rust_tokenizer:
               return

        A__		   =		self.get_tokenizer()
        A__		   =		self.get_rust_tokenizer()

        A__		   =		'''I was born in 92000, and this is falsé.'''

        A__		   =		tokenizer.tokenize(__lowerCamelCase					)
        A__		   =		rust_tokenizer.tokenize(__lowerCamelCase					)
        self.assertListEqual(__lowerCamelCase,__lowerCamelCase					)

        A__		   =		tokenizer.encode(__lowerCamelCase,add_special_tokens=__lowerCamelCase					)
        A__		   =		rust_tokenizer.encode(__lowerCamelCase,add_special_tokens=__lowerCamelCase					)
        self.assertListEqual(__lowerCamelCase,__lowerCamelCase					)

        A__		   =		self.get_rust_tokenizer()
        A__		   =		tokenizer.encode(__lowerCamelCase					)
        A__		   =		rust_tokenizer.encode(__lowerCamelCase					)
        self.assertListEqual(__lowerCamelCase,__lowerCamelCase					)



 def 							UpperCamelCase     (				self,__lowerCamelCase=15					):
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"					):
                      A__		   =		self.rust_tokenizer_class.from_pretrained(__lowerCamelCase,**__lowerCamelCase					)

                      # Simple input
                      A__		   =		'''This is a simple input'''
                      A__		   =		['''This is a simple input 1''', '''This is a simple input 2''']
                      A__		   =		('''This is a simple input''', '''This is a pair''')
                      A__		   =		[
                          ('''This is a simple input 1''', '''This is a simple input 2'''),
                          ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                      ]

                      # Simple input tests
                      self.assertRaises(__lowerCamelCase,tokenizer_r.encode,__lowerCamelCase,max_length=__lowerCamelCase,padding='''max_length'''					)

                      # Simple input
                      self.assertRaises(__lowerCamelCase,tokenizer_r.encode_plus,__lowerCamelCase,max_length=__lowerCamelCase,padding='''max_length'''					)

                      # Simple input
                      self.assertRaises(
                          __lowerCamelCase,tokenizer_r.batch_encode_plus,__lowerCamelCase,max_length=__lowerCamelCase,padding='''max_length''',)

                      # Pair input
                      self.assertRaises(__lowerCamelCase,tokenizer_r.encode,__lowerCamelCase,max_length=__lowerCamelCase,padding='''max_length'''					)

                      # Pair input
                      self.assertRaises(__lowerCamelCase,tokenizer_r.encode_plus,__lowerCamelCase,max_length=__lowerCamelCase,padding='''max_length'''					)

                      # Pair input
                      self.assertRaises(
                          __lowerCamelCase,tokenizer_r.batch_encode_plus,__lowerCamelCase,max_length=__lowerCamelCase,padding='''max_length''',)



 def 							UpperCamelCase     (				self					):
        pass



 def 							UpperCamelCase     (				self					):
        A__		   =		ReformerTokenizer(__lowerCamelCase,keep_accents=__lowerCamelCase					)

        A__		   =		tokenizer.tokenize('''This is a test'''					)
        self.assertListEqual(__lowerCamelCase,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']					)

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(__lowerCamelCase					),[285, 46, 10, 170, 382],)

        A__		   =		tokenizer.tokenize('''I was born in 92000, and this is falsé.'''					)
        self.assertListEqual(
            __lowerCamelCase,[
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''9''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''é''',
                '''.''',
            ],)
        A__		   =		tokenizer.convert_tokens_to_ids(__lowerCamelCase					)
        self.assertListEqual(
            __lowerCamelCase,[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4],)

        A__		   =		tokenizer.convert_ids_to_tokens(__lowerCamelCase					)
        self.assertListEqual(
            __lowerCamelCase,[
                SPIECE_UNDERLINE + '''I''',
                SPIECE_UNDERLINE + '''was''',
                SPIECE_UNDERLINE + '''b''',
                '''or''',
                '''n''',
                SPIECE_UNDERLINE + '''in''',
                SPIECE_UNDERLINE + '''''',
                '''<unk>''',
                '''2''',
                '''0''',
                '''0''',
                '''0''',
                ''',''',
                SPIECE_UNDERLINE + '''and''',
                SPIECE_UNDERLINE + '''this''',
                SPIECE_UNDERLINE + '''is''',
                SPIECE_UNDERLINE + '''f''',
                '''al''',
                '''s''',
                '''<unk>''',
                '''.''',
            ],)



 @cached_property
 def 							UpperCamelCase     (				self					):
        return ReformerTokenizer.from_pretrained('''google/reformer-crime-and-punishment'''					)



 @slow
 def 							UpperCamelCase     (				self					):
        A__		   =		'''Hello World!'''
        A__		   =		[126, 32, 262, 152, 38, 72, 287]

        self.assertListEqual(__lowerCamelCase,self.big_tokenizer.encode(__lowerCamelCase					)					)



 @slow
 def 							UpperCamelCase     (				self					):
        A__		   =		(
            '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
            ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
        )
        A__		   =		[
            108,
            265,
            24,
            111,
            4,
            258,
            156,
            35,
            28,
            275,
            3,
            259,
            297,
            260,
            84,
            4,
            35,
            110,
            44,
            8,
            259,
            91,
            268,
            21,
            11,
            209,
            274,
            109,
            266,
            277,
            117,
            86,
            93,
            315,
            258,
            278,
            258,
            277,
            258,
            0,
            258,
            288,
            258,
            319,
            258,
            0,
            258,
            0,
            258,
            0,
            258,
            0,
            258,
            287,
            258,
            315,
            258,
            289,
            258,
            278,
            99,
            269,
            266,
            262,
            8,
            259,
            241,
            4,
            217,
            230,
            268,
            266,
            55,
            168,
            106,
            75,
            193,
            266,
            223,
            27,
            49,
            26,
            282,
            25,
            264,
            299,
            19,
            26,
            0,
            258,
            277,
            117,
            86,
            93,
            176,
            183,
            270,
            11,
            262,
            42,
            61,
            265,
        ]

        self.assertListEqual(__lowerCamelCase,self.big_tokenizer.encode(__lowerCamelCase					)					)



 @require_torch
 @slow
 def 							UpperCamelCase     (				self					):
        import torch

        from transformers import ReformerConfig, ReformerModel

        # Build sequence
        A__		   =		list(self.big_tokenizer.get_vocab().keys()					)[:10]
        A__		   =		''' '''.join(__lowerCamelCase					)
        A__		   =		self.big_tokenizer.encode_plus(__lowerCamelCase,return_tensors='''pt'''					)
        A__		   =		self.big_tokenizer.batch_encode_plus([sequence, sequence],return_tensors='''pt'''					)

        A__		   =		ReformerConfig()
        # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
        A__		   =		encoded_sequence['''input_ids'''].shape
        A__		   =		ReformerModel(__lowerCamelCase					)

        # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
        assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

        with torch.no_grad():
               model(**__lowerCamelCase					)
               model(**__lowerCamelCase					)




 @slow
 def 							UpperCamelCase     (				self					):
        # fmt: off
        A__		   =		{'''input_ids''': [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
        # fmt: on

        # This tokenizer does not know some characters like ")".
        # That is the reason why we use very simple texts here.
        # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
        A__		   =		[
            '''This is a very simple sentence.''',
            '''The quick brown fox jumps over the lazy dog.''',
        ]

        self.tokenizer_integration_test_util(
            expected_encoding=__lowerCamelCase,model_name='''google/reformer-crime-and-punishment''',revision='''0e6c3decb8211d49bf881013425dc8b0448b3f5a''',padding=__lowerCamelCase,sequences=__lowerCamelCase,)












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


a__: List[Any]							   =						{
    'configuration_x_clip': [
        'XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'XCLIPConfig',
        'XCLIPTextConfig',
        'XCLIPVisionConfig',
    ],
    'processing_x_clip': ['XCLIPProcessor'],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a__: Any							   =						[
           'XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST',
           'XCLIPModel',
           'XCLIPPreTrainedModel',
           'XCLIPTextModel',
           'XCLIPVisionModel',
       ]

if TYPE_CHECKING:
       from .configuration_x_clip import (
           XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
           XCLIPConfig,
           XCLIPTextConfig,
           XCLIPVisionConfig,
       )
       from .processing_x_clip import XCLIPProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_x_clip import (
                  XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
                  XCLIPModel,
                  XCLIPPreTrainedModel,
                  XCLIPTextModel,
                  XCLIPVisionModel,
              )

else:
       import sys

       a__: Optional[Any]							   =						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





