





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCAmelCase       : str			     =       get_tests_dir("""fixtures/test_sentencepiece.model""")

@require_sentencepiece
@require_tokenizers
class      lowerCAmelCase							(      __UpperCamelCase,						unittest.TestCase    ):
 UpperCAmelCase__									=	XLNetTokenizer
 UpperCAmelCase__									=	XLNetTokenizerFast
 UpperCAmelCase__									=	True
 UpperCAmelCase__									=	True
 def   A_			(   self						:   Optional[Any]			)    ->			Dict:
     super().setUp()

     # We have a SentencePiece fixture for testing
     lowerCamelCase__					:   str      				=	XLNetTokenizer(UpperCAmelCase    ,  keep_accents=UpperCAmelCase			)
     tokenizer.sanitize_special_tokens()
     tokenizer.save_pretrained(self.tmpdirname			)
 def   A_			(   self						:   str			)    ->			Dict:
     lowerCamelCase__					:   Optional[Any]      				=	'<s>'
     lowerCamelCase__					:   int      				=	1

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase			)    ,  UpperCAmelCase			)
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase			)    ,  UpperCAmelCase			)
 def   A_			(   self						:   str			)    ->			Optional[int]:
     lowerCamelCase__					:   Optional[int]      				=	list(self.get_tokenizer().get_vocab().keys()			)

     self.assertEqual(vocab_keys[0]    ,  '<unk>'			)
     self.assertEqual(vocab_keys[1]    ,  '<s>'			)
     self.assertEqual(vocab_keys[-1]    ,  '<eod>'			)
     self.assertEqual(len(UpperCAmelCase			)    ,  1006			)
 def   A_			(   self						:   int			)    ->			Tuple:
     self.assertEqual(self.get_tokenizer().vocab_size    ,  1000			)
 def   A_			(   self						:   Tuple			)    ->			Optional[int]:
     lowerCamelCase__					:   Any      				=	XLNetTokenizer(UpperCAmelCase    ,  keep_accents=UpperCAmelCase			)

     lowerCamelCase__					:   Any      				=	tokenizer.tokenize('This is a test'			)
     self.assertListEqual(UpperCAmelCase    ,  ['▁This', '▁is', '▁a', '▁t', 'est']			)

     self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase			)    ,  [285, 46, 10, 170, 382]			)

     lowerCamelCase__					:   int      				=	tokenizer.tokenize('I was born in 92000, and this is falsé.'			)
     self.assertListEqual(
         UpperCAmelCase    ,  [
             SPIECE_UNDERLINE + 'I',
             SPIECE_UNDERLINE + 'was',
             SPIECE_UNDERLINE + 'b',
             'or',
             'n',
             SPIECE_UNDERLINE + 'in',
             SPIECE_UNDERLINE + '',
             '9',
             '2',
             '0',
             '0',
             '0',
             ',',
             SPIECE_UNDERLINE + 'and',
             SPIECE_UNDERLINE + 'this',
             SPIECE_UNDERLINE + 'is',
             SPIECE_UNDERLINE + 'f',
             'al',
             's',
             'é',
             '.',
         ]    ,  )
     lowerCamelCase__					:   Optional[int]      				=	tokenizer.convert_tokens_to_ids(UpperCAmelCase			)
     self.assertListEqual(UpperCAmelCase    ,  [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]			)

     lowerCamelCase__					:   Optional[int]      				=	tokenizer.convert_ids_to_tokens(UpperCAmelCase			)
     self.assertListEqual(
         UpperCAmelCase    ,  [
             SPIECE_UNDERLINE + 'I',
             SPIECE_UNDERLINE + 'was',
             SPIECE_UNDERLINE + 'b',
             'or',
             'n',
             SPIECE_UNDERLINE + 'in',
             SPIECE_UNDERLINE + '',
             '<unk>',
             '2',
             '0',
             '0',
             '0',
             ',',
             SPIECE_UNDERLINE + 'and',
             SPIECE_UNDERLINE + 'this',
             SPIECE_UNDERLINE + 'is',
             SPIECE_UNDERLINE + 'f',
             'al',
             's',
             '<unk>',
             '.',
         ]    ,  )
 def   A_			(   self						:   Optional[int]			)    ->			Any:
     lowerCamelCase__					:   Optional[int]      				=	XLNetTokenizer(UpperCAmelCase    ,  do_lower_case=UpperCAmelCase			)
     lowerCamelCase__					:   Optional[int]      				=	tokenizer.tokenize('I was born in 92000, and this is falsé.'			)
     self.assertListEqual(
         UpperCAmelCase    ,  [
             SPIECE_UNDERLINE + '',
             'i',
             SPIECE_UNDERLINE + 'was',
             SPIECE_UNDERLINE + 'b',
             'or',
             'n',
             SPIECE_UNDERLINE + 'in',
             SPIECE_UNDERLINE + '',
             '9',
             '2',
             '0',
             '0',
             '0',
             ',',
             SPIECE_UNDERLINE + 'and',
             SPIECE_UNDERLINE + 'this',
             SPIECE_UNDERLINE + 'is',
             SPIECE_UNDERLINE + 'f',
             'al',
             'se',
             '.',
         ]    ,  )
     self.assertListEqual(tokenizer.tokenize('H\u00E9llo'			)    ,  ['▁he', 'll', 'o']			)
 def   A_			(   self						:   Optional[Any]			)    ->			Dict:
     lowerCamelCase__					:   Tuple      				=	XLNetTokenizer(UpperCAmelCase    ,  do_lower_case=UpperCAmelCase			)
     lowerCamelCase__					:   Dict      				=	tokenizer.tokenize('I was born in 92000, and this is falsé.'			)
     self.assertListEqual(
         UpperCAmelCase    ,  [
             SPIECE_UNDERLINE + 'I',
             SPIECE_UNDERLINE + 'was',
             SPIECE_UNDERLINE + 'b',
             'or',
             'n',
             SPIECE_UNDERLINE + 'in',
             SPIECE_UNDERLINE + '',
             '9',
             '2',
             '0',
             '0',
             '0',
             ',',
             SPIECE_UNDERLINE + 'and',
             SPIECE_UNDERLINE + 'this',
             SPIECE_UNDERLINE + 'is',
             SPIECE_UNDERLINE + 'f',
             'al',
             'se',
             '.',
         ]    ,  )
 @slow
 def   A_			(   self						:   Tuple			)    ->			List[Any]:
     lowerCamelCase__					:   Dict      				=	XLNetTokenizer.from_pretrained('xlnet-base-cased'			)

     lowerCamelCase__					:   Optional[int]      				=	tokenizer.encode('sequence builders'    ,  add_special_tokens=UpperCAmelCase			)
     lowerCamelCase__					:   List[str]      				=	tokenizer.encode('multi-sequence build'    ,  add_special_tokens=UpperCAmelCase			)

     lowerCamelCase__					:   Optional[Any]      				=	tokenizer.build_inputs_with_special_tokens(UpperCAmelCase			)
     lowerCamelCase__					:   Union[str, Any]      				=	tokenizer.build_inputs_with_special_tokens(UpperCAmelCase    ,  UpperCAmelCase			)

     assert encoded_sentence == text + [4, 3]
     assert encoded_pair == text + [4] + text_a + [4, 3]







 @slow
 def   A_			(   self						:   List[str]			)    ->			Union[str, Any]:
     # fmt: off
     lowerCamelCase__					:   Optional[Any]      				=	{'input_ids': [[17, 21442, 270, 17, 10, 14645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 22018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 14431, 13, 5500, 11, 1176, 580, 13, 16819, 4797, 23, 17, 10, 17135, 658, 19, 457, 7932, 13, 184, 19, 3154, 17135, 6468, 19, 1404, 12269, 19, 4229, 5356, 16264, 46, 19, 17, 20545, 10395, 9, 9, 9, 11, 28, 6421, 9531, 20729, 17, 10, 353, 17022, 11, 21, 6421, 9531, 16949, 17, 10, 11509, 753, 11, 33, 95, 2421, 7385, 956, 14431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 24738, 19, 13203, 658, 218, 787, 21, 430, 18482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22178, 27, 1064, 22, 956, 13, 11101, 1429, 5854, 24313, 18953, 40, 422, 24366, 68, 1758, 37, 10483, 14257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 13894, 3380, 23, 95, 18, 17634, 2288, 9, 4, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=UpperCAmelCase    ,  model_name='xlnet-base-cased'    ,  revision='c841166438c31ec7ca9a106dee7bb312b73ae511'    ,  )












from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


_UpperCAmelCase       : Optional[Any]			     =       {
    """configuration_clip""": [
        """CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """CLIPConfig""",
        """CLIPOnnxConfig""",
        """CLIPTextConfig""",
        """CLIPVisionConfig""",
    ],
    """processing_clip""": ["""CLIPProcessor"""],
    """tokenization_clip""": ["""CLIPTokenizer"""],
}

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _UpperCAmelCase       : int			     =       ["""CLIPTokenizerFast"""]

try:
    if not is_vision_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _UpperCAmelCase       : Optional[int]			     =       ["""CLIPFeatureExtractor"""]
    _UpperCAmelCase       : Union[str, Any]			     =       ["""CLIPImageProcessor"""]

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _UpperCAmelCase       : int			     =       [
        """CLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
        """CLIPModel""",
        """CLIPPreTrainedModel""",
        """CLIPTextModel""",
        """CLIPTextModelWithProjection""",
        """CLIPVisionModel""",
        """CLIPVisionModelWithProjection""",
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _UpperCAmelCase       : Tuple			     =       [
        """TF_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
        """TFCLIPModel""",
        """TFCLIPPreTrainedModel""",
        """TFCLIPTextModel""",
        """TFCLIPVisionModel""",
    ]

try:
    if not is_flax_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _UpperCAmelCase       : Optional[int]			     =       [
        """FlaxCLIPModel""",
        """FlaxCLIPPreTrainedModel""",
        """FlaxCLIPTextModel""",
        """FlaxCLIPTextPreTrainedModel""",
        """FlaxCLIPVisionModel""",
        """FlaxCLIPVisionPreTrainedModel""",
    ]


if TYPE_CHECKING:
    from .configuration_clip import (
        CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
        CLIPConfig,
        CLIPOnnxConfig,
        CLIPTextConfig,
        CLIPVisionConfig,
    )
    from .processing_clip import CLIPProcessor
    from .tokenization_clip import CLIPTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_clip_fast import CLIPTokenizerFast

    try:
        if not is_vision_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .feature_extraction_clip import CLIPFeatureExtractor
        from .image_processing_clip import CLIPImageProcessor

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_clip import (
            CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
            CLIPModel,
            CLIPPreTrainedModel,
            CLIPTextModel,
            CLIPTextModelWithProjection,
            CLIPVisionModel,
            CLIPVisionModelWithProjection,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_clip import (
            TF_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
            TFCLIPModel,
            TFCLIPPreTrainedModel,
            TFCLIPTextModel,
            TFCLIPVisionModel,
        )

    try:
        if not is_flax_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_flax_clip import (
            FlaxCLIPModel,
            FlaxCLIPPreTrainedModel,
            FlaxCLIPTextModel,
            FlaxCLIPTextPreTrainedModel,
            FlaxCLIPVisionModel,
            FlaxCLIPVisionPreTrainedModel,
        )


else:
    import sys

    _UpperCAmelCase       : Optional[Any]			     =       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





