



import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase		       =      get_tests_dir("""fixtures/spiece.model""")


@require_sentencepiece
@require_tokenizers
class 				_a						(       SCREAMING_SNAKE_CASE       ,  unittest.TestCase			):





							'''simple docstring'''

							A				:List[Any] =       AlbertTokenizer
							A				:List[Any] =       AlbertTokenizerFast
							A				:Optional[Any] =       True
							A				:str =       True
							A				:List[Any] =       True



							def 		_A	(							self				):







											"""simple docstring"""



											super().setUp()

											# We have a SentencePiece fixture for testing
											a__	:							Any      						= AlbertTokenizer(__UpperCAmelCase				)
											tokenizer.save_pretrained(self.tmpdirname				)



							def 		_A	(							self ,					__UpperCAmelCase				):







											"""simple docstring"""



											a__	:							Tuple      						= "this is a test"
											a__	:							Optional[Any]      						= "this is a test"
											return input_text, output_text



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Dict      						= "<pad>"
											a__	:							Optional[Any]      						= 0

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCAmelCase				) ,					__UpperCAmelCase				)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCAmelCase				) ,					__UpperCAmelCase				)



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Any      						= list(self.get_tokenizer().get_vocab().keys()				)

											self.assertEqual(vocab_keys[0] ,					"<pad>"				)
											self.assertEqual(vocab_keys[1] ,					"<unk>"				)
											self.assertEqual(vocab_keys[-1] ,					"▁eloquent"				)
											self.assertEqual(len(__UpperCAmelCase				) ,					3_0000				)



							def 		_A	(							self				):







											"""simple docstring"""



											self.assertEqual(self.get_tokenizer().vocab_size ,					3_0000				)



							def 		_A	(							self				):







											"""simple docstring"""



											if not self.test_rust_tokenizer:
															return

											a__	:							Optional[Any]      						= self.get_tokenizer()
											a__	:							List[str]      						= self.get_rust_tokenizer()

											a__	:							Optional[Any]      						= "I was born in 92000, and this is falsé."

											a__	:							Any      						= tokenizer.tokenize(__UpperCAmelCase				)
											a__	:							int      						= rust_tokenizer.tokenize(__UpperCAmelCase				)
											self.assertListEqual(__UpperCAmelCase ,					__UpperCAmelCase				)

											a__	:							str      						= tokenizer.encode(__UpperCAmelCase ,					add_special_tokens=__UpperCAmelCase				)
											a__	:							Dict      						= rust_tokenizer.encode(__UpperCAmelCase ,					add_special_tokens=__UpperCAmelCase				)
											self.assertListEqual(__UpperCAmelCase ,					__UpperCAmelCase				)

											a__	:							Tuple      						= self.get_rust_tokenizer()
											a__	:							Any      						= tokenizer.encode(__UpperCAmelCase				)
											a__	:							str      						= rust_tokenizer.encode(__UpperCAmelCase				)
											self.assertListEqual(__UpperCAmelCase ,					__UpperCAmelCase				)



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							int      						= AlbertTokenizer(__UpperCAmelCase ,					keep_accents=__UpperCAmelCase				)

											a__	:							Optional[Any]      						= tokenizer.tokenize("This is a test"				)
											self.assertListEqual(__UpperCAmelCase ,					["▁this", "▁is", "▁a", "▁test"]				)

											self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase				) ,					[48, 25, 21, 1289]				)

											a__	:							Dict      						= tokenizer.tokenize("I was born in 92000, and this is falsé."				)
											self.assertListEqual(
											    __UpperCAmelCase ,					["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", "."]				)
											a__	:							Dict      						= tokenizer.convert_tokens_to_ids(__UpperCAmelCase				)
											self.assertListEqual(__UpperCAmelCase ,					[31, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]				)

											a__	:							int      						= tokenizer.convert_ids_to_tokens(__UpperCAmelCase				)
											self.assertListEqual(
											    __UpperCAmelCase ,					["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "."] ,					)



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Optional[Any]      						= AlbertTokenizer(__UpperCAmelCase				)

											a__	:							str      						= tokenizer.encode("sequence builders"				)
											a__	:							str      						= tokenizer.encode("multi-sequence build"				)

											a__	:							Optional[int]      						= tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase				)
											a__	:							List[str]      						= tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase ,					__UpperCAmelCase				)

											assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
											assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
											    tokenizer.sep_token_id
											]




							@slow
							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Tuple      						= {"attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "input_ids": [[2, 2_1970, 13, 5, 6092, 167, 28, 7103, 2153, 673, 8, 7028, 1_2051, 18, 17, 7103, 2153, 673, 8, 3515, 1_8684, 8, 4461, 6, 1927, 297, 8, 1_2060, 2607, 18, 13, 5, 4461, 15, 1_0538, 38, 8, 135, 15, 822, 58, 15, 993, 1_0363, 15, 1460, 8005, 4461, 15, 993, 255, 2328, 9, 9, 9, 6, 26, 1112, 816, 3260, 13, 5, 103, 2377, 6, 17, 1112, 816, 2782, 13, 5, 103, 1_0641, 6, 29, 84, 2512, 2430, 782, 1_8684, 2761, 19, 808, 2430, 2556, 17, 855, 1480, 9477, 4091, 128, 1_1712, 15, 7103, 2153, 673, 17, 2_4883, 9990, 9, 3], [2, 1_1502, 25, 1006, 20, 782, 8, 1_1809, 855, 1732, 1_9393, 1_8667, 37, 367, 2_1018, 69, 1854, 34, 1_1860, 1_9124, 27, 156, 225, 17, 193, 4141, 19, 65, 9124, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 2231, 886, 2385, 1_7659, 84, 14, 1_6792, 1952, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=__UpperCAmelCase ,					model_name="albert-base-v2" ,					revision="6b6560eaf5ff2e250b00c50f380c5389a9c2d82e" ,					)







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


lowerCamelCase		       =      {
    """configuration_blip""": [
        """BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """BlipConfig""",
        """BlipTextConfig""",
        """BlipVisionConfig""",
    ],
    """processing_blip""": ["""BlipProcessor"""],
}

try:
						if not is_vision_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCamelCase		       =      ["""BlipImageProcessor"""]


try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCamelCase		       =      [
						    """BLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """BlipModel""",
						    """BlipPreTrainedModel""",
						    """BlipForConditionalGeneration""",
						    """BlipForQuestionAnswering""",
						    """BlipVisionModel""",
						    """BlipTextModel""",
						    """BlipForImageTextRetrieval""",
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCamelCase		       =      [
						    """TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """TFBlipModel""",
						    """TFBlipPreTrainedModel""",
						    """TFBlipForConditionalGeneration""",
						    """TFBlipForQuestionAnswering""",
						    """TFBlipVisionModel""",
						    """TFBlipTextModel""",
						    """TFBlipForImageTextRetrieval""",
						]

if TYPE_CHECKING:
						from .configuration_blip import BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP, BlipConfig, BlipTextConfig, BlipVisionConfig
						from .processing_blip import BlipProcessor

						try:
												if not is_vision_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .image_processing_blip import BlipImageProcessor

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_blip import (
												    BLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
												    BlipForConditionalGeneration,
												    BlipForImageTextRetrieval,
												    BlipForQuestionAnswering,
												    BlipModel,
												    BlipPreTrainedModel,
												    BlipTextModel,
												    BlipVisionModel,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_blip import (
												    TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFBlipForConditionalGeneration,
												    TFBlipForImageTextRetrieval,
												    TFBlipForQuestionAnswering,
												    TFBlipModel,
												    TFBlipPreTrainedModel,
												    TFBlipTextModel,
												    TFBlipVisionModel,
												)

else:
						import sys

						lowerCamelCase		       =      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


