



import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


snake_case    :						int  =						'▁'

snake_case    :						Dict  =						get_tests_dir('fixtures/test_sentencepiece.model')




@require_sentencepiece
@require_tokenizers
class 							lowerCAmelCase__  (	UpperCamelCase					,							unittest.TestCase						):
					__A     :    Tuple           =						BigBirdTokenizer
					__A     :    Union[str, Any]           =						BigBirdTokenizerFast
					__A     :    Tuple           =						True
					__A     :    Tuple           =						True







					def  _lowercase			(       self      :     List[Any]):
											super().setUp()

											A__	:							List[str]						    = self.tokenizer_class(_A			,		keep_accents=_A)
											tokenizer.save_pretrained(self.tmpdirname)







					def  _lowercase			(       self      :     Tuple):
											A__	:							Tuple						    = "<s>"
											A__	:							str						    = 1

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A)			,		_A)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A)			,		_A)







					def  _lowercase			(       self      :     Dict):
											A__	:							Tuple						    = list(self.get_tokenizer().get_vocab().keys())

											self.assertEqual(vocab_keys[0]			,		"<unk>")
											self.assertEqual(vocab_keys[1]			,		"<s>")
											self.assertEqual(vocab_keys[-1]			,		"[MASK]")
											self.assertEqual(len(_A)			,		1004)







					def  _lowercase			(       self      :     List[Any]):
											self.assertEqual(self.get_tokenizer().vocab_size			,		1000)







					def  _lowercase			(       self      :     List[str]):
											if not self.test_rust_tokenizer:
																	return

											A__	:							Tuple						    = self.get_tokenizer()
											A__	:							Tuple						    = self.get_rust_tokenizer()

											A__	:							Union[str, Any]						    = "I was born in 92000, and this is falsé."

											A__	:							int						    = tokenizer.tokenize(_A)
											A__	:							Any						    = rust_tokenizer.tokenize(_A)
											self.assertListEqual(_A			,		_A)

											A__	:							int						    = tokenizer.encode(_A			,		add_special_tokens=_A)
											A__	:							Dict						    = rust_tokenizer.encode(_A			,		add_special_tokens=_A)
											self.assertListEqual(_A			,		_A)

											A__	:							Optional[Any]						    = self.get_rust_tokenizer()
											A__	:							Union[str, Any]						    = tokenizer.encode(_A)
											A__	:							Dict						    = rust_tokenizer.encode(_A)
											self.assertListEqual(_A			,		_A)







					def  _lowercase			(       self      :     List[str]):
											A__	:							Union[str, Any]						    = BigBirdTokenizer(_A			,		keep_accents=_A)

											A__	:							int						    = tokenizer.tokenize("This is a test")
											self.assertListEqual(_A			,		["▁This", "▁is", "▁a", "▁t", "est"])

											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(_A)			,		[285, 46, 10, 170, 382]			,		)

											A__	:							List[Any]						    = tokenizer.tokenize("I was born in 92000, and this is falsé.")
											self.assertListEqual(
											    _A			,		[
											        SPIECE_UNDERLINE + "I",
											        SPIECE_UNDERLINE + "was",
											        SPIECE_UNDERLINE + "b",
											        "or",
											        "n",
											        SPIECE_UNDERLINE + "in",
											        SPIECE_UNDERLINE + "",
											        "9",
											        "2",
											        "0",
											        "0",
											        "0",
											        ",",
											        SPIECE_UNDERLINE + "and",
											        SPIECE_UNDERLINE + "this",
											        SPIECE_UNDERLINE + "is",
											        SPIECE_UNDERLINE + "f",
											        "al",
											        "s",
											        "é",
											        ".",
											    ]			,		)
											A__	:							Optional[int]						    = tokenizer.convert_tokens_to_ids(_A)
											self.assertListEqual(
											    _A			,		[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]			,		)

											A__	:							List[str]						    = tokenizer.convert_ids_to_tokens(_A)
											self.assertListEqual(
											    _A			,		[
											        SPIECE_UNDERLINE + "I",
											        SPIECE_UNDERLINE + "was",
											        SPIECE_UNDERLINE + "b",
											        "or",
											        "n",
											        SPIECE_UNDERLINE + "in",
											        SPIECE_UNDERLINE + "",
											        "<unk>",
											        "2",
											        "0",
											        "0",
											        "0",
											        ",",
											        SPIECE_UNDERLINE + "and",
											        SPIECE_UNDERLINE + "this",
											        SPIECE_UNDERLINE + "is",
											        SPIECE_UNDERLINE + "f",
											        "al",
											        "s",
											        "<unk>",
											        ".",
											    ]			,		)







					@cached_property
					def  _lowercase			(       self      :     List[Any]):
											return BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base")







					@slow
					def  _lowercase			(       self      :     str):
											A__	:							Dict						    = "Hello World!"
											A__	:							Any						    = [65, 1_8536, 2260, 101, 66]

											self.assertListEqual(_A			,		self.big_tokenizer.encode(_A))







					@slow
					def  _lowercase			(       self      :     Any):
											A__	:							Union[str, Any]						    = (
											    "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
											    " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
											)
											# fmt: off
											A__	:							Dict						    = [65, 871, 419, 358, 946, 991, 2521, 452, 358, 1357, 387, 7751, 3536, 112, 985, 456, 126, 865, 938, 5400, 5734, 458, 1368, 467, 786, 2462, 5246, 1159, 633, 865, 4519, 457, 582, 852, 2557, 427, 916, 508, 405, 3_4324, 497, 391, 408, 1_1342, 1244, 385, 100, 938, 985, 456, 574, 362, 1_2597, 3200, 3129, 1172, 66]  # noqa: E231
											# fmt: on
											self.assertListEqual(_A			,		self.big_tokenizer.encode(_A))







					@require_torch
					@slow
					def  _lowercase			(       self      :     Tuple):
											import torch

											from transformers import BigBirdConfig, BigBirdModel

											# Build sequence
											A__	:							List[Any]						    = list(self.big_tokenizer.get_vocab().keys())[:10]
											A__	:							List[Any]						    = " ".join(_A)
											A__	:							Tuple						    = self.big_tokenizer.encode_plus(_A			,		return_tensors="pt"			,		return_token_type_ids=_A)
											A__	:							int						    = self.big_tokenizer.batch_encode_plus(
											    [sequence + " " + sequence]			,		return_tensors="pt"			,		return_token_type_ids=_A)

											A__	:							Tuple						    = BigBirdConfig(attention_type="original_full")
											A__	:							Any						    = BigBirdModel(_A)

											assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

											with torch.no_grad():
																	model(**_A)
																	model(**_A)







					@slow
					def  _lowercase			(       self      :     List[Any]):
											A__	:							Optional[int]						    = BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base")
											A__	:							Any						    = tokenizer.decode(tokenizer("Paris is the [MASK].").input_ids)

											self.assertTrue(decoded_text == "[CLS] Paris is the[MASK].[SEP]")







					@slow
					def  _lowercase			(       self      :     str):
											# fmt: off
											A__	:							str						    = {"input_ids": [[65, 3_9286, 458, 3_6335, 2001, 456, 1_3073, 1_3266, 455, 113, 7746, 1741, 1_1157, 391, 1_3073, 1_3266, 455, 113, 3967, 3_5412, 113, 4936, 109, 3870, 2377, 113, 3_0084, 4_5720, 458, 134, 1_7496, 112, 503, 1_1672, 113, 118, 112, 5665, 1_3347, 3_8687, 112, 1496, 3_1389, 112, 3268, 4_7264, 134, 962, 112, 1_6377, 8035, 2_3130, 430, 1_2169, 1_5518, 2_8592, 458, 146, 4_1697, 109, 391, 1_2169, 1_5518, 1_6689, 458, 146, 4_1358, 109, 452, 726, 4034, 111, 763, 3_5412, 5082, 388, 1903, 111, 9051, 391, 2870, 4_8918, 1900, 1123, 550, 998, 112, 9586, 1_5985, 455, 391, 410, 2_2955, 3_7636, 114, 66], [65, 448, 1_7496, 419, 3663, 385, 763, 113, 2_7533, 2870, 3283, 1_3043, 1639, 2_4713, 523, 656, 2_4013, 1_8550, 2521, 517, 2_7014, 2_1244, 420, 1212, 1465, 391, 927, 4833, 388, 578, 1_1786, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 484, 2169, 7687, 2_1932, 1_8146, 726, 363, 1_7032, 3391, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=_A			,		model_name="google/bigbird-roberta-base"			,		revision="215c99f1600e06f83acce68422f2035b2b5c3510"			,		)





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


snake_case    :						str  =						{
    'configuration_albert': ['ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'AlbertConfig', 'AlbertOnnxConfig'],
}

try:
		if not is_sentencepiece_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		snake_case    :						Any  =						['AlbertTokenizer']

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		snake_case    :						List[str]  =						['AlbertTokenizerFast']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		snake_case    :						List[str]  =						[
		    'ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'AlbertForMaskedLM',
		    'AlbertForMultipleChoice',
		    'AlbertForPreTraining',
		    'AlbertForQuestionAnswering',
		    'AlbertForSequenceClassification',
		    'AlbertForTokenClassification',
		    'AlbertModel',
		    'AlbertPreTrainedModel',
		    'load_tf_weights_in_albert',
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		snake_case    :						Union[str, Any]  =						[
		    'TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'TFAlbertForMaskedLM',
		    'TFAlbertForMultipleChoice',
		    'TFAlbertForPreTraining',
		    'TFAlbertForQuestionAnswering',
		    'TFAlbertForSequenceClassification',
		    'TFAlbertForTokenClassification',
		    'TFAlbertMainLayer',
		    'TFAlbertModel',
		    'TFAlbertPreTrainedModel',
		]

try:
		if not is_flax_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		snake_case    :						Any  =						[
		    'FlaxAlbertForMaskedLM',
		    'FlaxAlbertForMultipleChoice',
		    'FlaxAlbertForPreTraining',
		    'FlaxAlbertForQuestionAnswering',
		    'FlaxAlbertForSequenceClassification',
		    'FlaxAlbertForTokenClassification',
		    'FlaxAlbertModel',
		    'FlaxAlbertPreTrainedModel',
		]

if TYPE_CHECKING:
		from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig, AlbertOnnxConfig

		try:
				if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_albert import AlbertTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_albert_fast import AlbertTokenizerFast

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_albert import (
				    ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    AlbertForMaskedLM,
				    AlbertForMultipleChoice,
				    AlbertForPreTraining,
				    AlbertForQuestionAnswering,
				    AlbertForSequenceClassification,
				    AlbertForTokenClassification,
				    AlbertModel,
				    AlbertPreTrainedModel,
				    load_tf_weights_in_albert,
				)

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_albert import (
				    TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFAlbertForMaskedLM,
				    TFAlbertForMultipleChoice,
				    TFAlbertForPreTraining,
				    TFAlbertForQuestionAnswering,
				    TFAlbertForSequenceClassification,
				    TFAlbertForTokenClassification,
				    TFAlbertMainLayer,
				    TFAlbertModel,
				    TFAlbertPreTrainedModel,
				)

		try:
				if not is_flax_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_flax_albert import (
				    FlaxAlbertForMaskedLM,
				    FlaxAlbertForMultipleChoice,
				    FlaxAlbertForPreTraining,
				    FlaxAlbertForQuestionAnswering,
				    FlaxAlbertForSequenceClassification,
				    FlaxAlbertForTokenClassification,
				    FlaxAlbertModel,
				    FlaxAlbertPreTrainedModel,
				)
else:
		import sys

		snake_case    :						Any  =						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)