


"""simple docstring"""



import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case         =					"""▁"""

__snake_case         =					get_tests_dir("""fixtures/test_sentencepiece.model""")







@require_sentencepiece
@require_tokenizers
class _lowerCAmelCase       (   snake_case_					,							unittest.TestCase			):
   __UpperCAmelCase				: Optional[int]													=						BigBirdTokenizer
   __UpperCAmelCase				: Any													=						BigBirdTokenizerFast
   __UpperCAmelCase				: List[Any]													=						True
   __UpperCAmelCase				: Optional[int]													=						True







   def 				lowerCamelCase	(			self		)			->	Any:

       '''simple docstring'''




       super().setUp()

       snake_case					:   Optional[Any]         =					self.tokenizer_class(UpperCamelCase__		,			keep_accents=UpperCamelCase__		)
       tokenizer.save_pretrained(self.tmpdirname		)







   def 				lowerCamelCase	(			self		)			->	int:

       '''simple docstring'''




       snake_case					:   int         =					"<s>"
       snake_case					:   Union[str, Any]         =					1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__		)		,			UpperCamelCase__		)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__		)		,			UpperCamelCase__		)







   def 				lowerCamelCase	(			self		)			->	List[Any]:

       '''simple docstring'''




       snake_case					:   Union[str, Any]         =					list(self.get_tokenizer().get_vocab().keys()		)

       self.assertEqual(vocab_keys[0]		,			"<unk>"		)
       self.assertEqual(vocab_keys[1]		,			"<s>"		)
       self.assertEqual(vocab_keys[-1]		,			"[MASK]"		)
       self.assertEqual(len(UpperCamelCase__		)		,			1004		)







   def 				lowerCamelCase	(			self		)			->	List[Any]:

       '''simple docstring'''




       self.assertEqual(self.get_tokenizer().vocab_size		,			1000		)







   def 				lowerCamelCase	(			self		)			->	List[str]:

       '''simple docstring'''




       if not self.test_rust_tokenizer:
           return

       snake_case					:   Optional[int]         =					self.get_tokenizer()
       snake_case					:   Dict         =					self.get_rust_tokenizer()

       snake_case					:   Optional[int]         =					"I was born in 92000, and this is falsé."

       snake_case					:   Optional[Any]         =					tokenizer.tokenize(UpperCamelCase__		)
       snake_case					:   List[Any]         =					rust_tokenizer.tokenize(UpperCamelCase__		)
       self.assertListEqual(UpperCamelCase__		,			UpperCamelCase__		)

       snake_case					:   Union[str, Any]         =					tokenizer.encode(UpperCamelCase__		,			add_special_tokens=UpperCamelCase__		)
       snake_case					:   int         =					rust_tokenizer.encode(UpperCamelCase__		,			add_special_tokens=UpperCamelCase__		)
       self.assertListEqual(UpperCamelCase__		,			UpperCamelCase__		)

       snake_case					:   List[str]         =					self.get_rust_tokenizer()
       snake_case					:   Dict         =					tokenizer.encode(UpperCamelCase__		)
       snake_case					:   Dict         =					rust_tokenizer.encode(UpperCamelCase__		)
       self.assertListEqual(UpperCamelCase__		,			UpperCamelCase__		)







   def 				lowerCamelCase	(			self		)			->	List[Any]:

       '''simple docstring'''




       snake_case					:   List[Any]         =					BigBirdTokenizer(UpperCamelCase__		,			keep_accents=UpperCamelCase__		)

       snake_case					:   str         =					tokenizer.tokenize("This is a test"		)
       self.assertListEqual(UpperCamelCase__		,			["▁This", "▁is", "▁a", "▁t", "est"]		)

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(UpperCamelCase__		)		,			[285, 46, 10, 170, 382]		,			)

       snake_case					:   int         =					tokenizer.tokenize("I was born in 92000, and this is falsé."		)
       self.assertListEqual(
           UpperCamelCase__		,			[
               SPIECE_UNDERLINE + "I",
               SPIECE_UNDERLINE + "was",
               SPIECE_UNDERLINE + "b",
               "or",
               "n",
               SPIECE_UNDERLINE + "in",
               SPIECE_UNDERLINE + "",
               "9",
               "2",
               "0",
               "0",
               "0",
               ",",
               SPIECE_UNDERLINE + "and",
               SPIECE_UNDERLINE + "this",
               SPIECE_UNDERLINE + "is",
               SPIECE_UNDERLINE + "f",
               "al",
               "s",
               "é",
               ".",
           ]		,			)
       snake_case					:   List[str]         =					tokenizer.convert_tokens_to_ids(UpperCamelCase__		)
       self.assertListEqual(
           UpperCamelCase__		,			[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]		,			)

       snake_case					:   List[str]         =					tokenizer.convert_ids_to_tokens(UpperCamelCase__		)
       self.assertListEqual(
           UpperCamelCase__		,			[
               SPIECE_UNDERLINE + "I",
               SPIECE_UNDERLINE + "was",
               SPIECE_UNDERLINE + "b",
               "or",
               "n",
               SPIECE_UNDERLINE + "in",
               SPIECE_UNDERLINE + "",
               "<unk>",
               "2",
               "0",
               "0",
               "0",
               ",",
               SPIECE_UNDERLINE + "and",
               SPIECE_UNDERLINE + "this",
               SPIECE_UNDERLINE + "is",
               SPIECE_UNDERLINE + "f",
               "al",
               "s",
               "<unk>",
               ".",
           ]		,			)







   @cached_property
   def 				lowerCamelCase	(			self		)			->	Any:

       '''simple docstring'''




       return BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base"		)







   @slow
   def 				lowerCamelCase	(			self		)			->	Optional[int]:

       '''simple docstring'''




       snake_case					:   Optional[Any]         =					"Hello World!"
       snake_case					:   Dict         =					[65, 1_8536, 2260, 101, 66]

       self.assertListEqual(UpperCamelCase__		,			self.big_tokenizer.encode(UpperCamelCase__		)		)







   @slow
   def 				lowerCamelCase	(			self		)			->	Dict:

       '''simple docstring'''




       snake_case					:   List[Any]         =					(
           "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
           " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
       )
       # fmt: off
       snake_case					:   int         =					[65, 871, 419, 358, 946, 991, 2521, 452, 358, 1357, 387, 7751, 3536, 112, 985, 456, 126, 865, 938, 5400, 5734, 458, 1368, 467, 786, 2462, 5246, 1159, 633, 865, 4519, 457, 582, 852, 2557, 427, 916, 508, 405, 3_4324, 497, 391, 408, 1_1342, 1244, 385, 100, 938, 985, 456, 574, 362, 1_2597, 3200, 3129, 1172, 66]  # noqa: E231
       # fmt: on
       self.assertListEqual(UpperCamelCase__		,			self.big_tokenizer.encode(UpperCamelCase__		)		)







   @require_torch
   @slow
   def 				lowerCamelCase	(			self		)			->	Dict:

       '''simple docstring'''




       import torch

       from transformers import BigBirdConfig, BigBirdModel

       # Build sequence
       snake_case					:   Optional[Any]         =					list(self.big_tokenizer.get_vocab().keys()		)[:10]
       snake_case					:   Optional[Any]         =					" ".join(UpperCamelCase__		)
       snake_case					:   List[str]         =					self.big_tokenizer.encode_plus(UpperCamelCase__		,			return_tensors="pt"		,			return_token_type_ids=UpperCamelCase__		)
       snake_case					:   int         =					self.big_tokenizer.batch_encode_plus(
           [sequence + " " + sequence]		,			return_tensors="pt"		,			return_token_type_ids=UpperCamelCase__		)

       snake_case					:   Any         =					BigBirdConfig(attention_type="original_full"		)
       snake_case					:   Tuple         =					BigBirdModel(UpperCamelCase__		)

       assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

       with torch.no_grad():
           model(**UpperCamelCase__		)
           model(**UpperCamelCase__		)







   @slow
   def 				lowerCamelCase	(			self		)			->	List[Any]:

       '''simple docstring'''




       snake_case					:   int         =					BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base"		)
       snake_case					:   List[str]         =					tokenizer.decode(tokenizer("Paris is the [MASK]."		).input_ids		)

       self.assertTrue(decoded_text == "[CLS] Paris is the[MASK].[SEP]"		)







   @slow
   def 				lowerCamelCase	(			self		)			->	int:

       '''simple docstring'''




       snake_case					:   Any         =					{"input_ids": [[65, 3_9286, 458, 3_6335, 2001, 456, 1_3073, 1_3266, 455, 113, 7746, 1741, 1_1157, 391, 1_3073, 1_3266, 455, 113, 3967, 3_5412, 113, 4936, 109, 3870, 2377, 113, 3_0084, 4_5720, 458, 134, 1_7496, 112, 503, 1_1672, 113, 118, 112, 5665, 1_3347, 3_8687, 112, 1496, 3_1389, 112, 3268, 4_7264, 134, 962, 112, 1_6377, 8035, 2_3130, 430, 1_2169, 1_5518, 2_8592, 458, 146, 4_1697, 109, 391, 1_2169, 1_5518, 1_6689, 458, 146, 4_1358, 109, 452, 726, 4034, 111, 763, 3_5412, 5082, 388, 1903, 111, 9051, 391, 2870, 4_8918, 1900, 1123, 550, 998, 112, 9586, 1_5985, 455, 391, 410, 2_2955, 3_7636, 114, 66], [65, 448, 1_7496, 419, 3663, 385, 763, 113, 2_7533, 2870, 3283, 1_3043, 1639, 2_4713, 523, 656, 2_4013, 1_8550, 2521, 517, 2_7014, 2_1244, 420, 1212, 1465, 391, 927, 4833, 388, 578, 1_1786, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 484, 2169, 7687, 2_1932, 1_8146, 726, 363, 1_7032, 3391, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=UpperCamelCase__		,			model_name="google/bigbird-roberta-base"		,			revision="215c99f1600e06f83acce68422f2035b2b5c3510"		,			)









"""simple docstring"""



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__snake_case         =					{
    """configuration_distilbert""": [
        """DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """DistilBertConfig""",
        """DistilBertOnnxConfig""",
    ],
    """tokenization_distilbert""": ["""DistilBertTokenizer"""],
}

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __snake_case         =					["""DistilBertTokenizerFast"""]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __snake_case         =					[
      """DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """DistilBertForMaskedLM""",
      """DistilBertForMultipleChoice""",
      """DistilBertForQuestionAnswering""",
      """DistilBertForSequenceClassification""",
      """DistilBertForTokenClassification""",
      """DistilBertModel""",
      """DistilBertPreTrainedModel""",
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __snake_case         =					[
      """TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """TFDistilBertForMaskedLM""",
      """TFDistilBertForMultipleChoice""",
      """TFDistilBertForQuestionAnswering""",
      """TFDistilBertForSequenceClassification""",
      """TFDistilBertForTokenClassification""",
      """TFDistilBertMainLayer""",
      """TFDistilBertModel""",
      """TFDistilBertPreTrainedModel""",
  ]

try:
  if not is_flax_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __snake_case         =					[
      """FlaxDistilBertForMaskedLM""",
      """FlaxDistilBertForMultipleChoice""",
      """FlaxDistilBertForQuestionAnswering""",
      """FlaxDistilBertForSequenceClassification""",
      """FlaxDistilBertForTokenClassification""",
      """FlaxDistilBertModel""",
      """FlaxDistilBertPreTrainedModel""",
  ]


if TYPE_CHECKING:
  from .configuration_distilbert import (
      DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
      DistilBertConfig,
      DistilBertOnnxConfig,
  )
  from .tokenization_distilbert import DistilBertTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_distilbert_fast import DistilBertTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_distilbert import (
        DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        DistilBertForMaskedLM,
        DistilBertForMultipleChoice,
        DistilBertForQuestionAnswering,
        DistilBertForSequenceClassification,
        DistilBertForTokenClassification,
        DistilBertModel,
        DistilBertPreTrainedModel,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_distilbert import (
        TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFDistilBertForMaskedLM,
        TFDistilBertForMultipleChoice,
        TFDistilBertForQuestionAnswering,
        TFDistilBertForSequenceClassification,
        TFDistilBertForTokenClassification,
        TFDistilBertMainLayer,
        TFDistilBertModel,
        TFDistilBertPreTrainedModel,
    )

  try:
    if not is_flax_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_flax_distilbert import (
        FlaxDistilBertForMaskedLM,
        FlaxDistilBertForMultipleChoice,
        FlaxDistilBertForQuestionAnswering,
        FlaxDistilBertForSequenceClassification,
        FlaxDistilBertForTokenClassification,
        FlaxDistilBertModel,
        FlaxDistilBertPreTrainedModel,
    )

else:
  import sys

  __snake_case         =					_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





