






'''simple docstring'''






import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase:      Tuple		 =		"▁"

lowerCAmelCase:      str		 =		get_tests_dir('fixtures/test_sentencepiece.model')

@require_sentencepiece
@require_tokenizers
class 							a__(      _UpperCAmelCase	,				unittest.TestCase			):
    lowercase__				=					BigBirdTokenizer
    lowercase__				=					BigBirdTokenizerFast
    lowercase__				=					True
    lowercase__				=					True







    def     lowercase_ (   self				:						Union[str, Any]			):
      super().setUp()

      a		:      Tuple       =  self.tokenizer_class(lowerCamelCase_		,    keep_accents=lowerCamelCase_			)
      tokenizer.save_pretrained(self.tmpdirname			)







    def     lowercase_ (   self				:						Optional[int]			):
      a		:      Union[str, Any]       =  '''<s>'''
      a		:      Dict       =  1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase_			)		,    lowerCamelCase_			)
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase_			)		,    lowerCamelCase_			)







    def     lowercase_ (   self				:						List[Any]			):
      a		:      List[Any]       =  list(self.get_tokenizer().get_vocab().keys()			)

      self.assertEqual(vocab_keys[0]		,    '<unk>'			)
      self.assertEqual(vocab_keys[1]		,    '<s>'			)
      self.assertEqual(vocab_keys[-1]		,    '[MASK]'			)
      self.assertEqual(len(lowerCamelCase_			)		,    10_04			)







    def     lowercase_ (   self				:						Any			):
      self.assertEqual(self.get_tokenizer().vocab_size		,    10_00			)







    def     lowercase_ (   self				:						Dict			):




      if not self.test_rust_tokenizer:
        return

      a		:      int       =  self.get_tokenizer()
      a		:      List[str]       =  self.get_rust_tokenizer()

      a		:      str       =  '''I was born in 92000, and this is falsé.'''

      a		:      List[str]       =  tokenizer.tokenize(lowerCamelCase_			)
      a		:      Optional[int]       =  rust_tokenizer.tokenize(lowerCamelCase_			)
      self.assertListEqual(lowerCamelCase_		,    lowerCamelCase_			)

      a		:      List[str]       =  tokenizer.encode(lowerCamelCase_		,    add_special_tokens=lowerCamelCase_			)
      a		:      List[str]       =  rust_tokenizer.encode(lowerCamelCase_		,    add_special_tokens=lowerCamelCase_			)
      self.assertListEqual(lowerCamelCase_		,    lowerCamelCase_			)

      a		:      Dict       =  self.get_rust_tokenizer()
      a		:      Tuple       =  tokenizer.encode(lowerCamelCase_			)
      a		:      Tuple       =  rust_tokenizer.encode(lowerCamelCase_			)
      self.assertListEqual(lowerCamelCase_		,    lowerCamelCase_			)







    def     lowercase_ (   self				:						Any			):
      a		:      Any       =  BigBirdTokenizer(lowerCamelCase_		,    keep_accents=lowerCamelCase_			)

      a		:      List[Any]       =  tokenizer.tokenize('This is a test'			)
      self.assertListEqual(lowerCamelCase_		,    ['▁This', '▁is', '▁a', '▁t', 'est']			)

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(lowerCamelCase_			)		,    [2_85, 46, 10, 1_70, 3_82]		,    )

      a		:      List[Any]       =  tokenizer.tokenize('I was born in 92000, and this is falsé.'			)
      self.assertListEqual(
          lowerCamelCase_		,    [
              SPIECE_UNDERLINE + 'I',
              SPIECE_UNDERLINE + 'was',
              SPIECE_UNDERLINE + 'b',
              'or',
              'n',
              SPIECE_UNDERLINE + 'in',
              SPIECE_UNDERLINE + '',
              '9',
              '2',
              '0',
              '0',
              '0',
              ',',
              SPIECE_UNDERLINE + 'and',
              SPIECE_UNDERLINE + 'this',
              SPIECE_UNDERLINE + 'is',
              SPIECE_UNDERLINE + 'f',
              'al',
              's',
              'é',
              '.',
          ]		,    )
      a		:      List[Any]       =  tokenizer.convert_tokens_to_ids(lowerCamelCase_			)
      self.assertListEqual(
          lowerCamelCase_		,    [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]		,    )

      a		:      Optional[Any]       =  tokenizer.convert_ids_to_tokens(lowerCamelCase_			)
      self.assertListEqual(
          lowerCamelCase_		,    [
              SPIECE_UNDERLINE + 'I',
              SPIECE_UNDERLINE + 'was',
              SPIECE_UNDERLINE + 'b',
              'or',
              'n',
              SPIECE_UNDERLINE + 'in',
              SPIECE_UNDERLINE + '',
              '<unk>',
              '2',
              '0',
              '0',
              '0',
              ',',
              SPIECE_UNDERLINE + 'and',
              SPIECE_UNDERLINE + 'this',
              SPIECE_UNDERLINE + 'is',
              SPIECE_UNDERLINE + 'f',
              'al',
              's',
              '<unk>',
              '.',
          ]		,    )







    @cached_property
    def     lowercase_ (   self				:						Optional[Any]			):
      return BigBirdTokenizer.from_pretrained('google/bigbird-roberta-base'			)







    @slow
    def     lowercase_ (   self				:						int			):
      a		:      Dict       =  '''Hello World!'''
      a		:      Optional[Any]       =  [65, 1_85_36, 22_60, 1_01, 66]

      self.assertListEqual(lowerCamelCase_		,    self.big_tokenizer.encode(lowerCamelCase_			)			)







    @slow
    def     lowercase_ (   self				:						Union[str, Any]			):
      a		:      Union[str, Any]       =  (
          '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
          ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
      )
      # fmt: off
      a		:      Any       =  [65, 8_71, 4_19, 3_58, 9_46, 9_91, 25_21, 4_52, 3_58, 13_57, 3_87, 77_51, 35_36, 1_12, 9_85, 4_56, 1_26, 8_65, 9_38, 54_00, 57_34, 4_58, 13_68, 4_67, 7_86, 24_62, 52_46, 11_59, 6_33, 8_65, 45_19, 4_57, 5_82, 8_52, 25_57, 4_27, 9_16, 5_08, 4_05, 3_43_24, 4_97, 3_91, 4_08, 1_13_42, 12_44, 3_85, 1_00, 9_38, 9_85, 4_56, 5_74, 3_62, 1_25_97, 32_00, 31_29, 11_72, 66]  # noqa: E231
      # fmt: on
      self.assertListEqual(lowerCamelCase_		,    self.big_tokenizer.encode(lowerCamelCase_			)			)







    @require_torch
    @slow
    def     lowercase_ (   self				:						Optional[Any]			):
      import torch

      from transformers import BigBirdConfig, BigBirdModel

      # Build sequence
      a		:      int       =  list(self.big_tokenizer.get_vocab().keys()			)[:10]
      a		:      Dict       =  ''' '''.join(lowerCamelCase_			)
      a		:      Tuple       =  self.big_tokenizer.encode_plus(lowerCamelCase_		,    return_tensors='pt'		,    return_token_type_ids=lowerCamelCase_			)
      a		:      Any       =  self.big_tokenizer.batch_encode_plus(
          [sequence + ' ' + sequence]		,    return_tensors='pt'		,    return_token_type_ids=lowerCamelCase_			)

      a		:      Dict       =  BigBirdConfig(attention_type='original_full'			)
      a		:      Dict       =  BigBirdModel(lowerCamelCase_			)

      assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

      with torch.no_grad():
        model(**lowerCamelCase_			)
        model(**lowerCamelCase_			)







    @slow
    def     lowercase_ (   self				:						Dict			):
      a		:      int       =  BigBirdTokenizer.from_pretrained('google/bigbird-roberta-base'			)
      a		:      List[Any]       =  tokenizer.decode(tokenizer('Paris is the [MASK].'			).input_ids			)

      self.assertTrue(decoded_text == '[CLS] Paris is the[MASK].[SEP]'			)







    @slow
    def     lowercase_ (   self				:						Any			):
      a		:      str       =  {'''input_ids''': [[65, 3_92_86, 4_58, 3_63_35, 20_01, 4_56, 1_30_73, 1_32_66, 4_55, 1_13, 77_46, 17_41, 1_11_57, 3_91, 1_30_73, 1_32_66, 4_55, 1_13, 39_67, 3_54_12, 1_13, 49_36, 1_09, 38_70, 23_77, 1_13, 3_00_84, 4_57_20, 4_58, 1_34, 1_74_96, 1_12, 5_03, 1_16_72, 1_13, 1_18, 1_12, 56_65, 1_33_47, 3_86_87, 1_12, 14_96, 3_13_89, 1_12, 32_68, 4_72_64, 1_34, 9_62, 1_12, 1_63_77, 80_35, 2_31_30, 4_30, 1_21_69, 1_55_18, 2_85_92, 4_58, 1_46, 4_16_97, 1_09, 3_91, 1_21_69, 1_55_18, 1_66_89, 4_58, 1_46, 4_13_58, 1_09, 4_52, 7_26, 40_34, 1_11, 7_63, 3_54_12, 50_82, 3_88, 19_03, 1_11, 90_51, 3_91, 28_70, 4_89_18, 19_00, 11_23, 5_50, 9_98, 1_12, 95_86, 1_59_85, 4_55, 3_91, 4_10, 2_29_55, 3_76_36, 1_14, 66], [65, 4_48, 1_74_96, 4_19, 36_63, 3_85, 7_63, 1_13, 2_75_33, 28_70, 32_83, 1_30_43, 16_39, 2_47_13, 5_23, 6_56, 2_40_13, 1_85_50, 25_21, 5_17, 2_70_14, 2_12_44, 4_20, 12_12, 14_65, 3_91, 9_27, 48_33, 3_88, 5_78, 1_17_86, 1_14, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 4_84, 21_69, 76_87, 2_19_32, 1_81_46, 7_26, 3_63, 1_70_32, 33_91, 1_14, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=lowerCamelCase_		,    model_name='google/bigbird-roberta-base'		,    revision='215c99f1600e06f83acce68422f2035b2b5c3510'		,    )




'''simple docstring'''






import unittest

import numpy as np

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
     from transformers.models.bert.modeling_flax_bert import (
         FlaxBertForMaskedLM,
         FlaxBertForMultipleChoice,
         FlaxBertForNextSentencePrediction,
         FlaxBertForPreTraining,
         FlaxBertForQuestionAnswering,
         FlaxBertForSequenceClassification,
         FlaxBertForTokenClassification,
         FlaxBertModel,
     )

class 							a__(      unittest.TestCase			):







    def __init__(   self				:						Dict		,    __snake_case				:						Union[str, Any]		,    __snake_case				:						Any=13		,    __snake_case				:						Any=7		,    __snake_case				:						int=True		,    __snake_case				:						List[Any]=True		,    __snake_case				:						int=True		,    __snake_case				:						List[Any]=True		,    __snake_case				:						Optional[Any]=99		,    __snake_case				:						List[str]=32		,    __snake_case				:						List[Any]=5		,    __snake_case				:						int=4		,    __snake_case				:						List[str]=37		,    __snake_case				:						int="gelu"		,    __snake_case				:						Tuple=0.1		,    __snake_case				:						Union[str, Any]=0.1		,    __snake_case				:						Any=5_12		,    __snake_case				:						Tuple=16		,    __snake_case				:						Union[str, Any]=2		,    __snake_case				:						str=0.02		,    __snake_case				:						Union[str, Any]=4		,    ):
      a		:      List[Any]       =  parent
      a		:      List[str]       =  batch_size
      a		:      Dict       =  seq_length
      a		:      str       =  is_training
      a		:      Optional[int]       =  use_attention_mask
      a		:      Union[str, Any]       =  use_token_type_ids
      a		:      List[str]       =  use_labels
      a		:      Dict       =  vocab_size
      a		:      Tuple       =  hidden_size
      a		:      Optional[Any]       =  num_hidden_layers
      a		:      List[str]       =  num_attention_heads
      a		:      Tuple       =  intermediate_size
      a		:      Dict       =  hidden_act
      a		:      List[str]       =  hidden_dropout_prob
      a		:      str       =  attention_probs_dropout_prob
      a		:      str       =  max_position_embeddings
      a		:      Tuple       =  type_vocab_size
      a		:      Optional[Any]       =  type_sequence_label_size
      a		:      Union[str, Any]       =  initializer_range
      a		:      Union[str, Any]       =  num_choices







    def     lowercase_ (   self				:						Any			):
      a		:      List[Any]       =  ids_tensor([self.batch_size, self.seq_length]		,    self.vocab_size			)

      a		:      List[str]       =  None
      if self.use_attention_mask:
        a		:      List[Any]       =  random_attention_mask([self.batch_size, self.seq_length]			)

      a		:      Dict       =  None
      if self.use_token_type_ids:
        a		:      List[str]       =  ids_tensor([self.batch_size, self.seq_length]		,    self.type_vocab_size			)

      a		:      Dict       =  BertConfig(
          vocab_size=self.vocab_size		,    hidden_size=self.hidden_size		,    num_hidden_layers=self.num_hidden_layers		,    num_attention_heads=self.num_attention_heads		,    intermediate_size=self.intermediate_size		,    hidden_act=self.hidden_act		,    hidden_dropout_prob=self.hidden_dropout_prob		,    attention_probs_dropout_prob=self.attention_probs_dropout_prob		,    max_position_embeddings=self.max_position_embeddings		,    type_vocab_size=self.type_vocab_size		,    is_decoder=__snake_case		,    initializer_range=self.initializer_range		,    )

      return config, input_ids, token_type_ids, attention_mask







    def     lowercase_ (   self				:						Tuple			):
      a		:      Optional[int]       =  self.prepare_config_and_inputs()
      a     ,			a     ,			a     ,			a		:      Union[str, Any]       =  config_and_inputs
      a		:      str       =  {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': attention_mask}
      return config, inputs_dict







    def     lowercase_ (   self				:						List[str]			):
      a		:      int       =  self.prepare_config_and_inputs()
      a     ,			a     ,			a     ,			a		:      Tuple       =  config_and_inputs

      a		:      int       =  True
      a		:      int       =  floats_tensor([self.batch_size, self.seq_length, self.hidden_size]			)
      a		:      Optional[Any]       =  ids_tensor([self.batch_size, self.seq_length]		,    vocab_size=2			)

      return (
          config,
          input_ids,
          attention_mask,
          encoder_hidden_states,
          encoder_attention_mask,
      )

@require_flax
class 							a__(      lowerCamelCase__	,				unittest.TestCase			):
    lowercase__				=					True

    lowercase__				=					(
        (
            FlaxBertModel,
            FlaxBertForPreTraining,
            FlaxBertForMaskedLM,
            FlaxBertForMultipleChoice,
            FlaxBertForQuestionAnswering,
            FlaxBertForNextSentencePrediction,
            FlaxBertForSequenceClassification,
            FlaxBertForTokenClassification,
            FlaxBertForQuestionAnswering,
        )
        if is_flax_available()
        else ()
    )







    def     lowercase_ (   self				:						Union[str, Any]			):
      a		:      str       =  FlaxBertModelTester(self			)







    @slow
    def     lowercase_ (   self				:						Any			):
      # Only check this for base model, not necessary for all model classes.
      # This will also help speed-up tests.
      a		:      Optional[Any]       =  FlaxBertModel.from_pretrained('bert-base-cased'			)
      a		:      Union[str, Any]       =  model(np.ones((1, 1)			)			)
      self.assertIsNotNone(__snake_case			)