code: # coding=utf-8
  # Copyright 2020 The HuggingFace Inc. team.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  import inspect
  import os
  import shutil
  import tempfile
  import unittest
  from typing import List
  
  import numpy as np
  import pandas as pd
  
  from transformers import AddedToken
  from transformers.models.tapas.tokenization_tapas import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      TapasTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.testing_utils import (
      is_pt_tf_cross_test,
      require_pandas,
      require_scatter,
      require_tensorflow_probability,
      require_tokenizers,
      require_torch,
      slow,
  )
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english, merge_model_tokenizer_mappings
  
  
  @require_tokenizers
  @require_pandas
  class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
      tokenizer_class = TapasTokenizer
      test_rust_tokenizer = False
      space_between_special_tokens = True
      from_pretrained_filter = filter_non_english
      test_seq2seq = False
  
      def get_table(
          self,
          tokenizer: TapasTokenizer,
          length=5,
      ):
          toks = [tokenizer.decode([i], clean_up_tokenization_spaces=False) for i in range(len(tokenizer))]
  
          if length == 0:
              data = {}
          else:
              data = {toks[0]: [toks[tok] for tok in range(1, length)]}
  
          table = pd.DataFrame.from_dict(data)
  
          return table
  
      def get_table_and_query(
          self,
          tokenizer: TapasTokenizer,
          length=5,
      ):
          toks = [tokenizer.decode([i], clean_up_tokenization_spaces=False) for i in range(len(tokenizer))]
          table = self.get_table(tokenizer, length=length - 3)
          query = " ".join(toks[:3])
  
          return table, query
  
      def get_clean_sequence(
          self,
          tokenizer: TapasTokenizer,
          with_prefix_space=False,
          max_length=20,
          min_length=5,
          empty_table: bool = False,
          add_special_tokens: bool = True,
          return_table_and_query: bool = False,
      ):
  
          toks = [tokenizer.decode([i], clean_up_tokenization_spaces=False) for i in range(len(tokenizer))]
  
          if empty_table:
              table = pd.DataFrame.from_dict({})
              query = " ".join(toks[:min_length])
          else:
              data = {toks[0]: [toks[tok] for tok in range(1, min_length - 3)]}
              table = pd.DataFrame.from_dict(data)
              query = " ".join(toks[:3])
  
          output_ids = tokenizer.encode(table, query, add_special_tokens=add_special_tokens)
          output_txt = tokenizer.decode(output_ids)
  
          assert len(output_ids) >= min_length, "Update the code to generate the sequences so that they are larger"
          assert len(output_ids) <= max_length, "Update the code to generate the sequences so that they are smaller"
  
          if return_table_and_query:
              return output_txt, output_ids, table, query
  
          return output_txt, output_ids
  
      def setUp(self):
          super().setUp()
  
          vocab_tokens = [
              "[UNK]",
              "[CLS]",
              "[SEP]",
              "[PAD]",
              "[MASK]",
              "want",
              "##want",
              "##ed",
              "wa",
              "un",
              "runn",
              "##ing",
              ",",
              "low",
              "lowest",
          ]
          self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
          with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer:
              vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
  
      def get_input_output_texts(self, tokenizer):
          input_text = "UNwant\u00E9d,running"
          output_text = "unwanted, running"
          return input_text, output_text
  
      @require_tensorflow_probability
      def test_tf_encode_plus_sent_to_model(self):
          super().test_tf_encode_plus_sent_to_model()
  
      def test_rust_and_python_full_tokenizers(self):
          if not self.test_rust_tokenizer:
              return
  
          tokenizer = self.get_tokenizer()
          rust_tokenizer = self.get_rust_tokenizer()
  
          sequence = "UNwant\u00E9d,running"
  
          tokens = tokenizer.tokenize(sequence)
          rust_tokens = rust_tokenizer.tokenize(sequence)
          self.assertListEqual(tokens, rust_tokens)
  
          ids = tokenizer.encode(sequence, add_special_tokens=False)
          rust_ids = rust_tokenizer.encode(sequence, add_special_tokens=False)
          self.assertListEqual(ids, rust_ids)
  
          rust_tokenizer = self.get_rust_tokenizer()
          ids = tokenizer.encode(sequence)
          rust_ids = rust_tokenizer.encode(sequence)
          self.assertListEqual(ids, rust_ids)
  
          # With lower casing
          tokenizer = self.get_tokenizer(do_lower_case=True)
          rust_tokenizer = self.get_rust_tokenizer(do_lower_case=True)
  
          sequence = "UNwant\u00E9d,running"
  
          tokens = tokenizer.tokenize(sequence)
          rust_tokens = rust_tokenizer.tokenize(sequence)
          self.assertListEqual(tokens, rust_tokens)
  
          ids = tokenizer.encode(sequence, add_special_tokens=False)
          rust_ids = rust_tokenizer.encode(sequence, add_special_tokens=False)
          self.assertListEqual(ids, rust_ids)
  
          rust_tokenizer = self.get_rust_tokenizer()
          ids = tokenizer.encode(sequence)
          rust_ids = rust_tokenizer.encode(sequence)
          self.assertListEqual(ids, rust_ids)
  
      def test_chinese(self):
          tokenizer = BasicTokenizer()
  
          self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"), ["ah", "\u535A", "\u63A8", "zz"])
  
      def test_basic_tokenizer_lower(self):
          tokenizer = BasicTokenizer(do_lower_case=True)
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "), ["hello", "!", "how", "are", "you", "?"]
          )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["hello"])
  
      def test_basic_tokenizer_lower_strip_accents_false(self):
          tokenizer = BasicTokenizer(do_lower_case=True, strip_accents=False)
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "), ["hällo", "!", "how", "are", "you", "?"]
          )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["h\u00E9llo"])
  
      def test_basic_tokenizer_lower_strip_accents_true(self):
          tokenizer = BasicTokenizer(do_lower_case=True, strip_accents=True)
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "), ["hallo", "!", "how", "are", "you", "?"]
          )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["hello"])
  
      def test_basic_tokenizer_lower_strip_accents_default(self):
          tokenizer = BasicTokenizer(do_lower_case=True)
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "), ["hallo", "!", "how", "are", "you", "?"]
          )
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["hello"])
  
      def test_basic_tokenizer_no_lower(self):
          tokenizer = BasicTokenizer(do_lower_case=False)
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "), ["HeLLo", "!", "how", "Are", "yoU", "?"]
          )
  
      def test_basic_tokenizer_no_lower_strip_accents_false(self):
          tokenizer = BasicTokenizer(do_lower_case=False, strip_accents=False)
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "), ["HäLLo", "!", "how", "Are", "yoU", "?"]
          )
  
      def test_basic_tokenizer_no_lower_strip_accents_true(self):
          tokenizer = BasicTokenizer(do_lower_case=False, strip_accents=True)
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "), ["HaLLo", "!", "how", "Are", "yoU", "?"]
          )
  
      def test_basic_tokenizer_respects_never_split_tokens(self):
          tokenizer = BasicTokenizer(do_lower_case=False, never_split=["[UNK]"])
  
          self.assertListEqual(
              tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"), ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]
          )
  
      def test_wordpiece_tokenizer(self):
          vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]
  
          vocab = {}
          for i, token in enumerate(vocab_tokens):
              vocab[token] = i
          tokenizer = WordpieceTokenizer(vocab=vocab, unk_token="[UNK]")
  
          self.assertListEqual(tokenizer.tokenize(""), [])
  
          self.assertListEqual(tokenizer.tokenize("unwanted running"), ["un", "##want", "##ed", "runn", "##ing"])
  
          self.assertListEqual(tokenizer.tokenize("unwantedX running"), ["[UNK]", "runn", "##ing"])
  
      def test_is_whitespace(self):
          self.assertTrue(_is_whitespace(" "))
          self.assertTrue(_is_whitespace("\t"))
          self.assertTrue(_is_whitespace("\r"))
          self.assertTrue(_is_whitespace("\n"))
          self.assertTrue(_is_whitespace("\u00A0"))
  
          self.assertFalse(_is_whitespace("A"))
          self.assertFalse(_is_whitespace("-"))
  
      def test_is_control(self):
          self.assertTrue(_is_control("\u0005"))
  
          self.assertFalse(_is_control("A"))
          self.assertFalse(_is_control(" "))
          self.assertFalse(_is_control("\t"))
          self.assertFalse(_is_control("\r"))
  
      def test_is_punctuation(self):
          self.assertTrue(_is_punctuation("-"))
          self.assertTrue(_is_punctuation("$"))
          self.assertTrue(_is_punctuation("`"))
          self.assertTrue(_is_punctuation("."))
  
          self.assertFalse(_is_punctuation("A"))
          self.assertFalse(_is_punctuation(" "))
  
      def test_clean_text(self):
          tokenizer = self.get_tokenizer()
  
          # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
          self.assertListEqual(
              [tokenizer.tokenize(t) for t in ["Test", "\xad", "test"]], [["[UNK]"], ["[EMPTY]"], ["[UNK]"]]
          )
  
      @slow
      def test_sequence_builders(self):
          tokenizer = self.tokenizer_class.from_pretrained("google/tapas-base-finetuned-wtq")
  
          empty_table = self.get_table(tokenizer, length=0)
          table = self.get_table(tokenizer, length=10)
  
          text = tokenizer.encode(table, add_special_tokens=False)
          text_2 = tokenizer.encode(empty_table, "multi-sequence build", add_special_tokens=False)
  
          encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2)
  
          assert encoded_pair == [101] + text + [102] + text_2
  
      def test_offsets_with_special_characters(self):
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                  tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
  
                  sentence = f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                  tokens = tokenizer_r.encode_plus(
                      sentence,
                      return_attention_mask=False,
                      return_token_type_ids=False,
                      return_offsets_mapping=True,
                      add_special_tokens=True,
                  )
  
                  do_lower_case = tokenizer_r.do_lower_case if hasattr(tokenizer_r, "do_lower_case") else False
                  expected_results = (
                      [
                          ((0, 0), tokenizer_r.cls_token),
                          ((0, 1), "A"),
                          ((1, 2), ","),
                          ((3, 5), "na"),
                          ((5, 6), "##ï"),
                          ((6, 8), "##ve"),
                          ((9, 15), tokenizer_r.mask_token),
                          ((16, 21), "Allen"),
                          ((21, 23), "##NL"),
                          ((23, 24), "##P"),
                          ((25, 33), "sentence"),
                          ((33, 34), "."),
                          ((0, 0), tokenizer_r.sep_token),
                      ]
                      if not do_lower_case
                      else [
                          ((0, 0), tokenizer_r.cls_token),
                          ((0, 1), "a"),
                          ((1, 2), ","),
                          ((3, 8), "naive"),
                          ((9, 15), tokenizer_r.mask_token),
                          ((16, 21), "allen"),
                          ((21, 23), "##nl"),
                          ((23, 24), "##p"),
                          ((25, 33), "sentence"),
                          ((33, 34), "."),
                          ((0, 0), tokenizer_r.sep_token),
                      ]
                  )
  
                  self.assertEqual(
                      [e[1] for e in expected_results], tokenizer_r.convert_ids_to_tokens(tokens["input_ids"])
                  )
                  self.assertEqual([e[0] for e in expected_results], tokens["offset_mapping"])
  
      def test_add_special_tokens(self):
          tokenizers: List[TapasTokenizer] = self.get_tokenizers(do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  input_table = self.get_table(tokenizer, length=0)
  
                  special_token = "[SPECIAL_TOKEN]"
  
                  tokenizer.add_special_tokens({"cls_token": special_token})
                  encoded_special_token = tokenizer.encode(input_table, special_token, add_special_tokens=False)
                  self.assertEqual(len(encoded_special_token), 1)
  
                  decoded = tokenizer.decode(encoded_special_token, skip_special_tokens=True)
                  self.assertTrue(special_token not in decoded)
  
      def test_add_tokens_tokenizer(self):
          tokenizers: List[TapasTokenizer] = self.get_tokenizers(do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  table = self.get_table(tokenizer, length=0)
                  vocab_size = tokenizer.vocab_size
                  all_size = len(tokenizer)
  
                  self.assertNotEqual(vocab_size, 0)
  
                  # We usually have added tokens from the start in tests because our vocab fixtures are
                  # smaller than the original vocabs - let's not assert this
                  # self.assertEqual(vocab_size, all_size)
  
                  new_toks = ["aaaaa bbbbbb", "cccccccccdddddddd"]
                  added_toks = tokenizer.add_tokens(new_toks)
                  vocab_size_2 = tokenizer.vocab_size
                  all_size_2 = len(tokenizer)
  
                  self.assertNotEqual(vocab_size_2, 0)
                  self.assertEqual(vocab_size, vocab_size_2)
                  self.assertEqual(added_toks, len(new_toks))
                  self.assertEqual(all_size_2, all_size + len(new_toks))
  
                  tokens = tokenizer.encode(table, "aaaaa bbbbbb low cccccccccdddddddd l", add_special_tokens=False)
  
                  self.assertGreaterEqual(len(tokens), 4)
                  self.assertGreater(tokens[0], tokenizer.vocab_size - 1)
                  self.assertGreater(tokens[-2], tokenizer.vocab_size - 1)
  
                  new_toks_2 = {"eos_token": ">>>>|||<||<<|<<", "pad_token": "<<<<<|||>|>>>>|>"}
                  added_toks_2 = tokenizer.add_special_tokens(new_toks_2)
                  vocab_size_3 = tokenizer.vocab_size
                  all_size_3 = len(tokenizer)
  
                  self.assertNotEqual(vocab_size_3, 0)
                  self.assertEqual(vocab_size, vocab_size_3)
                  self.assertEqual(added_toks_2, len(new_toks_2))
                  self.assertEqual(all_size_3, all_size_2 + len(new_toks_2))
  
                  tokens = tokenizer.encode(
                      table,
                      ">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l",
                      add_special_tokens=False,
                  )
  
                  self.assertGreaterEqual(len(tokens), 6)
                  self.assertGreater(tokens[0], tokenizer.vocab_size - 1)
                  self.assertGreater(tokens[0], tokens[1])
                  self.assertGreater(tokens[-2], tokenizer.vocab_size - 1)
                  self.assertGreater(tokens[-2], tokens[-3])
                  self.assertEqual(tokens[0], tokenizer.eos_token_id)
                  self.assertEqual(tokens[-2], tokenizer.pad_token_id)
  
      @require_tokenizers
      def test_encode_decode_with_spaces(self):
          tokenizers = self.get_tokenizers(do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  table = self.get_table(tokenizer, length=0)
  
                  new_toks = [AddedToken("[ABC]", normalized=False), AddedToken("[DEF]", normalized=False)]
                  tokenizer.add_tokens(new_toks)
                  input = "[ABC][DEF][ABC][DEF]"
                  if self.space_between_special_tokens:
                      output = "[ABC] [DEF] [ABC] [DEF]"
                  else:
                      output = input
                  encoded = tokenizer.encode(table, input, add_special_tokens=False)
                  decoded = tokenizer.decode(encoded, spaces_between_special_tokens=self.space_between_special_tokens)
                  self.assertIn(decoded, [output, output.lower()])
  
      def test_encode_plus_with_padding(self):
          tokenizers = self.get_tokenizers(do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  table = self.get_table(tokenizer, length=0)
                  sequence = "Sequence"
  
                  # check correct behaviour if no pad_token_id exists and add it eventually
                  self._check_no_pad_token_padding(tokenizer, sequence)
  
                  padding_size = 10
                  padding_idx = tokenizer.pad_token_id
                  token_type_padding_idx = tokenizer.pad_token_type_id
  
                  encoded_sequence = tokenizer.encode_plus(table, sequence, return_special_tokens_mask=True)
                  input_ids = encoded_sequence["input_ids"]
                  special_tokens_mask = encoded_sequence["special_tokens_mask"]
                  sequence_length = len(input_ids)
  
                  # Test 'longest' and 'no_padding' don't do anything
                  tokenizer.padding_side = "right"
  
                  not_padded_sequence = tokenizer.encode_plus(
                      table,
                      sequence,
                      padding=False,
                      return_special_tokens_mask=True,
                  )
                  not_padded_input_ids = not_padded_sequence["input_ids"]
  
                  not_padded_special_tokens_mask = not_padded_sequence["special_tokens_mask"]
                  not_padded_sequence_length = len(not_padded_input_ids)
  
                  assert sequence_length == not_padded_sequence_length
                  assert input_ids == not_padded_input_ids
                  assert special_tokens_mask == not_padded_special_tokens_mask
  
                  not_padded_sequence = tokenizer.encode_plus(
                      table,
                      sequence,
                      padding=False,
                      return_special_tokens_mask=True,
                  )
                  not_padded_input_ids = not_padded_sequence["input_ids"]
  
                  not_padded_special_tokens_mask = not_padded_sequence["special_tokens_mask"]
                  not_padded_sequence_length = len(not_padded_input_ids)
  
                  assert sequence_length == not_padded_sequence_length
                  assert input_ids == not_padded_input_ids
                  assert special_tokens_mask == not_padded_special_tokens_mask
  
                  # Test right padding
                  tokenizer.padding_side = "right"
  
                  right_padded_sequence = tokenizer.encode_plus(
                      table,
                      sequence,
                      max_length=sequence_length + padding_size,
                      padding="max_length",
                      return_special_tokens_mask=True,
                  )
                  right_padded_input_ids = right_padded_sequence["input_ids"]
  
                  right_padded_special_tokens_mask = right_padded_sequence["special_tokens_mask"]
                  right_padded_sequence_length = len(right_padded_input_ids)
  
                  assert sequence_length + padding_size == right_padded_sequence_length
                  assert input_ids + [padding_idx] * padding_size == right_padded_input_ids
                  assert special_tokens_mask + [1] * padding_size == right_padded_special_tokens_mask
  
                  # Test left padding
                  tokenizer.padding_side = "left"
                  left_padded_sequence = tokenizer.encode_plus(
                      table,
                      sequence,
                      max_length=sequence_length + padding_size,
                      padding="max_length",
                      return_special_tokens_mask=True,
                  )
                  left_padded_input_ids = left_padded_sequence["input_ids"]
                  left_padded_special_tokens_mask = left_padded_sequence["special_tokens_mask"]
                  left_padded_sequence_length = len(left_padded_input_ids)
  
                  assert sequence_length + padding_size == left_padded_sequence_length
                  assert [padding_idx] * padding_size + input_ids == left_padded_input_ids
                  assert [1] * padding_size + special_tokens_mask == left_padded_special_tokens_mask
  
                  if "token_type_ids" in tokenizer.model_input_names:
                      token_type_ids = encoded_sequence["token_type_ids"]
                      left_padded_token_type_ids = left_padded_sequence["token_type_ids"]
                      right_padded_token_type_ids = right_padded_sequence["token_type_ids"]
  
                      assert (
                          token_type_ids + [[token_type_padding_idx] * 7] * padding_size == right_padded_token_type_ids
                      )
                      assert [[token_type_padding_idx] * 7] * padding_size + token_type_ids == left_padded_token_type_ids
  
                  if "attention_mask" in tokenizer.model_input_names:
                      attention_mask = encoded_sequence["attention_mask"]
                      right_padded_attention_mask = right_padded_sequence["attention_mask"]
                      left_padded_attention_mask = left_padded_sequence["attention_mask"]
  
                      assert attention_mask + [0] * padding_size == right_padded_attention_mask
                      assert [0] * padding_size + attention_mask == left_padded_attention_mask
  
      def test_internal_consistency(self):
          tokenizers = self.get_tokenizers()
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  table = self.get_table(tokenizer, length=0)
                  input_text, output_text = self.get_input_output_texts(tokenizer)
  
                  tokens = tokenizer.tokenize(input_text)
                  ids = tokenizer.convert_tokens_to_ids(tokens)
                  ids_2 = tokenizer.encode(table, input_text, add_special_tokens=False)
                  self.assertListEqual(ids, ids_2)
  
                  tokens_2 = tokenizer.convert_ids_to_tokens(ids)
                  self.assertNotEqual(len(tokens_2), 0)
                  text_2 = tokenizer.decode(ids)
                  self.assertIsInstance(text_2, str)
  
                  self.assertEqual(text_2, output_text)
  
      def test_mask_output(self):
          tokenizers = self.get_tokenizers(fast=False, do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  table, query = self.get_table_and_query(tokenizer)
  
                  if (
                      tokenizer.build_inputs_with_special_tokens.__qualname__.split(".")[0] != "PreTrainedTokenizer"
                      and "token_type_ids" in tokenizer.model_input_names
                  ):
                      information = tokenizer.encode_plus(table, query, add_special_tokens=True)
                      sequences, mask = information["input_ids"], information["token_type_ids"]
                      self.assertEqual(len(sequences), len(mask))
  
      @unittest.skip("TAPAS tokenizer only handles two sequences.")
      def test_maximum_encoding_length_pair_input(self):
          pass
  
      @unittest.skip("TAPAS tokenizer only handles two sequences.")
      def test_maximum_encoding_length_single_input(self):
          pass
  
      def test_number_of_added_tokens(self):
          tokenizers = self.get_tokenizers(do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
  
                  table, query = self.get_table_and_query(tokenizer)
  
                  sequences = tokenizer.encode(table, query, add_special_tokens=False)
                  attached_sequences = tokenizer.encode(table, query, add_special_tokens=True)
  
                  # Method is implemented (e.g. not GPT-2)
                  if len(attached_sequences) != 2:
                      self.assertEqual(
                          tokenizer.num_special_tokens_to_add(pair=True), len(attached_sequences) - len(sequences)
                      )
  
      def test_padding_to_max_length(self):
          """We keep this test for backward compatibility but it should be removed when `pad_to_max_length` will be deprecated"""
          tokenizers = self.get_tokenizers(do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  table = self.get_table(tokenizer)
                  sequence = "Sequence"
                  padding_size = 10
  
                  # check correct behaviour if no pad_token_id exists and add it eventually
                  self._check_no_pad_token_padding(tokenizer, sequence)
  
                  padding_idx = tokenizer.pad_token_id
  
                  # Check that it correctly pads when a maximum length is specified along with the padding flag set to True
                  tokenizer.padding_side = "right"
                  encoded_sequence = tokenizer.encode(table, sequence)
                  sequence_length = len(encoded_sequence)
                  # FIXME: the next line should be padding(max_length) to avoid warning
                  padded_sequence = tokenizer.encode(
                      table, sequence, max_length=sequence_length + padding_size, padding=True
                  )
                  padded_sequence_length = len(padded_sequence)
                  assert sequence_length + padding_size == padded_sequence_length
                  assert encoded_sequence + [padding_idx] * padding_size == padded_sequence
  
                  # Check that nothing is done when a maximum length is not specified
                  encoded_sequence = tokenizer.encode(table, sequence)
                  sequence_length = len(encoded_sequence)
  
                  tokenizer.padding_side = "right"
                  padded_sequence_right = tokenizer.encode(table, sequence, pad_to_max_length=True)
                  padded_sequence_right_length = len(padded_sequence_right)
                  assert sequence_length == padded_sequence_right_length
                  assert encoded_sequence == padded_sequence_right
  
      def test_call(self):
          # Tests that all call wrap to encode_plus and batch_encode_plus
          tokenizers = self.get_tokenizers(do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  sequences = [
                      "Testing batch encode plus",
                      "Testing batch encode plus with different sequence lengths",
                      "Testing batch encode plus with different sequence lengths correctly pads",
                  ]
  
                  # Test not batched
                  table = self.get_table(tokenizer, length=0)
                  encoded_sequences_1 = tokenizer.encode_plus(table, sequences[0])
                  encoded_sequences_2 = tokenizer(table, sequences[0])
                  self.assertEqual(encoded_sequences_1, encoded_sequences_2)
  
                  # Test not batched pairs
                  table = self.get_table(tokenizer, length=10)
                  encoded_sequences_1 = tokenizer.encode_plus(table, sequences[1])
                  encoded_sequences_2 = tokenizer(table, sequences[1])
                  self.assertEqual(encoded_sequences_1, encoded_sequences_2)
  
                  # Test batched
                  table = self.get_table(tokenizer, length=0)
                  encoded_sequences_1 = tokenizer.batch_encode_plus(table, sequences)
                  encoded_sequences_2 = tokenizer(table, sequences)
                  self.assertEqual(encoded_sequences_1, encoded_sequences_2)
  
      def test_batch_encode_plus_batch_sequence_length(self):
          # Tests that all encoded values have the correct size
          tokenizers = self.get_tokenizers(do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  table = self.get_table(tokenizer, length=0)
                  sequences = [
                      "Testing batch encode plus",
                      "Testing batch encode plus with different sequence lengths",
                      "Testing batch encode plus with different sequence lengths correctly pads",
                  ]
  
                  encoded_sequences = [tokenizer.encode_plus(table, sequence) for sequence in sequences]
                  encoded_sequences_batch = tokenizer.batch_encode_plus(table, sequences, padding=False)
                  self.assertListEqual(
                      encoded_sequences, self.convert_batch_encode_plus_format_to_encode_plus(encoded_sequences_batch)
                  )
  
                  maximum_length = len(
                      max([encoded_sequence["input_ids"] for encoded_sequence in encoded_sequences], key=len)
                  )
  
                  # check correct behaviour if no pad_token_id exists and add it eventually
                  self._check_no_pad_token_padding(tokenizer, sequences)
  
                  encoded_sequences_padded = [
                      tokenizer.encode_plus(table, sequence, max_length=maximum_length, padding="max_length")
                      for sequence in sequences
                  ]
  
                  encoded_sequences_batch_padded = tokenizer.batch_encode_plus(table, sequences, padding=True)
                  self.assertListEqual(
                      encoded_sequences_padded,
                      self.convert_batch_encode_plus_format_to_encode_plus(encoded_sequences_batch_padded),
                  )
  
                  # check 'longest' is unsensitive to a max length
                  encoded_sequences_batch_padded_1 = tokenizer.batch_encode_plus(table, sequences, padding=True)
                  encoded_sequences_batch_padded_2 = tokenizer.batch_encode_plus(
                      table, sequences, max_length=maximum_length + 10, padding="longest"
                  )
                  for key in encoded_sequences_batch_padded_1.keys():
                      self.assertListEqual(
                          encoded_sequences_batch_padded_1[key],
                          encoded_sequences_batch_padded_2[key],
                      )
  
                  # check 'no_padding' is unsensitive to a max length
                  encoded_sequences_batch_padded_1 = tokenizer.batch_encode_plus(table, sequences, padding=False)
                  encoded_sequences_batch_padded_2 = tokenizer.batch_encode_plus(
                      table, sequences, max_length=maximum_length + 10, padding=False
                  )
                  for key in encoded_sequences_batch_padded_1.keys():
                      self.assertListEqual(
                          encoded_sequences_batch_padded_1[key],
                          encoded_sequences_batch_padded_2[key],
                      )
  
      @unittest.skip("batch_encode_plus does not handle overflowing tokens.")
      def test_batch_encode_plus_overflowing_tokens(self):
          pass
  
      def test_batch_encode_plus_padding(self):
          # Test that padded sequences are equivalent between batch_encode_plus and encode_plus
  
          # Right padding tests
          tokenizers = self.get_tokenizers(do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  table = self.get_table(tokenizer, length=0)
                  sequences = [
                      "Testing batch encode plus",
                      "Testing batch encode plus with different sequence lengths",
                      "Testing batch encode plus with different sequence lengths correctly pads",
                  ]
  
                  max_length = 100
  
                  # check correct behaviour if no pad_token_id exists and add it eventually
                  self._check_no_pad_token_padding(tokenizer, sequences)
  
                  encoded_sequences = [
                      tokenizer.encode_plus(table, sequence, max_length=max_length, padding="max_length")
                      for sequence in sequences
                  ]
                  encoded_sequences_batch = tokenizer.batch_encode_plus(
                      table, sequences, max_length=max_length, padding="max_length"
                  )
                  self.assertListEqual(
                      encoded_sequences, self.convert_batch_encode_plus_format_to_encode_plus(encoded_sequences_batch)
                  )
  
          # Left padding tests
          tokenizers = self.get_tokenizers(do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  tokenizer.padding_side = "left"
                  sequences = [
                      "Testing batch encode plus",
                      "Testing batch encode plus with different sequence lengths",
                      "Testing batch encode plus with different sequence lengths correctly pads",
                  ]
  
                  max_length = 100
  
                  # check correct behaviour if no pad_token_id exists and add it eventually
                  self._check_no_pad_token_padding(tokenizer, sequences)
  
                  encoded_sequences = [
                      tokenizer.encode_plus(table, sequence, max_length=max_length, padding="max_length")
                      for sequence in sequences
                  ]
                  encoded_sequences_batch = tokenizer.batch_encode_plus(
                      table, sequences, max_length=max_length, padding="max_length"
                  )
                  self.assertListEqual(
                      encoded_sequences, self.convert_batch_encode_plus_format_to_encode_plus(encoded_sequences_batch)
                  )
  
      def test_padding_to_multiple_of(self):
          tokenizers = self.get_tokenizers()
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  table = self.get_table(tokenizer, length=0)
                  if tokenizer.pad_token is None:
                      self.skipTest("No padding token.")
                  else:
                      empty_tokens = tokenizer(table, padding=True, pad_to_multiple_of=8)
                      normal_tokens = tokenizer(table, "This is a sample input", padding=True, pad_to_multiple_of=8)
                      for key, value in empty_tokens.items():
                          self.assertEqual(len(value) % 8, 0, f"BatchEncoding.{key} is not multiple of 8")
                      for key, value in normal_tokens.items():
                          self.assertEqual(len(value) % 8, 0, f"BatchEncoding.{key} is not multiple of 8")
  
                      normal_tokens = tokenizer(table, "This", pad_to_multiple_of=8)
                      for key, value in normal_tokens.items():
                          self.assertNotEqual(len(value) % 8, 0, f"BatchEncoding.{key} is not multiple of 8")
  
                      # Should also work with truncation
                      normal_tokens = tokenizer(table, "This", padding=True, truncation=True, pad_to_multiple_of=8)
                      for key, value in normal_tokens.items():
                          self.assertEqual(len(value) % 8, 0, f"BatchEncoding.{key} is not multiple of 8")
  
      @unittest.skip("TAPAS cannot handle `prepare_for_model` without passing by `encode_plus` or `batch_encode_plus`")
      def test_prepare_for_model(self):
          pass
  
      def test_tokenizer_slow_store_full_signature(self):
          signature = inspect.signature(self.tokenizer_class.__init__)
          tokenizer = self.get_tokenizer()
  
          for parameter_name, parameter in signature.parameters.items():
              if parameter.default != inspect.Parameter.empty:
                  self.assertIn(parameter_name, tokenizer.init_kwargs)
  
      def test_special_tokens_mask_input_pairs(self):
          tokenizers = self.get_tokenizers(do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  sequence_0 = "Encode this."
                  empty_table = self.get_table(tokenizer, length=0)
                  table = self.get_table(tokenizer, length=10)
                  encoded_sequence = tokenizer.encode(empty_table, sequence_0, add_special_tokens=False)
                  encoded_sequence += tokenizer.encode(table, "", add_special_tokens=False)
                  encoded_sequence_dict = tokenizer.encode_plus(
                      table,
                      sequence_0,
                      add_special_tokens=True,
                      return_special_tokens_mask=True,
                      # add_prefix_space=False,
                  )
                  encoded_sequence_w_special = encoded_sequence_dict["input_ids"]
                  special_tokens_mask = encoded_sequence_dict["special_tokens_mask"]
                  self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special))
  
                  filtered_sequence = [
                      (x if not special_tokens_mask[i] else None) for i, x in enumerate(encoded_sequence_w_special)
                  ]
                  filtered_sequence = [x for x in filtered_sequence if x is not None]
                  self.assertEqual(encoded_sequence, filtered_sequence)
  
      def test_special_tokens_mask(self):
          tokenizers = self.get_tokenizers(do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  table = self.get_table(tokenizer, length=0)
                  sequence_0 = "Encode this."
                  # Testing single inputs
                  encoded_sequence = tokenizer.encode(table, sequence_0, add_special_tokens=False)
                  encoded_sequence_dict = tokenizer.encode_plus(
                      table, sequence_0, add_special_tokens=True, return_special_tokens_mask=True
                  )
                  encoded_sequence_w_special = encoded_sequence_dict["input_ids"]
                  special_tokens_mask = encoded_sequence_dict["special_tokens_mask"]
                  self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special))
  
                  filtered_sequence = [x for i, x in enumerate(encoded_sequence_w_special) if not special_tokens_mask[i]]
                  self.assertEqual(encoded_sequence, filtered_sequence)
  
      def test_save_and_load_tokenizer(self):
          # safety check on max_len default value so we are sure the test works
          tokenizers = self.get_tokenizers()
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  self.assertNotEqual(tokenizer.model_max_length, 42)
  
          # Now let's start the test
          tokenizers = self.get_tokenizers()
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  # Isolate this from the other tests because we save additional tokens/etc
                  table = self.get_table(tokenizer, length=0)
                  tmpdirname = tempfile.mkdtemp()
  
                  sample_text = " He is very happy, UNwant\u00E9d,running"
                  before_tokens = tokenizer.encode(table, sample_text, add_special_tokens=False)
                  before_vocab = tokenizer.get_vocab()
                  tokenizer.save_pretrained(tmpdirname)
  
                  after_tokenizer = tokenizer.__class__.from_pretrained(tmpdirname)
                  after_tokens = after_tokenizer.encode(table, sample_text, add_special_tokens=False)
                  after_vocab = after_tokenizer.get_vocab()
                  self.assertListEqual(before_tokens, after_tokens)
                  self.assertDictEqual(before_vocab, after_vocab)
  
                  shutil.rmtree(tmpdirname)
  
      @unittest.skip("Not implemented")
      def test_right_and_left_truncation(self):
          pass
  
      def test_right_and_left_padding(self):
          tokenizers = self.get_tokenizers(do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  table = self.get_table(tokenizer, length=0)
                  sequence = "Sequence"
                  padding_size = 10
  
                  # check correct behaviour if no pad_token_id exists and add it eventually
                  self._check_no_pad_token_padding(tokenizer, sequence)
  
                  padding_idx = tokenizer.pad_token_id
  
                  # RIGHT PADDING - Check that it correctly pads when a maximum length is specified along with the padding flag set to True
                  tokenizer.padding_side = "right"
                  encoded_sequence = tokenizer.encode(table, sequence)
                  sequence_length = len(encoded_sequence)
                  padded_sequence = tokenizer.encode(
                      table, sequence, max_length=sequence_length + padding_size, padding="max_length"
                  )
                  padded_sequence_length = len(padded_sequence)
                  assert sequence_length + padding_size == padded_sequence_length
                  assert encoded_sequence + [padding_idx] * padding_size == padded_sequence
  
                  # LEFT PADDING - Check that it correctly pads when a maximum length is specified along with the padding flag set to True
                  tokenizer.padding_side = "left"
                  encoded_sequence = tokenizer.encode(table, sequence)
                  sequence_length = len(encoded_sequence)
                  padded_sequence = tokenizer.encode(
                      table, sequence, max_length=sequence_length + padding_size, padding="max_length"
                  )
                  padded_sequence_length = len(padded_sequence)
                  assert sequence_length + padding_size == padded_sequence_length
                  assert [padding_idx] * padding_size + encoded_sequence == padded_sequence
  
                  # RIGHT & LEFT PADDING - Check that nothing is done for 'longest' and 'no_padding'
                  encoded_sequence = tokenizer.encode(table, sequence)
                  sequence_length = len(encoded_sequence)
  
                  tokenizer.padding_side = "right"
                  padded_sequence_right = tokenizer.encode(table, sequence, padding=True)
                  padded_sequence_right_length = len(padded_sequence_right)
                  assert sequence_length == padded_sequence_right_length
                  assert encoded_sequence == padded_sequence_right
  
                  tokenizer.padding_side = "left"
                  padded_sequence_left = tokenizer.encode(table, sequence, padding="longest")
                  padded_sequence_left_length = len(padded_sequence_left)
                  assert sequence_length == padded_sequence_left_length
                  assert encoded_sequence == padded_sequence_left
  
                  tokenizer.padding_side = "right"
                  padded_sequence_right = tokenizer.encode(table, sequence)
                  padded_sequence_right_length = len(padded_sequence_right)
                  assert sequence_length == padded_sequence_right_length
                  assert encoded_sequence == padded_sequence_right
  
                  tokenizer.padding_side = "left"
                  padded_sequence_left = tokenizer.encode(table, sequence, padding=False)
                  padded_sequence_left_length = len(padded_sequence_left)
                  assert sequence_length == padded_sequence_left_length
                  assert encoded_sequence == padded_sequence_left
  
      def test_token_type_ids(self):
          tokenizers = self.get_tokenizers()
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  empty_table = self.get_table(tokenizer, length=0)
                  seq_0 = "Test this method."
  
                  # We want to have sequence 0 and sequence 1 are tagged
                  # respectively with 0 and 1 token_ids
                  # (regardless of whether the model use token type ids)
                  # We use this assumption in the QA pipeline among other place
                  output = tokenizer(empty_table, seq_0, return_token_type_ids=True)
  
                  # Assert that the token type IDs have the same length as the input IDs
                  self.assertEqual(len(output["token_type_ids"]), len(output["input_ids"]))
  
                  # Assert that each token type ID has 7 values
                  self.assertTrue(all(len(token_type_ids) == 7 for token_type_ids in output["token_type_ids"]))
  
                  # Do the same test as modeling common.
                  self.assertIn(0, output["token_type_ids"][0])
  
      @require_torch
      @slow
      @require_scatter
      def test_torch_encode_plus_sent_to_model(self):
          import torch
  
          from transformers import MODEL_MAPPING, TOKENIZER_MAPPING
  
          MODEL_TOKENIZER_MAPPING = merge_model_tokenizer_mappings(MODEL_MAPPING, TOKENIZER_MAPPING)
  
          tokenizers = self.get_tokenizers(do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
  
                  if tokenizer.__class__ not in MODEL_TOKENIZER_MAPPING:
                      return
  
                  config_class, model_class = MODEL_TOKENIZER_MAPPING[tokenizer.__class__]
                  config = config_class()
  
                  if config.is_encoder_decoder or config.pad_token_id is None:
                      return
  
                  model = model_class(config)
  
                  # Make sure the model contains at least the full vocabulary size in its embedding matrix
                  is_using_common_embeddings = hasattr(model.get_input_embeddings(), "weight")
                  assert (
                      (model.get_input_embeddings().weight.shape[0] >= len(tokenizer))
                      if is_using_common_embeddings
                      else True
                  )
  
                  # Build sequence
                  first_ten_tokens = list(tokenizer.get_vocab().keys())[:10]
                  sequence = " ".join(first_ten_tokens)
                  table = self.get_table(tokenizer, length=0)
                  encoded_sequence = tokenizer.encode_plus(table, sequence, return_tensors="pt")
                  batch_encoded_sequence = tokenizer.batch_encode_plus(table, [sequence, sequence], return_tensors="pt")
                  # This should not fail
  
                  with torch.no_grad():  # saves some time
                      model(**encoded_sequence)
                      model(**batch_encoded_sequence)
  
      @unittest.skip("TAPAS doesn't handle pre-tokenized inputs.")
      def test_pretokenized_inputs(self):
          pass
  
      @slow
      def test_tapas_truncation_integration_test(self):
          data = {
              "Actors": ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"],
              "Age": ["56", "45", "59"],
              "Number of movies": ["87", "53", "69"],
              "Date of birth": ["18 december 1963", "11 november 1974", "6 may 1961"],
          }
          queries = [
              "When was Brad Pitt born?",
              "Which actor appeared in the least number of movies?",
              "What is the average number of movies?",
          ]
          table = pd.DataFrame.from_dict(data)
  
          tokenizer = TapasTokenizer.from_pretrained("lysandre/tapas-temporary-repo", model_max_length=512)
  
          for i in range(12):
              # The table cannot even encode the headers, so raise an error
              with self.assertRaises(ValueError):
                  tokenizer.encode(table=table, query=queries[0], max_length=i, truncation="drop_rows_to_fit")
  
          for i in range(12, 512):
              new_encoded_inputs = tokenizer.encode(
                  table=table, query=queries[0], max_length=i, truncation="drop_rows_to_fit"
              )
  
              # Ensure that the input IDs are less than the max length defined.
              self.assertLessEqual(len(new_encoded_inputs), i)
  
          tokenizer.model_max_length = 20
          new_encoded_inputs = tokenizer.encode(table=table, query=queries[0], truncation=True)
          dropped_encoded_inputs = tokenizer.encode(table=table, query=queries[0], truncation="drop_rows_to_fit")
  
          # Ensure that the input IDs are still truncated when no max_length is specified
          self.assertListEqual(new_encoded_inputs, dropped_encoded_inputs)
          self.assertLessEqual(len(new_encoded_inputs), 20)
  
      @slow
      def test_min_max_question_length(self):
          data = {
              "Actors": ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"],
              "Age": ["56", "45", "59"],
              "Number of movies": ["87", "53", "69"],
              "Date of birth": ["18 december 1963", "11 november 1974", "6 may 1961"],
          }
          queries = "When was Brad Pitt born?"
          table = pd.DataFrame.from_dict(data)
  
          # test max_question_length
          tokenizer = TapasTokenizer.from_pretrained("lysandre/tapas-temporary-repo", max_question_length=2)
  
          encoding = tokenizer(table=table, queries=queries)
  
          # query should not be tokenized as it's longer than the specified max_question_length
          expected_results = [101, 102]
  
          self.assertListEqual(encoding.input_ids[:2], expected_results)
  
          # test min_question_length
          tokenizer = TapasTokenizer.from_pretrained("lysandre/tapas-temporary-repo", min_question_length=30)
  
          encoding = tokenizer(table=table, queries=queries)
  
          # query should not be tokenized as it's shorter than the specified min_question_length
          expected_results = [101, 102]
  
          self.assertListEqual(encoding.input_ids[:2], expected_results)
  
      @is_pt_tf_cross_test
      def test_batch_encode_plus_tensors(self):
          tokenizers = self.get_tokenizers(do_lower_case=False)
          for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"):
                  sequences = [
                      "Testing batch encode plus",
                      "Testing batch encode plus with different sequence lengths",
                      "Testing batch encode plus with different sequence lengths correctly pads",
                  ]
  
                  table = self.get_table(tokenizer, length=0)
  
                  # A Tensor cannot be build by sequences which are not the same size
                  self.assertRaises(ValueError, tokenizer.batch_encode_plus, table, sequences, return_tensors="pt")
                  self.assertRaises(ValueError, tokenizer.batch_encode_plus, table, sequences, return_tensors="tf")
  
                  if tokenizer.pad_token_id is None:
                      self.assertRaises(
                          ValueError,
                          tokenizer.batch_encode_plus,
                          table,
                          sequences,
                          padding=True,
                          return_tensors="pt",
                      )
                      self.assertRaises(
                          ValueError,
                          tokenizer.batch_encode_plus,
                          table,
                          sequences,
                          padding="longest",
                          return_tensors="tf",
                      )
                  else:
                      pytorch_tensor = tokenizer.batch_encode_plus(table, sequences, padding=True, return_tensors="pt")
                      tensorflow_tensor = tokenizer.batch_encode_plus(
                          table, sequences, padding="longest", return_tensors="tf"
                      )
                      encoded_sequences = tokenizer.batch_encode_plus(table, sequences, padding=True)
  
                      for key in encoded_sequences.keys():
                          pytorch_value = pytorch_tensor[key].tolist()
                          tensorflow_value = tensorflow_tensor[key].numpy().tolist()
                          encoded_value = encoded_sequences[key]
  
                          self.assertEqual(pytorch_value, tensorflow_value, encoded_value)
  
      @slow
      def test_tapas_integration_test(self):
          data = {
              "Actors": ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"],
              "Age": ["56", "45", "59"],
              "Number of movies": ["87", "53", "69"],
              "Date of birth": ["18 december 1963", "11 november 1974", "6 may 1961"],
          }
          queries = [
              "When was Brad Pitt born?",
              "Which actor appeared in the least number of movies?",
              "What is the average number of movies?",
          ]
          table = pd.DataFrame.from_dict(data)
  
          tokenizer = TapasTokenizer.from_pretrained("google/tapas-base-finetuned-wtq", model_max_length=512)
  
          # fmt: off
          expected_results = {'input_ids':[101,2043,2001,8226,15091,2141,1029,102,5889,2287,2193,1997,5691,3058,1997,4182,8226,15091,5179,6584,2324,2285,3699,14720,4487,6178,9488,3429,5187,2340,2281,3326,2577,18856,7828,3240,5354,6353,1020,2089,3777],'attention_mask':[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],'token_type_ids':[[0,0,0,0,0,0,0],[0,0,0,0,0,0,0],[0,0,0,0,0,0,0],[0,0,0,0,0,0,0],[0,0,0,0,0,0,0],[0,0,0,0,0,0,0],[0,0,0,0,0,0,0],[0,0,0,0,0,0,0],[1,1,0,0,0,0,0],[1,2,0,0,0,0,0],[1,3,0,0,0,0,0],[1,3,0,0,0,0,0],[1,3,0,0,0,0,0],[1,4,0,0,0,0,0],[1,4,0,0,0,0,0],[1,4,0,0,0,0,0],[1,1,1,0,0,0,0],[1,1,1,0,0,0,0],[1,2,1,0,2,2,0],[1,3,1,0,3,1,0],[1,4,1,0,2,2,0],[1,4,1,0,2,2,0],[1,4,1,0,2,2,0],[1,1,2,0,0,0,0],[1,1,2,0,0,0,0],[1,1,2,0,0,0,0],[1,1,2,0,0,0,0],[1,2,2,0,1,3,0],[1,3,2,0,1,3,0],[1,4,2,0,3,1,0],[1,4,2,0,3,1,0],[1,4,2,0,3,1,0],[1,1,3,0,0,0,0],[1,1,3,0,0,0,0],[1,1,3,0,0,0,0],[1,1,3,0,0,0,0],[1,2,3,0,3,1,0],[1,3,3,0,2,2,0],[1,4,3,0,1,3,0],[1,4,3,0,1,3,0],[1,4,3,0,1,3,0]]}  # noqa: E231
          # fmt: on
  
          new_encoded_inputs = tokenizer.encode_plus(table=table, query=queries[0])
  
          self.assertDictEqual(dict(new_encoded_inputs), expected_results)
  
      @slow
      def test_full_tokenizer(self):
          data = [
              ["Pos", "No", "Driver", "Team", "Laps", "Time/Retired", "Grid", "Points"],
              ["1", "32", "Patrick Carpentier", "Team Player's", "87", "1:48:11.023", "1", "22"],
              ["2", "1", "Bruno Junqueira", "Newman/Haas Racing", "87", "+0.8 secs", "2", "17"],
              ["3", "3", "Paul Tracy", "Team Player's", "87", "+28.6 secs", "3", "14"],
              ["4", "9", "Michel Jourdain, Jr.", "Team Rahal", "87", "+40.8 secs", "13", "12"],
              ["5", "34", "Mario Haberfeld", "Mi-Jack Conquest Racing", "87", "+42.1 secs", "6", "10"],
              ["6", "20", "Oriol Servia", "Patrick Racing", "87", "+1:00.2", "10", "8"],
              ["7", "51", "Adrian Fernandez", "Fernandez Racing", "87", "+1:01.4", "5", "6"],
              ["8", "12", "Jimmy Vasser", "American Spirit Team Johansson", "87", "+1:01.8", "8", "5"],
              ["9", "7", "Tiago Monteiro", "Fittipaldi-Dingman Racing", "86", "+ 1 Lap", "15", "4"],
              ["10", "55", "Mario Dominguez", "Herdez Competition", "86", "+ 1 Lap", "11", "3"],
              ["11", "27", "Bryan Herta", "PK Racing", "86", "+ 1 Lap", "12", "2"],
              ["12", "31", "Ryan Hunter-Reay", "American Spirit Team Johansson", "86", "+ 1 Lap", "17", "1"],
              ["13", "19", "Joel Camathias", "Dale Coyne Racing", "85", "+ 2 Laps", "18", "0"],
              ["14", "33", "Alex Tagliani", "Rocketsports Racing", "85", "+ 2 Laps", "14", "0"],
              ["15", "4", "Roberto Moreno", "Herdez Competition", "85", "+ 2 Laps", "9", "0"],
              ["16", "11", "Geoff Boss", "Dale Coyne Racing", "83", "Mechanical", "19", "0"],
              ["17", "2", "Sebastien Bourdais", "Newman/Haas Racing", "77", "Mechanical", "4", "0"],
              ["18", "15", "Darren Manning", "Walker Racing", "12", "Mechanical", "7", "0"],
              ["19", "5", "Rodolfo Lavin", "Walker Racing", "10", "Mechanical", "16", "0"],
          ]
          query = "what were the drivers names?"
          table = pd.DataFrame.from_records(data[1:], columns=data[0])
  
          tokenizer = TapasTokenizer.from_pretrained("google/tapas-base-finetuned-wtq", model_max_length=512)
          model_inputs = tokenizer(table, query, padding="max_length")
  
          input_ids = model_inputs["input_ids"]
          token_type_ids = np.array(model_inputs["token_type_ids"])
          segment_ids = token_type_ids[:, 0]
          column_ids = token_type_ids[:, 1]
          row_ids = token_type_ids[:, 2]
  
          # fmt: off
          expected_results = {'input_ids':[101,2054,2020,1996,6853,3415,1029,102,13433,2015,2053,4062,2136,10876,2051,1013,3394,8370,2685,1015,3590,4754,29267,4765,3771,2136,2447,1005,1055,6584,1015,1024,4466,1024,2340,1012,6185,2509,1015,2570,1016,1015,10391,12022,4226,7895,10625,1013,22996,3868,6584,1009,1014,1012,1022,10819,2015,1016,2459,1017,1017,2703,10555,2136,2447,1005,1055,6584,1009,2654,1012,1020,10819,2015,1017,2403,1018,1023,8709,8183,3126,21351,2078,1010,3781,1012,2136,10958,8865,6584,1009,2871,1012,1022,10819,2015,2410,2260,1019,4090,7986,5292,5677,8151,2771,1011,2990,9187,3868,6584,1009,4413,1012,1015,10819,2015,1020,2184,1020,2322,2030,20282,14262,9035,4754,3868,6584,1009,1015,1024,4002,1012,1016,2184,1022,1021,4868,7918,12023,12023,3868,6584,1009,1015,1024,5890,1012,1018,1019,1020,1022,2260,5261,12436,18116,2137,4382,2136,26447,6584,1009,1015,1024,5890,1012,1022,1022,1019,1023,1021,27339,3995,10125,9711,4906,25101,24657,1011,22033,2386,3868,6564,1009,1015,5001,2321,1018,2184,4583,7986,14383,2075,29488,14906,9351,2971,6564,1009,1015,5001,2340,1017,2340,2676,8527,2014,2696,1052,2243,3868,6564,1009,1015,5001,2260,1016,2260,2861,4575,4477,1011,2128,4710,2137,4382,2136,26447,6564,1009,1015,5001,2459,1015,2410,2539,8963,11503,25457,3022,8512,2522,9654,3868,5594,1009,1016,10876,2324,1014,2403,3943,4074,6415,15204,2072,12496,25378,3868,5594,1009,1016,10876,2403,1014,2321,1018,10704,17921,14906,9351,2971,5594,1009,1016,10876,1023,1014,2385,2340,14915,5795,8512,2522,9654,3868,6640,6228,2539,1014,2459,1016,28328,8945,3126,21351,2015,10625,1013,22996,3868,6255,6228,1018,1014,2324,2321,12270,11956,5232,3868,2260,6228,1021,1014,2539,1019,8473,28027,2080,2474,6371,5232,3868,2184,6228,2385,1014,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],'column_ids':[0,0,0,0,0,0,0,0,1,1,2,3,4,5,6,6,6,7,8,1,2,3,3,3,3,4,4,4,4,5,6,6,6,6,6,6,6,6,7,8,1,2,3,3,3,3,4,4,4,4,5,6,6,6,6,6,6,7,8,1,2,3,3,4,4,4,4,5,6,6,6,6,6,6,7,8,1,2,3,3,3,3,3,3,3,3,4,4,4,5,6,6,6,6,6,6,7,8,1,2,3,3,3,3,4,4,4,4,4,5,6,6,6,6,6,6,7,8,1,2,3,3,3,3,4,4,5,6,6,6,6,6,6,7,8,1,2,3,3,4,4,5,6,6,6,6,6,6,7,8,1,2,3,3,3,4,4,4,4,5,6,6,6,6,6,6,7,8,1,2,3,3,3,3,4,4,4,4,4,4,4,5,6,6,6,7,8,1,2,3,3,3,3,4,4,4,5,6,6,6,7,8,1,2,3,3,3,4,4,4,5,6,6,6,7,8,1,2,3,3,3,3,3,4,4,4,4,5,6,6,6,7,8,1,2,3,3,3,3,4,4,4,4,5,6,6,6,7,8,1,2,3,3,3,3,4,4,4,5,6,6,6,7,8,1,2,3,3,4,4,4,5,6,6,6,7,8,1,2,3,3,4,4,4,4,5,6,7,8,1,2,3,3,3,3,3,4,4,4,4,5,6,7,8,1,2,3,3,4,4,5,6,7,8,1,2,3,3,3,3,3,4,4,5,6,7,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],'row_ids':[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,18,18,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,19,19,19,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],'segment_ids':[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}  # noqa: E231
          # fmt: on
  
          self.assertListEqual(input_ids, expected_results["input_ids"])
          self.assertListEqual(segment_ids.tolist(), expected_results["segment_ids"])
          self.assertListEqual(column_ids.tolist(), expected_results["column_ids"])
          self.assertListEqual(row_ids.tolist(), expected_results["row_ids"])
  
      @unittest.skip("Skip this test while all models are still to be uploaded.")
      def test_pretrained_model_lists(self):
          pass
  
      @unittest.skip("Doesn't support another framework than PyTorch")
      def test_np_encode_plus_sent_to_model(self):
          pass
   # coding=utf-8
  # Copyright 2021 The HuggingFace Inc. team. All rights reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  """ Testing suite for the PyTorch BigBird model. """
  
  
  import unittest
  
  from transformers import BigBirdConfig, is_torch_available
  from transformers.models.auto import get_values
  from transformers.models.big_bird.tokenization_big_bird import BigBirdTokenizer
  from transformers.testing_utils import require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
  
  
  if is_torch_available():
      import torch
  
      from transformers import (
          MODEL_FOR_PRETRAINING_MAPPING,
          BigBirdForCausalLM,
          BigBirdForMaskedLM,
          BigBirdForMultipleChoice,
          BigBirdForPreTraining,
          BigBirdForQuestionAnswering,
          BigBirdForSequenceClassification,
          BigBirdForTokenClassification,
          BigBirdModel,
      )
      from transformers.models.big_bird.modeling_big_bird import BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  class BigBirdModelTester:
      def __init__(
          self,
          parent,
          batch_size=7,
          seq_length=128,
          is_training=True,
          use_input_mask=True,
          use_token_type_ids=True,
          use_labels=True,
          vocab_size=99,
          hidden_size=32,
          num_hidden_layers=2,
          num_attention_heads=4,
          intermediate_size=37,
          hidden_act="gelu_new",
          hidden_dropout_prob=0.1,
          attention_probs_dropout_prob=0.1,
          max_position_embeddings=256,
          type_vocab_size=16,
          type_sequence_label_size=2,
          initializer_range=0.02,
          num_labels=3,
          num_choices=4,
          attention_type="block_sparse",
          use_bias=True,
          rescale_embeddings=False,
          block_size=8,
          num_rand_blocks=3,
          position_embedding_type="absolute",
          scope=None,
      ):
          self.parent = parent
          self.batch_size = batch_size
          self.seq_length = seq_length
          self.is_training = is_training
          self.use_input_mask = use_input_mask
          self.use_token_type_ids = use_token_type_ids
          self.use_labels = use_labels
          self.vocab_size = vocab_size
          self.hidden_size = hidden_size
          self.num_hidden_layers = num_hidden_layers
          self.num_attention_heads = num_attention_heads
          self.intermediate_size = intermediate_size
          self.hidden_act = hidden_act
          self.hidden_dropout_prob = hidden_dropout_prob
          self.attention_probs_dropout_prob = attention_probs_dropout_prob
          self.max_position_embeddings = max_position_embeddings
          self.type_vocab_size = type_vocab_size
          self.type_sequence_label_size = type_sequence_label_size
          self.initializer_range = initializer_range
          self.num_labels = num_labels
          self.num_choices = num_choices
          self.scope = scope
  
          self.attention_type = attention_type
          self.use_bias = use_bias
          self.rescale_embeddings = rescale_embeddings
          self.block_size = block_size
          self.num_rand_blocks = num_rand_blocks
          self.position_embedding_type = position_embedding_type
  
      def prepare_config_and_inputs(self):
          input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
  
          input_mask = None
          if self.use_input_mask:
              input_mask = random_attention_mask([self.batch_size, self.seq_length])
  
          token_type_ids = None
          if self.use_token_type_ids:
              token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size)
  
          sequence_labels = None
          token_labels = None
          choice_labels = None
          if self.use_labels:
              sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
              token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
              choice_labels = ids_tensor([self.batch_size], self.num_choices)
  
          config = self.get_config()
  
          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
  
      def get_config(self):
          return BigBirdConfig(
              vocab_size=self.vocab_size,
              hidden_size=self.hidden_size,
              num_hidden_layers=self.num_hidden_layers,
              num_attention_heads=self.num_attention_heads,
              intermediate_size=self.intermediate_size,
              hidden_act=self.hidden_act,
              hidden_dropout_prob=self.hidden_dropout_prob,
              attention_probs_dropout_prob=self.attention_probs_dropout_prob,
              max_position_embeddings=self.max_position_embeddings,
              type_vocab_size=self.type_vocab_size,
              is_encoder_decoder=False,
              initializer_range=self.initializer_range,
              attention_type=self.attention_type,
              use_bias=self.use_bias,
              rescale_embeddings=self.rescale_embeddings,
              block_size=self.block_size,
              num_random_blocks=self.num_rand_blocks,
              position_embedding_type=self.position_embedding_type,
          )
  
      def prepare_config_and_inputs_for_decoder(self):
          (
              config,
              input_ids,
              token_type_ids,
              input_mask,
              sequence_labels,
              token_labels,
              choice_labels,
          ) = self.prepare_config_and_inputs()
  
          config.is_decoder = True
          encoder_hidden_states = floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
          encoder_attention_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2)
  
          return (
              config,
              input_ids,
              token_type_ids,
              input_mask,
              sequence_labels,
              token_labels,
              choice_labels,
              encoder_hidden_states,
              encoder_attention_mask,
          )
  
      def create_and_check_model(
          self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
      ):
          model = BigBirdModel(config=config)
          model.to(torch_device)
          model.eval()
          result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids)
          result = model(input_ids, token_type_ids=token_type_ids)
          result = model(input_ids)
          self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
  
      def create_and_check_for_pretraining(
          self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
      ):
          model = BigBirdForPreTraining(config=config)
          model.to(torch_device)
          model.eval()
          result = model(
              input_ids,
              attention_mask=input_mask,
              token_type_ids=token_type_ids,
              labels=token_labels,
              next_sentence_label=sequence_labels,
          )
          self.parent.assertEqual(result.prediction_logits.shape, (self.batch_size, self.seq_length, self.vocab_size))
          self.parent.assertEqual(result.seq_relationship_logits.shape, (self.batch_size, config.num_labels))
  
      def create_and_check_model_as_decoder(
          self,
          config,
          input_ids,
          token_type_ids,
          input_mask,
          sequence_labels,
          token_labels,
          choice_labels,
          encoder_hidden_states,
          encoder_attention_mask,
      ):
          config.add_cross_attention = True
          model = BigBirdModel(config)
          model.to(torch_device)
          model.eval()
          result = model(
              input_ids,
              attention_mask=input_mask,
              token_type_ids=token_type_ids,
              encoder_hidden_states=encoder_hidden_states,
              encoder_attention_mask=encoder_attention_mask,
          )
          result = model(
              input_ids,
              attention_mask=input_mask,
              token_type_ids=token_type_ids,
              encoder_hidden_states=encoder_hidden_states,
          )
          result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids)
          self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
  
      def create_and_check_for_causal_lm(
          self,
          config,
          input_ids,
          token_type_ids,
          input_mask,
          sequence_labels,
          token_labels,
          choice_labels,
          encoder_hidden_states,
          encoder_attention_mask,
      ):
          model = BigBirdForCausalLM(config=config)
          model.to(torch_device)
          model.eval()
          result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
          self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))
  
      def create_and_check_for_masked_lm(
          self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
      ):
          model = BigBirdForMaskedLM(config=config)
          model.to(torch_device)
          model.eval()
          result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
          self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))
  
      def create_and_check_decoder_model_past_large_inputs(
          self,
          config,
          input_ids,
          token_type_ids,
          input_mask,
          sequence_labels,
          token_labels,
          choice_labels,
          encoder_hidden_states,
          encoder_attention_mask,
      ):
          config.is_decoder = True
          config.add_cross_attention = True
          model = BigBirdForCausalLM(config=config)
          model.to(torch_device)
          model.eval()
  
          # first forward pass
          outputs = model(
              input_ids,
              attention_mask=input_mask,
              encoder_hidden_states=encoder_hidden_states,
              encoder_attention_mask=encoder_attention_mask,
              use_cache=True,
          )
          past_key_values = outputs.past_key_values
  
          # create hypothetical multiple next token and extent to next_input_ids
          next_tokens = ids_tensor((self.batch_size, 3), config.vocab_size)
          next_mask = ids_tensor((self.batch_size, 3), vocab_size=2)
  
          # append to next input_ids and
          next_input_ids = torch.cat([input_ids, next_tokens], dim=-1)
          next_attention_mask = torch.cat([input_mask, next_mask], dim=-1)
  
          output_from_no_past = model(
              next_input_ids,
              attention_mask=next_attention_mask,
              encoder_hidden_states=encoder_hidden_states,
              encoder_attention_mask=encoder_attention_mask,
              output_hidden_states=True,
          )["hidden_states"][0]
          output_from_past = model(
              next_tokens,
              attention_mask=next_attention_mask,
              encoder_hidden_states=encoder_hidden_states,
              encoder_attention_mask=encoder_attention_mask,
              past_key_values=past_key_values,
              output_hidden_states=True,
          )["hidden_states"][0]
  
          # select random slice
          random_slice_idx = ids_tensor((1,), output_from_past.shape[-1]).item()
          output_from_no_past_slice = output_from_no_past[:, -3:, random_slice_idx].detach()
          output_from_past_slice = output_from_past[:, :, random_slice_idx].detach()
  
          self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])
  
          # test that outputs are equal for slice
          self.parent.assertTrue(torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-3))
  
      def create_and_check_for_question_answering(
          self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
      ):
          model = BigBirdForQuestionAnswering(config=config)
          model.to(torch_device)
          model.eval()
          result = model(
              input_ids,
              attention_mask=input_mask,
              token_type_ids=token_type_ids,
              start_positions=sequence_labels,
              end_positions=sequence_labels,
          )
          self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
          self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))
  
      def create_and_check_for_sequence_classification(
          self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
      ):
          config.num_labels = self.num_labels
          model = BigBirdForSequenceClassification(config)
          model.to(torch_device)
          model.eval()
          result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels)
          self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels))
  
      def create_and_check_for_token_classification(
          self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
      ):
          config.num_labels = self.num_labels
          model = BigBirdForTokenClassification(config=config)
          model.to(torch_device)
          model.eval()
          result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels)
          self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))
  
      def create_and_check_for_multiple_choice(
          self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
      ):
          config.num_choices = self.num_choices
          model = BigBirdForMultipleChoice(config=config)
          model.to(torch_device)
          model.eval()
          multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
          multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
          multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
          result = model(
              multiple_choice_inputs_ids,
              attention_mask=multiple_choice_input_mask,
              token_type_ids=multiple_choice_token_type_ids,
              labels=choice_labels,
          )
          self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices))
  
      def prepare_config_and_inputs_for_common(self):
          config_and_inputs = self.prepare_config_and_inputs()
          (
              config,
              input_ids,
              token_type_ids,
              input_mask,
              sequence_labels,
              token_labels,
              choice_labels,
          ) = config_and_inputs
          inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
          return config, inputs_dict
  
      def create_and_check_for_auto_padding(
          self,
          config,
          input_ids,
          token_type_ids,
          input_mask,
          sequence_labels,
          token_labels,
          choice_labels,
      ):
          model = BigBirdModel(config)
          model.to(torch_device)
          model.eval()
          result = model(input_ids)
          self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
  
      def create_and_check_for_change_to_full_attn(
          self,
          config,
          input_ids,
          token_type_ids,
          input_mask,
          sequence_labels,
          token_labels,
          choice_labels,
      ):
          model = BigBirdModel(config)
          model.to(torch_device)
          model.eval()
          result = model(input_ids)
          self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
          # the config should not be changed
          self.parent.assertTrue(model.config.attention_type == "block_sparse")
  
  
  @require_torch
  class BigBirdModelTest(ModelTesterMixin, unittest.TestCase):
  
      # head masking & pruning is currently not supported for big bird
      test_head_masking = False
      test_pruning = False
  
      # torchscript should be possible, but takes prohibitively long to test.
      # Also torchscript is not an important feature to have in the beginning.
      test_torchscript = False
  
      all_model_classes = (
          (
              BigBirdModel,
              BigBirdForPreTraining,
              BigBirdForMaskedLM,
              BigBirdForCausalLM,
              BigBirdForMultipleChoice,
              BigBirdForQuestionAnswering,
              BigBirdForSequenceClassification,
              BigBirdForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      all_generative_model_classes = (BigBirdForCausalLM,) if is_torch_available() else ()
  
      # special case for ForPreTraining model
      def _prepare_for_class(self, inputs_dict, model_class, return_labels=False):
          inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels)
  
          if return_labels:
              if model_class in get_values(MODEL_FOR_PRETRAINING_MAPPING):
                  inputs_dict["labels"] = torch.zeros(
                      (self.model_tester.batch_size, self.model_tester.seq_length), dtype=torch.long, device=torch_device
                  )
                  inputs_dict["next_sentence_label"] = torch.zeros(
                      self.model_tester.batch_size, dtype=torch.long, device=torch_device
                  )
          return inputs_dict
  
      def setUp(self):
          self.model_tester = BigBirdModelTester(self)
          self.config_tester = ConfigTester(self, config_class=BigBirdConfig, hidden_size=37)
  
      def test_config(self):
          self.config_tester.run_common_tests()
  
      def test_model(self):
          config_and_inputs = self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*config_and_inputs)
  
      def test_for_pretraining(self):
          config_and_inputs = self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_pretraining(*config_and_inputs)
  
      def test_for_masked_lm(self):
          config_and_inputs = self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*config_and_inputs)
  
      def test_for_multiple_choice(self):
          config_and_inputs = self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*config_and_inputs)
  
      def test_decoder_model_past_with_large_inputs(self):
          config_and_inputs = self.model_tester.prepare_config_and_inputs_for_decoder()
          self.model_tester.create_and_check_decoder_model_past_large_inputs(*config_and_inputs)
  
      def test_for_question_answering(self):
          config_and_inputs = self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*config_and_inputs)
  
      def test_for_sequence_classification(self):
          config_and_inputs = self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*config_and_inputs)
  
      def test_for_token_classification(self):
          config_and_inputs = self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*config_and_inputs)
  
      def test_model_as_decoder(self):
          config_and_inputs = self.model_tester.prepare_config_and_inputs_for_decoder()
          self.model_tester.create_and_check_model_as_decoder(*config_and_inputs)
  
      def test_model_as_decoder_with_default_input_mask(self):
          # This regression test was failing with PyTorch < 1.3
          (
              config,
              input_ids,
              token_type_ids,
              input_mask,
              sequence_labels,
              token_labels,
              choice_labels,
              encoder_hidden_states,
              encoder_attention_mask,
          ) = self.model_tester.prepare_config_and_inputs_for_decoder()
  
          input_mask = None
  
          self.model_tester.create_and_check_model_as_decoder(
              config,
              input_ids,
              token_type_ids,
              input_mask,
              sequence_labels,
              token_labels,
              choice_labels,
              encoder_hidden_states,
              encoder_attention_mask,
          )
  
      def test_retain_grad_hidden_states_attentions(self):
          # bigbird cannot keep gradients in attentions when `attention_type=block_sparse`
  
          if self.model_tester.attention_type == "original_full":
              super().test_retain_grad_hidden_states_attentions()
  
      @slow
      def test_model_from_pretrained(self):
          for model_name in BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              model = BigBirdForPreTraining.from_pretrained(model_name)
              self.assertIsNotNone(model)
  
      def test_model_various_attn_type(self):
          config_and_inputs = self.model_tester.prepare_config_and_inputs()
          for type in ["original_full", "block_sparse"]:
              config_and_inputs[0].attention_type = type
              self.model_tester.create_and_check_model(*config_and_inputs)
  
      def test_fast_integration(self):
          # fmt: off
          input_ids = torch.tensor(
              [[6, 117, 33, 36, 70, 22, 63, 31, 71, 72, 88, 58, 109, 49, 48, 116, 92, 6, 19, 95, 118, 100, 80, 111, 93, 2, 31, 84, 26, 5, 6, 82, 46, 96, 109, 4, 39, 19, 109, 13, 92, 31, 36, 90, 111, 18, 75, 6, 56, 74, 16, 42, 56, 92, 69, 108, 127, 81, 82, 41, 106, 19, 44, 24, 82, 121, 120, 65, 36, 26, 72, 13, 36, 98, 43, 64, 8, 53, 100, 92, 51, 122, 66, 17, 61, 50, 104, 127, 26, 35, 94, 23, 110, 71, 80, 67, 109, 111, 44, 19, 51, 41, 86, 71, 76, 44, 18, 68, 44, 77, 107, 81, 98, 126, 100, 2, 49, 98, 84, 39, 23, 98, 52, 46, 10, 82, 121, 73],[6, 117, 33, 36, 70, 22, 63, 31, 71, 72, 88, 58, 109, 49, 48, 116, 92, 6, 19, 95, 118, 100, 80, 111, 93, 2, 31, 84, 26, 5, 6, 82, 46, 96, 109, 4, 39, 19, 109, 13, 92, 31, 36, 90, 111, 18, 75, 6, 56, 74, 16, 42, 56, 92, 69, 108, 127, 81, 82, 41, 106, 19, 44, 24, 82, 121, 120, 65, 36, 26, 72, 13, 36, 98, 43, 64, 8, 53, 100, 92, 51, 12, 66, 17, 61, 50, 104, 127, 26, 35, 94, 23, 110, 71, 80, 67, 109, 111, 44, 19, 51, 41, 86, 71, 76, 28, 18, 68, 44, 77, 107, 81, 98, 126, 100, 2, 49, 18, 84, 39, 23, 98, 52, 46, 10, 82, 121, 73]],  # noqa: E231
              dtype=torch.long,
              device=torch_device,
          )
          # fmt: on
          input_ids = input_ids % self.model_tester.vocab_size
          input_ids[1] = input_ids[1] - 1
  
          attention_mask = torch.ones((input_ids.shape), device=torch_device)
          attention_mask[:, :-10] = 0
  
          config, _, _, _, _, _, _ = self.model_tester.prepare_config_and_inputs()
          torch.manual_seed(0)
          model = BigBirdModel(config).eval().to(torch_device)
  
          with torch.no_grad():
              hidden_states = model(input_ids, attention_mask=attention_mask).last_hidden_state
              self.assertTrue(
                  torch.allclose(
                      hidden_states[0, 0, :5],
                      torch.tensor([1.4825, 0.0774, 0.8226, -0.2962, -0.9593], device=torch_device),
                      atol=1e-3,
                  )
              )
  
      def test_auto_padding(self):
          self.model_tester.seq_length = 241
          config_and_inputs = self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_auto_padding(*config_and_inputs)
  
      def test_for_change_to_full_attn(self):
          self.model_tester.seq_length = 9
          config_and_inputs = self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_change_to_full_attn(*config_and_inputs)
  
      # overwrite from common in order to skip the check on `attentions`
      # also use `5e-5` to avoid flaky test failure
      def check_pt_flax_outputs(self, fx_outputs, pt_outputs, model_class, tol=5e-5, name="outputs", attributes=None):
          # `bigbird_block_sparse_attention` in `FlaxBigBird` returns `attention_probs = None`, while in PyTorch version,
          # an effort was done to return `attention_probs` (yet to be verified).
          if name.startswith("outputs.attentions"):
              return
          else:
              super().check_pt_flax_outputs(fx_outputs, pt_outputs, model_class, tol, name, attributes)
  
  
  @require_torch
  @slow
  class BigBirdModelIntegrationTest(unittest.TestCase):
      # we can have this true once block_sparse attn_probs works accurately
      test_attention_probs = False
  
      def _get_dummy_input_ids(self):
          # fmt: off
          ids = torch.tensor(
              [[6, 117, 33, 36, 70, 22, 63, 31, 71, 72, 88, 58, 109, 49, 48, 116, 92, 6, 19, 95, 118, 100, 80, 111, 93, 2, 31, 84, 26, 5, 6, 82, 46, 96, 109, 4, 39, 19, 109, 13, 92, 31, 36, 90, 111, 18, 75, 6, 56, 74, 16, 42, 56, 92, 69, 108, 127, 81, 82, 41, 106, 19, 44, 24, 82, 121, 120, 65, 36, 26, 72, 13, 36, 98, 43, 64, 8, 53, 100, 92, 51, 122, 66, 17, 61, 50, 104, 127, 26, 35, 94, 23, 110, 71, 80, 67, 109, 111, 44, 19, 51, 41, 86, 71, 76, 44, 18, 68, 44, 77, 107, 81, 98, 126, 100, 2, 49, 98, 84, 39, 23, 98, 52, 46, 10, 82, 121, 73]],  # noqa: E231
              dtype=torch.long,
              device=torch_device,
          )
          # fmt: on
          return ids
  
      def test_inference_block_sparse_pretraining(self):
          model = BigBirdForPreTraining.from_pretrained("google/bigbird-roberta-base", attention_type="block_sparse")
          model.to(torch_device)
  
          input_ids = torch.tensor([[20920, 232, 328, 1437] * 1024], dtype=torch.long, device=torch_device)
          outputs = model(input_ids)
          prediction_logits = outputs.prediction_logits
          seq_relationship_logits = outputs.seq_relationship_logits
  
          self.assertEqual(prediction_logits.shape, torch.Size((1, 4096, 50358)))
          self.assertEqual(seq_relationship_logits.shape, torch.Size((1, 2)))
  
          expected_prediction_logits_slice = torch.tensor(
              [
                  [-0.2420, -0.6048, -0.0614, 7.8422],
                  [-0.0596, -0.0104, -1.8408, 9.3352],
                  [1.0588, 0.7999, 5.0770, 8.7555],
                  [-0.1385, -1.7199, -1.7613, 6.1094],
              ],
              device=torch_device,
          )
          self.assertTrue(
              torch.allclose(prediction_logits[0, 128:132, 128:132], expected_prediction_logits_slice, atol=1e-4)
          )
  
          expected_seq_relationship_logits = torch.tensor([[58.8196, 56.3629]], device=torch_device)
          self.assertTrue(torch.allclose(seq_relationship_logits, expected_seq_relationship_logits, atol=1e-4))
  
      def test_inference_full_pretraining(self):
          model = BigBirdForPreTraining.from_pretrained("google/bigbird-roberta-base", attention_type="original_full")
          model.to(torch_device)
  
          input_ids = torch.tensor([[20920, 232, 328, 1437] * 512], dtype=torch.long, device=torch_device)
          outputs = model(input_ids)
          prediction_logits = outputs.prediction_logits
          seq_relationship_logits = outputs.seq_relationship_logits
  
          self.assertEqual(prediction_logits.shape, torch.Size((1, 512 * 4, 50358)))
          self.assertEqual(seq_relationship_logits.shape, torch.Size((1, 2)))
  
          expected_prediction_logits_slice = torch.tensor(
              [
                  [0.1499, -1.1217, 0.1990, 8.4499],
                  [-2.7757, -3.0687, -4.8577, 7.5156],
                  [1.5446, 0.1982, 4.3016, 10.4281],
                  [-1.3705, -4.0130, -3.9629, 5.1526],
              ],
              device=torch_device,
          )
          self.assertTrue(
              torch.allclose(prediction_logits[0, 128:132, 128:132], expected_prediction_logits_slice, atol=1e-4)
          )
  
          expected_seq_relationship_logits = torch.tensor([[41.4503, 41.2406]], device=torch_device)
          self.assertTrue(torch.allclose(seq_relationship_logits, expected_seq_relationship_logits, atol=1e-4))
  
      def test_block_sparse_attention_probs(self):
          """
          Asserting if outputted attention matrix is similar to hard coded attention matrix
          """
  
          if not self.test_attention_probs:
              return
  
          model = BigBirdModel.from_pretrained(
              "google/bigbird-roberta-base", attention_type="block_sparse", num_random_blocks=3, block_size=16
          )
          model.to(torch_device)
          model.eval()
          config = model.config
  
          input_ids = self._get_dummy_input_ids()
  
          hidden_states = model.embeddings(input_ids)
  
          batch_size, seqlen, _ = hidden_states.size()
          attn_mask = torch.ones(batch_size, seqlen, device=torch_device, dtype=torch.float)
          to_seq_length = from_seq_length = seqlen
          from_block_size = to_block_size = config.block_size
  
          blocked_mask, band_mask, from_mask, to_mask = model.create_masks_for_block_sparse_attn(
              attn_mask, config.block_size
          )
          from_blocked_mask = to_blocked_mask = blocked_mask
  
          for i in range(config.num_hidden_layers):
              pointer = model.encoder.layer[i].attention.self
  
              query_layer = pointer.transpose_for_scores(pointer.query(hidden_states))
              key_layer = pointer.transpose_for_scores(pointer.key(hidden_states))
              value_layer = pointer.transpose_for_scores(pointer.value(hidden_states))
  
              context_layer, attention_probs = pointer.bigbird_block_sparse_attention(
                  query_layer,
                  key_layer,
                  value_layer,
                  band_mask,
                  from_mask,
                  to_mask,
                  from_blocked_mask,
                  to_blocked_mask,
                  pointer.num_attention_heads,
                  pointer.num_random_blocks,
                  pointer.attention_head_size,
                  from_block_size,
                  to_block_size,
                  batch_size,
                  from_seq_length,
                  to_seq_length,
                  seed=pointer.seed,
                  plan_from_length=None,
                  plan_num_rand_blocks=None,
                  output_attentions=True,
              )
  
              context_layer = context_layer.contiguous().view(batch_size, from_seq_length, -1)
              cl = torch.einsum("bhqk,bhkd->bhqd", attention_probs, value_layer)
              cl = cl.view(context_layer.size())
  
              self.assertTrue(torch.allclose(context_layer, cl, atol=0.001))
  
      def test_block_sparse_context_layer(self):
          model = BigBirdModel.from_pretrained(
              "google/bigbird-roberta-base", attention_type="block_sparse", num_random_blocks=3, block_size=16
          )
          model.to(torch_device)
          model.eval()
          config = model.config
  
          input_ids = self._get_dummy_input_ids()
          dummy_hidden_states = model.embeddings(input_ids)
  
          attn_mask = torch.ones_like(input_ids, device=torch_device)
          blocked_mask, band_mask, from_mask, to_mask = model.create_masks_for_block_sparse_attn(
              attn_mask, config.block_size
          )
          targeted_cl = torch.tensor(
              [
                  [0.1874, 1.5260, 0.2335, -0.0473, -0.0961, 1.8384, -0.0141, 0.1250, 0.0085, -0.0048],
                  [-0.0554, 0.0728, 0.1683, -0.1332, 0.1741, 0.1337, -0.2380, -0.1849, -0.0390, -0.0259],
                  [-0.0419, 0.0767, 0.1591, -0.1399, 0.1789, 0.1257, -0.2406, -0.1772, -0.0261, -0.0079],
                  [0.1860, 1.5172, 0.2326, -0.0473, -0.0953, 1.8291, -0.0147, 0.1245, 0.0082, -0.0046],
                  [0.1879, 1.5296, 0.2335, -0.0471, -0.0975, 1.8433, -0.0136, 0.1260, 0.0086, -0.0054],
                  [0.1854, 1.5147, 0.2334, -0.0480, -0.0956, 1.8250, -0.0149, 0.1222, 0.0082, -0.0060],
                  [0.1859, 1.5184, 0.2334, -0.0474, -0.0955, 1.8297, -0.0143, 0.1234, 0.0079, -0.0054],
                  [0.1885, 1.5336, 0.2335, -0.0467, -0.0979, 1.8481, -0.0130, 0.1269, 0.0085, -0.0049],
                  [0.1881, 1.5305, 0.2335, -0.0471, -0.0976, 1.8445, -0.0135, 0.1262, 0.0086, -0.0053],
                  [0.1852, 1.5148, 0.2333, -0.0480, -0.0949, 1.8254, -0.0151, 0.1225, 0.0079, -0.0055],
                  [0.1877, 1.5292, 0.2335, -0.0470, -0.0972, 1.8431, -0.0135, 0.1259, 0.0084, -0.0052],
                  [0.1874, 1.5261, 0.2334, -0.0472, -0.0968, 1.8393, -0.0140, 0.1251, 0.0084, -0.0052],
                  [0.1853, 1.5151, 0.2331, -0.0478, -0.0948, 1.8256, -0.0154, 0.1228, 0.0086, -0.0052],
                  [0.1867, 1.5233, 0.2334, -0.0475, -0.0965, 1.8361, -0.0139, 0.1247, 0.0084, -0.0054],
              ],
              device=torch_device,
          )
  
          context_layer = model.encoder.layer[0].attention.self(
              dummy_hidden_states,
              band_mask=band_mask,
              from_mask=from_mask,
              to_mask=to_mask,
              from_blocked_mask=blocked_mask,
              to_blocked_mask=blocked_mask,
          )
          context_layer = context_layer[0]
  
          self.assertEqual(context_layer.shape, torch.Size((1, 128, 768)))
          self.assertTrue(torch.allclose(context_layer[0, 64:78, 300:310], targeted_cl, atol=0.0001))
  
      def test_tokenizer_inference(self):
          tokenizer = BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base")
          model = BigBirdModel.from_pretrained(
              "google/bigbird-roberta-base", attention_type="block_sparse", num_random_blocks=3, block_size=16
          )
          model.to(torch_device)
  
          text = [
              "Transformer-based models are unable to process long sequences due to their self-attention operation,"
              " which scales quadratically with the sequence length. To address this limitation, we introduce the"
              " Longformer with an attention mechanism that scales linearly with sequence length, making it easy to"
              " process documents of thousands of tokens or longer. Longformer’s attention mechanism is a drop-in"
              " replacement for the standard self-attention and combines a local windowed attention with a task"
              " motivated global attention. Following prior work on long-sequence transformers, we evaluate Longformer"
              " on character-level language modeling and achieve state-of-the-art results on text8 and enwik8. In"
              " contrast to most prior work, we also pretrain Longformer and finetune it on a variety of downstream"
              " tasks. Our pretrained Longformer consistently outperforms RoBERTa on long document tasks and sets new"
              " state-of-the-art results on WikiHop and TriviaQA."
          ]
          inputs = tokenizer(text)
  
          for k in inputs:
              inputs[k] = torch.tensor(inputs[k], device=torch_device, dtype=torch.long)
  
          prediction = model(**inputs)
          prediction = prediction[0]
  
          self.assertEqual(prediction.shape, torch.Size((1, 199, 768)))
  
          expected_prediction = torch.tensor(
              [
                  [-0.0213, -0.2213, -0.0061, 0.0687],
                  [0.0977, 0.1858, 0.2374, 0.0483],
                  [0.2112, -0.2524, 0.5793, 0.0967],
                  [0.2473, -0.5070, -0.0630, 0.2174],
                  [0.2885, 0.1139, 0.6071, 0.2991],
                  [0.2328, -0.2373, 0.3648, 0.1058],
                  [0.2517, -0.0689, 0.0555, 0.0880],
                  [0.1021, -0.1495, -0.0635, 0.1891],
                  [0.0591, -0.0722, 0.2243, 0.2432],
                  [-0.2059, -0.2679, 0.3225, 0.6183],
                  [0.2280, -0.2618, 0.1693, 0.0103],
                  [0.0183, -0.1375, 0.2284, -0.1707],
              ],
              device=torch_device,
          )
          self.assertTrue(torch.allclose(prediction[0, 52:64, 320:324], expected_prediction, atol=1e-4))
  
      def test_inference_question_answering(self):
          tokenizer = BigBirdTokenizer.from_pretrained("google/bigbird-base-trivia-itc")
          model = BigBirdForQuestionAnswering.from_pretrained(
              "google/bigbird-base-trivia-itc", attention_type="block_sparse", block_size=16, num_random_blocks=3
          )
          model.to(torch_device)
  
          context = (
              "The BigBird model was proposed in Big Bird: Transformers for Longer Sequences by Zaheer, Manzil and"
              " Guruganesh, Guru and Dubey, Kumar Avinava and Ainslie, Joshua and Alberti, Chris and Ontanon, Santiago"
              " and Pham, Philip and Ravula, Anirudh and Wang, Qifan and Yang, Li and others. BigBird, is a"
              " sparse-attention based transformer which extends Transformer based models, such as BERT to much longer"
              " sequences. In addition to sparse attention, BigBird also applies global attention as well as random"
              " attention to the input sequence. Theoretically, it has been shown that applying sparse, global, and"
              " random attention approximates full attention, while being computationally much more efficient for longer"
              " sequences. As a consequence of the capability to handle longer context, BigBird has shown improved"
              " performance on various long document NLP tasks, such as question answering and summarization, compared"
              " to BERT or RoBERTa."
          )
  
          question = [
              "Which is better for longer sequences- BigBird or BERT?",
              "What is the benefit of using BigBird over BERT?",
          ]
          inputs = tokenizer(
              question,
              [context, context],
              padding=True,
              return_tensors="pt",
              add_special_tokens=True,
              max_length=256,
              truncation=True,
          )
  
          inputs = {k: v.to(torch_device) for k, v in inputs.items()}
  
          start_logits, end_logits = model(**inputs).to_tuple()
  
          # fmt: off
          target_start_logits = torch.tensor(
              [[-8.9304, -10.3849, -14.4997, -9.6497, -13.9469, -7.8134, -8.9687, -13.3585, -9.7987, -13.8869, -9.2632, -8.9294, -13.6721, -7.3198, -9.5434, -11.2641, -14.3245, -9.5705, -12.7367, -8.6168, -11.083, -13.7573, -8.1151, -14.5329, -7.6876, -15.706, -12.8558, -9.1135, 8.0909, -3.1925, -11.5812, -9.4822], [-11.5595, -14.5591, -10.2978, -14.8445, -10.2092, -11.1899, -13.8356, -10.5644, -14.7706, -9.9841, -11.0052, -14.1862, -8.8173, -11.1098, -12.4686, -15.0531, -11.0196, -13.6614, -10.0236, -11.8151, -14.8744, -9.5123, -15.1605, -8.6472, -15.4184, -8.898, -9.6328, -7.0258, -11.3365, -14.4065, -10.2587, -8.9103]],  # noqa: E231
              device=torch_device,
          )
          target_end_logits = torch.tensor(
              [[-12.4131, -8.5959, -15.7163, -11.1524, -15.9913, -12.2038, -7.8902, -16.0296, -12.164, -16.5017, -13.3332, -6.9488, -15.7756, -13.8506, -11.0779, -9.2893, -15.0426, -10.1963, -17.3292, -12.2945, -11.5337, -16.4514, -9.1564, -17.5001, -9.1562, -16.2971, -13.3199, -7.5724, -5.1175, 7.2168, -10.3804, -11.9873], [-10.8654, -14.9967, -11.4144, -16.9189, -14.2673, -9.7068, -15.0182, -12.8846, -16.8716, -13.665, -10.3113, -15.1436, -14.9069, -13.3364, -11.2339, -16.0118, -11.8331, -17.0613, -13.8852, -12.4163, -16.8978, -10.7772, -17.2324, -10.6979, -16.9811, -10.3427, -9.497, -13.7104, -11.1107, -13.2936, -13.855, -14.1264]],  # noqa: E231
              device=torch_device,
          )
          # fmt: on
  
          self.assertTrue(torch.allclose(start_logits[:, 64:96], target_start_logits, atol=1e-4))
          self.assertTrue(torch.allclose(end_logits[:, 64:96], target_end_logits, atol=1e-4))
  
          input_ids = inputs["input_ids"].tolist()
          answer = [
              input_ids[i][torch.argmax(start_logits, dim=-1)[i] : torch.argmax(end_logits, dim=-1)[i] + 1]
              for i in range(len(input_ids))
          ]
          answer = tokenizer.batch_decode(answer)
  
          self.assertTrue(answer == ["BigBird", "global attention"])
  
      def test_fill_mask(self):
          tokenizer = BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base")
          model = BigBirdForMaskedLM.from_pretrained("google/bigbird-roberta-base")
          model.to(torch_device)
  
          input_ids = tokenizer("The goal of life is [MASK] .", return_tensors="pt").input_ids.to(torch_device)
          logits = model(input_ids).logits
  
          # [MASK] is token at 6th position
          pred_token = tokenizer.decode(torch.argmax(logits[0, 6:7], axis=-1))
          self.assertEqual(pred_token, "happiness")
  
      def test_auto_padding(self):
          model = BigBirdModel.from_pretrained(
              "google/bigbird-roberta-base", attention_type="block_sparse", num_random_blocks=3, block_size=16
          )
          model.to(torch_device)
          model.eval()
  
          input_ids = torch.tensor([200 * [10] + 40 * [2] + [1]], device=torch_device, dtype=torch.long)
          output = model(input_ids).to_tuple()[0]
  
          # fmt: off
          target = torch.tensor(
              [[-0.045136, -0.068013, 0.12246, -0.01356, 0.018386, 0.025333, -0.0044439, -0.0030996, -0.064031, 0.0006439], [-0.045018, -0.067638, 0.12317, -0.013998, 0.019216, 0.025695, -0.0043705, -0.0031895, -0.063153, 0.00088899], [-0.045042, -0.067305, 0.1234, -0.014512, 0.020057, 0.026084, -0.004615, -0.0031728, -0.062442, 0.0010263], [-0.044589, -0.067655, 0.12416, -0.014287, 0.019416, 0.026065, -0.0050958, -0.002702, -0.063158, 0.0004827], [-0.044627, -0.067535, 0.1239, -0.014319, 0.019491, 0.026213, -0.0059482, -0.0025906, -0.063116, 0.00014669], [-0.044899, -0.067704, 0.12337, -0.014231, 0.019256, 0.026345, -0.0065565, -0.0022938, -0.063433, -0.00011409], [-0.045599, -0.067764, 0.12235, -0.014151, 0.019206, 0.026417, -0.0068965, -0.0024494, -0.063313, -4.4499e-06], [-0.045557, -0.068372, 0.12199, -0.013747, 0.017962, 0.026103, -0.0070607, -0.0023552, -0.06447, -0.00048756], [-0.045334, -0.068913, 0.1217, -0.013566, 0.01693, 0.025745, -0.006311, -0.0024903, -0.065575, -0.0006719], [-0.045171, -0.068726, 0.12164, -0.013688, 0.017139, 0.025629, -0.005213, -0.0029412, -0.065237, -0.00020669], [-0.044411, -0.069267, 0.12206, -0.013645, 0.016212, 0.025589, -0.0044121, -0.002972, -0.066277, -0.00067963], [-0.043487, -0.069792, 0.1232, -0.013663, 0.015303, 0.02613, -0.0036294, -0.0030616, -0.067483, -0.0012642], [-0.042622, -0.069287, 0.12469, -0.013936, 0.016204, 0.026474, -0.0040534, -0.0027365, -0.066994, -0.0014148], [-0.041879, -0.070031, 0.12593, -0.014047, 0.015082, 0.027751, -0.0040683, -0.0027189, -0.068985, -0.0027146]],  # noqa: E231
              device=torch_device,
          )
          # fmt: on
  
          self.assertEqual(output.shape, torch.Size((1, 241, 768)))
          self.assertTrue(torch.allclose(output[0, 64:78, 300:310], target, atol=0.0001))
  
apis: numpy.array pandas.DataFrame.from_records torch.no_grad pandas.DataFrame.from_dict
apis: torch.ones_like torch.ones torch.Size torch.argmax torch.manual_seed torch.no_grad torch.tensor torch.zeros torch.einsum torch.cat torch.allclose