

"""simple docstring"""
import copy
from typing import Any, Dict, List, Optional, Union

import numpy as np
import torch

from ...audio_utils import mel_filter_bank, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import TensorType, logging


_A							=  logging.get_logger(__name__)

class       lowerCamelCase	(       lowerCAmelCase__   ):


  '''simple docstring'''







  SCREAMING_SNAKE_CASE					      =  ['input_features', 'is_longer']



  def __init__(self							,       _lowerCamelCase=64							,       _lowerCamelCase=48000							,       _lowerCamelCase=480							,       _lowerCamelCase=10							,       _lowerCamelCase=1024							,       _lowerCamelCase=0.0							,       _lowerCamelCase=False							,       _lowerCamelCase = 0							,       _lowerCamelCase = 14000							,       _lowerCamelCase = None							,       _lowerCamelCase = "fusion"							,       _lowerCamelCase = "repeatpad"							,       **_lowerCamelCase							,       ):


        """simple docstring"""

        super().__init__(
            feature_size=_lowerCamelCase							,       sampling_rate=_lowerCamelCase							,       padding_value=_lowerCamelCase							,       return_attention_mask=_lowerCamelCase							,       **_lowerCamelCase							,       )
        UpperCAmelCase__			:							Union[str, Any]							=    top_db
        UpperCAmelCase__			:							Dict							=    truncation
        UpperCAmelCase__			:							Optional[int]							=    padding
        UpperCAmelCase__			:							str							=    fft_window_size
        UpperCAmelCase__			:							Optional[int]							=    (fft_window_size >> 1) + 1
        UpperCAmelCase__			:							Any							=    hop_length
        UpperCAmelCase__			:							Any							=    max_length_s
        UpperCAmelCase__			:							List[str]							=    max_length_s * sampling_rate
        UpperCAmelCase__			:							List[str]							=    sampling_rate
        UpperCAmelCase__			:							Optional[int]							=    frequency_min
        UpperCAmelCase__			:							int							=    frequency_max
        UpperCAmelCase__			:							int							=    mel_filter_bank(
            num_frequency_bins=self.nb_frequency_bins							,       num_mel_filters=_lowerCamelCase							,       min_frequency=_lowerCamelCase							,       max_frequency=_lowerCamelCase							,       sampling_rate=_lowerCamelCase							,       norm=_lowerCamelCase							,       mel_scale="""htk"""							,       )
        UpperCAmelCase__			:							Dict							=    mel_filter_bank(
            num_frequency_bins=self.nb_frequency_bins							,       num_mel_filters=_lowerCamelCase							,       min_frequency=_lowerCamelCase							,       max_frequency=_lowerCamelCase							,       sampling_rate=_lowerCamelCase							,       norm="""slaney"""							,       mel_scale="""slaney"""							,       )



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							int							=    copy.deepcopy(self.__dict__	)
        UpperCAmelCase__			:							Dict							=    self.__class__.__name__
        if "mel_filters" in output:
              del output["mel_filters"]
        if "mel_filters_slaney" in output:
              del output["mel_filters_slaney"]
        return output



  def  _a      (self							,       _lowerCamelCase							,       _lowerCamelCase = None	):


        """simple docstring"""

        UpperCAmelCase__			:							Optional[Any]							=    spectrogram(
            _lowerCamelCase							,       window_function(self.fft_window_size							,       """hann"""	)							,       frame_length=self.fft_window_size							,       hop_length=self.hop_length							,       power=2.0							,       mel_filters=_lowerCamelCase							,       log_mel="""dB"""							,       )
        return log_mel_spectrogram.T



  def  _a      (self							,       _lowerCamelCase							,       _lowerCamelCase							,       _lowerCamelCase	):


        """simple docstring"""

        UpperCAmelCase__			:							Optional[Any]							=    np.array_split(list(range(0							,       total_frames - chunk_frames + 1	)	)							,       3	)
        if len(ranges[1]	) == 0:
              # if the audio is too short, we just use the first chunk
              UpperCAmelCase__			:							List[Any]							=    [0]
        if len(ranges[2]	) == 0:
              # if the audio is too short, we just use the first chunk
              UpperCAmelCase__			:							Tuple							=    [0]
        # randomly choose index for each part
        UpperCAmelCase__			:							Optional[int]							=    np.random.choice(ranges[0]	)
        UpperCAmelCase__			:							Any							=    np.random.choice(ranges[1]	)
        UpperCAmelCase__			:							List[Any]							=    np.random.choice(ranges[2]	)

        UpperCAmelCase__			:							Tuple							=    mel[idx_front : idx_front + chunk_frames, :]
        UpperCAmelCase__			:							List[str]							=    mel[idx_middle : idx_middle + chunk_frames, :]
        UpperCAmelCase__			:							Tuple							=    mel[idx_back : idx_back + chunk_frames, :]

        UpperCAmelCase__			:							str							=    torch.tensor(mel[None, None, :]	)
        UpperCAmelCase__			:							Optional[Any]							=    torch.nn.functional.interpolate(
            _lowerCamelCase							,       size=[chunk_frames, 64]							,       mode="""bilinear"""							,       align_corners=_lowerCamelCase	)
        UpperCAmelCase__			:							List[str]							=    mel_shrink[0][0].numpy()
        UpperCAmelCase__			:							str							=    np.stack([mel_shrink, mel_chunk_front, mel_chunk_middle, mel_chunk_back]							,       axis=0	)
        return mel_fusion



  def  _a      (self							,       _lowerCamelCase							,       _lowerCamelCase							,       _lowerCamelCase							,       _lowerCamelCase	):


        """simple docstring"""

        if waveform.shape[0] > max_length:
              if truncation == "rand_trunc":
                    UpperCAmelCase__			:							List[Any]							=    True
                    # random crop to max_length (for compatibility) -> this should be handled by self.pad
                    UpperCAmelCase__			:							List[Any]							=    len(_lowerCamelCase	) - max_length
                    UpperCAmelCase__			:							Union[str, Any]							=    np.random.randint(0							,       overflow + 1	)
                    UpperCAmelCase__			:							Dict							=    waveform[idx : idx + max_length]
                    UpperCAmelCase__			:							Optional[Any]							=    self._np_extract_fbank_features(_lowerCamelCase							,       self.mel_filters_slaney	)[None, :]
              elif truncation == "fusion":
                    UpperCAmelCase__			:							Dict							=    self._np_extract_fbank_features(_lowerCamelCase							,       self.mel_filters	)
                    UpperCAmelCase__			:							Optional[Any]							=    max_length // self.hop_length + 1  # the +1 related to how the spectrogram is computed
                    UpperCAmelCase__			:							List[str]							=    mel.shape[0]
                    if chunk_frames == total_frames:
                          # there is a corner case where the audio length is larger than max_length but smaller than max_length+hop_length.
                          # In this case, we just use the whole audio.
                          UpperCAmelCase__			:							Union[str, Any]							=    np.stack([mel, mel, mel, mel]							,       axis=0	)
                          UpperCAmelCase__			:							List[str]							=    False
                    else:
                          UpperCAmelCase__			:							Any							=    self._random_mel_fusion(_lowerCamelCase							,       _lowerCamelCase							,       _lowerCamelCase	)
                          UpperCAmelCase__			:							Union[str, Any]							=    True
              else:
                    raise NotImplementedError(F"""data_truncating {truncation} not implemented"""	)

        else:
              UpperCAmelCase__			:							str							=    False
              # only use repeat as a new possible value for padding. you repeat the audio before applying the usual max_length padding
              if waveform.shape[0] < max_length:
                    if padding == "repeat":
                          UpperCAmelCase__			:							Optional[Any]							=    int(max_length / len(_lowerCamelCase	)	)
                          UpperCAmelCase__			:							str							=    np.stack(np.tile(_lowerCamelCase							,       n_repeat + 1	)	)[:max_length]
                    if padding == "repeatpad":
                          UpperCAmelCase__			:							str							=    int(max_length / len(_lowerCamelCase	)	)
                          UpperCAmelCase__			:							Optional[int]							=    np.stack(np.tile(_lowerCamelCase							,       _lowerCamelCase	)	)
                    UpperCAmelCase__			:							Optional[Any]							=    np.pad(_lowerCamelCase							,       (0, max_length - waveform.shape[0])							,       mode="""constant"""							,       constant_values=0	)

              if truncation == "fusion":
                    UpperCAmelCase__			:							int							=    self._np_extract_fbank_features(_lowerCamelCase							,       self.mel_filters	)
                    UpperCAmelCase__			:							List[Any]							=    np.stack([input_mel, input_mel, input_mel, input_mel]							,       axis=0	)
              else:
                    UpperCAmelCase__			:							Dict							=    self._np_extract_fbank_features(_lowerCamelCase							,       self.mel_filters_slaney	)[None, :]

        return input_mel, longer




  def __call__(self							,       _lowerCamelCase							,       _lowerCamelCase = None							,       _lowerCamelCase = None							,       _lowerCamelCase = None							,       _lowerCamelCase = None							,       _lowerCamelCase = None							,       **_lowerCamelCase							,       ):


        """simple docstring"""

        UpperCAmelCase__			:							List[str]							=    truncation if truncation is not None else self.truncation
        UpperCAmelCase__			:							List[str]							=    padding if padding else self.padding

        if sampling_rate is not None:
              if sampling_rate != self.sampling_rate:
                    raise ValueError(
                        F"""The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"""
                        F""" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"""
                        F""" was sampled with {self.sampling_rate} and not {sampling_rate}."""	)
        else:
              logger.warning(
                  """It is strongly recommended to pass the `sampling_rate` argument to this function. """
                  """Failing to do so can result in silent errors that might be hard to debug."""	)

        UpperCAmelCase__			:							List[str]							=    isinstance(_lowerCamelCase							,       np.ndarray	) and len(raw_speech.shape	) > 1
        if is_batched_numpy and len(raw_speech.shape	) > 2:
              raise ValueError(F"""Only mono-channel audio is supported for input to {self}"""	)
        UpperCAmelCase__			:							List[str]							=    is_batched_numpy or (
            isinstance(_lowerCamelCase							,       (list, tuple)	) and (isinstance(raw_speech[0]							,       (np.ndarray, tuple, list)	))
        )

        if is_batched:
              UpperCAmelCase__			:							Dict							=    [np.asarray(_lowerCamelCase							,       dtype=np.floataa	) for speech in raw_speech]
        elif not is_batched and not isinstance(_lowerCamelCase							,       np.ndarray	):
              UpperCAmelCase__			:							Any							=    np.asarray(_lowerCamelCase							,       dtype=np.floataa	)
        elif isinstance(_lowerCamelCase							,       np.ndarray	) and raw_speech.dtype is np.dtype(np.floataa	):
              UpperCAmelCase__			:							Union[str, Any]							=    raw_speech.astype(np.floataa	)

        # always return batch
        if not is_batched:
              UpperCAmelCase__			:							str							=    [np.asarray(_lowerCamelCase	)]

        # convert to mel spectrogram, truncate and pad if needed.
        UpperCAmelCase__			:							List[Any]							=    [
            self._get_input_mel(_lowerCamelCase							,       max_length if max_length else self.nb_max_samples							,       _lowerCamelCase							,       _lowerCamelCase	)
            for waveform in raw_speech
        ]

        UpperCAmelCase__			:							List[str]							=    []
        UpperCAmelCase__			:							int							=    []
        for mel, longer in padded_inputs:
              input_mel.append(_lowerCamelCase	)
              is_longer.append(_lowerCamelCase	)

        if truncation == "fusion" and sum(_lowerCamelCase	) == 0:
              # if no audio is longer than 10s, then randomly select one audio to be longer
              UpperCAmelCase__			:							Dict							=    np.random.randint(0							,       len(_lowerCamelCase	)	)
              UpperCAmelCase__			:							Union[str, Any]							=    True

        if isinstance(input_mel[0]							,       _lowerCamelCase	):
              UpperCAmelCase__			:							str							=    [np.asarray(_lowerCamelCase							,       dtype=np.floataa	) for feature in input_mel]

        # is_longer is a list of bool
        UpperCAmelCase__			:							int							=    [[longer] for longer in is_longer]

        UpperCAmelCase__			:							Dict							=    {"""input_features""": input_mel, """is_longer""": is_longer}
        UpperCAmelCase__			:							Any							=    BatchFeature(_lowerCamelCase	)

        if return_tensors is not None:
              UpperCAmelCase__			:							Optional[int]							=    input_features.convert_to_tensors(_lowerCamelCase	)

        return input_features




"""simple docstring"""
import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_A							=  """▁"""

_A							=  get_tests_dir("""fixtures/test_sentencepiece.model""")

@require_sentencepiece
@require_tokenizers
class       lowerCamelCase	(       lowerCAmelCase__					,					unittest.TestCase   ):


  '''simple docstring'''







  SCREAMING_SNAKE_CASE					      =  BigBirdTokenizer
  SCREAMING_SNAKE_CASE					      =  BigBirdTokenizerFast
  SCREAMING_SNAKE_CASE					      =  True
  SCREAMING_SNAKE_CASE					      =  True



  def  _a      (self	):


        """simple docstring"""

        super().setUp()

        UpperCAmelCase__			:							Any							=    self.tokenizer_class(_lowerCamelCase							,       keep_accents=_lowerCamelCase	)
        tokenizer.save_pretrained(self.tmpdirname	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Union[str, Any]							=    """<s>"""
        UpperCAmelCase__			:							Optional[int]							=    1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCamelCase	)							,       _lowerCamelCase	)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCamelCase	)							,       _lowerCamelCase	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							List[Any]							=    list(self.get_tokenizer().get_vocab().keys()	)

        self.assertEqual(vocab_keys[0]							,       """<unk>"""	)
        self.assertEqual(vocab_keys[1]							,       """<s>"""	)
        self.assertEqual(vocab_keys[-1]							,       """[MASK]"""	)
        self.assertEqual(len(_lowerCamelCase	)							,       1004	)



  def  _a      (self	):


        """simple docstring"""

        self.assertEqual(self.get_tokenizer().vocab_size							,       1000	)



  def  _a      (self	):


        """simple docstring"""

        if not self.test_rust_tokenizer:
              return

        UpperCAmelCase__			:							List[Any]							=    self.get_tokenizer()
        UpperCAmelCase__			:							Optional[Any]							=    self.get_rust_tokenizer()

        UpperCAmelCase__			:							Dict							=    """I was born in 92000, and this is falsé."""

        UpperCAmelCase__			:							Dict							=    tokenizer.tokenize(_lowerCamelCase	)
        UpperCAmelCase__			:							int							=    rust_tokenizer.tokenize(_lowerCamelCase	)
        self.assertListEqual(_lowerCamelCase							,       _lowerCamelCase	)

        UpperCAmelCase__			:							int							=    tokenizer.encode(_lowerCamelCase							,       add_special_tokens=_lowerCamelCase	)
        UpperCAmelCase__			:							str							=    rust_tokenizer.encode(_lowerCamelCase							,       add_special_tokens=_lowerCamelCase	)
        self.assertListEqual(_lowerCamelCase							,       _lowerCamelCase	)

        UpperCAmelCase__			:							Any							=    self.get_rust_tokenizer()
        UpperCAmelCase__			:							List[Any]							=    tokenizer.encode(_lowerCamelCase	)
        UpperCAmelCase__			:							Union[str, Any]							=    rust_tokenizer.encode(_lowerCamelCase	)
        self.assertListEqual(_lowerCamelCase							,       _lowerCamelCase	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							str							=    BigBirdTokenizer(_lowerCamelCase							,       keep_accents=_lowerCamelCase	)

        UpperCAmelCase__			:							Union[str, Any]							=    tokenizer.tokenize("""This is a test"""	)
        self.assertListEqual(_lowerCamelCase							,       ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]	)

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(_lowerCamelCase	)							,       [285, 46, 10, 170, 382]							,       )

        UpperCAmelCase__			:							Dict							=    tokenizer.tokenize("""I was born in 92000, and this is falsé."""	)
        self.assertListEqual(
            _lowerCamelCase							,       [
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """9""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """é""",
                """.""",
            ]							,       )
        UpperCAmelCase__			:							Tuple							=    tokenizer.convert_tokens_to_ids(_lowerCamelCase	)
        self.assertListEqual(
            _lowerCamelCase							,       [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]							,       )

        UpperCAmelCase__			:							Tuple							=    tokenizer.convert_ids_to_tokens(_lowerCamelCase	)
        self.assertListEqual(
            _lowerCamelCase							,       [
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """<unk>""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """<unk>""",
                """.""",
            ]							,       )



  @cached_property
  def  _a      (self	):


        """simple docstring"""

        return BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base"""	)



  @slow
  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Tuple							=    """Hello World!"""
        UpperCAmelCase__			:							Union[str, Any]							=    [65, 18536, 2260, 101, 66]

        self.assertListEqual(_lowerCamelCase							,       self.big_tokenizer.encode(_lowerCamelCase	)	)



  @slow
  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Union[str, Any]							=    (
            """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
            """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
        )
        # fmt: off
        UpperCAmelCase__			:							Dict							=    [65, 871, 419, 358, 946, 991, 2521, 452, 358, 1357, 387, 7751, 3536, 112, 985, 456, 126, 865, 938, 5400, 5734, 458, 1368, 467, 786, 2462, 5246, 1159, 633, 865, 4519, 457, 582, 852, 2557, 427, 916, 508, 405, 34324, 497, 391, 408, 11342, 1244, 385, 100, 938, 985, 456, 574, 362, 12597, 3200, 3129, 1172, 66]  # noqa: E231
        # fmt: on
        self.assertListEqual(_lowerCamelCase							,       self.big_tokenizer.encode(_lowerCamelCase	)	)



  @require_torch
  @slow
  def  _a      (self	):


        """simple docstring"""

        import torch

        from transformers import BigBirdConfig, BigBirdModel

        # Build sequence
        UpperCAmelCase__			:							int							=    list(self.big_tokenizer.get_vocab().keys()	)[:10]
        UpperCAmelCase__			:							str							=    """ """.join(_lowerCamelCase	)
        UpperCAmelCase__			:							int							=    self.big_tokenizer.encode_plus(_lowerCamelCase							,       return_tensors="""pt"""							,       return_token_type_ids=_lowerCamelCase	)
        UpperCAmelCase__			:							str							=    self.big_tokenizer.batch_encode_plus(
            [sequence + """ """ + sequence]							,       return_tensors="""pt"""							,       return_token_type_ids=_lowerCamelCase	)

        UpperCAmelCase__			:							Optional[int]							=    BigBirdConfig(attention_type="""original_full"""	)
        UpperCAmelCase__			:							Optional[int]							=    BigBirdModel(_lowerCamelCase	)

        assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

        with torch.no_grad():
              model(**_lowerCamelCase	)
              model(**_lowerCamelCase	)



  @slow
  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Union[str, Any]							=    BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base"""	)
        UpperCAmelCase__			:							int							=    tokenizer.decode(tokenizer("""Paris is the [MASK]."""	).input_ids	)

        self.assertTrue(decoded_text == """[CLS] Paris is the[MASK].[SEP]"""	)




  @slow
  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							List[str]							=    {"""input_ids""": [[65, 39286, 458, 36335, 2001, 456, 13073, 13266, 455, 113, 7746, 1741, 11157, 391, 13073, 13266, 455, 113, 3967, 35412, 113, 4936, 109, 3870, 2377, 113, 30084, 45720, 458, 134, 17496, 112, 503, 11672, 113, 118, 112, 5665, 13347, 38687, 112, 1496, 31389, 112, 3268, 47264, 134, 962, 112, 16377, 8035, 23130, 430, 12169, 15518, 28592, 458, 146, 41697, 109, 391, 12169, 15518, 16689, 458, 146, 41358, 109, 452, 726, 4034, 111, 763, 35412, 5082, 388, 1903, 111, 9051, 391, 2870, 48918, 1900, 1123, 550, 998, 112, 9586, 15985, 455, 391, 410, 22955, 37636, 114, 66], [65, 448, 17496, 419, 3663, 385, 763, 113, 27533, 2870, 3283, 13043, 1639, 24713, 523, 656, 24013, 18550, 2521, 517, 27014, 21244, 420, 1212, 1465, 391, 927, 4833, 388, 578, 11786, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 484, 2169, 7687, 21932, 18146, 726, 363, 17032, 3391, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=_lowerCamelCase							,       model_name="""google/bigbird-roberta-base"""							,       revision="""215c99f1600e06f83acce68422f2035b2b5c3510"""							,       )

