




# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig




def 		lowerCAmelCase_ (						lowerCamelCase						,  lowerCamelCase						,  lowerCamelCase=0 ):
				# Format the message.
				if name is None:
								__magic_name__						:						List[str]										=None
				else:
								__magic_name__						:						Optional[Any]										=""".""" * max(0						,  spaces - 2 ) + """# {:""" + str(50 - spaces ) + """s}"""
								__magic_name__						:						Optional[int]										=fmt.format(lowerCamelCase )

				# Print and recurse (if needed).
				if isinstance(lowerCamelCase						,  lowerCamelCase ):
								if msg is not None:
												print(lowerCamelCase )
								for k in val.keys():
												recursive_print(lowerCamelCase						,  val[k]						,  spaces + 2 )
				elif isinstance(lowerCamelCase						,  torch.Tensor ):
								print(lowerCamelCase						,  """:"""						,  val.size() )
				else:
								print(lowerCamelCase						,  """:"""						,  lowerCamelCase )




def 		lowerCAmelCase_ (						lowerCamelCase						,  lowerCamelCase						,  lowerCamelCase						,  lowerCamelCase						,  lowerCamelCase ):
				# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
				# for compatibility with later versions of NVIDIA Megatron-LM.
				# The inverse operation is performed inside Megatron-LM to read checkpoints:
				# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
				# If param is the weight tensor of the self-attention block, the returned tensor
				# will have to be transposed one more time to be read by HuggingFace GPT2.
				__magic_name__						:						Union[str, Any]										=param.size()
				if checkpoint_version == 1.0:
								# version 1.0 stores [num_heads * hidden_size * num_splits, :]
								__magic_name__						:						int										=(num_heads, hidden_size, num_splits) + input_shape[1:]
								__magic_name__						:						List[str]										=param.view(*lowerCamelCase )
								__magic_name__						:						List[Any]										=param.transpose(0						,  2 )
								__magic_name__						:						int										=param.transpose(1						,  2 ).contiguous()
				elif checkpoint_version >= 2.0:
								# other versions store [num_heads * num_splits * hidden_size, :]
								__magic_name__						:						Any										=(num_heads, num_splits, hidden_size) + input_shape[1:]
								__magic_name__						:						int										=param.view(*lowerCamelCase )
								__magic_name__						:						str										=param.transpose(0						,  1 ).contiguous()
				__magic_name__						:						List[Any]										=param.view(*lowerCamelCase )
				return param




def 		lowerCAmelCase_ (						lowerCamelCase						,  lowerCamelCase						,  lowerCamelCase ):
				# The converted output model.
				__magic_name__						:						Optional[int]										={}

				# old versions did not store training args
				__magic_name__						:						Any										=input_state_dict.get("""args"""						,  lowerCamelCase )
				if ds_args is not None:
								# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
								# from pprint import pprint
								# pprint(vars(ds_args))

								__magic_name__						:						List[str]										=ds_args.padded_vocab_size
								__magic_name__						:						List[str]										=ds_args.max_position_embeddings
								__magic_name__						:						List[Any]										=ds_args.hidden_size
								__magic_name__						:						Union[str, Any]										=ds_args.num_layers
								__magic_name__						:						Tuple										=ds_args.num_attention_heads
								__magic_name__						:						List[str]										=ds_args.ffn_hidden_size
								# pprint(config)

				# The number of heads.
				__magic_name__						:						Dict										=config.n_head
				# The hidden_size per head.
				__magic_name__						:						int										=config.n_embd // config.n_head
				# Megatron-LM checkpoint version
				if "checkpoint_version" in input_state_dict.keys():
								__magic_name__						:						Union[str, Any]										=input_state_dict["""checkpoint_version"""]
				else:
								__magic_name__						:						Any										=0.0

				# The model.
				__magic_name__						:						Dict										=input_state_dict["""model"""]
				# The language model.
				__magic_name__						:						Tuple										=model["""language_model"""]
				# The embeddings.
				__magic_name__						:						List[str]										=lm["""embedding"""]

				# The word embeddings.
				__magic_name__						:						str										=embeddings["""word_embeddings"""]["""weight"""]
				# Truncate the embedding table to vocab_size rows.
				__magic_name__						:						Dict										=word_embeddings[: config.vocab_size, :]
				__magic_name__						:						Optional[int]										=word_embeddings

				# The position embeddings.
				__magic_name__						:						str										=embeddings["""position_embeddings"""]["""weight"""]
				# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
				__magic_name__						:						Union[str, Any]										=pos_embeddings.size(0 )
				if n_positions != config.n_positions:
								raise ValueError(
								    F"pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match" )
				# Store the position embeddings.
				__magic_name__						:						Optional[int]										=pos_embeddings

				# The transformer.
				__magic_name__						:						Dict										=lm["""transformer"""] if """transformer""" in lm.keys() else lm["""encoder"""]

				# The regex to extract layer names.
				__magic_name__						:						List[str]										=re.compile(R"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)""" )

				# The simple map of names for "automated" rules.
				__magic_name__						:						List[Any]										={
				    """attention.dense""": """.attn.c_proj.""",
				    """self_attention.dense""": """.attn.c_proj.""",
				    """mlp.dense_h_to_4h""": """.mlp.c_fc.""",
				    """mlp.dense_4h_to_h""": """.mlp.c_proj.""",
				}

				# Extract the layers.
				for key, val in transformer.items():
								# Match the name.
								__magic_name__						:						List[Any]										=layer_re.match(lowerCamelCase )

								# Stop if that's not a layer
								if m is None:
												break

								# The index of the layer.
								__magic_name__						:						str										=int(m.group(1 ) )
								# The name of the operation.
								__magic_name__						:						Dict										=m.group(2 )
								# Is it a weight or a bias?
								__magic_name__						:						Dict										=m.group(3 )

								# The name of the layer.
								__magic_name__						:						List[str]										=F"transformer.h.{layer_idx}"

								# For layernorm(s), simply store the layer norm.
								if op_name.endswith("""layernorm""" ):
												__magic_name__						:						List[Any]										="""ln_1""" if op_name.startswith("""input""" ) else """ln_2"""
												__magic_name__						:						Optional[int]										=val

								# Transpose the QKV matrix.
								elif (
								    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
								) and weight_or_bias == "weight":
												# Insert a tensor of 1x1xDxD bias.
												__magic_name__						:						Optional[Any]										=torch.tril(torch.ones((n_positions, n_positions)						,  dtype=torch.floataa ) ).view(
												    1						,  1						,  lowerCamelCase						,  lowerCamelCase )
												__magic_name__						:						Dict										=causal_mask

												# Insert a "dummy" tensor for masked_bias.
												__magic_name__						:						Any										=torch.tensor(-1E4						,  dtype=torch.floataa )
												__magic_name__						:						Dict										=masked_bias

												__magic_name__						:						List[str]										=fix_query_key_value_ordering(lowerCamelCase						,  lowerCamelCase						,  3						,  lowerCamelCase						,  lowerCamelCase )
												# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
												__magic_name__						:						Optional[Any]										=out_val.transpose(0						,  1 ).contiguous()
												# Store.
												__magic_name__						:						Any										=out_val

								# Transpose the bias.
								elif (
								    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
								) and weight_or_bias == "bias":
												__magic_name__						:						Dict										=fix_query_key_value_ordering(lowerCamelCase						,  lowerCamelCase						,  3						,  lowerCamelCase						,  lowerCamelCase )
												# Store. No change of shape.
												__magic_name__						:						Union[str, Any]										=out_val

								# Transpose the weights.
								elif weight_or_bias == "weight":
												__magic_name__						:						Tuple										=megatron_to_transformers[op_name]
												__magic_name__						:						List[str]										=val.transpose(0						,  1 )

								# Copy the bias.
								elif weight_or_bias == "bias":
												__magic_name__						:						Optional[int]										=megatron_to_transformers[op_name]
												__magic_name__						:						Dict										=val

    # DEBUG.
				assert config.n_layer == layer_idx + 1

				# The final layernorm.
				__magic_name__						:						Any										=transformer["""final_layernorm.weight"""]
				__magic_name__						:						Tuple										=transformer["""final_layernorm.bias"""]

				# For LM head, transformers' wants the matrix to weight embeddings.
				__magic_name__						:						int										=word_embeddings

				# It should be done!
				return output_state_dict




def 		lowerCAmelCase_ (						):
				# Create the argument parser.
				__magic_name__						:						Union[str, Any]										=argparse.ArgumentParser()
				parser.add_argument("""--print-checkpoint-structure"""						,  action="""store_true""" )
				parser.add_argument(
				    """path_to_checkpoint"""						,  type=lowerCamelCase						,  help="""Path to the checkpoint file (.zip archive or direct .pt file)"""						,  )
				parser.add_argument(
				    """--config_file"""						,  default=""""""						,  type=lowerCamelCase						,  help="""An optional config json file describing the pre-trained model."""						,  )
				__magic_name__						:						Any										=parser.parse_args()

				# Extract the basename.
				__magic_name__						:						Tuple										=os.path.dirname(args.path_to_checkpoint )

				# Load the model.
				# the .zip is very optional, let's keep it for backward compatibility
				print(F"Extracting PyTorch state dictionary from {args.path_to_checkpoint}" )
				if args.path_to_checkpoint.endswith(""".zip""" ):
								with zipfile.ZipFile(args.path_to_checkpoint						,  """r""" ) as checkpoint:
												with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt""" ) as pytorch_dict:
																__magic_name__						:						Optional[Any]										=torch.load(lowerCamelCase						,  map_location="""cpu""" )
				else:
								__magic_name__						:						Dict										=torch.load(args.path_to_checkpoint						,  map_location="""cpu""" )

				__magic_name__						:						Optional[Any]										=input_state_dict.get("""args"""						,  lowerCamelCase )

				# Read the config, or default to the model released by NVIDIA.
				if args.config_file == "":
								if ds_args is not None:
												if ds_args.bias_gelu_fusion:
																__magic_name__						:						Optional[Any]										="""gelu_fast"""
												elif ds_args.openai_gelu:
																__magic_name__						:						Optional[Any]										="""gelu_new"""
												else:
																__magic_name__						:						List[Any]										="""gelu"""
								else:
												# in the very early days this used to be "gelu_new"
												__magic_name__						:						Dict										="""gelu_new"""

								# Spell out all parameters in case the defaults change.
								__magic_name__						:						Any										=GPTaConfig(
								    vocab_size=50257						,  n_positions=1024						,  n_embd=1024						,  n_layer=24						,  n_head=16						,  n_inner=4096						,  activation_function=lowerCamelCase						,  resid_pdrop=0.1						,  embd_pdrop=0.1						,  attn_pdrop=0.1						,  layer_norm_epsilon=1E-5						,  initializer_range=0.0_2						,  summary_type="""cls_index"""						,  summary_use_proj=lowerCamelCase						,  summary_activation=lowerCamelCase						,  summary_proj_to_labels=lowerCamelCase						,  summary_first_dropout=0.1						,  scale_attn_weights=lowerCamelCase						,  use_cache=lowerCamelCase						,  bos_token_id=50256						,  eos_token_id=50256						,  )
				else:
								__magic_name__						:						Optional[int]										=GPTaConfig.from_json_file(args.config_file )

				__magic_name__						:						Tuple										=["""GPT2LMHeadModel"""]

				# Convert.
				print("""Converting""" )
				__magic_name__						:						int										=convert_megatron_checkpoint(lowerCamelCase						,  lowerCamelCase						,  lowerCamelCase )

				# Print the structure of converted state dict.
				if args.print_checkpoint_structure:
								recursive_print(lowerCamelCase						,  lowerCamelCase )

				# Add tokenizer class info to config
				# see https://github.com/huggingface/transformers/issues/13906)
				if ds_args is not None:
								__magic_name__						:						Union[str, Any]										=ds_args.tokenizer_type
								if tokenizer_type == "GPT2BPETokenizer":
												__magic_name__						:						List[Any]										="""gpt2"""
								elif tokenizer_type == "PretrainedFromHF":
												__magic_name__						:						Optional[Any]										=ds_args.tokenizer_name_or_path
								else:
												raise ValueError(F"Unrecognized tokenizer_type {tokenizer_type}" )
				else:
								__magic_name__						:						List[Any]										="""gpt2"""

				__magic_name__						:						Dict										=AutoTokenizer.from_pretrained(lowerCamelCase )
				__magic_name__						:						Tuple										=type(lowerCamelCase ).__name__
				__magic_name__						:						List[Any]										=tokenizer_class

				# Store the config to file.
				print("""Saving config""" )
				config.save_pretrained(lowerCamelCase )

				# Save tokenizer based on args
				print(F"Adding {tokenizer_class} tokenizer files" )
				tokenizer.save_pretrained(lowerCamelCase )

				# Store the state_dict to file.
				__magic_name__						:						Optional[int]										=os.path.join(lowerCamelCase						,  """pytorch_model.bin""" )
				print(F"Saving checkpoint to \"{output_checkpoint_file}\"" )
				torch.save(lowerCamelCase						,  lowerCamelCase )


####################################################################################################

if __name__ == "__main__":
							main()

####################################################################################################







"""simple docstring"""
import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_A							=  """▁"""

_A							=  get_tests_dir("""fixtures/test_sentencepiece.model""")

@require_sentencepiece
@require_tokenizers
class       lowerCamelCase	(       lowerCAmelCase__					,					unittest.TestCase   ):


  '''simple docstring'''







  SCREAMING_SNAKE_CASE					      =  BigBirdTokenizer
  SCREAMING_SNAKE_CASE					      =  BigBirdTokenizerFast
  SCREAMING_SNAKE_CASE					      =  True
  SCREAMING_SNAKE_CASE					      =  True



  def  _a      (self	):


        """simple docstring"""

        super().setUp()

        UpperCAmelCase__			:							Any							=    self.tokenizer_class(_lowerCamelCase							,       keep_accents=_lowerCamelCase	)
        tokenizer.save_pretrained(self.tmpdirname	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Union[str, Any]							=    """<s>"""
        UpperCAmelCase__			:							Optional[int]							=    1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCamelCase	)							,       _lowerCamelCase	)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCamelCase	)							,       _lowerCamelCase	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							List[Any]							=    list(self.get_tokenizer().get_vocab().keys()	)

        self.assertEqual(vocab_keys[0]							,       """<unk>"""	)
        self.assertEqual(vocab_keys[1]							,       """<s>"""	)
        self.assertEqual(vocab_keys[-1]							,       """[MASK]"""	)
        self.assertEqual(len(_lowerCamelCase	)							,       1004	)



  def  _a      (self	):


        """simple docstring"""

        self.assertEqual(self.get_tokenizer().vocab_size							,       1000	)



  def  _a      (self	):


        """simple docstring"""

        if not self.test_rust_tokenizer:
              return

        UpperCAmelCase__			:							List[Any]							=    self.get_tokenizer()
        UpperCAmelCase__			:							Optional[Any]							=    self.get_rust_tokenizer()

        UpperCAmelCase__			:							Dict							=    """I was born in 92000, and this is falsé."""

        UpperCAmelCase__			:							Dict							=    tokenizer.tokenize(_lowerCamelCase	)
        UpperCAmelCase__			:							int							=    rust_tokenizer.tokenize(_lowerCamelCase	)
        self.assertListEqual(_lowerCamelCase							,       _lowerCamelCase	)

        UpperCAmelCase__			:							int							=    tokenizer.encode(_lowerCamelCase							,       add_special_tokens=_lowerCamelCase	)
        UpperCAmelCase__			:							str							=    rust_tokenizer.encode(_lowerCamelCase							,       add_special_tokens=_lowerCamelCase	)
        self.assertListEqual(_lowerCamelCase							,       _lowerCamelCase	)

        UpperCAmelCase__			:							Any							=    self.get_rust_tokenizer()
        UpperCAmelCase__			:							List[Any]							=    tokenizer.encode(_lowerCamelCase	)
        UpperCAmelCase__			:							Union[str, Any]							=    rust_tokenizer.encode(_lowerCamelCase	)
        self.assertListEqual(_lowerCamelCase							,       _lowerCamelCase	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							str							=    BigBirdTokenizer(_lowerCamelCase							,       keep_accents=_lowerCamelCase	)

        UpperCAmelCase__			:							Union[str, Any]							=    tokenizer.tokenize("""This is a test"""	)
        self.assertListEqual(_lowerCamelCase							,       ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]	)

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(_lowerCamelCase	)							,       [285, 46, 10, 170, 382]							,       )

        UpperCAmelCase__			:							Dict							=    tokenizer.tokenize("""I was born in 92000, and this is falsé."""	)
        self.assertListEqual(
            _lowerCamelCase							,       [
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """9""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """é""",
                """.""",
            ]							,       )
        UpperCAmelCase__			:							Tuple							=    tokenizer.convert_tokens_to_ids(_lowerCamelCase	)
        self.assertListEqual(
            _lowerCamelCase							,       [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]							,       )

        UpperCAmelCase__			:							Tuple							=    tokenizer.convert_ids_to_tokens(_lowerCamelCase	)
        self.assertListEqual(
            _lowerCamelCase							,       [
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """<unk>""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """<unk>""",
                """.""",
            ]							,       )



  @cached_property
  def  _a      (self	):


        """simple docstring"""

        return BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base"""	)



  @slow
  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Tuple							=    """Hello World!"""
        UpperCAmelCase__			:							Union[str, Any]							=    [65, 18536, 2260, 101, 66]

        self.assertListEqual(_lowerCamelCase							,       self.big_tokenizer.encode(_lowerCamelCase	)	)



  @slow
  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Union[str, Any]							=    (
            """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
            """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
        )
        # fmt: off
        UpperCAmelCase__			:							Dict							=    [65, 871, 419, 358, 946, 991, 2521, 452, 358, 1357, 387, 7751, 3536, 112, 985, 456, 126, 865, 938, 5400, 5734, 458, 1368, 467, 786, 2462, 5246, 1159, 633, 865, 4519, 457, 582, 852, 2557, 427, 916, 508, 405, 34324, 497, 391, 408, 11342, 1244, 385, 100, 938, 985, 456, 574, 362, 12597, 3200, 3129, 1172, 66]  # noqa: E231
        # fmt: on
        self.assertListEqual(_lowerCamelCase							,       self.big_tokenizer.encode(_lowerCamelCase	)	)



  @require_torch
  @slow
  def  _a      (self	):


        """simple docstring"""

        import torch

        from transformers import BigBirdConfig, BigBirdModel

        # Build sequence
        UpperCAmelCase__			:							int							=    list(self.big_tokenizer.get_vocab().keys()	)[:10]
        UpperCAmelCase__			:							str							=    """ """.join(_lowerCamelCase	)
        UpperCAmelCase__			:							int							=    self.big_tokenizer.encode_plus(_lowerCamelCase							,       return_tensors="""pt"""							,       return_token_type_ids=_lowerCamelCase	)
        UpperCAmelCase__			:							str							=    self.big_tokenizer.batch_encode_plus(
            [sequence + """ """ + sequence]							,       return_tensors="""pt"""							,       return_token_type_ids=_lowerCamelCase	)

        UpperCAmelCase__			:							Optional[int]							=    BigBirdConfig(attention_type="""original_full"""	)
        UpperCAmelCase__			:							Optional[int]							=    BigBirdModel(_lowerCamelCase	)

        assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

        with torch.no_grad():
              model(**_lowerCamelCase	)
              model(**_lowerCamelCase	)



  @slow
  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Union[str, Any]							=    BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base"""	)
        UpperCAmelCase__			:							int							=    tokenizer.decode(tokenizer("""Paris is the [MASK]."""	).input_ids	)

        self.assertTrue(decoded_text == """[CLS] Paris is the[MASK].[SEP]"""	)




  @slow
  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							List[str]							=    {"""input_ids""": [[65, 39286, 458, 36335, 2001, 456, 13073, 13266, 455, 113, 7746, 1741, 11157, 391, 13073, 13266, 455, 113, 3967, 35412, 113, 4936, 109, 3870, 2377, 113, 30084, 45720, 458, 134, 17496, 112, 503, 11672, 113, 118, 112, 5665, 13347, 38687, 112, 1496, 31389, 112, 3268, 47264, 134, 962, 112, 16377, 8035, 23130, 430, 12169, 15518, 28592, 458, 146, 41697, 109, 391, 12169, 15518, 16689, 458, 146, 41358, 109, 452, 726, 4034, 111, 763, 35412, 5082, 388, 1903, 111, 9051, 391, 2870, 48918, 1900, 1123, 550, 998, 112, 9586, 15985, 455, 391, 410, 22955, 37636, 114, 66], [65, 448, 17496, 419, 3663, 385, 763, 113, 27533, 2870, 3283, 13043, 1639, 24713, 523, 656, 24013, 18550, 2521, 517, 27014, 21244, 420, 1212, 1465, 391, 927, 4833, 388, 578, 11786, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 484, 2169, 7687, 21932, 18146, 726, 363, 17032, 3391, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=_lowerCamelCase							,       model_name="""google/bigbird-roberta-base"""							,       revision="""215c99f1600e06f83acce68422f2035b2b5c3510"""							,       )

