
import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

from seqaseq_trainer import SeqaSeqTrainer
from seqaseq_training_args import SeqaSeqTrainingArguments

import transformers
from transformers import (
    AutoConfig,
    AutoModelForSeqaSeqLM,
    AutoTokenizer,
    HfArgumentParser,
    MBartTokenizer,
    MBartTokenizerFast,
    set_seed,
)
from transformers.trainer_utils import EvaluationStrategy, is_main_process
from transformers.training_args import ParallelMode
from utils import (
    SeqaSeqDataCollator,
    SeqaSeqDataset,
    assert_all_frozen,
    build_compute_metrics_fn,
    check_output_dir,
    freeze_embeds,
    freeze_params,
    lmap,
    save_json,
    use_task_specific_params,
    write_txt_file,
)


_lowercase											=    logging.getLogger(__name__)


@dataclass
class 			lowercase_		:
       __lowerCamelCase				       =						field(
           metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}       )
       __lowerCamelCase				       =						field(
           default=A  ,      metadata={"help": "Pretrained config name or path if not the same as model_name"}       )
       __lowerCamelCase				       =						field(
           default=A  ,      metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}       )
       __lowerCamelCase				       =						field(
           default=A  ,      metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}  ,      )
       __lowerCamelCase				       =						field(default=A  ,      metadata={"help": "Whether tp freeze the encoder."}       )
       __lowerCamelCase				       =						field(default=A  ,      metadata={"help": "Whether  to freeze the embeddings."}       )






@dataclass
class 			lowercase_		:
       __lowerCamelCase				       =						field(
           metadata={"help": "The input data dir. Should contain the .tsv files (or other data files) for the task."}       )
       __lowerCamelCase				       =						field(
           default="summarization"  ,      metadata={"help": "Task name, summarization (or summarization_{dataset} for pegasus) or translation"}  ,      )
       __lowerCamelCase				       =						field(
           default=1_0_2_4  ,      metadata={
               "help": (
                   "The maximum total input sequence length after tokenization. Sequences longer "
                   "than this will be truncated, sequences shorter will be padded."
               )
           }  ,      )
       __lowerCamelCase				       =						field(
           default=1_2_8  ,      metadata={
               "help": (
                   "The maximum total sequence length for target text after tokenization. Sequences longer "
                   "than this will be truncated, sequences shorter will be padded."
               )
           }  ,      )
       __lowerCamelCase				       =						field(
           default=1_4_2  ,      metadata={
               "help": (
                   "The maximum total sequence length for validation target text after tokenization. Sequences longer "
                   "than this will be truncated, sequences shorter will be padded. "
                   "This argument is also used to override the ``max_length`` param of ``model.generate``, which is used "
                   "during ``evaluate`` and ``predict``."
               )
           }  ,      )
       __lowerCamelCase				       =						field(
           default=1_4_2  ,      metadata={
               "help": (
                   "The maximum total sequence length for test target text after tokenization. Sequences longer "
                   "than this will be truncated, sequences shorter will be padded."
               )
           }  ,      )
       __lowerCamelCase				       =						field(default=-1  ,      metadata={"help": "# training examples. -1 means use all."}       )
       __lowerCamelCase				       =						field(default=-1  ,      metadata={"help": "# validation examples. -1 means use all."}       )
       __lowerCamelCase				       =						field(default=-1  ,      metadata={"help": "# test examples. -1 means use all."}       )
       __lowerCamelCase				       =						field(default=A  ,      metadata={"help": "Source language id for translation."}       )
       __lowerCamelCase				       =						field(default=A  ,      metadata={"help": "Target language id for translation."}       )
       __lowerCamelCase				       =						field(default=A  ,      metadata={"help": "# num_beams to use for evaluation."}       )
       __lowerCamelCase				       =						field(
           default=A  ,      metadata={"help": "If only pad tokens should be ignored. This assumes that `config.pad_token_id` is defined."}  ,      )






def 							SCREAMING_SNAKE_CASE_	(					UpperCAmelCase_	:     Optional[Any]			,							UpperCAmelCase_	:     int			,							UpperCAmelCase_	:     Tuple				)			->						str:
       logger.info(f'***** {split} metrics *****'				)
       for key in sorted(metrics.keys()				):
              logger.info(f'  {key} = {metrics[key]}'				)
       save_json(UpperCAmelCase_			,							os.path.join(UpperCAmelCase_			,							f'{split}_results.json'				)				)
def 							SCREAMING_SNAKE_CASE_	(					)			->						Optional[int]:
       # See all possible arguments in src/transformers/training_args.py
       # or by passing the --help flag to this script.
       # We now keep distinct sets of args, for a cleaner separation of concerns.

       SCREAMING_SNAKE_CASE_							:				List[str]		=HfArgumentParser((ModelArguments, DataTrainingArguments, SeqaSeqTrainingArguments)				)

       if len(sys.argv				) == 2 and sys.argv[1].endswith('''.json'''				):
              # If we pass only one argument to the script and it's the path to a json file,
              # let's parse it to get our arguments.
              SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_							:				Dict		=parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]				)				)
       else:
              SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_	,     SCREAMING_SNAKE_CASE_							:				Tuple		=parser.parse_args_into_dataclasses()

       check_output_dir(UpperCAmelCase_				)

       # Setup logging
       logging.basicConfig(
           format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''			,							datefmt='''%m/%d/%Y %H:%M:%S'''			,							level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN			,							)
       logger.warning(
           '''Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s'''			,							training_args.local_rank			,							training_args.device			,							training_args.n_gpu			,							bool(training_args.parallel_mode == ParallelMode.DISTRIBUTED				)			,							training_args.fpaa			,							)
       transformers.utils.logging.enable_default_handler()
       transformers.utils.logging.enable_explicit_format()
       # Set the verbosity to info of the Transformers logger (on main process only):
       if is_main_process(training_args.local_rank				):
              transformers.utils.logging.set_verbosity_info()
       logger.info('''Training/evaluation parameters %s'''			,							UpperCAmelCase_				)

       # Set seed
       set_seed(training_args.seed				)

       # Load pretrained model and tokenizer
       #
       # Distributed training:
       # The .from_pretrained methods guarantee that only one local process can concurrently
       # download model & vocab.

       SCREAMING_SNAKE_CASE_							:				Tuple		=AutoConfig.from_pretrained(
           model_args.config_name if model_args.config_name else model_args.model_name_or_path			,							cache_dir=model_args.cache_dir			,							)

       SCREAMING_SNAKE_CASE_							:				Dict		=('''encoder_layerdrop''', '''decoder_layerdrop''', '''dropout''', '''attention_dropout''')
       for p in extra_model_params:
              if getattr(UpperCAmelCase_			,							UpperCAmelCase_			,							UpperCAmelCase_				):
                     assert hasattr(UpperCAmelCase_			,							UpperCAmelCase_				), f'({config.__class__.__name__}) doesn\'t have a `{p}` attribute'
                     setattr(UpperCAmelCase_			,							UpperCAmelCase_			,							getattr(UpperCAmelCase_			,							UpperCAmelCase_				)				)

       SCREAMING_SNAKE_CASE_							:				Tuple		=AutoTokenizer.from_pretrained(
           model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path			,							cache_dir=model_args.cache_dir			,							)
       SCREAMING_SNAKE_CASE_							:				str		=AutoModelForSeqaSeqLM.from_pretrained(
           model_args.model_name_or_path			,							from_tf='''.ckpt''' in model_args.model_name_or_path			,							config=UpperCAmelCase_			,							cache_dir=model_args.cache_dir			,							)

       # use task specific params
       use_task_specific_params(UpperCAmelCase_			,							data_args.task				)

       # set num_beams for evaluation
       if data_args.eval_beams is None:
              SCREAMING_SNAKE_CASE_							:				List[str]		=model.config.num_beams

       # set decoder_start_token_id for MBart
       if model.config.decoder_start_token_id is None and isinstance(UpperCAmelCase_			,							(MBartTokenizer, MBartTokenizerFast)				):
              assert (
                  data_args.tgt_lang is not None and data_args.src_lang is not None
              ), "mBart requires --tgt_lang and --src_lang"
              if isinstance(UpperCAmelCase_			,							UpperCAmelCase_				):
                     SCREAMING_SNAKE_CASE_							:				int		=tokenizer.lang_code_to_id[data_args.tgt_lang]
              else:
                     SCREAMING_SNAKE_CASE_							:				Optional[int]		=tokenizer.convert_tokens_to_ids(data_args.tgt_lang				)

       if model_args.freeze_embeds:
              freeze_embeds(UpperCAmelCase_				)
       if model_args.freeze_encoder:
              freeze_params(model.get_encoder()				)
              assert_all_frozen(model.get_encoder()				)

       SCREAMING_SNAKE_CASE_							:				int		=SeqaSeqDataset

       # Get datasets
       SCREAMING_SNAKE_CASE_							:				Optional[int]		=(
           dataset_class(
               UpperCAmelCase_			,							type_path='''train'''			,							data_dir=data_args.data_dir			,							n_obs=data_args.n_train			,							max_target_length=data_args.max_target_length			,							max_source_length=data_args.max_source_length			,							prefix=model.config.prefix or ''''''			,							)
           if training_args.do_train
           else None
       )
       SCREAMING_SNAKE_CASE_							:				Tuple		=(
           dataset_class(
               UpperCAmelCase_			,							type_path='''val'''			,							data_dir=data_args.data_dir			,							n_obs=data_args.n_val			,							max_target_length=data_args.val_max_target_length			,							max_source_length=data_args.max_source_length			,							prefix=model.config.prefix or ''''''			,							)
           if training_args.do_eval or training_args.evaluation_strategy != EvaluationStrategy.NO
           else None
       )
       SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=(
           dataset_class(
               UpperCAmelCase_			,							type_path='''test'''			,							data_dir=data_args.data_dir			,							n_obs=data_args.n_test			,							max_target_length=data_args.test_max_target_length			,							max_source_length=data_args.max_source_length			,							prefix=model.config.prefix or ''''''			,							)
           if training_args.do_predict
           else None
       )

       # Initialize our Trainer
       SCREAMING_SNAKE_CASE_							:				List[Any]		=(
           build_compute_metrics_fn(data_args.task			,							UpperCAmelCase_				) if training_args.predict_with_generate else None
       )
       SCREAMING_SNAKE_CASE_							:				Optional[int]		=SeqaSeqTrainer(
           model=UpperCAmelCase_			,							args=UpperCAmelCase_			,							data_args=UpperCAmelCase_			,							train_dataset=UpperCAmelCase_			,							eval_dataset=UpperCAmelCase_			,							data_collator=SeqaSeqDataCollator(
               UpperCAmelCase_			,							UpperCAmelCase_			,							model.config.decoder_start_token_id			,							training_args.tpu_num_cores				)			,							compute_metrics=UpperCAmelCase_			,							tokenizer=UpperCAmelCase_			,							)

       SCREAMING_SNAKE_CASE_							:				List[Any]		={}
       # Training
       if training_args.do_train:
              logger.info('''*** Train ***'''				)

              SCREAMING_SNAKE_CASE_							:				List[Any]		=trainer.train(
                  model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path				) else None				)
              SCREAMING_SNAKE_CASE_							:				Optional[int]		=train_result.metrics
              SCREAMING_SNAKE_CASE_							:				int		=data_args.n_train

              trainer.save_model()  # this also saves the tokenizer

              if trainer.is_world_process_zero():
                     handle_metrics('''train'''			,							UpperCAmelCase_			,							training_args.output_dir				)
                     all_metrics.update(UpperCAmelCase_				)

                     # Need to save the state, since Trainer.save_model saves only the tokenizer with the model
                     trainer.state.save_to_json(os.path.join(training_args.output_dir			,							'''trainer_state.json'''				)				)

                     # For convenience, we also re-save the tokenizer to the same directory,
                     # so that you can share your model easily on huggingface.co/models =)
                     tokenizer.save_pretrained(training_args.output_dir				)

    # Evaluation
       if training_args.do_eval:
              logger.info('''*** Evaluate ***'''				)

              SCREAMING_SNAKE_CASE_							:				Optional[int]		=trainer.evaluate(metric_key_prefix='''val'''				)
              SCREAMING_SNAKE_CASE_							:				Dict		=data_args.n_val
              SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=round(metrics['''val_loss''']			,							4				)

              if trainer.is_world_process_zero():
                     handle_metrics('''val'''			,							UpperCAmelCase_			,							training_args.output_dir				)
                     all_metrics.update(UpperCAmelCase_				)

       if training_args.do_predict:
              logger.info('''*** Predict ***'''				)

              SCREAMING_SNAKE_CASE_							:				Tuple		=trainer.predict(test_dataset=UpperCAmelCase_			,							metric_key_prefix='''test'''				)
              SCREAMING_SNAKE_CASE_							:				List[str]		=test_output.metrics
              SCREAMING_SNAKE_CASE_							:				str		=data_args.n_test

              if trainer.is_world_process_zero():
                     SCREAMING_SNAKE_CASE_							:				List[Any]		=round(metrics['''test_loss''']			,							4				)
                     handle_metrics('''test'''			,							UpperCAmelCase_			,							training_args.output_dir				)
                     all_metrics.update(UpperCAmelCase_				)

                     if training_args.predict_with_generate:
                            SCREAMING_SNAKE_CASE_							:				Optional[Any]		=tokenizer.batch_decode(
                                test_output.predictions			,							skip_special_tokens=UpperCAmelCase_			,							clean_up_tokenization_spaces=UpperCAmelCase_				)
                            SCREAMING_SNAKE_CASE_							:				List[Any]		=lmap(str.strip			,							UpperCAmelCase_				)
                            write_txt_file(UpperCAmelCase_			,							os.path.join(training_args.output_dir			,							'''test_generations.txt'''				)				)

       if trainer.is_world_process_zero():
              save_json(UpperCAmelCase_			,							os.path.join(training_args.output_dir			,							'''all_results.json'''				)				)

       return all_metrics
def 							SCREAMING_SNAKE_CASE_	(					UpperCAmelCase_	:     Tuple				)			->						int:
       # For xla_spawn (TPUs)
       main()


if __name__ == "__main__":
 main()


import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_lowercase											=    """▁"""

_lowercase											=    get_tests_dir("""fixtures/test_sentencepiece.model""")


@require_sentencepiece
@require_tokenizers
class 			lowercase_		(      A  ,      unittest.TestCase       ):
       __lowerCamelCase				       =						BigBirdTokenizer
       __lowerCamelCase				       =						BigBirdTokenizerFast
       __lowerCamelCase				       =						True
       __lowerCamelCase				       =						True


       def 	_snake_case				(    self				)						->    Tuple:
              super().setUp()

              SCREAMING_SNAKE_CASE_							:				str		=self.tokenizer_class(__A						,       keep_accents=__A				)
              tokenizer.save_pretrained(self.tmpdirname				)


       def 	_snake_case				(    self				)						->    List[Any]:
              SCREAMING_SNAKE_CASE_							:				str		='''<s>'''
              SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=1

              self.assertEqual(self.get_tokenizer()._convert_token_to_id(__A				)						,       __A				)
              self.assertEqual(self.get_tokenizer()._convert_id_to_token(__A				)						,       __A				)


       def 	_snake_case				(    self				)						->    Any:
              SCREAMING_SNAKE_CASE_							:				str		=list(self.get_tokenizer().get_vocab().keys()				)

              self.assertEqual(vocab_keys[0]						,       '''<unk>'''				)
              self.assertEqual(vocab_keys[1]						,       '''<s>'''				)
              self.assertEqual(vocab_keys[-1]						,       '''[MASK]'''				)
              self.assertEqual(len(__A				)						,       1_004				)


       def 	_snake_case				(    self				)						->    Optional[Any]:
              self.assertEqual(self.get_tokenizer().vocab_size						,       1_000				)


       def 	_snake_case				(    self				)						->    str:
              if not self.test_rust_tokenizer:
                     return

              SCREAMING_SNAKE_CASE_							:				Dict		=self.get_tokenizer()
              SCREAMING_SNAKE_CASE_							:				List[Any]		=self.get_rust_tokenizer()

              SCREAMING_SNAKE_CASE_							:				Union[str, Any]		='''I was born in 92000, and this is falsé.'''

              SCREAMING_SNAKE_CASE_							:				List[Any]		=tokenizer.tokenize(__A				)
              SCREAMING_SNAKE_CASE_							:				Optional[int]		=rust_tokenizer.tokenize(__A				)
              self.assertListEqual(__A						,       __A				)

              SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=tokenizer.encode(__A						,       add_special_tokens=__A				)
              SCREAMING_SNAKE_CASE_							:				List[Any]		=rust_tokenizer.encode(__A						,       add_special_tokens=__A				)
              self.assertListEqual(__A						,       __A				)

              SCREAMING_SNAKE_CASE_							:				Dict		=self.get_rust_tokenizer()
              SCREAMING_SNAKE_CASE_							:				int		=tokenizer.encode(__A				)
              SCREAMING_SNAKE_CASE_							:				Dict		=rust_tokenizer.encode(__A				)
              self.assertListEqual(__A						,       __A				)


       def 	_snake_case				(    self				)						->    str:
              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=BigBirdTokenizer(__A						,       keep_accents=__A				)

              SCREAMING_SNAKE_CASE_							:				str		=tokenizer.tokenize('''This is a test'''				)
              self.assertListEqual(__A						,       ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

              self.assertListEqual(
                  tokenizer.convert_tokens_to_ids(__A				)						,       [285, 46, 10, 170, 382]						,       )

              SCREAMING_SNAKE_CASE_							:				List[Any]		=tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
              self.assertListEqual(
                  __A						,       [
                      SPIECE_UNDERLINE + '''I''',
                      SPIECE_UNDERLINE + '''was''',
                      SPIECE_UNDERLINE + '''b''',
                      '''or''',
                      '''n''',
                      SPIECE_UNDERLINE + '''in''',
                      SPIECE_UNDERLINE + '''''',
                      '''9''',
                      '''2''',
                      '''0''',
                      '''0''',
                      '''0''',
                      ''',''',
                      SPIECE_UNDERLINE + '''and''',
                      SPIECE_UNDERLINE + '''this''',
                      SPIECE_UNDERLINE + '''is''',
                      SPIECE_UNDERLINE + '''f''',
                      '''al''',
                      '''s''',
                      '''é''',
                      '''.''',
                  ]						,       )
              SCREAMING_SNAKE_CASE_							:				Dict		=tokenizer.convert_tokens_to_ids(__A				)
              self.assertListEqual(
                  __A						,       [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]						,       )

              SCREAMING_SNAKE_CASE_							:				Optional[Any]		=tokenizer.convert_ids_to_tokens(__A				)
              self.assertListEqual(
                  __A						,       [
                      SPIECE_UNDERLINE + '''I''',
                      SPIECE_UNDERLINE + '''was''',
                      SPIECE_UNDERLINE + '''b''',
                      '''or''',
                      '''n''',
                      SPIECE_UNDERLINE + '''in''',
                      SPIECE_UNDERLINE + '''''',
                      '''<unk>''',
                      '''2''',
                      '''0''',
                      '''0''',
                      '''0''',
                      ''',''',
                      SPIECE_UNDERLINE + '''and''',
                      SPIECE_UNDERLINE + '''this''',
                      SPIECE_UNDERLINE + '''is''',
                      SPIECE_UNDERLINE + '''f''',
                      '''al''',
                      '''s''',
                      '''<unk>''',
                      '''.''',
                  ]						,       )


       @cached_property
       def 	_snake_case				(    self				)						->    Union[str, Any]:
              return BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base'''				)


       @slow
       def 	_snake_case				(    self				)						->    Any:
              SCREAMING_SNAKE_CASE_							:				str		='''Hello World!'''
              SCREAMING_SNAKE_CASE_							:				Dict		=[65, 18_536, 2_260, 101, 66]

              self.assertListEqual(__A						,       self.big_tokenizer.encode(__A				)				)


       @slow
       def 	_snake_case				(    self				)						->    Optional[int]:
              SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=(
                  '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
                  ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
              )
              # fmt: off
              SCREAMING_SNAKE_CASE_							:				List[Any]		=[65, 871, 419, 358, 946, 991, 2_521, 452, 358, 1_357, 387, 7_751, 3_536, 112, 985, 456, 126, 865, 938, 5_400, 5_734, 458, 1_368, 467, 786, 2_462, 5_246, 1_159, 633, 865, 4_519, 457, 582, 852, 2_557, 427, 916, 508, 405, 34_324, 497, 391, 408, 11_342, 1_244, 385, 100, 938, 985, 456, 574, 362, 12_597, 3_200, 3_129, 1_172, 66]  # noqa: E231
              # fmt: on
              self.assertListEqual(__A						,       self.big_tokenizer.encode(__A				)				)


       @require_torch
       @slow
       def 	_snake_case				(    self				)						->    int:
              import torch

              from transformers import BigBirdConfig, BigBirdModel

              # Build sequence
              SCREAMING_SNAKE_CASE_							:				Optional[int]		=list(self.big_tokenizer.get_vocab().keys()				)[:10]
              SCREAMING_SNAKE_CASE_							:				Any		=''' '''.join(__A				)
              SCREAMING_SNAKE_CASE_							:				int		=self.big_tokenizer.encode_plus(__A						,       return_tensors='''pt'''						,       return_token_type_ids=__A				)
              SCREAMING_SNAKE_CASE_							:				Tuple		=self.big_tokenizer.batch_encode_plus(
                  [sequence + ''' ''' + sequence]						,       return_tensors='''pt'''						,       return_token_type_ids=__A				)

              SCREAMING_SNAKE_CASE_							:				Union[str, Any]		=BigBirdConfig(attention_type='''original_full'''				)
              SCREAMING_SNAKE_CASE_							:				Tuple		=BigBirdModel(__A				)

              assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

              with torch.no_grad():
                     model(**__A				)
                     model(**__A				)


       @slow
       def 	_snake_case				(    self				)						->    Tuple:
              SCREAMING_SNAKE_CASE_							:				Any		=BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base'''				)
              SCREAMING_SNAKE_CASE_							:				List[Any]		=tokenizer.decode(tokenizer('''Paris is the [MASK].'''				).input_ids				)

              self.assertTrue(decoded_text == '''[CLS] Paris is the[MASK].[SEP]'''				)





       @slow
       def 	_snake_case				(    self				)						->    Any:
              # fmt: off
              SCREAMING_SNAKE_CASE_							:				Optional[int]		={'''input_ids''': [[65, 39_286, 458, 36_335, 2_001, 456, 13_073, 13_266, 455, 113, 7_746, 1_741, 11_157, 391, 13_073, 13_266, 455, 113, 3_967, 35_412, 113, 4_936, 109, 3_870, 2_377, 113, 30_084, 45_720, 458, 134, 17_496, 112, 503, 11_672, 113, 118, 112, 5_665, 13_347, 38_687, 112, 1_496, 31_389, 112, 3_268, 47_264, 134, 962, 112, 16_377, 8_035, 23_130, 430, 12_169, 15_518, 28_592, 458, 146, 41_697, 109, 391, 12_169, 15_518, 16_689, 458, 146, 41_358, 109, 452, 726, 4_034, 111, 763, 35_412, 5_082, 388, 1_903, 111, 9_051, 391, 2_870, 48_918, 1_900, 1_123, 550, 998, 112, 9_586, 15_985, 455, 391, 410, 22_955, 37_636, 114, 66], [65, 448, 17_496, 419, 3_663, 385, 763, 113, 27_533, 2_870, 3_283, 13_043, 1_639, 24_713, 523, 656, 24_013, 18_550, 2_521, 517, 27_014, 21_244, 420, 1_212, 1_465, 391, 927, 4_833, 388, 578, 11_786, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 484, 2_169, 7_687, 21_932, 18_146, 726, 363, 17_032, 3_391, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
              # fmt: on

              self.tokenizer_integration_test_util(
                  expected_encoding=__A						,       model_name='''google/bigbird-roberta-base'''						,       revision='''215c99f1600e06f83acce68422f2035b2b5c3510'''						,       )
