

"""simple docstring"""


import argparse
import collections

import numpy as np
import torch
from flax import traverse_util
from tax import checkpoints

from transformers import MTaConfig, UMTaEncoderModel, UMTaForConditionalGeneration
from transformers.utils import logging


logging.set_verbosity_info()



def   _A					(	_a	:						Any     ,   _a	:						Dict     ,   _a	:						Tuple				):



      """simple docstring"""






      return params[f'{prefix}/{prefix}/relpos_bias/rel_embedding'][:, i, :]



def   _A					(	_a	:						List[str]     ,   _a	:						str     ,   _a	:						Tuple     ,   _a	:						Optional[int]="attention"				):



      """simple docstring"""






      A				  =		A				  =		np.ascontiguousarray(params[f'{prefix}/{prefix}/{layer_name}/key/kernel'][:, i, :, :]				)
      A				  =		k_tmp.reshape(k_tmp.shape[0]     ,   k_tmp.shape[1] * k_tmp.shape[2]				)
      A				  =		np.ascontiguousarray(params[f'{prefix}/{prefix}/{layer_name}/out/kernel'][:, i, :, :]				)
      A				  =		o_tmp.reshape(o_tmp.shape[0] * o_tmp.shape[1]     ,   o_tmp.shape[2]				)
      A				  =		np.ascontiguousarray(params[f'{prefix}/{prefix}/{layer_name}/query/kernel'][:, i, :, :]				)
      A				  =		q_tmp.reshape(q_tmp.shape[0]     ,   q_tmp.shape[1] * q_tmp.shape[2]				)
      A				  =		np.ascontiguousarray(params[f'{prefix}/{prefix}/{layer_name}/value/kernel'][:, i, :, :]				)
      A				  =		v_tmp.reshape(v_tmp.shape[0]     ,   v_tmp.shape[1] * v_tmp.shape[2]				)
      return k, o, q, v



def   _A					(	_a	:						str     ,   _a	:						List[str]     ,   _a	:						Union[str, Any]     ,   _a	:						Optional[Any]=False				):



      """simple docstring"""






      if split_mlp_wi:
            A				  =		params[f'{prefix}/{prefix}/mlp/wi_0/kernel'][:, i, :]
            A				  =		params[f'{prefix}/{prefix}/mlp/wi_1/kernel'][:, i, :]
            A				  =		(wi_a, wi_a)
      else:
            A				  =		params[f'{prefix}/{prefix}/mlp/wi/kernel'][:, i, :]

      A				  =		params[f'{prefix}/{prefix}/mlp/wo/kernel'][:, i, :]
      return wi, wo



def   _A					(	_a	:						Optional[Any]     ,   _a	:						Dict     ,   _a	:						Optional[Any]     ,   _a	:						Optional[Any]				):



      """simple docstring"""






      return params[f'{prefix}/{prefix}/{layer_name}/scale'][:, i]



def   _A					(	_a	:						dict     ,   *, _a	:						int     ,   _a	:						bool     ,   _a	:						bool = False				):



      """simple docstring"""






      A				  =		traverse_util.flatten_dict(variables["""target"""]				)
      A				  =		{"""/""".join(__lowerCAmelCase				): v for k, v in old.items()}

      # v1.1 models have a gated GeLU with wi_0 and wi_1 instead of wi
      A				  =		"""encoder/encoder/mlp/wi_0/kernel""" in old
      print("""Split MLP:"""     ,   __lowerCAmelCase				)

      A				  =		collections.OrderedDict()

      # Shared embeddings.
      A				  =		old["""token_embedder/embedding"""]

      # Encoder.
      for i in range(__lowerCAmelCase				):
            # Block i, layer 0 (Self Attention).
            A				  =		tax_layer_norm_lookup(__lowerCAmelCase     ,   __lowerCAmelCase     ,   """encoder"""     ,   """pre_attention_layer_norm"""				)
            A  ,			A  ,			A  ,			A				  =		tax_attention_lookup(__lowerCAmelCase     ,   __lowerCAmelCase     ,   """encoder"""     ,   """attention"""				)
            A				  =		layer_norm
            A				  =		k.T
            A				  =		o.T
            A				  =		q.T
            A				  =		v.T

            # Block i, layer 1 (MLP).
            A				  =		tax_layer_norm_lookup(__lowerCAmelCase     ,   __lowerCAmelCase     ,   """encoder"""     ,   """pre_mlp_layer_norm"""				)
            A  ,			A				  =		tax_mlp_lookup(__lowerCAmelCase     ,   __lowerCAmelCase     ,   """encoder"""     ,   __lowerCAmelCase				)
            A				  =		layer_norm
            if split_mlp_wi:
                  A				  =		wi[0].T
                  A				  =		wi[1].T
            else:
                  A				  =		wi.T
            A				  =		wo.T
            if scalable_attention:
                  # convert the rel_embedding of each layer
                  A				  =		tax_relpos_bias_lookup(
                      __lowerCAmelCase     ,   __lowerCAmelCase     ,   """encoder"""				).T

      A				  =		old["""encoder/encoder_norm/scale"""]

      if not scalable_attention:
            A				  =		tax_relpos_bias_lookup(
                __lowerCAmelCase     ,   0     ,   """encoder"""				).T
            A				  =		tax_relpos_bias_lookup(
                __lowerCAmelCase     ,   0     ,   """decoder"""				).T

      if not is_encoder_only:
            # Decoder.
            for i in range(__lowerCAmelCase				):
                  # Block i, layer 0 (Self Attention).
                  A				  =		tax_layer_norm_lookup(__lowerCAmelCase     ,   __lowerCAmelCase     ,   """decoder"""     ,   """pre_self_attention_layer_norm"""				)
                  A  ,			A  ,			A  ,			A				  =		tax_attention_lookup(__lowerCAmelCase     ,   __lowerCAmelCase     ,   """decoder"""     ,   """self_attention"""				)
                  A				  =		layer_norm
                  A				  =		k.T
                  A				  =		o.T
                  A				  =		q.T
                  A				  =		v.T

                  # Block i, layer 1 (Cross Attention).
                  A				  =		tax_layer_norm_lookup(__lowerCAmelCase     ,   __lowerCAmelCase     ,   """decoder"""     ,   """pre_cross_attention_layer_norm"""				)
                  A  ,			A  ,			A  ,			A				  =		tax_attention_lookup(__lowerCAmelCase     ,   __lowerCAmelCase     ,   """decoder"""     ,   """encoder_decoder_attention"""				)
                  A				  =		layer_norm
                  A				  =		k.T
                  A				  =		o.T
                  A				  =		q.T
                  A				  =		v.T

                  # Block i, layer 2 (MLP).
                  A				  =		tax_layer_norm_lookup(__lowerCAmelCase     ,   __lowerCAmelCase     ,   """decoder"""     ,   """pre_mlp_layer_norm"""				)
                  A  ,			A				  =		tax_mlp_lookup(__lowerCAmelCase     ,   __lowerCAmelCase     ,   """decoder"""     ,   __lowerCAmelCase				)
                  A				  =		layer_norm
                  if split_mlp_wi:
                        A				  =		wi[0].T
                        A				  =		wi[1].T
                  else:
                        A				  =		wi.T
                  A				  =		wo.T

                  if scalable_attention:
                        # convert the rel_embedding of each layer
                        A				  =		tax_relpos_bias_lookup(__lowerCAmelCase     ,   __lowerCAmelCase     ,   """decoder"""				).T

            A				  =		old["""decoder/decoder_norm/scale"""]

            # LM Head (only in v1.1 checkpoints, in v1.0 embeddings are used instead)
            if "decoder/logits_dense/kernel" in old:
                  A				  =		old["""decoder/logits_dense/kernel"""].T

      return new



def   _A					(	_a	:						Optional[Any]     ,   _a	:						bool				):



      """simple docstring"""






      A				  =		collections.OrderedDict([(k, torch.from_numpy(v.copy()				)) for (k, v) in converted_params.items()]				)

      # Add what is missing.
      if "encoder.embed_tokens.weight" not in state_dict:
            A				  =		state_dict["""shared.weight"""]

      if not is_encoder_only:
            if "decoder.embed_tokens.weight" not in state_dict:
                  A				  =		state_dict["""shared.weight"""]

            if "lm_head.weight" not in state_dict:  # For old 1.0 models.
                  print("""Using shared word embeddings as lm_head."""				)
                  A				  =		state_dict["""shared.weight"""]

      return state_dict



def   _A					(	_a	:						Any     ,   _a	:						Dict     ,   _a	:						Dict     ,   _a	:						Dict     ,   _a	:						List[Any]				):



      """simple docstring"""






      A				  =		checkpoints.load_tax_checkpoint(__lowerCAmelCase				)
      A				  =		convert_tax_to_pytorch(
          __lowerCAmelCase     ,   num_layers=config.num_layers     ,   is_encoder_only=__lowerCAmelCase     ,   scalable_attention=__lowerCAmelCase				)
      A				  =		make_state_dict(__lowerCAmelCase     ,   __lowerCAmelCase				)
      model.load_state_dict(__lowerCAmelCase     ,   strict=__lowerCAmelCase				)



def   _A					(	_a	:						str     ,   _a	:						List[Any]     ,   _a	:						Union[str, Any]     ,   _a	:						bool = False     ,   _a	:						bool = False     ,   ):



      """simple docstring"""






      A				  =		MTaConfig.from_json_file(__lowerCAmelCase				)
      print(f'Building PyTorch model from configuration: {config}'				)
      # Non-v1.1 checkpoints could also use T5Model, but this works for all.
      # The v1.0 checkpoints will simply have an LM head that is the word embeddings.
      if is_encoder_only:
            A				  =		UMTaEncoderModel(__lowerCAmelCase				)
      else:
            A				  =		UMTaForConditionalGeneration(__lowerCAmelCase				)

      # Load weights from tf checkpoint
      load_tax_weights_in_ta(__lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase     ,   __lowerCAmelCase				)

      # Save pytorch-model
      print(f'Save PyTorch model to {pytorch_dump_path}'				)
      model.save_pretrained(__lowerCAmelCase				)

      # Verify that we can load the checkpoint.
      model.from_pretrained(__lowerCAmelCase				)
      print("""Done"""				)


if __name__ == "__main__":
  UpperCAmelCase													=argparse.ArgumentParser(description="Converts a native T5X checkpoint into a PyTorch checkpoint.")
  # Required parameters
  parser.add_argument(
      "--t5x_checkpoint_path", default=None, type=str, required=True, help="Path to the T5X checkpoint."
  )
  parser.add_argument(
      "--config_file",
      default=None,
      type=str,
      required=True,
      help="The config json file corresponding to the pre-trained T5 model.\nThis specifies the model architecture.",
  )
  parser.add_argument(
      "--pytorch_dump_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
  )
  parser.add_argument(
      "--is_encoder_only", action="store_true", help="Check if the model is encoder-decoder model", default=False
  )
  parser.add_argument(
      "--scalable_attention",
      action="store_true",
      help="Whether the model uses scaled attention (umt5 model)",
      default=False,
  )
  UpperCAmelCase													=parser.parse_args()
  convert_tax_checkpoint_to_pytorch(
      args.tax_checkpoint_path,
      args.config_file,
      args.pytorch_dump_path,
      args.is_encoder_only,
      args.scalable_attention,
  )











'''simple docstring'''




import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_SCREAMING_SNAKE_CASE							     =			get_tests_dir("fixtures/test_sentencepiece.model")
@require_sentencepiece
@require_tokenizers
class 			_lowerCAmelCase		(			A__							,     unittest.TestCase	):






     """simple docstring"""






     snake_case_						      =   XLNetTokenizer
     snake_case_						      =   XLNetTokenizerFast
     snake_case_						      =   True
     snake_case_						      =   True



     def      lowerCAmelCase			(   self							:			List[Any]						)->	Union[str, Any]:
        super().setUp()

        # We have a SentencePiece fixture for testing
        snake_case 						=   XLNetTokenizer(__snake_case   ,       keep_accents=__snake_case						)
        tokenizer.sanitize_special_tokens()
        tokenizer.save_pretrained(self.tmpdirname						)



     def      lowerCAmelCase			(   self							:			List[Any]						)->	str:
        snake_case 						=   """<s>"""
        snake_case 						=   1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case						)   ,       __snake_case						)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case						)   ,       __snake_case						)



     def      lowerCAmelCase			(   self							:			Tuple						)->	str:
        snake_case 						=   list(self.get_tokenizer().get_vocab().keys()						)

        self.assertEqual(vocab_keys[0]   ,       """<unk>"""						)
        self.assertEqual(vocab_keys[1]   ,       """<s>"""						)
        self.assertEqual(vocab_keys[-1]   ,       """<eod>"""						)
        self.assertEqual(len(__snake_case						)   ,       10_06						)



     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	Tuple:
        self.assertEqual(self.get_tokenizer().vocab_size   ,       10_00						)



     def      lowerCAmelCase			(   self							:			List[str]						)->	int:
        snake_case 						=   XLNetTokenizer(__snake_case   ,       keep_accents=__snake_case						)

        snake_case 						=   tokenizer.tokenize("""This is a test"""						)
        self.assertListEqual(__snake_case   ,       ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]						)

        self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case						)   ,       [2_85, 46, 10, 1_70, 3_82]						)

        snake_case 						=   tokenizer.tokenize("""I was born in 92000, and this is falsé."""						)
        self.assertListEqual(
            __snake_case   ,       [
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """9""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """é""",
                """.""",
            ]   ,       )
        snake_case 						=   tokenizer.convert_tokens_to_ids(__snake_case						)
        self.assertListEqual(__snake_case   ,       [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]						)

        snake_case 						=   tokenizer.convert_ids_to_tokens(__snake_case						)
        self.assertListEqual(
            __snake_case   ,       [
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """<unk>""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """<unk>""",
                """.""",
            ]   ,       )



     def      lowerCAmelCase			(   self							:			str						)->	str:
        snake_case 						=   XLNetTokenizer(__snake_case   ,       do_lower_case=__snake_case						)
        snake_case 						=   tokenizer.tokenize("""I was born in 92000, and this is falsé."""						)
        self.assertListEqual(
            __snake_case   ,       [
                SPIECE_UNDERLINE + """""",
                """i""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """9""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """se""",
                """.""",
            ]   ,       )
        self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)   ,       ["""▁he""", """ll""", """o"""]						)



     def      lowerCAmelCase			(   self							:			List[str]						)->	str:
        snake_case 						=   XLNetTokenizer(__snake_case   ,       do_lower_case=__snake_case						)
        snake_case 						=   tokenizer.tokenize("""I was born in 92000, and this is falsé."""						)
        self.assertListEqual(
            __snake_case   ,       [
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """9""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """se""",
                """.""",
            ]   ,       )



     @slow
     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	Dict:
        snake_case 						=   XLNetTokenizer.from_pretrained("""xlnet-base-cased"""						)

        snake_case 						=   tokenizer.encode("""sequence builders"""   ,       add_special_tokens=__snake_case						)
        snake_case 						=   tokenizer.encode("""multi-sequence build"""   ,       add_special_tokens=__snake_case						)

        snake_case 						=   tokenizer.build_inputs_with_special_tokens(__snake_case						)
        snake_case 						=   tokenizer.build_inputs_with_special_tokens(__snake_case   ,       __snake_case						)

        assert encoded_sentence == text + [4, 3]
        assert encoded_pair == text + [4] + text_a + [4, 3]



     @slow
     def      lowerCAmelCase			(   self							:			Optional[int]						)->	Union[str, Any]:
        # fmt: off
        snake_case 						=   {"""input_ids""": [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=__snake_case   ,       model_name="""xlnet-base-cased"""   ,       revision="""c841166438c31ec7ca9a106dee7bb312b73ae511"""   ,       )

