






"""simple docstring"""







import unittest

from transformers import MraConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_torch_available():
				import torch

				from transformers import (
				    MraForMaskedLM,
				    MraForMultipleChoice,
				    MraForQuestionAnswering,
				    MraForSequenceClassification,
				    MraForTokenClassification,
				    MraModel,
				)
				from transformers.models.mra.modeling_mra import MRA_PRETRAINED_MODEL_ARCHIVE_LIST


class      UpperCamelCase		:






						def __init__(		self  :int			, __magic_name__  :Tuple			, __magic_name__  :int=2			, __magic_name__  :Union[str, Any]=8			, __magic_name__  :List[Any]=True			, __magic_name__  :str=True			, __magic_name__  :Tuple=True			, __magic_name__  :Union[str, Any]=True			, __magic_name__  :Optional[int]=99			, __magic_name__  :Union[str, Any]=16			, __magic_name__  :Optional[Any]=5			, __magic_name__  :int=2			, __magic_name__  :Tuple=36			, __magic_name__  :Dict="gelu"			, __magic_name__  :Optional[int]=0.0			, __magic_name__  :List[Any]=0.0			, __magic_name__  :List[Any]=512			, __magic_name__  :Dict=16			, __magic_name__  :List[Any]=2			, __magic_name__  :Any=0.02			, __magic_name__  :Union[str, Any]=3			, __magic_name__  :int=4			, __magic_name__  :Dict=None			, )      ->int:
										lowercase    :      Tuple       = parent
										lowercase    :      List[Any]       = batch_size
										lowercase    :      Dict       = seq_length
										lowercase    :      int       = is_training
										lowercase    :      str       = use_input_mask
										lowercase    :      Optional[int]       = use_token_type_ids
										lowercase    :      str       = use_labels
										lowercase    :      List[str]       = vocab_size
										lowercase    :      Tuple       = hidden_size
										lowercase    :      List[str]       = num_hidden_layers
										lowercase    :      Dict       = num_attention_heads
										lowercase    :      Optional[int]       = intermediate_size
										lowercase    :      List[Any]       = hidden_act
										lowercase    :      Union[str, Any]       = hidden_dropout_prob
										lowercase    :      List[Any]       = attention_probs_dropout_prob
										lowercase    :      str       = max_position_embeddings
										lowercase    :      Union[str, Any]       = type_vocab_size
										lowercase    :      str       = type_sequence_label_size
										lowercase    :      Optional[int]       = initializer_range
										lowercase    :      Dict       = num_labels
										lowercase    :      str       = num_choices
										lowercase    :      List[Any]       = scope






						def      __snake_case      (		self  :int				)      ->Union[str, Any]:
										lowercase    :      Tuple       = ids_tensor([self.batch_size, self.seq_length]			, self.vocab_size				)

										lowercase    :      str       = None
										if self.use_input_mask:
														lowercase    :      int       = random_attention_mask([self.batch_size, self.seq_length]				)

										lowercase    :      List[Any]       = None
										if self.use_token_type_ids:
														lowercase    :      Optional[Any]       = ids_tensor([self.batch_size, self.seq_length]			, self.type_vocab_size				)

										lowercase    :      List[str]       = None
										lowercase    :      str       = None
										lowercase    :      Tuple       = None
										if self.use_labels:
														lowercase    :      int       = ids_tensor([self.batch_size]			, self.type_sequence_label_size				)
														lowercase    :      Tuple       = ids_tensor([self.batch_size, self.seq_length]			, self.num_labels				)
														lowercase    :      List[Any]       = ids_tensor([self.batch_size]			, self.num_choices				)

										lowercase    :      Tuple       = self.get_config()

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






						def      __snake_case      (		self  :Tuple				)      ->Any:
										return MraConfig(
										    vocab_size=self.vocab_size			, hidden_size=self.hidden_size			, num_hidden_layers=self.num_hidden_layers			, num_attention_heads=self.num_attention_heads			, intermediate_size=self.intermediate_size			, hidden_act=self.hidden_act			, hidden_dropout_prob=self.hidden_dropout_prob			, attention_probs_dropout_prob=self.attention_probs_dropout_prob			, max_position_embeddings=self.max_position_embeddings			, type_vocab_size=self.type_vocab_size			, is_decoder=__magic_name__			, initializer_range=self.initializer_range			, )






						def      __snake_case      (		self  :Dict				)      ->List[Any]:
										lowercase    :      Optional[Any]       = self.get_config()
										lowercase    :      Optional[Any]       = 300
										return config






						def      __snake_case      (		self  :Tuple				)      ->Optional[Any]:
										(
										    (
										    lowercase
										)      ,      (
										    lowercase
										)      ,      (
										    lowercase
										)      ,      (
										    lowercase
										)      ,      (
										    lowercase
										)      ,      (
										    lowercase
										)      ,      (
										    lowercase
										)      ,      
										)    :      Tuple       = self.prepare_config_and_inputs()

										lowercase    :      List[Any]       = True
										lowercase    :      Optional[int]       = floats_tensor([self.batch_size, self.seq_length, self.hidden_size]				)
										lowercase    :      str       = ids_tensor([self.batch_size, self.seq_length]			, vocab_size=2				)

										return (
										    config,
										    input_ids,
										    token_type_ids,
										    input_mask,
										    sequence_labels,
										    token_labels,
										    choice_labels,
										    encoder_hidden_states,
										    encoder_attention_mask,
										)






						def      __snake_case      (		self  :Dict			, __magic_name__  :Tuple			, __magic_name__  :str			, __magic_name__  :Dict			, __magic_name__  :Union[str, Any]			, __magic_name__  :Union[str, Any]			, __magic_name__  :Optional[Any]			, __magic_name__  :Tuple				)      ->List[str]:
										lowercase    :      Union[str, Any]       = MraModel(config=__magic_name__				)
										model.to(__magic_name__				)
										model.eval()
										lowercase    :      Tuple       = model(__magic_name__			, attention_mask=__magic_name__			, token_type_ids=__magic_name__				)
										lowercase    :      Union[str, Any]       = model(__magic_name__			, token_type_ids=__magic_name__				)
										lowercase    :      List[Any]       = model(__magic_name__				)
										self.parent.assertEqual(result.last_hidden_state.shape			, (self.batch_size, self.seq_length, self.hidden_size)				)






						def      __snake_case      (		self  :Any			, __magic_name__  :Dict			, __magic_name__  :Optional[int]			, __magic_name__  :Optional[int]			, __magic_name__  :Dict			, __magic_name__  :Union[str, Any]			, __magic_name__  :int			, __magic_name__  :Optional[Any]			, __magic_name__  :int			, __magic_name__  :List[str]			, )      ->int:
										lowercase    :      Optional[int]       = True
										lowercase    :      List[str]       = MraModel(__magic_name__				)
										model.to(__magic_name__				)
										model.eval()
										lowercase    :      Dict       = model(
										    __magic_name__			, attention_mask=__magic_name__			, token_type_ids=__magic_name__			, encoder_hidden_states=__magic_name__			, encoder_attention_mask=__magic_name__			, )
										lowercase    :      Tuple       = model(
										    __magic_name__			, attention_mask=__magic_name__			, token_type_ids=__magic_name__			, encoder_hidden_states=__magic_name__			, )
										lowercase    :      Dict       = model(__magic_name__			, attention_mask=__magic_name__			, token_type_ids=__magic_name__				)
										self.parent.assertEqual(result.last_hidden_state.shape			, (self.batch_size, self.seq_length, self.hidden_size)				)






						def      __snake_case      (		self  :Dict			, __magic_name__  :int			, __magic_name__  :str			, __magic_name__  :Dict			, __magic_name__  :int			, __magic_name__  :Any			, __magic_name__  :Optional[Any]			, __magic_name__  :Dict				)      ->Dict:
										lowercase    :      int       = MraForMaskedLM(config=__magic_name__				)
										model.to(__magic_name__				)
										model.eval()
										lowercase    :      str       = model(__magic_name__			, attention_mask=__magic_name__			, token_type_ids=__magic_name__			, labels=__magic_name__				)
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.seq_length, self.vocab_size)				)






						def      __snake_case      (		self  :Optional[int]			, __magic_name__  :Union[str, Any]			, __magic_name__  :Tuple			, __magic_name__  :Tuple			, __magic_name__  :List[str]			, __magic_name__  :List[Any]			, __magic_name__  :Optional[int]			, __magic_name__  :Union[str, Any]				)      ->Optional[int]:
										lowercase    :      List[Any]       = MraForQuestionAnswering(config=__magic_name__				)
										model.to(__magic_name__				)
										model.eval()
										lowercase    :      Tuple       = model(
										    __magic_name__			, attention_mask=__magic_name__			, token_type_ids=__magic_name__			, start_positions=__magic_name__			, end_positions=__magic_name__			, )
										self.parent.assertEqual(result.start_logits.shape			, (self.batch_size, self.seq_length)				)
										self.parent.assertEqual(result.end_logits.shape			, (self.batch_size, self.seq_length)				)






						def      __snake_case      (		self  :Dict			, __magic_name__  :Tuple			, __magic_name__  :List[str]			, __magic_name__  :List[Any]			, __magic_name__  :Any			, __magic_name__  :str			, __magic_name__  :Optional[Any]			, __magic_name__  :Dict				)      ->Optional[int]:
										lowercase    :      Optional[int]       = self.num_labels
										lowercase    :      Union[str, Any]       = MraForSequenceClassification(__magic_name__				)
										model.to(__magic_name__				)
										model.eval()
										lowercase    :      Optional[int]       = model(__magic_name__			, attention_mask=__magic_name__			, token_type_ids=__magic_name__			, labels=__magic_name__				)
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.num_labels)				)






						def      __snake_case      (		self  :int			, __magic_name__  :Dict			, __magic_name__  :Optional[Any]			, __magic_name__  :int			, __magic_name__  :Tuple			, __magic_name__  :List[str]			, __magic_name__  :Union[str, Any]			, __magic_name__  :int				)      ->Dict:
										lowercase    :      int       = self.num_labels
										lowercase    :      int       = MraForTokenClassification(config=__magic_name__				)
										model.to(__magic_name__				)
										model.eval()
										lowercase    :      Optional[Any]       = model(__magic_name__			, attention_mask=__magic_name__			, token_type_ids=__magic_name__			, labels=__magic_name__				)
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.seq_length, self.num_labels)				)






						def      __snake_case      (		self  :str			, __magic_name__  :Any			, __magic_name__  :Any			, __magic_name__  :str			, __magic_name__  :int			, __magic_name__  :Tuple			, __magic_name__  :Any			, __magic_name__  :List[Any]				)      ->List[str]:
										lowercase    :      str       = self.num_choices
										lowercase    :      Any       = MraForMultipleChoice(config=__magic_name__				)
										model.to(__magic_name__				)
										model.eval()
										lowercase    :      int       = input_ids.unsqueeze(1				).expand(-1			, self.num_choices			, -1				).contiguous()
										lowercase    :      List[str]       = token_type_ids.unsqueeze(1				).expand(-1			, self.num_choices			, -1				).contiguous()
										lowercase    :      List[str]       = input_mask.unsqueeze(1				).expand(-1			, self.num_choices			, -1				).contiguous()
										lowercase    :      List[Any]       = model(
										    __magic_name__			, attention_mask=__magic_name__			, token_type_ids=__magic_name__			, labels=__magic_name__			, )
										self.parent.assertEqual(result.logits.shape			, (self.batch_size, self.num_choices)				)






						def      __snake_case      (		self  :Any				)      ->List[str]:
										lowercase    :      Optional[Any]       = self.prepare_config_and_inputs()
										(
										    (
										    lowercase
										)      ,      (
										    lowercase
										)      ,      (
										    lowercase
										)      ,      (
										    lowercase
										)      ,      (
										    lowercase
										)      ,      (
										    lowercase
										)      ,      (
										    lowercase
										)      ,      
										)    :      Union[str, Any]       = config_and_inputs
										lowercase    :      Optional[int]       = {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
										return config, inputs_dict


@require_torch
class      UpperCamelCase		(__snake_case						,						unittest.TestCase     ):
						_SCREAMING_SNAKE_CASE  : List[str]												=		(
						    (
						        MraModel,
						        MraForMaskedLM,
						        MraForMultipleChoice,
						        MraForQuestionAnswering,
						        MraForSequenceClassification,
						        MraForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						_SCREAMING_SNAKE_CASE  : Tuple												=		False
						_SCREAMING_SNAKE_CASE  : Tuple												=		False
						_SCREAMING_SNAKE_CASE  : Tuple												=		False
						_SCREAMING_SNAKE_CASE  : Tuple												=		False

						_SCREAMING_SNAKE_CASE  : Optional[int]												=		()






						def      __snake_case      (		self  :Any				)      ->Union[str, Any]:
										lowercase    :      List[str]       = MraModelTester(self				)
										lowercase    :      Tuple       = ConfigTester(self			, config_class=__magic_name__			, hidden_size=37				)






						def      __snake_case      (		self  :Any				)      ->List[Any]:
										self.config_tester.run_common_tests()






						def      __snake_case      (		self  :Tuple				)      ->List[Any]:
										lowercase    :      List[Any]       = self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*__magic_name__				)






						def      __snake_case      (		self  :Union[str, Any]				)      ->int:
										lowercase    :      Any       = self.model_tester.prepare_config_and_inputs()
										for type in ["absolute", "relative_key", "relative_key_query"]:
														lowercase    :      Any       = type
														self.model_tester.create_and_check_model(*__magic_name__				)






						def      __snake_case      (		self  :Optional[Any]				)      ->List[str]:
										lowercase    :      List[str]       = self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_masked_lm(*__magic_name__				)






						def      __snake_case      (		self  :str				)      ->Dict:
										lowercase    :      List[Any]       = self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_multiple_choice(*__magic_name__				)






						def      __snake_case      (		self  :Optional[int]				)      ->int:
										lowercase    :      Any       = self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_question_answering(*__magic_name__				)






						def      __snake_case      (		self  :List[str]				)      ->Optional[int]:
										lowercase    :      int       = self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_sequence_classification(*__magic_name__				)






						def      __snake_case      (		self  :Union[str, Any]				)      ->Tuple:
										lowercase    :      Union[str, Any]       = self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_token_classification(*__magic_name__				)






						@slow
						def      __snake_case      (		self  :Optional[int]				)      ->Optional[int]:
										for model_name in MRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
														lowercase    :      List[Any]       = MraModel.from_pretrained(__magic_name__				)
														self.assertIsNotNone(__magic_name__				)






						@unittest.skip(reason="""MRA does not output attentions"""				)
						def      __snake_case      (		self  :int				)      ->Optional[Any]:
										return


@require_torch
class      UpperCamelCase		(unittest.TestCase     ):






						@slow
						def      __snake_case      (		self  :Union[str, Any]				)      ->Optional[int]:
										lowercase    :      List[Any]       = MraModel.from_pretrained("""uw-madison/mra-base-512-4"""				)
										lowercase    :      Union[str, Any]       = torch.arange(256				).unsqueeze(0				)

										with torch.no_grad():
														lowercase    :      Tuple       = model(__magic_name__				)[0]

										lowercase    :      Union[str, Any]       = torch.Size((1, 256, 768)				)
										self.assertEqual(output.shape			, __magic_name__				)

										lowercase    :      Optional[int]       = torch.tensor(
										    [[[-0.0140, 0.0830, -0.0381], [0.1546, 0.1402, 0.0220], [0.1162, 0.0851, 0.0165]]]				)

										self.assertTrue(torch.allclose(output[:, :3, :3]			, __magic_name__			, atol=1E-4				)				)






						@slow
						def      __snake_case      (		self  :List[Any]				)      ->List[str]:
										lowercase    :      Tuple       = MraForMaskedLM.from_pretrained("""uw-madison/mra-base-512-4"""				)
										lowercase    :      List[str]       = torch.arange(256				).unsqueeze(0				)

										with torch.no_grad():
														lowercase    :      Optional[Any]       = model(__magic_name__				)[0]

										lowercase    :      Optional[Any]       = 50_265

										lowercase    :      Union[str, Any]       = torch.Size((1, 256, vocab_size)				)
										self.assertEqual(output.shape			, __magic_name__				)

										lowercase    :      Any       = torch.tensor(
										    [[[9.2595, -3.6038, 11.8819], [9.3869, -3.2693, 11.0956], [11.8524, -3.4938, 13.1210]]]				)

										self.assertTrue(torch.allclose(output[:, :3, :3]			, __magic_name__			, atol=1E-4				)				)






						@slow
						def      __snake_case      (		self  :int				)      ->str:
										lowercase    :      List[Any]       = MraForMaskedLM.from_pretrained("""uw-madison/mra-base-4096-8-d3"""				)
										lowercase    :      str       = torch.arange(4_096				).unsqueeze(0				)

										with torch.no_grad():
														lowercase    :      Tuple       = model(__magic_name__				)[0]

										lowercase    :      List[Any]       = 50_265

										lowercase    :      List[str]       = torch.Size((1, 4_096, vocab_size)				)
										self.assertEqual(output.shape			, __magic_name__				)

										lowercase    :      int       = torch.tensor(
										    [[[5.4789, -2.3564, 7.5064], [7.9067, -1.3369, 9.9668], [9.0712, -1.8106, 7.0380]]]				)

										self.assertTrue(torch.allclose(output[:, :3, :3]			, __magic_name__			, atol=1E-4				)				)








"""simple docstring"""







import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCAmelCase            =	get_tests_dir('fixtures/test_sentencepiece.model')


@require_sentencepiece
class      UpperCamelCase		(__snake_case						,						unittest.TestCase     ):
						_SCREAMING_SNAKE_CASE  : Optional[Any]												=		XLMProphetNetTokenizer
						_SCREAMING_SNAKE_CASE  : Optional[int]												=		False
						_SCREAMING_SNAKE_CASE  : Optional[int]												=		True






						def      __snake_case      (		self  :Optional[int]				)      ->List[str]:
										super().setUp()

										# We have a SentencePiece fixture for testing
										lowercase    :      List[str]       = XLMProphetNetTokenizer(__magic_name__			, keep_accents=__magic_name__				)
										tokenizer.save_pretrained(self.tmpdirname				)






						def      __snake_case      (		self  :int				)      ->Any:
										lowercase    :      Dict       = """[PAD]"""
										lowercase    :      Tuple       = 0

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(__magic_name__				)			, __magic_name__				)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(__magic_name__				)			, __magic_name__				)






						def      __snake_case      (		self  :Any				)      ->Union[str, Any]:
										lowercase    :      Optional[int]       = list(self.get_tokenizer().get_vocab().keys()				)

										self.assertEqual(vocab_keys[0]			, """[PAD]"""				)
										self.assertEqual(vocab_keys[1]			, """[CLS]"""				)
										self.assertEqual(vocab_keys[-1]			, """j"""				)
										self.assertEqual(len(__magic_name__				)			, 1_012				)






						def      __snake_case      (		self  :int				)      ->int:
										self.assertEqual(self.get_tokenizer().vocab_size			, 1_012				)






						def      __snake_case      (		self  :Optional[Any]				)      ->List[Any]:
										lowercase    :      Optional[Any]       = XLMProphetNetTokenizer(__magic_name__			, keep_accents=__magic_name__				)

										lowercase    :      Union[str, Any]       = tokenizer.tokenize("""This is a test"""				)
										self.assertListEqual(__magic_name__			, ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]				)

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(__magic_name__				)			, [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			, )

										lowercase    :      Optional[Any]       = tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
										self.assertListEqual(
										    __magic_name__			, [
										        SPIECE_UNDERLINE + """I""",
										        SPIECE_UNDERLINE + """was""",
										        SPIECE_UNDERLINE + """b""",
										        """or""",
										        """n""",
										        SPIECE_UNDERLINE + """in""",
										        SPIECE_UNDERLINE + """""",
										        """9""",
										        """2""",
										        """0""",
										        """0""",
										        """0""",
										        """,""",
										        SPIECE_UNDERLINE + """and""",
										        SPIECE_UNDERLINE + """this""",
										        SPIECE_UNDERLINE + """is""",
										        SPIECE_UNDERLINE + """f""",
										        """al""",
										        """s""",
										        """é""",
										        """.""",
										    ]			, )
										lowercase    :      Optional[int]       = tokenizer.convert_tokens_to_ids(__magic_name__				)
										self.assertListEqual(
										    __magic_name__			, [
										        value + tokenizer.fairseq_offset
										        for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
										    ]			, )

										lowercase    :      Union[str, Any]       = tokenizer.convert_ids_to_tokens(__magic_name__				)
										self.assertListEqual(
										    __magic_name__			, [
										        SPIECE_UNDERLINE + """I""",
										        SPIECE_UNDERLINE + """was""",
										        SPIECE_UNDERLINE + """b""",
										        """or""",
										        """n""",
										        SPIECE_UNDERLINE + """in""",
										        SPIECE_UNDERLINE + """""",
										        """[UNK]""",
										        """2""",
										        """0""",
										        """0""",
										        """0""",
										        """,""",
										        SPIECE_UNDERLINE + """and""",
										        SPIECE_UNDERLINE + """this""",
										        SPIECE_UNDERLINE + """is""",
										        SPIECE_UNDERLINE + """f""",
										        """al""",
										        """s""",
										        """[UNK]""",
										        """.""",
										    ]			, )






						@cached_property
						def      __snake_case      (		self  :List[Any]				)      ->str:
										return XLMProphetNetTokenizer.from_pretrained("""microsoft/xprophetnet-large-wiki100-cased"""				)






						@slow
						def      __snake_case      (		self  :int				)      ->Optional[Any]:
										lowercase    :      str       = """Hello World!"""
										lowercase    :      Optional[Any]       = [35_389, 6_672, 49, 2]
										self.assertListEqual(__magic_name__			, self.big_tokenizer.encode(__magic_name__				)				)






						@slow
						def      __snake_case      (		self  :Optional[int]				)      ->Union[str, Any]:
										# fmt: off
										lowercase    :      Optional[int]       = {"""input_ids""": [[11_073, 82_783, 18, 26, 82_783, 549, 51_540, 248, 17_209, 1_301, 217, 20, 215_186, 1_325, 147, 17_209, 1_301, 217, 20, 56_370, 53, 122_020, 20, 16_477, 27, 87_355, 4_548, 20, 4_728, 78_392, 17, 159_969, 18, 26, 24_491, 629, 15, 538, 22_704, 5_439, 15, 2_788, 24_491, 9_885, 15, 43_534, 605, 15, 814, 18_403, 33_200, 29, 15, 43_534, 24_458, 12_410, 111, 24_966, 83_669, 9_637, 144_068, 26, 850, 22_346, 27, 147, 24_966, 83_669, 83_490, 26, 39_113, 735, 27, 689, 656, 2_800, 1_339, 4_600, 53, 122_020, 115_785, 34, 816, 1_339, 46_887, 18, 147, 53_905, 1_951, 42_238, 41_170, 17_732, 834, 436, 15, 27_523, 98_733, 217, 147, 5_542, 4_981, 930, 17_347, 16, 2], [20_091, 629, 94, 82_786, 58, 490, 20, 1_528, 84, 53_905, 344, 80_592, 110_128, 18_822, 5_267, 1_306, 62, 152_537, 308, 7_997, 401, 124_427, 549, 35_442, 225, 109, 15_055, 25_748, 147, 7_119, 43_712, 34, 767, 135_366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63_784, 119_466, 17, 147_808, 88_214, 18, 656, 81, 32, 3_296, 10_280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=__magic_name__			, model_name="""microsoft/xprophetnet-large-wiki100-cased"""			, revision="""1acad1643ddd54a44df6a1b797ada8373685d90e"""			, )
