





'''simple docstring'''






from __future__ import annotations

import random
import unittest

from transformers import TransfoXLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import (
         TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
         TFTransfoXLForSequenceClassification,
         TFTransfoXLLMHeadModel,
         TFTransfoXLModel,
     )


class   _lowercase						:


   def __init__( self    ,   _UpperCAmelCase    ,   ):
       A       :       int							=						parent
       A       :       int							=						13
       A       :       int							=						7
       A       :       List[Any]							=						30
       A       :       Any							=						self.seq_length + self.mem_len
       A       :       List[Any]							=						15
       A       :       Union[str, Any]							=						True
       A       :       List[Any]							=						True
       A       :       Optional[Any]							=						99
       A       :       List[str]							=						[10, 50, 80]
       A       :       Dict							=						32
       A       :       Optional[int]							=						32
       A       :       List[Any]							=						4
       A       :       Dict							=						8
       A       :       str							=						128
       A       :       str							=						2
       A       :       int							=						2
       A       :       str							=						None
       A       :       Dict							=						1
       A       :       Optional[Any]							=						0
       A       :       Any							=						3
       A       :       Any							=						self.vocab_size - 1
       A       :       Optional[Any]							=						0.01


   def 					snake_case				( self					):
       A       :       Union[str, Any]							=						ids_tensor([self.batch_size, self.seq_length]    ,   self.vocab_size					)
       A       :       Optional[int]							=						ids_tensor([self.batch_size, self.seq_length]    ,   self.vocab_size					)

       A       :       str							=						None
       if self.use_labels:
           A       :       Optional[int]							=						ids_tensor([self.batch_size, self.seq_length]    ,   self.vocab_size					)

       A       :       Optional[int]							=						TransfoXLConfig(
           vocab_size=self.vocab_size    ,   mem_len=self.mem_len    ,   clamp_len=self.clamp_len    ,   cutoffs=self.cutoffs    ,   d_model=self.hidden_size    ,   d_embed=self.d_embed    ,   n_head=self.num_attention_heads    ,   d_head=self.d_head    ,   d_inner=self.d_inner    ,   div_val=self.div_val    ,   n_layer=self.num_hidden_layers    ,   eos_token_id=self.eos_token_id    ,   pad_token_id=self.vocab_size - 1    ,   init_range=self.init_range    ,   num_labels=self.num_labels    ,   )

       return (config, input_ids_a, input_ids_a, lm_labels)


   def 					snake_case				( self					):
       random.seed(self.seed					)
       tf.random.set_seed(self.seed					)


   def 					snake_case				( self    ,   _UpperCAmelCase    ,   _UpperCAmelCase    ,   _UpperCAmelCase    ,   _UpperCAmelCase					):
       A       :       Tuple							=						TFTransfoXLModel(_UpperCAmelCase					)

       A,	A       :       Any							=						model(_UpperCAmelCase					).to_tuple()

       A       :       Optional[int]							=						{'''input_ids''': input_ids_a, '''mems''': mems_a}

       A,	A       :       List[str]							=						model(_UpperCAmelCase					).to_tuple()

       self.parent.assertEqual(hidden_states_a.shape    ,   (self.batch_size, self.seq_length, self.hidden_size)					)
       self.parent.assertEqual(hidden_states_a.shape    ,   (self.batch_size, self.seq_length, self.hidden_size)					)
       self.parent.assertListEqual(
           [mem.shape for mem in mems_a]    ,   [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers    ,   )
       self.parent.assertListEqual(
           [mem.shape for mem in mems_a]    ,   [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers    ,   )


   def 					snake_case				( self    ,   _UpperCAmelCase    ,   _UpperCAmelCase    ,   _UpperCAmelCase    ,   _UpperCAmelCase					):
       A       :       Union[str, Any]							=						TFTransfoXLLMHeadModel(_UpperCAmelCase					)

       A,	A       :       Union[str, Any]							=						model(_UpperCAmelCase					).to_tuple()

       A       :       Union[str, Any]							=						{'''input_ids''': input_ids_a, '''labels''': lm_labels}
       A,	A       :       Any							=						model(_UpperCAmelCase					).to_tuple()

       A,	A       :       Tuple							=						model([input_ids_a, mems_a]					).to_tuple()

       A       :       List[str]							=						{'''input_ids''': input_ids_a, '''mems''': mems_a, '''labels''': lm_labels}

       A,	A       :       Dict							=						model(_UpperCAmelCase					).to_tuple()

       self.parent.assertEqual(lm_logits_a.shape    ,   (self.batch_size, self.seq_length, self.vocab_size)					)
       self.parent.assertListEqual(
           [mem.shape for mem in mems_a]    ,   [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers    ,   )

       self.parent.assertEqual(lm_logits_a.shape    ,   (self.batch_size, self.seq_length, self.vocab_size)					)
       self.parent.assertListEqual(
           [mem.shape for mem in mems_a]    ,   [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers    ,   )


   def 					snake_case				( self    ,   _UpperCAmelCase    ,   _UpperCAmelCase    ,   _UpperCAmelCase    ,   _UpperCAmelCase					):
       A       :       Optional[int]							=						TFTransfoXLForSequenceClassification(_UpperCAmelCase					)
       A       :       List[str]							=						model(_UpperCAmelCase					)
       self.parent.assertEqual(result.logits.shape    ,   (self.batch_size, self.num_labels)					)


   def 					snake_case				( self					):
       A       :       str							=						self.prepare_config_and_inputs()
       ((A),	(A),	(A),	(A))       :       Dict							=						config_and_inputs
       A       :       Optional[int]							=						{'''input_ids''': input_ids_a}
       return config, inputs_dict







@require_tf
class   _lowercase						(							snake_case_		,					snake_case_		,					unittest.TestCase				):
   _UpperCamelCase       		=   (
       (TFTransfoXLModel, TFTransfoXLLMHeadModel, TFTransfoXLForSequenceClassification) if is_tf_available() else ()
   )
   _UpperCamelCase       		=   () if is_tf_available() else ()
   _UpperCamelCase       		=   (
       {
           """feature-extraction""": TFTransfoXLModel,
           """text-classification""": TFTransfoXLForSequenceClassification,
           """text-generation""": TFTransfoXLLMHeadModel,
           """zero-shot""": TFTransfoXLForSequenceClassification,
       }
       if is_tf_available()
       else {}
   )
   # TODO: add this test when TFTransfoXLLMHead has a linear output layer implemented
   _UpperCamelCase       		=   False
   _UpperCamelCase       		=   False
   _UpperCamelCase       		=   False
   _UpperCamelCase       		=   False


   def 					snake_case				( self    ,   _UpperCAmelCase    ,   _UpperCAmelCase    ,   _UpperCAmelCase    ,   _UpperCAmelCase    ,   _UpperCAmelCase					):
       if pipeline_test_casse_name == "TextGenerationPipelineTests":
           # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`.
           # `TransfoXLConfig` was never used in pipeline tests: cannot create a simple
           # tokenizer.
           return True

       return False


   def 					snake_case				( self					):
       A       :       List[Any]							=						TFTransfoXLModelTester(self					)
       A       :       Optional[Any]							=						ConfigTester(self    ,   config_class=_UpperCAmelCase    ,   d_embed=37					)


   def 					snake_case				( self					):
       self.config_tester.run_common_tests()


   def 					snake_case				( self					):
       self.model_tester.set_seed()
       A       :       List[Any]							=						self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_transfo_xl_model(*_UpperCAmelCase					)


   def 					snake_case				( self					):
       self.model_tester.set_seed()
       A       :       Optional[int]							=						self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_transfo_xl_lm_head(*_UpperCAmelCase					)


   def 					snake_case				( self					):
       A       :       Dict							=						self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_transfo_xl_for_sequence_classification(*_UpperCAmelCase					)


   def 					snake_case				( self					):
       A,	A       :       int							=						self.model_tester.prepare_config_and_inputs_for_common()
       A       :       Dict							=						[TFTransfoXLForSequenceClassification]

       for model_class in self.all_model_classes:
           A       :       Union[str, Any]							=						model_class(_UpperCAmelCase					)
           assert isinstance(model.get_input_embeddings()    ,   tf.keras.layers.Layer					)
           if model_class in list_other_models_with_output_ebd:
               A       :       int							=						model.get_output_embeddings()
               assert isinstance(_UpperCAmelCase    ,   tf.keras.layers.Layer					)
               A       :       int							=						model.get_bias()
               assert name is None
           else:
               A       :       Dict							=						model.get_output_embeddings()
               assert x is None
               A       :       Dict							=						model.get_bias()
               assert name is None


   def 					snake_case				( self					):
       # TODO JP: Make TransfoXL XLA compliant
       pass


   @slow
   def 					snake_case				( self					):
       for model_name in TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           A       :       Tuple							=						TFTransfoXLModel.from_pretrained(_UpperCAmelCase					)
           self.assertIsNotNone(_UpperCAmelCase					)


   @unittest.skip(reason='''This model doesn\'t play well with fit() due to not returning a single loss.'''					)
   def 					snake_case				( self					):
       pass







@require_tf
class   _lowercase						(							unittest.TestCase				):


   @unittest.skip('''Skip test until #12651 is resolved.'''					)
   @slow
   def 					snake_case				( self					):
       A       :       Dict							=						TFTransfoXLLMHeadModel.from_pretrained('''transfo-xl-wt103'''					)
       # fmt: off
       A       :       List[Any]							=						tf.convert_to_tensor([[33,1_297,2,1,1_009,4,1_109,11_739,4_762,358,5,25,245,22,1_706,17,20_098,5,3_215,21,37,1_110,3,13,1_041,4,24,603,490,2,71_477,20_098,104_447,2,20_961,1,2_604,4,1,329,3,6_224,831,16_002,2,8,603,78_967,29_546,23,803,20,25,416,5,8,232,4,277,6,1_855,4_601,3,29_546,54,8,3_609,5,57_211,49,4,1,277,18,8,1_755,15_691,3,341,25,416,693,42_573,71,17,401,94,31,17_919,2,29_546,7_873,18,1,435,23,11_011,755,5,5_167,3,7_983,98,84,2,29_546,3_267,8,3_609,4,1,4_865,1_075,2,6_087,71,6,346,8,5_854,3,29_546,824,1_400,1_868,2,19,160,2,311,8,5_496,2,20_920,17,25,15_097,3,24,24,0]]    ,   dtype=tf.intaa					)  # noqa: E231
       # fmt: on
       #  In 1991 , the remains of Russian Tsar Nicholas II and his family
       #  ( except for Alexei and Maria ) are discovered .
       #  The voice of Nicholas's young son , Tsarevich Alexei Nikolaevich , narrates the
       #  remainder of the story . 1883 Western Siberia ,
       #  a young Grigori Rasputin is asked by his father and a group of men to perform magic .
       #  Rasputin has a vision and denounces one of the men as a horse thief . Although his
       #  father initially slaps him for making such an accusation , Rasputin watches as the
       #  man is chased outside and beaten . Twenty years later , Rasputin sees a vision of
       #  the Virgin Mary , prompting him to become a priest . Rasputin quickly becomes famous ,
       #  with people , even a bishop , begging for his blessing . <eod> </s> <eos>

       # fmt: off
       A       :       List[str]							=						[33,1_297,2,1,1_009,4,1_109,11_739,4_762,358,5,25,245,22,1_706,17,20_098,5,3_215,21,37,1_110,3,13,1_041,4,24,603,490,2,71_477,20_098,104_447,2,20_961,1,2_604,4,1,329,3,6_224,831,16_002,2,8,603,78_967,29_546,23,803,20,25,416,5,8,232,4,277,6,1_855,4_601,3,29_546,54,8,3_609,5,57_211,49,4,1,277,18,8,1_755,15_691,3,341,25,416,693,42_573,71,17,401,94,31,17_919,2,29_546,7_873,18,1,435,23,11_011,755,5,5_167,3,7_983,98,84,2,29_546,3_267,8,3_609,4,1,4_865,1_075,2,6_087,71,6,346,8,5_854,3,29_546,824,1_400,1_868,2,19,160,2,311,8,5_496,2,20_920,17,25,15_097,3,24,24,0,33,1,1_857,2,1,1_009,4,1_109,11_739,4_762,358,5,25,245,28,1_110,3,13,1_041,4,24,603,490,2,71_477,20_098,104_447,2,20_961,1,2_604,4,1,329,3,0]  # noqa: E231
       # fmt: on
       #  In 1991, the remains of Russian Tsar Nicholas II and his family (
       #  except for Alexei and Maria ) are discovered. The voice of young son,
       #  Tsarevich Alexei Nikolaevich, narrates the remainder of the story.
       #  1883 Western Siberia, a young Grigori Rasputin is asked by his father
       #  and a group of men to perform magic. Rasputin has a vision and
       #  denounces one of the men as a horse thief. Although his father initially
       #  slaps him for making such an accusation, Rasputin watches as the man
       #  is chased outside and beaten. Twenty years later, Rasputin sees a vision
       #  of the Virgin Mary, prompting him to become a priest.
       #  Rasputin quickly becomes famous, with people, even a bishop, begging for
       #  his blessing. <unk> <unk> <eos> In the 1990s, the remains of Russian Tsar
       # Nicholas II and his family were discovered. The voice of <unk> young son,
       # Tsarevich Alexei Nikolaevich, narrates the remainder of the story.<eos>

       A       :       Union[str, Any]							=						model.generate(_UpperCAmelCase    ,   max_length=200    ,   do_sample=_UpperCAmelCase					)
       self.assertListEqual(output_ids[0].numpy().tolist()    ,   _UpperCAmelCase					)







'''simple docstring'''






import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


snake_case_     =	get_tests_dir("""fixtures/test_sentencepiece.model""")


@require_sentencepiece
class   _lowercase						(							a		,					unittest.TestCase				):
   _UpperCamelCase       		=   XLMProphetNetTokenizer
   _UpperCamelCase       		=   False
   _UpperCamelCase       		=   True


   def 					snake_case				( self					):
       super().setUp()

       # We have a SentencePiece fixture for testing
       A       :       Dict							=						XLMProphetNetTokenizer(_UpperCAmelCase    ,   keep_accents=_UpperCAmelCase					)
       tokenizer.save_pretrained(self.tmpdirname					)


   def 					snake_case				( self					):
       A       :       Union[str, Any]							=						'''[PAD]'''
       A       :       Optional[int]							=						0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCAmelCase					)    ,   _UpperCAmelCase					)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCAmelCase					)    ,   _UpperCAmelCase					)


   def 					snake_case				( self					):
       A       :       Union[str, Any]							=						list(self.get_tokenizer().get_vocab().keys()					)

       self.assertEqual(vocab_keys[0]    ,   '''[PAD]'''					)
       self.assertEqual(vocab_keys[1]    ,   '''[CLS]'''					)
       self.assertEqual(vocab_keys[-1]    ,   '''j'''					)
       self.assertEqual(len(_UpperCAmelCase					)    ,   1_012					)


   def 					snake_case				( self					):
       self.assertEqual(self.get_tokenizer().vocab_size    ,   1_012					)


   def 					snake_case				( self					):
       A       :       Dict							=						XLMProphetNetTokenizer(_UpperCAmelCase    ,   keep_accents=_UpperCAmelCase					)

       A       :       int							=						tokenizer.tokenize('''This is a test'''					)
       self.assertListEqual(_UpperCAmelCase    ,   ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']					)

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(_UpperCAmelCase					)    ,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]    ,   )

       A       :       Optional[Any]							=						tokenizer.tokenize('''I was born in 92000, and this is falsé.'''					)
       self.assertListEqual(
           _UpperCAmelCase    ,   [
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''9''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''é''',
               '''.''',
           ]    ,   )
       A       :       int							=						tokenizer.convert_tokens_to_ids(_UpperCAmelCase					)
       self.assertListEqual(
           _UpperCAmelCase    ,   [
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
           ]    ,   )

       A       :       Union[str, Any]							=						tokenizer.convert_ids_to_tokens(_UpperCAmelCase					)
       self.assertListEqual(
           _UpperCAmelCase    ,   [
               SPIECE_UNDERLINE + '''I''',
               SPIECE_UNDERLINE + '''was''',
               SPIECE_UNDERLINE + '''b''',
               '''or''',
               '''n''',
               SPIECE_UNDERLINE + '''in''',
               SPIECE_UNDERLINE + '''''',
               '''[UNK]''',
               '''2''',
               '''0''',
               '''0''',
               '''0''',
               ''',''',
               SPIECE_UNDERLINE + '''and''',
               SPIECE_UNDERLINE + '''this''',
               SPIECE_UNDERLINE + '''is''',
               SPIECE_UNDERLINE + '''f''',
               '''al''',
               '''s''',
               '''[UNK]''',
               '''.''',
           ]    ,   )


   @cached_property
   def 					snake_case				( self					):
       return XLMProphetNetTokenizer.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased'''					)


   @slow
   def 					snake_case				( self					):
       A       :       Union[str, Any]							=						'''Hello World!'''
       A       :       Any							=						[35_389, 6_672, 49, 2]
       self.assertListEqual(_UpperCAmelCase    ,   self.big_tokenizer.encode(_UpperCAmelCase					)					)


   @slow
   def 					snake_case				( self					):
       # fmt: off
       A       :       List[Any]							=						{'''input_ids''': [[11_073, 82_783, 18, 26, 82_783, 549, 51_540, 248, 17_209, 1_301, 217, 20, 215_186, 1_325, 147, 17_209, 1_301, 217, 20, 56_370, 53, 122_020, 20, 16_477, 27, 87_355, 4_548, 20, 4_728, 78_392, 17, 159_969, 18, 26, 24_491, 629, 15, 538, 22_704, 5_439, 15, 2_788, 24_491, 9_885, 15, 43_534, 605, 15, 814, 18_403, 33_200, 29, 15, 43_534, 24_458, 12_410, 111, 24_966, 83_669, 9_637, 144_068, 26, 850, 22_346, 27, 147, 24_966, 83_669, 83_490, 26, 39_113, 735, 27, 689, 656, 2_800, 1_339, 4_600, 53, 122_020, 115_785, 34, 816, 1_339, 46_887, 18, 147, 53_905, 1_951, 42_238, 41_170, 17_732, 834, 436, 15, 27_523, 98_733, 217, 147, 5_542, 4_981, 930, 17_347, 16, 2], [20_091, 629, 94, 82_786, 58, 490, 20, 1_528, 84, 53_905, 344, 80_592, 110_128, 18_822, 5_267, 1_306, 62, 152_537, 308, 7_997, 401, 124_427, 549, 35_442, 225, 109, 15_055, 25_748, 147, 7_119, 43_712, 34, 767, 135_366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63_784, 119_466, 17, 147_808, 88_214, 18, 656, 81, 32, 3_296, 10_280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=_UpperCAmelCase    ,   model_name='''microsoft/xprophetnet-large-wiki100-cased'''    ,   revision='''1acad1643ddd54a44df6a1b797ada8373685d90e'''    ,   )
