


'''simple docstring'''






import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import BlenderbotConfig, is_flax_available
from transformers.testing_utils import jax_device, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
							import os

							# The slow tests are often failing with OOM error on GPU
							# This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
							# but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
							__A	:	List[str]     =     '''platform'''

							import jax
							import jax.numpy as jnp

							from transformers import BlenderbotTokenizer
							from transformers.models.blenderbot.modeling_flax_blenderbot import (
							    FlaxBlenderbotForConditionalGeneration,
							    FlaxBlenderbotModel,
							    shift_tokens_right,
							)







def      UpperCAmelCase      (		lowerCamelCase_  :Union[str, Any]				,      lowerCamelCase_  :List[str]				,      lowerCamelCase_  :Optional[Any]=None				,      lowerCamelCase_  :List[Any]=None				,      lowerCamelCase_  :List[str]=None				,      lowerCamelCase_  :Optional[int]=None				,      lowerCamelCase_  :Tuple=None				,      lowerCamelCase_  :str=None				,      ):







		'''simple docstring'''






		if attention_mask is None:
				snake_case_		:							str			       =						np.where(input_ids != config.pad_token_id				,      1				,      0 )
		if decoder_attention_mask is None:
				snake_case_		:							Dict			       =						np.where(decoder_input_ids != config.pad_token_id				,      1				,      0 )
		if head_mask is None:
				snake_case_		:							Union[str, Any]			       =						np.ones((config.encoder_layers, config.encoder_attention_heads) )
		if decoder_head_mask is None:
				snake_case_		:							Tuple			       =						np.ones((config.decoder_layers, config.decoder_attention_heads) )
		if cross_attn_head_mask is None:
				snake_case_		:							List[Any]			       =						np.ones((config.decoder_layers, config.decoder_attention_heads) )
		return {
		    "input_ids": input_ids,
		    "decoder_input_ids": decoder_input_ids,
		    "attention_mask": attention_mask,
		    "decoder_attention_mask": attention_mask,
		}
class 						__UpperCamelCase	:





							def __init__(  self		:Tuple  ,_UpperCamelCase		:str  ,_UpperCamelCase		:Optional[int]=1_3  ,_UpperCamelCase		:Any=7  ,_UpperCamelCase		:List[str]=True  ,_UpperCamelCase		:str=False  ,_UpperCamelCase		:Dict=9_9  ,_UpperCamelCase		:Union[str, Any]=1_6  ,_UpperCamelCase		:str=2  ,_UpperCamelCase		:Optional[int]=4  ,_UpperCamelCase		:int=4  ,_UpperCamelCase		:Dict="gelu"  ,_UpperCamelCase		:Union[str, Any]=0.1  ,_UpperCamelCase		:Any=0.1  ,_UpperCamelCase		:Any=3_2  ,_UpperCamelCase		:List[str]=2  ,_UpperCamelCase		:Union[str, Any]=1  ,_UpperCamelCase		:Dict=0  ,_UpperCamelCase		:Union[str, Any]=0.02  ,):
									snake_case_		:							Any			       =						parent
									snake_case_		:							Optional[Any]			       =						batch_size
									snake_case_		:							Any			       =						seq_length
									snake_case_		:							Tuple			       =						is_training
									snake_case_		:							Optional[Any]			       =						use_labels
									snake_case_		:							int			       =						vocab_size
									snake_case_		:							List[Any]			       =						hidden_size
									snake_case_		:							int			       =						num_hidden_layers
									snake_case_		:							Optional[Any]			       =						num_attention_heads
									snake_case_		:							List[str]			       =						intermediate_size
									snake_case_		:							Optional[int]			       =						hidden_act
									snake_case_		:							Optional[int]			       =						hidden_dropout_prob
									snake_case_		:							Tuple			       =						attention_probs_dropout_prob
									snake_case_		:							Dict			       =						max_position_embeddings
									snake_case_		:							Optional[Any]			       =						eos_token_id
									snake_case_		:							Tuple			       =						pad_token_id
									snake_case_		:							Dict			       =						bos_token_id
									snake_case_		:							Union[str, Any]			       =						initializer_range





							def 		a__   (  self		:Tuple			):
									snake_case_		:							Dict			       =						np.clip(ids_tensor([self.batch_size, self.seq_length - 1]  ,self.vocab_size			)  ,3  ,self.vocab_size			)
									snake_case_		:							Optional[Any]			       =						np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1)  ,dtype=np.intaa			))  ,-1			)

									snake_case_		:							Union[str, Any]			       =						shift_tokens_right(lowerCAmelCase__  ,1  ,2			)

									snake_case_		:							List[str]			       =						BlenderbotConfig(
									    vocab_size=self.vocab_size  ,d_model=self.hidden_size  ,encoder_layers=self.num_hidden_layers  ,decoder_layers=self.num_hidden_layers  ,encoder_attention_heads=self.num_attention_heads  ,decoder_attention_heads=self.num_attention_heads  ,encoder_ffn_dim=self.intermediate_size  ,decoder_ffn_dim=self.intermediate_size  ,dropout=self.hidden_dropout_prob  ,attention_dropout=self.attention_probs_dropout_prob  ,max_position_embeddings=self.max_position_embeddings  ,eos_token_id=self.eos_token_id  ,bos_token_id=self.bos_token_id  ,pad_token_id=self.pad_token_id  ,initializer_range=self.initializer_range  ,use_cache=lowerCAmelCase__  ,)
									snake_case_		:							Optional[int]			       =						prepare_blenderbot_inputs_dict(lowerCAmelCase__  ,lowerCAmelCase__  ,lowerCAmelCase__			)
									return config, inputs_dict





							def 		a__   (  self		:List[str]			):
									snake_case_		:							List[str]			       =						self.prepare_config_and_inputs()
									return config, inputs_dict





							def 		a__   (  self		:Dict  ,_UpperCamelCase		:Optional[int]  ,_UpperCamelCase		:Dict  ,_UpperCamelCase		:Optional[Any]			):
									snake_case_		:							Dict			       =						2_0
									snake_case_		:							Dict			       =						model_class_name(lowerCAmelCase__			)

									snake_case_		:							Optional[Any]			       =						model.encode(inputs_dict["""input_ids"""]			)

									snake_case_		:							Optional[int]			       =						(
									    inputs_dict["decoder_input_ids"],
									    inputs_dict["decoder_attention_mask"],
									)

									snake_case_		:							List[Any]			       =						model.init_cache(decoder_input_ids.shape[0]  ,lowerCAmelCase__  ,lowerCAmelCase__			)
									snake_case_		:							List[Any]			       =						jnp.ones((decoder_input_ids.shape[0], max_decoder_length)  ,dtype="""i4"""			)

									snake_case_		:							Tuple			       =						jnp.broadcast_to(
									    jnp.arange(decoder_input_ids.shape[-1] - 1			)[None, :]  ,(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)  ,)
									snake_case_		:							List[Any]			       =						model.decode(
									    decoder_input_ids[:, :-1]  ,lowerCAmelCase__  ,decoder_attention_mask=lowerCAmelCase__  ,past_key_values=lowerCAmelCase__  ,decoder_position_ids=lowerCAmelCase__  ,)

									snake_case_		:							Tuple			       =						jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]  ,dtype="""i4"""			)
									snake_case_		:							Dict			       =						model.decode(
									    decoder_input_ids[:, -1:]  ,lowerCAmelCase__  ,decoder_attention_mask=lowerCAmelCase__  ,past_key_values=outputs_cache.past_key_values  ,decoder_position_ids=lowerCAmelCase__  ,)

									snake_case_		:							List[Any]			       =						model.decode(lowerCAmelCase__  ,lowerCAmelCase__			)

									snake_case_		:							List[str]			       =						np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])			)			)
									self.parent.assertTrue(diff < 1E-3  ,msg=F'''Max diff is {diff}'''			)





							def 		a__   (  self		:Union[str, Any]  ,_UpperCamelCase		:str  ,_UpperCamelCase		:Union[str, Any]  ,_UpperCamelCase		:List[Any]			):
									snake_case_		:							List[Any]			       =						2_0
									snake_case_		:							Optional[Any]			       =						model_class_name(lowerCAmelCase__			)

									snake_case_		:							Optional[int]			       =						model.encode(inputs_dict["""input_ids"""]			)

									snake_case_		:							Union[str, Any]			       =						(
									    inputs_dict["decoder_input_ids"],
									    inputs_dict["decoder_attention_mask"],
									)

									snake_case_		:							List[str]			       =						jnp.concatenate(
									    [
									        decoder_attention_mask,
									        jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1])			),
									    ]  ,axis=-1  ,)

									snake_case_		:							Union[str, Any]			       =						model.init_cache(decoder_input_ids.shape[0]  ,lowerCAmelCase__  ,lowerCAmelCase__			)
									snake_case_		:							Dict			       =						jnp.broadcast_to(
									    jnp.arange(decoder_input_ids.shape[-1] - 1			)[None, :]  ,(decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1)  ,)

									snake_case_		:							int			       =						model.decode(
									    decoder_input_ids[:, :-1]  ,lowerCAmelCase__  ,decoder_attention_mask=lowerCAmelCase__  ,past_key_values=lowerCAmelCase__  ,decoder_position_ids=lowerCAmelCase__  ,)
									snake_case_		:							List[Any]			       =						jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]]  ,dtype="""i4"""			)
									snake_case_		:							List[Any]			       =						model.decode(
									    decoder_input_ids[:, -1:]  ,lowerCAmelCase__  ,past_key_values=outputs_cache.past_key_values  ,decoder_attention_mask=lowerCAmelCase__  ,decoder_position_ids=lowerCAmelCase__  ,)

									snake_case_		:							Union[str, Any]			       =						model.decode(lowerCAmelCase__  ,lowerCAmelCase__  ,decoder_attention_mask=lowerCAmelCase__			)

									snake_case_		:							Union[str, Any]			       =						np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])			)			)
									self.parent.assertTrue(diff < 1E-3  ,msg=F'''Max diff is {diff}'''			)





@require_flax
class 						__UpperCamelCase	(						unittest.TestCase							):
							lowercase			:		int   		=  9_9





							def 		a__   (  self		:str			):
									snake_case_		:							int			       =						np.array(
									    [
									        [7_1, 8_2, 1_8, 3_3, 4_6, 9_1, 2],
									        [6_8, 3_4, 2_6, 5_8, 3_0, 8_2, 2],
									        [5, 9_7, 1_7, 3_9, 9_4, 4_0, 2],
									        [7_6, 8_3, 9_4, 2_5, 7_0, 7_8, 2],
									        [8_7, 5_9, 4_1, 3_5, 4_8, 6_6, 2],
									        [5_5, 1_3, 1_6, 5_8, 5, 2, 1],  # note padding
									        [6_4, 2_7, 3_1, 5_1, 1_2, 7_5, 2],
									        [5_2, 6_4, 8_6, 1_7, 8_3, 3_9, 2],
									        [4_8, 6_1, 9, 2_4, 7_1, 8_2, 2],
									        [2_6, 1, 6_0, 4_8, 2_2, 1_3, 2],
									        [2_1, 5, 6_2, 2_8, 1_4, 7_6, 2],
									        [4_5, 9_8, 3_7, 8_6, 5_9, 4_8, 2],
									        [7_0, 7_0, 5_0, 9, 2_8, 0, 2],
									    ]  ,dtype=np.intaa  ,)

									snake_case_		:							List[Any]			       =						input_ids.shape[0]
									snake_case_		:							Any			       =						BlenderbotConfig(
									    vocab_size=self.vocab_size  ,d_model=2_4  ,encoder_layers=2  ,decoder_layers=2  ,encoder_attention_heads=2  ,decoder_attention_heads=2  ,encoder_ffn_dim=3_2  ,decoder_ffn_dim=3_2  ,max_position_embeddings=4_8  ,eos_token_id=2  ,pad_token_id=1  ,bos_token_id=0  ,)
									return config, input_ids, batch_size





							def 		a__   (  self		:List[Any]			):
									snake_case_		:							str			       =						self._get_config_and_data()
									snake_case_		:							str			       =						FlaxBlenderbotForConditionalGeneration(lowerCAmelCase__			)
									snake_case_		:							List[str]			       =						lm_model(input_ids=lowerCAmelCase__			)
									snake_case_		:							Optional[Any]			       =						(batch_size, input_ids.shape[1], config.vocab_size)
									self.assertEqual(outputs["""logits"""].shape  ,lowerCAmelCase__			)





							def 		a__   (  self		:Union[str, Any]			):
									snake_case_		:							Dict			       =						BlenderbotConfig(
									    vocab_size=self.vocab_size  ,d_model=1_4  ,encoder_layers=2  ,decoder_layers=2  ,encoder_attention_heads=2  ,decoder_attention_heads=2  ,encoder_ffn_dim=8  ,decoder_ffn_dim=8  ,max_position_embeddings=4_8  ,)
									snake_case_		:							List[Any]			       =						FlaxBlenderbotForConditionalGeneration(lowerCAmelCase__			)
									snake_case_		:							List[Any]			       =						np.array([[7_1, 8_2, 1_8, 3_3, 4_6, 9_1, 2], [6_8, 3_4, 2_6, 5_8, 3_0, 2, 1]]  ,dtype=np.intaa			)
									snake_case_		:							str			       =						np.array([[8_2, 7_1, 8_2, 1_8, 2], [5_8, 6_8, 2, 1, 1]]  ,dtype=np.intaa			)
									snake_case_		:							List[Any]			       =						lm_model(input_ids=lowerCAmelCase__  ,decoder_input_ids=lowerCAmelCase__			)
									snake_case_		:							List[Any]			       =						(*summary.shape, config.vocab_size)
									self.assertEqual(outputs["""logits"""].shape  ,lowerCAmelCase__			)





							def 		a__   (  self		:Optional[int]			):
									snake_case_		:							Optional[int]			       =						np.array([[7_1, 8_2, 1_8, 3_3, 2, 1, 1], [6_8, 3_4, 2_6, 5_8, 3_0, 8_2, 2]]  ,dtype=np.intaa			)
									snake_case_		:							Optional[Any]			       =						shift_tokens_right(lowerCAmelCase__  ,1  ,2			)
									snake_case_		:							Dict			       =						np.equal(lowerCAmelCase__  ,1			).astype(np.floataa			).sum()
									snake_case_		:							Optional[Any]			       =						np.equal(lowerCAmelCase__  ,1			).astype(np.floataa			).sum()
									self.assertEqual(shifted.shape  ,input_ids.shape			)
									self.assertEqual(lowerCAmelCase__  ,n_pad_before - 1			)
									self.assertTrue(np.equal(shifted[:, 0]  ,2			).all()			)





@require_flax
class 						__UpperCamelCase	(						a__						,  unittest.TestCase						,  a__							):
							lowercase			:		Optional[int]   		=  True
							lowercase			:		Dict   		=  (
							    (
							        FlaxBlenderbotModel,
							        FlaxBlenderbotForConditionalGeneration,
							    )
							    if is_flax_available()
							    else ()
							)
							lowercase			:		Dict   		=  (FlaxBlenderbotForConditionalGeneration,) if is_flax_available() else ()





							def 		a__   (  self		:Optional[int]			):
									snake_case_		:							Union[str, Any]			       =						FlaxBlenderbotModelTester(self			)





							def 		a__   (  self		:int			):
									snake_case_		:							Tuple			       =						self.model_tester.prepare_config_and_inputs()
									for model_class in self.all_model_classes:
											self.model_tester.check_use_cache_forward(lowerCAmelCase__  ,lowerCAmelCase__  ,lowerCAmelCase__			)





							def 		a__   (  self		:str			):
									snake_case_		:							Dict			       =						self.model_tester.prepare_config_and_inputs()
									for model_class in self.all_model_classes:
											self.model_tester.check_use_cache_forward_with_attn_mask(lowerCAmelCase__  ,lowerCAmelCase__  ,lowerCAmelCase__			)





							def 		a__   (  self		:Union[str, Any]			):
									snake_case_		:							Any			       =						self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
											with self.subTest(model_class.__name__			):
													snake_case_		:							Any			       =						self._prepare_for_class(lowerCAmelCase__  ,lowerCAmelCase__			)
													snake_case_		:							int			       =						model_class(lowerCAmelCase__			)

													@jax.jit
													def encode_jitted(_UpperCamelCase		:Any  ,_UpperCamelCase		:int=None  ,**_UpperCamelCase		:Optional[int]			):
															return model.encode(input_ids=lowerCAmelCase__  ,attention_mask=lowerCAmelCase__			)

													with self.subTest("""JIT Enabled"""			):
															snake_case_		:							Union[str, Any]			       =						encode_jitted(**lowerCAmelCase__			).to_tuple()

													with self.subTest("""JIT Disabled"""			):
															with jax.disable_jit():
																	snake_case_		:							Optional[Any]			       =						encode_jitted(**lowerCAmelCase__			).to_tuple()

													self.assertEqual(len(lowerCAmelCase__			)  ,len(lowerCAmelCase__			)			)
													for jitted_output, output in zip(lowerCAmelCase__  ,lowerCAmelCase__			):
															self.assertEqual(jitted_output.shape  ,output.shape			)





							def 		a__   (  self		:Dict			):
									snake_case_		:							Optional[Any]			       =						self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
											with self.subTest(model_class.__name__			):
													snake_case_		:							Optional[int]			       =						model_class(lowerCAmelCase__			)
													snake_case_		:							List[str]			       =						model.encode(inputs_dict["""input_ids"""]  ,inputs_dict["""attention_mask"""]			)

													snake_case_		:							Dict			       =						{
													    "decoder_input_ids": inputs_dict["decoder_input_ids"],
													    "decoder_attention_mask": inputs_dict["decoder_attention_mask"],
													    "encoder_outputs": encoder_outputs,
													}

													@jax.jit
													def decode_jitted(_UpperCamelCase		:Optional[Any]  ,_UpperCamelCase		:List[str]  ,_UpperCamelCase		:List[Any]			):
															return model.decode(
															    decoder_input_ids=lowerCAmelCase__  ,decoder_attention_mask=lowerCAmelCase__  ,encoder_outputs=lowerCAmelCase__  ,)

													with self.subTest("""JIT Enabled"""			):
															snake_case_		:							List[str]			       =						decode_jitted(**lowerCAmelCase__			).to_tuple()

													with self.subTest("""JIT Disabled"""			):
															with jax.disable_jit():
																	snake_case_		:							Union[str, Any]			       =						decode_jitted(**lowerCAmelCase__			).to_tuple()

													self.assertEqual(len(lowerCAmelCase__			)  ,len(lowerCAmelCase__			)			)
													for jitted_output, output in zip(lowerCAmelCase__  ,lowerCAmelCase__			):
															self.assertEqual(jitted_output.shape  ,output.shape			)





							@slow
							def 		a__   (  self		:Tuple			):
									for model_class_name in self.all_model_classes:
											snake_case_		:							Optional[int]			       =						model_class_name.from_pretrained("""facebook/blenderbot-400M-distill"""			)
											# FlaxBlenderbotForSequenceClassification expects eos token in input_ids
											snake_case_		:							Union[str, Any]			       =						np.ones((1, 1)			) * model.config.eos_token_id
											snake_case_		:							str			       =						model(lowerCAmelCase__			)
											self.assertIsNotNone(lowerCAmelCase__			)





							@unittest.skipUnless(jax_device != """cpu"""  ,"""3B test too slow on CPU."""			)
							@slow
							def 		a__   (  self		:Optional[Any]			):
									snake_case_		:							Optional[Any]			       =						{"num_beams": 1, "early_stopping": True, "min_length": 1_5, "max_length": 2_5}
									snake_case_		:							Tuple			       =						{"skip_special_tokens": True, "clean_up_tokenization_spaces": True}

									snake_case_		:							Optional[int]			       =						FlaxBlenderbotForConditionalGeneration.from_pretrained("""facebook/blenderbot-3B"""  ,from_pt=lowerCAmelCase__			)
									snake_case_		:							Tuple			       =						BlenderbotTokenizer.from_pretrained("""facebook/blenderbot-3B"""			)

									snake_case_		:							Union[str, Any]			       =						["Sam"]
									snake_case_		:							Union[str, Any]			       =						tokenizer(lowerCAmelCase__  ,return_tensors="""jax"""			)

									snake_case_		:							int			       =						model.generate(**lowerCAmelCase__  ,**lowerCAmelCase__			)
									snake_case_		:							str			       =						"Sam is a great name. It means \"sun\" in Gaelic."

									snake_case_		:							Optional[int]			       =						tokenizer.batch_decode(lowerCAmelCase__  ,**lowerCAmelCase__			)
									assert generated_txt[0].strip() == tgt_text



'''simple docstring'''






import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin
class 						__UpperCamelCase	(						lowercase__						,  unittest.TestCase							):
							lowercase			:		Dict   		=  TransfoXLTokenizer
							lowercase			:		Optional[Any]   		=  False
							lowercase			:		Dict   		=  False





							def 		a__   (  self		:Union[str, Any]			):
									super().setUp()

									snake_case_		:							Optional[int]			       =						[
									    """<unk>""",
									    """[CLS]""",
									    """[SEP]""",
									    """want""",
									    """unwanted""",
									    """wa""",
									    """un""",
									    """running""",
									    """,""",
									    """low""",
									    """l""",
									]
									snake_case_		:							Optional[int]			       =						os.path.join(self.tmpdirname  ,VOCAB_FILES_NAMES["""vocab_file"""]			)
									with open(self.vocab_file  ,"""w"""  ,encoding="""utf-8"""			) as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]			)			)





							def 		a__   (  self		:List[Any]  ,**_UpperCamelCase		:Optional[Any]			):
									snake_case_		:							Tuple			       =						True
									return TransfoXLTokenizer.from_pretrained(self.tmpdirname  ,**_UpperCamelCase			)





							def 		a__   (  self		:Tuple  ,_UpperCamelCase		:Union[str, Any]			):
									snake_case_		:							Any			       =						"""<unk> UNwanted , running"""
									snake_case_		:							Optional[int]			       =						"""<unk> unwanted, running"""
									return input_text, output_text





							def 		a__   (  self		:Dict			):
									snake_case_		:							Dict			       =						TransfoXLTokenizer(vocab_file=self.vocab_file  ,lower_case=_UpperCamelCase			)

									snake_case_		:							Dict			       =						tokenizer.tokenize("""<unk> UNwanted , running"""			)
									self.assertListEqual(_UpperCamelCase  ,["""<unk>""", """unwanted""", """,""", """running"""]			)

									self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase			)  ,[0, 4, 8, 7]			)





							def 		a__   (  self		:Optional[Any]			):
									snake_case_		:							Dict			       =						TransfoXLTokenizer(lower_case=_UpperCamelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """			)  ,["""hello""", """!""", """how""", """are""", """you""", """?"""]			)





							def 		a__   (  self		:Any			):
									snake_case_		:							List[Any]			       =						TransfoXLTokenizer(lower_case=_UpperCamelCase			)

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """			)  ,["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]			)





							def 		a__   (  self		:List[str]			):
									snake_case_		:							str			       =						TransfoXLTokenizer(lower_case=_UpperCamelCase			)
									snake_case_		:							List[str]			       =						"""Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
									snake_case_		:							Optional[int]			       =						[
									    """Hello""",
									    """(""",
									    """bracket""",
									    """)""",
									    """and""",
									    """side""",
									    """@-@""",
									    """scrolled""",
									    """[""",
									    """and""",
									    """]""",
									    """Henry""",
									    """'s""",
									    """$""",
									    """5""",
									    """@,@""",
									    """000""",
									    """with""",
									    """3""",
									    """@.@""",
									    """34""",
									    """m""",
									    """.""",
									    """What""",
									    """'s""",
									    """up""",
									    """!""",
									    """?""",
									]

									self.assertListEqual(tokenizer.tokenize(_UpperCamelCase			)  ,_UpperCamelCase			)

									self.assertEqual(tokenizer.convert_tokens_to_string(_UpperCamelCase			)  ,_UpperCamelCase			)





							def 		a__   (  self		:Dict			):
									snake_case_		:							Union[str, Any]			       =						self.get_tokenizer()
									snake_case_		:							Dict			       =						len(_UpperCamelCase			)

									tokenizer.add_tokens(["""new1""", """new2"""]			)
									tokenizer.move_added_token("""new1"""  ,1			)

									# Check that moved token is not copied (duplicate)
									self.assertEqual(len(_UpperCamelCase			)  ,original_len + 2			)
									# Check that token is moved to specified id
									self.assertEqual(tokenizer.encode("""new1"""			)  ,[1]			)
									self.assertEqual(tokenizer.decode([1]			)  ,"""new1"""			)