from __future__ import annotations

import unittest

from transformers import LEDConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import TFLEDForConditionalGeneration, TFLEDModel
@require_tf
class      UpperCamelCase_						:
					lowerCAmelCase_							=		LEDConfig
					lowerCAmelCase_							=		{}
					lowerCAmelCase_							=		"""gelu"""




					def __init__(						self			,  lowerCAmelCase_			,  lowerCAmelCase_=13			,  lowerCAmelCase_=7			,  lowerCAmelCase_=True			,  lowerCAmelCase_=False			,  lowerCAmelCase_=99			,  lowerCAmelCase_=32			,  lowerCAmelCase_=2			,  lowerCAmelCase_=4			,  lowerCAmelCase_=37			,  lowerCAmelCase_=0.1			,  lowerCAmelCase_=0.1			,  lowerCAmelCase_=20			,  lowerCAmelCase_=2			,  lowerCAmelCase_=1			,  lowerCAmelCase_=0			,  lowerCAmelCase_=4			,  )						->							Optional[int]:
										_snake_case													=			parent
										_snake_case													=			batch_size
										_snake_case													=			seq_length
										_snake_case													=			is_training
										_snake_case													=			use_labels
										_snake_case													=			vocab_size
										_snake_case													=			hidden_size
										_snake_case													=			num_hidden_layers
										_snake_case													=			num_attention_heads
										_snake_case													=			intermediate_size

										_snake_case													=			hidden_dropout_prob
										_snake_case													=			attention_probs_dropout_prob
										_snake_case													=			max_position_embeddings
										_snake_case													=			eos_token_id
										_snake_case													=			pad_token_id
										_snake_case													=			bos_token_id
										_snake_case													=			attention_window

										# `ModelTesterMixin.test_attention_outputs` is expecting attention tensors to be of size
										# [num_attention_heads, encoder_seq_length, encoder_key_length], but TFLongformerSelfAttention
										# returns attention of shape [num_attention_heads, encoder_seq_length, self.attention_window + 1]
										# because its local attention only attends to `self.attention_window` and one before and one after
										_snake_case													=			self.attention_window + 2

										# because of padding `encoder_seq_length`, is different from `seq_length`. Relevant for
										# the `test_attention_outputs` and `test_hidden_states_output` tests
										_snake_case													=			(
										    self.seq_length + (self.attention_window - self.seq_length % self.attention_window) % self.attention_window
										)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			ids_tensor([self.batch_size, self.seq_length - 1]			,  self.vocab_size					)
										_snake_case													=			tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size					)			,  1					)
										_snake_case													=			tf.concat([input_ids, eos_tensor]			,  axis=1					)

										_snake_case													=			ids_tensor([self.batch_size, self.seq_length]			,  self.vocab_size					)

										_snake_case													=			self.config_cls(
										    vocab_size=self.vocab_size			,  d_model=self.hidden_size			,  encoder_layers=self.num_hidden_layers			,  decoder_layers=self.num_hidden_layers			,  encoder_attention_heads=self.num_attention_heads			,  decoder_attention_heads=self.num_attention_heads			,  encoder_ffn_dim=self.intermediate_size			,  decoder_ffn_dim=self.intermediate_size			,  dropout=self.hidden_dropout_prob			,  attention_dropout=self.attention_probs_dropout_prob			,  max_position_embeddings=self.max_position_embeddings			,  eos_token_ids=[2]			,  bos_token_id=self.bos_token_id			,  pad_token_id=self.pad_token_id			,  decoder_start_token_id=self.pad_token_id			,  attention_window=self.attention_window			,  **self.config_updates			,  )
										_snake_case													=			prepare_led_inputs_dict(_lowercase			,  _lowercase			,  _lowercase					)
										_snake_case													=			tf.concat(
										    [tf.zeros_like(_lowercase					)[:, :-1], tf.ones_like(_lowercase					)[:, -1:]]			,  axis=-1			,  )
										_snake_case													=			global_attention_mask
										return config, inputs_dict






					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_					)						->							Union[str, Any]:
										_snake_case													=			TFLEDModel(config=_lowercase					).get_decoder()
										_snake_case													=			inputs_dict['input_ids']

										_snake_case													=			input_ids[:1, :]
										_snake_case													=			inputs_dict['attention_mask'][:1, :]
										_snake_case													=			1

										# first forward pass
										_snake_case													=			model(_lowercase			,  attention_mask=_lowercase			,  use_cache=_lowercase					)

										_snake_case							,       _snake_case													=			outputs.to_tuple()

										# create hypothetical next token and extent to next_input_ids
										_snake_case													=			ids_tensor((self.batch_size, 3)			,  config.vocab_size					)
										_snake_case													=			tf.cast(ids_tensor((self.batch_size, 3)			,  2					)			,  tf.inta					)

										# append to next input_ids and
										_snake_case													=			tf.concat([input_ids, next_tokens]			,  axis=-1					)
										_snake_case													=			tf.concat([attention_mask, next_attn_mask]			,  axis=-1					)

										_snake_case													=			model(_lowercase			,  attention_mask=_lowercase					)[0]
										_snake_case													=			model(_lowercase			,  attention_mask=_lowercase			,  past_key_values=_lowercase					)[0]

										self.parent.assertEqual(next_tokens.shape[1]			,  output_from_past.shape[1]					)

										# select random slice
										_snake_case													=			int(ids_tensor((1,)			,  output_from_past.shape[-1]					)					)
										_snake_case													=			output_from_no_past[:, -3:, random_slice_idx]
										_snake_case													=			output_from_past[:, :, random_slice_idx]

										# test that outputs are equal for slice
										tf.debugging.assert_near(_lowercase			,  _lowercase			,  rtol=1E-3					)





def   lowerCamelCase__       (       UpperCamelCase__     :    Optional[int]    ,      UpperCamelCase__     :    Union[str, Any]    ,      UpperCamelCase__     :    Union[str, Any]    ,      UpperCamelCase__     :    int=None    ,      UpperCamelCase__     :    Any=None    ,      UpperCamelCase__     :    Dict=None    ,      UpperCamelCase__     :    Dict=None    ,      )     ->	Optional[Any]:



					'''simple docstring'''




					if attention_mask is None:
										_snake_case													=			tf.cast(tf.math.not_equal(SCREAMING_SNAKE_CASE_    ,      config.pad_token_id  )    ,      tf.inta  )
					if decoder_attention_mask is None:
										_snake_case													=			tf.concat(
										    [
										        tf.ones(decoder_input_ids[:, :1].shape    ,      dtype=tf.inta  ),
										        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]    ,      config.pad_token_id  )    ,      tf.inta  ),
										    ]    ,      axis=-1    ,      )
					if head_mask is None:
										_snake_case													=			tf.ones((config.encoder_layers, config.encoder_attention_heads)  )
					if decoder_head_mask is None:
										_snake_case													=			tf.ones((config.decoder_layers, config.decoder_attention_heads)  )
					return {
					    "input_ids": input_ids,
					    "attention_mask": attention_mask,
					    "decoder_input_ids": decoder_input_ids,
					    "decoder_attention_mask": decoder_attention_mask,
					    "head_mask": head_mask,
					    "decoder_head_mask": decoder_head_mask,
					}
@require_tf
class      UpperCamelCase_						(					_lowerCamelCase							,					_lowerCamelCase							,					unittest.TestCase   ):
					lowerCAmelCase_							=		(TFLEDForConditionalGeneration, TFLEDModel) if is_tf_available() else ()
					lowerCAmelCase_							=		(TFLEDForConditionalGeneration,) if is_tf_available() else ()
					lowerCAmelCase_							=		(
					    {
					        """conversational""": TFLEDForConditionalGeneration,
					        """feature-extraction""": TFLEDModel,
					        """summarization""": TFLEDForConditionalGeneration,
					        """text2text-generation""": TFLEDForConditionalGeneration,
					        """translation""": TFLEDForConditionalGeneration,
					    }
					    if is_tf_available()
					    else {}
					)
					lowerCAmelCase_							=		True
					lowerCAmelCase_							=		False
					lowerCAmelCase_							=		False
					lowerCAmelCase_							=		False




					def 		lowerCAmelCase     (						self					)						->							Any:
										_snake_case													=			TFLEDModelTester(self					)
										_snake_case													=			ConfigTester(self			,  config_class=_lowercase					)




					def 		lowerCAmelCase     (						self					)						->							Optional[int]:
										self.config_tester.run_common_tests()




					def 		lowerCAmelCase     (						self					)						->							str:
										_snake_case													=			self.model_tester.prepare_config_and_inputs_for_common()
										self.model_tester.check_decoder_model_past_large_inputs(*_lowercase					)




					def 		lowerCAmelCase     (						self					)						->							Dict:
										_snake_case							,       _snake_case													=			self.model_tester.prepare_config_and_inputs_for_common()
										_snake_case													=			tf.zeros_like(inputs_dict['attention_mask']					)
										_snake_case													=			2
										_snake_case													=			tf.where(
										    tf.range(self.model_tester.seq_length					)[None, :] < num_global_attn_indices			,  1			,  inputs_dict['global_attention_mask']			,  )

										_snake_case													=			True
										_snake_case													=			self.model_tester.seq_length
										_snake_case													=			self.model_tester.encoder_seq_length

										def check_decoder_attentions_output(lowerCAmelCase_					):
															_snake_case													=			outputs.decoder_attentions
															self.assertEqual(len(_lowercase					)			,  self.model_tester.num_hidden_layers					)
															self.assertListEqual(
															    list(decoder_attentions[0].shape[-3:]					)			,  [self.model_tester.num_attention_heads, seq_length, seq_length]			,  )

										def check_encoder_attentions_output(lowerCAmelCase_					):
															_snake_case													=			[t.numpy() for t in outputs.encoder_attentions]
															_snake_case													=			[t.numpy() for t in outputs.encoder_global_attentions]
															self.assertEqual(len(_lowercase					)			,  self.model_tester.num_hidden_layers					)
															self.assertEqual(len(_lowercase					)			,  self.model_tester.num_hidden_layers					)
															self.assertListEqual(
															    list(attentions[0].shape[-3:]					)			,  [self.model_tester.num_attention_heads, seq_length, seq_length]			,  )
															self.assertListEqual(
															    list(global_attentions[0].shape[-3:]					)			,  [self.model_tester.num_attention_heads, encoder_seq_length, num_global_attn_indices]			,  )

										for model_class in self.all_model_classes:
															_snake_case													=			True
															_snake_case													=			False
															_snake_case													=			False
															_snake_case													=			model_class(_lowercase					)
															_snake_case													=			model(self._prepare_for_class(_lowercase			,  _lowercase					)					)
															_snake_case													=			len(_lowercase					)
															self.assertEqual(config.output_hidden_states			,  _lowercase					)
															check_encoder_attentions_output(_lowercase					)

															if self.is_encoder_decoder:
																				_snake_case													=			model_class(_lowercase					)
																				_snake_case													=			model(self._prepare_for_class(_lowercase			,  _lowercase					)					)
																				self.assertEqual(config.output_hidden_states			,  _lowercase					)
																				check_decoder_attentions_output(_lowercase					)

															# Check that output attentions can also be changed via the config
															del inputs_dict["output_attentions"]
															_snake_case													=			True
															_snake_case													=			model_class(_lowercase					)
															_snake_case													=			model(self._prepare_for_class(_lowercase			,  _lowercase					)					)
															self.assertEqual(config.output_hidden_states			,  _lowercase					)
															check_encoder_attentions_output(_lowercase					)

															# Check attention is always last and order is fine
															_snake_case													=			True
															_snake_case													=			True
															_snake_case													=			model_class(_lowercase					)
															_snake_case													=			model(self._prepare_for_class(_lowercase			,  _lowercase					)					)

															self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)			,  len(_lowercase					)					)
															self.assertEqual(model.config.output_hidden_states			,  _lowercase					)
															check_encoder_attentions_output(_lowercase					)




					@unittest.skip('LED keeps using potentially symbolic tensors in conditionals and breaks tracing.'					)
					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										pass






					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										# TODO: Head-masking not yet implement
										pass





def   lowerCamelCase__       (       UpperCamelCase__     :    List[str]  )     ->	str:



					'''simple docstring'''




					return tf.constant(SCREAMING_SNAKE_CASE_    ,      dtype=tf.intaa  )


UpperCAmelCase_ 							=       1E-4




@slow
@require_tf
class      UpperCamelCase_						(					unittest.TestCase   ):




					def 		lowerCAmelCase     (						self					)						->							str:
										_snake_case													=			TFLEDForConditionalGeneration.from_pretrained('allenai/led-base-16384'					).led

										# change to intended input here
										_snake_case													=			_long_tensor([512 * [0, 3_1414, 232, 328, 740, 1140, 1_2695, 69]]					)
										_snake_case													=			_long_tensor([128 * [0, 3_1414, 232, 328, 740, 1140, 1_2695, 69]]					)
										_snake_case													=			prepare_led_inputs_dict(model.config			,  _lowercase			,  _lowercase					)
										_snake_case													=			model(**_lowercase					)[0]
										_snake_case													=			(1, 1024, 768)
										self.assertEqual(output.shape			,  _lowercase					)
										# change to expected output here
										_snake_case													=			tf.convert_to_tensor(
										    [[2.30_50, 2.82_79, 0.65_31], [-1.84_57, -0.14_55, -3.56_61], [-1.01_86, 0.45_86, -2.20_43]]			,  )
										tf.debugging.assert_near(output[:, :3, :3]			,  _lowercase			,  atol=1E-3					)






					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			TFLEDForConditionalGeneration.from_pretrained('allenai/led-base-16384'					)

										# change to intended input here
										_snake_case													=			_long_tensor([512 * [0, 3_1414, 232, 328, 740, 1140, 1_2695, 69]]					)
										_snake_case													=			_long_tensor([128 * [0, 3_1414, 232, 328, 740, 1140, 1_2695, 69]]					)
										_snake_case													=			prepare_led_inputs_dict(model.config			,  _lowercase			,  _lowercase					)
										_snake_case													=			model(**_lowercase					)[0]
										_snake_case													=			(1, 1024, model.config.vocab_size)
										self.assertEqual(output.shape			,  _lowercase					)
										# change to expected output here
										_snake_case													=			tf.convert_to_tensor(
										    [[33.65_07, 6.45_72, 16.80_89], [5.87_39, -2.42_38, 11.29_02], [-3.21_39, -4.31_49, 4.27_83]]			,  )
										tf.debugging.assert_near(output[:, :3, :3]			,  _lowercase			,  atol=1E-3			,  rtol=1E-3					)










import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class      UpperCamelCase_						(					_lowerCamelCase							,					unittest.TestCase   ):
					lowerCAmelCase_							=		LEDTokenizer
					lowerCAmelCase_							=		LEDTokenizerFast
					lowerCAmelCase_							=		True




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										super().setUp()
										_snake_case													=			[
										    'l',
										    'o',
										    'w',
										    'e',
										    'r',
										    's',
										    't',
										    'i',
										    'd',
										    'n',
										    '\u0120',
										    '\u0120l',
										    '\u0120n',
										    '\u0120lo',
										    '\u0120low',
										    'er',
										    '\u0120lowest',
										    '\u0120newer',
										    '\u0120wider',
										    '<unk>',
										]
										_snake_case													=			dict(zip(lowerCAmelCase_			,  range(len(lowerCAmelCase_					)					)					)					)
										_snake_case													=			['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
										_snake_case													=			{'unk_token': '<unk>'}

										_snake_case													=			os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES['vocab_file']					)
										_snake_case													=			os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES['merges_file']					)
										with open(self.vocab_file			,  'w'			,  encoding='utf-8'					) as fp:
															fp.write(json.dumps(lowerCAmelCase_					) + '\n'					)
										with open(self.merges_file			,  'w'			,  encoding='utf-8'					) as fp:
															fp.write('\n'.join(lowerCAmelCase_					)					)




					def 		lowerCAmelCase     (						self			,  **lowerCAmelCase_					)						->							List[str]:
										kwargs.update(self.special_tokens_map					)
										return self.tokenizer_class.from_pretrained(self.tmpdirname			,  **lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self			,  **lowerCAmelCase_					)						->							str:
										kwargs.update(self.special_tokens_map					)
										return self.rust_tokenizer_class.from_pretrained(self.tmpdirname			,  **lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							Union[str, Any]:
										return "lower newer", "lower newer"




					@cached_property
					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										return LEDTokenizer.from_pretrained('allenai/led-base-16384'					)




					@cached_property
					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										return LEDTokenizerFast.from_pretrained('allenai/led-base-16384'					)




					@require_torch
					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			['A long paragraph for summarization.', 'Another paragraph for summarization.']
										_snake_case													=			[0, 250, 251, 1_7818, 13, 3_9186, 1938, 4, 2]

										for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
															_snake_case													=			tokenizer(lowerCAmelCase_			,  max_length=len(lowerCAmelCase_					)			,  padding=lowerCAmelCase_			,  return_tensors='pt'					)
															self.assertIsInstance(lowerCAmelCase_			,  lowerCAmelCase_					)

															self.assertEqual((2, 9)			,  batch.input_ids.shape					)
															self.assertEqual((2, 9)			,  batch.attention_mask.shape					)
															_snake_case													=			batch.input_ids.tolist()[0]
															self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)




					@require_torch
					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										_snake_case													=			['A long paragraph for summarization.', 'Another paragraph for summarization.']
										for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
															_snake_case													=			tokenizer(lowerCAmelCase_			,  padding=lowerCAmelCase_			,  return_tensors='pt'					)
															self.assertIn('input_ids'			,  lowerCAmelCase_					)
															self.assertIn('attention_mask'			,  lowerCAmelCase_					)
															self.assertNotIn('labels'			,  lowerCAmelCase_					)
															self.assertNotIn('decoder_attention_mask'			,  lowerCAmelCase_					)




					@require_torch
					def 		lowerCAmelCase     (						self					)						->							Optional[int]:
										_snake_case													=			[
										    'Summary of the text.',
										    'Another summary.',
										]
										for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
															_snake_case													=			tokenizer(text_target=lowerCAmelCase_			,  max_length=32			,  padding='max_length'			,  return_tensors='pt'					)
															self.assertEqual(32			,  targets['input_ids'].shape[1]					)




					@require_torch
					def 		lowerCAmelCase     (						self					)						->							List[str]:
										for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
															_snake_case													=			tokenizer(
															    ['I am a small frog' * 1024, 'I am a small frog']			,  padding=lowerCAmelCase_			,  truncation=lowerCAmelCase_			,  return_tensors='pt'					)
															self.assertIsInstance(lowerCAmelCase_			,  lowerCAmelCase_					)
															self.assertEqual(batch.input_ids.shape			,  (2, 5122)					)




					@require_torch
					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			['A long paragraph for summarization.']
										_snake_case													=			[
										    'Summary of the text.',
										]
										for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
															_snake_case													=			tokenizer(lowerCAmelCase_			,  return_tensors='pt'					)
															_snake_case													=			tokenizer(text_target=lowerCAmelCase_			,  return_tensors='pt'					)
															_snake_case													=			inputs['input_ids']
															_snake_case													=			targets['input_ids']
															self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()					)
															self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()					)
															self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()					)
															self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()					)




					@require_torch
					def 		lowerCAmelCase     (						self					)						->							List[str]:
										for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
															_snake_case													=			['Summary of the text.', 'Another summary.']
															_snake_case													=			[[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

															_snake_case													=			tokenizer(lowerCAmelCase_			,  padding=lowerCAmelCase_					)
															_snake_case													=			[[0] * len(lowerCAmelCase_					) for x in encoded_output['input_ids']]
															_snake_case													=			tokenizer.pad(lowerCAmelCase_					)
															self.assertSequenceEqual(outputs['global_attention_mask']			,  lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										pass






					def 		lowerCAmelCase     (						self					)						->							str:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)
																				_snake_case													=			self.tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)
																				_snake_case													=			'A, <mask> AllenNLP sentence.'
																				_snake_case													=			tokenizer_r.encode_plus(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_			,  return_token_type_ids=lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.encode_plus(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_			,  return_token_type_ids=lowerCAmelCase_					)
																				self.assertEqual(sum(tokens_r['token_type_ids']					)			,  sum(tokens_p['token_type_ids']					)					)
																				self.assertEqual(
																				    sum(tokens_r['attention_mask']					) / len(tokens_r['attention_mask']					)			,  sum(tokens_p['attention_mask']					) / len(tokens_p['attention_mask']					)			,  )

																				_snake_case													=			tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']					)
																				_snake_case													=			tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']					)
																				self.assertSequenceEqual(tokens_p['input_ids']			,  [0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]					)
																				self.assertSequenceEqual(tokens_r['input_ids']			,  [0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]					)

																				self.assertSequenceEqual(
																				    lowerCAmelCase_			,  ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']					)
																				self.assertSequenceEqual(
																				    lowerCAmelCase_			,  ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']					)



