

from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import (
					    TFConvBertForMaskedLM,
					    TFConvBertForMultipleChoice,
					    TFConvBertForQuestionAnswering,
					    TFConvBertForSequenceClassification,
					    TFConvBertForTokenClassification,
					    TFConvBertModel,
					)







class 				lowercase       :




				def __init__(      self   ,     snake_case   ,     snake_case=13   ,     snake_case=7   ,     snake_case=True   ,     snake_case=True   ,     snake_case=True   ,     snake_case=True   ,     snake_case=99   ,     snake_case=32   ,     snake_case=2   ,     snake_case=4   ,     snake_case=37   ,     snake_case="gelu"   ,     snake_case=0.1   ,     snake_case=0.1   ,     snake_case=512   ,     snake_case=16   ,     snake_case=2   ,     snake_case=0.02   ,     snake_case=3   ,     snake_case=4   ,     snake_case=None   ,     ):
											snake_case_           =					parent
											snake_case_           =					13
											snake_case_           =					7
											snake_case_           =					True
											snake_case_           =					True
											snake_case_           =					True
											snake_case_           =					True
											snake_case_           =					99
											snake_case_           =					384
											snake_case_           =					2
											snake_case_           =					4
											snake_case_           =					37
											snake_case_           =					'''gelu'''
											snake_case_           =					0.1
											snake_case_           =					0.1
											snake_case_           =					512
											snake_case_           =					16
											snake_case_           =					2
											snake_case_           =					0.02
											snake_case_           =					3
											snake_case_           =					4
											snake_case_           =					128
											snake_case_           =					2
											snake_case_           =					9
											snake_case_           =					1
											snake_case_           =					None




				def 					a							(      self ):
											snake_case_           =					ids_tensor([self.batch_size, self.seq_length]   ,     self.vocab_size )

											snake_case_           =					None
											if self.use_input_mask:
																		snake_case_           =					random_attention_mask([self.batch_size, self.seq_length] )

											snake_case_           =					None
											if self.use_token_type_ids:
																		snake_case_           =					ids_tensor([self.batch_size, self.seq_length]   ,     self.type_vocab_size )

											snake_case_           =					None
											snake_case_           =					None
											snake_case_           =					None
											if self.use_labels:
																		snake_case_           =					ids_tensor([self.batch_size]   ,     self.type_sequence_label_size )
																		snake_case_           =					ids_tensor([self.batch_size, self.seq_length]   ,     self.num_labels )
																		snake_case_           =					ids_tensor([self.batch_size]   ,     self.num_choices )

											snake_case_           =					ConvBertConfig(
											    vocab_size=self.vocab_size   ,     hidden_size=self.hidden_size   ,     num_hidden_layers=self.num_hidden_layers   ,     num_attention_heads=self.num_attention_heads   ,     intermediate_size=self.intermediate_size   ,     hidden_act=self.hidden_act   ,     hidden_dropout_prob=self.hidden_dropout_prob   ,     attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,     max_position_embeddings=self.max_position_embeddings   ,     type_vocab_size=self.type_vocab_size   ,     initializer_range=self.initializer_range   ,     return_dict=__snake_case   ,     )

											return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					TFConvBertModel(config=__snake_case )
											snake_case_           =					{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

											snake_case_           =					[input_ids, input_mask]
											snake_case_           =					model(__snake_case )

											snake_case_           =					model(__snake_case )

											self.parent.assertEqual(result.last_hidden_state.shape   ,     (self.batch_size, self.seq_length, self.hidden_size) )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					TFConvBertForMaskedLM(config=__snake_case )
											snake_case_           =					{
											    '''input_ids''': input_ids,
											    '''attention_mask''': input_mask,
											    '''token_type_ids''': token_type_ids,
											}
											snake_case_           =					model(__snake_case )
											self.parent.assertEqual(result.logits.shape   ,     (self.batch_size, self.seq_length, self.vocab_size) )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					self.num_labels
											snake_case_           =					TFConvBertForSequenceClassification(config=__snake_case )
											snake_case_           =					{
											    '''input_ids''': input_ids,
											    '''attention_mask''': input_mask,
											    '''token_type_ids''': token_type_ids,
											}

											snake_case_           =					model(__snake_case )
											self.parent.assertEqual(result.logits.shape   ,     (self.batch_size, self.num_labels) )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					self.num_choices
											snake_case_           =					TFConvBertForMultipleChoice(config=__snake_case )
											snake_case_           =					tf.tile(tf.expand_dims(__snake_case   ,     1 )   ,     (1, self.num_choices, 1) )
											snake_case_           =					tf.tile(tf.expand_dims(__snake_case   ,     1 )   ,     (1, self.num_choices, 1) )
											snake_case_           =					tf.tile(tf.expand_dims(__snake_case   ,     1 )   ,     (1, self.num_choices, 1) )
											snake_case_           =					{
											    '''input_ids''': multiple_choice_inputs_ids,
											    '''attention_mask''': multiple_choice_input_mask,
											    '''token_type_ids''': multiple_choice_token_type_ids,
											}
											snake_case_           =					model(__snake_case )
											self.parent.assertEqual(result.logits.shape   ,     (self.batch_size, self.num_choices) )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					self.num_labels
											snake_case_           =					TFConvBertForTokenClassification(config=__snake_case )
											snake_case_           =					{
											    '''input_ids''': input_ids,
											    '''attention_mask''': input_mask,
											    '''token_type_ids''': token_type_ids,
											}
											snake_case_           =					model(__snake_case )
											self.parent.assertEqual(result.logits.shape   ,     (self.batch_size, self.seq_length, self.num_labels) )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					TFConvBertForQuestionAnswering(config=__snake_case )
											snake_case_           =					{
											    '''input_ids''': input_ids,
											    '''attention_mask''': input_mask,
											    '''token_type_ids''': token_type_ids,
											}

											snake_case_           =					model(__snake_case )
											self.parent.assertEqual(result.start_logits.shape   ,     (self.batch_size, self.seq_length) )
											self.parent.assertEqual(result.end_logits.shape   ,     (self.batch_size, self.seq_length) )




				def 					a							(      self ):
											snake_case_           =					self.prepare_config_and_inputs()
											(
											    snake_case_    
											)           =					config_and_inputs
											snake_case_           =					{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
											return config, inputs_dict







@require_tf
class 				lowercase       ( __SCREAMING_SNAKE_CASE			,						__SCREAMING_SNAKE_CASE			,						unittest.TestCase    ):
				__SCREAMING_SNAKE_CASE      :				List[str]     =    (
				    (
				        TFConvBertModel,
				        TFConvBertForMaskedLM,
				        TFConvBertForQuestionAnswering,
				        TFConvBertForSequenceClassification,
				        TFConvBertForTokenClassification,
				        TFConvBertForMultipleChoice,
				    )
				    if is_tf_available()
				    else ()
				)
				__SCREAMING_SNAKE_CASE      :				List[str]     =    (
				    {
				        '''feature-extraction''': TFConvBertModel,
				        '''fill-mask''': TFConvBertForMaskedLM,
				        '''question-answering''': TFConvBertForQuestionAnswering,
				        '''text-classification''': TFConvBertForSequenceClassification,
				        '''token-classification''': TFConvBertForTokenClassification,
				        '''zero-shot''': TFConvBertForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)
				__SCREAMING_SNAKE_CASE      :				Union[str, Any]     =    False
				__SCREAMING_SNAKE_CASE      :				Optional[int]     =    False
				__SCREAMING_SNAKE_CASE      :				Union[str, Any]     =    False




				def 					a							(      self ):
											snake_case_           =					TFConvBertModelTester(self )
											snake_case_           =					ConfigTester(self   ,     config_class=__snake_case   ,     hidden_size=37 )




				def 					a							(      self ):
											self.config_tester.run_common_tests()




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*__snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*__snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_multiple_choice(*__snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*__snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*__snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*__snake_case )




				@slow
				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs_for_common()
											snake_case_           =					True
											snake_case_           =					True

											if hasattr(__snake_case   ,     'use_cache' ):
																		snake_case_           =					True

											snake_case_           =					getattr(self.model_tester   ,     'encoder_seq_length'   ,     self.model_tester.seq_length )
											snake_case_           =					getattr(self.model_tester   ,     'key_length'   ,     __snake_case )

											for model_class in self.all_model_classes:
																		snake_case_           =					self._prepare_for_class(__snake_case   ,     __snake_case )
																		snake_case_           =					model_class(__snake_case )
																		snake_case_           =					len(model(__snake_case ) )

																		with tempfile.TemporaryDirectory() as tmpdirname:
																									model.save_pretrained(__snake_case   ,     saved_model=__snake_case )
																									snake_case_           =					os.path.join(__snake_case   ,     'saved_model'   ,     '1' )
																									snake_case_           =					tf.keras.models.load_model(__snake_case )
																									snake_case_           =					model(__snake_case )

																									if self.is_encoder_decoder:
																																snake_case_           =					outputs['''encoder_hidden_states''']
																																snake_case_           =					outputs['''encoder_attentions''']
																									else:
																																snake_case_           =					outputs['''hidden_states''']
																																snake_case_           =					outputs['''attentions''']

																									self.assertEqual(len(__snake_case )   ,     __snake_case )

																									snake_case_           =					getattr(
																									    self.model_tester   ,     'expected_num_hidden_layers'   ,     self.model_tester.num_hidden_layers + 1 )

																									self.assertEqual(len(__snake_case )   ,     __snake_case )
																									self.assertListEqual(
																									    list(output_hidden_states[0].shape[-2:] )   ,     [self.model_tester.seq_length, self.model_tester.hidden_size]   ,     )

																									self.assertEqual(len(__snake_case )   ,     self.model_tester.num_hidden_layers )
																									self.assertListEqual(
																									    list(output_attentions[0].shape[-3:] )   ,     [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]   ,     )




				@slow
				def 					a							(      self ):
											snake_case_           =					TFConvBertModel.from_pretrained('YituTech/conv-bert-base' )
											self.assertIsNotNone(__snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs_for_common()
											snake_case_           =					True
											snake_case_           =					getattr(self.model_tester   ,     'decoder_seq_length'   ,     self.model_tester.seq_length )
											snake_case_           =					getattr(self.model_tester   ,     'encoder_seq_length'   ,     self.model_tester.seq_length )
											snake_case_           =					getattr(self.model_tester   ,     'key_length'   ,     __snake_case )
											snake_case_           =					getattr(self.model_tester   ,     'key_length'   ,     __snake_case )

											def check_decoder_attentions_output(snake_case ):
																		snake_case_           =					len(__snake_case )
																		self.assertEqual(out_len % 2   ,     0 )
																		snake_case_           =					outputs.decoder_attentions
																		self.assertEqual(len(__snake_case )   ,     self.model_tester.num_hidden_layers )
																		self.assertListEqual(
																		    list(decoder_attentions[0].shape[-3:] )   ,     [self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]   ,     )

											def check_encoder_attentions_output(snake_case ):
																		snake_case_           =					[
																		    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
																		]
																		self.assertEqual(len(__snake_case )   ,     self.model_tester.num_hidden_layers )
																		self.assertListEqual(
																		    list(attentions[0].shape[-3:] )   ,     [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]   ,     )

											for model_class in self.all_model_classes:
																		snake_case_           =					True
																		snake_case_           =					False
																		snake_case_           =					model_class(__snake_case )
																		snake_case_           =					model(self._prepare_for_class(__snake_case   ,     __snake_case ) )
																		snake_case_           =					len(__snake_case )
																		self.assertEqual(config.output_hidden_states   ,     __snake_case )
																		check_encoder_attentions_output(__snake_case )

																		if self.is_encoder_decoder:
																									snake_case_           =					model_class(__snake_case )
																									snake_case_           =					model(self._prepare_for_class(__snake_case   ,     __snake_case ) )
																									self.assertEqual(config.output_hidden_states   ,     __snake_case )
																									check_decoder_attentions_output(__snake_case )

																		# Check that output attentions can also be changed via the config
																		del inputs_dict["output_attentions"]
																		snake_case_           =					True
																		snake_case_           =					model_class(__snake_case )
																		snake_case_           =					model(self._prepare_for_class(__snake_case   ,     __snake_case ) )
																		self.assertEqual(config.output_hidden_states   ,     __snake_case )
																		check_encoder_attentions_output(__snake_case )

																		# Check attention is always last and order is fine
																		snake_case_           =					True
																		snake_case_           =					True
																		snake_case_           =					model_class(__snake_case )
																		snake_case_           =					model(self._prepare_for_class(__snake_case   ,     __snake_case ) )

																		self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)   ,     len(__snake_case ) )
																		self.assertEqual(model.config.output_hidden_states   ,     __snake_case )
																		check_encoder_attentions_output(__snake_case )







@require_tf
class 				lowercase       ( unittest.TestCase    ):




				@slow
				def 					a							(      self ):
											snake_case_           =					TFConvBertModel.from_pretrained('YituTech/conv-bert-base' )
											snake_case_           =					tf.constant([[0, 1, 2, 3, 4, 5]] )
											snake_case_           =					model(__snake_case )[0]

											snake_case_           =					[1, 6, 768]
											self.assertEqual(output.shape   ,     __snake_case )

											snake_case_           =					tf.constant(
											    [
											        [
											            [-0.03_47_54_93, -0.4_68_60_34, -0.30_63_88_32],
											            [0.22_63_72_48, -0.26_98_86_46, -0.7_42_34_24],
											            [0.10_32_48_68, -0.45_01_35_08, -0.58_28_07_84],
											        ]
											    ] )
											tf.debugging.assert_near(output[:, :3, :3]   ,     __snake_case   ,     atol=1e-4 )









import json
import os
import unittest

from transformers import BatchEncoding, MvpTokenizer, MvpTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin, filter_roberta_detectors







@require_tokenizers
class 				lowercase       ( lowercase_			,						unittest.TestCase    ):
				__SCREAMING_SNAKE_CASE      :				Optional[int]     =    MvpTokenizer
				__SCREAMING_SNAKE_CASE      :				Optional[int]     =    MvpTokenizerFast
				__SCREAMING_SNAKE_CASE      :				Optional[int]     =    True
				__SCREAMING_SNAKE_CASE      :				Tuple     =    filter_roberta_detectors




				def 					a							(      self ):
											super().setUp()
											snake_case_           =					[
											    'l',
											    'o',
											    'w',
											    'e',
											    'r',
											    's',
											    't',
											    'i',
											    'd',
											    'n',
											    '\u0120',
											    '\u0120l',
											    '\u0120n',
											    '\u0120lo',
											    '\u0120low',
											    'er',
											    '\u0120lowest',
											    '\u0120newer',
											    '\u0120wider',
											    '<unk>',
											]
											snake_case_           =					dict(zip(snake_case   ,     range(len(snake_case ) ) ) )
											snake_case_           =					['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
											snake_case_           =					{'unk_token': '<unk>'}

											snake_case_           =					os.path.join(self.tmpdirname   ,     VOCAB_FILES_NAMES['vocab_file'] )
											snake_case_           =					os.path.join(self.tmpdirname   ,     VOCAB_FILES_NAMES['merges_file'] )
											with open(self.vocab_file   ,     'w'   ,     encoding='utf-8' ) as fp:
																		fp.write(json.dumps(snake_case ) + '\n' )
											with open(self.merges_file   ,     'w'   ,     encoding='utf-8' ) as fp:
																		fp.write('\n'.join(snake_case ) )




				def 					a							(      self   ,     **snake_case ):
											kwargs.update(self.special_tokens_map )
											return self.tokenizer_class.from_pretrained(self.tmpdirname   ,     **snake_case )




				def 					a							(      self   ,     **snake_case ):
											kwargs.update(self.special_tokens_map )
											return self.rust_tokenizer_class.from_pretrained(self.tmpdirname   ,     **snake_case )




				def 					a							(      self   ,     snake_case ):
											return "lower newer", "lower newer"




				@cached_property
				def 					a							(      self ):
											return MvpTokenizer.from_pretrained('RUCAIBox/mvp' )




				@cached_property
				def 					a							(      self ):
											return MvpTokenizerFast.from_pretrained('RUCAIBox/mvp' )




				@require_torch
				def 					a							(      self ):
											snake_case_           =					['A long paragraph for summarization.', 'Another paragraph for summarization.']
											snake_case_           =					[0, 250, 251, 1_7818, 13, 3_9186, 1938, 4, 2]

											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		snake_case_           =					tokenizer(snake_case   ,     max_length=len(snake_case )   ,     padding=snake_case   ,     return_tensors='pt' )
																		self.assertIsInstance(snake_case   ,     snake_case )

																		self.assertEqual((2, 9)   ,     batch.input_ids.shape )
																		self.assertEqual((2, 9)   ,     batch.attention_mask.shape )
																		snake_case_           =					batch.input_ids.tolist()[0]
																		self.assertListEqual(snake_case   ,     snake_case )
																		# Test that special tokens are reset




				@require_torch
				def 					a							(      self ):
											snake_case_           =					['A long paragraph for summarization.', 'Another paragraph for summarization.']
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		snake_case_           =					tokenizer(snake_case   ,     padding=snake_case   ,     return_tensors='pt' )
																		# check if input_ids are returned and no labels
																		self.assertIn('input_ids'   ,     snake_case )
																		self.assertIn('attention_mask'   ,     snake_case )
																		self.assertNotIn('labels'   ,     snake_case )
																		self.assertNotIn('decoder_attention_mask'   ,     snake_case )




				@require_torch
				def 					a							(      self ):
											snake_case_           =					[
											    'Summary of the text.',
											    'Another summary.',
											]
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		snake_case_           =					tokenizer(text_target=snake_case   ,     max_length=32   ,     padding='max_length'   ,     return_tensors='pt' )
																		self.assertEqual(32   ,     targets['input_ids'].shape[1] )




				@require_torch
				def 					a							(      self ):
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		snake_case_           =					tokenizer(
																		    ['I am a small frog' * 1024, 'I am a small frog']   ,     padding=snake_case   ,     truncation=snake_case   ,     return_tensors='pt' )
																		self.assertIsInstance(snake_case   ,     snake_case )
																		self.assertEqual(batch.input_ids.shape   ,     (2, 1024) )




				@require_torch
				def 					a							(      self ):
											snake_case_           =					['A long paragraph for summarization.']
											snake_case_           =					[
											    'Summary of the text.',
											]
											for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
																		snake_case_           =					tokenizer(snake_case   ,     text_target=snake_case   ,     return_tensors='pt' )
																		snake_case_           =					inputs['input_ids']
																		snake_case_           =					inputs['labels']
																		self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item() )
																		self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item() )
																		self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item() )
																		self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item() )




				def 					a							(      self ):
											pass




				def 					a							(      self ):
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
																									snake_case_           =					self.rust_tokenizer_class.from_pretrained(snake_case   ,     **snake_case )
																									snake_case_           =					self.tokenizer_class.from_pretrained(snake_case   ,     **snake_case )
																									snake_case_           =					'A, <mask> AllenNLP sentence.'
																									snake_case_           =					tokenizer_r.encode_plus(snake_case   ,     add_special_tokens=snake_case   ,     return_token_type_ids=snake_case )
																									snake_case_           =					tokenizer_p.encode_plus(snake_case   ,     add_special_tokens=snake_case   ,     return_token_type_ids=snake_case )

																									# token_type_ids should put 0 everywhere
																									self.assertEqual(sum(tokens_r['token_type_ids'] )   ,     sum(tokens_p['token_type_ids'] ) )

																									# attention_mask should put 1 everywhere, so sum over length should be 1
																									self.assertEqual(
																									    sum(tokens_r['attention_mask'] ) / len(tokens_r['attention_mask'] )   ,     sum(tokens_p['attention_mask'] ) / len(tokens_p['attention_mask'] )   ,     )

																									snake_case_           =					tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids'] )
																									snake_case_           =					tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids'] )

																									# Rust correctly handles the space before the mask while python doesnt
																									self.assertSequenceEqual(tokens_p['input_ids']   ,     [0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2] )
																									self.assertSequenceEqual(tokens_r['input_ids']   ,     [0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2] )

																									self.assertSequenceEqual(
																									    snake_case   ,     ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>'] )
																									self.assertSequenceEqual(
																									    snake_case   ,     ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>'] )




