


import unittest

from transformers import SqueezeBertConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    SQUEEZEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
							    SqueezeBertForMaskedLM,
							    SqueezeBertForMultipleChoice,
							    SqueezeBertForQuestionAnswering,
							    SqueezeBertForSequenceClassification,
							    SqueezeBertForTokenClassification,
							    SqueezeBertModel,
							)

class 						A__							(		UpperCamelCase__  ):




						def __init__(       self					:      Tuple				,    _a					:      Optional[Any]				,    _a					:      int=13				,    _a					:      List[str]=7				,    _a					:      List[Any]=True				,    _a					:      Any=True				,    _a					:      Optional[Any]=False				,    _a					:      Optional[Any]=True				,    _a					:      List[Any]=99				,    _a					:      int=32				,    _a					:      Union[str, Any]=5				,    _a					:      Tuple=4				,    _a					:      str=64				,    _a					:      List[Any]="gelu"				,    _a					:      int=0.1				,    _a					:      Tuple=0.1				,    _a					:      Tuple=512				,    _a					:      Any=16				,    _a					:      List[str]=2				,    _a					:      Any=0.02				,    _a					:      str=3				,    _a					:      Tuple=4				,    _a					:      Tuple=None				,    _a					:      Any=2				,    _a					:      List[str]=2				,    _a					:      Any=2				,    _a					:      Dict=2				,    _a					:      Dict=4				,    _a					:      Optional[int]=1				,    )							->     Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =parent
										_SCREAMING_SNAKE_CASE			  =batch_size
										_SCREAMING_SNAKE_CASE			  =seq_length
										_SCREAMING_SNAKE_CASE			  =is_training
										_SCREAMING_SNAKE_CASE			  =use_input_mask
										_SCREAMING_SNAKE_CASE			  =use_token_type_ids
										_SCREAMING_SNAKE_CASE			  =use_labels
										_SCREAMING_SNAKE_CASE			  =vocab_size
										_SCREAMING_SNAKE_CASE			  =hidden_size
										_SCREAMING_SNAKE_CASE			  =num_hidden_layers
										_SCREAMING_SNAKE_CASE			  =num_attention_heads
										_SCREAMING_SNAKE_CASE			  =intermediate_size
										_SCREAMING_SNAKE_CASE			  =hidden_act
										_SCREAMING_SNAKE_CASE			  =hidden_dropout_prob
										_SCREAMING_SNAKE_CASE			  =attention_probs_dropout_prob
										_SCREAMING_SNAKE_CASE			  =max_position_embeddings
										_SCREAMING_SNAKE_CASE			  =type_vocab_size
										_SCREAMING_SNAKE_CASE			  =type_sequence_label_size
										_SCREAMING_SNAKE_CASE			  =initializer_range
										_SCREAMING_SNAKE_CASE			  =num_labels
										_SCREAMING_SNAKE_CASE			  =num_choices
										_SCREAMING_SNAKE_CASE			  =scope
										_SCREAMING_SNAKE_CASE			  =q_groups
										_SCREAMING_SNAKE_CASE			  =k_groups
										_SCREAMING_SNAKE_CASE			  =v_groups
										_SCREAMING_SNAKE_CASE			  =post_attention_groups
										_SCREAMING_SNAKE_CASE			  =intermediate_groups
										_SCREAMING_SNAKE_CASE			  =output_groups




						def 							__UpperCamelCase				(       self					:      int  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =ids_tensor([self.batch_size, self.seq_length]				,    self.vocab_size  )

										_SCREAMING_SNAKE_CASE			  =None
										if self.use_input_mask:
														_SCREAMING_SNAKE_CASE			  =random_attention_mask([self.batch_size, self.seq_length]  )

										_SCREAMING_SNAKE_CASE			  =None
										_SCREAMING_SNAKE_CASE			  =None
										_SCREAMING_SNAKE_CASE			  =None
										if self.use_labels:
														_SCREAMING_SNAKE_CASE			  =ids_tensor([self.batch_size]				,    self.type_sequence_label_size  )
														_SCREAMING_SNAKE_CASE			  =ids_tensor([self.batch_size, self.seq_length]				,    self.num_labels  )
														_SCREAMING_SNAKE_CASE			  =ids_tensor([self.batch_size]				,    self.num_choices  )

										_SCREAMING_SNAKE_CASE			  =self.get_config()

										return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels




						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     List[str]:


										"""simple docstring"""





										return SqueezeBertConfig(
										    embedding_size=self.hidden_size				,    vocab_size=self.vocab_size				,    hidden_size=self.hidden_size				,    num_hidden_layers=self.num_hidden_layers				,    num_attention_heads=self.num_attention_heads				,    intermediate_size=self.intermediate_size				,    hidden_act=self.hidden_act				,    attention_probs_dropout_prob=self.hidden_dropout_prob				,    attention_dropout=self.attention_probs_dropout_prob				,    max_position_embeddings=self.max_position_embeddings				,    initializer_range=self.initializer_range				,    q_groups=self.q_groups				,    k_groups=self.k_groups				,    v_groups=self.v_groups				,    post_attention_groups=self.post_attention_groups				,    intermediate_groups=self.intermediate_groups				,    output_groups=self.output_groups				,    )




						def 							__UpperCamelCase				(       self					:      Union[str, Any]				,    _a					:      Any				,    _a					:      List[str]				,    _a					:      Union[str, Any]				,    _a					:      str				,    _a					:      Optional[int]				,    _a					:      Optional[Any]  )							->     List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =SqueezeBertModel(config=_a  )
										model.to(_a  )
										model.eval()
										_SCREAMING_SNAKE_CASE			  =model(_a				,    _a  )
										_SCREAMING_SNAKE_CASE			  =model(_a  )
										self.parent.assertEqual(result.last_hidden_state.shape				,    (self.batch_size, self.seq_length, self.hidden_size)  )




						def 							__UpperCamelCase				(       self					:      Optional[int]				,    _a					:      Tuple				,    _a					:      Optional[Any]				,    _a					:      List[Any]				,    _a					:      List[Any]				,    _a					:      Tuple				,    _a					:      Any  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =SqueezeBertForMaskedLM(config=_a  )
										model.to(_a  )
										model.eval()
										_SCREAMING_SNAKE_CASE			  =model(_a				,    attention_mask=_a				,    labels=_a  )
										self.parent.assertEqual(result.logits.shape				,    (self.batch_size, self.seq_length, self.vocab_size)  )




						def 							__UpperCamelCase				(       self					:      List[str]				,    _a					:      Optional[Any]				,    _a					:      int				,    _a					:      Optional[int]				,    _a					:      Optional[int]				,    _a					:      Optional[Any]				,    _a					:      Tuple  )							->     List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =SqueezeBertForQuestionAnswering(config=_a  )
										model.to(_a  )
										model.eval()
										_SCREAMING_SNAKE_CASE			  =model(
										    _a				,    attention_mask=_a				,    start_positions=_a				,    end_positions=_a  )
										self.parent.assertEqual(result.start_logits.shape				,    (self.batch_size, self.seq_length)  )
										self.parent.assertEqual(result.end_logits.shape				,    (self.batch_size, self.seq_length)  )




						def 							__UpperCamelCase				(       self					:      Any				,    _a					:      Optional[Any]				,    _a					:      List[str]				,    _a					:      str				,    _a					:      Union[str, Any]				,    _a					:      List[Any]				,    _a					:      Dict  )							->     Any:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.num_labels
										_SCREAMING_SNAKE_CASE			  =SqueezeBertForSequenceClassification(_a  )
										model.to(_a  )
										model.eval()
										_SCREAMING_SNAKE_CASE			  =model(_a				,    attention_mask=_a				,    labels=_a  )
										self.parent.assertEqual(result.logits.shape				,    (self.batch_size, self.num_labels)  )




						def 							__UpperCamelCase				(       self					:      Dict				,    _a					:      Union[str, Any]				,    _a					:      Dict				,    _a					:      Union[str, Any]				,    _a					:      Tuple				,    _a					:      Union[str, Any]				,    _a					:      Any  )							->     Any:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.num_labels
										_SCREAMING_SNAKE_CASE			  =SqueezeBertForTokenClassification(config=_a  )
										model.to(_a  )
										model.eval()

										_SCREAMING_SNAKE_CASE			  =model(_a				,    attention_mask=_a				,    labels=_a  )
										self.parent.assertEqual(result.logits.shape				,    (self.batch_size, self.seq_length, self.num_labels)  )




						def 							__UpperCamelCase				(       self					:      str				,    _a					:      Dict				,    _a					:      Union[str, Any]				,    _a					:      Tuple				,    _a					:      Tuple				,    _a					:      Dict				,    _a					:      Tuple  )							->     List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.num_choices
										_SCREAMING_SNAKE_CASE			  =SqueezeBertForMultipleChoice(config=_a  )
										model.to(_a  )
										model.eval()
										_SCREAMING_SNAKE_CASE			  =input_ids.unsqueeze(1  ).expand(-1				,    self.num_choices				,    -1  ).contiguous()
										_SCREAMING_SNAKE_CASE			  =input_mask.unsqueeze(1  ).expand(-1				,    self.num_choices				,    -1  ).contiguous()
										_SCREAMING_SNAKE_CASE			  =model(
										    _a				,    attention_mask=_a				,    labels=_a				,    )
										self.parent.assertEqual(result.logits.shape				,    (self.batch_size, self.num_choices)  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.prepare_config_and_inputs()
										((_SCREAMING_SNAKE_CASE)  ,				(_SCREAMING_SNAKE_CASE)  ,				(_SCREAMING_SNAKE_CASE)  ,				(_SCREAMING_SNAKE_CASE)  ,				(_SCREAMING_SNAKE_CASE)  ,				(_SCREAMING_SNAKE_CASE))			  =config_and_inputs
										_SCREAMING_SNAKE_CASE			  ={'''input_ids''': input_ids, '''attention_mask''': input_mask}
										return config, inputs_dict

@require_torch
class 						A__							(		UpperCamelCase__   ,     UpperCamelCase__   ,     unittest.TestCase  ):
						UpperCAmelCase             =						(
						    (
						        SqueezeBertModel,
						        SqueezeBertForMaskedLM,
						        SqueezeBertForMultipleChoice,
						        SqueezeBertForQuestionAnswering,
						        SqueezeBertForSequenceClassification,
						        SqueezeBertForTokenClassification,
						    )
						    if is_torch_available()
						    else None
						)
						UpperCAmelCase             =						(
						    {
						        "feature-extraction": SqueezeBertModel,
						        "fill-mask": SqueezeBertForMaskedLM,
						        "question-answering": SqueezeBertForQuestionAnswering,
						        "text-classification": SqueezeBertForSequenceClassification,
						        "token-classification": SqueezeBertForTokenClassification,
						        "zero-shot": SqueezeBertForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)
						UpperCAmelCase             =						False
						UpperCAmelCase             =						True
						UpperCAmelCase             =						False




						def 							__UpperCamelCase				(       self					:      Union[str, Any]  )							->     Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =SqueezeBertModelTester(self  )
										_SCREAMING_SNAKE_CASE			  =ConfigTester(self				,    config_class=_a				,    dim=37  )




						def 							__UpperCamelCase				(       self					:      Tuple  )							->     List[str]:


										"""simple docstring"""





										self.config_tester.run_common_tests()




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_squeezebert_model(*_a  )




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_squeezebert_for_masked_lm(*_a  )




						def 							__UpperCamelCase				(       self					:      Any  )							->     List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_squeezebert_for_question_answering(*_a  )




						def 							__UpperCamelCase				(       self					:      Tuple  )							->     Any:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_squeezebert_for_sequence_classification(*_a  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_squeezebert_for_token_classification(*_a  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_squeezebert_for_multiple_choice(*_a  )




						@slow
						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     List[str]:


										"""simple docstring"""





										for model_name in SQUEEZEBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
														_SCREAMING_SNAKE_CASE			  =SqueezeBertModel.from_pretrained(_a  )
														self.assertIsNotNone(_a  )

@require_sentencepiece
@require_tokenizers
@require_torch
class 						A__							(		unittest.TestCase  ):




						@slow
						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     Optional[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =SqueezeBertForSequenceClassification.from_pretrained('''squeezebert/squeezebert-mnli'''  )

										_SCREAMING_SNAKE_CASE			  =torch.tensor([[1, 2_9414, 232, 328, 740, 1140, 1_2695, 69, 13, 1588, 2]]  )
										_SCREAMING_SNAKE_CASE			  =model(_a  )[0]
										_SCREAMING_SNAKE_CASE			  =torch.Size((1, 3)  )
										self.assertEqual(output.shape				,    _a  )
										_SCREAMING_SNAKE_CASE			  =torch.tensor([[0.64_01, -0.03_49, -0.60_41]]  )
										self.assertTrue(torch.allclose(_a				,    _a				,    atol=1E-4  )  )



import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin

class 						A__							(		UpperCamelCase__   ,     unittest.TestCase  ):
						UpperCAmelCase             =						ProphetNetTokenizer
						UpperCAmelCase             =						False




						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     int:


										"""simple docstring"""





										super().setUp()

										_SCREAMING_SNAKE_CASE			  =[
										    '''[UNK]''',
										    '''[CLS]''',
										    '''[SEP]''',
										    '''[PAD]''',
										    '''[MASK]''',
										    '''want''',
										    '''##want''',
										    '''##ed''',
										    '''wa''',
										    '''un''',
										    '''runn''',
										    '''##ing''',
										    ''',''',
										    '''low''',
										    '''lowest''',
										]
										_SCREAMING_SNAKE_CASE			  =os.path.join(self.tmpdirname				,    VOCAB_FILES_NAMES['''vocab_file''']  )
										with open(self.vocab_file				,    '''w'''				,    encoding='''utf-8'''  ) as vocab_writer:
														vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )




						def 							__UpperCamelCase				(       self					:      str				,    _a					:      Union[str, Any]  )							->     List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''UNwant\u00E9d,running'''
										_SCREAMING_SNAKE_CASE			  ='''unwanted, running'''
										return input_text, output_text




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.tokenizer_class(self.vocab_file  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.tokenize('''UNwant\u00E9d,running'''  )
										self.assertListEqual(_a				,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(_a  )				,    [9, 6, 7, 12, 10, 11]  )




						def 							__UpperCamelCase				(       self					:      Tuple  )							->     Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''  )				,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )				,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )				,    ['''hello''']  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a				,    strip_accents=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )				,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )				,    ['''h\u00E9llo''']  )




						def 							__UpperCamelCase				(       self					:      List[str]  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a				,    strip_accents=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )				,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )				,    ['''hello''']  )




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )				,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )				,    ['''hello''']  )




						def 							__UpperCamelCase				(       self					:      Any  )							->     Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )				,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a				,    strip_accents=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )				,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )




						def 							__UpperCamelCase				(       self					:      Optional[Any]  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a				,    strip_accents=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )				,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )




						def 							__UpperCamelCase				(       self					:      List[str]  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a				,    never_split=['''[UNK]''']  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''  )				,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']  )




						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

										_SCREAMING_SNAKE_CASE			  ={}
										for i, token in enumerate(_a  ):
														_SCREAMING_SNAKE_CASE			  =i
										_SCREAMING_SNAKE_CASE			  =WordpieceTokenizer(vocab=_a				,    unk_token='''[UNK]'''  )

										self.assertListEqual(tokenizer.tokenize(''''''  )				,    []  )

										self.assertListEqual(tokenizer.tokenize('''unwanted running'''  )				,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']  )

										self.assertListEqual(tokenizer.tokenize('''unwantedX running'''  )				,    ['''[UNK]''', '''runn''', '''##ing''']  )




						@require_torch
						def 							__UpperCamelCase				(       self					:      Any  )							->     Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''  )

										_SCREAMING_SNAKE_CASE			  =['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
										_SCREAMING_SNAKE_CASE			  =[1037, 2146, 2_0423, 2005, 7680, 7849, 3989, 1012, 102]
										_SCREAMING_SNAKE_CASE			  =tokenizer(_a				,    padding=_a				,    return_tensors='''pt'''  )
										self.assertIsInstance(_a				,    _a  )
										_SCREAMING_SNAKE_CASE			  =list(batch.input_ids.numpy()[0]  )
										self.assertListEqual(_a				,    _a  )

										self.assertEqual((2, 9)				,    batch.input_ids.shape  )
										self.assertEqual((2, 9)				,    batch.attention_mask.shape  )




						def 							__UpperCamelCase				(       self					:      List[str]  )							->     Optional[int]:


										"""simple docstring"""





										self.assertTrue(_is_whitespace(''' '''  )  )
										self.assertTrue(_is_whitespace('''\t'''  )  )
										self.assertTrue(_is_whitespace('''\r'''  )  )
										self.assertTrue(_is_whitespace('''\n'''  )  )
										self.assertTrue(_is_whitespace('''\u00A0'''  )  )

										self.assertFalse(_is_whitespace('''A'''  )  )
										self.assertFalse(_is_whitespace('''-'''  )  )




						def 							__UpperCamelCase				(       self					:      int  )							->     Dict:


										"""simple docstring"""





										self.assertTrue(_is_control('''\u0005'''  )  )

										self.assertFalse(_is_control('''A'''  )  )
										self.assertFalse(_is_control(''' '''  )  )
										self.assertFalse(_is_control('''\t'''  )  )
										self.assertFalse(_is_control('''\r'''  )  )




						def 							__UpperCamelCase				(       self					:      List[str]  )							->     List[str]:


										"""simple docstring"""





										self.assertTrue(_is_punctuation('''-'''  )  )
										self.assertTrue(_is_punctuation('''$'''  )  )
										self.assertTrue(_is_punctuation('''`'''  )  )
										self.assertTrue(_is_punctuation('''.'''  )  )

										self.assertFalse(_is_punctuation('''A'''  )  )
										self.assertFalse(_is_punctuation(''' '''  )  )




						@slow
						def 							__UpperCamelCase				(       self					:      Tuple  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.encode('''sequence builders'''				,    add_special_tokens=_a  )
										_SCREAMING_SNAKE_CASE			  =tokenizer.encode('''multi-sequence build'''				,    add_special_tokens=_a  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.build_inputs_with_special_tokens(_a  )
										_SCREAMING_SNAKE_CASE			  =tokenizer.build_inputs_with_special_tokens(_a				,    _a  )

										assert encoded_sentence == text + [102]
										assert encoded_pair == text + [102] + text_a + [102]