




"""simple docstring"""

import math
import os
import unittest

from transformers import MegatronBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           MODEL_FOR_PRETRAINING_MAPPING,
           MegatronBertForCausalLM,
           MegatronBertForMaskedLM,
           MegatronBertForMultipleChoice,
           MegatronBertForNextSentencePrediction,
           MegatronBertForPreTraining,
           MegatronBertForQuestionAnswering,
           MegatronBertForSequenceClassification,
           MegatronBertForTokenClassification,
           MegatronBertModel,
       )





class 							lowercase_			:



 '''simple docstring'''

 def __init__(					self			:			Union[str, Any]       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Any=13       ,				_UpperCAmelCase			:			Union[str, Any]=7       ,				_UpperCAmelCase			:			List[Any]=True       ,				_UpperCAmelCase			:			int=True       ,				_UpperCAmelCase			:			List[str]=True       ,				_UpperCAmelCase			:			str=True       ,				_UpperCAmelCase			:			Tuple=99       ,				_UpperCAmelCase			:			Any=64       ,				_UpperCAmelCase			:			Dict=32       ,				_UpperCAmelCase			:			Union[str, Any]=5       ,				_UpperCAmelCase			:			Optional[Any]=4       ,				_UpperCAmelCase			:			int=37       ,				_UpperCAmelCase			:			str="gelu"       ,				_UpperCAmelCase			:			Union[str, Any]=0.1       ,				_UpperCAmelCase			:			str=0.1       ,				_UpperCAmelCase			:			List[Any]=512       ,				_UpperCAmelCase			:			Any=16       ,				_UpperCAmelCase			:			Tuple=2       ,				_UpperCAmelCase			:			Tuple=0.02       ,				_UpperCAmelCase			:			Any=3       ,				_UpperCAmelCase			:			Any=4       ,				_UpperCAmelCase			:			Tuple=None       ,				):
       _A     =  parent
       _A     =  batch_size
       _A     =  seq_length
       _A     =  is_training
       _A     =  use_input_mask
       _A     =  use_token_type_ids
       _A     =  use_labels
       _A     =  vocab_size
       _A     =  hidden_size
       _A     =  embedding_size
       _A     =  num_hidden_layers
       _A     =  num_attention_heads
       _A     =  intermediate_size
       _A     =  hidden_act
       _A     =  hidden_dropout_prob
       _A     =  attention_probs_dropout_prob
       _A     =  max_position_embeddings
       _A     =  type_vocab_size
       _A     =  type_sequence_label_size
       _A     =  initializer_range
       _A     =  num_labels
       _A     =  num_choices
       _A     =  scope

 def      lowerCAmelCase_				(					self			:			Optional[Any]						):
       _A     =  ids_tensor([self.batch_size, self.seq_length]       ,				self.vocab_size						)

       _A     =  None
       if self.use_input_mask:
             _A     =  random_attention_mask([self.batch_size, self.seq_length]						)

       _A     =  None
       if self.use_token_type_ids:
             _A     =  ids_tensor([self.batch_size, self.seq_length]       ,				self.type_vocab_size						)

       _A     =  None
       _A     =  None
       _A     =  None
       if self.use_labels:
             _A     =  ids_tensor([self.batch_size]       ,				self.type_sequence_label_size						)
             _A     =  ids_tensor([self.batch_size, self.seq_length]       ,				self.num_labels						)
             _A     =  ids_tensor([self.batch_size]       ,				self.num_choices						)

       _A     =  self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

 def      lowerCAmelCase_				(					self			:			Dict						):
       return MegatronBertConfig(
           vocab_size=self.vocab_size       ,				hidden_size=self.hidden_size       ,				num_hidden_layers=self.num_hidden_layers       ,				num_attention_heads=self.num_attention_heads       ,				intermediate_size=self.intermediate_size       ,				embedding_size=self.embedding_size       ,				hidden_act=self.hidden_act       ,				hidden_dropout_prob=self.hidden_dropout_prob       ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,				max_position_embeddings=self.max_position_embeddings       ,				type_vocab_size=self.type_vocab_size       ,				is_decoder=_UpperCAmelCase       ,				initializer_range=self.initializer_range       ,				)

 def      lowerCAmelCase_				(					self			:			Union[str, Any]       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			Tuple       ,				_UpperCAmelCase			:			Tuple       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			Optional[int]       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			Optional[Any]						):
       _A     =  MegatronBertModel(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(_UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase						)
       _A     =  model(_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase						)
       _A     =  model(_UpperCAmelCase						)

       self.parent.assertEqual(result.last_hidden_state.shape       ,				(self.batch_size, self.seq_length, self.hidden_size)						)
       self.parent.assertEqual(result.pooler_output.shape       ,				(self.batch_size, self.hidden_size)						)

 def      lowerCAmelCase_				(					self			:			Any       ,				_UpperCAmelCase			:			Optional[int]       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Tuple       ,				_UpperCAmelCase			:			Dict       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			Optional[int]						):
       _A     =  MegatronBertForMaskedLM(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(_UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				labels=_UpperCAmelCase						)
       self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.seq_length, self.vocab_size)						)

 def      lowerCAmelCase_				(					self			:			int       ,				_UpperCAmelCase			:			Optional[Any]       ,				_UpperCAmelCase			:			Tuple       ,				_UpperCAmelCase			:			Optional[int]       ,				_UpperCAmelCase			:			Tuple       ,				_UpperCAmelCase			:			int       ,				_UpperCAmelCase			:			int       ,				_UpperCAmelCase			:			int						):
       _A     =  MegatronBertForCausalLM(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(_UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				labels=_UpperCAmelCase						)
       self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.seq_length, self.vocab_size)						)

 def      lowerCAmelCase_				(					self			:			Union[str, Any]       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Optional[Any]       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			Optional[int]       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			Any						):
       _A     =  MegatronBertForNextSentencePrediction(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(
           _UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				labels=_UpperCAmelCase       ,				)
       self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, 2)						)

 def      lowerCAmelCase_				(					self			:			Tuple       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			List[Any]       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Tuple       ,				_UpperCAmelCase			:			Dict						):
       _A     =  MegatronBertForPreTraining(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(
           _UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				labels=_UpperCAmelCase       ,				next_sentence_label=_UpperCAmelCase       ,				)
       self.parent.assertEqual(result.prediction_logits.shape       ,				(self.batch_size, self.seq_length, self.vocab_size)						)
       self.parent.assertEqual(result.seq_relationship_logits.shape       ,				(self.batch_size, 2)						)

 def      lowerCAmelCase_				(					self			:			List[str]       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			List[str]						):
       _A     =  MegatronBertForQuestionAnswering(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(
           _UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				start_positions=_UpperCAmelCase       ,				end_positions=_UpperCAmelCase       ,				)
       self.parent.assertEqual(result.start_logits.shape       ,				(self.batch_size, self.seq_length)						)
       self.parent.assertEqual(result.end_logits.shape       ,				(self.batch_size, self.seq_length)						)

 def      lowerCAmelCase_				(					self			:			List[Any]       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			str						):
       _A     =  self.num_labels
       _A     =  MegatronBertForSequenceClassification(_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(_UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				labels=_UpperCAmelCase						)
       self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.num_labels)						)

 def      lowerCAmelCase_				(					self			:			Tuple       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Optional[int]       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			Optional[Any]       ,				_UpperCAmelCase			:			Optional[Any]       ,				_UpperCAmelCase			:			Optional[Any]       ,				_UpperCAmelCase			:			Tuple						):
       _A     =  self.num_labels
       _A     =  MegatronBertForTokenClassification(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  model(_UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				labels=_UpperCAmelCase						)
       self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.seq_length, self.num_labels)						)

 def      lowerCAmelCase_				(					self			:			List[Any]       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			Dict       ,				_UpperCAmelCase			:			str       ,				_UpperCAmelCase			:			Optional[int]       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			Union[str, Any]       ,				_UpperCAmelCase			:			Any						):
       _A     =  self.num_choices
       _A     =  MegatronBertForMultipleChoice(config=_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.eval()
       _A     =  input_ids.unsqueeze(1						).expand(-1       ,				self.num_choices       ,				-1						).contiguous()
       _A     =  token_type_ids.unsqueeze(1						).expand(-1       ,				self.num_choices       ,				-1						).contiguous()
       _A     =  input_mask.unsqueeze(1						).expand(-1       ,				self.num_choices       ,				-1						).contiguous()
       _A     =  model(
           _UpperCAmelCase       ,				attention_mask=_UpperCAmelCase       ,				token_type_ids=_UpperCAmelCase       ,				labels=_UpperCAmelCase       ,				)
       self.parent.assertEqual(result.logits.shape       ,				(self.batch_size, self.num_choices)						)



 def      lowerCAmelCase_				(					self			:			Any						):
       _A     =  self.prepare_config_and_inputs()
       (
           (
           _A
       )  ,       (
           _A
       )  ,       (
           _A
       )  ,       (
           _A
       )  ,       (
           _A
       )  ,       (
           _A
       )  ,       (
           _A
       )  ,       
       )     =  config_and_inputs
       _A     =  {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
       return config, inputs_dict





@require_torch
class 							lowercase_			(	__lowerCAmelCase     ,      __lowerCAmelCase     ,      unittest.TestCase				):



 '''simple docstring'''
 UpperCAmelCase       :	Tuple     =      (
     (
         MegatronBertModel,
         MegatronBertForMaskedLM,
         MegatronBertForCausalLM,
         MegatronBertForMultipleChoice,
         MegatronBertForNextSentencePrediction,
         MegatronBertForPreTraining,
         MegatronBertForQuestionAnswering,
         MegatronBertForSequenceClassification,
         MegatronBertForTokenClassification,
     )
     if is_torch_available()
     else ()
 )
 UpperCAmelCase       :	Optional[int]     =      (
     {
         '''feature-extraction''': MegatronBertModel,
         '''fill-mask''': MegatronBertForMaskedLM,
         '''question-answering''': MegatronBertForQuestionAnswering,
         '''text-classification''': MegatronBertForSequenceClassification,
         '''text-generation''': MegatronBertForCausalLM,
         '''token-classification''': MegatronBertForTokenClassification,
         '''zero-shot''': MegatronBertForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 UpperCAmelCase       :	int     =      True
 # test_resize_embeddings = False
 UpperCAmelCase       :	str     =      False

 def      lowerCAmelCase_				(					self			:			Union[str, Any]       ,				_UpperCAmelCase			:			Any       ,				_UpperCAmelCase			:			List[str]       ,				_UpperCAmelCase			:			List[Any]=False						):
       _A     =  super()._prepare_for_class(_UpperCAmelCase       ,				_UpperCAmelCase       ,				return_labels=_UpperCAmelCase						)

       if return_labels:
             if model_class in get_values(_UpperCAmelCase						):
                   _A     =  torch.zeros(
                       (self.model_tester.batch_size, self.model_tester.seq_length)       ,				dtype=torch.long       ,				device=_UpperCAmelCase						)
                   _A     =  torch.zeros(
                       self.model_tester.batch_size       ,				dtype=torch.long       ,				device=_UpperCAmelCase						)
       return inputs_dict

 def      lowerCAmelCase_				(					self			:			Any						):
       _A     =  MegatronBertModelTester(self						)
       _A     =  ConfigTester(self       ,				config_class=_UpperCAmelCase       ,				hidden_size=37						)

 def      lowerCAmelCase_				(					self			:			Optional[int]						):
       self.config_tester.run_common_tests()

 def      lowerCAmelCase_				(					self			:			Dict						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_model(*_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			Dict						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_masked_lm(*_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			Tuple						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_multiple_choice(*_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			List[str]						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_next_sequence_prediction(*_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			Optional[int]						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_pretraining(*_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			List[str]						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_question_answering(*_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			List[Any]						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_sequence_classification(*_UpperCAmelCase						)



 def      lowerCAmelCase_				(					self			:			List[Any]						):
       _A     =  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_megatron_bert_for_token_classification(*_UpperCAmelCase						)






def 		_snake_case	(   _snake_case				:   Any				)       ->   List[Any]:






      '''simple docstring'''


      return torch.tensor(
          _snake_case							,    dtype=torch.long							,    device=_snake_case							,    )


a							      =     1e-4







@require_torch
@require_sentencepiece
@require_tokenizers
class 							lowercase_			(	unittest.TestCase				):



 '''simple docstring'''

 @slow
 @unittest.skip('Model is not available.'						)
 def      lowerCAmelCase_				(					self			:			Optional[int]						):
       _A     =  'nvidia/megatron-bert-uncased-345m'
       if "MYDIR" in os.environ:
             _A     =  os.path.join(os.environ['MYDIR']       ,				_UpperCAmelCase						)
       _A     =  MegatronBertModel.from_pretrained(_UpperCAmelCase						)
       model.to(_UpperCAmelCase						)
       model.half()
       _A     =  _long_tensor([[101, 7_110, 1_005, 1_056, 2_023, 11_333, 17_413, 1_029, 102]]						)
       with torch.no_grad():
             _A     =  model(_UpperCAmelCase						)[0]
       _A     =  torch.Size((1, 9, 1_024)						)
       self.assertEqual(output.shape       ,				_UpperCAmelCase						)

       _A     =  [-0.6040, -0.2517, -0.1025, 0.3420, -0.6758, -0.0017, -0.1089, -0.1990, 0.5728]
       for ii in range(3						):
             for jj in range(3						):
                   _A     =  output[0, ii, jj]
                   _A     =  expected[3 * ii + jj]
                   _A     =  'ii={} jj={} a={} b={}'.format(_UpperCAmelCase       ,				_UpperCAmelCase       ,				_UpperCAmelCase       ,				_UpperCAmelCase						)
                   self.assertTrue(math.isclose(_UpperCAmelCase       ,				_UpperCAmelCase       ,				rel_tol=_UpperCAmelCase       ,				abs_tol=_UpperCAmelCase						)       ,				msg=_UpperCAmelCase						)







"""simple docstring"""

import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


a							      =     get_tests_dir('''fixtures/test_sentencepiece_bpe_char.model''')





@require_sentencepiece
@require_tokenizers
class 							lowercase_			(	__lowerCAmelCase     ,      unittest.TestCase				):



 '''simple docstring'''
 UpperCAmelCase       :	List[str]     =      SpeechTaTokenizer
 UpperCAmelCase       :	Tuple     =      False
 UpperCAmelCase       :	Optional[int]     =      True

 def      lowerCAmelCase_				(					self			:			Tuple						):
       super().setUp()

       # We have a SentencePiece fixture for testing
       _A     =  SpeechTaTokenizer(_UpperCAmelCase						)

       _A     =  AddedToken('<mask>'       ,				lstrip=_UpperCAmelCase       ,				rstrip=_UpperCAmelCase						)
       _A     =  mask_token
       tokenizer.add_special_tokens({'mask_token': mask_token}						)
       tokenizer.add_tokens(['<ctc_blank>']						)

       tokenizer.save_pretrained(self.tmpdirname						)

 def      lowerCAmelCase_				(					self			:			Optional[Any]       ,				_UpperCAmelCase			:			Tuple						):
       _A     =  'this is a test'
       _A     =  'this is a test'
       return input_text, output_text

 def      lowerCAmelCase_				(					self			:			List[Any]       ,				_UpperCAmelCase			:			Optional[int]       ,				_UpperCAmelCase			:			Union[str, Any]=False       ,				_UpperCAmelCase			:			Dict=20       ,				_UpperCAmelCase			:			str=5						):
       _A  ,       _A     =  self.get_input_output_texts(_UpperCAmelCase						)
       _A     =  tokenizer.encode(_UpperCAmelCase       ,				add_special_tokens=_UpperCAmelCase						)
       _A     =  tokenizer.decode(_UpperCAmelCase       ,				clean_up_tokenization_spaces=_UpperCAmelCase						)
       return text, ids

 def      lowerCAmelCase_				(					self			:			Optional[Any]						):
       _A     =  '<pad>'
       _A     =  1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCAmelCase						)       ,				_UpperCAmelCase						)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCAmelCase						)       ,				_UpperCAmelCase						)

 def      lowerCAmelCase_				(					self			:			Optional[Any]						):
       _A     =  list(self.get_tokenizer().get_vocab().keys()						)

       self.assertEqual(vocab_keys[0]       ,				'<s>'						)
       self.assertEqual(vocab_keys[1]       ,				'<pad>'						)
       self.assertEqual(vocab_keys[-4]       ,				'œ'						)
       self.assertEqual(vocab_keys[-2]       ,				'<mask>'						)
       self.assertEqual(vocab_keys[-1]       ,				'<ctc_blank>'						)
       self.assertEqual(len(_UpperCAmelCase						)       ,				81						)

 def      lowerCAmelCase_				(					self			:			Optional[Any]						):
       self.assertEqual(self.get_tokenizer().vocab_size       ,				79						)

 def      lowerCAmelCase_				(					self			:			Any						):
       _A     =  self.get_tokenizers(do_lower_case=_UpperCAmelCase						)
       for tokenizer in tokenizers:
             with self.subTest(F'''{tokenizer.__class__.__name__}'''						):
                   _A     =  tokenizer.vocab_size
                   _A     =  len(_UpperCAmelCase						)

                   self.assertNotEqual(_UpperCAmelCase       ,				0						)

                   # We usually have added tokens from the start in tests because our vocab fixtures are
                   # smaller than the original vocabs - let's not assert this
                   # self.assertEqual(vocab_size, all_size)

                   _A     =  ['aaaaa bbbbbb', 'cccccccccdddddddd']
                   _A     =  tokenizer.add_tokens(_UpperCAmelCase						)
                   _A     =  tokenizer.vocab_size
                   _A     =  len(_UpperCAmelCase						)

                   self.assertNotEqual(_UpperCAmelCase       ,				0						)
                   self.assertEqual(_UpperCAmelCase       ,				_UpperCAmelCase						)
                   self.assertEqual(_UpperCAmelCase       ,				len(_UpperCAmelCase						)						)
                   self.assertEqual(_UpperCAmelCase       ,				all_size + len(_UpperCAmelCase						)						)

                   _A     =  tokenizer.encode('aaaaa bbbbbb low cccccccccdddddddd l'       ,				add_special_tokens=_UpperCAmelCase						)

                   self.assertGreaterEqual(len(_UpperCAmelCase						)       ,				4						)
                   self.assertGreater(tokens[0]       ,				tokenizer.vocab_size - 1						)
                   self.assertGreater(tokens[-3]       ,				tokenizer.vocab_size - 1						)

                   _A     =  {'eos_token': '>>>>|||<||<<|<<', 'pad_token': '<<<<<|||>|>>>>|>'}
                   _A     =  tokenizer.add_special_tokens(_UpperCAmelCase						)
                   _A     =  tokenizer.vocab_size
                   _A     =  len(_UpperCAmelCase						)

                   self.assertNotEqual(_UpperCAmelCase       ,				0						)
                   self.assertEqual(_UpperCAmelCase       ,				_UpperCAmelCase						)
                   self.assertEqual(_UpperCAmelCase       ,				len(_UpperCAmelCase						)						)
                   self.assertEqual(_UpperCAmelCase       ,				all_size_a + len(_UpperCAmelCase						)						)

                   _A     =  tokenizer.encode(
                       '>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'       ,				add_special_tokens=_UpperCAmelCase						)

                   self.assertGreaterEqual(len(_UpperCAmelCase						)       ,				6						)
                   self.assertGreater(tokens[0]       ,				tokenizer.vocab_size - 1						)
                   self.assertGreater(tokens[0]       ,				tokens[1]						)
                   self.assertGreater(tokens[-3]       ,				tokenizer.vocab_size - 1						)
                   self.assertGreater(tokens[-3]       ,				tokens[-4]						)
                   self.assertEqual(tokens[0]       ,				tokenizer.eos_token_id						)
                   self.assertEqual(tokens[-3]       ,				tokenizer.pad_token_id						)

 def      lowerCAmelCase_				(					self			:			str						):
       pass

 def      lowerCAmelCase_				(					self			:			Any						):
       pass

 def      lowerCAmelCase_				(					self			:			Dict						):
       _A     =  self.get_tokenizer()

       _A     =  tokenizer.tokenize('This is a test'						)
       # fmt: off
       self.assertListEqual(_UpperCAmelCase       ,				[SPIECE_UNDERLINE, 'T', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'a', SPIECE_UNDERLINE, 't', 'e', 's', 't']						)
       # fmt: on

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)       ,				[4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]       ,				)

       _A     =  tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
       self.assertListEqual(
           _UpperCAmelCase       ,				[SPIECE_UNDERLINE, 'I', SPIECE_UNDERLINE, 'w', 'a', 's', SPIECE_UNDERLINE, 'b', 'o', 'r', 'n', SPIECE_UNDERLINE, 'i', 'n', SPIECE_UNDERLINE, '92000', ',', SPIECE_UNDERLINE, 'a', 'n', 'd', SPIECE_UNDERLINE, 't', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'f', 'a', 'l', 's', 'é', '.']						)

       _A     =  tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)
       # fmt: off
       self.assertListEqual(_UpperCAmelCase       ,				[4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26]						)
       # fmt: on

       _A     =  tokenizer.convert_ids_to_tokens(_UpperCAmelCase						)
       self.assertListEqual(
           _UpperCAmelCase       ,				[SPIECE_UNDERLINE, 'I', SPIECE_UNDERLINE, 'w', 'a', 's', SPIECE_UNDERLINE, 'b', 'o', 'r', 'n', SPIECE_UNDERLINE, 'i', 'n', SPIECE_UNDERLINE, '<unk>', ',', SPIECE_UNDERLINE, 'a', 'n', 'd', SPIECE_UNDERLINE, 't', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'f', 'a', 'l', 's', 'é', '.']						)



 @slow
 def      lowerCAmelCase_				(					self			:			List[Any]						):
       # Use custom sequence because this tokenizer does not handle numbers.
       _A     =  [
           'Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '
           'general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural '
           'Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained '
           'models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.',
           'BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '
           'conditioning on both left and right context in all layers.',
           'The quick brown fox jumps over the lazy dog.',
       ]

       # fmt: off
       _A     =  {
           'input_ids': [
               [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
               [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
               [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
           ],
           'attention_mask': [
               [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
               [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
               [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
           ]
       }
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=_UpperCAmelCase       ,				model_name='microsoft/speecht5_asr'       ,				revision='c5ef64c71905caeccde0e4462ef3f9077224c524'       ,				sequences=_UpperCAmelCase       ,				)

