
from typing import Optional

import torch
import torch.utils.checkpoint
from torch import Tensor, nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
from ...modeling_outputs import (
    BaseModelOutputWithNoAttention,
    BaseModelOutputWithPoolingAndNoAttention,
    ImageClassifierOutputWithNoAttention,
)
from ...modeling_utils import PreTrainedModel
from ...utils import logging
from .configuration_regnet import RegNetConfig


UpperCAmelCase       =					logging.get_logger(__name__)

# General docstring
UpperCAmelCase       =					'''RegNetConfig'''

# Base docstring
UpperCAmelCase       =					'''facebook/regnet-y-040'''
UpperCAmelCase       =					[1, 1088, 7, 7]

# Image classification docstring
UpperCAmelCase       =					'''facebook/regnet-y-040'''
UpperCAmelCase       =					'''tabby, tabby cat'''

UpperCAmelCase       =					[
    '''facebook/regnet-y-040''',
    # See all regnet models at https://huggingface.co/models?filter=regnet
]







class 			A_  (     nn.Module  ):




	'''simple docstring'''


	def __init__(			self						,       snake_case						,       snake_case						,       snake_case = 3						,       snake_case = 1						,       snake_case = 1						,       snake_case = "relu"						,       ):
		super().__init__()
		lowercase	    =       nn.Convad(
		    snake_case						,       snake_case						,       kernel_size=snake_case						,       stride=snake_case						,       padding=kernel_size // 2						,       groups=snake_case						,       bias=snake_case						,       )
		lowercase	    =       nn.BatchNormad(snake_case			)
		lowercase	    =       ACTaFN[activation] if activation is not None else nn.Identity()





	def       SCREAMING_SNAKE_CASE__		(			self						,       snake_case			):
		lowercase	    =       self.convolution(snake_case			)
		lowercase	    =       self.normalization(snake_case			)
		lowercase	    =       self.activation(snake_case			)
		return hidden_state







class 			A_  (     nn.Module  ):




	'''simple docstring'''


	def __init__(			self						,       snake_case			):
		super().__init__()
		lowercase	    =       RegNetConvLayer(
		    config.num_channels						,       config.embedding_size						,       kernel_size=3						,       stride=2						,       activation=config.hidden_act			)
		lowercase	    =       config.num_channels





	def       SCREAMING_SNAKE_CASE__		(			self						,       snake_case			):
		lowercase	    =       pixel_values.shape[1]
		if num_channels != self.num_channels:
			raise ValueError(
			    'Make sure that the channel dimension of the pixel values match with the one set in the configuration.'			)
		lowercase	    =       self.embedder(snake_case			)
		return hidden_state







class 			A_  (     nn.Module  ):




	'''simple docstring'''


	def __init__(			self						,       snake_case						,       snake_case						,       snake_case = 2			):
		super().__init__()
		lowercase	    =       nn.Convad(snake_case						,       snake_case						,       kernel_size=1						,       stride=snake_case						,       bias=snake_case			)
		lowercase	    =       nn.BatchNormad(snake_case			)





	def       SCREAMING_SNAKE_CASE__		(			self						,       snake_case			):
		lowercase	    =       self.convolution(snake_case			)
		lowercase	    =       self.normalization(snake_case			)
		return hidden_state







class 			A_  (     nn.Module  ):




	'''simple docstring'''


	def __init__(			self						,       snake_case						,       snake_case			):
		super().__init__()

		lowercase	    =       nn.AdaptiveAvgPoolad((1, 1)			)
		lowercase	    =       nn.Sequential(
		    nn.Convad(snake_case						,       snake_case						,       kernel_size=1			)						,       nn.ReLU()						,       nn.Convad(snake_case						,       snake_case						,       kernel_size=1			)						,       nn.Sigmoid()						,       )





	def       SCREAMING_SNAKE_CASE__		(			self						,       snake_case			):
		# b c h w -> b c 1 1
		lowercase	    =       self.pooler(snake_case			)
		lowercase	    =       self.attention(snake_case			)
		lowercase	    =       hidden_state * attention
		return hidden_state







class 			A_  (     nn.Module  ):




	'''simple docstring'''


	def __init__(			self						,       snake_case						,       snake_case						,       snake_case						,       snake_case = 1			):
		super().__init__()
		lowercase	    =       in_channels != out_channels or stride != 1
		lowercase	    =       max(1						,       out_channels // config.groups_width			)
		lowercase	    =       (
		    RegNetShortCut(snake_case						,       snake_case						,       stride=snake_case			) if should_apply_shortcut else nn.Identity()
		)
		lowercase	    =       nn.Sequential(
		    RegNetConvLayer(snake_case						,       snake_case						,       kernel_size=1						,       activation=config.hidden_act			)						,       RegNetConvLayer(snake_case						,       snake_case						,       stride=snake_case						,       groups=snake_case						,       activation=config.hidden_act			)						,       RegNetConvLayer(snake_case						,       snake_case						,       kernel_size=1						,       activation=snake_case			)						,       )
		lowercase	    =       ACTaFN[config.hidden_act]





	def       SCREAMING_SNAKE_CASE__		(			self						,       snake_case			):
		lowercase	    =       hidden_state
		lowercase	    =       self.layer(snake_case			)
		lowercase	    =       self.shortcut(snake_case			)
		hidden_state += residual
		lowercase	    =       self.activation(snake_case			)
		return hidden_state







class 			A_  (     nn.Module  ):




	'''simple docstring'''


	def __init__(			self						,       snake_case						,       snake_case						,       snake_case						,       snake_case = 1			):
		super().__init__()
		lowercase	    =       in_channels != out_channels or stride != 1
		lowercase	    =       max(1						,       out_channels // config.groups_width			)
		lowercase	    =       (
		    RegNetShortCut(snake_case						,       snake_case						,       stride=snake_case			) if should_apply_shortcut else nn.Identity()
		)
		lowercase	    =       nn.Sequential(
		    RegNetConvLayer(snake_case						,       snake_case						,       kernel_size=1						,       activation=config.hidden_act			)						,       RegNetConvLayer(snake_case						,       snake_case						,       stride=snake_case						,       groups=snake_case						,       activation=config.hidden_act			)						,       RegNetSELayer(snake_case						,       reduced_channels=int(round(in_channels / 4			)			)			)						,       RegNetConvLayer(snake_case						,       snake_case						,       kernel_size=1						,       activation=snake_case			)						,       )
		lowercase	    =       ACTaFN[config.hidden_act]





	def       SCREAMING_SNAKE_CASE__		(			self						,       snake_case			):
		lowercase	    =       hidden_state
		lowercase	    =       self.layer(snake_case			)
		lowercase	    =       self.shortcut(snake_case			)
		hidden_state += residual
		lowercase	    =       self.activation(snake_case			)
		return hidden_state







class 			A_  (     nn.Module  ):




	'''simple docstring'''


	def __init__(			self						,       snake_case						,       snake_case						,       snake_case						,       snake_case = 2						,       snake_case = 2						,       ):
		super().__init__()

		lowercase	    =       RegNetXLayer if config.layer_type == 'x' else RegNetYLayer

		lowercase	    =       nn.Sequential(
		    # downsampling is done in the first layer with stride of 2
		    layer(
		        snake_case						,       snake_case						,       snake_case						,       stride=snake_case						,       )						,       *[layer(snake_case						,       snake_case						,       snake_case			) for _ in range(depth - 1			)]						,       )





	def       SCREAMING_SNAKE_CASE__		(			self						,       snake_case			):
		lowercase	    =       self.layers(snake_case			)
		return hidden_state







class 			A_  (     nn.Module  ):




	'''simple docstring'''


	def __init__(			self						,       snake_case			):
		super().__init__()
		lowercase	    =       nn.ModuleList([]			)
		# based on `downsample_in_first_stage`, the first layer of the first stage may or may not downsample the input
		self.stages.append(
		    RegNetStage(
		        snake_case						,       config.embedding_size						,       config.hidden_sizes[0]						,       stride=2 if config.downsample_in_first_stage else 1						,       depth=config.depths[0]						,       )			)
		lowercase	    =       zip(config.hidden_sizes						,       config.hidden_sizes[1:]			)
		for (in_channels, out_channels), depth in zip(snake_case						,       config.depths[1:]			):
			self.stages.append(RegNetStage(snake_case						,       snake_case						,       snake_case						,       depth=snake_case			)			)





	def       SCREAMING_SNAKE_CASE__		(			self						,       snake_case						,       snake_case = False						,       snake_case = True			):
		lowercase	    =       () if output_hidden_states else None

		for stage_module in self.stages:
			if output_hidden_states:
				lowercase	    =       hidden_states + (hidden_state,)

			lowercase	    =       stage_module(snake_case			)

		if output_hidden_states:
			lowercase	    =       hidden_states + (hidden_state,)

		if not return_dict:
			return tuple(v for v in [hidden_state, hidden_states] if v is not None			)

		return BaseModelOutputWithNoAttention(last_hidden_state=snake_case						,       hidden_states=snake_case			)







class 			A_  (     __lowerCamelCase  ):




	'''simple docstring'''


	_UpperCamelCase	:	str									=			RegNetConfig
	_UpperCamelCase	:	Optional[Any]									=			"""regnet"""
	_UpperCamelCase	:	Tuple									=			"""pixel_values"""
	_UpperCamelCase	:	str									=			True





	def       SCREAMING_SNAKE_CASE__		(			self						,       snake_case			):
		if isinstance(snake_case						,       nn.Convad			):
			nn.init.kaiming_normal_(module.weight						,       mode='fan_out'						,       nonlinearity='relu'			)
		elif isinstance(snake_case						,       (nn.BatchNormad, nn.GroupNorm)			):
			nn.init.constant_(module.weight						,       1			)
			nn.init.constant_(module.bias						,       0			)





	def       SCREAMING_SNAKE_CASE__		(			self						,       snake_case						,       snake_case=False			):
		if isinstance(snake_case						,       snake_case			):
			lowercase	    =       value




UpperCAmelCase       =					R'''
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`RegNetConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
'''

UpperCAmelCase       =					R'''
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`ConvNextImageProcessor.__call__`] for details.

        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple.
'''







@add_start_docstrings(
    """The bare RegNet model outputting raw features without any specific head on top.""" ,      __lowerCamelCase ,      )
# Copied from transformers.models.resnet.modeling_resnet.ResNetModel with RESNET->REGNET,ResNet->RegNet
class 			A_  (     __lowerCamelCase  ):




	'''simple docstring'''


	def __init__(			self						,       snake_case			):
		super().__init__(snake_case			)
		lowercase	    =       config
		lowercase	    =       RegNetEmbeddings(snake_case			)
		lowercase	    =       RegNetEncoder(snake_case			)
		lowercase	    =       nn.AdaptiveAvgPoolad((1, 1)			)
		# Initialize weights and apply final processing
		self.post_init()





	@add_start_docstrings_to_model_forward(snake_case			)
	@add_code_sample_docstrings(
	    checkpoint=_CHECKPOINT_FOR_DOC						,       output_type=snake_case						,       config_class=_CONFIG_FOR_DOC						,       modality='vision'						,       expected_output=_EXPECTED_OUTPUT_SHAPE						,       )
	def       SCREAMING_SNAKE_CASE__		(			self						,       snake_case						,       snake_case = None						,       snake_case = None			):
		lowercase	    =       (
		    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
		)
		lowercase	    =       return_dict if return_dict is not None else self.config.use_return_dict

		lowercase	    =       self.embedder(snake_case			)

		lowercase	    =       self.encoder(
		    snake_case						,       output_hidden_states=snake_case						,       return_dict=snake_case			)

		lowercase	    =       encoder_outputs[0]

		lowercase	    =       self.pooler(snake_case			)

		if not return_dict:
			return (last_hidden_state, pooled_output) + encoder_outputs[1:]

		return BaseModelOutputWithPoolingAndNoAttention(
		    last_hidden_state=snake_case						,       pooler_output=snake_case						,       hidden_states=encoder_outputs.hidden_states						,       )







@add_start_docstrings(
    """
    RegNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    """ ,      __lowerCamelCase ,      )
# Copied from transformers.models.resnet.modeling_resnet.ResNetForImageClassification with RESNET->REGNET,ResNet->RegNet,resnet->regnet
class 			A_  (     __lowerCamelCase  ):




	'''simple docstring'''


	def __init__(			self						,       snake_case			):
		super().__init__(snake_case			)
		lowercase	    =       config.num_labels
		lowercase	    =       RegNetModel(snake_case			)
		# classification head
		lowercase	    =       nn.Sequential(
		    nn.Flatten()						,       nn.Linear(config.hidden_sizes[-1]						,       config.num_labels			) if config.num_labels > 0 else nn.Identity()						,       )
		# initialize weights and apply final processing
		self.post_init()





	@add_start_docstrings_to_model_forward(snake_case			)
	@add_code_sample_docstrings(
	    checkpoint=_IMAGE_CLASS_CHECKPOINT						,       output_type=snake_case						,       config_class=_CONFIG_FOR_DOC						,       expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT						,       )
	def       SCREAMING_SNAKE_CASE__		(			self						,       snake_case = None						,       snake_case = None						,       snake_case = None						,       snake_case = None						,       ):
		lowercase	    =       return_dict if return_dict is not None else self.config.use_return_dict

		lowercase	    =       self.regnet(snake_case						,       output_hidden_states=snake_case						,       return_dict=snake_case			)

		lowercase	    =       outputs.pooler_output if return_dict else outputs[1]

		lowercase	    =       self.classifier(snake_case			)

		lowercase	    =       None

		if labels is not None:
			if self.config.problem_type is None:
				if self.num_labels == 1:
					lowercase	    =       'regression'
				elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
					lowercase	    =       'single_label_classification'
				else:
					lowercase	    =       'multi_label_classification'
			if self.config.problem_type == "regression":
				lowercase	    =       MSELoss()
				if self.num_labels == 1:
					lowercase	    =       loss_fct(logits.squeeze()						,       labels.squeeze()			)
				else:
					lowercase	    =       loss_fct(snake_case						,       snake_case			)
			elif self.config.problem_type == "single_label_classification":
				lowercase	    =       CrossEntropyLoss()
				lowercase	    =       loss_fct(logits.view(-1						,       self.num_labels			)						,       labels.view(-1			)			)
			elif self.config.problem_type == "multi_label_classification":
				lowercase	    =       BCEWithLogitsLoss()
				lowercase	    =       loss_fct(snake_case						,       snake_case			)

		if not return_dict:
			lowercase	    =       (logits,) + outputs[2:]
			return (loss,) + output if loss is not None else output

		return ImageClassifierOutputWithNoAttention(loss=snake_case						,       logits=snake_case						,       hidden_states=outputs.hidden_states			)









"""simple docstring"""


import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


__SCREAMING_SNAKE_CASE   =   get_tests_dir('fixtures/test_sentencepiece_bpe_char.model')




@require_sentencepiece
@require_tokenizers
class 						__UpperCamelCase     (     UpperCamelCase   ,     unittest.TestCase						):
					lowercase_    :				Optional[int]      =  SpeechTaTokenizer
					lowercase_    :				Dict      =  False
					lowercase_    :				Any      =  True
					def        UpperCAmelCase__ (							self							:	Tuple   )	->       Optional[Any]:
											super().setUp()

											# We have a SentencePiece fixture for testing
											lowerCAmelCase   :Union[str, Any]						=			SpeechTaTokenizer(UpperCAmelCase   )

											lowerCAmelCase   :Any						=			AddedToken('<mask>'     , lstrip=UpperCAmelCase     , rstrip=UpperCAmelCase   )
											lowerCAmelCase   :List[Any]						=			mask_token
											tokenizer.add_special_tokens({'mask_token': mask_token}   )
											tokenizer.add_tokens(['<ctc_blank>']   )

											tokenizer.save_pretrained(self.tmpdirname   )
					def        UpperCAmelCase__ (							self							:	str     , UpperCAmelCase							:	Any   )	->       Union[str, Any]:
											lowerCAmelCase   :Tuple						=			'this is a test'
											lowerCAmelCase   :Union[str, Any]						=			'this is a test'
											return input_text, output_text
					def        UpperCAmelCase__ (							self							:	Optional[Any]     , UpperCAmelCase							:	Optional[int]     , UpperCAmelCase							:	Any=False     , UpperCAmelCase							:	List[Any]=20     , UpperCAmelCase							:	str=5   )	->       Optional[Any]:
											lowerCAmelCase	, lowerCAmelCase   :Dict						=			self.get_input_output_texts(UpperCAmelCase   )
											lowerCAmelCase   :str						=			tokenizer.encode(UpperCAmelCase     , add_special_tokens=UpperCAmelCase   )
											lowerCAmelCase   :Union[str, Any]						=			tokenizer.decode(UpperCAmelCase     , clean_up_tokenization_spaces=UpperCAmelCase   )
											return text, ids
					def        UpperCAmelCase__ (							self							:	Any   )	->       Optional[Any]:
											lowerCAmelCase   :List[Any]						=			'<pad>'
											lowerCAmelCase   :Any						=			1

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase   )     , UpperCAmelCase   )
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase   )     , UpperCAmelCase   )
					def        UpperCAmelCase__ (							self							:	Any   )	->       Any:
											lowerCAmelCase   :Dict						=			list(self.get_tokenizer().get_vocab().keys()   )

											self.assertEqual(vocab_keys[0]     , '<s>'   )
											self.assertEqual(vocab_keys[1]     , '<pad>'   )
											self.assertEqual(vocab_keys[-4]     , 'œ'   )
											self.assertEqual(vocab_keys[-2]     , '<mask>'   )
											self.assertEqual(vocab_keys[-1]     , '<ctc_blank>'   )
											self.assertEqual(len(UpperCAmelCase   )     , 81   )
					def        UpperCAmelCase__ (							self							:	Tuple   )	->       Union[str, Any]:
											self.assertEqual(self.get_tokenizer().vocab_size     , 79   )
					def        UpperCAmelCase__ (							self							:	Any   )	->       List[str]:
											lowerCAmelCase   :Any						=			self.get_tokenizers(do_lower_case=UpperCAmelCase   )
											for tokenizer in tokenizers:
																	with self.subTest(f"""{tokenizer.__class__.__name__}"""   ):
																							lowerCAmelCase   :Optional[int]						=			tokenizer.vocab_size
																							lowerCAmelCase   :Union[str, Any]						=			len(UpperCAmelCase   )

																							self.assertNotEqual(UpperCAmelCase     , 0   )

																							# We usually have added tokens from the start in tests because our vocab fixtures are
																							# smaller than the original vocabs - let's not assert this
																							# self.assertEqual(vocab_size, all_size)

																							lowerCAmelCase   :Optional[Any]						=			['aaaaa bbbbbb', 'cccccccccdddddddd']
																							lowerCAmelCase   :Union[str, Any]						=			tokenizer.add_tokens(UpperCAmelCase   )
																							lowerCAmelCase   :str						=			tokenizer.vocab_size
																							lowerCAmelCase   :List[Any]						=			len(UpperCAmelCase   )

																							self.assertNotEqual(UpperCAmelCase     , 0   )
																							self.assertEqual(UpperCAmelCase     , UpperCAmelCase   )
																							self.assertEqual(UpperCAmelCase     , len(UpperCAmelCase   )   )
																							self.assertEqual(UpperCAmelCase     , all_size + len(UpperCAmelCase   )   )

																							lowerCAmelCase   :int						=			tokenizer.encode('aaaaa bbbbbb low cccccccccdddddddd l'     , add_special_tokens=UpperCAmelCase   )

																							self.assertGreaterEqual(len(UpperCAmelCase   )     , 4   )
																							self.assertGreater(tokens[0]     , tokenizer.vocab_size - 1   )
																							self.assertGreater(tokens[-3]     , tokenizer.vocab_size - 1   )

																							lowerCAmelCase   :List[Any]						=			{'eos_token': '>>>>|||<||<<|<<', 'pad_token': '<<<<<|||>|>>>>|>'}
																							lowerCAmelCase   :str						=			tokenizer.add_special_tokens(UpperCAmelCase   )
																							lowerCAmelCase   :Optional[Any]						=			tokenizer.vocab_size
																							lowerCAmelCase   :Tuple						=			len(UpperCAmelCase   )

																							self.assertNotEqual(UpperCAmelCase     , 0   )
																							self.assertEqual(UpperCAmelCase     , UpperCAmelCase   )
																							self.assertEqual(UpperCAmelCase     , len(UpperCAmelCase   )   )
																							self.assertEqual(UpperCAmelCase     , all_size_a + len(UpperCAmelCase   )   )

																							lowerCAmelCase   :List[str]						=			tokenizer.encode(
																							    '>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'     , add_special_tokens=UpperCAmelCase   )

																							self.assertGreaterEqual(len(UpperCAmelCase   )     , 6   )
																							self.assertGreater(tokens[0]     , tokenizer.vocab_size - 1   )
																							self.assertGreater(tokens[0]     , tokens[1]   )
																							self.assertGreater(tokens[-3]     , tokenizer.vocab_size - 1   )
																							self.assertGreater(tokens[-3]     , tokens[-4]   )
																							self.assertEqual(tokens[0]     , tokenizer.eos_token_id   )
																							self.assertEqual(tokens[-3]     , tokenizer.pad_token_id   )
					def        UpperCAmelCase__ (							self							:	Union[str, Any]   )	->       Optional[Any]:
											pass
					def        UpperCAmelCase__ (							self							:	str   )	->       int:
											pass
					def        UpperCAmelCase__ (							self							:	Optional[int]   )	->       List[Any]:
											lowerCAmelCase   :Optional[int]						=			self.get_tokenizer()

											lowerCAmelCase   :List[str]						=			tokenizer.tokenize('This is a test'   )
											# fmt: off
											self.assertListEqual(UpperCAmelCase     , [SPIECE_UNDERLINE, 'T', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'a', SPIECE_UNDERLINE, 't', 'e', 's', 't']   )
											# fmt: on

											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(UpperCAmelCase   )     , [4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]     , )

											lowerCAmelCase   :List[str]						=			tokenizer.tokenize('I was born in 92000, and this is falsé.'   )
											self.assertListEqual(
											    UpperCAmelCase     , [SPIECE_UNDERLINE, 'I', SPIECE_UNDERLINE, 'w', 'a', 's', SPIECE_UNDERLINE, 'b', 'o', 'r', 'n', SPIECE_UNDERLINE, 'i', 'n', SPIECE_UNDERLINE, '92000', ',', SPIECE_UNDERLINE, 'a', 'n', 'd', SPIECE_UNDERLINE, 't', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'f', 'a', 'l', 's', 'é', '.']   )

											lowerCAmelCase   :Optional[int]						=			tokenizer.convert_tokens_to_ids(UpperCAmelCase   )
											# fmt: off
											self.assertListEqual(UpperCAmelCase     , [4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26]   )
											# fmt: on

											lowerCAmelCase   :int						=			tokenizer.convert_ids_to_tokens(UpperCAmelCase   )
											self.assertListEqual(
											    UpperCAmelCase     , [SPIECE_UNDERLINE, 'I', SPIECE_UNDERLINE, 'w', 'a', 's', SPIECE_UNDERLINE, 'b', 'o', 'r', 'n', SPIECE_UNDERLINE, 'i', 'n', SPIECE_UNDERLINE, '<unk>', ',', SPIECE_UNDERLINE, 'a', 'n', 'd', SPIECE_UNDERLINE, 't', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'f', 'a', 'l', 's', 'é', '.']   )
					@slow
					def        UpperCAmelCase__ (							self							:	str   )	->       Tuple:
											# Use custom sequence because this tokenizer does not handle numbers.
											lowerCAmelCase   :Any						=			[
											    'Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '
											    'general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural '
											    'Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained '
											    'models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.',
											    'BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '
											    'conditioning on both left and right context in all layers.',
											    'The quick brown fox jumps over the lazy dog.',
											]

											# fmt: off
											lowerCAmelCase   :List[str]						=			{
											    'input_ids': [
											        [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
											        [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
											        [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
											    ],
											    'attention_mask': [
											        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
											        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											    ]
											}
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=UpperCAmelCase     , model_name='microsoft/speecht5_asr'     , revision='c5ef64c71905caeccde0e4462ef3f9077224c524'     , sequences=UpperCAmelCase     , )