






"""simple docstring"""


import os
from pickle import UnpicklingError
from typing import Dict, Tuple

import jax
import jax.numpy as jnp
import numpy as np
from flax.serialization import from_bytes
from flax.traverse_util import flatten_dict, unflatten_dict

import transformers

from .utils import logging


__SCREAMING_SNAKE_CASE   =   logging.get_logger(__name__)


def        UpperCAmelCase      (					a__     ,      a__     ,      a__     ,      a__=False ):





						'''simple docstring'''






						try:
												import torch  # noqa: F401
						except ImportError:
												logger.error(
												    'Loading a PyTorch model in Flax, requires both PyTorch and Flax to be installed. Please see'
												    ' https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation'
												    ' instructions.' )
												raise

						if not is_sharded:
												lowerCAmelCase   :List[str]						=			os.path.abspath(a__ )
												logger.info(F"""Loading PyTorch weights from {pt_path}""" )

												lowerCAmelCase   :int						=			torch.load(a__     ,      map_location='cpu' )
												logger.info(F"""PyTorch checkpoint contains {sum(t.numel() for t in pt_state_dict.values() ):,} parameters.""" )

												lowerCAmelCase   :Optional[Any]						=			convert_pytorch_state_dict_to_flax(a__     ,      a__ )
						else:
												# model is sharded and pytorch_checkpoint_path already contains the list of .pt shard files
												lowerCAmelCase   :int						=			convert_pytorch_sharded_state_dict_to_flax(a__     ,      a__ )
						return flax_state_dict


def        UpperCAmelCase      (					a__     ,      a__     ,      a__     ,      a__     ,      ):





						'''simple docstring'''






						def is_key_or_prefix_key_in_dict(a__ ) -> bool:
												return len(set(a__ ) & {key, (model_prefix,) + key} ) > 0

						# layer norm
						lowerCAmelCase   :List[Any]						=			pt_tuple_key[:-1] + ('scale',)
						if pt_tuple_key[-1] in ["weight", "gamma"] and is_key_or_prefix_key_in_dict(a__ ):
												return renamed_pt_tuple_key, pt_tensor

						# batch norm layer mean
						lowerCAmelCase   :str						=			pt_tuple_key[:-1] + ('mean',)
						if pt_tuple_key[-1] == "running_mean" and not is_key_or_prefix_key_in_dict(a__ ):
												return renamed_pt_tuple_key, pt_tensor

						# batch norm layer var
						lowerCAmelCase   :List[Any]						=			pt_tuple_key[:-1] + ('var',)
						if pt_tuple_key[-1] == "running_var" and not is_key_or_prefix_key_in_dict(a__ ):
												return renamed_pt_tuple_key, pt_tensor

						# embedding
						lowerCAmelCase   :List[Any]						=			pt_tuple_key[:-1] + ('embedding',)
						if pt_tuple_key[-1] == "weight" and is_key_or_prefix_key_in_dict(a__ ):
												return renamed_pt_tuple_key, pt_tensor

						# conv layer
						lowerCAmelCase   :Any						=			pt_tuple_key[:-1] + ('kernel',)
						if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4 and not is_key_or_prefix_key_in_dict(a__ ):
												lowerCAmelCase   :str						=			pt_tensor.transpose(2     ,      3     ,      1     ,      0 )
												return renamed_pt_tuple_key, pt_tensor

						# linear layer
						lowerCAmelCase   :int						=			pt_tuple_key[:-1] + ('kernel',)
						if pt_tuple_key[-1] == "weight" and not is_key_or_prefix_key_in_dict(a__ ):
												lowerCAmelCase   :Dict						=			pt_tensor.T
												return renamed_pt_tuple_key, pt_tensor

						# old PyTorch layer norm weight
						lowerCAmelCase   :Optional[Any]						=			pt_tuple_key[:-1] + ('weight',)
						if pt_tuple_key[-1] == "gamma":
												return renamed_pt_tuple_key, pt_tensor

						# old PyTorch layer norm bias
						lowerCAmelCase   :Any						=			pt_tuple_key[:-1] + ('bias',)
						if pt_tuple_key[-1] == "beta":
												return renamed_pt_tuple_key, pt_tensor

						# New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
						lowerCAmelCase   :Optional[int]						=			None
						if pt_tuple_key[-3::2] == ("parametrizations", "original0"):
												lowerCAmelCase   :List[str]						=			pt_tuple_key[-2] + '_g'
						elif pt_tuple_key[-3::2] == ("parametrizations", "original1"):
												lowerCAmelCase   :Any						=			pt_tuple_key[-2] + '_v'
						if name is not None:
												lowerCAmelCase   :Union[str, Any]						=			pt_tuple_key[:-3] + (name,)
												return renamed_pt_tuple_key, pt_tensor

						return pt_tuple_key, pt_tensor


def        UpperCAmelCase      (					a__     ,      a__ ):





						'''simple docstring'''






						lowerCAmelCase   :Union[str, Any]						=			{k: v.numpy() for k, v in pt_state_dict.items()}

						lowerCAmelCase   :Dict						=			flax_model.base_model_prefix

						# use params dict if the model contains batch norm layers
						if "params" in flax_model.params:
												lowerCAmelCase   :Optional[Any]						=			flax_model.params['params']
						else:
												lowerCAmelCase   :Optional[Any]						=			flax_model.params
						lowerCAmelCase   :Tuple						=			flatten_dict(a__ )

						# add batch_stats keys,values to dict
						if "batch_stats" in flax_model.params:
												lowerCAmelCase   :List[str]						=			flatten_dict(flax_model.params['batch_stats'] )
												random_flax_state_dict.update(a__ )

						lowerCAmelCase   :Union[str, Any]						=			{}

						lowerCAmelCase   :int						=			(model_prefix not in flax_model_params) and (
						    model_prefix in {k.split('.' )[0] for k in pt_state_dict.keys()}
						)
						lowerCAmelCase   :Union[str, Any]						=			(model_prefix in flax_model_params) and (
						    model_prefix not in {k.split('.' )[0] for k in pt_state_dict.keys()}
						)

						# Need to change some parameters name to match Flax names
						for pt_key, pt_tensor in pt_state_dict.items():
												lowerCAmelCase   :Union[str, Any]						=			tuple(pt_key.split('.' ) )

												# remove base model prefix if necessary
												lowerCAmelCase   :List[Any]						=			pt_tuple_key[0] == model_prefix
												if load_model_with_head_into_base_model and has_base_model_prefix:
																		lowerCAmelCase   :List[str]						=			pt_tuple_key[1:]

												# Correctly rename weight parameters
												lowerCAmelCase	, lowerCAmelCase   :List[Any]						=			rename_key_and_reshape_tensor(
												    a__     ,      a__     ,      a__     ,      a__ )

												# add model prefix if necessary
												lowerCAmelCase   :List[str]						=			(model_prefix,) + flax_key in random_flax_state_dict
												if load_base_model_into_model_with_head and require_base_model_prefix:
																		lowerCAmelCase   :Dict						=			(model_prefix,) + flax_key

												if flax_key in random_flax_state_dict:
																		if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
																								raise ValueError(
																								    F"""PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape """
																								    F"""{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.""" )

        # add batch stats if the model contains batchnorm layers
												if "batch_stats" in flax_model.params:
																		if "mean" in flax_key[-1] or "var" in flax_key[-1]:
																								lowerCAmelCase   :Optional[int]						=			jnp.asarray(a__ )
																								continue
																		# remove num_batches_tracked key
																		if "num_batches_tracked" in flax_key[-1]:
																								flax_state_dict.pop(a__     ,      a__ )
																								continue

																		# also add unexpected weight so that warning is thrown
																		lowerCAmelCase   :Any						=			jnp.asarray(a__ )

												else:
																		# also add unexpected weight so that warning is thrown
																		lowerCAmelCase   :str						=			jnp.asarray(a__ )

						return unflatten_dict(a__ )


def        UpperCAmelCase      (					a__     ,      a__ ):





						'''simple docstring'''






						import torch

						# Load the index
						lowerCAmelCase   :Optional[Any]						=			{}
						for shard_file in shard_filenames:
												# load using msgpack utils
												lowerCAmelCase   :List[Any]						=			torch.load(a__ )
												lowerCAmelCase   :int						=			{k: v.numpy() for k, v in pt_state_dict.items()}

												lowerCAmelCase   :Optional[Any]						=			flax_model.base_model_prefix

												# use params dict if the model contains batch norm layers and then add batch_stats keys,values to dict
												if "batch_stats" in flax_model.params:
																		lowerCAmelCase   :Optional[Any]						=			flax_model.params['params']

																		lowerCAmelCase   :List[Any]						=			flatten_dict(a__ )
																		random_flax_state_dict.update(flatten_dict(flax_model.params['batch_stats'] ) )
												else:
																		lowerCAmelCase   :Optional[int]						=			flax_model.params
																		lowerCAmelCase   :Dict						=			flatten_dict(a__ )

												lowerCAmelCase   :Tuple						=			(model_prefix not in flax_model_params) and (
												    model_prefix in {k.split('.' )[0] for k in pt_state_dict.keys()}
												)
												lowerCAmelCase   :Optional[Any]						=			(model_prefix in flax_model_params) and (
												    model_prefix not in {k.split('.' )[0] for k in pt_state_dict.keys()}
												)
												# Need to change some parameters name to match Flax names
												for pt_key, pt_tensor in pt_state_dict.items():
																		lowerCAmelCase   :Dict						=			tuple(pt_key.split('.' ) )

																		# remove base model prefix if necessary
																		lowerCAmelCase   :List[str]						=			pt_tuple_key[0] == model_prefix
																		if load_model_with_head_into_base_model and has_base_model_prefix:
																								lowerCAmelCase   :List[Any]						=			pt_tuple_key[1:]

																		# Correctly rename weight parameters
																		lowerCAmelCase	, lowerCAmelCase   :Optional[int]						=			rename_key_and_reshape_tensor(
																		    a__     ,      a__     ,      a__     ,      a__ )
																		# add model prefix if necessary
																		lowerCAmelCase   :int						=			(model_prefix,) + flax_key in random_flax_state_dict
																		if load_base_model_into_model_with_head and require_base_model_prefix:
																								lowerCAmelCase   :Tuple						=			(model_prefix,) + flax_key

																		if flax_key in random_flax_state_dict:
																								if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
																														raise ValueError(
																														    F"""PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape """
																														    F"""{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.""" )

            # add batch stats if the model contains batchnorm layers
																		if "batch_stats" in flax_model.params:
																								if "mean" in flax_key[-1]:
																														lowerCAmelCase   :Optional[int]						=			jnp.asarray(a__ )
																														continue
																								if "var" in flax_key[-1]:
																														lowerCAmelCase   :str						=			jnp.asarray(a__ )
																														continue
																								# remove num_batches_tracked key
																								if "num_batches_tracked" in flax_key[-1]:
																														flax_state_dict.pop(a__     ,      a__ )
																														continue

																								# also add unexpected weight so that warning is thrown
																								lowerCAmelCase   :Optional[int]						=			jnp.asarray(a__ )

																		else:
																								# also add unexpected weight so that warning is thrown
																								lowerCAmelCase   :str						=			jnp.asarray(a__ )
						return unflatten_dict(a__ )


def        UpperCAmelCase      (					a__     ,      a__ ):





						'''simple docstring'''






						lowerCAmelCase   :Tuple						=			os.path.abspath(a__ )
						logger.info(F"""Loading Flax weights from {flax_checkpoint_path}""" )

						# import correct flax class
						lowerCAmelCase   :List[str]						=			getattr(a__     ,      'Flax' + model.__class__.__name__ )

						# load flax weight dict
						with open(a__     ,      'rb' ) as state_f:
												try:
																		lowerCAmelCase   :List[str]						=			from_bytes(a__     ,      state_f.read() )
												except UnpicklingError:
																		raise EnvironmentError(F"""Unable to convert {flax_checkpoint_path} to Flax deserializable object. """ )

						return load_flax_weights_in_pytorch_model(a__     ,      a__ )







def        UpperCAmelCase      (					a__     ,      a__ ):





						'''simple docstring'''






						try:
												import torch  # noqa: F401
						except ImportError:
												logger.error(
												    'Loading a Flax weights in PyTorch, requires both PyTorch and Flax to be installed. Please see'
												    ' https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation'
												    ' instructions.' )
												raise

						# check if we have bf16 weights
						lowerCAmelCase   :Optional[int]						=			flatten_dict(jax.tree_util.tree_map(lambda a__ : x.dtype == jnp.bfloataa     ,      a__ ) ).values()
						if any(a__ ):
												# convert all weights to fp32 if the are bf16 since torch.from_numpy can-not handle bf16
												# and bf16 is not fully supported in PT yet.
												logger.warning(
												    'Found ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` '
												    'before loading those in PyTorch model.' )
												lowerCAmelCase   :Dict						=			jax.tree_util.tree_map(
												    lambda a__ : params.astype(np.floataa ) if params.dtype == jnp.bfloataa else params     ,      a__ )

						lowerCAmelCase   :Tuple						=			flatten_dict(a__ )
						lowerCAmelCase   :Any						=			pt_model.state_dict()

						lowerCAmelCase   :str						=			(pt_model.base_model_prefix in flax_state) and (
						    pt_model.base_model_prefix not in {k.split('.' )[0] for k in pt_model_dict.keys()}
						)
						lowerCAmelCase   :List[Any]						=			(pt_model.base_model_prefix not in flax_state) and (
						    pt_model.base_model_prefix in {k.split('.' )[0] for k in pt_model_dict.keys()}
						)

						# keep track of unexpected & missing keys
						lowerCAmelCase   :Any						=			[]
						lowerCAmelCase   :List[Any]						=			set(pt_model_dict.keys() )

						for flax_key_tuple, flax_tensor in flax_state_dict.items():
												lowerCAmelCase   :Optional[int]						=			flax_key_tuple[0] == pt_model.base_model_prefix
												lowerCAmelCase   :Dict						=			'.'.join((pt_model.base_model_prefix,) + flax_key_tuple ) in pt_model_dict

												# adapt flax_key to prepare for loading from/to base model only
												if load_model_with_head_into_base_model and has_base_model_prefix:
																		lowerCAmelCase   :Tuple						=			flax_key_tuple[1:]
												elif load_base_model_into_model_with_head and require_base_model_prefix:
																		lowerCAmelCase   :int						=			(pt_model.base_model_prefix,) + flax_key_tuple

												# rename flax weights to PyTorch format
												if flax_key_tuple[-1] == "kernel" and flax_tensor.ndim == 4 and ".".join(a__ ) not in pt_model_dict:
																		# conv layer
																		lowerCAmelCase   :Optional[Any]						=			flax_key_tuple[:-1] + ('weight',)
																		lowerCAmelCase   :Union[str, Any]						=			jnp.transpose(a__     ,      (3, 2, 0, 1) )
												elif flax_key_tuple[-1] == "kernel" and ".".join(a__ ) not in pt_model_dict:
																		# linear layer
																		lowerCAmelCase   :List[str]						=			flax_key_tuple[:-1] + ('weight',)
																		lowerCAmelCase   :int						=			flax_tensor.T
												elif flax_key_tuple[-1] in ["scale", "embedding"]:
																		lowerCAmelCase   :Dict						=			flax_key_tuple[:-1] + ('weight',)

												# adding batch stats from flax batch norm to pt
												elif "mean" in flax_key_tuple[-1]:
																		lowerCAmelCase   :List[Any]						=			flax_key_tuple[:-1] + ('running_mean',)
												elif "var" in flax_key_tuple[-1]:
																		lowerCAmelCase   :Optional[Any]						=			flax_key_tuple[:-1] + ('running_var',)

												if "batch_stats" in flax_state:
																		lowerCAmelCase   :Optional[Any]						=			'.'.join(flax_key_tuple[1:] )  # Remove the params/batch_stats header
												else:
																		lowerCAmelCase   :Tuple						=			'.'.join(a__ )

												# We also need to look at `pt_model_dict` and see if there are keys requiring further transformation.
												lowerCAmelCase   :int						=			{}
												# New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
												for key in pt_model_dict:
																		lowerCAmelCase   :List[str]						=			key.split('.' )
																		lowerCAmelCase   :List[Any]						=			None
																		if key_components[-3::2] == ["parametrizations", "original0"]:
																								lowerCAmelCase   :Union[str, Any]						=			key_components[-2] + '_g'
																		elif key_components[-3::2] == ["parametrizations", "original1"]:
																								lowerCAmelCase   :Optional[Any]						=			key_components[-2] + '_v'
																		if name is not None:
																								lowerCAmelCase   :Optional[int]						=			key_components[:-3] + [name]
																								lowerCAmelCase   :Union[str, Any]						=			'.'.join(a__ )
																								lowerCAmelCase   :str						=			key

												if flax_key in special_pt_names:
																		lowerCAmelCase   :Tuple						=			special_pt_names[flax_key]

												if flax_key in pt_model_dict:
																		if flax_tensor.shape != pt_model_dict[flax_key].shape:
																								raise ValueError(
																								    F"""Flax checkpoint seems to be incorrect. Weight {flax_key_tuple} was expected """
																								    F"""to be of shape {pt_model_dict[flax_key].shape}, but is {flax_tensor.shape}.""" )
																		else:
																								# add weight to pytorch dict
																								lowerCAmelCase   :str						=			np.asarray(a__ ) if not isinstance(a__     ,      np.ndarray ) else flax_tensor
																								lowerCAmelCase   :Optional[Any]						=			torch.from_numpy(a__ )
																								# remove from missing keys
																								missing_keys.remove(a__ )
												else:
																		# weight is not expected by PyTorch model
																		unexpected_keys.append(a__ )

						pt_model.load_state_dict(a__ )

						# re-transform missing_keys to list
						lowerCAmelCase   :Dict						=			list(a__ )

						if len(a__ ) > 0:
												logger.warning(
												    'Some weights of the Flax model were not used when initializing the PyTorch model'
												    F""" {pt_model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are initializing"""
												    F""" {pt_model.__class__.__name__} from a Flax model trained on another task or with another architecture"""
												    ' (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).\n- This'
												    F""" IS NOT expected if you are initializing {pt_model.__class__.__name__} from a Flax model that you expect"""
												    ' to be exactly identical (e.g. initializing a BertForSequenceClassification model from a'
												    ' FlaxBertForSequenceClassification model).' )
						else:
												logger.warning(F"""All Flax model weights were used when initializing {pt_model.__class__.__name__}.\n""" )
						if len(a__ ) > 0:
												logger.warning(
												    F"""Some weights of {pt_model.__class__.__name__} were not initialized from the Flax model and are newly"""
												    F""" initialized: {missing_keys}\nYou should probably TRAIN this model on a down-stream task to be able to"""
												    ' use it for predictions and inference.' )
						else:
												logger.warning(
												    F"""All the weights of {pt_model.__class__.__name__} were initialized from the Flax model.\n"""
												    'If your task is similar to the task the model of the checkpoint was trained on, '
												    F"""you can already use {pt_model.__class__.__name__} for predictions without further training.""" )

						return pt_model







"""simple docstring"""


import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


__SCREAMING_SNAKE_CASE   =   get_tests_dir('fixtures/test_sentencepiece_bpe_char.model')




@require_sentencepiece
@require_tokenizers
class 						__UpperCamelCase     (     UpperCamelCase   ,     unittest.TestCase						):
					lowercase_    :				Optional[int]      =  SpeechTaTokenizer
					lowercase_    :				Dict      =  False
					lowercase_    :				Any      =  True
					def        UpperCAmelCase__ (							self							:	Tuple   )	->       Optional[Any]:
											super().setUp()

											# We have a SentencePiece fixture for testing
											lowerCAmelCase   :Union[str, Any]						=			SpeechTaTokenizer(UpperCAmelCase   )

											lowerCAmelCase   :Any						=			AddedToken('<mask>'     , lstrip=UpperCAmelCase     , rstrip=UpperCAmelCase   )
											lowerCAmelCase   :List[Any]						=			mask_token
											tokenizer.add_special_tokens({'mask_token': mask_token}   )
											tokenizer.add_tokens(['<ctc_blank>']   )

											tokenizer.save_pretrained(self.tmpdirname   )
					def        UpperCAmelCase__ (							self							:	str     , UpperCAmelCase							:	Any   )	->       Union[str, Any]:
											lowerCAmelCase   :Tuple						=			'this is a test'
											lowerCAmelCase   :Union[str, Any]						=			'this is a test'
											return input_text, output_text
					def        UpperCAmelCase__ (							self							:	Optional[Any]     , UpperCAmelCase							:	Optional[int]     , UpperCAmelCase							:	Any=False     , UpperCAmelCase							:	List[Any]=20     , UpperCAmelCase							:	str=5   )	->       Optional[Any]:
											lowerCAmelCase	, lowerCAmelCase   :Dict						=			self.get_input_output_texts(UpperCAmelCase   )
											lowerCAmelCase   :str						=			tokenizer.encode(UpperCAmelCase     , add_special_tokens=UpperCAmelCase   )
											lowerCAmelCase   :Union[str, Any]						=			tokenizer.decode(UpperCAmelCase     , clean_up_tokenization_spaces=UpperCAmelCase   )
											return text, ids
					def        UpperCAmelCase__ (							self							:	Any   )	->       Optional[Any]:
											lowerCAmelCase   :List[Any]						=			'<pad>'
											lowerCAmelCase   :Any						=			1

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase   )     , UpperCAmelCase   )
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase   )     , UpperCAmelCase   )
					def        UpperCAmelCase__ (							self							:	Any   )	->       Any:
											lowerCAmelCase   :Dict						=			list(self.get_tokenizer().get_vocab().keys()   )

											self.assertEqual(vocab_keys[0]     , '<s>'   )
											self.assertEqual(vocab_keys[1]     , '<pad>'   )
											self.assertEqual(vocab_keys[-4]     , 'œ'   )
											self.assertEqual(vocab_keys[-2]     , '<mask>'   )
											self.assertEqual(vocab_keys[-1]     , '<ctc_blank>'   )
											self.assertEqual(len(UpperCAmelCase   )     , 81   )
					def        UpperCAmelCase__ (							self							:	Tuple   )	->       Union[str, Any]:
											self.assertEqual(self.get_tokenizer().vocab_size     , 79   )
					def        UpperCAmelCase__ (							self							:	Any   )	->       List[str]:
											lowerCAmelCase   :Any						=			self.get_tokenizers(do_lower_case=UpperCAmelCase   )
											for tokenizer in tokenizers:
																	with self.subTest(f"""{tokenizer.__class__.__name__}"""   ):
																							lowerCAmelCase   :Optional[int]						=			tokenizer.vocab_size
																							lowerCAmelCase   :Union[str, Any]						=			len(UpperCAmelCase   )

																							self.assertNotEqual(UpperCAmelCase     , 0   )

																							# We usually have added tokens from the start in tests because our vocab fixtures are
																							# smaller than the original vocabs - let's not assert this
																							# self.assertEqual(vocab_size, all_size)

																							lowerCAmelCase   :Optional[Any]						=			['aaaaa bbbbbb', 'cccccccccdddddddd']
																							lowerCAmelCase   :Union[str, Any]						=			tokenizer.add_tokens(UpperCAmelCase   )
																							lowerCAmelCase   :str						=			tokenizer.vocab_size
																							lowerCAmelCase   :List[Any]						=			len(UpperCAmelCase   )

																							self.assertNotEqual(UpperCAmelCase     , 0   )
																							self.assertEqual(UpperCAmelCase     , UpperCAmelCase   )
																							self.assertEqual(UpperCAmelCase     , len(UpperCAmelCase   )   )
																							self.assertEqual(UpperCAmelCase     , all_size + len(UpperCAmelCase   )   )

																							lowerCAmelCase   :int						=			tokenizer.encode('aaaaa bbbbbb low cccccccccdddddddd l'     , add_special_tokens=UpperCAmelCase   )

																							self.assertGreaterEqual(len(UpperCAmelCase   )     , 4   )
																							self.assertGreater(tokens[0]     , tokenizer.vocab_size - 1   )
																							self.assertGreater(tokens[-3]     , tokenizer.vocab_size - 1   )

																							lowerCAmelCase   :List[Any]						=			{'eos_token': '>>>>|||<||<<|<<', 'pad_token': '<<<<<|||>|>>>>|>'}
																							lowerCAmelCase   :str						=			tokenizer.add_special_tokens(UpperCAmelCase   )
																							lowerCAmelCase   :Optional[Any]						=			tokenizer.vocab_size
																							lowerCAmelCase   :Tuple						=			len(UpperCAmelCase   )

																							self.assertNotEqual(UpperCAmelCase     , 0   )
																							self.assertEqual(UpperCAmelCase     , UpperCAmelCase   )
																							self.assertEqual(UpperCAmelCase     , len(UpperCAmelCase   )   )
																							self.assertEqual(UpperCAmelCase     , all_size_a + len(UpperCAmelCase   )   )

																							lowerCAmelCase   :List[str]						=			tokenizer.encode(
																							    '>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'     , add_special_tokens=UpperCAmelCase   )

																							self.assertGreaterEqual(len(UpperCAmelCase   )     , 6   )
																							self.assertGreater(tokens[0]     , tokenizer.vocab_size - 1   )
																							self.assertGreater(tokens[0]     , tokens[1]   )
																							self.assertGreater(tokens[-3]     , tokenizer.vocab_size - 1   )
																							self.assertGreater(tokens[-3]     , tokens[-4]   )
																							self.assertEqual(tokens[0]     , tokenizer.eos_token_id   )
																							self.assertEqual(tokens[-3]     , tokenizer.pad_token_id   )
					def        UpperCAmelCase__ (							self							:	Union[str, Any]   )	->       Optional[Any]:
											pass
					def        UpperCAmelCase__ (							self							:	str   )	->       int:
											pass
					def        UpperCAmelCase__ (							self							:	Optional[int]   )	->       List[Any]:
											lowerCAmelCase   :Optional[int]						=			self.get_tokenizer()

											lowerCAmelCase   :List[str]						=			tokenizer.tokenize('This is a test'   )
											# fmt: off
											self.assertListEqual(UpperCAmelCase     , [SPIECE_UNDERLINE, 'T', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'a', SPIECE_UNDERLINE, 't', 'e', 's', 't']   )
											# fmt: on

											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(UpperCAmelCase   )     , [4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]     , )

											lowerCAmelCase   :List[str]						=			tokenizer.tokenize('I was born in 92000, and this is falsé.'   )
											self.assertListEqual(
											    UpperCAmelCase     , [SPIECE_UNDERLINE, 'I', SPIECE_UNDERLINE, 'w', 'a', 's', SPIECE_UNDERLINE, 'b', 'o', 'r', 'n', SPIECE_UNDERLINE, 'i', 'n', SPIECE_UNDERLINE, '92000', ',', SPIECE_UNDERLINE, 'a', 'n', 'd', SPIECE_UNDERLINE, 't', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'f', 'a', 'l', 's', 'é', '.']   )

											lowerCAmelCase   :Optional[int]						=			tokenizer.convert_tokens_to_ids(UpperCAmelCase   )
											# fmt: off
											self.assertListEqual(UpperCAmelCase     , [4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26]   )
											# fmt: on

											lowerCAmelCase   :int						=			tokenizer.convert_ids_to_tokens(UpperCAmelCase   )
											self.assertListEqual(
											    UpperCAmelCase     , [SPIECE_UNDERLINE, 'I', SPIECE_UNDERLINE, 'w', 'a', 's', SPIECE_UNDERLINE, 'b', 'o', 'r', 'n', SPIECE_UNDERLINE, 'i', 'n', SPIECE_UNDERLINE, '<unk>', ',', SPIECE_UNDERLINE, 'a', 'n', 'd', SPIECE_UNDERLINE, 't', 'h', 'i', 's', SPIECE_UNDERLINE, 'i', 's', SPIECE_UNDERLINE, 'f', 'a', 'l', 's', 'é', '.']   )
					@slow
					def        UpperCAmelCase__ (							self							:	str   )	->       Tuple:
											# Use custom sequence because this tokenizer does not handle numbers.
											lowerCAmelCase   :Any						=			[
											    'Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '
											    'general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural '
											    'Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained '
											    'models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.',
											    'BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '
											    'conditioning on both left and right context in all layers.',
											    'The quick brown fox jumps over the lazy dog.',
											]

											# fmt: off
											lowerCAmelCase   :List[str]						=			{
											    'input_ids': [
											        [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
											        [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
											        [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
											    ],
											    'attention_mask': [
											        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
											        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											    ]
											}
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=UpperCAmelCase     , model_name='microsoft/speecht5_asr'     , revision='c5ef64c71905caeccde0e4462ef3f9077224c524'     , sequences=UpperCAmelCase     , )