"""simple docstring"""




from typing import Optional, Tuple, Union

import tensorflow as tf

from ...activations_tf import ACTaFN
from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
from ...modeling_tf_outputs import (
    TFBaseModelOutputWithNoAttention,
    TFBaseModelOutputWithPoolingAndNoAttention,
    TFSequenceClassifierOutput,
)
from ...modeling_tf_utils import TFPreTrainedModel, TFSequenceClassificationLoss, keras_serializable, unpack_inputs
from ...tf_utils import shape_list
from ...utils import logging
from .configuration_regnet import RegNetConfig


__lowerCamelCase											=  logging.get_logger(__name__)

# General docstring
__lowerCamelCase											=  "RegNetConfig"

# Base docstring
__lowerCamelCase											=  "facebook/regnet-y-040"
__lowerCamelCase											=  [1, 10_88, 7, 7]

# Image classification docstring
__lowerCamelCase											=  "facebook/regnet-y-040"
__lowerCamelCase											=  "tabby, tabby cat"

__lowerCamelCase											=  [
    "facebook/regnet-y-040",
    # See all regnet models at https://huggingface.co/models?filter=regnet
]
class     UpperCamelCase__(			tf.keras.layers.Layer ):





      def __init__(       self		,__UpperCAmelCase		,__UpperCAmelCase = 3		,__UpperCAmelCase = 1		,__UpperCAmelCase = 1		,__UpperCAmelCase = "relu"		,**__UpperCAmelCase		,)					->				str:
             super().__init__(**__UpperCAmelCase     )
             # The padding and conv has been verified in
             # https://colab.research.google.com/gist/sayakpaul/854bc10eeaf21c9ee2119e0b9f3841a7/scratchpad.ipynb
             A__	     =							tf.keras.layers.ZeroPaddingaD(padding=kernel_size // 2     )
             A__	     =							tf.keras.layers.ConvaD(
                 filters=__UpperCAmelCase		,kernel_size=__UpperCAmelCase		,strides=__UpperCAmelCase		,padding='VALID'		,groups=__UpperCAmelCase		,use_bias=__UpperCAmelCase		,name='convolution'		,)
             A__	     =							tf.keras.layers.BatchNormalization(epsilon=1e-5		,momentum=0.9		,name='normalization'     )
             A__	     =							ACTaFN[activation] if activation is not None else tf.identity





      def 				snake_case__	(       self		,__UpperCAmelCase     )					->				Optional[Any]:
             A__	     =							self.convolution(self.padding(__UpperCAmelCase     )     )
             A__	     =							self.normalization(__UpperCAmelCase     )
             A__	     =							self.activation(__UpperCAmelCase     )
             return hidden_state






class     UpperCamelCase__(			tf.keras.layers.Layer ):





      def __init__(       self		,__UpperCAmelCase		,**__UpperCAmelCase     )					->				Optional[Any]:
             super().__init__(**__UpperCAmelCase     )
             A__	     =							config.num_channels
             A__	     =							TFRegNetConvLayer(
                 out_channels=config.embedding_size		,kernel_size=3		,stride=2		,activation=config.hidden_act		,name='embedder'		,)





      def 				snake_case__	(       self		,__UpperCAmelCase     )					->				Any:
             A__	     =							shape_list(__UpperCAmelCase     )[1]
             if tf.executing_eagerly() and num_channels != self.num_channels:
                    raise ValueError(
                        'Make sure that the channel dimension of the pixel values match with the one set in the configuration.'     )

             # When running on CPU, `tf.keras.layers.Conv2D` doesn't support `NCHW` format.
             # So change the input format from `NCHW` to `NHWC`.
             # shape = (batch_size, in_height, in_width, in_channels=num_channels)
             A__	     =							tf.transpose(__UpperCAmelCase		,perm=(0, 2, 3, 1)     )
             A__	     =							self.embedder(__UpperCAmelCase     )
             return hidden_state






class     UpperCamelCase__(			tf.keras.layers.Layer ):





      def __init__(       self		,__UpperCAmelCase		,__UpperCAmelCase = 2		,**__UpperCAmelCase     )					->				Optional[Any]:
             super().__init__(**__UpperCAmelCase     )
             A__	     =							tf.keras.layers.ConvaD(
                 filters=__UpperCAmelCase		,kernel_size=1		,strides=__UpperCAmelCase		,use_bias=__UpperCAmelCase		,name='convolution'     )
             A__	     =							tf.keras.layers.BatchNormalization(epsilon=1e-5		,momentum=0.9		,name='normalization'     )





      def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase = False     )					->				tf.Tensor:
             return self.normalization(self.convolution(__UpperCAmelCase     )		,training=__UpperCAmelCase     )






class     UpperCamelCase__(			tf.keras.layers.Layer ):





      def __init__(       self		,__UpperCAmelCase		,__UpperCAmelCase		,**__UpperCAmelCase     )					->				Optional[int]:
             super().__init__(**__UpperCAmelCase     )
             A__	     =							tf.keras.layers.GlobalAveragePoolingaD(keepdims=__UpperCAmelCase		,name='pooler'     )
             A__	     =							[
                 tf.keras.layers.ConvaD(filters=__UpperCAmelCase		,kernel_size=1		,activation='relu'		,name='attention.0'     ),
                 tf.keras.layers.ConvaD(filters=__UpperCAmelCase		,kernel_size=1		,activation='sigmoid'		,name='attention.2'     ),
             ]





      def 				snake_case__	(       self		,__UpperCAmelCase     )					->				int:
             # [batch_size, h, w, num_channels] -> [batch_size, 1, 1, num_channels]
             A__	     =							self.pooler(__UpperCAmelCase     )
             for layer_module in self.attention:
                    A__	     =							layer_module(__UpperCAmelCase     )
             A__	     =							hidden_state * pooled
             return hidden_state






class     UpperCamelCase__(			tf.keras.layers.Layer ):





      def __init__(       self		,__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase = 1		,**__UpperCAmelCase     )					->				List[str]:
             super().__init__(**__UpperCAmelCase     )
             A__	     =							in_channels != out_channels or stride != 1
             A__	     =							max(1		,out_channels // config.groups_width     )
             A__	     =							(
                 TFRegNetShortCut(__UpperCAmelCase		,stride=__UpperCAmelCase		,name='shortcut'     )
                 if should_apply_shortcut
                 else tf.keras.layers.Activation('linear'		,name='shortcut'     )
             )
             # `self.layers` instead of `self.layer` because that is a reserved argument.
             A__	     =							[
                 TFRegNetConvLayer(__UpperCAmelCase		,kernel_size=1		,activation=config.hidden_act		,name='layer.0'     ),
                 TFRegNetConvLayer(
                     __UpperCAmelCase		,stride=__UpperCAmelCase		,groups=__UpperCAmelCase		,activation=config.hidden_act		,name='layer.1'     ),
                 TFRegNetConvLayer(__UpperCAmelCase		,kernel_size=1		,activation=__UpperCAmelCase		,name='layer.2'     ),
             ]
             A__	     =							ACTaFN[config.hidden_act]





      def 				snake_case__	(       self		,__UpperCAmelCase     )					->				List[Any]:
             A__	     =							hidden_state
             for layer_module in self.layers:
                    A__	     =							layer_module(__UpperCAmelCase     )
             A__	     =							self.shortcut(__UpperCAmelCase     )
             hidden_state += residual
             A__	     =							self.activation(__UpperCAmelCase     )
             return hidden_state






class     UpperCamelCase__(			tf.keras.layers.Layer ):





      def __init__(       self		,__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase = 1		,**__UpperCAmelCase     )					->				Optional[int]:
             super().__init__(**__UpperCAmelCase     )
             A__	     =							in_channels != out_channels or stride != 1
             A__	     =							max(1		,out_channels // config.groups_width     )
             A__	     =							(
                 TFRegNetShortCut(__UpperCAmelCase		,stride=__UpperCAmelCase		,name='shortcut'     )
                 if should_apply_shortcut
                 else tf.keras.layers.Activation('linear'		,name='shortcut'     )
             )
             A__	     =							[
                 TFRegNetConvLayer(__UpperCAmelCase		,kernel_size=1		,activation=config.hidden_act		,name='layer.0'     ),
                 TFRegNetConvLayer(
                     __UpperCAmelCase		,stride=__UpperCAmelCase		,groups=__UpperCAmelCase		,activation=config.hidden_act		,name='layer.1'     ),
                 TFRegNetSELayer(__UpperCAmelCase		,reduced_channels=int(round(in_channels / 4     )     )		,name='layer.2'     ),
                 TFRegNetConvLayer(__UpperCAmelCase		,kernel_size=1		,activation=__UpperCAmelCase		,name='layer.3'     ),
             ]
             A__	     =							ACTaFN[config.hidden_act]





      def 				snake_case__	(       self		,__UpperCAmelCase     )					->				Tuple:
             A__	     =							hidden_state
             for layer_module in self.layers:
                    A__	     =							layer_module(__UpperCAmelCase     )
             A__	     =							self.shortcut(__UpperCAmelCase     )
             hidden_state += residual
             A__	     =							self.activation(__UpperCAmelCase     )
             return hidden_state






class     UpperCamelCase__(			tf.keras.layers.Layer ):





      def __init__(       self		,__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase = 2		,__UpperCAmelCase = 2		,**__UpperCAmelCase     )					->				Dict:
             super().__init__(**__UpperCAmelCase     )

             A__	     =							TFRegNetXLayer if config.layer_type == 'x' else TFRegNetYLayer
             A__	     =							[
                 # downsampling is done in the first layer with stride of 2
                 layer(__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase		,stride=__UpperCAmelCase		,name='layers.0'     ),
                 *[layer(__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase		,name=f'''layers.{i+1}'''     ) for i in range(depth - 1     )],
             ]





      def 				snake_case__	(       self		,__UpperCAmelCase     )					->				Dict:
             for layer_module in self.layers:
                    A__	     =							layer_module(__UpperCAmelCase     )
             return hidden_state






class     UpperCamelCase__(			tf.keras.layers.Layer ):





      def __init__(       self		,__UpperCAmelCase		,**__UpperCAmelCase     )					->				Optional[int]:
             super().__init__(**__UpperCAmelCase     )
             A__	     =							[]
             # based on `downsample_in_first_stage`, the first layer of the first stage may or may not downsample the input
             self.stages.append(
                 TFRegNetStage(
                     __UpperCAmelCase		,config.embedding_size		,config.hidden_sizes[0]		,stride=2 if config.downsample_in_first_stage else 1		,depth=config.depths[0]		,name='stages.0'		,)     )
             A__	     =							zip(config.hidden_sizes		,config.hidden_sizes[1:]     )
             for i, ((in_channels, out_channels), depth) in enumerate(zip(__UpperCAmelCase		,config.depths[1:]     )     ):
                    self.stages.append(TFRegNetStage(__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase		,depth=__UpperCAmelCase		,name=f'''stages.{i+1}'''     )     )





      def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase = False		,__UpperCAmelCase = True     )					->				TFBaseModelOutputWithNoAttention:
             A__	     =							() if output_hidden_states else None

             for stage_module in self.stages:
                    if output_hidden_states:
                           A__	     =							hidden_states + (hidden_state,)

                    A__	     =							stage_module(__UpperCAmelCase     )

             if output_hidden_states:
                    A__	     =							hidden_states + (hidden_state,)

             if not return_dict:
                    return tuple(v for v in [hidden_state, hidden_states] if v is not None     )

             return TFBaseModelOutputWithNoAttention(last_hidden_state=__UpperCAmelCase		,hidden_states=__UpperCAmelCase     )






@keras_serializable
class     UpperCamelCase__(			tf.keras.layers.Layer ):
      lowerCAmelCase__  :						List[str]													=		RegNetConfig





      def __init__(       self		,__UpperCAmelCase		,**__UpperCAmelCase     )					->				Optional[int]:
             super().__init__(**__UpperCAmelCase     )
             A__	     =							config
             A__	     =							TFRegNetEmbeddings(__UpperCAmelCase		,name='embedder'     )
             A__	     =							TFRegNetEncoder(__UpperCAmelCase		,name='encoder'     )
             A__	     =							tf.keras.layers.GlobalAveragePoolingaD(keepdims=__UpperCAmelCase		,name='pooler'     )





      @unpack_inputs
      def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase = None		,__UpperCAmelCase = None		,__UpperCAmelCase = False		,)					->				TFBaseModelOutputWithPoolingAndNoAttention:
             A__	     =							(
                 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
             )
             A__	     =							return_dict if return_dict is not None else self.config.use_return_dict

             A__	     =							self.embedder(__UpperCAmelCase		,training=__UpperCAmelCase     )

             A__	     =							self.encoder(
                 __UpperCAmelCase		,output_hidden_states=__UpperCAmelCase		,return_dict=__UpperCAmelCase		,training=__UpperCAmelCase     )

             A__	     =							encoder_outputs[0]
             A__	     =							self.pooler(__UpperCAmelCase     )

             # Change to NCHW output format have uniformity in the modules
             A__	     =							tf.transpose(__UpperCAmelCase		,perm=(0, 3, 1, 2)     )
             A__	     =							tf.transpose(__UpperCAmelCase		,perm=(0, 3, 1, 2)     )

             # Change the other hidden state outputs to NCHW as well
             if output_hidden_states:
                    A__	     =							tuple([tf.transpose(__UpperCAmelCase		,perm=(0, 3, 1, 2)     ) for h in encoder_outputs[1]]     )

             if not return_dict:
                    return (last_hidden_state, pooled_output) + encoder_outputs[1:]

             return TFBaseModelOutputWithPoolingAndNoAttention(
                 last_hidden_state=__UpperCAmelCase		,pooler_output=__UpperCAmelCase		,hidden_states=hidden_states if output_hidden_states else encoder_outputs.hidden_states		,)






class     UpperCamelCase__(			__A ):
      lowerCAmelCase__  :						Any													=		RegNetConfig
      lowerCAmelCase__  :						List[str]													=		'regnet'
      lowerCAmelCase__  :						Union[str, Any]													=		'pixel_values'





      @property
      def 				snake_case__	(       self     )					->				str:
             return {"pixel_values": tf.TensorSpec(shape=(None, self.config.num_channels, 2_24, 2_24)		,dtype=tf.floataa     )}






__lowerCamelCase											=  R"\n    Parameters:\n    This model is a Tensorflow\n    [tf.keras.layers.Layer](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Layer) sub-class. Use it as a\n    regular Tensorflow Module and refer to the Tensorflow documentation for all matter related to general usage and\n    behavior.\n        config ([`RegNetConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.\n"

__lowerCamelCase											=  R"\n    Args:\n        pixel_values (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`ConveNextImageProcessor.__call__`] for details.\n        output_hidden_states (`bool`, *optional*):\n            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n            more detail.\n        return_dict (`bool`, *optional*):\n            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.\n"
@add_start_docstrings(
    'The bare RegNet model outputting raw features without any specific head on top.'		,						__A		,						)
class     UpperCamelCase__(			__A ):





      def __init__(       self		,__UpperCAmelCase		,*__UpperCAmelCase		,**__UpperCAmelCase     )					->				int:
             super().__init__(__UpperCAmelCase		,*__UpperCAmelCase		,**__UpperCAmelCase     )
             A__	     =							TFRegNetMainLayer(__UpperCAmelCase		,name='regnet'     )





      @unpack_inputs
      @add_start_docstrings_to_model_forward(__UpperCAmelCase     )
      @add_code_sample_docstrings(
          checkpoint=_CHECKPOINT_FOR_DOC		,output_type=__UpperCAmelCase		,config_class=_CONFIG_FOR_DOC		,modality='vision'		,expected_output=_EXPECTED_OUTPUT_SHAPE		,)
      def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase = None		,__UpperCAmelCase = None		,__UpperCAmelCase=False		,)					->				Union[TFBaseModelOutputWithPoolingAndNoAttention, Tuple[tf.Tensor]]:
             A__	     =							(
                 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
             )
             A__	     =							return_dict if return_dict is not None else self.config.use_return_dict

             A__	     =							self.regnet(
                 pixel_values=__UpperCAmelCase		,output_hidden_states=__UpperCAmelCase		,return_dict=__UpperCAmelCase		,training=__UpperCAmelCase		,)
             if not return_dict:
                    return (outputs[0],) + outputs[1:]

             return TFBaseModelOutputWithPoolingAndNoAttention(
                 last_hidden_state=outputs.last_hidden_state		,pooler_output=outputs.pooler_output		,hidden_states=outputs.hidden_states		,)






@add_start_docstrings(
    '\n    RegNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for\n    ImageNet.\n    '		,						__A		,						)
class     UpperCamelCase__(			__A		,						__A ):





      def __init__(       self		,__UpperCAmelCase		,*__UpperCAmelCase		,**__UpperCAmelCase     )					->				Dict:
             super().__init__(__UpperCAmelCase		,*__UpperCAmelCase		,**__UpperCAmelCase     )
             A__	     =							config.num_labels
             A__	     =							TFRegNetMainLayer(__UpperCAmelCase		,name='regnet'     )
             # classification head
             A__	     =							[
                 tf.keras.layers.Flatten(),
                 tf.keras.layers.Dense(config.num_labels		,name='classifier.1'     ) if config.num_labels > 0 else tf.identity,
             ]





      @unpack_inputs
      @add_start_docstrings_to_model_forward(__UpperCAmelCase     )
      @add_code_sample_docstrings(
          checkpoint=_IMAGE_CLASS_CHECKPOINT		,output_type=__UpperCAmelCase		,config_class=_CONFIG_FOR_DOC		,expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT		,)
      def 				snake_case__	(       self		,__UpperCAmelCase = None		,__UpperCAmelCase = None		,__UpperCAmelCase = None		,__UpperCAmelCase = None		,__UpperCAmelCase=False		,)					->				Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]:
             A__	     =							(
                 output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
             )
             A__	     =							return_dict if return_dict is not None else self.config.use_return_dict

             A__	     =							self.regnet(
                 __UpperCAmelCase		,output_hidden_states=__UpperCAmelCase		,return_dict=__UpperCAmelCase		,training=__UpperCAmelCase     )

             A__	     =							outputs.pooler_output if return_dict else outputs[1]

             A__	     =							self.classifier[0](__UpperCAmelCase     )
             A__	     =							self.classifier[1](__UpperCAmelCase     )

             A__	     =							None if labels is None else self.hf_compute_loss(labels=__UpperCAmelCase		,logits=__UpperCAmelCase     )

             if not return_dict:
                    A__	     =							(logits,) + outputs[2:]
                    return ((loss,) + output) if loss is not None else output

             return TFSequenceClassifierOutput(loss=__UpperCAmelCase		,logits=__UpperCAmelCase		,hidden_states=outputs.hidden_states     )






"""simple docstring"""




import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
				__lowerCamelCase											=  "pt"
elif is_tf_available():
				__lowerCamelCase											=  "tf"
else:
				__lowerCamelCase											=  "jax"
class     UpperCamelCase__(			__A		,						unittest.TestCase ):
						lowerCAmelCase__  :						List[Any]													=		ByTaTokenizer
						lowerCAmelCase__  :						str													=		False





						def 				snake_case__	(       self     )					->				int:
													super().setUp()
													A__	     =							ByTaTokenizer()
													tokenizer.save_pretrained(self.tmpdirname     )





						@cached_property
						def 				snake_case__	(       self     )					->				List[Any]:
													return ByTaTokenizer.from_pretrained('google/byt5-small'     )





						def 				snake_case__	(       self		,**__UpperCAmelCase     )					->				ByTaTokenizer:
													return self.tokenizer_class.from_pretrained(self.tmpdirname		,**__UpperCAmelCase     )





						def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase=False		,__UpperCAmelCase=20		,__UpperCAmelCase=5     )					->				Tuple[str, list]:
													# XXX The default common tokenizer tests assume that every ID is decodable on its own.
													# This assumption is invalid for ByT5 because single bytes might not be
													# valid utf-8 (byte 128 for instance).
													# Here we're overriding the smallest possible method to provide
													# a clean sequence without making the same assumption.

													A__	     =							[]
													for i in range(len(__UpperCAmelCase     )     ):
																				try:
																											A__	     =							tokenizer.decode([i]		,clean_up_tokenization_spaces=__UpperCAmelCase     )
																				except UnicodeDecodeError:
																											pass
																				toks.append((i, tok)     )

													A__	     =							list(filter(lambda __UpperCAmelCase     : re.match(R'^[ a-zA-Z]+$'		,t[1]     )		,__UpperCAmelCase     )     )
													A__	     =							list(filter(lambda __UpperCAmelCase     : [t[0]] == tokenizer.encode(t[1]		,add_special_tokens=__UpperCAmelCase     )		,__UpperCAmelCase     )     )
													if max_length is not None and len(__UpperCAmelCase     ) > max_length:
																				A__	     =							toks[:max_length]
													if min_length is not None and len(__UpperCAmelCase     ) < min_length and len(__UpperCAmelCase     ) > 0:
																				while len(__UpperCAmelCase     ) < min_length:
																											A__	     =							toks + toks
        # toks_str = [t[1] for t in toks]
													A__	     =							[t[0] for t in toks]

													# Ensure consistency
													A__	     =							tokenizer.decode(__UpperCAmelCase		,clean_up_tokenization_spaces=__UpperCAmelCase     )
													if " " not in output_txt and len(__UpperCAmelCase     ) > 1:
																				A__	     =							(
																				    tokenizer.decode([toks_ids[0]]		,clean_up_tokenization_spaces=__UpperCAmelCase     )
																				    + ' '
																				    + tokenizer.decode(toks_ids[1:]		,clean_up_tokenization_spaces=__UpperCAmelCase     )
																				)
													if with_prefix_space:
																				A__	     =							' ' + output_txt
													A__	     =							tokenizer.encode(__UpperCAmelCase		,add_special_tokens=__UpperCAmelCase     )
													return output_txt, output_ids





						def 				snake_case__	(       self     )					->				Dict:
													A__	     =							self.ta_base_tokenizer
													A__	     =							tokenizer(['hi</s>', 'I went to the gym</s>', '</s>']     )
													A__	     =							tokenizer(['hi', 'I went to the gym', '']     )
													self.assertListEqual(batch_with_eos_added['input_ids']		,batch_without_eos_added['input_ids']     )





						def 				snake_case__	(       self     )					->				Dict:
													A__	     =							self.ta_base_tokenizer
													A__	     =							'Unicode €.'
													A__	     =							tokenizer(__UpperCAmelCase     )
													A__	     =							[88, 1_13, 1_08, 1_02, 1_14, 1_03, 1_04, 35, 2_29, 1_33, 1_75, 49, 1]
													self.assertEqual(encoded['input_ids']		,__UpperCAmelCase     )

													# decoding
													A__	     =							tokenizer.decode(__UpperCAmelCase     )
													self.assertEqual(__UpperCAmelCase		,'Unicode €.</s>'     )

													A__	     =							tokenizer('e è é ê ë'     )
													A__	     =							[1_04, 35, 1_98, 1_71, 35, 1_98, 1_72, 35, 1_98, 1_73, 35, 1_98, 1_74, 1]
													self.assertEqual(encoded['input_ids']		,__UpperCAmelCase     )
													# decoding
													A__	     =							tokenizer.decode(__UpperCAmelCase     )
													self.assertEqual(__UpperCAmelCase		,'e è é ê ë</s>'     )

													# encode/decode, but with `encode` instead of `__call__`
													self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'     )     )		,'e è é ê ë</s>'     )





						def 				snake_case__	(       self     )					->				Optional[int]:
													A__	     =							self.ta_base_tokenizer
													A__	     =							['A long paragraph for summarization.', 'Another paragraph for summarization.']
													# fmt: off
													A__	     =							[68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 1, 0]
													# fmt: on
													A__	     =							tokenizer(__UpperCAmelCase		,padding=__UpperCAmelCase		,return_tensors=__UpperCAmelCase     )
													self.assertIsInstance(__UpperCAmelCase		,__UpperCAmelCase     )

													if FRAMEWORK != "jax":
																				A__	     =							list(batch.input_ids.numpy()[0]     )
													else:
																				A__	     =							list(batch.input_ids.tolist()[0]     )

													self.assertListEqual(__UpperCAmelCase		,__UpperCAmelCase     )

													self.assertEqual((2, 37)		,batch.input_ids.shape     )
													self.assertEqual((2, 37)		,batch.attention_mask.shape     )





						def 				snake_case__	(       self     )					->				List[str]:
													A__	     =							self.ta_base_tokenizer
													A__	     =							['A long paragraph for summarization.', 'Another paragraph for summarization.']
													A__	     =							tokenizer(__UpperCAmelCase		,padding=__UpperCAmelCase		,return_tensors=__UpperCAmelCase     )
													# check if input_ids are returned and no decoder_input_ids
													self.assertIn('input_ids'		,__UpperCAmelCase     )
													self.assertIn('attention_mask'		,__UpperCAmelCase     )
													self.assertNotIn('decoder_input_ids'		,__UpperCAmelCase     )
													self.assertNotIn('decoder_attention_mask'		,__UpperCAmelCase     )





						def 				snake_case__	(       self     )					->				Any:
													A__	     =							self.ta_base_tokenizer
													A__	     =							[
													    'Summary of the text.',
													    'Another summary.',
													]
													A__	     =							tokenizer(
													    text_target=__UpperCAmelCase		,max_length=32		,padding='max_length'		,truncation=__UpperCAmelCase		,return_tensors=__UpperCAmelCase     )
													self.assertEqual(32		,targets['input_ids'].shape[1]     )





						def 				snake_case__	(       self     )					->				Optional[int]:
													A__	     =							self.ta_base_tokenizer
													A__	     =							['A long paragraph for summarization. </s>']
													A__	     =							['Summary of the text. </s>']
													# fmt: off
													A__	     =							[68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 35, 1]
													A__	     =							[86, 1_20, 1_12, 1_12, 1_00, 1_17, 1_24, 35, 1_14, 1_05, 35, 1_19, 1_07, 1_04, 35, 1_19, 1_04, 1_23, 1_19, 49, 35, 1]
													# fmt: on

													A__	     =							tokenizer(__UpperCAmelCase		,text_target=__UpperCAmelCase     )

													self.assertEqual(__UpperCAmelCase		,batch['input_ids'][0]     )
													self.assertEqual(__UpperCAmelCase		,batch['labels'][0]     )





						def 				snake_case__	(       self     )					->				Tuple:
													# safety check on max_len default value so we are sure the test works
													A__	     =							self.get_tokenizers()
													for tokenizer in tokenizers:
																				with self.subTest(f'''{tokenizer.__class__.__name__}'''     ):
																											self.assertNotEqual(tokenizer.model_max_length		,42     )

        # Now let's start the test
													A__	     =							self.get_tokenizers()
													for tokenizer in tokenizers:
																				with self.subTest(f'''{tokenizer.__class__.__name__}'''     ):
																											# Isolate this from the other tests because we save additional tokens/etc
																											A__	     =							tempfile.mkdtemp()

																											A__	     =							' He is very happy, UNwant\u00E9d,running'
																											A__	     =							tokenizer.encode(__UpperCAmelCase		,add_special_tokens=__UpperCAmelCase     )
																											tokenizer.save_pretrained(__UpperCAmelCase     )

																											A__	     =							tokenizer.__class__.from_pretrained(__UpperCAmelCase     )
																											A__	     =							after_tokenizer.encode(__UpperCAmelCase		,add_special_tokens=__UpperCAmelCase     )
																											self.assertListEqual(__UpperCAmelCase		,__UpperCAmelCase     )

																											shutil.rmtree(__UpperCAmelCase     )

													A__	     =							self.get_tokenizers(model_max_length=42     )
													for tokenizer in tokenizers:
																				with self.subTest(f'''{tokenizer.__class__.__name__}'''     ):
																											# Isolate this from the other tests because we save additional tokens/etc
																											A__	     =							tempfile.mkdtemp()

																											A__	     =							' He is very happy, UNwant\u00E9d,running'
																											tokenizer.add_tokens(['bim', 'bambam']     )
																											A__	     =							tokenizer.additional_special_tokens
																											additional_special_tokens.append('new_additional_special_token'     )
																											tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}     )
																											A__	     =							tokenizer.encode(__UpperCAmelCase		,add_special_tokens=__UpperCAmelCase     )
																											tokenizer.save_pretrained(__UpperCAmelCase     )

																											A__	     =							tokenizer.__class__.from_pretrained(__UpperCAmelCase     )
																											A__	     =							after_tokenizer.encode(__UpperCAmelCase		,add_special_tokens=__UpperCAmelCase     )
																											self.assertListEqual(__UpperCAmelCase		,__UpperCAmelCase     )
																											self.assertIn('new_additional_special_token'		,after_tokenizer.additional_special_tokens     )
																											self.assertEqual(after_tokenizer.model_max_length		,42     )

																											A__	     =							tokenizer.__class__.from_pretrained(__UpperCAmelCase		,model_max_length=43     )
																											self.assertEqual(tokenizer.model_max_length		,43     )

																											shutil.rmtree(__UpperCAmelCase     )





						def 				snake_case__	(       self     )					->				Optional[int]:
													A__	     =							[]
													if self.test_slow_tokenizer:
																				tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())     )

													if self.test_rust_tokenizer:
																				tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())     )

													for tokenizer_class, tokenizer_utils in tokenizer_list:
																				with tempfile.TemporaryDirectory() as tmp_dir:
																											tokenizer_utils.save_pretrained(__UpperCAmelCase     )

																											with open(os.path.join(__UpperCAmelCase		,'special_tokens_map.json'     )		,encoding='utf-8'     ) as json_file:
																																		A__	     =							json.load(__UpperCAmelCase     )

																											with open(os.path.join(__UpperCAmelCase		,'tokenizer_config.json'     )		,encoding='utf-8'     ) as json_file:
																																		A__	     =							json.load(__UpperCAmelCase     )

																											A__	     =							[f'''<extra_id_{i}>''' for i in range(1_25     )]

																											A__	     =							added_tokens_extra_ids + [
																											    'an_additional_special_token'
																											]
																											A__	     =							added_tokens_extra_ids + [
																											    'an_additional_special_token'
																											]

																											with open(os.path.join(__UpperCAmelCase		,'special_tokens_map.json'     )		,'w'		,encoding='utf-8'     ) as outfile:
																																		json.dump(__UpperCAmelCase		,__UpperCAmelCase     )
																											with open(os.path.join(__UpperCAmelCase		,'tokenizer_config.json'     )		,'w'		,encoding='utf-8'     ) as outfile:
																																		json.dump(__UpperCAmelCase		,__UpperCAmelCase     )

																											# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																											# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																											# "special_tokens_map.json" files
																											A__	     =							tokenizer_class.from_pretrained(
																											    __UpperCAmelCase		,)
																											self.assertIn(
																											    'an_additional_special_token'		,tokenizer_without_change_in_init.additional_special_tokens     )
																											# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																											self.assertEqual(
																											    ['an_additional_special_token']		,tokenizer_without_change_in_init.convert_ids_to_tokens(
																											        tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']     )     )		,)

																											# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																											A__	     =							added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'		,lstrip=__UpperCAmelCase     )]
																											A__	     =							tokenizer_class.from_pretrained(
																											    __UpperCAmelCase		,additional_special_tokens=__UpperCAmelCase		,)

																											self.assertIn('a_new_additional_special_token'		,tokenizer.additional_special_tokens     )
																											self.assertEqual(
																											    ['a_new_additional_special_token']		,tokenizer.convert_ids_to_tokens(
																											        tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']     )     )		,)





						def 				snake_case__	(       self     )					->				str:
													A__	     =							[]
													if self.test_slow_tokenizer:
																				tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())     )

													if self.test_rust_tokenizer:
																				tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())     )

													for tokenizer_class, tokenizer_utils in tokenizer_list:
																				with tempfile.TemporaryDirectory() as tmp_dir:
																											tokenizer_utils.save_pretrained(__UpperCAmelCase     )

																											A__	     =							tokenizer_class.from_pretrained(__UpperCAmelCase     )

																											self.assertTrue(tokenizer.decode([2_55]     ) == ''     )





						def 				snake_case__	(       self     )					->				Any:
													pass





						def 				snake_case__	(       self     )					->				Any:
													pass





						def 				snake_case__	(       self     )					->				str:
													pass





						def 				snake_case__	(       self     )					->				int:
													pass





						def 				snake_case__	(       self     )					->				Dict:
													# The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
													# and special added tokens as tokens
													A__	     =							self.get_tokenizers(fast=__UpperCAmelCase		,do_lower_case=__UpperCAmelCase     )
													for tokenizer in tokenizers:
																				with self.subTest(f'''{tokenizer.__class__.__name__}'''     ):
																											A__	     =							['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
																											A__	     =							tokenizer.convert_tokens_to_string(__UpperCAmelCase     )

																											self.assertIsInstance(__UpperCAmelCase		,__UpperCAmelCase     )





						def 				snake_case__	(       self     )					->				Tuple:
													A__	     =							self.get_tokenizers()
													for tokenizer in tokenizers:
																				with self.subTest(f'''{tokenizer.__class__.__name__}'''     ):
																											A__	     =							[
																											    'bos_token',
																											    'eos_token',
																											    'unk_token',
																											    'sep_token',
																											    'pad_token',
																											    'cls_token',
																											    'mask_token',
																											]

																											A__	     =							0
																											A__	     =							tokenizer.convert_ids_to_tokens(
																											    __UpperCAmelCase		,skip_special_tokens=__UpperCAmelCase     )

																											for attr in attributes_list:
																																		setattr(__UpperCAmelCase		,attr + '_id'		,__UpperCAmelCase     )
																																		self.assertEqual(getattr(__UpperCAmelCase		,__UpperCAmelCase     )		,__UpperCAmelCase     )
																																		self.assertEqual(getattr(__UpperCAmelCase		,attr + '_id'     )		,__UpperCAmelCase     )

																																		setattr(__UpperCAmelCase		,attr + '_id'		,__UpperCAmelCase     )
																																		self.assertEqual(getattr(__UpperCAmelCase		,__UpperCAmelCase     )		,__UpperCAmelCase     )
																																		self.assertEqual(getattr(__UpperCAmelCase		,attr + '_id'     )		,__UpperCAmelCase     )

																											setattr(__UpperCAmelCase		,'additional_special_tokens_ids'		,[]     )
																											self.assertListEqual(getattr(__UpperCAmelCase		,'additional_special_tokens'     )		,[]     )
																											self.assertListEqual(getattr(__UpperCAmelCase		,'additional_special_tokens_ids'     )		,[]     )

																											setattr(__UpperCAmelCase		,'additional_special_tokens_ids'		,[token_id_to_test_setters]     )
																											self.assertListEqual(getattr(__UpperCAmelCase		,'additional_special_tokens'     )		,[token_to_test_setters]     )
																											self.assertListEqual(getattr(__UpperCAmelCase		,'additional_special_tokens_ids'     )		,[token_id_to_test_setters]     )





