


from typing import Optional

import torch
import torch.utils.checkpoint
from torch import Tensor, nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import (
    BackboneOutput,
    BaseModelOutputWithNoAttention,
    BaseModelOutputWithPoolingAndNoAttention,
    ImageClassifierOutputWithNoAttention,
)
from ...modeling_utils import PreTrainedModel
from ...utils import (
    add_code_sample_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    logging,
    replace_return_docstrings,
)
from ...utils.backbone_utils import BackboneMixin
from .configuration_resnet import ResNetConfig


UpperCAmelCase   :    str     =		logging.get_logger(__name__)

# General docstring
UpperCAmelCase   :    Union[str, Any]     =		"ResNetConfig"

# Base docstring
UpperCAmelCase   :    Optional[int]     =		"microsoft/resnet-50"
UpperCAmelCase   :    Optional[int]     =		[1, 20_48, 7, 7]

# Image classification docstring
UpperCAmelCase   :    int     =		"microsoft/resnet-50"
UpperCAmelCase   :    Dict     =		"tiger cat"

UpperCAmelCase   :    int     =		[
    "microsoft/resnet-50",
    # See all resnet models at https://huggingface.co/models?filter=resnet
]







class        __lowercase       (							nn.Module  ):



		"""simple docstring"""

		def __init__( self		,      A		,      A		,      A = 3		,      A = 1		,      A = "relu"	)  -> int:







					'''simple docstring'''







					super().__init__()
					lowerCamelCase    		=  nn.Convad(
					    __SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      kernel_size=__SCREAMING_SNAKE_CASE		,      stride=__SCREAMING_SNAKE_CASE		,      padding=kernel_size // 2		,      bias=__SCREAMING_SNAKE_CASE	)
					lowerCamelCase    		=  nn.BatchNormad(__SCREAMING_SNAKE_CASE	)
					lowerCamelCase    		=  ACTaFN[activation] if activation is not None else nn.Identity()







		def 					__A   ( self		,      A	)  -> Tensor:







					'''simple docstring'''







					lowerCamelCase    		=  self.convolution(__SCREAMING_SNAKE_CASE	)
					lowerCamelCase    		=  self.normalization(__SCREAMING_SNAKE_CASE	)
					lowerCamelCase    		=  self.activation(__SCREAMING_SNAKE_CASE	)
					return hidden_state







class        __lowercase       (							nn.Module  ):



		"""simple docstring"""

		def __init__( self		,      A	)  -> Any:







					'''simple docstring'''







					super().__init__()
					lowerCamelCase    		=  ResNetConvLayer(
					    config.num_channels		,      config.embedding_size		,      kernel_size=7		,      stride=2		,      activation=config.hidden_act	)
					lowerCamelCase    		=  nn.MaxPoolad(kernel_size=3		,      stride=2		,      padding=1	)
					lowerCamelCase    		=  config.num_channels







		def 					__A   ( self		,      A	)  -> Tensor:







					'''simple docstring'''







					lowerCamelCase    		=  pixel_values.shape[1]
					if num_channels != self.num_channels:
								raise ValueError(
								    """Make sure that the channel dimension of the pixel values match with the one set in the configuration."""	)
					lowerCamelCase    		=  self.embedder(__SCREAMING_SNAKE_CASE	)
					lowerCamelCase    		=  self.pooler(__SCREAMING_SNAKE_CASE	)
					return embedding







class        __lowercase       (							nn.Module  ):



		"""simple docstring"""

		def __init__( self		,      A		,      A		,      A = 2	)  -> Any:







					'''simple docstring'''







					super().__init__()
					lowerCamelCase    		=  nn.Convad(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      kernel_size=1		,      stride=__SCREAMING_SNAKE_CASE		,      bias=__SCREAMING_SNAKE_CASE	)
					lowerCamelCase    		=  nn.BatchNormad(__SCREAMING_SNAKE_CASE	)







		def 					__A   ( self		,      A	)  -> Tensor:







					'''simple docstring'''







					lowerCamelCase    		=  self.convolution(__SCREAMING_SNAKE_CASE	)
					lowerCamelCase    		=  self.normalization(__SCREAMING_SNAKE_CASE	)
					return hidden_state







class        __lowercase       (							nn.Module  ):



		"""simple docstring"""

		def __init__( self		,      A		,      A		,      A = 1		,      A = "relu"	)  -> int:







					'''simple docstring'''







					super().__init__()
					lowerCamelCase    		=  in_channels != out_channels or stride != 1
					lowerCamelCase    		=  (
					    ResNetShortCut(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      stride=__SCREAMING_SNAKE_CASE	) if should_apply_shortcut else nn.Identity()
					)
					lowerCamelCase    		=  nn.Sequential(
					    ResNetConvLayer(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      stride=__SCREAMING_SNAKE_CASE	)		,      ResNetConvLayer(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      activation=__SCREAMING_SNAKE_CASE	)		,      )
					lowerCamelCase    		=  ACTaFN[activation]







		def 					__A   ( self		,      A	)  -> str:







					'''simple docstring'''







					lowerCamelCase    		=  hidden_state
					lowerCamelCase    		=  self.layer(__SCREAMING_SNAKE_CASE	)
					lowerCamelCase    		=  self.shortcut(__SCREAMING_SNAKE_CASE	)
					hidden_state += residual
					lowerCamelCase    		=  self.activation(__SCREAMING_SNAKE_CASE	)
					return hidden_state







class        __lowercase       (							nn.Module  ):



		"""simple docstring"""

		def __init__( self		,      A		,      A		,      A = 1		,      A = "relu"		,      A = 4	)  -> Optional[Any]:







					'''simple docstring'''







					super().__init__()
					lowerCamelCase    		=  in_channels != out_channels or stride != 1
					lowerCamelCase    		=  out_channels // reduction
					lowerCamelCase    		=  (
					    ResNetShortCut(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      stride=__SCREAMING_SNAKE_CASE	) if should_apply_shortcut else nn.Identity()
					)
					lowerCamelCase    		=  nn.Sequential(
					    ResNetConvLayer(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      kernel_size=1	)		,      ResNetConvLayer(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      stride=__SCREAMING_SNAKE_CASE	)		,      ResNetConvLayer(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      kernel_size=1		,      activation=__SCREAMING_SNAKE_CASE	)		,      )
					lowerCamelCase    		=  ACTaFN[activation]







		def 					__A   ( self		,      A	)  -> Optional[Any]:







					'''simple docstring'''







					lowerCamelCase    		=  hidden_state
					lowerCamelCase    		=  self.layer(__SCREAMING_SNAKE_CASE	)
					lowerCamelCase    		=  self.shortcut(__SCREAMING_SNAKE_CASE	)
					hidden_state += residual
					lowerCamelCase    		=  self.activation(__SCREAMING_SNAKE_CASE	)
					return hidden_state







class        __lowercase       (							nn.Module  ):



		"""simple docstring"""

		def __init__( self		,      A		,      A		,      A		,      A = 2		,      A = 2		,      )  -> int:







					'''simple docstring'''







					super().__init__()

					lowerCamelCase    		=  ResNetBottleNeckLayer if config.layer_type == '''bottleneck''' else ResNetBasicLayer

					lowerCamelCase    		=  nn.Sequential(
					    # downsampling is done in the first layer with stride of 2
					    layer(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      stride=__SCREAMING_SNAKE_CASE		,      activation=config.hidden_act	)		,      *[layer(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      activation=config.hidden_act	) for _ in range(depth - 1	)]		,      )







		def 					__A   ( self		,      A	)  -> Tensor:







					'''simple docstring'''







					lowerCamelCase    		=  input
					for layer in self.layers:
								lowerCamelCase    		=  layer(__SCREAMING_SNAKE_CASE	)
					return hidden_state







class        __lowercase       (							nn.Module  ):



		"""simple docstring"""

		def __init__( self		,      A	)  -> str:







					'''simple docstring'''







					super().__init__()
					lowerCamelCase    		=  nn.ModuleList([]	)
					# based on `downsample_in_first_stage` the first layer of the first stage may or may not downsample the input
					self.stages.append(
					    ResNetStage(
					        __SCREAMING_SNAKE_CASE		,      config.embedding_size		,      config.hidden_sizes[0]		,      stride=2 if config.downsample_in_first_stage else 1		,      depth=config.depths[0]		,      )	)
					lowerCamelCase    		=  zip(config.hidden_sizes		,      config.hidden_sizes[1:]	)
					for (in_channels, out_channels), depth in zip(__SCREAMING_SNAKE_CASE		,      config.depths[1:]	):
								self.stages.append(ResNetStage(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE		,      depth=__SCREAMING_SNAKE_CASE	)	)







		def 					__A   ( self		,      A		,      A = False		,      A = True	)  -> BaseModelOutputWithNoAttention:







					'''simple docstring'''







					lowerCamelCase    		=  () if output_hidden_states else None

					for stage_module in self.stages:
								if output_hidden_states:
											lowerCamelCase    		=  hidden_states + (hidden_state,)

								lowerCamelCase    		=  stage_module(__SCREAMING_SNAKE_CASE	)

					if output_hidden_states:
								lowerCamelCase    		=  hidden_states + (hidden_state,)

					if not return_dict:
								return tuple(v for v in [hidden_state, hidden_states] if v is not None	)

					return BaseModelOutputWithNoAttention(
					    last_hidden_state=__SCREAMING_SNAKE_CASE		,      hidden_states=__SCREAMING_SNAKE_CASE		,      )







class        __lowercase       (							SCREAMING_SNAKE_CASE_  ):



		"""simple docstring"""

		UpperCamelCase     :					Optional[Any]			       =		ResNetConfig
		UpperCamelCase     :					Dict			       =		"resnet"
		UpperCamelCase     :					List[str]			       =		"pixel_values"
		UpperCamelCase     :					Optional[int]			       =		True







		def 					__A   ( self		,      A	)  -> Dict:







					'''simple docstring'''







					if isinstance(__SCREAMING_SNAKE_CASE		,      nn.Convad	):
								nn.init.kaiming_normal_(module.weight		,      mode="""fan_out"""		,      nonlinearity="""relu"""	)
					elif isinstance(__SCREAMING_SNAKE_CASE		,      (nn.BatchNormad, nn.GroupNorm)	):
								nn.init.constant_(module.weight		,      1	)
								nn.init.constant_(module.bias		,      0	)







		def 					__A   ( self		,      A		,      A=False	)  -> Any:







					'''simple docstring'''







					if isinstance(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE	):
								lowerCamelCase    		=  value






UpperCAmelCase   :    Union[str, Any]     =		r"\n    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it\n    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and\n    behavior.\n\n    Parameters:\n        config ([`ResNetConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n"

UpperCAmelCase   :    Any     =		r"\n    Args:\n        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`ConvNextImageProcessor.__call__`] for details.\n\n        output_hidden_states (`bool`, *optional*):\n            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for\n            more detail.\n        return_dict (`bool`, *optional*):\n            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.\n"







@add_start_docstrings(
    "The bare ResNet model outputting raw features without any specific head on top."			,							SCREAMING_SNAKE_CASE_			,							)
class        __lowercase       (							SCREAMING_SNAKE_CASE_  ):



		"""simple docstring"""

		def __init__( self		,      A	)  -> List[Any]:







					'''simple docstring'''







					super().__init__(__SCREAMING_SNAKE_CASE	)
					lowerCamelCase    		=  config
					lowerCamelCase    		=  ResNetEmbeddings(__SCREAMING_SNAKE_CASE	)
					lowerCamelCase    		=  ResNetEncoder(__SCREAMING_SNAKE_CASE	)
					lowerCamelCase    		=  nn.AdaptiveAvgPoolad((1, 1)	)
					# Initialize weights and apply final processing
					self.post_init()







		@add_start_docstrings_to_model_forward(__SCREAMING_SNAKE_CASE	)
		@add_code_sample_docstrings(
		    checkpoint=_CHECKPOINT_FOR_DOC		,      output_type=__SCREAMING_SNAKE_CASE		,      config_class=_CONFIG_FOR_DOC		,      modality="""vision"""		,      expected_output=_EXPECTED_OUTPUT_SHAPE		,      )
		def 					__A   ( self		,      A		,      A = None		,      A = None	)  -> BaseModelOutputWithPoolingAndNoAttention:







					'''simple docstring'''







					lowerCamelCase    		=  (
					    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
					)
					lowerCamelCase    		=  return_dict if return_dict is not None else self.config.use_return_dict

					lowerCamelCase    		=  self.embedder(__SCREAMING_SNAKE_CASE	)

					lowerCamelCase    		=  self.encoder(
					    __SCREAMING_SNAKE_CASE		,      output_hidden_states=__SCREAMING_SNAKE_CASE		,      return_dict=__SCREAMING_SNAKE_CASE	)

					lowerCamelCase    		=  encoder_outputs[0]

					lowerCamelCase    		=  self.pooler(__SCREAMING_SNAKE_CASE	)

					if not return_dict:
								return (last_hidden_state, pooled_output) + encoder_outputs[1:]

					return BaseModelOutputWithPoolingAndNoAttention(
					    last_hidden_state=__SCREAMING_SNAKE_CASE		,      pooler_output=__SCREAMING_SNAKE_CASE		,      hidden_states=encoder_outputs.hidden_states		,      )







@add_start_docstrings(
    "\n    ResNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for\n    ImageNet.\n    "			,							SCREAMING_SNAKE_CASE_			,							)
class        __lowercase       (							SCREAMING_SNAKE_CASE_  ):



		"""simple docstring"""

		def __init__( self		,      A	)  -> List[Any]:







					'''simple docstring'''







					super().__init__(__SCREAMING_SNAKE_CASE	)
					lowerCamelCase    		=  config.num_labels
					lowerCamelCase    		=  ResNetModel(__SCREAMING_SNAKE_CASE	)
					# classification head
					lowerCamelCase    		=  nn.Sequential(
					    nn.Flatten()		,      nn.Linear(config.hidden_sizes[-1]		,      config.num_labels	) if config.num_labels > 0 else nn.Identity()		,      )
					# initialize weights and apply final processing
					self.post_init()







		@add_start_docstrings_to_model_forward(__SCREAMING_SNAKE_CASE	)
		@add_code_sample_docstrings(
		    checkpoint=_IMAGE_CLASS_CHECKPOINT		,      output_type=__SCREAMING_SNAKE_CASE		,      config_class=_CONFIG_FOR_DOC		,      expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT		,      )
		def 					__A   ( self		,      A = None		,      A = None		,      A = None		,      A = None		,      )  -> ImageClassifierOutputWithNoAttention:







					'''simple docstring'''







					lowerCamelCase    		=  return_dict if return_dict is not None else self.config.use_return_dict

					lowerCamelCase    		=  self.resnet(__SCREAMING_SNAKE_CASE		,      output_hidden_states=__SCREAMING_SNAKE_CASE		,      return_dict=__SCREAMING_SNAKE_CASE	)

					lowerCamelCase    		=  outputs.pooler_output if return_dict else outputs[1]

					lowerCamelCase    		=  self.classifier(__SCREAMING_SNAKE_CASE	)

					lowerCamelCase    		=  None

					if labels is not None:
								if self.config.problem_type is None:
											if self.num_labels == 1:
														lowerCamelCase    		=  '''regression'''
											elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
														lowerCamelCase    		=  '''single_label_classification'''
											else:
														lowerCamelCase    		=  '''multi_label_classification'''
								if self.config.problem_type == "regression":
											lowerCamelCase    		=  MSELoss()
											if self.num_labels == 1:
														lowerCamelCase    		=  loss_fct(logits.squeeze()		,      labels.squeeze()	)
											else:
														lowerCamelCase    		=  loss_fct(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE	)
								elif self.config.problem_type == "single_label_classification":
											lowerCamelCase    		=  CrossEntropyLoss()
											lowerCamelCase    		=  loss_fct(logits.view(-1		,      self.num_labels	)		,      labels.view(-1	)	)
								elif self.config.problem_type == "multi_label_classification":
											lowerCamelCase    		=  BCEWithLogitsLoss()
											lowerCamelCase    		=  loss_fct(__SCREAMING_SNAKE_CASE		,      __SCREAMING_SNAKE_CASE	)

					if not return_dict:
								lowerCamelCase    		=  (logits,) + outputs[2:]
								return (loss,) + output if loss is not None else output

					return ImageClassifierOutputWithNoAttention(loss=__SCREAMING_SNAKE_CASE		,      logits=__SCREAMING_SNAKE_CASE		,      hidden_states=outputs.hidden_states	)







@add_start_docstrings(
    "\n    ResNet backbone, to be used with frameworks like DETR and MaskFormer.\n    "			,							SCREAMING_SNAKE_CASE_			,							)
class        __lowercase       (							SCREAMING_SNAKE_CASE_			,							SCREAMING_SNAKE_CASE_  ):



		"""simple docstring"""

		def __init__( self		,      A	)  -> List[Any]:







					'''simple docstring'''







					super().__init__(__SCREAMING_SNAKE_CASE	)
					super()._init_backbone(__SCREAMING_SNAKE_CASE	)

					lowerCamelCase    		=  [config.embedding_size] + config.hidden_sizes
					lowerCamelCase    		=  ResNetEmbeddings(__SCREAMING_SNAKE_CASE	)
					lowerCamelCase    		=  ResNetEncoder(__SCREAMING_SNAKE_CASE	)

					# initialize weights and apply final processing
					self.post_init()







		@add_start_docstrings_to_model_forward(__SCREAMING_SNAKE_CASE	)
		@replace_return_docstrings(output_type=__SCREAMING_SNAKE_CASE		,      config_class=_CONFIG_FOR_DOC	)
		def 					__A   ( self		,      A		,      A = None		,      A = None	)  -> BackboneOutput:







					'''simple docstring'''







					lowerCamelCase    		=  return_dict if return_dict is not None else self.config.use_return_dict
					lowerCamelCase    		=  (
					    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
					)

					lowerCamelCase    		=  self.embedder(__SCREAMING_SNAKE_CASE	)

					lowerCamelCase    		=  self.encoder(__SCREAMING_SNAKE_CASE		,      output_hidden_states=__SCREAMING_SNAKE_CASE		,      return_dict=__SCREAMING_SNAKE_CASE	)

					lowerCamelCase    		=  outputs.hidden_states

					lowerCamelCase    		=  ()
					for idx, stage in enumerate(self.stage_names	):
								if stage in self.out_features:
											feature_maps += (hidden_states[idx],)

					if not return_dict:
								lowerCamelCase    		=  (feature_maps,)
								if output_hidden_states:
											output += (outputs.hidden_states,)
								return output

					return BackboneOutput(
					    feature_maps=__SCREAMING_SNAKE_CASE		,      hidden_states=outputs.hidden_states if output_hidden_states else None		,      attentions=__SCREAMING_SNAKE_CASE		,      )









'''simple docstring'''







from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import (
		    TFConvBertForMaskedLM,
		    TFConvBertForMultipleChoice,
		    TFConvBertForQuestionAnswering,
		    TFConvBertForSequenceClassification,
		    TFConvBertForTokenClassification,
		    TFConvBertModel,
		)





class 						SCREAMING_SNAKE_CASE       :






			def __init__(			self   :      Union[str, Any]	,    __SCREAMING_SNAKE_CASE   :      Tuple	,    __SCREAMING_SNAKE_CASE   :      str=13	,    __SCREAMING_SNAKE_CASE   :      Optional[int]=7	,    __SCREAMING_SNAKE_CASE   :      List[str]=True	,    __SCREAMING_SNAKE_CASE   :      Optional[int]=True	,    __SCREAMING_SNAKE_CASE   :      Any=True	,    __SCREAMING_SNAKE_CASE   :      int=True	,    __SCREAMING_SNAKE_CASE   :      List[str]=99	,    __SCREAMING_SNAKE_CASE   :      Optional[int]=32	,    __SCREAMING_SNAKE_CASE   :      Tuple=2	,    __SCREAMING_SNAKE_CASE   :      str=4	,    __SCREAMING_SNAKE_CASE   :      int=37	,    __SCREAMING_SNAKE_CASE   :      Any="gelu"	,    __SCREAMING_SNAKE_CASE   :      List[Any]=0.1	,    __SCREAMING_SNAKE_CASE   :      List[Any]=0.1	,    __SCREAMING_SNAKE_CASE   :      Tuple=512	,    __SCREAMING_SNAKE_CASE   :      Optional[Any]=16	,    __SCREAMING_SNAKE_CASE   :      Dict=2	,    __SCREAMING_SNAKE_CASE   :      List[Any]=0.02	,    __SCREAMING_SNAKE_CASE   :      List[Any]=3	,    __SCREAMING_SNAKE_CASE   :      List[Any]=4	,    __SCREAMING_SNAKE_CASE   :      Dict=None	,    )      ->      Tuple:
							a_		:	Any									=   parent
							a_		:	List[str]									=   13
							a_		:	Dict									=   7
							a_		:	List[str]									=   True
							a_		:	str									=   True
							a_		:	List[str]									=   True
							a_		:	Any									=   True
							a_		:	List[Any]									=   99
							a_		:	Optional[Any]									=   384
							a_		:	List[str]									=   2
							a_		:	Optional[int]									=   4
							a_		:	Optional[int]									=   37
							a_		:	Union[str, Any]									=   '''gelu'''
							a_		:	List[str]									=   0.1
							a_		:	str									=   0.1
							a_		:	Optional[Any]									=   512
							a_		:	Union[str, Any]									=   16
							a_		:	Optional[Any]									=   2
							a_		:	Optional[int]									=   0.02
							a_		:	Union[str, Any]									=   3
							a_		:	Union[str, Any]									=   4
							a_		:	List[Any]									=   128
							a_		:	Optional[Any]									=   2
							a_		:	Any									=   9
							a_		:	Optional[int]									=   1
							a_		:	int									=   None






			def       SCREAMING_SNAKE_CASE   (			self   :      Dict       )      ->      Union[str, Any]:
							a_		:	Dict									=   ids_tensor([self.batch_size, self.seq_length]	,    self.vocab_size       )

							a_		:	int									=   None
							if self.use_input_mask:
											a_		:	Dict									=   random_attention_mask([self.batch_size, self.seq_length]       )

							a_		:	Optional[int]									=   None
							if self.use_token_type_ids:
											a_		:	Any									=   ids_tensor([self.batch_size, self.seq_length]	,    self.type_vocab_size       )

							a_		:	Optional[Any]									=   None
							a_		:	Optional[Any]									=   None
							a_		:	Union[str, Any]									=   None
							if self.use_labels:
											a_		:	List[str]									=   ids_tensor([self.batch_size]	,    self.type_sequence_label_size       )
											a_		:	int									=   ids_tensor([self.batch_size, self.seq_length]	,    self.num_labels       )
											a_		:	Tuple									=   ids_tensor([self.batch_size]	,    self.num_choices       )

							a_		:	Union[str, Any]									=   ConvBertConfig(
							    vocab_size=self.vocab_size	,    hidden_size=self.hidden_size	,    num_hidden_layers=self.num_hidden_layers	,    num_attention_heads=self.num_attention_heads	,    intermediate_size=self.intermediate_size	,    hidden_act=self.hidden_act	,    hidden_dropout_prob=self.hidden_dropout_prob	,    attention_probs_dropout_prob=self.attention_probs_dropout_prob	,    max_position_embeddings=self.max_position_embeddings	,    type_vocab_size=self.type_vocab_size	,    initializer_range=self.initializer_range	,    return_dict=__SCREAMING_SNAKE_CASE	,    )

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






			def       SCREAMING_SNAKE_CASE   (			self   :      Optional[Any]	,    __SCREAMING_SNAKE_CASE   :      Any	,    __SCREAMING_SNAKE_CASE   :      Tuple	,    __SCREAMING_SNAKE_CASE   :      Any	,    __SCREAMING_SNAKE_CASE   :      Any	,    __SCREAMING_SNAKE_CASE   :      int	,    __SCREAMING_SNAKE_CASE   :      Any	,    __SCREAMING_SNAKE_CASE   :      Optional[Any]       )      ->      Union[str, Any]:
							a_		:	Tuple									=   TFConvBertModel(config=__SCREAMING_SNAKE_CASE       )
							a_		:	str									=   {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

							a_		:	Optional[Any]									=   [input_ids, input_mask]
							a_		:	Tuple									=   model(__SCREAMING_SNAKE_CASE       )

							a_		:	int									=   model(__SCREAMING_SNAKE_CASE       )

							self.parent.assertEqual(result.last_hidden_state.shape	,    (self.batch_size, self.seq_length, self.hidden_size)       )






			def       SCREAMING_SNAKE_CASE   (			self   :      Optional[Any]	,    __SCREAMING_SNAKE_CASE   :      Tuple	,    __SCREAMING_SNAKE_CASE   :      Any	,    __SCREAMING_SNAKE_CASE   :      Any	,    __SCREAMING_SNAKE_CASE   :      List[str]	,    __SCREAMING_SNAKE_CASE   :      Tuple	,    __SCREAMING_SNAKE_CASE   :      Tuple	,    __SCREAMING_SNAKE_CASE   :      Dict       )      ->      str:
							a_		:	List[Any]									=   TFConvBertForMaskedLM(config=__SCREAMING_SNAKE_CASE       )
							a_		:	Dict									=   {
							    '''input_ids''': input_ids,
							    '''attention_mask''': input_mask,
							    '''token_type_ids''': token_type_ids,
							}
							a_		:	str									=   model(__SCREAMING_SNAKE_CASE       )
							self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.seq_length, self.vocab_size)       )






			def       SCREAMING_SNAKE_CASE   (			self   :      Optional[int]	,    __SCREAMING_SNAKE_CASE   :      str	,    __SCREAMING_SNAKE_CASE   :      int	,    __SCREAMING_SNAKE_CASE   :      str	,    __SCREAMING_SNAKE_CASE   :      List[str]	,    __SCREAMING_SNAKE_CASE   :      Dict	,    __SCREAMING_SNAKE_CASE   :      Tuple	,    __SCREAMING_SNAKE_CASE   :      int       )      ->      Any:
							a_		:	Union[str, Any]									=   self.num_labels
							a_		:	List[str]									=   TFConvBertForSequenceClassification(config=__SCREAMING_SNAKE_CASE       )
							a_		:	Optional[Any]									=   {
							    '''input_ids''': input_ids,
							    '''attention_mask''': input_mask,
							    '''token_type_ids''': token_type_ids,
							}

							a_		:	Tuple									=   model(__SCREAMING_SNAKE_CASE       )
							self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.num_labels)       )






			def       SCREAMING_SNAKE_CASE   (			self   :      List[Any]	,    __SCREAMING_SNAKE_CASE   :      List[str]	,    __SCREAMING_SNAKE_CASE   :      Optional[Any]	,    __SCREAMING_SNAKE_CASE   :      str	,    __SCREAMING_SNAKE_CASE   :      str	,    __SCREAMING_SNAKE_CASE   :      Dict	,    __SCREAMING_SNAKE_CASE   :      Tuple	,    __SCREAMING_SNAKE_CASE   :      Tuple       )      ->      Optional[int]:
							a_		:	Optional[Any]									=   self.num_choices
							a_		:	int									=   TFConvBertForMultipleChoice(config=__SCREAMING_SNAKE_CASE       )
							a_		:	List[str]									=   tf.tile(tf.expand_dims(__SCREAMING_SNAKE_CASE	,    1       )	,    (1, self.num_choices, 1)       )
							a_		:	Tuple									=   tf.tile(tf.expand_dims(__SCREAMING_SNAKE_CASE	,    1       )	,    (1, self.num_choices, 1)       )
							a_		:	List[Any]									=   tf.tile(tf.expand_dims(__SCREAMING_SNAKE_CASE	,    1       )	,    (1, self.num_choices, 1)       )
							a_		:	Union[str, Any]									=   {
							    '''input_ids''': multiple_choice_inputs_ids,
							    '''attention_mask''': multiple_choice_input_mask,
							    '''token_type_ids''': multiple_choice_token_type_ids,
							}
							a_		:	Optional[int]									=   model(__SCREAMING_SNAKE_CASE       )
							self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.num_choices)       )






			def       SCREAMING_SNAKE_CASE   (			self   :      List[Any]	,    __SCREAMING_SNAKE_CASE   :      List[str]	,    __SCREAMING_SNAKE_CASE   :      Dict	,    __SCREAMING_SNAKE_CASE   :      Optional[Any]	,    __SCREAMING_SNAKE_CASE   :      Union[str, Any]	,    __SCREAMING_SNAKE_CASE   :      Union[str, Any]	,    __SCREAMING_SNAKE_CASE   :      List[Any]	,    __SCREAMING_SNAKE_CASE   :      Tuple       )      ->      str:
							a_		:	Optional[Any]									=   self.num_labels
							a_		:	Optional[int]									=   TFConvBertForTokenClassification(config=__SCREAMING_SNAKE_CASE       )
							a_		:	Tuple									=   {
							    '''input_ids''': input_ids,
							    '''attention_mask''': input_mask,
							    '''token_type_ids''': token_type_ids,
							}
							a_		:	Tuple									=   model(__SCREAMING_SNAKE_CASE       )
							self.parent.assertEqual(result.logits.shape	,    (self.batch_size, self.seq_length, self.num_labels)       )






			def       SCREAMING_SNAKE_CASE   (			self   :      List[str]	,    __SCREAMING_SNAKE_CASE   :      Optional[Any]	,    __SCREAMING_SNAKE_CASE   :      List[Any]	,    __SCREAMING_SNAKE_CASE   :      List[Any]	,    __SCREAMING_SNAKE_CASE   :      str	,    __SCREAMING_SNAKE_CASE   :      Optional[int]	,    __SCREAMING_SNAKE_CASE   :      Tuple	,    __SCREAMING_SNAKE_CASE   :      Union[str, Any]       )      ->      Tuple:
							a_		:	List[Any]									=   TFConvBertForQuestionAnswering(config=__SCREAMING_SNAKE_CASE       )
							a_		:	Optional[Any]									=   {
							    '''input_ids''': input_ids,
							    '''attention_mask''': input_mask,
							    '''token_type_ids''': token_type_ids,
							}

							a_		:	Dict									=   model(__SCREAMING_SNAKE_CASE       )
							self.parent.assertEqual(result.start_logits.shape	,    (self.batch_size, self.seq_length)       )
							self.parent.assertEqual(result.end_logits.shape	,    (self.batch_size, self.seq_length)       )







			def       SCREAMING_SNAKE_CASE   (			self   :      List[str]       )      ->      Any:
							a_		:	List[Any]									=   self.prepare_config_and_inputs()
							(
							    (
							    a_
							)					,      (
							    a_
							)					,      (
							    a_
							)					,      (
							    a_
							)					,      (
							    a_
							)					,      (
							    a_
							)					,      (
							    a_
							)					,      
							)		:	List[Any]									=   config_and_inputs
							a_		:	Any									=   {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
							return config, inputs_dict





@require_tf
class 						SCREAMING_SNAKE_CASE       (       SCREAMING_SNAKE_CASE_						, SCREAMING_SNAKE_CASE_						, unittest.TestCase							):
			snake_case__	=					(
			    (
			        TFConvBertModel,
			        TFConvBertForMaskedLM,
			        TFConvBertForQuestionAnswering,
			        TFConvBertForSequenceClassification,
			        TFConvBertForTokenClassification,
			        TFConvBertForMultipleChoice,
			    )
			    if is_tf_available()
			    else ()
			)
			snake_case__	=					(
			    {
			        "feature-extraction": TFConvBertModel,
			        "fill-mask": TFConvBertForMaskedLM,
			        "question-answering": TFConvBertForQuestionAnswering,
			        "text-classification": TFConvBertForSequenceClassification,
			        "token-classification": TFConvBertForTokenClassification,
			        "zero-shot": TFConvBertForSequenceClassification,
			    }
			    if is_tf_available()
			    else {}
			)
			snake_case__	=					False
			snake_case__	=					False
			snake_case__	=					False






			def       SCREAMING_SNAKE_CASE   (			self   :      List[str]       )      ->      List[Any]:
							a_		:	Optional[int]									=   TFConvBertModelTester(self       )
							a_		:	Any									=   ConfigTester(self	,    config_class=__SCREAMING_SNAKE_CASE	,    hidden_size=37       )






			def       SCREAMING_SNAKE_CASE   (			self   :      str       )      ->      Optional[Any]:
							self.config_tester.run_common_tests()






			def       SCREAMING_SNAKE_CASE   (			self   :      Tuple       )      ->      str:
							a_		:	Any									=   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE       )






			def       SCREAMING_SNAKE_CASE   (			self   :      Optional[Any]       )      ->      List[str]:
							a_		:	str									=   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*__SCREAMING_SNAKE_CASE       )






			def       SCREAMING_SNAKE_CASE   (			self   :      Any       )      ->      str:
							a_		:	Dict									=   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*__SCREAMING_SNAKE_CASE       )






			def       SCREAMING_SNAKE_CASE   (			self   :      str       )      ->      Optional[Any]:
							a_		:	Optional[Any]									=   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*__SCREAMING_SNAKE_CASE       )






			def       SCREAMING_SNAKE_CASE   (			self   :      Any       )      ->      str:
							a_		:	List[str]									=   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*__SCREAMING_SNAKE_CASE       )






			def       SCREAMING_SNAKE_CASE   (			self   :      str       )      ->      Optional[int]:
							a_		:	Any									=   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*__SCREAMING_SNAKE_CASE       )






			@slow
			def       SCREAMING_SNAKE_CASE   (			self   :      Optional[int]       )      ->      Union[str, Any]:
							a_					,      a_		:	Optional[Any]									=   self.model_tester.prepare_config_and_inputs_for_common()
							a_		:	Any									=   True
							a_		:	Optional[int]									=   True

							if hasattr(__SCREAMING_SNAKE_CASE	,    '''use_cache'''       ):
											a_		:	Dict									=   True

							a_		:	Any									=   getattr(self.model_tester	,    '''encoder_seq_length'''	,    self.model_tester.seq_length       )
							a_		:	List[Any]									=   getattr(self.model_tester	,    '''key_length'''	,    __SCREAMING_SNAKE_CASE       )

							for model_class in self.all_model_classes:
											a_		:	str									=   self._prepare_for_class(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )
											a_		:	Optional[int]									=   model_class(__SCREAMING_SNAKE_CASE       )
											a_		:	Any									=   len(model(__SCREAMING_SNAKE_CASE       )       )

											with tempfile.TemporaryDirectory() as tmpdirname:
															model.save_pretrained(__SCREAMING_SNAKE_CASE	,    saved_model=__SCREAMING_SNAKE_CASE       )
															a_		:	Union[str, Any]									=   os.path.join(__SCREAMING_SNAKE_CASE	,    '''saved_model'''	,    '''1'''       )
															a_		:	Optional[int]									=   tf.keras.models.load_model(__SCREAMING_SNAKE_CASE       )
															a_		:	Any									=   model(__SCREAMING_SNAKE_CASE       )

															if self.is_encoder_decoder:
																			a_		:	Optional[int]									=   outputs['''encoder_hidden_states''']
																			a_		:	Dict									=   outputs['''encoder_attentions''']
															else:
																			a_		:	Union[str, Any]									=   outputs['''hidden_states''']
																			a_		:	Tuple									=   outputs['''attentions''']

															self.assertEqual(len(__SCREAMING_SNAKE_CASE       )	,    __SCREAMING_SNAKE_CASE       )

															a_		:	Union[str, Any]									=   getattr(
															    self.model_tester	,    '''expected_num_hidden_layers'''	,    self.model_tester.num_hidden_layers + 1       )

															self.assertEqual(len(__SCREAMING_SNAKE_CASE       )	,    __SCREAMING_SNAKE_CASE       )
															self.assertListEqual(
															    list(output_hidden_states[0].shape[-2:]       )	,    [self.model_tester.seq_length, self.model_tester.hidden_size]	,    )

															self.assertEqual(len(__SCREAMING_SNAKE_CASE       )	,    self.model_tester.num_hidden_layers       )
															self.assertListEqual(
															    list(output_attentions[0].shape[-3:]       )	,    [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]	,    )






			@slow
			def       SCREAMING_SNAKE_CASE   (			self   :      str       )      ->      Optional[int]:
							a_		:	Tuple									=   TFConvBertModel.from_pretrained('''YituTech/conv-bert-base'''       )
							self.assertIsNotNone(__SCREAMING_SNAKE_CASE       )







			def       SCREAMING_SNAKE_CASE   (			self   :      Any       )      ->      List[Any]:
							a_					,      a_		:	Union[str, Any]									=   self.model_tester.prepare_config_and_inputs_for_common()
							a_		:	Optional[Any]									=   True
							a_		:	Dict									=   getattr(self.model_tester	,    '''decoder_seq_length'''	,    self.model_tester.seq_length       )
							a_		:	List[str]									=   getattr(self.model_tester	,    '''encoder_seq_length'''	,    self.model_tester.seq_length       )
							a_		:	Dict									=   getattr(self.model_tester	,    '''key_length'''	,    __SCREAMING_SNAKE_CASE       )
							a_		:	Any									=   getattr(self.model_tester	,    '''key_length'''	,    __SCREAMING_SNAKE_CASE       )

							def check_decoder_attentions_output(__SCREAMING_SNAKE_CASE   :      List[str]       ):
											a_		:	int									=   len(__SCREAMING_SNAKE_CASE       )
											self.assertEqual(out_len % 2	,    0       )
											a_		:	Dict									=   outputs.decoder_attentions
											self.assertEqual(len(__SCREAMING_SNAKE_CASE       )	,    self.model_tester.num_hidden_layers       )
											self.assertListEqual(
											    list(decoder_attentions[0].shape[-3:]       )	,    [self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]	,    )

							def check_encoder_attentions_output(__SCREAMING_SNAKE_CASE   :      List[str]       ):
											a_		:	Optional[int]									=   [
											    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
											]
											self.assertEqual(len(__SCREAMING_SNAKE_CASE       )	,    self.model_tester.num_hidden_layers       )
											self.assertListEqual(
											    list(attentions[0].shape[-3:]       )	,    [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]	,    )

							for model_class in self.all_model_classes:
											a_		:	Tuple									=   True
											a_		:	Tuple									=   False
											a_		:	Optional[Any]									=   model_class(__SCREAMING_SNAKE_CASE       )
											a_		:	List[Any]									=   model(self._prepare_for_class(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )       )
											a_		:	Union[str, Any]									=   len(__SCREAMING_SNAKE_CASE       )
											self.assertEqual(config.output_hidden_states	,    __SCREAMING_SNAKE_CASE       )
											check_encoder_attentions_output(__SCREAMING_SNAKE_CASE       )

											if self.is_encoder_decoder:
															a_		:	Any									=   model_class(__SCREAMING_SNAKE_CASE       )
															a_		:	List[Any]									=   model(self._prepare_for_class(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )       )
															self.assertEqual(config.output_hidden_states	,    __SCREAMING_SNAKE_CASE       )
															check_decoder_attentions_output(__SCREAMING_SNAKE_CASE       )

											# Check that output attentions can also be changed via the config
											del inputs_dict["output_attentions"]
											a_		:	Union[str, Any]									=   True
											a_		:	Union[str, Any]									=   model_class(__SCREAMING_SNAKE_CASE       )
											a_		:	int									=   model(self._prepare_for_class(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )       )
											self.assertEqual(config.output_hidden_states	,    __SCREAMING_SNAKE_CASE       )
											check_encoder_attentions_output(__SCREAMING_SNAKE_CASE       )

											# Check attention is always last and order is fine
											a_		:	Dict									=   True
											a_		:	str									=   True
											a_		:	int									=   model_class(__SCREAMING_SNAKE_CASE       )
											a_		:	List[Any]									=   model(self._prepare_for_class(__SCREAMING_SNAKE_CASE	,    __SCREAMING_SNAKE_CASE       )       )

											self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)	,    len(__SCREAMING_SNAKE_CASE       )       )
											self.assertEqual(model.config.output_hidden_states	,    __SCREAMING_SNAKE_CASE       )
											check_encoder_attentions_output(__SCREAMING_SNAKE_CASE       )





@require_tf
class 						SCREAMING_SNAKE_CASE       (       unittest.TestCase							):






			@slow
			def       SCREAMING_SNAKE_CASE   (			self   :      int       )      ->      Any:
							a_		:	int									=   TFConvBertModel.from_pretrained('''YituTech/conv-bert-base'''       )
							a_		:	List[Any]									=   tf.constant([[0, 1, 2, 3, 4, 5]]       )
							a_		:	Any									=   model(__SCREAMING_SNAKE_CASE       )[0]

							a_		:	str									=   [1, 6, 768]
							self.assertEqual(output.shape	,    __SCREAMING_SNAKE_CASE       )

							a_		:	Dict									=   tf.constant(
							    [
							        [
							            [-0.0347_5493, -0.468_6034, -0.3063_8832],
							            [0.2263_7248, -0.2698_8646, -0.742_3424],
							            [0.1032_4868, -0.4501_3508, -0.5828_0784],
							        ]
							    ]       )
							tf.debugging.assert_near(output[:, :3, :3]	,    __SCREAMING_SNAKE_CASE	,    atol=1e-4       )




