'''simple docstring'''






import inspect
import unittest

import numpy as np

from transformers import BeitConfig
from transformers.testing_utils import require_flax, require_vision, slow
from transformers.utils import cached_property, is_flax_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor


if is_flax_available():
							import jax

							from transformers import FlaxBeitForImageClassification, FlaxBeitForMaskedImageModeling, FlaxBeitModel

if is_vision_available():
							from PIL import Image

							from transformers import BeitImageProcessor
class 						__UpperCamelCase	(						unittest.TestCase							):





							def __init__(  self		:Optional[int]  ,_UpperCamelCase		:Union[str, Any]  ,_UpperCamelCase		:Dict=1_0_0  ,_UpperCamelCase		:List[str]=1_3  ,_UpperCamelCase		:Optional[int]=3_0  ,_UpperCamelCase		:Optional[Any]=2  ,_UpperCamelCase		:str=3  ,_UpperCamelCase		:List[str]=True  ,_UpperCamelCase		:int=True  ,_UpperCamelCase		:Any=3_2  ,_UpperCamelCase		:str=5  ,_UpperCamelCase		:str=4  ,_UpperCamelCase		:str=3_7  ,_UpperCamelCase		:Union[str, Any]="gelu"  ,_UpperCamelCase		:List[Any]=0.1  ,_UpperCamelCase		:List[Any]=0.1  ,_UpperCamelCase		:Union[str, Any]=1_0  ,_UpperCamelCase		:Tuple=0.02  ,_UpperCamelCase		:Any=3  ,):
									snake_case_		:							List[Any]			       =						parent
									snake_case_		:							List[Any]			       =						vocab_size
									snake_case_		:							Any			       =						batch_size
									snake_case_		:							Dict			       =						image_size
									snake_case_		:							Dict			       =						patch_size
									snake_case_		:							List[Any]			       =						num_channels
									snake_case_		:							str			       =						is_training
									snake_case_		:							Any			       =						use_labels
									snake_case_		:							Dict			       =						hidden_size
									snake_case_		:							int			       =						num_hidden_layers
									snake_case_		:							str			       =						num_attention_heads
									snake_case_		:							List[str]			       =						intermediate_size
									snake_case_		:							int			       =						hidden_act
									snake_case_		:							int			       =						hidden_dropout_prob
									snake_case_		:							Optional[int]			       =						attention_probs_dropout_prob
									snake_case_		:							Optional[int]			       =						type_sequence_label_size
									snake_case_		:							Dict			       =						initializer_range

									# in BeiT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
									snake_case_		:							str			       =						(image_size // patch_size) ** 2
									snake_case_		:							int			       =						num_patches + 1





							def 		a__   (  self		:List[str]			):
									snake_case_		:							Any			       =						floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]			)

									snake_case_		:							List[str]			       =						None
									if self.use_labels:
											snake_case_		:							Any			       =						ids_tensor([self.batch_size]  ,self.type_sequence_label_size			)

									snake_case_		:							List[str]			       =						BeitConfig(
									    vocab_size=self.vocab_size  ,image_size=self.image_size  ,patch_size=self.patch_size  ,num_channels=self.num_channels  ,hidden_size=self.hidden_size  ,num_hidden_layers=self.num_hidden_layers  ,num_attention_heads=self.num_attention_heads  ,intermediate_size=self.intermediate_size  ,hidden_act=self.hidden_act  ,hidden_dropout_prob=self.hidden_dropout_prob  ,attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,is_decoder=snake_case__  ,initializer_range=self.initializer_range  ,)

									return config, pixel_values, labels





							def 		a__   (  self		:List[Any]  ,_UpperCamelCase		:List[Any]  ,_UpperCamelCase		:str  ,_UpperCamelCase		:Optional[int]			):
									snake_case_		:							Dict			       =						FlaxBeitModel(config=snake_case__			)
									snake_case_		:							int			       =						model(snake_case__			)
									self.parent.assertEqual(result.last_hidden_state.shape  ,(self.batch_size, self.seq_length, self.hidden_size)			)





							def 		a__   (  self		:int  ,_UpperCamelCase		:Any  ,_UpperCamelCase		:str  ,_UpperCamelCase		:Optional[int]			):
									snake_case_		:							Optional[Any]			       =						FlaxBeitForMaskedImageModeling(config=snake_case__			)
									snake_case_		:							Any			       =						model(snake_case__			)
									self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.seq_length - 1, self.vocab_size)			)





							def 		a__   (  self		:Optional[Any]  ,_UpperCamelCase		:Any  ,_UpperCamelCase		:int  ,_UpperCamelCase		:Tuple			):
									snake_case_		:							str			       =						self.type_sequence_label_size
									snake_case_		:							Optional[Any]			       =						FlaxBeitForImageClassification(config=snake_case__			)
									snake_case_		:							str			       =						model(snake_case__			)
									self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.type_sequence_label_size)			)

									# test greyscale images
									snake_case_		:							List[Any]			       =						1
									snake_case_		:							Dict			       =						FlaxBeitForImageClassification(snake_case__			)

									snake_case_		:							Union[str, Any]			       =						floats_tensor([self.batch_size, 1, self.image_size, self.image_size]			)
									snake_case_		:							int			       =						model(snake_case__			)





							def 		a__   (  self		:List[str]			):
									snake_case_		:							List[str]			       =						self.prepare_config_and_inputs()
									(
									    (
									    snake_case_						
									)       ,      (
									    snake_case_						
									)       ,      (
									    snake_case_						
									)       ,      						
									)		:							Optional[int]			       =						config_and_inputs
									snake_case_		:							Dict			       =						{"""pixel_values""": pixel_values}
									return config, inputs_dict





@require_flax
class 						__UpperCamelCase	(						__a						,  unittest.TestCase							):
							lowercase			:		Optional[Any]   		=  (
							    (FlaxBeitModel, FlaxBeitForImageClassification, FlaxBeitForMaskedImageModeling) if is_flax_available() else ()
							)





							def 		a__   (  self		:Union[str, Any]			):
									snake_case_		:							Union[str, Any]			       =						FlaxBeitModelTester(self			)
									snake_case_		:							str			       =						ConfigTester(self  ,config_class=snake_case__  ,has_text_modality=snake_case__  ,hidden_size=3_7			)





							def 		a__   (  self		:List[str]			):
									self.config_tester.run_common_tests()





							def 		a__   (  self		:Optional[Any]			):
									snake_case_       ,      snake_case_		:							Optional[Any]			       =						self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
											snake_case_		:							Any			       =						model_class(snake_case__			)
											snake_case_		:							Any			       =						inspect.signature(model.__call__			)
											# signature.parameters is an OrderedDict => so arg_names order is deterministic
											snake_case_		:							List[Any]			       =						[*signature.parameters.keys()]

											snake_case_		:							str			       =						["""pixel_values"""]
											self.assertListEqual(arg_names[:1]  ,snake_case__			)





							def 		a__   (  self		:int			):
									snake_case_       ,      snake_case_		:							Optional[int]			       =						self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
											with self.subTest(model_class.__name__			):
													snake_case_		:							str			       =						self._prepare_for_class(snake_case__  ,snake_case__			)
													snake_case_		:							Union[str, Any]			       =						model_class(snake_case__			)

													@jax.jit
													def model_jitted(_UpperCamelCase		:Optional[int]  ,**_UpperCamelCase		:str			):
															return model(pixel_values=snake_case__  ,**snake_case__			)

													with self.subTest("""JIT Enabled"""			):
															snake_case_		:							Tuple			       =						model_jitted(**snake_case__			).to_tuple()

													with self.subTest("""JIT Disabled"""			):
															with jax.disable_jit():
																	snake_case_		:							Dict			       =						model_jitted(**snake_case__			).to_tuple()

													self.assertEqual(len(snake_case__			)  ,len(snake_case__			)			)
													for jitted_output, output in zip(snake_case__  ,snake_case__			):
															self.assertEqual(jitted_output.shape  ,output.shape			)





							def 		a__   (  self		:Union[str, Any]			):
									snake_case_		:							Optional[int]			       =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*snake_case__			)





							def 		a__   (  self		:Tuple			):
									snake_case_		:							List[str]			       =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_masked_lm(*snake_case__			)





							def 		a__   (  self		:List[str]			):
									snake_case_		:							Union[str, Any]			       =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_image_classification(*snake_case__			)





							@slow
							def 		a__   (  self		:Any			):
									for model_class_name in self.all_model_classes:
											snake_case_		:							str			       =						model_class_name.from_pretrained("""microsoft/beit-base-patch16-224"""			)
											snake_case_		:							List[Any]			       =						model(np.ones((1, 3, 2_2_4, 2_2_4)			)			)
											self.assertIsNotNone(snake_case__			)





def      UpperCAmelCase      (		):







		'''simple docstring'''
		snake_case_		:							Union[str, Any]			       =						Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png""" )
		return image






@require_vision
@require_flax
class 						__UpperCamelCase	(						unittest.TestCase							):





							@cached_property
							def 		a__   (  self		:int			):
									return BeitImageProcessor.from_pretrained("""microsoft/beit-base-patch16-224"""			) if is_vision_available() else None





							@slow
							def 		a__   (  self		:int			):
									snake_case_		:							Tuple			       =						FlaxBeitForMaskedImageModeling.from_pretrained("""microsoft/beit-base-patch16-224-pt22k"""			)

									snake_case_		:							int			       =						self.default_image_processor
									snake_case_		:							List[Any]			       =						prepare_img()
									snake_case_		:							Tuple			       =						image_processor(images=snake_case__  ,return_tensors="""np"""			).pixel_values

									# prepare bool_masked_pos
									snake_case_		:							Optional[int]			       =						np.ones((1, 1_9_6)  ,dtype=snake_case__			)

									# forward pass
									snake_case_		:							Tuple			       =						model(pixel_values=snake_case__  ,bool_masked_pos=snake_case__			)
									snake_case_		:							str			       =						outputs.logits

									# verify the logits
									snake_case_		:							Optional[Any]			       =						(1, 1_9_6, 8_1_9_2)
									self.assertEqual(logits.shape  ,snake_case__			)

									snake_case_		:							List[str]			       =						np.array(
									    [[-3.24_37, 0.50_72, -13.91_74], [-3.24_56, 0.49_48, -13.94_01], [-3.20_33, 0.51_21, -13.85_50]]			)

									self.assertTrue(np.allclose(logits[bool_masked_pos][:3, :3]  ,snake_case__  ,atol=1E-2			)			)





							@slow
							def 		a__   (  self		:Tuple			):
									snake_case_		:							int			       =						FlaxBeitForImageClassification.from_pretrained("""microsoft/beit-base-patch16-224"""			)

									snake_case_		:							List[Any]			       =						self.default_image_processor
									snake_case_		:							Optional[Any]			       =						prepare_img()
									snake_case_		:							Dict			       =						image_processor(images=snake_case__  ,return_tensors="""np"""			)

									# forward pass
									snake_case_		:							Dict			       =						model(**snake_case__			)
									snake_case_		:							str			       =						outputs.logits

									# verify the logits
									snake_case_		:							Any			       =						(1, 1_0_0_0)
									self.assertEqual(logits.shape  ,snake_case__			)

									snake_case_		:							Optional[Any]			       =						np.array([-1.23_85, -1.09_87, -1.01_08]			)

									self.assertTrue(np.allclose(logits[0, :3]  ,snake_case__  ,atol=1E-4			)			)

									snake_case_		:							int			       =						2_8_1
									self.assertEqual(logits.argmax(-1			).item()  ,snake_case__			)





							@slow
							def 		a__   (  self		:Optional[int]			):
									snake_case_		:							int			       =						FlaxBeitForImageClassification.from_pretrained("""microsoft/beit-large-patch16-224-pt22k-ft22k"""			)

									snake_case_		:							Optional[int]			       =						self.default_image_processor
									snake_case_		:							Optional[Any]			       =						prepare_img()
									snake_case_		:							Tuple			       =						image_processor(images=snake_case__  ,return_tensors="""np"""			)

									# forward pass
									snake_case_		:							Optional[Any]			       =						model(**snake_case__			)
									snake_case_		:							str			       =						outputs.logits

									# verify the logits
									snake_case_		:							List[str]			       =						(1, 2_1_8_4_1)
									self.assertEqual(logits.shape  ,snake_case__			)

									snake_case_		:							Dict			       =						np.array([1.68_81, -0.27_87, 0.59_01]			)

									self.assertTrue(np.allclose(logits[0, :3]  ,snake_case__  ,atol=1E-4			)			)

									snake_case_		:							Any			       =						2_3_9_6
									self.assertEqual(logits.argmax(-1			).item()  ,snake_case__			)



'''simple docstring'''






import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


__A	:	Any     =     logging.getLogger(__name__)
class 						__UpperCamelCase	(						lowercase__							):





							def 		a__   (  self		:int  ,_UpperCamelCase		:Union[str, Any]  ,_UpperCamelCase		:Optional[Any]  ,_UpperCamelCase		:List[Any]=None  ,_UpperCamelCase		:Any=None			):
									snake_case_		:							List[str]			       =						self.layer[current_layer](_UpperCamelCase  ,_UpperCamelCase  ,head_mask[current_layer]			)

									snake_case_		:							Tuple			       =						layer_outputs[0]

									return hidden_states





@add_start_docstrings(
    'The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.'						,  lowercase__						,  )
class 						__UpperCamelCase	(						lowercase__							):





							def __init__(  self		:Dict  ,_UpperCamelCase		:Optional[Any]			):
									super().__init__(_UpperCamelCase			)

									snake_case_		:							List[Any]			       =						BertEncoderWithPabee(_UpperCamelCase			)

									self.init_weights()
									snake_case_		:							int			       =						0
									snake_case_		:							Optional[Any]			       =						0
									snake_case_		:							Any			       =						0

									snake_case_		:							str			       =						0





							def 		a__   (  self		:Tuple  ,_UpperCamelCase		:int			):
									snake_case_		:							Optional[Any]			       =						threshold





							def 		a__   (  self		:Optional[Any]  ,_UpperCamelCase		:Union[str, Any]			):
									snake_case_		:							str			       =						patience





							def 		a__   (  self		:Optional[int]			):
									snake_case_		:							List[str]			       =						0
									snake_case_		:							List[Any]			       =						0





							def 		a__   (  self		:List[Any]			):
									snake_case_		:							str			       =						self.inference_layers_num / self.inference_instances_num
									snake_case_		:							List[str]			       =						(
									    F'''*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='''
									    F''' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'''
									)
									print(_UpperCamelCase			)





							@add_start_docstrings_to_model_forward(_UpperCamelCase			)
							def 		a__   (  self		:Optional[Any]  ,_UpperCamelCase		:Any=None  ,_UpperCamelCase		:List[str]=None  ,_UpperCamelCase		:Any=None  ,_UpperCamelCase		:Optional[int]=None  ,_UpperCamelCase		:Optional[int]=None  ,_UpperCamelCase		:int=None  ,_UpperCamelCase		:Optional[int]=None  ,_UpperCamelCase		:Any=None  ,_UpperCamelCase		:int=None  ,_UpperCamelCase		:Union[str, Any]=None  ,_UpperCamelCase		:int=False  ,):

									if input_ids is not None and inputs_embeds is not None:
											raise ValueError("""You cannot specify both input_ids and inputs_embeds at the same time"""			)
									elif input_ids is not None:
											snake_case_		:							Any			       =						input_ids.size()
									elif inputs_embeds is not None:
											snake_case_		:							Union[str, Any]			       =						inputs_embeds.size()[:-1]
									else:
											raise ValueError("""You have to specify either input_ids or inputs_embeds"""			)

									snake_case_		:							Tuple			       =						input_ids.device if input_ids is not None else inputs_embeds.device

									if attention_mask is None:
											snake_case_		:							Any			       =						torch.ones(_UpperCamelCase  ,device=_UpperCamelCase			)
									if token_type_ids is None:
											snake_case_		:							str			       =						torch.zeros(_UpperCamelCase  ,dtype=torch.long  ,device=_UpperCamelCase			)

									# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
									# ourselves in which case we just need to make it broadcastable to all heads.
									snake_case_		:							torch.Tensor			       =						self.get_extended_attention_mask(_UpperCamelCase  ,_UpperCamelCase  ,_UpperCamelCase			)

									# If a 2D ou 3D attention mask is provided for the cross-attention
									# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
									if self.config.is_decoder and encoder_hidden_states is not None:
											snake_case_       ,      snake_case_       ,      snake_case_		:							List[Any]			       =						encoder_hidden_states.size()
											snake_case_		:							str			       =						(encoder_batch_size, encoder_sequence_length)
											if encoder_attention_mask is None:
													snake_case_		:							List[str]			       =						torch.ones(_UpperCamelCase  ,device=_UpperCamelCase			)
											snake_case_		:							Optional[Any]			       =						self.invert_attention_mask(_UpperCamelCase			)
									else:
											snake_case_		:							List[str]			       =						None

									# Prepare head mask if needed
									# 1.0 in head_mask indicate we keep the head
									# attention_probs has shape bsz x n_heads x N x N
									# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
									# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
									snake_case_		:							List[Any]			       =						self.get_head_mask(_UpperCamelCase  ,self.config.num_hidden_layers			)

									snake_case_		:							Dict			       =						self.embeddings(
									    input_ids=_UpperCamelCase  ,position_ids=_UpperCamelCase  ,token_type_ids=_UpperCamelCase  ,inputs_embeds=_UpperCamelCase			)
									snake_case_		:							Optional[int]			       =						embedding_output

									if self.training:
											snake_case_		:							Any			       =						[]
											for i in range(self.config.num_hidden_layers			):
													snake_case_		:							Any			       =						self.encoder.adaptive_forward(
													    _UpperCamelCase  ,current_layer=_UpperCamelCase  ,attention_mask=_UpperCamelCase  ,head_mask=_UpperCamelCase			)

													snake_case_		:							Dict			       =						self.pooler(_UpperCamelCase			)
													snake_case_		:							str			       =						output_layers[i](output_dropout(_UpperCamelCase			)			)
													res.append(_UpperCamelCase			)
									elif self.patience == 0:  # Use all layers for inference
											snake_case_		:							List[str]			       =						self.encoder(
											    _UpperCamelCase  ,attention_mask=_UpperCamelCase  ,head_mask=_UpperCamelCase  ,encoder_hidden_states=_UpperCamelCase  ,encoder_attention_mask=_UpperCamelCase  ,)
											snake_case_		:							int			       =						self.pooler(encoder_outputs[0]			)
											snake_case_		:							Tuple			       =						[output_layers[self.config.num_hidden_layers - 1](_UpperCamelCase			)]
									else:
											snake_case_		:							Any			       =						0
											snake_case_		:							List[str]			       =						None
											snake_case_		:							Any			       =						0
											for i in range(self.config.num_hidden_layers			):
													calculated_layer_num += 1
													snake_case_		:							Dict			       =						self.encoder.adaptive_forward(
													    _UpperCamelCase  ,current_layer=_UpperCamelCase  ,attention_mask=_UpperCamelCase  ,head_mask=_UpperCamelCase			)

													snake_case_		:							Union[str, Any]			       =						self.pooler(_UpperCamelCase			)
													snake_case_		:							Optional[Any]			       =						output_layers[i](_UpperCamelCase			)
													if regression:
															snake_case_		:							Union[str, Any]			       =						logits.detach()
															if patient_result is not None:
																	snake_case_		:							Union[str, Any]			       =						patient_result.detach()
															if (patient_result is not None) and torch.abs(patient_result - labels			) < self.regression_threshold:
																	patient_counter += 1
															else:
																	snake_case_		:							Union[str, Any]			       =						0
													else:
															snake_case_		:							Any			       =						logits.detach().argmax(dim=1			)
															if patient_result is not None:
																	snake_case_		:							str			       =						patient_result.detach().argmax(dim=1			)
															if (patient_result is not None) and torch.all(labels.eq(_UpperCamelCase			)			):
																	patient_counter += 1
															else:
																	snake_case_		:							List[str]			       =						0

													snake_case_		:							List[Any]			       =						logits
													if patient_counter == self.patience:
															break
											snake_case_		:							int			       =						[patient_result]
											self.inference_layers_num += calculated_layer_num
											self.inference_instances_num += 1

									return res





@add_start_docstrings(
    'Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. '						,  lowercase__						,  )
class 						__UpperCamelCase	(						lowercase__							):





							def __init__(  self		:Tuple  ,_UpperCamelCase		:List[str]			):
									super().__init__(_UpperCamelCase			)
									snake_case_		:							List[str]			       =						config.num_labels

									snake_case_		:							Dict			       =						BertModelWithPabee(_UpperCamelCase			)
									snake_case_		:							List[str]			       =						nn.Dropout(config.hidden_dropout_prob			)
									snake_case_		:							str			       =						nn.ModuleList(
									    [nn.Linear(config.hidden_size  ,self.config.num_labels			) for _ in range(config.num_hidden_layers			)]			)

									self.init_weights()





							@add_start_docstrings_to_model_forward(_UpperCamelCase			)
							def 		a__   (  self		:Union[str, Any]  ,_UpperCamelCase		:Tuple=None  ,_UpperCamelCase		:Dict=None  ,_UpperCamelCase		:Union[str, Any]=None  ,_UpperCamelCase		:Union[str, Any]=None  ,_UpperCamelCase		:Tuple=None  ,_UpperCamelCase		:int=None  ,_UpperCamelCase		:Union[str, Any]=None  ,):
									snake_case_		:							int			       =						self.bert(
									    input_ids=_UpperCamelCase  ,attention_mask=_UpperCamelCase  ,token_type_ids=_UpperCamelCase  ,position_ids=_UpperCamelCase  ,head_mask=_UpperCamelCase  ,inputs_embeds=_UpperCamelCase  ,output_dropout=self.dropout  ,output_layers=self.classifiers  ,regression=self.num_labels == 1  ,)

									snake_case_		:							Optional[int]			       =						(logits[-1],)

									if labels is not None:
											snake_case_		:							Optional[Any]			       =						None
											snake_case_		:							Optional[int]			       =						0
											for ix, logits_item in enumerate(_UpperCamelCase			):
													if self.num_labels == 1:
															#  We are doing regression
															snake_case_		:							Dict			       =						MSELoss()
															snake_case_		:							List[str]			       =						loss_fct(logits_item.view(-1			)  ,labels.view(-1			)			)
													else:
															snake_case_		:							Any			       =						CrossEntropyLoss()
															snake_case_		:							str			       =						loss_fct(logits_item.view(-1  ,self.num_labels			)  ,labels.view(-1			)			)
													if total_loss is None:
															snake_case_		:							str			       =						loss
													else:
															total_loss += loss * (ix + 1)
													total_weights += ix + 1
											snake_case_		:							str			       =						(total_loss / total_weights,) + outputs

									return outputs