




'''simple docstring'''




import inspect
import unittest

from datasets import load_dataset
from packaging import version

from transformers import BeitConfig
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch
					from torch import nn

					from transformers import (
					    MODEL_MAPPING,
					    BeitForImageClassification,
					    BeitForMaskedImageModeling,
					    BeitForSemanticSegmentation,
					    BeitModel,
					)
					from transformers.models.beit.modeling_beit import BEIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
					import PIL
					from PIL import Image

					from transformers import BeitImageProcessor






class      UpperCAmelCase_     :


	"""simple docstring"""


	def __init__(						self						, UpperCAmelCase_						, UpperCAmelCase_=1_00						, UpperCAmelCase_=13						, UpperCAmelCase_=30						, UpperCAmelCase_=2						, UpperCAmelCase_=3						, UpperCAmelCase_=True						, UpperCAmelCase_=True						, UpperCAmelCase_=32						, UpperCAmelCase_=4						, UpperCAmelCase_=4						, UpperCAmelCase_=37						, UpperCAmelCase_="gelu"						, UpperCAmelCase_=0.1						, UpperCAmelCase_=0.1						, UpperCAmelCase_=10						, UpperCAmelCase_=0.02						, UpperCAmelCase_=3						, UpperCAmelCase_=None						, UpperCAmelCase_=[0, 1, 2, 3]						, ):
						snake_case_	=      parent
						snake_case_	=      1_00
						snake_case_	=      batch_size
						snake_case_	=      image_size
						snake_case_	=      patch_size
						snake_case_	=      num_channels
						snake_case_	=      is_training
						snake_case_	=      use_labels
						snake_case_	=      hidden_size
						snake_case_	=      num_hidden_layers
						snake_case_	=      num_attention_heads
						snake_case_	=      intermediate_size
						snake_case_	=      hidden_act
						snake_case_	=      hidden_dropout_prob
						snake_case_	=      attention_probs_dropout_prob
						snake_case_	=      type_sequence_label_size
						snake_case_	=      initializer_range
						snake_case_	=      scope
						snake_case_	=      out_indices
						snake_case_	=      num_labels

						# in BeiT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
						snake_case_	=      (image_size // patch_size) ** 2
						snake_case_	=      num_patches + 1







	def  _lowercase    (						self				):
						snake_case_	=      floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]				)

						snake_case_	=      None
						snake_case_	=      None
						if self.use_labels:
											snake_case_	=      ids_tensor([self.batch_size]						, self.type_sequence_label_size				)
											snake_case_	=      ids_tensor([self.batch_size, self.image_size, self.image_size]						, self.num_labels				)

						snake_case_	=      self.get_config()

						return config, pixel_values, labels, pixel_labels







	def  _lowercase    (						self				):
						return BeitConfig(
						    vocab_size=self.vocab_size						, image_size=self.image_size						, patch_size=self.patch_size						, num_channels=self.num_channels						, hidden_size=self.hidden_size						, num_hidden_layers=self.num_hidden_layers						, num_attention_heads=self.num_attention_heads						, intermediate_size=self.intermediate_size						, hidden_act=self.hidden_act						, hidden_dropout_prob=self.hidden_dropout_prob						, attention_probs_dropout_prob=self.attention_probs_dropout_prob						, is_decoder=_lowercase						, initializer_range=self.initializer_range						, out_indices=self.out_indices						, )







	def  _lowercase    (						self						, UpperCAmelCase_						, UpperCAmelCase_						, UpperCAmelCase_						, UpperCAmelCase_				):
						snake_case_	=      BeitModel(config=_lowercase				)
						model.to(_lowercase				)
						model.eval()
						snake_case_	=      model(_lowercase				)
						self.parent.assertEqual(result.last_hidden_state.shape						, (self.batch_size, self.seq_length, self.hidden_size)				)







	def  _lowercase    (						self						, UpperCAmelCase_						, UpperCAmelCase_						, UpperCAmelCase_						, UpperCAmelCase_				):
						snake_case_	=      BeitForMaskedImageModeling(config=_lowercase				)
						model.to(_lowercase				)
						model.eval()
						snake_case_	=      model(_lowercase				)
						self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.seq_length - 1, self.vocab_size)				)







	def  _lowercase    (						self						, UpperCAmelCase_						, UpperCAmelCase_						, UpperCAmelCase_						, UpperCAmelCase_				):
						snake_case_	=      self.type_sequence_label_size
						snake_case_	=      BeitForImageClassification(_lowercase				)
						model.to(_lowercase				)
						model.eval()
						snake_case_	=      model(_lowercase						, labels=_lowercase				)
						self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.type_sequence_label_size)				)

						# test greyscale images
						snake_case_	=      1
						snake_case_	=      BeitForImageClassification(_lowercase				)
						model.to(_lowercase				)
						model.eval()

						snake_case_	=      floats_tensor([self.batch_size, 1, self.image_size, self.image_size]				)
						snake_case_	=      model(_lowercase						, labels=_lowercase				)
						self.parent.assertEqual(result.logits.shape						, (self.batch_size, self.type_sequence_label_size)				)







	def  _lowercase    (						self						, UpperCAmelCase_						, UpperCAmelCase_						, UpperCAmelCase_						, UpperCAmelCase_				):
						snake_case_	=      self.num_labels
						snake_case_	=      BeitForSemanticSegmentation(_lowercase				)
						model.to(_lowercase				)
						model.eval()
						snake_case_	=      model(_lowercase				)
						self.parent.assertEqual(
						    result.logits.shape						, (self.batch_size, self.num_labels, self.image_size * 2, self.image_size * 2)				)
						snake_case_	=      model(_lowercase						, labels=_lowercase				)
						self.parent.assertEqual(
						    result.logits.shape						, (self.batch_size, self.num_labels, self.image_size * 2, self.image_size * 2)				)







	def  _lowercase    (						self				):
						snake_case_	=      self.prepare_config_and_inputs()
						snake_case_	,					snake_case_	,					snake_case_	,					snake_case_	=      config_and_inputs
						snake_case_	=      {"pixel_values": pixel_values}
						return config, inputs_dict






@require_torch
class      UpperCAmelCase_     (  SCREAMING_SNAKE_CASE__       ,	SCREAMING_SNAKE_CASE__       ,	unittest.TestCase     ):


	"""simple docstring"""


	snake_case									=		(
	    (BeitModel, BeitForImageClassification, BeitForMaskedImageModeling, BeitForSemanticSegmentation)
	    if is_torch_available()
	    else ()
	)
	snake_case									=		(
	    {
	        """feature-extraction""": BeitModel,
	        """image-classification""": BeitForImageClassification,
	        """image-segmentation""": BeitForSemanticSegmentation,
	    }
	    if is_torch_available()
	    else {}
	)

	snake_case									=		False
	snake_case									=		False
	snake_case									=		False







	def  _lowercase    (						self				):
						snake_case_	=      BeitModelTester(self				)
						snake_case_	=      ConfigTester(self						, config_class=_lowercase						, has_text_modality=_lowercase						, hidden_size=37				)







	def  _lowercase    (						self				):
						self.config_tester.run_common_tests()







	@unittest.skip(reason="BEiT does not use inputs_embeds"				)
	def  _lowercase    (						self				):
						pass







	@require_torch_multi_gpu
	@unittest.skip(reason="BEiT has some layers using `add_module` which doesn\'t work well with `nn.DataParallel`"				)
	def  _lowercase    (						self				):
						pass







	def  _lowercase    (						self				):
						snake_case_	,					snake_case_	=      self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
											snake_case_	=      model_class(_lowercase				)
											self.assertIsInstance(model.get_input_embeddings()						, (nn.Module)				)
											snake_case_	=      model.get_output_embeddings()
											self.assertTrue(x is None or isinstance(_lowercase						, nn.Linear				)				)







	def  _lowercase    (						self				):
						snake_case_	,					snake_case_	=      self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
											snake_case_	=      model_class(_lowercase				)
											snake_case_	=      inspect.signature(model.forward				)
											# signature.parameters is an OrderedDict => so arg_names order is deterministic
											snake_case_	=      [*signature.parameters.keys()]

											snake_case_	=      ["pixel_values"]
											self.assertListEqual(arg_names[:1]						, _lowercase				)







	def  _lowercase    (						self				):
						snake_case_	=      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*_lowercase				)







	def  _lowercase    (						self				):
						snake_case_	=      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_masked_lm(*_lowercase				)







	def  _lowercase    (						self				):
						snake_case_	=      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_image_classification(*_lowercase				)







	def  _lowercase    (						self				):
						snake_case_	=      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_semantic_segmentation(*_lowercase				)







	def  _lowercase    (						self				):
						if not self.model_tester.is_training:
											return

						snake_case_	,					snake_case_	=      self.model_tester.prepare_config_and_inputs_for_common()
						snake_case_	=      True

						for model_class in self.all_model_classes:
											# we don't test BeitForMaskedImageModeling
											if model_class in [*get_values(_lowercase				), BeitForMaskedImageModeling]:
																continue

											snake_case_	=      model_class(_lowercase				)
											model.to(_lowercase				)
											model.train()
											snake_case_	=      self._prepare_for_class(_lowercase						, _lowercase						, return_labels=_lowercase				)
											snake_case_	=      model(**_lowercase				).loss
											loss.backward()







	def  _lowercase    (						self				):
						snake_case_	,					snake_case_	=      self.model_tester.prepare_config_and_inputs_for_common()
						if not self.model_tester.is_training:
											return

						snake_case_	=      False
						snake_case_	=      True

						for model_class in self.all_model_classes:
											# we don't test BeitForMaskedImageModeling
											if (
											    model_class in [*get_values(_lowercase				), BeitForMaskedImageModeling]
											    or not model_class.supports_gradient_checkpointing
											):
																continue

											snake_case_	=      model_class(_lowercase				)
											model.gradient_checkpointing_enable()
											model.to(_lowercase				)
											model.train()
											snake_case_	=      self._prepare_for_class(_lowercase						, _lowercase						, return_labels=_lowercase				)
											snake_case_	=      model(**_lowercase				).loss
											loss.backward()







	def  _lowercase    (						self				):
						snake_case_	,					snake_case_	=      self.model_tester.prepare_config_and_inputs_for_common()

						snake_case_	=      _config_zero_init(_lowercase				)
						for model_class in self.all_model_classes:
											snake_case_	=      model_class(config=_lowercase				)
											for name, param in model.named_parameters():
																# we skip lambda parameters as these require special initial values
																# determined by config.layer_scale_init_value
																if "lambda" in name:
																					continue
																if param.requires_grad:
																					self.assertIn(
																					    ((param.data.mean() * 1e9).round() / 1e9).item()						, [0.0, 1.0]						, msg=f'''Parameter {name} of model {model_class} seems not properly initialized'''						, )







	@slow
	def  _lowercase    (						self				):
						for model_name in BEIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											snake_case_	=      BeitModel.from_pretrained(_lowercase				)
											self.assertIsNotNone(_lowercase				)



def      __snake_case       (   ):
					snake_case_	=      Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"  )
					return image






@require_torch
@require_vision
class      UpperCAmelCase_     (  unittest.TestCase     ):


	"""simple docstring"""


	@cached_property
	def  _lowercase    (						self				):
						return BeitImageProcessor.from_pretrained("microsoft/beit-base-patch16-224"				) if is_vision_available() else None







	@slow
	def  _lowercase    (						self				):
						snake_case_	=      BeitForMaskedImageModeling.from_pretrained("microsoft/beit-base-patch16-224-pt22k"				).to(_lowercase				)

						snake_case_	=      self.default_image_processor
						snake_case_	=      prepare_img()
						snake_case_	=      image_processor(images=_lowercase						, return_tensors="pt"				).pixel_values.to(_lowercase				)

						# prepare bool_masked_pos
						snake_case_	=      torch.ones((1, 1_96)						, dtype=torch.bool				).to(_lowercase				)

						# forward pass
						with torch.no_grad():
											snake_case_	=      model(pixel_values=_lowercase						, bool_masked_pos=_lowercase				)
						snake_case_	=      outputs.logits

						# verify the logits
						snake_case_	=      torch.Size((1, 1_96, 81_92)				)
						self.assertEqual(logits.shape						, _lowercase				)

						snake_case_	=      torch.tensor(
						    [[-3.2_437, 0.5_072, -13.9_174], [-3.2_456, 0.4_948, -13.9_401], [-3.2_033, 0.5_121, -13.8_550]]				).to(_lowercase				)

						self.assertTrue(torch.allclose(logits[bool_masked_pos][:3, :3]						, _lowercase						, atol=1e-2				)				)







	@slow
	def  _lowercase    (						self				):
						snake_case_	=      BeitForImageClassification.from_pretrained("microsoft/beit-base-patch16-224"				).to(_lowercase				)

						snake_case_	=      self.default_image_processor
						snake_case_	=      prepare_img()
						snake_case_	=      image_processor(images=_lowercase						, return_tensors="pt"				).to(_lowercase				)

						# forward pass
						with torch.no_grad():
											snake_case_	=      model(**_lowercase				)
						snake_case_	=      outputs.logits

						# verify the logits
						snake_case_	=      torch.Size((1, 10_00)				)
						self.assertEqual(logits.shape						, _lowercase				)

						snake_case_	=      torch.tensor([-1.2_385, -1.0_987, -1.0_108]				).to(_lowercase				)

						self.assertTrue(torch.allclose(logits[0, :3]						, _lowercase						, atol=1e-4				)				)

						snake_case_	=      2_81
						self.assertEqual(logits.argmax(-1				).item()						, _lowercase				)







	@slow
	def  _lowercase    (						self				):
						snake_case_	=      BeitForImageClassification.from_pretrained("microsoft/beit-large-patch16-224-pt22k-ft22k"				).to(
						    _lowercase				)

						snake_case_	=      self.default_image_processor
						snake_case_	=      prepare_img()
						snake_case_	=      image_processor(images=_lowercase						, return_tensors="pt"				).to(_lowercase				)

						# forward pass
						with torch.no_grad():
											snake_case_	=      model(**_lowercase				)
						snake_case_	=      outputs.logits

						# verify the logits
						snake_case_	=      torch.Size((1, 2_18_41)				)
						self.assertEqual(logits.shape						, _lowercase				)

						snake_case_	=      torch.tensor([1.6_881, -0.2_787, 0.5_901]				).to(_lowercase				)

						self.assertTrue(torch.allclose(logits[0, :3]						, _lowercase						, atol=1e-4				)				)

						snake_case_	=      23_96
						self.assertEqual(logits.argmax(-1				).item()						, _lowercase				)







	@slow
	def  _lowercase    (						self				):
						snake_case_	=      BeitForSemanticSegmentation.from_pretrained("microsoft/beit-base-finetuned-ade-640-640"				)
						snake_case_	=      model.to(_lowercase				)

						snake_case_	=      BeitImageProcessor(do_resize=_lowercase						, size=6_40						, do_center_crop=_lowercase				)

						snake_case_	=      load_dataset("hf-internal-testing/fixtures_ade20k"						, split="test"				)
						snake_case_	=      Image.open(ds[0]["file"]				)
						snake_case_	=      image_processor(images=_lowercase						, return_tensors="pt"				).to(_lowercase				)

						# forward pass
						with torch.no_grad():
											snake_case_	=      model(**_lowercase				)
						snake_case_	=      outputs.logits

						# verify the logits
						snake_case_	=      torch.Size((1, 1_50, 1_60, 1_60)				)
						self.assertEqual(logits.shape						, _lowercase				)

						snake_case_	=      version.parse(PIL.__version__				) < version.parse("9.0.0"				)

						if is_pillow_less_than_a:
											snake_case_	=      torch.tensor(
											    [
											        [[-4.9_225, -2.3_954, -3.0_522], [-2.8_822, -1.0_046, -1.7_561], [-2.9_549, -1.3_228, -2.1_347]],
											        [[-5.8_168, -3.4_129, -4.0_778], [-3.8_651, -2.2_214, -3.0_277], [-3.8_356, -2.4_643, -3.3_535]],
											        [[-0.0_078, 3.9_952, 4.0_754], [2.9_856, 4.6_944, 5.0_035], [3.2_413, 4.7_813, 4.9_969]],
											    ]						, device=_lowercase						, )
						else:
											snake_case_	=      torch.tensor(
											    [
											        [[-4.8_960, -2.3_688, -3.0_355], [-2.8_478, -0.9_836, -1.7_418], [-2.9_449, -1.3_332, -2.1_456]],
											        [[-5.8_081, -3.4_124, -4.1_006], [-3.8_561, -2.2_081, -3.0_323], [-3.8_365, -2.4_601, -3.3_669]],
											        [[-0.0_309, 3.9_868, 4.0_540], [2.9_640, 4.6_877, 4.9_976], [3.2_081, 4.7_690, 4.9_942]],
											    ]						, device=_lowercase						, )

						self.assertTrue(torch.allclose(logits[0, :3, :3, :3]						, _lowercase						, atol=1e-4				)				)







	@slow
	def  _lowercase    (						self				):
						snake_case_	=      BeitForSemanticSegmentation.from_pretrained("microsoft/beit-base-finetuned-ade-640-640"				)
						snake_case_	=      model.to(_lowercase				)

						snake_case_	=      BeitImageProcessor(do_resize=_lowercase						, size=6_40						, do_center_crop=_lowercase				)

						snake_case_	=      load_dataset("hf-internal-testing/fixtures_ade20k"						, split="test"				)
						snake_case_	=      Image.open(ds[0]["file"]				)
						snake_case_	=      image_processor(images=_lowercase						, return_tensors="pt"				).to(_lowercase				)

						# forward pass
						with torch.no_grad():
											snake_case_	=      model(**_lowercase				)

						snake_case_	=      outputs.logits.detach().cpu()

						snake_case_	=      image_processor.post_process_semantic_segmentation(outputs=_lowercase						, target_sizes=[(5_00, 3_00)]				)
						snake_case_	=      torch.Size((5_00, 3_00)				)
						self.assertEqual(segmentation[0].shape						, _lowercase				)

						snake_case_	=      image_processor.post_process_semantic_segmentation(outputs=_lowercase				)
						snake_case_	=      torch.Size((1_60, 1_60)				)
						self.assertEqual(segmentation[0].shape						, _lowercase				)











'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE			=						logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE			=						{"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/config.json"}
class   __UpperCAmelCase				(     lowerCAmelCase				):



					'''simple docstring'''





					_UpperCamelCase							=		"""openai-gpt"""
					_UpperCamelCase							=		{
					    """max_position_embeddings""": """n_positions""",
					    """hidden_size""": """n_embd""",
					    """num_attention_heads""": """n_head""",
					    """num_hidden_layers""": """n_layer""",
					}



					def __init__(	self     :						Any      ,       _lowercase     :						List[Any]=40_478      ,       _lowercase     :						Tuple=512      ,       _lowercase     :						Dict=768      ,       _lowercase     :						List[str]=12      ,       _lowercase     :						Optional[int]=12      ,       _lowercase     :						Tuple="gelu"      ,       _lowercase     :						int=0.1      ,       _lowercase     :						Tuple=0.1      ,       _lowercase     :						Any=0.1      ,       _lowercase     :						List[str]=1E-5      ,       _lowercase     :						str=0.02      ,       _lowercase     :						Optional[Any]="cls_index"      ,       _lowercase     :						int=True      ,       _lowercase     :						str=None      ,       _lowercase     :						Tuple=True      ,       _lowercase     :						List[Any]=0.1      ,       **_lowercase     :						int      ,       )    ->							Optional[int]:
								A_									= vocab_size
								A_									= n_positions
								A_									= n_embd
								A_									= n_layer
								A_									= n_head
								A_									= afn
								A_									= resid_pdrop
								A_									= embd_pdrop
								A_									= attn_pdrop
								A_									= layer_norm_epsilon
								A_									= initializer_range
								A_									= summary_type
								A_									= summary_use_proj
								A_									= summary_activation
								A_									= summary_first_dropout
								A_									= summary_proj_to_labels
								super().__init__(**_lowercase)




