






import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    GPTNeoXForCausalLM,
							    GPTNeoXForQuestionAnswering,
							    GPTNeoXForSequenceClassification,
							    GPTNeoXForTokenClassification,
							    GPTNeoXModel,
							)






class 			_UpperCamelCase:

		def __init__(					self		:    Tuple    ,       SCREAMING_SNAKE_CASE__		:    Any    ,       SCREAMING_SNAKE_CASE__		:    Tuple=1_3    ,       SCREAMING_SNAKE_CASE__		:    Union[str, Any]=7    ,       SCREAMING_SNAKE_CASE__		:    List[Any]=True    ,       SCREAMING_SNAKE_CASE__		:    int=True    ,       SCREAMING_SNAKE_CASE__		:    Optional[Any]=True    ,       SCREAMING_SNAKE_CASE__		:    List[str]=True    ,       SCREAMING_SNAKE_CASE__		:    Tuple=9_9    ,       SCREAMING_SNAKE_CASE__		:    Union[str, Any]=6_4    ,       SCREAMING_SNAKE_CASE__		:    str=5    ,       SCREAMING_SNAKE_CASE__		:    Optional[Any]=4    ,       SCREAMING_SNAKE_CASE__		:    Optional[int]=3_7    ,       SCREAMING_SNAKE_CASE__		:    str="gelu"    ,       SCREAMING_SNAKE_CASE__		:    Union[str, Any]=0.1    ,       SCREAMING_SNAKE_CASE__		:    int=0.1    ,       SCREAMING_SNAKE_CASE__		:    Tuple=5_1_2    ,       SCREAMING_SNAKE_CASE__		:    List[Any]=1_6    ,       SCREAMING_SNAKE_CASE__		:    int=2    ,       SCREAMING_SNAKE_CASE__		:    Dict=0.02    ,       SCREAMING_SNAKE_CASE__		:    Optional[int]=3    ,       SCREAMING_SNAKE_CASE__		:    str=4    ,       SCREAMING_SNAKE_CASE__		:    Dict=None    ,       ):



					'''simple docstring'''



					__a						:  Tuple									=     parent
					__a						:  str									=     batch_size
					__a						:  int									=     seq_length
					__a						:  Union[str, Any]									=     is_training
					__a						:  str									=     use_input_mask
					__a						:  Dict									=     use_token_type_ids
					__a						:  int									=     use_labels
					__a						:  Tuple									=     vocab_size
					__a						:  Union[str, Any]									=     hidden_size
					__a						:  Optional[Any]									=     num_hidden_layers
					__a						:  List[str]									=     num_attention_heads
					__a						:  List[str]									=     intermediate_size
					__a						:  Dict									=     hidden_act
					__a						:  str									=     hidden_dropout_prob
					__a						:  str									=     attention_probs_dropout_prob
					__a						:  List[str]									=     max_position_embeddings
					__a						:  List[Any]									=     type_vocab_size
					__a						:  List[Any]									=     type_sequence_label_size
					__a						:  str									=     initializer_range
					__a						:  Optional[Any]									=     num_labels
					__a						:  List[str]									=     num_choices
					__a						:  Tuple									=     scope
					__a						:  int									=     vocab_size - 1

		def    __lowerCAmelCase  (					self		:    Union[str, Any]    ):



					'''simple docstring'''



					__a						:  Optional[Any]									=     ids_tensor([self.batch_size, self.seq_length]    ,       self.vocab_size    )

					__a						:  Optional[Any]									=     None
					if self.use_input_mask:
								__a						:  Tuple									=     random_attention_mask([self.batch_size, self.seq_length]    )

					__a						:  Tuple									=     None
					if self.use_labels:
								__a						:  Tuple									=     ids_tensor([self.batch_size, self.seq_length]    ,       self.num_labels    )

					__a						:  Dict									=     self.get_config()

					return config, input_ids, input_mask, token_labels

		def    __lowerCAmelCase  (					self		:    Any    ):



					'''simple docstring'''



					return GPTNeoXConfig(
					    vocab_size=self.vocab_size    ,       hidden_size=self.hidden_size    ,       num_hidden_layers=self.num_hidden_layers    ,       num_attention_heads=self.num_attention_heads    ,       intermediate_size=self.intermediate_size    ,       hidden_act=self.hidden_act    ,       hidden_dropout_prob=self.hidden_dropout_prob    ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,       max_position_embeddings=self.max_position_embeddings    ,       type_vocab_size=self.type_vocab_size    ,       is_decoder=SCREAMING_SNAKE_CASE__    ,       initializer_range=self.initializer_range    ,       pad_token_id=self.pad_token_id    ,       )

		def    __lowerCAmelCase  (					self		:    int    ):



					'''simple docstring'''



					__a						,					__a						,					__a						,					__a						:  Optional[int]									=     self.prepare_config_and_inputs()

					__a						:  str									=     True

					return config, input_ids, input_mask, token_labels

		def    __lowerCAmelCase  (					self		:    Optional[Any]    ,       SCREAMING_SNAKE_CASE__		:    int    ,       SCREAMING_SNAKE_CASE__		:    List[str]    ,       SCREAMING_SNAKE_CASE__		:    str    ):



					'''simple docstring'''



					__a						:  Tuple									=     GPTNeoXModel(config=SCREAMING_SNAKE_CASE__    )
					model.to(SCREAMING_SNAKE_CASE__    )
					model.eval()
					__a						:  Dict									=     model(SCREAMING_SNAKE_CASE__    ,       attention_mask=SCREAMING_SNAKE_CASE__    )
					__a						:  Tuple									=     model(SCREAMING_SNAKE_CASE__    )
					self.parent.assertEqual(result.last_hidden_state.shape    ,       (self.batch_size, self.seq_length, self.hidden_size)    )

		def    __lowerCAmelCase  (					self		:    List[str]    ,       SCREAMING_SNAKE_CASE__		:    Tuple    ,       SCREAMING_SNAKE_CASE__		:    str    ,       SCREAMING_SNAKE_CASE__		:    str    ):



					'''simple docstring'''



					__a						:  int									=     True
					__a						:  str									=     GPTNeoXModel(SCREAMING_SNAKE_CASE__    )
					model.to(SCREAMING_SNAKE_CASE__    )
					model.eval()
					__a						:  Optional[Any]									=     model(SCREAMING_SNAKE_CASE__    ,       attention_mask=SCREAMING_SNAKE_CASE__    )
					self.parent.assertEqual(result.last_hidden_state.shape    ,       (self.batch_size, self.seq_length, self.hidden_size)    )

		def    __lowerCAmelCase  (					self		:    int    ,       SCREAMING_SNAKE_CASE__		:    Any    ,       SCREAMING_SNAKE_CASE__		:    str    ,       SCREAMING_SNAKE_CASE__		:    Optional[int]    ,       SCREAMING_SNAKE_CASE__		:    List[str]    ):



					'''simple docstring'''



					__a						:  str									=     GPTNeoXForCausalLM(config=SCREAMING_SNAKE_CASE__    )
					model.to(SCREAMING_SNAKE_CASE__    )
					model.eval()
					__a						:  Union[str, Any]									=     model(SCREAMING_SNAKE_CASE__    ,       attention_mask=SCREAMING_SNAKE_CASE__    ,       labels=SCREAMING_SNAKE_CASE__    )
					self.parent.assertEqual(result.logits.shape    ,       (self.batch_size, self.seq_length, self.vocab_size)    )

		def    __lowerCAmelCase  (					self		:    str    ,       SCREAMING_SNAKE_CASE__		:    int    ,       SCREAMING_SNAKE_CASE__		:    str    ,       SCREAMING_SNAKE_CASE__		:    Optional[int]    ,       SCREAMING_SNAKE_CASE__		:    str    ):



					'''simple docstring'''



					__a						:  Any									=     self.num_labels
					__a						:  int									=     GPTNeoXForQuestionAnswering(SCREAMING_SNAKE_CASE__    )
					model.to(SCREAMING_SNAKE_CASE__    )
					model.eval()
					__a						:  List[str]									=     model(SCREAMING_SNAKE_CASE__    ,       attention_mask=SCREAMING_SNAKE_CASE__    )
					self.parent.assertEqual(result.start_logits.shape    ,       (self.batch_size, self.seq_length)    )
					self.parent.assertEqual(result.end_logits.shape    ,       (self.batch_size, self.seq_length)    )

		def    __lowerCAmelCase  (					self		:    Union[str, Any]    ,       SCREAMING_SNAKE_CASE__		:    List[str]    ,       SCREAMING_SNAKE_CASE__		:    Optional[Any]    ,       SCREAMING_SNAKE_CASE__		:    int    ,       SCREAMING_SNAKE_CASE__		:    Dict    ):



					'''simple docstring'''



					__a						:  Dict									=     self.num_labels
					__a						:  List[str]									=     GPTNeoXForSequenceClassification(SCREAMING_SNAKE_CASE__    )
					model.to(SCREAMING_SNAKE_CASE__    )
					model.eval()
					__a						:  str									=     ids_tensor([self.batch_size]    ,       self.type_sequence_label_size    )
					__a						:  List[str]									=     model(SCREAMING_SNAKE_CASE__    ,       attention_mask=SCREAMING_SNAKE_CASE__    ,       labels=SCREAMING_SNAKE_CASE__    )
					self.parent.assertEqual(result.logits.shape    ,       (self.batch_size, self.num_labels)    )

		def    __lowerCAmelCase  (					self		:    List[Any]    ,       SCREAMING_SNAKE_CASE__		:    int    ,       SCREAMING_SNAKE_CASE__		:    Dict    ,       SCREAMING_SNAKE_CASE__		:    Any    ,       SCREAMING_SNAKE_CASE__		:    Dict    ):



					'''simple docstring'''



					__a						:  Dict									=     self.num_labels
					__a						:  Dict									=     GPTNeoXForTokenClassification(SCREAMING_SNAKE_CASE__    )
					model.to(SCREAMING_SNAKE_CASE__    )
					model.eval()
					__a						:  str									=     model(SCREAMING_SNAKE_CASE__    ,       attention_mask=SCREAMING_SNAKE_CASE__    ,       labels=SCREAMING_SNAKE_CASE__    )
					self.parent.assertEqual(result.logits.shape    ,       (self.batch_size, self.seq_length, self.num_labels)    )

		def    __lowerCAmelCase  (					self		:    Tuple    ,       SCREAMING_SNAKE_CASE__		:    Optional[int]    ,       SCREAMING_SNAKE_CASE__		:    Optional[int]    ,       SCREAMING_SNAKE_CASE__		:    str    ):



					'''simple docstring'''



					__a						:  Optional[Any]									=     True
					__a						:  str									=     GPTNeoXForCausalLM(config=SCREAMING_SNAKE_CASE__    )
					model.to(SCREAMING_SNAKE_CASE__    )
					model.eval()

					# first forward pass
					__a						:  Optional[Any]									=     model(SCREAMING_SNAKE_CASE__    ,       attention_mask=SCREAMING_SNAKE_CASE__    ,       use_cache=SCREAMING_SNAKE_CASE__    )
					__a						:  Tuple									=     outputs.past_key_values

					# create hypothetical multiple next token and extent to next_input_ids
					__a						:  Optional[Any]									=     ids_tensor((self.batch_size, 3)    ,       config.vocab_size    )
					__a						:  List[Any]									=     ids_tensor((self.batch_size, 3)    ,       vocab_size=2    )

					# append to next input_ids and
					__a						:  str									=     torch.cat([input_ids, next_tokens]    ,       dim=-1    )
					__a						:  Dict									=     torch.cat([input_mask, next_mask]    ,       dim=-1    )

					__a						:  int									=     model(SCREAMING_SNAKE_CASE__    ,       attention_mask=SCREAMING_SNAKE_CASE__    ,       output_hidden_states=SCREAMING_SNAKE_CASE__    )
					__a						:  Dict									=     output_from_no_past['hidden_states'][0]
					__a						:  List[str]									=     model(
					    SCREAMING_SNAKE_CASE__    ,       attention_mask=SCREAMING_SNAKE_CASE__    ,       past_key_values=SCREAMING_SNAKE_CASE__    ,       output_hidden_states=SCREAMING_SNAKE_CASE__    ,       )['hidden_states'][0]

					# select random slice
					__a						:  Any									=     ids_tensor((1,)    ,       output_from_past.shape[-1]    ).item()
					__a						:  Union[str, Any]									=     output_from_no_past[:, -3:, random_slice_idx].detach()
					__a						:  List[str]									=     output_from_past[:, :, random_slice_idx].detach()

					self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]    )

					# test that outputs are equal for slice
					self.parent.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ,       atol=1e-3    )    )



		def    __lowerCAmelCase  (					self		:    str    ):



					'''simple docstring'''



					__a						:  Union[str, Any]									=     self.prepare_config_and_inputs()
					__a						,					__a						,					__a						,					__a						:  str									=     config_and_inputs
					__a						:  Tuple									=     {'input_ids': input_ids, 'attention_mask': input_mask}
					return config, inputs_dict






@require_torch
class 			_UpperCamelCase(       __lowerCamelCase		,				__lowerCamelCase		,				__lowerCamelCase		,				unittest.TestCase		):
		__SCREAMING_SNAKE_CASE  :       Optional[int]							=							(
		    (
		        GPTNeoXModel,
		        GPTNeoXForCausalLM,
		        GPTNeoXForQuestionAnswering,
		        GPTNeoXForSequenceClassification,
		        GPTNeoXForTokenClassification,
		    )
		    if is_torch_available()
		    else ()
		)
		__SCREAMING_SNAKE_CASE  :       str							=							(GPTNeoXForCausalLM,) if is_torch_available() else ()
		__SCREAMING_SNAKE_CASE  :       Optional[int]							=							(
		    {
		        '''feature-extraction''': GPTNeoXModel,
		        '''question-answering''': GPTNeoXForQuestionAnswering,
		        '''text-classification''': GPTNeoXForSequenceClassification,
		        '''text-generation''': GPTNeoXForCausalLM,
		        '''token-classification''': GPTNeoXForTokenClassification,
		        '''zero-shot''': GPTNeoXForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		__SCREAMING_SNAKE_CASE  :       Union[str, Any]							=							False
		__SCREAMING_SNAKE_CASE  :       Optional[int]							=							False
		__SCREAMING_SNAKE_CASE  :       str							=							False
		__SCREAMING_SNAKE_CASE  :       Any							=							False

		def    __lowerCAmelCase  (					self		:    Union[str, Any]    ):



					'''simple docstring'''



					__a						:  str									=     GPTNeoXModelTester(self    )
					__a						:  Union[str, Any]									=     ConfigTester(self    ,       config_class=SCREAMING_SNAKE_CASE__    ,       hidden_size=6_4    ,       num_attention_heads=8    )

		def    __lowerCAmelCase  (					self		:    Dict    ):



					'''simple docstring'''



					self.config_tester.run_common_tests()

		def    __lowerCAmelCase  (					self		:    Any    ):



					'''simple docstring'''



					__a						,					__a						,					__a						,					__a						:  str									=     self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_model(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    )

		def    __lowerCAmelCase  (					self		:    Dict    ):



					'''simple docstring'''



					__a						,					__a						,					__a						,					__a						:  Dict									=     self.model_tester.prepare_config_and_inputs_for_decoder()
					self.model_tester.create_and_check_model_as_decoder(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    )

		def    __lowerCAmelCase  (					self		:    Dict    ):



					'''simple docstring'''



					__a						,					__a						,					__a						,					__a						:  Union[str, Any]									=     self.model_tester.prepare_config_and_inputs_for_decoder()

					__a						:  int									=     None

					self.model_tester.create_and_check_model_as_decoder(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    )

		def    __lowerCAmelCase  (					self		:    Optional[int]    ):



					'''simple docstring'''



					__a						,					__a						,					__a						,					__a						:  Union[str, Any]									=     self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_decoder_model_past_large_inputs(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    )

		def    __lowerCAmelCase  (					self		:    Optional[Any]    ):



					'''simple docstring'''



					__a						:  List[str]									=     self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_causal_lm(*SCREAMING_SNAKE_CASE__    )

		def    __lowerCAmelCase  (					self		:    Any    ):



					'''simple docstring'''



					__a						:  Any									=     self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE__    )

		def    __lowerCAmelCase  (					self		:    List[str]    ):



					'''simple docstring'''



					__a						:  Tuple									=     self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE__    )

		def    __lowerCAmelCase  (					self		:    int    ):



					'''simple docstring'''



					__a						:  Any									=     self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE__    )

		@unittest.skip(reason='Feed forward chunking is not implemented'    )
		def    __lowerCAmelCase  (					self		:    Tuple    ):



					'''simple docstring'''



					pass



		@parameterized.expand([('linear',), ('dynamic',)]    )
		def    __lowerCAmelCase  (					self		:    str    ,       SCREAMING_SNAKE_CASE__		:    Any    ):



					'''simple docstring'''



					__a						,					__a						:  Tuple									=     self.model_tester.prepare_config_and_inputs_for_common()
					__a						:  int									=     ids_tensor([1, 1_0]    ,       config.vocab_size    )
					__a						:  Optional[Any]									=     ids_tensor([1, int(config.max_position_embeddings * 1.5    )]    ,       config.vocab_size    )

					set_seed(4_2    )  # Fixed seed at init time so the two models get the same random weights
					__a						:  List[str]									=     GPTNeoXModel(SCREAMING_SNAKE_CASE__    )
					original_model.to(SCREAMING_SNAKE_CASE__    )
					original_model.eval()
					__a						:  Union[str, Any]									=     original_model(SCREAMING_SNAKE_CASE__    ).last_hidden_state
					__a						:  int									=     original_model(SCREAMING_SNAKE_CASE__    ).last_hidden_state

					set_seed(4_2    )  # Fixed seed at init time so the two models get the same random weights
					__a						:  str									=     {'type': scaling_type, 'factor': 10.0}
					__a						:  Any									=     GPTNeoXModel(SCREAMING_SNAKE_CASE__    )
					scaled_model.to(SCREAMING_SNAKE_CASE__    )
					scaled_model.eval()
					__a						:  Optional[int]									=     scaled_model(SCREAMING_SNAKE_CASE__    ).last_hidden_state
					__a						:  Union[str, Any]									=     scaled_model(SCREAMING_SNAKE_CASE__    ).last_hidden_state

					# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
					# maximum sequence length, so the outputs for the short input should match.
					if scaling_type == "dynamic":
								self.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ,       atol=1e-5    )    )
					else:
								self.assertFalse(torch.allclose(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ,       atol=1e-5    )    )

					# The output should be different for long inputs
					self.assertFalse(torch.allclose(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ,       atol=1e-5    )    )






@require_torch
class 			_UpperCamelCase(       unittest.TestCase		):

		@slow
		def    __lowerCAmelCase  (					self		:    Optional[Any]    ):



					'''simple docstring'''



					__a						:  str									=     AutoTokenizer.from_pretrained('EleutherAI/pythia-410m-deduped'    )
					for checkpointing in [True, False]:
								__a						:  str									=     GPTNeoXForCausalLM.from_pretrained('EleutherAI/pythia-410m-deduped'    )

								if checkpointing:
											model.gradient_checkpointing_enable()
								else:
											model.gradient_checkpointing_disable()
								model.to(SCREAMING_SNAKE_CASE__    )

								__a						:  Dict									=     tokenizer('My favorite food is'    ,       return_tensors='pt'    ).to(SCREAMING_SNAKE_CASE__    )
								# The hub repo. is updated on 2023-04-04, resulting in poor outputs.
								# See: https://github.com/huggingface/transformers/pull/24193
								__a						:  Optional[int]									=     'My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI\'m not sure'

								__a						:  Tuple									=     model.generate(**SCREAMING_SNAKE_CASE__    ,       do_sample=SCREAMING_SNAKE_CASE__    ,       max_new_tokens=2_0    )
								__a						:  int									=     tokenizer.batch_decode(SCREAMING_SNAKE_CASE__    )[0]

								self.assertEqual(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    )












'''simple docstring'''


from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE									=				logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE									=				{
    "microsoft/swinv2-tiny-patch4-window8-256": (
        "https://huggingface.co/microsoft/swinv2-tiny-patch4-window8-256/resolve/main/config.json"
    ),
}

class  lowerCAmelCase_		(		__magic_name__		):
 __lowerCamelCase    :				Union[str, Any]								=       "swinv2"

 __lowerCamelCase    :				int								=       {
     "num_attention_heads": "num_heads",
     "num_hidden_layers": "num_layers",
 }




 def __init__(						self      ,  _lowerCAmelCase=224      ,  _lowerCAmelCase=4      ,  _lowerCAmelCase=3      ,  _lowerCAmelCase=96      ,  _lowerCAmelCase=[2, 2, 6, 2]      ,  _lowerCAmelCase=[3, 6, 12, 24]      ,  _lowerCAmelCase=7      ,  _lowerCAmelCase=4.0      ,  _lowerCAmelCase=True      ,  _lowerCAmelCase=0.0      ,  _lowerCAmelCase=0.0      ,  _lowerCAmelCase=0.1      ,  _lowerCAmelCase="gelu"      ,  _lowerCAmelCase=False      ,  _lowerCAmelCase=0.02      ,  _lowerCAmelCase=1E-5      ,  _lowerCAmelCase=32      ,  **_lowerCAmelCase      ,  )     ->				Tuple:
     super().__init__(**_lowerCAmelCase		)

     _lowerCAmelCase						      =   image_size
     _lowerCAmelCase						      =   patch_size
     _lowerCAmelCase						      =   num_channels
     _lowerCAmelCase						      =   embed_dim
     _lowerCAmelCase						      =   depths
     _lowerCAmelCase						      =   len(_lowerCAmelCase		)
     _lowerCAmelCase						      =   num_heads
     _lowerCAmelCase						      =   window_size
     _lowerCAmelCase						      =   mlp_ratio
     _lowerCAmelCase						      =   qkv_bias
     _lowerCAmelCase						      =   hidden_dropout_prob
     _lowerCAmelCase						      =   attention_probs_dropout_prob
     _lowerCAmelCase						      =   drop_path_rate
     _lowerCAmelCase						      =   hidden_act
     _lowerCAmelCase						      =   use_absolute_embeddings
     _lowerCAmelCase						      =   layer_norm_eps
     _lowerCAmelCase						      =   initializer_range
     _lowerCAmelCase						      =   encoder_stride
     # we set the hidden_size attribute in order to make Swinv2 work with VisionEncoderDecoderModel
     # this indicates the channel dimension after the last stage of the model
     _lowerCAmelCase						      =   int(embed_dim * 2 ** (len(_lowerCAmelCase		) - 1)		)
     _lowerCAmelCase						      =   (0, 0, 0, 0)


