






"""simple docstring"""




import collections
import inspect
import unittest
from typing import Dict, List, Tuple

from transformers import MaskFormerSwinConfig
from transformers.testing_utils import require_torch, require_torch_multi_gpu, torch_device
from transformers.utils import is_torch_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch
							from torch import nn

							from transformers import MaskFormerSwinBackbone
							from transformers.models.maskformer import MaskFormerSwinModel


class 	__UpperCAmelCase		:


	'''simple docstring'''





	def __init__(     self      ,      _A      ,      _A=1_3      ,      _A=3_2      ,      _A=2      ,      _A=3      ,      _A=1_6      ,      _A=[1, 2, 1]      ,      _A=[2, 2, 4]      ,      _A=2      ,      _A=2.0      ,      _A=True      ,      _A=0.0      ,      _A=0.0      ,      _A=0.1      ,      _A="gelu"      ,      _A=False      ,      _A=True      ,      _A=0.02      ,      _A=1E-5      ,      _A=True      ,      _A=None      ,      _A=True      ,      _A=1_0      ,      _A=8      ,      _A=["stage1", "stage2", "stage3"]      ,      _A=[1, 2, 3]      ,      ):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=parent
		_SCREAMING_SNAKE_CASE						=batch_size
		_SCREAMING_SNAKE_CASE						=image_size
		_SCREAMING_SNAKE_CASE						=patch_size
		_SCREAMING_SNAKE_CASE						=num_channels
		_SCREAMING_SNAKE_CASE						=embed_dim
		_SCREAMING_SNAKE_CASE						=depths
		_SCREAMING_SNAKE_CASE						=num_heads
		_SCREAMING_SNAKE_CASE						=window_size
		_SCREAMING_SNAKE_CASE						=mlp_ratio
		_SCREAMING_SNAKE_CASE						=qkv_bias
		_SCREAMING_SNAKE_CASE						=hidden_dropout_prob
		_SCREAMING_SNAKE_CASE						=attention_probs_dropout_prob
		_SCREAMING_SNAKE_CASE						=drop_path_rate
		_SCREAMING_SNAKE_CASE						=hidden_act
		_SCREAMING_SNAKE_CASE						=use_absolute_embeddings
		_SCREAMING_SNAKE_CASE						=patch_norm
		_SCREAMING_SNAKE_CASE						=layer_norm_eps
		_SCREAMING_SNAKE_CASE						=initializer_range
		_SCREAMING_SNAKE_CASE						=is_training
		_SCREAMING_SNAKE_CASE						=scope
		_SCREAMING_SNAKE_CASE						=use_labels
		_SCREAMING_SNAKE_CASE						=type_sequence_label_size
		_SCREAMING_SNAKE_CASE						=encoder_stride
		_SCREAMING_SNAKE_CASE						=out_features
		_SCREAMING_SNAKE_CASE						=out_indices


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]					)

		_SCREAMING_SNAKE_CASE						=None
		if self.use_labels:
			_SCREAMING_SNAKE_CASE						=ids_tensor([self.batch_size]      ,      self.type_sequence_label_size					)

		_SCREAMING_SNAKE_CASE						=self.get_config()

		return config, pixel_values, labels


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		return MaskFormerSwinConfig(
		    image_size=self.image_size      ,      patch_size=self.patch_size      ,      num_channels=self.num_channels      ,      embed_dim=self.embed_dim      ,      depths=self.depths      ,      num_heads=self.num_heads      ,      window_size=self.window_size      ,      mlp_ratio=self.mlp_ratio      ,      qkv_bias=self.qkv_bias      ,      hidden_dropout_prob=self.hidden_dropout_prob      ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,      drop_path_rate=self.drop_path_rate      ,      hidden_act=self.hidden_act      ,      use_absolute_embeddings=self.use_absolute_embeddings      ,      path_norm=self.patch_norm      ,      layer_norm_eps=self.layer_norm_eps      ,      initializer_range=self.initializer_range      ,      encoder_stride=self.encoder_stride      ,      out_features=self.out_features      ,      out_indices=self.out_indices      ,      )


	def 				UpperCamelCase_  (     self      ,      _A      ,      _A      ,      _A					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=MaskFormerSwinModel(config=__lowerCAmelCase					)
		model.to(__lowerCAmelCase					)
		model.eval()
		_SCREAMING_SNAKE_CASE						=model(__lowerCAmelCase					)

		_SCREAMING_SNAKE_CASE						=((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths					) - 1))
		_SCREAMING_SNAKE_CASE						=int(config.embed_dim * 2 ** (len(config.depths					) - 1)					)

		self.parent.assertEqual(result.last_hidden_state.shape      ,      (self.batch_size, expected_seq_len, expected_dim)					)


	def 				UpperCamelCase_  (     self      ,      _A      ,      _A      ,      _A					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=MaskFormerSwinBackbone(config=__lowerCAmelCase					)
		model.to(__lowerCAmelCase					)
		model.eval()
		_SCREAMING_SNAKE_CASE						=model(__lowerCAmelCase					)

		# verify feature maps
		self.parent.assertEqual(len(result.feature_maps					)      ,      len(config.out_features					)					)
		self.parent.assertListEqual(list(result.feature_maps[0].shape					)      ,      [1_3, 1_6, 1_6, 1_6]					)

		# verify channels
		self.parent.assertEqual(len(model.channels					)      ,      len(config.out_features					)					)
		self.parent.assertListEqual(model.channels      ,      [1_6, 3_2, 6_4]					)

		# verify ValueError
		with self.parent.assertRaises(__lowerCAmelCase					):
			_SCREAMING_SNAKE_CASE						=['''stem''']
			_SCREAMING_SNAKE_CASE						=MaskFormerSwinBackbone(config=__lowerCAmelCase					)







	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=self.prepare_config_and_inputs()
		_SCREAMING_SNAKE_CASE		,     _SCREAMING_SNAKE_CASE		,     _SCREAMING_SNAKE_CASE						=config_and_inputs
		_SCREAMING_SNAKE_CASE						={'''pixel_values''': pixel_values}
		return config, inputs_dict




@require_torch
class 	__UpperCAmelCase		(				_A,  _A,  unittest.TestCase	):


	'''simple docstring'''





	lowercase		:	List[Any]							     =				(
	    (
	        MaskFormerSwinModel,
	        MaskFormerSwinBackbone,
	    )
	    if is_torch_available()
	    else ()
	)
	lowercase		:	Dict							     =				{"feature-extraction": MaskFormerSwinModel} if is_torch_available() else {}
	lowercase		:	Union[str, Any]							     =				False
	lowercase		:	str							     =				False
	lowercase		:	str							     =				False
	lowercase		:	List[Any]							     =				False
	lowercase		:	Optional[Any]							     =				False


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=MaskFormerSwinModelTester(self					)
		_SCREAMING_SNAKE_CASE						=ConfigTester(self      ,      config_class=__lowerCAmelCase      ,      embed_dim=3_7					)


	@require_torch_multi_gpu
	@unittest.skip(
	    reason=(
	        '''`MaskFormerSwinModel` outputs `hidden_states_spatial_dimensions` which doesn\'t work well with'''
	        ''' `nn.DataParallel`'''
	    )					)
	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		pass


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		self.create_and_test_config_common_properties()
		self.config_tester.create_and_test_config_to_json_string()
		self.config_tester.create_and_test_config_to_json_file()
		self.config_tester.create_and_test_config_from_and_save_pretrained()
		self.config_tester.create_and_test_config_with_num_labels()
		self.config_tester.check_config_can_be_init_without_params()
		self.config_tester.check_config_arguments_init()


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		return


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_model(*__lowerCAmelCase					)


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_backbone(*__lowerCAmelCase					)


	@unittest.skip('''Swin does not use inputs_embeds'''					)
	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		pass


	@unittest.skip('''Swin does not support feedforward chunking'''					)
	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		pass


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE		,     _SCREAMING_SNAKE_CASE						=self.model_tester.prepare_config_and_inputs_for_common()

		for model_class in self.all_model_classes:
			_SCREAMING_SNAKE_CASE						=model_class(__lowerCAmelCase					)
			self.assertIsInstance(model.get_input_embeddings()      ,      (nn.Module)					)
			_SCREAMING_SNAKE_CASE						=model.get_output_embeddings()
			self.assertTrue(x is None or isinstance(__lowerCAmelCase      ,      nn.Linear					)					)


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE		,     _SCREAMING_SNAKE_CASE						=self.model_tester.prepare_config_and_inputs_for_common()

		for model_class in self.all_model_classes:
			_SCREAMING_SNAKE_CASE						=model_class(__lowerCAmelCase					)
			_SCREAMING_SNAKE_CASE						=inspect.signature(model.forward					)
			# signature.parameters is an OrderedDict => so arg_names order is deterministic
			_SCREAMING_SNAKE_CASE						=[*signature.parameters.keys()]

			_SCREAMING_SNAKE_CASE						=['''pixel_values''']
			self.assertListEqual(arg_names[:1]      ,      __lowerCAmelCase					)


	@unittest.skip(reason='''MaskFormerSwin is only used as backbone and doesn\'t support output_attentions'''					)
	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		pass


	@unittest.skip(reason='''MaskFormerSwin is only used as an internal backbone'''					)
	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		pass


	def 				UpperCamelCase_  (     self      ,      _A      ,      _A      ,      _A      ,      _A					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=model_class(__lowerCAmelCase					)
		model.to(__lowerCAmelCase					)
		model.eval()

		with torch.no_grad():
			_SCREAMING_SNAKE_CASE						=model(**self._prepare_for_class(__lowerCAmelCase      ,      __lowerCAmelCase					)					)

		_SCREAMING_SNAKE_CASE						=outputs.hidden_states

		_SCREAMING_SNAKE_CASE						=getattr(
		    self.model_tester      ,      '''expected_num_hidden_layers'''      ,      len(self.model_tester.depths					) + 1					)
		self.assertEqual(len(__lowerCAmelCase					)      ,      __lowerCAmelCase					)

		# Swin has a different seq_length
		_SCREAMING_SNAKE_CASE						=(
		    config.patch_size
		    if isinstance(config.patch_size      ,      collections.abc.Iterable					)
		    else (config.patch_size, config.patch_size)
		)

		_SCREAMING_SNAKE_CASE						=(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

		self.assertListEqual(
		    list(hidden_states[0].shape[-2:]					)      ,      [num_patches, self.model_tester.embed_dim]      ,      )


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE		,     _SCREAMING_SNAKE_CASE						=self.model_tester.prepare_config_and_inputs_for_common()

		_SCREAMING_SNAKE_CASE						=(
		    self.model_tester.image_size
		    if isinstance(self.model_tester.image_size      ,      collections.abc.Iterable					)
		    else (self.model_tester.image_size, self.model_tester.image_size)
		)

		for model_class in self.all_model_classes:
			_SCREAMING_SNAKE_CASE						=True
			self.check_hidden_states_output(__lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase					)

			# check that output_hidden_states also work using config
			del inputs_dict["output_hidden_states"]
			_SCREAMING_SNAKE_CASE						=True

			self.check_hidden_states_output(__lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase					)


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE		,     _SCREAMING_SNAKE_CASE						=self.model_tester.prepare_config_and_inputs_for_common()
		_SCREAMING_SNAKE_CASE						=3

		_SCREAMING_SNAKE_CASE						=(
		    self.model_tester.image_size
		    if isinstance(self.model_tester.image_size      ,      collections.abc.Iterable					)
		    else (self.model_tester.image_size, self.model_tester.image_size)
		)
		_SCREAMING_SNAKE_CASE						=(
		    config.patch_size
		    if isinstance(config.patch_size      ,      collections.abc.Iterable					)
		    else (config.patch_size, config.patch_size)
		)

		_SCREAMING_SNAKE_CASE						=image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
		_SCREAMING_SNAKE_CASE						=image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

		for model_class in self.all_model_classes:
			_SCREAMING_SNAKE_CASE						=True
			self.check_hidden_states_output(__lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase      ,      (padded_height, padded_width)					)

			# check that output_hidden_states also work using config
			del inputs_dict["output_hidden_states"]
			_SCREAMING_SNAKE_CASE						=True
			self.check_hidden_states_output(__lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase      ,      (padded_height, padded_width)					)


	@unittest.skip(reason='''MaskFormerSwin doesn\'t have pretrained checkpoints'''					)
	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		pass


	@unittest.skip(reason='''This will be fixed once MaskFormerSwin is replaced by native Swin'''					)
	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		pass


	@unittest.skip(reason='''This will be fixed once MaskFormerSwin is replaced by native Swin'''					)
	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		pass







	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE		,     _SCREAMING_SNAKE_CASE						=self.model_tester.prepare_config_and_inputs_for_common()

		def set_nan_tensor_to_zero(_A					):
			_SCREAMING_SNAKE_CASE						=0
			return t

		def check_equivalence(_A      ,      _A      ,      _A      ,      _A={}					):
			with torch.no_grad():
				_SCREAMING_SNAKE_CASE						=model(**__lowerCAmelCase      ,      return_dict=__lowerCAmelCase      ,      **__lowerCAmelCase					)
				_SCREAMING_SNAKE_CASE						=model(**__lowerCAmelCase      ,      return_dict=__lowerCAmelCase      ,      **__lowerCAmelCase					).to_tuple()

				def recursive_check(_A      ,      _A					):
					if isinstance(__lowerCAmelCase      ,      (List, Tuple)					):
						for tuple_iterable_value, dict_iterable_value in zip(__lowerCAmelCase      ,      __lowerCAmelCase					):
							recursive_check(__lowerCAmelCase      ,      __lowerCAmelCase					)
					elif isinstance(__lowerCAmelCase      ,      __lowerCAmelCase					):
						for tuple_iterable_value, dict_iterable_value in zip(
						    tuple_object.values()      ,      dict_object.values()					):
							recursive_check(__lowerCAmelCase      ,      __lowerCAmelCase					)
					elif tuple_object is None:
						return
					else:
						self.assertTrue(
						    torch.allclose(
						        set_nan_tensor_to_zero(__lowerCAmelCase					)      ,      set_nan_tensor_to_zero(__lowerCAmelCase					)      ,      atol=1E-5					)      ,      msg=(
						        '''Tuple and dict output are not equal. Difference:'''
						        f""" {torch.max(torch.abs(tuple_object - dict_object					)					)}. Tuple has `nan`:"""
						        f""" {torch.isnan(__lowerCAmelCase					).any()} and `inf`: {torch.isinf(__lowerCAmelCase					)}. Dict has"""
						        f""" `nan`: {torch.isnan(__lowerCAmelCase					).any()} and `inf`: {torch.isinf(__lowerCAmelCase					)}."""
						    )      ,      )

				recursive_check(__lowerCAmelCase      ,      __lowerCAmelCase					)

		for model_class in self.all_model_classes:
			_SCREAMING_SNAKE_CASE						=model_class(__lowerCAmelCase					)
			model.to(__lowerCAmelCase					)
			model.eval()

			_SCREAMING_SNAKE_CASE						=self._prepare_for_class(__lowerCAmelCase      ,      __lowerCAmelCase					)
			_SCREAMING_SNAKE_CASE						=self._prepare_for_class(__lowerCAmelCase      ,      __lowerCAmelCase					)
			check_equivalence(__lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase					)

			_SCREAMING_SNAKE_CASE						=self._prepare_for_class(__lowerCAmelCase      ,      __lowerCAmelCase      ,      return_labels=__lowerCAmelCase					)
			_SCREAMING_SNAKE_CASE						=self._prepare_for_class(__lowerCAmelCase      ,      __lowerCAmelCase      ,      return_labels=__lowerCAmelCase					)
			check_equivalence(__lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase					)

			_SCREAMING_SNAKE_CASE						=self._prepare_for_class(__lowerCAmelCase      ,      __lowerCAmelCase					)
			_SCREAMING_SNAKE_CASE						=self._prepare_for_class(__lowerCAmelCase      ,      __lowerCAmelCase					)
			check_equivalence(__lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase      ,      {'''output_hidden_states''': True}					)

			_SCREAMING_SNAKE_CASE						=self._prepare_for_class(__lowerCAmelCase      ,      __lowerCAmelCase      ,      return_labels=__lowerCAmelCase					)
			_SCREAMING_SNAKE_CASE						=self._prepare_for_class(__lowerCAmelCase      ,      __lowerCAmelCase      ,      return_labels=__lowerCAmelCase					)
			check_equivalence(__lowerCAmelCase      ,      __lowerCAmelCase      ,      __lowerCAmelCase      ,      {'''output_hidden_states''': True}					)




@require_torch
class 	__UpperCAmelCase		(				unittest.TestCase,  _A	):


	'''simple docstring'''





	lowercase		:	List[str]							     =				(MaskFormerSwinBackbone,) if is_torch_available() else ()
	lowercase		:	Any							     =				MaskFormerSwinConfig


	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE						=MaskFormerSwinModelTester(self					)







	def 				UpperCamelCase_  (     self					):



		'''simple docstring'''

		_SCREAMING_SNAKE_CASE		,     _SCREAMING_SNAKE_CASE						=self.model_tester.prepare_config_and_inputs_for_common()
		_SCREAMING_SNAKE_CASE						=inputs_dict['''pixel_values'''].shape[0]

		for backbone_class in self.all_model_classes:
			_SCREAMING_SNAKE_CASE						=backbone_class(__lowerCAmelCase					)
			backbone.to(__lowerCAmelCase					)
			backbone.eval()

			_SCREAMING_SNAKE_CASE						=backbone(**__lowerCAmelCase					)

			# Test default outputs and verify feature maps
			self.assertIsInstance(outputs.feature_maps      ,      __lowerCAmelCase					)
			self.assertTrue(len(outputs.feature_maps					) == len(backbone.channels					)					)
			for feature_map, n_channels in zip(outputs.feature_maps      ,      backbone.channels					):
				self.assertTrue(feature_map.shape[:2]      ,      (batch_size, n_channels)					)
			self.assertIsNone(outputs.hidden_states					)
			self.assertIsNone(outputs.attentions					)

			# Test output_hidden_states=True
			_SCREAMING_SNAKE_CASE						=backbone(**__lowerCAmelCase      ,      output_hidden_states=__lowerCAmelCase					)
			self.assertIsNotNone(outputs.hidden_states					)
			self.assertTrue(len(outputs.hidden_states					)      ,      len(backbone.stage_names					)					)
			# We skip the stem layer
			for hidden_states, n_channels in zip(outputs.hidden_states[1:]      ,      backbone.channels					):
				for hidden_state in hidden_states:
					# Hidden states are in the format (batch_size, (height * width), n_channels)
					_SCREAMING_SNAKE_CASE		,     _SCREAMING_SNAKE_CASE		,     _SCREAMING_SNAKE_CASE						=hidden_state.shape
					self.assertTrue((h_batch_size, h_n_channels)      ,      (batch_size, n_channels)					)

				  # Test output_attentions=True
			if self.has_attentions:
				_SCREAMING_SNAKE_CASE						=backbone(**__lowerCAmelCase      ,      output_attentions=__lowerCAmelCase					)
				self.assertIsNotNone(outputs.attentions					)









import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_		 =					logging.get_logger(__name__)

UpperCAmelCase_		 =					{
    """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/config.json""",
    """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/config.json""",
}


class 	lowerCamelCase__	(       _A):



 """simple docstring"""

 a__					: Any =  "xlnet"
 a__					: Dict =  ["mems"]
 a__					: List[str] =  {
     "n_token": "vocab_size",  # Backward compatibility
     "hidden_size": "d_model",
     "num_attention_heads": "n_head",
     "num_hidden_layers": "n_layer",
 }

 def __init__(    self					:					int			,						__lowerCAmelCase					:					Dict=3_20_00			,						__lowerCAmelCase					:					List[str]=10_24			,						__lowerCAmelCase					:					Dict=24			,						__lowerCAmelCase					:					Optional[Any]=16			,						__lowerCAmelCase					:					Dict=40_96			,						__lowerCAmelCase					:					Any="gelu"			,						__lowerCAmelCase					:					int=True			,						__lowerCAmelCase					:					List[str]="bi"			,						__lowerCAmelCase					:					Dict=0.02			,						__lowerCAmelCase					:					Union[str, Any]=1E-12			,						__lowerCAmelCase					:					Optional[Any]=0.1			,						__lowerCAmelCase					:					Optional[Any]=5_12			,						__lowerCAmelCase					:					List[str]=None			,						__lowerCAmelCase					:					Tuple=True			,						__lowerCAmelCase					:					Tuple=False			,						__lowerCAmelCase					:					Optional[Any]=False			,						__lowerCAmelCase					:					Union[str, Any]=-1			,						__lowerCAmelCase					:					Optional[Any]=False			,						__lowerCAmelCase					:					Any="last"			,						__lowerCAmelCase					:					List[Any]=True			,						__lowerCAmelCase					:					Tuple="tanh"			,						__lowerCAmelCase					:					int=0.1			,						__lowerCAmelCase					:					str=5			,						__lowerCAmelCase					:					str=5			,						__lowerCAmelCase					:					List[str]=5			,						__lowerCAmelCase					:					List[str]=1			,						__lowerCAmelCase					:					Optional[int]=2			,						**__lowerCAmelCase					:					List[str]			,						)   ->	Tuple:
    _A      =       vocab_size
    _A      =       d_model
    _A      =       n_layer
    _A      =       n_head
    if d_model % n_head != 0:
       raise ValueError(f'''\'d_model % n_head\' ({d_model % n_head}) should be equal to 0'''      )
    if "d_head" in kwargs:
       if kwargs["d_head"] != d_model // n_head:
          raise ValueError(
              f'''`d_head` ({kwargs['d_head']}) should be equal to `d_model // n_head` ({d_model // n_head})'''      )
    _A      =       d_model // n_head
    _A      =       ff_activation
    _A      =       d_inner
    _A      =       untie_r
    _A      =       attn_type

    _A      =       initializer_range
    _A      =       layer_norm_eps

    _A      =       dropout
    _A      =       mem_len
    _A      =       reuse_len
    _A      =       bi_data
    _A      =       clamp_len
    _A      =       same_length

    _A      =       summary_type
    _A      =       summary_use_proj
    _A      =       summary_activation
    _A      =       summary_last_dropout
    _A      =       start_n_top
    _A      =       end_n_top

    _A      =       bos_token_id
    _A      =       pad_token_id
    _A      =       eos_token_id

    if "use_cache" in kwargs:
       warnings.warn(
           '''The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`'''
           ''' instead.'''			,						__lowerCAmelCase			,						)
       _A      =       kwargs['''use_cache''']

    _A      =       use_mems_eval
    _A      =       use_mems_train
    super().__init__(pad_token_id=__lowerCAmelCase			,						bos_token_id=__lowerCAmelCase			,						eos_token_id=__lowerCAmelCase			,						**__lowerCAmelCase      )

 @property
 def     snake_case_      (    self					:					Optional[Any]      )   ->	Union[str, Any]:
    logger.info(f'''The model {self.model_type} is one of the few models that has no sequence length limit.'''      )
    return -1

 @max_position_embeddings.setter
 def     snake_case_      (    self					:					Tuple			,						__lowerCAmelCase					:					Optional[Any]      )   ->	Dict:
    # Message copied from Transformer-XL documentation
    raise NotImplementedError(
        f'''The model {self.model_type} is one of the few models that has no sequence length limit.'''      )



