





'''simple docstring'''


from ...configuration_utils import PretrainedConfig
from ...utils import logging


a__			:					Union[str, Any]       						=						logging.get_logger(__name__)

a__			:					Optional[int]       						=						{
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}






class     lowerCAmelCase__ (				UpperCAmelCase_    ):

       '''simple docstring'''
       _lowerCamelCase								="megatron-bert"

       def __init__( self					:				Dict       ,     a__					:				Union[str, Any]=29056       ,     a__					:				Dict=1024       ,     a__					:				str=24       ,     a__					:				Any=16       ,     a__					:				Tuple=4096       ,     a__					:				Optional[int]="gelu"       ,     a__					:				Tuple=0.1       ,     a__					:				Tuple=0.1       ,     a__					:				Any=512       ,     a__					:				Optional[Any]=2       ,     a__					:				str=0.02       ,     a__					:				Optional[int]=1e-1_2       ,     a__					:				Union[str, Any]=0       ,     a__					:				Optional[Any]="absolute"       ,     a__					:				Dict=True       ,     **a__					:				Dict       ,     ):
              super().__init__(pad_token_id=a__       ,     **a__						)

              UpperCAmelCase  =						vocab_size
              UpperCAmelCase  =						hidden_size
              UpperCAmelCase  =						num_hidden_layers
              UpperCAmelCase  =						num_attention_heads
              UpperCAmelCase  =						hidden_act
              UpperCAmelCase  =						intermediate_size
              UpperCAmelCase  =						hidden_dropout_prob
              UpperCAmelCase  =						attention_probs_dropout_prob
              UpperCAmelCase  =						max_position_embeddings
              UpperCAmelCase  =						type_vocab_size
              UpperCAmelCase  =						initializer_range
              UpperCAmelCase  =						layer_norm_eps
              UpperCAmelCase  =						position_embedding_type
              UpperCAmelCase  =						use_cache




"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_lowercase				 =					{
    '''configuration_xlm_roberta_xl''': [
        '''XLM_ROBERTA_XL_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''XLMRobertaXLConfig''',
        '''XLMRobertaXLOnnxConfig''',
    ],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_lowercase				 =					[
		    '''XLM_ROBERTA_XL_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''XLMRobertaXLForCausalLM''',
		    '''XLMRobertaXLForMaskedLM''',
		    '''XLMRobertaXLForMultipleChoice''',
		    '''XLMRobertaXLForQuestionAnswering''',
		    '''XLMRobertaXLForSequenceClassification''',
		    '''XLMRobertaXLForTokenClassification''',
		    '''XLMRobertaXLModel''',
		    '''XLMRobertaXLPreTrainedModel''',
		]

if TYPE_CHECKING:
		from .configuration_xlm_roberta_xl import (
		    XLM_ROBERTA_XL_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    XLMRobertaXLConfig,
		    XLMRobertaXLOnnxConfig,
		)

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_xlm_roberta_xl import (
				    XLM_ROBERTA_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
				    XLMRobertaXLForCausalLM,
				    XLMRobertaXLForMaskedLM,
				    XLMRobertaXLForMultipleChoice,
				    XLMRobertaXLForQuestionAnswering,
				    XLMRobertaXLForSequenceClassification,
				    XLMRobertaXLForTokenClassification,
				    XLMRobertaXLModel,
				    XLMRobertaXLPreTrainedModel,
				)

else:
		import sys

		_lowercase				 =					_LazyModule(__name__, globals()['''__file__'''], _import_structure)