


from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase					:		Any            =						logging.get_logger(__name__)

lowerCamelCase					:		Union[str, Any]            =						{
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}


class snake_case__			(    UpperCamelCase_					):
       _lowerCAmelCase   ='megatron-bert'




       def __init__( self				:    Tuple    ,				_lowerCamelCase				:    List[Any]=2_9_0_5_6    ,				_lowerCamelCase				:    Any=1_0_2_4    ,				_lowerCamelCase				:    Tuple=2_4    ,				_lowerCamelCase				:    Optional[Any]=1_6    ,				_lowerCamelCase				:    List[str]=4_0_9_6    ,				_lowerCamelCase				:    Union[str, Any]="gelu"    ,				_lowerCamelCase				:    int=0.1    ,				_lowerCamelCase				:    Dict=0.1    ,				_lowerCamelCase				:    str=5_1_2    ,				_lowerCamelCase				:    Optional[Any]=2    ,				_lowerCamelCase				:    Any=0.02    ,				_lowerCamelCase				:    Optional[Any]=1E-12    ,				_lowerCamelCase				:    str=0    ,				_lowerCamelCase				:    Optional[Any]="absolute"    ,				_lowerCamelCase				:    Optional[Any]=True    ,				**_lowerCamelCase				:    Any    ,				):
              super().__init__(pad_token_id=snake_case__    ,				**snake_case__			)

              snake_case__ :       Union[str, Any]	   =							vocab_size
              snake_case__ :       Union[str, Any]	   =							hidden_size
              snake_case__ :       Optional[Any]	   =							num_hidden_layers
              snake_case__ :       Tuple	   =							num_attention_heads
              snake_case__ :       List[Any]	   =							hidden_act
              snake_case__ :       Optional[int]	   =							intermediate_size
              snake_case__ :       Dict	   =							hidden_dropout_prob
              snake_case__ :       str	   =							attention_probs_dropout_prob
              snake_case__ :       str	   =							max_position_embeddings
              snake_case__ :       List[str]	   =							type_vocab_size
              snake_case__ :       Tuple	   =							initializer_range
              snake_case__ :       str	   =							layer_norm_eps
              snake_case__ :       List[Any]	   =							position_embedding_type
              snake_case__ :       Dict	   =							use_cache













"""simple docstring"""



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_lowerCAmelCase	:	Tuple				=      {
    """configuration_lxmert""": ["""LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """LxmertConfig"""],
    """tokenization_lxmert""": ["""LxmertTokenizer"""],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase	:	int				=      ["""LxmertTokenizerFast"""]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase	:	Tuple				=      [
     """LxmertEncoder""",
     """LxmertForPreTraining""",
     """LxmertForQuestionAnswering""",
     """LxmertModel""",
     """LxmertPreTrainedModel""",
     """LxmertVisualFeatureEncoder""",
     """LxmertXLayer""",
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase	:	Optional[int]				=      [
     """TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """TFLxmertForPreTraining""",
     """TFLxmertMainLayer""",
     """TFLxmertModel""",
     """TFLxmertPreTrainedModel""",
     """TFLxmertVisualFeatureEncoder""",
 ]


if TYPE_CHECKING:
 from .configuration_lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig
 from .tokenization_lxmert import LxmertTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_lxmert_fast import LxmertTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_lxmert import (
      LxmertEncoder,
      LxmertForPreTraining,
      LxmertForQuestionAnswering,
      LxmertModel,
      LxmertPreTrainedModel,
      LxmertVisualFeatureEncoder,
      LxmertXLayer,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_lxmert import (
      TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFLxmertForPreTraining,
      TFLxmertMainLayer,
      TFLxmertModel,
      TFLxmertPreTrainedModel,
      TFLxmertVisualFeatureEncoder,
  )

else:
 import sys

 _lowerCAmelCase	:	Any				=      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


