





"""simple docstring"""


from ...configuration_utils import PretrainedConfig
from ...utils import logging


__magic_name__       :					Union[str, Any]						=			logging.get_logger(__name__)

__magic_name__       :					List[Any]						=			{
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}






class 							lowercase__			(   __SCREAMING_SNAKE_CASE	):

			"""simple docstring"""
			__lowerCAmelCase     :				Any										=  """megatron-bert"""







			def __init__(      self						,   _A=2_9_0_5_6						,   _A=1_0_2_4						,   _A=2_4						,   _A=1_6						,   _A=4_0_9_6						,   _A="gelu"						,   _A=0.1						,   _A=0.1						,   _A=5_1_2						,   _A=2						,   _A=0.02						,   _A=1e-1_2						,   _A=0						,   _A="absolute"						,   _A=True						,   **_A						,   ):






									'''simple docstring'''







									super().__init__(pad_token_id=_A						,   **_A    )

									UpperCamelCase				:		int				   =			vocab_size
									UpperCamelCase				:		int				   =			hidden_size
									UpperCamelCase				:		Tuple				   =			num_hidden_layers
									UpperCamelCase				:		Tuple				   =			num_attention_heads
									UpperCamelCase				:		Any				   =			hidden_act
									UpperCamelCase				:		Dict				   =			intermediate_size
									UpperCamelCase				:		Union[str, Any]				   =			hidden_dropout_prob
									UpperCamelCase				:		List[Any]				   =			attention_probs_dropout_prob
									UpperCamelCase				:		Union[str, Any]				   =			max_position_embeddings
									UpperCamelCase				:		int				   =			type_vocab_size
									UpperCamelCase				:		List[Any]				   =			initializer_range
									UpperCamelCase				:		Union[str, Any]				   =			layer_norm_eps
									UpperCamelCase				:		Optional[int]				   =			position_embedding_type
									UpperCamelCase				:		int				   =			use_cache







'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__A   :    str											= {
    "configuration_roc_bert": ["ROC_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "RoCBertConfig"],
    "tokenization_roc_bert": ["RoCBertTokenizer"],
}

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  pass

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __A   :    Optional[int]											= [
      "ROC_BERT_PRETRAINED_MODEL_ARCHIVE_LIST",
      "RoCBertForCausalLM",
      "RoCBertForMaskedLM",
      "RoCBertForMultipleChoice",
      "RoCBertForPreTraining",
      "RoCBertForQuestionAnswering",
      "RoCBertForSequenceClassification",
      "RoCBertForTokenClassification",
      "RoCBertLayer",
      "RoCBertModel",
      "RoCBertPreTrainedModel",
      "load_tf_weights_in_roc_bert",
  ]

if TYPE_CHECKING:
  from .configuration_roc_bert import ROC_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RoCBertConfig
  from .tokenization_roc_bert import RoCBertTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    raise OptionalDependencyNotAvailable()

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_roc_bert import (
        ROC_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        RoCBertForCausalLM,
        RoCBertForMaskedLM,
        RoCBertForMultipleChoice,
        RoCBertForPreTraining,
        RoCBertForQuestionAnswering,
        RoCBertForSequenceClassification,
        RoCBertForTokenClassification,
        RoCBertLayer,
        RoCBertModel,
        RoCBertPreTrainedModel,
        load_tf_weights_in_roc_bert,
    )


else:
  import sys

  __A   :    Optional[int]											= _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



