
'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_lowercase          =						{
    """configuration_megatron_bert""": ["""MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MegatronBertConfig"""],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_lowercase          =						[
						    """MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """MegatronBertForCausalLM""",
						    """MegatronBertForMaskedLM""",
						    """MegatronBertForMultipleChoice""",
						    """MegatronBertForNextSentencePrediction""",
						    """MegatronBertForPreTraining""",
						    """MegatronBertForQuestionAnswering""",
						    """MegatronBertForSequenceClassification""",
						    """MegatronBertForTokenClassification""",
						    """MegatronBertModel""",
						    """MegatronBertPreTrainedModel""",
						]

if TYPE_CHECKING:
						from .configuration_megatron_bert import MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MegatronBertConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_megatron_bert import (
												    MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    MegatronBertForCausalLM,
												    MegatronBertForMaskedLM,
												    MegatronBertForMultipleChoice,
												    MegatronBertForNextSentencePrediction,
												    MegatronBertForPreTraining,
												    MegatronBertForQuestionAnswering,
												    MegatronBertForSequenceClassification,
												    MegatronBertForTokenClassification,
												    MegatronBertModel,
												    MegatronBertPreTrainedModel,
												)

else:
						import sys

						_lowercase          =						_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)







"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE__   =  logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__   =  {
    "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/config.json",
    "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/config.json"
    # See all FNet models at https://huggingface.co/models?filter=fnet
}

class 						lowercase   (   _UpperCAmelCase   ):
      _SCREAMING_SNAKE_CASE							      = 'fnet'





      def __init__(			self ,		lowercase=32_000 ,		lowercase=768 ,		lowercase=12 ,		lowercase=3_072 ,		lowercase="gelu_new" ,		lowercase=0.1 ,		lowercase=512 ,		lowercase=4 ,		lowercase=0.02 ,		lowercase=1e-12 ,		lowercase=False ,		lowercase=512 ,		lowercase=3 ,		lowercase=1 ,		lowercase=2 ,		**lowercase ,		)						->	int:
            super().__init__(pad_token_id=lowercase ,		bos_token_id=lowercase ,		eos_token_id=lowercase ,		**lowercase					)

            lowerCAmelCase    		=			vocab_size
            lowerCAmelCase    		=			max_position_embeddings
            lowerCAmelCase    		=			hidden_size
            lowerCAmelCase    		=			num_hidden_layers
            lowerCAmelCase    		=			intermediate_size
            lowerCAmelCase    		=			hidden_act
            lowerCAmelCase    		=			hidden_dropout_prob
            lowerCAmelCase    		=			initializer_range
            lowerCAmelCase    		=			type_vocab_size
            lowerCAmelCase    		=			layer_norm_eps
            lowerCAmelCase    		=			use_tpu_fourier_optimizations
            lowerCAmelCase    		=			tpu_short_seq_length






