





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__						:			Tuple									=logging.get_logger(__name__)

UpperCAmelCase__						:			Union[str, Any]									={
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}

class       __A     (						_UpperCAmelCase    ):
				__A      =				"""megatron-bert"""


				def __init__(		self   , UpperCAmelCase_=29056   , UpperCAmelCase_=1024   , UpperCAmelCase_=24   , UpperCAmelCase_=16   , UpperCAmelCase_=4096   , UpperCAmelCase_="gelu"   , UpperCAmelCase_=0.1   , UpperCAmelCase_=0.1   , UpperCAmelCase_=512   , UpperCAmelCase_=2   , UpperCAmelCase_=0.0_2   , UpperCAmelCase_=1E-12   , UpperCAmelCase_=0   , UpperCAmelCase_="absolute"   , UpperCAmelCase_=True   , **UpperCAmelCase_   , ):
					super().__init__(pad_token_id=lowercase_   , **lowercase_							)

					lowerCamelCase	 =vocab_size
					lowerCamelCase	 =hidden_size
					lowerCamelCase	 =num_hidden_layers
					lowerCamelCase	 =num_attention_heads
					lowerCamelCase	 =hidden_act
					lowerCamelCase	 =intermediate_size
					lowerCamelCase	 =hidden_dropout_prob
					lowerCamelCase	 =attention_probs_dropout_prob
					lowerCamelCase	 =max_position_embeddings
					lowerCamelCase	 =type_vocab_size
					lowerCamelCase	 =initializer_range
					lowerCamelCase	 =layer_norm_eps
					lowerCamelCase	 =position_embedding_type
					lowerCamelCase	 =use_cache












def      _lowercase					(						_UpperCAmelCase			,      _UpperCAmelCase				)						->  list[str]:
				return [sentence[i : i + ngram_size] for i in range(len(_UpperCAmelCase				) - ngram_size + 1				)]


if __name__ == "__main__":
							from doctest import testmod

							testmod()





