





"""simple docstring"""




from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A     =    logging.get_logger(__name__)

__A     =    {
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''
 lowercase_		=      "megatron-bert"

 def __init__(self    :       Any	,			UpperCAmelCase_    :       List[str]=29_056	,			UpperCAmelCase_    :       Optional[int]=1_024	,			UpperCAmelCase_    :       Any=24	,			UpperCAmelCase_    :       Optional[Any]=16	,			UpperCAmelCase_    :       List[str]=4_096	,			UpperCAmelCase_    :       Any="gelu"	,			UpperCAmelCase_    :       Any=0.1	,			UpperCAmelCase_    :       Optional[int]=0.1	,			UpperCAmelCase_    :       str=512	,			UpperCAmelCase_    :       Tuple=2	,			UpperCAmelCase_    :       List[Any]=0.02	,			UpperCAmelCase_    :       Union[str, Any]=1E-1_2	,			UpperCAmelCase_    :       int=0	,			UpperCAmelCase_    :       List[str]="absolute"	,			UpperCAmelCase_    :       int=True	,			**UpperCAmelCase_    :       Tuple	,			)	->List[str]:



   '''simple docstring'''



   super().__init__(pad_token_id=UpperCAmelCase_	,			**UpperCAmelCase_)

   lowerCamelCase__:				Tuple        =vocab_size
   lowerCamelCase__:				int        =hidden_size
   lowerCamelCase__:				Optional[Any]        =num_hidden_layers
   lowerCamelCase__:				List[Any]        =num_attention_heads
   lowerCamelCase__:				Dict        =hidden_act
   lowerCamelCase__:				Optional[Any]        =intermediate_size
   lowerCamelCase__:				Any        =hidden_dropout_prob
   lowerCamelCase__:				Dict        =attention_probs_dropout_prob
   lowerCamelCase__:				List[str]        =max_position_embeddings
   lowerCamelCase__:				Union[str, Any]        =type_vocab_size
   lowerCamelCase__:				List[Any]        =initializer_range
   lowerCamelCase__:				str        =layer_norm_eps
   lowerCamelCase__:				str        =position_embedding_type
   lowerCamelCase__:				Any        =use_cache








import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
__A     =    logging.getLogger(__name__)

if __name__ == "__main__":
     __A     =    argparse.ArgumentParser(
         description="Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"
     )
     parser.add_argument(
         "--data_file", type=str, default="data/dump.bert-base-uncased.pickle", help="The binarized dataset."
     )
     parser.add_argument(
         "--token_counts_dump", type=str, default="data/token_counts.bert-base-uncased.pickle", help="The dump file."
     )
     parser.add_argument("--vocab_size", default=3_0522, type=int)
     __A     =    parser.parse_args()

     logger.info(f'Loading data from {args.data_file}')
     with open(args.data_file, "rb") as fp:
          __A     =    pickle.load(fp)

     logger.info("Counting occurrences for MLM.")
     __A     =    Counter()
     for tk_ids in data:
          counter.update(tk_ids)
     __A     =    [0] * args.vocab_size
     for k, v in counter.items():
          __A     =    v

     logger.info(f'Dump to {args.token_counts_dump}')
     with open(args.token_counts_dump, "wb") as handle:
          pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)

