






from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE__				     =			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__				     =			{'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/config.json'''}






class 			_UpperCamelCase(       __lowerCamelCase		):
		__SCREAMING_SNAKE_CASE  :       Any							=							'''openai-gpt'''
		__SCREAMING_SNAKE_CASE  :       Union[str, Any]							=							{
		    '''max_position_embeddings''': '''n_positions''',
		    '''hidden_size''': '''n_embd''',
		    '''num_attention_heads''': '''n_head''',
		    '''num_hidden_layers''': '''n_layer''',
		}

		def __init__(					self		:    Tuple    ,       SCREAMING_SNAKE_CASE__		:    Optional[Any]=4_0_4_7_8    ,       SCREAMING_SNAKE_CASE__		:    Dict=5_1_2    ,       SCREAMING_SNAKE_CASE__		:    Optional[Any]=7_6_8    ,       SCREAMING_SNAKE_CASE__		:    List[str]=1_2    ,       SCREAMING_SNAKE_CASE__		:    str=1_2    ,       SCREAMING_SNAKE_CASE__		:    Union[str, Any]="gelu"    ,       SCREAMING_SNAKE_CASE__		:    Union[str, Any]=0.1    ,       SCREAMING_SNAKE_CASE__		:    Tuple=0.1    ,       SCREAMING_SNAKE_CASE__		:    List[Any]=0.1    ,       SCREAMING_SNAKE_CASE__		:    Optional[Any]=1e-5    ,       SCREAMING_SNAKE_CASE__		:    Dict=0.02    ,       SCREAMING_SNAKE_CASE__		:    Optional[int]="cls_index"    ,       SCREAMING_SNAKE_CASE__		:    Tuple=True    ,       SCREAMING_SNAKE_CASE__		:    Tuple=None    ,       SCREAMING_SNAKE_CASE__		:    str=True    ,       SCREAMING_SNAKE_CASE__		:    str=0.1    ,       **SCREAMING_SNAKE_CASE__		:    Union[str, Any]    ,       ):



					'''simple docstring'''



					__a						:  Union[str, Any]									=     vocab_size
					__a						:  List[str]									=     n_positions
					__a						:  Union[str, Any]									=     n_embd
					__a						:  List[str]									=     n_layer
					__a						:  Optional[Any]									=     n_head
					__a						:  Any									=     afn
					__a						:  Union[str, Any]									=     resid_pdrop
					__a						:  str									=     embd_pdrop
					__a						:  str									=     attn_pdrop
					__a						:  Union[str, Any]									=     layer_norm_epsilon
					__a						:  Optional[int]									=     initializer_range
					__a						:  Optional[int]									=     summary_type
					__a						:  str									=     summary_use_proj
					__a						:  Any									=     summary_activation
					__a						:  List[str]									=     summary_first_dropout
					__a						:  List[str]									=     summary_proj_to_labels
					super().__init__(**SCREAMING_SNAKE_CASE__    )













"""simple docstring"""







import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO
)
_lowerCAmelCase            =	logging.getLogger(__name__)

if __name__ == "__main__":
				_lowerCAmelCase            =	argparse.ArgumentParser(
				    description='Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)'
				)
				parser.add_argument(
				    '--data_file', type=str, default='data/dump.bert-base-uncased.pickle', help='The binarized dataset.'
				)
				parser.add_argument(
				    '--token_counts_dump', type=str, default='data/token_counts.bert-base-uncased.pickle', help='The dump file.'
				)
				parser.add_argument('--vocab_size', default=3_05_22, type=int)
				_lowerCAmelCase            =	parser.parse_args()

				logger.info(F'Loading data from {args.data_file}')
				with open(args.data_file, 'rb') as fp:
								_lowerCAmelCase            =	pickle.load(fp)

				logger.info('Counting occurrences for MLM.')
				_lowerCAmelCase            =	Counter()
				for tk_ids in data:
								counter.update(tk_ids)
				_lowerCAmelCase            =	[0] * args.vocab_size
				for k, v in counter.items():
								_lowerCAmelCase            =	v

				logger.info(F'Dump to {args.token_counts_dump}')
				with open(args.token_counts_dump, 'wb') as handle:
								pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)
