





"""simple docstring"""




from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


__A     =    logging.get_logger(__name__)


__A     =    {
    "Salesforce/codegen-350M-nl": "https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json",
    "Salesforce/codegen-350M-multi": "https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json",
    "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json",
    "Salesforce/codegen-2B-nl": "https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json",
    "Salesforce/codegen-2B-multi": "https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json",
    "Salesforce/codegen-2B-mono": "https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json",
    "Salesforce/codegen-6B-nl": "https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json",
    "Salesforce/codegen-6B-multi": "https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json",
    "Salesforce/codegen-6B-mono": "https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json",
    "Salesforce/codegen-16B-nl": "https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json",
    "Salesforce/codegen-16B-multi": "https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json",
    "Salesforce/codegen-16B-mono": "https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json",
}



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''
 lowercase_		=      "codegen"
 lowercase_		=      {
     "max_position_embeddings": "n_positions",
     "hidden_size": "n_embd",
     "num_attention_heads": "n_head",
     "num_hidden_layers": "n_layer",
 }

 def __init__(self    :       Union[str, Any]	,			UpperCAmelCase_    :       Optional[Any]=50_400	,			UpperCAmelCase_    :       Optional[Any]=2_048	,			UpperCAmelCase_    :       str=2_048	,			UpperCAmelCase_    :       Union[str, Any]=4_096	,			UpperCAmelCase_    :       List[str]=28	,			UpperCAmelCase_    :       Union[str, Any]=16	,			UpperCAmelCase_    :       Dict=64	,			UpperCAmelCase_    :       Any=None	,			UpperCAmelCase_    :       Optional[int]="gelu_new"	,			UpperCAmelCase_    :       List[Any]=0.0	,			UpperCAmelCase_    :       Optional[int]=0.0	,			UpperCAmelCase_    :       Any=0.0	,			UpperCAmelCase_    :       Optional[Any]=1E-5	,			UpperCAmelCase_    :       Tuple=0.02	,			UpperCAmelCase_    :       List[str]=True	,			UpperCAmelCase_    :       Any=50_256	,			UpperCAmelCase_    :       Union[str, Any]=50_256	,			UpperCAmelCase_    :       Union[str, Any]=False	,			**UpperCAmelCase_    :       List[str]	,			)	->Optional[int]:



   '''simple docstring'''



   lowerCamelCase__:				Optional[Any]        =vocab_size
   lowerCamelCase__:				int        =n_ctx
   lowerCamelCase__:				str        =n_positions
   lowerCamelCase__:				str        =n_embd
   lowerCamelCase__:				Optional[int]        =n_layer
   lowerCamelCase__:				Dict        =n_head
   lowerCamelCase__:				Tuple        =n_inner
   lowerCamelCase__:				Optional[int]        =rotary_dim
   lowerCamelCase__:				Tuple        =activation_function
   lowerCamelCase__:				Optional[Any]        =resid_pdrop
   lowerCamelCase__:				int        =embd_pdrop
   lowerCamelCase__:				Tuple        =attn_pdrop
   lowerCamelCase__:				Union[str, Any]        =layer_norm_epsilon
   lowerCamelCase__:				List[Any]        =initializer_range
   lowerCamelCase__:				Any        =use_cache

   lowerCamelCase__:				Optional[Any]        =bos_token_id
   lowerCamelCase__:				int        =eos_token_id

   super().__init__(
       bos_token_id=UpperCAmelCase_	,			eos_token_id=UpperCAmelCase_	,			tie_word_embeddings=UpperCAmelCase_	,			**UpperCAmelCase_)



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''

 def __init__(self    :       List[Any]	,			UpperCAmelCase_    :       PretrainedConfig	,			UpperCAmelCase_    :       str = "default"	,			UpperCAmelCase_    :       List[PatchingSpec] = None	,			UpperCAmelCase_    :       bool = False	,			)	->Optional[int]:



   '''simple docstring'''



   super().__init__(UpperCAmelCase_	,			task=UpperCAmelCase_	,			patching_specs=UpperCAmelCase_	,			use_past=UpperCAmelCase_)
   if not getattr(self._config	,			"pad_token_id"	,			UpperCAmelCase_):
     # TODO: how to do that better?
     lowerCamelCase__:				Optional[Any]        =0

 @property
 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[int])	->Mapping[str, Mapping[int, str]]:



   '''simple docstring'''



   lowerCamelCase__:				Any        =OrderedDict({"input_ids": {0: "batch", 1: "sequence"}})
   if self.use_past:
     self.fill_with_past_key_values_(UpperCAmelCase_	,			direction="inputs")
     lowerCamelCase__:				Optional[Any]        ={0: "batch", 1: "past_sequence + sequence"}
   else:
     lowerCamelCase__:				Tuple        ={0: "batch", 1: "sequence"}

   return common_inputs

 @property
 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[Any])	->int:



   '''simple docstring'''



   return self._config.n_layer

 @property
 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[int])	->int:



   '''simple docstring'''



   return self._config.n_head

 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[Any]	,			UpperCAmelCase_    :       PreTrainedTokenizer	,			UpperCAmelCase_    :       int = -1	,			UpperCAmelCase_    :       int = -1	,			UpperCAmelCase_    :       bool = False	,			UpperCAmelCase_    :       Optional[TensorType] = None	,			)	->Mapping[str, Any]:



   '''simple docstring'''



   lowerCamelCase__:				Union[str, Any]        =super(UpperCAmelCase_	,			self).generate_dummy_inputs(
       UpperCAmelCase_	,			batch_size=UpperCAmelCase_	,			seq_length=UpperCAmelCase_	,			is_pair=UpperCAmelCase_	,			framework=UpperCAmelCase_)

   # We need to order the input in the way they appears in the forward()
   lowerCamelCase__:				List[str]        =OrderedDict({"input_ids": common_inputs["input_ids"]})

   # Need to add the past_keys
   if self.use_past:
     if not is_torch_available():
       raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed.")
     else:
       import torch

       lowerCamelCase__:				Union[str, Any]        =common_inputs["input_ids"].shape
       # Not using the same length for past_key_values
       lowerCamelCase__:				List[Any]        =seqlen + 2
       lowerCamelCase__:				str        =(
           batch,
           self.num_attention_heads,
           past_key_values_length,
           self._config.hidden_size // self.num_attention_heads,
       )
       lowerCamelCase__:				Tuple        =[
           (torch.zeros(UpperCAmelCase_), torch.zeros(UpperCAmelCase_)) for _ in range(self.num_layers)
       ]

   lowerCamelCase__:				str        =common_inputs["attention_mask"]
   if self.use_past:
     lowerCamelCase__:				List[Any]        =ordered_inputs["attention_mask"].dtype
     lowerCamelCase__:				Union[str, Any]        =torch.cat(
         [ordered_inputs["attention_mask"], torch.ones(UpperCAmelCase_	,			UpperCAmelCase_	,			dtype=UpperCAmelCase_)]	,			dim=1)

   return ordered_inputs







 @property
 def 							SCREAMING_SNAKE_CASE_ (self    :       Union[str, Any])	->int:



   '''simple docstring'''



   return 13








import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
__A     =    logging.getLogger(__name__)

if __name__ == "__main__":
     __A     =    argparse.ArgumentParser(
         description="Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"
     )
     parser.add_argument(
         "--data_file", type=str, default="data/dump.bert-base-uncased.pickle", help="The binarized dataset."
     )
     parser.add_argument(
         "--token_counts_dump", type=str, default="data/token_counts.bert-base-uncased.pickle", help="The dump file."
     )
     parser.add_argument("--vocab_size", default=3_0522, type=int)
     __A     =    parser.parse_args()

     logger.info(f'Loading data from {args.data_file}')
     with open(args.data_file, "rb") as fp:
          __A     =    pickle.load(fp)

     logger.info("Counting occurrences for MLM.")
     __A     =    Counter()
     for tk_ids in data:
          counter.update(tk_ids)
     __A     =    [0] * args.vocab_size
     for k, v in counter.items():
          __A     =    v

     logger.info(f'Dump to {args.token_counts_dump}')
     with open(args.token_counts_dump, "wb") as handle:
          pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)

