





import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
__A     =    logging.getLogger(__name__)





def 					lowerCAmelCase_			(	)					->		Dict:







  """simple docstring"""





  lowerCamelCase__:				Tuple        =argparse.ArgumentParser(
      description="Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids)."		)
  parser.add_argument("--file_path"     ,   type=__a     ,   default="data/dump.txt"     ,   help="The path to the data."		)
  parser.add_argument("--tokenizer_type"     ,   type=__a     ,   default="bert"     ,   choices=["bert", "roberta", "gpt2"]		)
  parser.add_argument("--tokenizer_name"     ,   type=__a     ,   default="bert-base-uncased"     ,   help="The tokenizer to use."		)
  parser.add_argument("--dump_file"     ,   type=__a     ,   default="data/dump"     ,   help="The dump file prefix."		)
  lowerCamelCase__:				Any        =parser.parse_args()

  logger.info(F"""Loading Tokenizer ({args.tokenizer_name})"""		)
  if args.tokenizer_type == "bert":
    lowerCamelCase__:				List[str]        =BertTokenizer.from_pretrained(args.tokenizer_name		)
    lowerCamelCase__:				List[Any]        =tokenizer.special_tokens_map["cls_token"]  # `[CLS]`
    lowerCamelCase__:				Dict        =tokenizer.special_tokens_map["sep_token"]  # `[SEP]`
  elif args.tokenizer_type == "roberta":
    lowerCamelCase__:				Union[str, Any]        =RobertaTokenizer.from_pretrained(args.tokenizer_name		)
    lowerCamelCase__:				str        =tokenizer.special_tokens_map["cls_token"]  # `<s>`
    lowerCamelCase__:				Tuple        =tokenizer.special_tokens_map["sep_token"]  # `</s>`
  elif args.tokenizer_type == "gpt2":
    lowerCamelCase__:				str        =GPTaTokenizer.from_pretrained(args.tokenizer_name		)
    lowerCamelCase__:				Tuple        =tokenizer.special_tokens_map["bos_token"]  # `<|endoftext|>`
    lowerCamelCase__:				Optional[int]        =tokenizer.special_tokens_map["eos_token"]  # `<|endoftext|>`

  logger.info(F"""Loading text from {args.file_path}"""		)
  with open(args.file_path     ,   "r"     ,   encoding="utf8"		) as fp:
    lowerCamelCase__:				Dict        =fp.readlines()

  logger.info("Start encoding"		)
  logger.info(F"""{len(__a		)} examples to process."""		)

  lowerCamelCase__:				int        =[]
  lowerCamelCase__:				Any        =0
  lowerCamelCase__:				List[Any]        =10000
  lowerCamelCase__:				Optional[int]        =time.time()
  for text in data:
    lowerCamelCase__:				int        =F"""{bos} {text.strip()} {sep}"""
    lowerCamelCase__:				Optional[Any]        =tokenizer.encode(__a     ,   add_special_tokens=__a		)
    rslt.append(__a		)

    iter += 1
    if iter % interval == 0:
      lowerCamelCase__:				Any        =time.time()
      logger.info(F"""{iter} examples processed. - {(end-start):.2f}s/{interval}expl"""		)
      lowerCamelCase__:				Any        =time.time()
  logger.info("Finished binarization"		)
  logger.info(F"""{len(__a		)} examples processed."""		)

  lowerCamelCase__:				Optional[int]        =F"""{args.dump_file}.{args.tokenizer_name}.pickle"""
  lowerCamelCase__:				Optional[int]        =tokenizer.vocab_size
  if vocab_size < (1 << 16):
    lowerCamelCase__:				str        =[np.uintaa(__a		) for d in rslt]
  else:
    lowerCamelCase__:				str        =[np.intaa(__a		) for d in rslt]
  random.shuffle(rslt_		)
  logger.info(F"""Dump to {dp_file}"""		)
  with open(__a     ,   "wb"		) as handle:
    pickle.dump(rslt_     ,   __a     ,   protocol=pickle.HIGHEST_PROTOCOL		)


if __name__ == "__main__":
     main()








import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
__A     =    logging.getLogger(__name__)

if __name__ == "__main__":
     __A     =    argparse.ArgumentParser(
         description="Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"
     )
     parser.add_argument(
         "--data_file", type=str, default="data/dump.bert-base-uncased.pickle", help="The binarized dataset."
     )
     parser.add_argument(
         "--token_counts_dump", type=str, default="data/token_counts.bert-base-uncased.pickle", help="The dump file."
     )
     parser.add_argument("--vocab_size", default=3_0522, type=int)
     __A     =    parser.parse_args()

     logger.info(f'Loading data from {args.data_file}')
     with open(args.data_file, "rb") as fp:
          __A     =    pickle.load(fp)

     logger.info("Counting occurrences for MLM.")
     __A     =    Counter()
     for tk_ids in data:
          counter.update(tk_ids)
     __A     =    [0] * args.vocab_size
     for k, v in counter.items():
          __A     =    v

     logger.info(f'Dump to {args.token_counts_dump}')
     with open(args.token_counts_dump, "wb") as handle:
          pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)

