


'''simple docstring'''
from importlib import import_module

from .logging import get_logger


UpperCAmelCase_ :     Any								=      get_logger(__name__)

class 						_lowerCamelCase					:




 '''simple docstring'''


 def __init__(      self      ,      __lowercase      ,      __lowercase=None						):

  """simple docstring"""







  __A  :		Any       =			attrs or []
  if module is not None:
   for key in module.__dict__:
    if key in attrs or not key.startswith('__'						):
     setattr(self      ,      UpperCamelCase__      ,      getattr(UpperCamelCase__      ,      UpperCamelCase__						)						)
  __A  :		Any       =			module._original_module if isinstance(UpperCamelCase__      ,      _PatchedModuleObj						) else module


class 						_lowerCamelCase					:




 '''simple docstring'''
 __lowercase				:		List[Any]    							=						[]


 def __init__(      self      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase=None						):

  """simple docstring"""







  __A  :		Optional[int]       =			obj
  __A  :		Optional[int]       =			target
  __A  :		Optional[Any]       =			new
  __A  :		int       =			target.split('.'						)[0]
  __A  :		Union[str, Any]       =			{}
  __A  :		Any       =			attrs or []


 def __enter__(      self						):

  """simple docstring"""







  __A  :		List[str]       =			self.target.split('.'						)

  # Patch modules:
  # it's used to patch attributes of submodules like "os.path.join";
  # in this case we need to patch "os" and "os.path"

  for i in range(len(UpperCamelCase__						)						):
   try:
    __A  :		Optional[int]       =			import_module('.'.join(submodules[: i + 1]						)						)
   except ModuleNotFoundError:
    continue
   # We iterate over all the globals in self.obj in case we find "os" or "os.path"
   for attr in self.obj.__dir__():
    __A  :		str       =			getattr(self.obj      ,      UpperCamelCase__						)
    # We don't check for the name of the global, but rather if its value *is* "os" or "os.path".
    # This allows to patch renamed modules like "from os import path as ospath".
    if obj_attr is submodule or (
        (isinstance(UpperCamelCase__      ,      _PatchedModuleObj						) and obj_attr._original_module is submodule)
    ):
     __A  :		Optional[int]       =			obj_attr
     # patch at top level
     setattr(self.obj      ,      UpperCamelCase__      ,      _PatchedModuleObj(UpperCamelCase__      ,      attrs=self.attrs						)						)
     __A  :		List[Any]       =			getattr(self.obj      ,      UpperCamelCase__						)
     # construct lower levels patches
     for key in submodules[i + 1 :]:
      setattr(UpperCamelCase__      ,      UpperCamelCase__      ,      _PatchedModuleObj(getattr(UpperCamelCase__      ,      UpperCamelCase__      ,      UpperCamelCase__						)      ,      attrs=self.attrs						)						)
      __A  :		Union[str, Any]       =			getattr(UpperCamelCase__      ,      UpperCamelCase__						)
     # finally set the target attribute
     setattr(UpperCamelCase__      ,      UpperCamelCase__      ,      self.new						)

        # Patch attribute itself:
        # it's used for builtins like "open",
        # and also to patch "os.path.join" we may also need to patch "join"
        # itself if it was imported as "from os.path import join".

  if submodules:  # if it's an attribute of a submodule like "os.path.join"
   try:
    __A  :		Optional[int]       =			getattr(import_module('.'.join(UpperCamelCase__						)						)      ,      UpperCamelCase__						)
   except (AttributeError, ModuleNotFoundError):
    return
   # We iterate over all the globals in self.obj in case we find "os.path.join"
   for attr in self.obj.__dir__():
    # We don't check for the name of the global, but rather if its value *is* "os.path.join".
    # This allows to patch renamed attributes like "from os.path import join as pjoin".
    if getattr(self.obj      ,      UpperCamelCase__						) is attr_value:
     __A  :		Optional[int]       =			getattr(self.obj      ,      UpperCamelCase__						)
     setattr(self.obj      ,      UpperCamelCase__      ,      self.new						)
  elif target_attr in globals()["__builtins__"]:  # if it'a s builtin like "open"
   __A  :		Dict       =			globals()['__builtins__'][target_attr]
   setattr(self.obj      ,      UpperCamelCase__      ,      self.new						)
  else:
   raise RuntimeError(F"""Tried to patch attribute {target_attr} instead of a submodule."""						)


 def __exit__(      self      ,      *__lowercase						):

  """simple docstring"""







  for attr in list(self.original						):
   setattr(self.obj      ,      UpperCamelCase__      ,      self.original.pop(UpperCamelCase__						)						)


 def  snake_case__   (      self						):

  """simple docstring"""







  self.__enter__()
  self._active_patches.append(self						)







 def  snake_case__   (      self						):

  """simple docstring"""







  try:
   self._active_patches.remove(self						)
  except ValueError:
   # If the patch hasn't been started this will fail
   return None

  return self.__exit__()










'''simple docstring'''
import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO
)
UpperCAmelCase_ :     Any								=      logging.getLogger(__name__)


def 			_lowercase       (				):
 __A  :		Tuple       =			argparse.ArgumentParser(
     description='Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'			)
 parser.add_argument('--file_path',				type=UpperCamelCase__,				default='data/dump.txt',				help='The path to the data.'			)
 parser.add_argument('--tokenizer_type',				type=UpperCamelCase__,				default='bert',				choices=['bert', 'roberta', 'gpt2']			)
 parser.add_argument('--tokenizer_name',				type=UpperCamelCase__,				default='bert-base-uncased',				help='The tokenizer to use.'			)
 parser.add_argument('--dump_file',				type=UpperCamelCase__,				default='data/dump',				help='The dump file prefix.'			)
 __A  :		Tuple       =			parser.parse_args()

 logger.info(f"""Loading Tokenizer ({args.tokenizer_name})"""			)
 if args.tokenizer_type == "bert":
  __A  :		Any       =			BertTokenizer.from_pretrained(args.tokenizer_name			)
  __A  :		Any       =			tokenizer.special_tokens_map['cls_token']  # `[CLS]`
  __A  :		List[Any]       =			tokenizer.special_tokens_map['sep_token']  # `[SEP]`
 elif args.tokenizer_type == "roberta":
  __A  :		Any       =			RobertaTokenizer.from_pretrained(args.tokenizer_name			)
  __A  :		List[Any]       =			tokenizer.special_tokens_map['cls_token']  # `<s>`
  __A  :		Dict       =			tokenizer.special_tokens_map['sep_token']  # `</s>`
 elif args.tokenizer_type == "gpt2":
  __A  :		List[str]       =			GPTaTokenizer.from_pretrained(args.tokenizer_name			)
  __A  :		Any       =			tokenizer.special_tokens_map['bos_token']  # `<|endoftext|>`
  __A  :		Union[str, Any]       =			tokenizer.special_tokens_map['eos_token']  # `<|endoftext|>`

 logger.info(f"""Loading text from {args.file_path}"""			)
 with open(args.file_path,				'r',				encoding='utf8'			) as fp:
  __A  :		Optional[int]       =			fp.readlines()

 logger.info('Start encoding'			)
 logger.info(f"""{len(UpperCamelCase__			)} examples to process."""			)

 __A  :		int       =			[]
 __A  :		List[str]       =			0
 __A  :		Tuple       =			10000
 __A  :		List[str]       =			time.time()
 for text in data:
  __A  :		int       =			f"""{bos} {text.strip()} {sep}"""
  __A  :		List[Any]       =			tokenizer.encode(UpperCamelCase__,				add_special_tokens=UpperCamelCase__			)
  rslt.append(UpperCamelCase__			)

  iter += 1
  if iter % interval == 0:
   __A  :		Any       =			time.time()
   logger.info(f"""{iter} examples processed. - {(end-start):.2f}s/{interval}expl"""			)
   __A  :		Any       =			time.time()
 logger.info('Finished binarization'			)
 logger.info(f"""{len(UpperCamelCase__			)} examples processed."""			)

 __A  :		List[str]       =			f"""{args.dump_file}.{args.tokenizer_name}.pickle"""
 __A  :		str       =			tokenizer.vocab_size
 if vocab_size < (1 << 16):
  __A  :		Optional[int]       =			[np.uintaa(UpperCamelCase__			) for d in rslt]
 else:
  __A  :		int       =			[np.intaa(UpperCamelCase__			) for d in rslt]
 random.shuffle(rslt_			)
 logger.info(f"""Dump to {dp_file}"""			)
 with open(UpperCamelCase__,				'wb'			) as handle:
  pickle.dump(rslt_,				UpperCamelCase__,				protocol=pickle.HIGHEST_PROTOCOL			)


if __name__ == "__main__":
 main()






