



'''simple docstring'''





import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
A					:       Optional[Any]									=     logging.getLogger(__name__)
def 						_a       (    ):
   snake_case       :				Dict		    =argparse.ArgumentParser(
       description='''Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'''				)
   parser.add_argument('''--file_path'''	,		type=__a	,		default='''data/dump.txt'''	,		help='''The path to the data.'''				)
   parser.add_argument('''--tokenizer_type'''	,		type=__a	,		default='''bert'''	,		choices=['''bert''', '''roberta''', '''gpt2''']				)
   parser.add_argument('''--tokenizer_name'''	,		type=__a	,		default='''bert-base-uncased'''	,		help='''The tokenizer to use.'''				)
   parser.add_argument('''--dump_file'''	,		type=__a	,		default='''data/dump'''	,		help='''The dump file prefix.'''				)
   snake_case       :				Optional[int]		    =parser.parse_args()

   logger.info(F'''Loading Tokenizer ({args.tokenizer_name})'''				)
   if args.tokenizer_type == "bert":
      snake_case       :				int		    =BertTokenizer.from_pretrained(args.tokenizer_name				)
      snake_case       :				Union[str, Any]		    =tokenizer.special_tokens_map['''cls_token''']  # `[CLS]`
      snake_case       :				Tuple		    =tokenizer.special_tokens_map['''sep_token''']  # `[SEP]`
   elif args.tokenizer_type == "roberta":
      snake_case       :				Optional[Any]		    =RobertaTokenizer.from_pretrained(args.tokenizer_name				)
      snake_case       :				str		    =tokenizer.special_tokens_map['''cls_token''']  # `<s>`
      snake_case       :				Optional[Any]		    =tokenizer.special_tokens_map['''sep_token''']  # `</s>`
   elif args.tokenizer_type == "gpt2":
      snake_case       :				List[str]		    =GPTaTokenizer.from_pretrained(args.tokenizer_name				)
      snake_case       :				List[Any]		    =tokenizer.special_tokens_map['''bos_token''']  # `<|endoftext|>`
      snake_case       :				List[str]		    =tokenizer.special_tokens_map['''eos_token''']  # `<|endoftext|>`

   logger.info(F'''Loading text from {args.file_path}'''				)
   with open(args.file_path	,		'''r'''	,		encoding='''utf8'''				) as fp:
      snake_case       :				int		    =fp.readlines()

   logger.info('''Start encoding'''				)
   logger.info(F'''{len(__a				)} examples to process.'''				)

   snake_case       :				Any		    =[]
   snake_case       :				Union[str, Any]		    =0
   snake_case       :				List[Any]		    =1_00_00
   snake_case       :				int		    =time.time()
   for text in data:
      snake_case       :				Optional[int]		    =F'''{bos} {text.strip()} {sep}'''
      snake_case       :				Tuple		    =tokenizer.encode(__a	,		add_special_tokens=__a				)
      rslt.append(__a				)

      iter += 1
      if iter % interval == 0:
         snake_case       :				Optional[int]		    =time.time()
         logger.info(F'''{iter} examples processed. - {(end-start):.2f}s/{interval}expl'''				)
         snake_case       :				Union[str, Any]		    =time.time()
   logger.info('''Finished binarization'''				)
   logger.info(F'''{len(__a				)} examples processed.'''				)

   snake_case       :				Dict		    =F'''{args.dump_file}.{args.tokenizer_name}.pickle'''
   snake_case       :				Optional[Any]		    =tokenizer.vocab_size
   if vocab_size < (1 << 16):
      snake_case       :				Optional[Any]		    =[np.uintaa(__a				) for d in rslt]
   else:
      snake_case       :				Union[str, Any]		    =[np.intaa(__a				) for d in rslt]
   random.shuffle(rslt_				)
   logger.info(F'''Dump to {dp_file}'''				)
   with open(__a	,		'''wb'''				) as handle:
      pickle.dump(rslt_	,		__a	,		protocol=pickle.HIGHEST_PROTOCOL				)


if __name__ == "__main__":
      main()









'''simple docstring'''
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


__snake_case				 =	{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__snake_case				 =	{
    '''vocab_file''': {
        '''google/electra-small-generator''': (
            '''https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt'''
        ),
        '''google/electra-base-generator''': '''https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt''',
        '''google/electra-large-generator''': (
            '''https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt'''
        ),
        '''google/electra-small-discriminator''': (
            '''https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt'''
        ),
        '''google/electra-base-discriminator''': (
            '''https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt'''
        ),
        '''google/electra-large-discriminator''': (
            '''https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''google/electra-small-generator''': (
            '''https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-base-generator''': (
            '''https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-large-generator''': (
            '''https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-small-discriminator''': (
            '''https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-base-discriminator''': (
            '''https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json'''
        ),
        '''google/electra-large-discriminator''': (
            '''https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json'''
        ),
    },
}

__snake_case				 =	{
    '''google/electra-small-generator''': 512,
    '''google/electra-base-generator''': 512,
    '''google/electra-large-generator''': 512,
    '''google/electra-small-discriminator''': 512,
    '''google/electra-base-discriminator''': 512,
    '''google/electra-large-discriminator''': 512,
}

__snake_case				 =	{
    '''google/electra-small-generator''': {'''do_lower_case''': True},
    '''google/electra-base-generator''': {'''do_lower_case''': True},
    '''google/electra-large-generator''': {'''do_lower_case''': True},
    '''google/electra-small-discriminator''': {'''do_lower_case''': True},
    '''google/electra-base-discriminator''': {'''do_lower_case''': True},
    '''google/electra-large-discriminator''': {'''do_lower_case''': True},
}
class   lowercase      (				A__   ):

		"""simple docstring"""

		_a       					=							VOCAB_FILES_NAMES
		_a       					=							PRETRAINED_VOCAB_FILES_MAP
		_a       					=							PRETRAINED_INIT_CONFIGURATION
		_a       					=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_a       					=							ElectraTokenizer




		def __init__(						self       ,  UpperCamelCase_=None       ,  UpperCamelCase_=None       ,  UpperCamelCase_=True       ,  UpperCamelCase_="[UNK]"       ,  UpperCamelCase_="[SEP]"       ,  UpperCamelCase_="[PAD]"       ,  UpperCamelCase_="[CLS]"       ,  UpperCamelCase_="[MASK]"       ,  UpperCamelCase_=True       ,  UpperCamelCase_=None       ,  **UpperCamelCase_       ,  ):

							'''simple docstring'''


							super().__init__(
							    UpperCamelCase_       ,  tokenizer_file=UpperCamelCase_       ,  do_lower_case=UpperCamelCase_       ,  unk_token=UpperCamelCase_       ,  sep_token=UpperCamelCase_       ,  pad_token=UpperCamelCase_       ,  cls_token=UpperCamelCase_       ,  mask_token=UpperCamelCase_       ,  tokenize_chinese_chars=UpperCamelCase_       ,  strip_accents=UpperCamelCase_       ,  **UpperCamelCase_       ,  )

							UpperCamelCase__  :str				   =					json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
							if (
							    normalizer_state.get('''lowercase'''       ,  UpperCamelCase_				) != do_lower_case
							    or normalizer_state.get('''strip_accents'''       ,  UpperCamelCase_				) != strip_accents
							    or normalizer_state.get('''handle_chinese_chars'''       ,  UpperCamelCase_				) != tokenize_chinese_chars
							):
												UpperCamelCase__  :Any				   =					getattr(UpperCamelCase_       ,  normalizer_state.pop('''type'''				)				)
												UpperCamelCase__  :Any				   =					do_lower_case
												UpperCamelCase__  :Dict				   =					strip_accents
												UpperCamelCase__  :Optional[int]				   =					tokenize_chinese_chars
												UpperCamelCase__  :Dict				   =					normalizer_class(**UpperCamelCase_				)

							UpperCamelCase__  :str				   =					do_lower_case




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_=None				):

							'''simple docstring'''


							UpperCamelCase__  :Dict				   =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

							return output




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_ = None				):

							'''simple docstring'''


							UpperCamelCase__  :Union[str, Any]				   =					[self.sep_token_id]
							UpperCamelCase__  :str				   =					[self.cls_token_id]
							if token_ids_a is None:
												return len(cls + token_ids_a + sep				) * [0]
							return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_ = None				):

							'''simple docstring'''


							UpperCamelCase__  :str				   =					self._tokenizer.model.save(UpperCamelCase_       ,  name=UpperCamelCase_				)
							return tuple(UpperCamelCase_				)