






"""simple docstring"""






import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class       snake_case_       (    _lowerCamelCase    ,							unittest.TestCase       ):



	"""simple docstring"""

	SCREAMING_SNAKE_CASE_:							Dict   			=					OpenAIGPTTokenizer
	SCREAMING_SNAKE_CASE_:							str   			=					OpenAIGPTTokenizerFast
	SCREAMING_SNAKE_CASE_:							int   			=					True
	SCREAMING_SNAKE_CASE_:							Union[str, Any]   			=					False


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					super().setUp()

					# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
					A__							 =       [
					    'l',
					    'o',
					    'w',
					    'e',
					    'r',
					    's',
					    't',
					    'i',
					    'd',
					    'n',
					    'w</w>',
					    'r</w>',
					    't</w>',
					    'lo',
					    'low',
					    'er</w>',
					    'low</w>',
					    'lowest</w>',
					    'newer</w>',
					    'wider</w>',
					    '<unk>',
					]
					A__							 =       dict(zip(__a  ,					range(len(__a						)						)						)						)
					A__							 =       ['#version: 0.2', 'l o', 'lo w', 'e r</w>', '']

					A__							 =       os.path.join(self.tmpdirname  ,					VOCAB_FILES_NAMES['vocab_file']						)
					A__							 =       os.path.join(self.tmpdirname  ,					VOCAB_FILES_NAMES['merges_file']						)
					with open(self.vocab_file  ,					'w'						) as fp:
									fp.write(json.dumps(__a						)						)
					with open(self.merges_file  ,					'w'						) as fp:
									fp.write('\n'.join(__a						)						)


	def       _UpperCAmelCase						(      self  ,					__a						):


					"""simple docstring"""

					return "lower newer", "lower newer"


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       OpenAIGPTTokenizer(self.vocab_file  ,					self.merges_file						)

					A__							 =       'lower'
					A__							 =       ['low', 'er</w>']
					A__							 =       tokenizer.tokenize(__a						)
					self.assertListEqual(__a  ,					__a						)

					A__							 =       tokens + ['<unk>']
					A__							 =       [14, 15, 20]
					self.assertListEqual(tokenizer.convert_tokens_to_ids(__a						)  ,					__a						)


	def       _UpperCAmelCase						(      self  ,					__a=15						):


					"""simple docstring"""

					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
													A__							 =       self.rust_tokenizer_class.from_pretrained(__a  ,					**__a						)

													# Simple input
													A__							 =       'This is a simple input'
													A__							 =       ['This is a simple input 1', 'This is a simple input 2']
													A__							 =       ('This is a simple input', 'This is a pair')
													A__							 =       [
													    ('This is a simple input 1', 'This is a simple input 2'),
													    ('This is a simple pair 1', 'This is a simple pair 2'),
													]

													# Simple input tests
													self.assertRaises(__a  ,					tokenizer_r.encode  ,					__a  ,					max_length=__a  ,					padding='max_length'						)

													# Simple input
													self.assertRaises(__a  ,					tokenizer_r.encode_plus  ,					__a  ,					max_length=__a  ,					padding='max_length'						)

													# Simple input
													self.assertRaises(
													    __a  ,					tokenizer_r.batch_encode_plus  ,					__a  ,					max_length=__a  ,					padding='max_length'  ,					)

													# Pair input
													self.assertRaises(__a  ,					tokenizer_r.encode  ,					__a  ,					max_length=__a  ,					padding='max_length'						)

													# Pair input
													self.assertRaises(__a  ,					tokenizer_r.encode_plus  ,					__a  ,					max_length=__a  ,					padding='max_length'						)

													# Pair input
													self.assertRaises(
													    __a  ,					tokenizer_r.batch_encode_plus  ,					__a  ,					max_length=__a  ,					padding='max_length'  ,					)






	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					pass





@require_ftfy
@require_spacy
@require_tokenizers
class       snake_case_       (    _lowerCamelCase       ):



	"""simple docstring"""

	pass










"""simple docstring"""






import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
SCREAMING_SNAKE_CASE						:						Optional[int]					   =       logging.getLogger(__name__)



def  __lowerCamelCase					(					):
				A__							 =       argparse.ArgumentParser(
				    description='Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'				)
				parser.add_argument('--file_path'       ,type=lowerCAmelCase__       ,default='data/dump.txt'       ,help='The path to the data.'				)
				parser.add_argument('--tokenizer_type'       ,type=lowerCAmelCase__       ,default='bert'       ,choices=['bert', 'roberta', 'gpt2']				)
				parser.add_argument('--tokenizer_name'       ,type=lowerCAmelCase__       ,default='bert-base-uncased'       ,help='The tokenizer to use.'				)
				parser.add_argument('--dump_file'       ,type=lowerCAmelCase__       ,default='data/dump'       ,help='The dump file prefix.'				)
				A__							 =       parser.parse_args()

				logger.info(f'''Loading Tokenizer ({args.tokenizer_name})'''				)
				if args.tokenizer_type == "bert":
								A__							 =       BertTokenizer.from_pretrained(args.tokenizer_name				)
								A__							 =       tokenizer.special_tokens_map['cls_token']  # `[CLS]`
								A__							 =       tokenizer.special_tokens_map['sep_token']  # `[SEP]`
				elif args.tokenizer_type == "roberta":
								A__							 =       RobertaTokenizer.from_pretrained(args.tokenizer_name				)
								A__							 =       tokenizer.special_tokens_map['cls_token']  # `<s>`
								A__							 =       tokenizer.special_tokens_map['sep_token']  # `</s>`
				elif args.tokenizer_type == "gpt2":
								A__							 =       GPTaTokenizer.from_pretrained(args.tokenizer_name				)
								A__							 =       tokenizer.special_tokens_map['bos_token']  # `<|endoftext|>`
								A__							 =       tokenizer.special_tokens_map['eos_token']  # `<|endoftext|>`

				logger.info(f'''Loading text from {args.file_path}'''				)
				with open(args.file_path       ,'r'       ,encoding='utf8'				) as fp:
								A__							 =       fp.readlines()

				logger.info('Start encoding'				)
				logger.info(f'''{len(lowerCAmelCase__				)} examples to process.'''				)

				A__							 =       []
				A__							 =       0
				A__							 =       1_0000
				A__							 =       time.time()
				for text in data:
								A__							 =       f'''{bos} {text.strip()} {sep}'''
								A__							 =       tokenizer.encode(lowerCAmelCase__       ,add_special_tokens=lowerCAmelCase__				)
								rslt.append(lowerCAmelCase__				)

								iter += 1
								if iter % interval == 0:
												A__							 =       time.time()
												logger.info(f'''{iter} examples processed. - {(end-start):.2f}s/{interval}expl'''				)
												A__							 =       time.time()
				logger.info('Finished binarization'				)
				logger.info(f'''{len(lowerCAmelCase__				)} examples processed.'''				)

				A__							 =       f'''{args.dump_file}.{args.tokenizer_name}.pickle'''
				A__							 =       tokenizer.vocab_size
				if vocab_size < (1 << 16):
								A__							 =       [np.uintaa(lowerCAmelCase__				) for d in rslt]
				else:
								A__							 =       [np.intaa(lowerCAmelCase__				) for d in rslt]
				random.shuffle(rslt_				)
				logger.info(f'''Dump to {dp_file}'''				)
				with open(lowerCAmelCase__       ,'wb'				) as handle:
								pickle.dump(rslt_       ,lowerCAmelCase__       ,protocol=pickle.HIGHEST_PROTOCOL				)


if __name__ == "__main__":
					main()


