




'''simple docstring'''



import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
UpperCAmelCase_							:	Any	     =		logging.getLogger(__name__)







def       _UpperCamelCase			()-> Union[str, Any]:




     '''simple docstring'''





     __snake_case            =	argparse.ArgumentParser(
         description='''Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'''						)
     parser.add_argument('''--file_path''' ,						type=_lowerCamelCase ,						default='''data/dump.txt''' ,						help='''The path to the data.'''						)
     parser.add_argument('''--tokenizer_type''' ,						type=_lowerCamelCase ,						default='''bert''' ,						choices=['''bert''', '''roberta''', '''gpt2''']						)
     parser.add_argument('''--tokenizer_name''' ,						type=_lowerCamelCase ,						default='''bert-base-uncased''' ,						help='''The tokenizer to use.'''						)
     parser.add_argument('''--dump_file''' ,						type=_lowerCamelCase ,						default='''data/dump''' ,						help='''The dump file prefix.'''						)
     __snake_case            =	parser.parse_args()

     logger.info(f'''Loading Tokenizer ({args.tokenizer_name})'''						)
     if args.tokenizer_type == "bert":
          __snake_case            =	BertTokenizer.from_pretrained(args.tokenizer_name						)
          __snake_case            =	tokenizer.special_tokens_map['''cls_token''']  # `[CLS]`
          __snake_case            =	tokenizer.special_tokens_map['''sep_token''']  # `[SEP]`
     elif args.tokenizer_type == "roberta":
          __snake_case            =	RobertaTokenizer.from_pretrained(args.tokenizer_name						)
          __snake_case            =	tokenizer.special_tokens_map['''cls_token''']  # `<s>`
          __snake_case            =	tokenizer.special_tokens_map['''sep_token''']  # `</s>`
     elif args.tokenizer_type == "gpt2":
          __snake_case            =	GPTaTokenizer.from_pretrained(args.tokenizer_name						)
          __snake_case            =	tokenizer.special_tokens_map['''bos_token''']  # `<|endoftext|>`
          __snake_case            =	tokenizer.special_tokens_map['''eos_token''']  # `<|endoftext|>`

     logger.info(f'''Loading text from {args.file_path}'''						)
     with open(args.file_path ,						'''r''' ,						encoding='''utf8'''						) as fp:
          __snake_case            =	fp.readlines()

     logger.info('''Start encoding'''						)
     logger.info(f'''{len(_lowerCamelCase						)} examples to process.'''						)

     __snake_case            =	[]
     __snake_case            =	0
     __snake_case            =	1_00_00
     __snake_case            =	time.time()
     for text in data:
          __snake_case            =	f'''{bos} {text.strip()} {sep}'''
          __snake_case            =	tokenizer.encode(_lowerCamelCase ,						add_special_tokens=_lowerCamelCase						)
          rslt.append(_lowerCamelCase						)

          iter += 1
          if iter % interval == 0:
               __snake_case            =	time.time()
               logger.info(f'''{iter} examples processed. - {(end-start):.2f}s/{interval}expl'''						)
               __snake_case            =	time.time()
     logger.info('''Finished binarization'''						)
     logger.info(f'''{len(_lowerCamelCase						)} examples processed.'''						)

     __snake_case            =	f'''{args.dump_file}.{args.tokenizer_name}.pickle'''
     __snake_case            =	tokenizer.vocab_size
     if vocab_size < (1 << 16):
          __snake_case            =	[np.uintaa(_lowerCamelCase						) for d in rslt]
     else:
          __snake_case            =	[np.intaa(_lowerCamelCase						) for d in rslt]
     random.shuffle(rslt_						)
     logger.info(f'''Dump to {dp_file}'''						)
     with open(_lowerCamelCase ,						'''wb'''						) as handle:
          pickle.dump(rslt_ ,						_lowerCamelCase ,						protocol=pickle.HIGHEST_PROTOCOL						)


if __name__ == "__main__":
     main()







'''simple docstring'''


import importlib
import os
import sys


# This is required to make the module import works (when the python process is running from the root of the repo)
sys.path.append(".")





def    lowerCAmelCase       (		UpperCamelCase__					: Dict							):


	"""simple docstring"""
	__UpperCAmelCase      					=      test_file.split(os.path.sep							)
	if components[0:2] != ["tests", "models"]:
		raise ValueError(
		    '''`test_file` should start with `tests/models/` (with `/` being the OS specific path separator). Got '''
		    f"""{test_file} instead."""							)
	__UpperCAmelCase      					=      components[-1]
	if not test_fn.endswith('''py'''							):
		raise ValueError(f"""`test_file` should be a python file. Got {test_fn} instead."""							)
	if not test_fn.startswith('''test_modeling_'''							):
		raise ValueError(
		    f"""`test_file` should point to a file name of the form `test_modeling_*.py`. Got {test_fn} instead."""							)

	__UpperCAmelCase      					=      components[:-1] + [test_fn.replace('''.py'''     ,       ''''''							)]
	__UpperCAmelCase      					=      '''.'''.join(UpperCamelCase__							)

	return test_module_path





def    lowerCAmelCase       (		UpperCamelCase__					: int							):


	"""simple docstring"""
	__UpperCAmelCase      					=      get_module_path(UpperCamelCase__							)
	__UpperCAmelCase      					=      importlib.import_module(UpperCamelCase__							)

	return test_module





def    lowerCAmelCase       (		UpperCamelCase__					: Tuple							):


	"""simple docstring"""
	__UpperCAmelCase      					=      []
	__UpperCAmelCase      					=      get_test_module(UpperCamelCase__							)
	for attr in dir(UpperCamelCase__							):
		if attr.endswith('''ModelTester'''							):
			tester_classes.append(getattr(UpperCamelCase__     ,       UpperCamelCase__							)							)

    # sort with class names
	return sorted(UpperCamelCase__     ,       key=lambda UpperCamelCase__							: x.__name__							)





def    lowerCAmelCase       (		UpperCamelCase__					: List[Any]							):


	"""simple docstring"""
	__UpperCAmelCase      					=      []
	__UpperCAmelCase      					=      get_test_module(UpperCamelCase__							)
	for attr in dir(UpperCamelCase__							):
		__UpperCAmelCase      					=      getattr(UpperCamelCase__     ,       UpperCamelCase__							)
		# (TF/Flax)ModelTesterMixin is also an attribute in specific model test module. Let's exclude them by checking
		# `all_model_classes` is not empty (which also excludes other special classes).
		__UpperCAmelCase      					=      getattr(UpperCamelCase__     ,       '''all_model_classes'''     ,       []							)
		if len(UpperCamelCase__							) > 0:
			test_classes.append(UpperCamelCase__							)

    # sort with class names
	return sorted(UpperCamelCase__     ,       key=lambda UpperCamelCase__							: x.__name__							)





def    lowerCAmelCase       (		UpperCamelCase__					: str							):


	"""simple docstring"""
	__UpperCAmelCase      					=      get_test_classes(UpperCamelCase__							)
	__UpperCAmelCase      					=      set()
	for test_class in test_classes:
		model_classes.update(test_class.all_model_classes							)

	# sort with class names
	return sorted(UpperCamelCase__     ,       key=lambda UpperCamelCase__							: x.__name__							)





def    lowerCAmelCase       (		UpperCamelCase__					: Any							):


	"""simple docstring"""
	__UpperCAmelCase      					=      test_class()
	if hasattr(UpperCamelCase__     ,       '''setUp'''							):
		test.setUp()

	__UpperCAmelCase      					=      None
	if hasattr(UpperCamelCase__     ,       '''model_tester'''							):
		# `(TF/Flax)ModelTesterMixin` has this attribute default to `None`. Let's skip this case.
		if test.model_tester is not None:
			__UpperCAmelCase      					=      test.model_tester.__class__

	return model_tester





def    lowerCAmelCase       (		UpperCamelCase__					: Dict     ,       UpperCamelCase__					: int							):


	"""simple docstring"""
	__UpperCAmelCase      					=      get_test_classes(UpperCamelCase__							)

	__UpperCAmelCase      					=      []
	for test_class in test_classes:
		if model_class in test_class.all_model_classes:
			target_test_classes.append(UpperCamelCase__							)

    # sort with class names
	return sorted(UpperCamelCase__     ,       key=lambda UpperCamelCase__							: x.__name__							)





def    lowerCAmelCase       (		UpperCamelCase__					: Tuple     ,       UpperCamelCase__					: Any							):


	"""simple docstring"""
	__UpperCAmelCase      					=      get_test_classes_for_model(UpperCamelCase__     ,       UpperCamelCase__							)

	__UpperCAmelCase      					=      []
	for test_class in test_classes:
		__UpperCAmelCase      					=      get_model_tester_from_test_class(UpperCamelCase__							)
		if tester_class is not None:
			tester_classes.append(UpperCamelCase__							)

    # sort with class names
	return sorted(UpperCamelCase__     ,       key=lambda UpperCamelCase__							: x.__name__							)





def    lowerCAmelCase       (		UpperCamelCase__					: int							):


	"""simple docstring"""
	__UpperCAmelCase      					=      get_test_classes(UpperCamelCase__							)
	__UpperCAmelCase      					=      {test_class: get_model_tester_from_test_class(UpperCamelCase__							) for test_class in test_classes}
	return test_tester_mapping





def    lowerCAmelCase       (		UpperCamelCase__					: str							):


	"""simple docstring"""
	__UpperCAmelCase      					=      get_model_classes(UpperCamelCase__							)
	__UpperCAmelCase      					=      {
	    model_class: get_test_classes_for_model(UpperCamelCase__     ,       UpperCamelCase__							) for model_class in model_classes
	}
	return model_test_mapping





def    lowerCAmelCase       (		UpperCamelCase__					: Optional[int]							):


	"""simple docstring"""
	__UpperCAmelCase      					=      get_model_classes(UpperCamelCase__							)
	__UpperCAmelCase      					=      {
	    model_class: get_tester_classes_for_model(UpperCamelCase__     ,       UpperCamelCase__							) for model_class in model_classes
	}
	return model_to_tester_mapping





def    lowerCAmelCase       (		UpperCamelCase__					: Tuple							):


	"""simple docstring"""
	if isinstance(UpperCamelCase__     ,       UpperCamelCase__							):
		return o
	elif isinstance(UpperCamelCase__     ,       UpperCamelCase__							):
		return o.__name__
	elif isinstance(UpperCamelCase__     ,       (list, tuple)							):
		return [to_json(UpperCamelCase__							) for x in o]
	elif isinstance(UpperCamelCase__     ,       UpperCamelCase__							):
		return {to_json(UpperCamelCase__							): to_json(UpperCamelCase__							) for k, v in o.items()}
	else:
		return o
