


'''simple docstring'''
import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
__snake_case				 =	logging.getLogger(__name__)


def      a (     )    ->      List[Any]:



					'''simple docstring'''






					UpperCamelCase__  :Dict				   =					argparse.ArgumentParser(
					    description='''Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'''  )
					parser.add_argument('''--file_path'''					,   type=__a					,   default='''data/dump.txt'''					,   help='''The path to the data.'''  )
					parser.add_argument('''--tokenizer_type'''					,   type=__a					,   default='''bert'''					,   choices=['''bert''', '''roberta''', '''gpt2''']  )
					parser.add_argument('''--tokenizer_name'''					,   type=__a					,   default='''bert-base-uncased'''					,   help='''The tokenizer to use.'''  )
					parser.add_argument('''--dump_file'''					,   type=__a					,   default='''data/dump'''					,   help='''The dump file prefix.'''  )
					UpperCamelCase__  :Optional[int]				   =					parser.parse_args()

					logger.info(f'''Loading Tokenizer ({args.tokenizer_name})'''  )
					if args.tokenizer_type == "bert":
										UpperCamelCase__  :int				   =					BertTokenizer.from_pretrained(args.tokenizer_name  )
										UpperCamelCase__  :Union[str, Any]				   =					tokenizer.special_tokens_map['''cls_token''']  # `[CLS]`
										UpperCamelCase__  :Tuple				   =					tokenizer.special_tokens_map['''sep_token''']  # `[SEP]`
					elif args.tokenizer_type == "roberta":
										UpperCamelCase__  :Optional[Any]				   =					RobertaTokenizer.from_pretrained(args.tokenizer_name  )
										UpperCamelCase__  :str				   =					tokenizer.special_tokens_map['''cls_token''']  # `<s>`
										UpperCamelCase__  :Optional[Any]				   =					tokenizer.special_tokens_map['''sep_token''']  # `</s>`
					elif args.tokenizer_type == "gpt2":
										UpperCamelCase__  :List[str]				   =					GPTaTokenizer.from_pretrained(args.tokenizer_name  )
										UpperCamelCase__  :List[Any]				   =					tokenizer.special_tokens_map['''bos_token''']  # `<|endoftext|>`
										UpperCamelCase__  :List[str]				   =					tokenizer.special_tokens_map['''eos_token''']  # `<|endoftext|>`

					logger.info(f'''Loading text from {args.file_path}'''  )
					with open(args.file_path					,   '''r'''					,   encoding='''utf8'''  ) as fp:
										UpperCamelCase__  :int				   =					fp.readlines()

					logger.info('''Start encoding'''  )
					logger.info(f'''{len(__a  )} examples to process.'''  )

					UpperCamelCase__  :Any				   =					[]
					UpperCamelCase__  :Union[str, Any]				   =					0
					UpperCamelCase__  :List[Any]				   =					10000
					UpperCamelCase__  :int				   =					time.time()
					for text in data:
										UpperCamelCase__  :Optional[int]				   =					f'''{bos} {text.strip()} {sep}'''
										UpperCamelCase__  :Tuple				   =					tokenizer.encode(__a					,   add_special_tokens=__a  )
										rslt.append(__a  )

										iter += 1
										if iter % interval == 0:
															UpperCamelCase__  :Optional[int]				   =					time.time()
															logger.info(f'''{iter} examples processed. - {(end-start):.2f}s/{interval}expl'''  )
															UpperCamelCase__  :Union[str, Any]				   =					time.time()
					logger.info('''Finished binarization'''  )
					logger.info(f'''{len(__a  )} examples processed.'''  )

					UpperCamelCase__  :Dict				   =					f'''{args.dump_file}.{args.tokenizer_name}.pickle'''
					UpperCamelCase__  :Optional[Any]				   =					tokenizer.vocab_size
					if vocab_size < (1 << 16):
										UpperCamelCase__  :Optional[Any]				   =					[np.uintaa(__a  ) for d in rslt]
					else:
										UpperCamelCase__  :Union[str, Any]				   =					[np.intaa(__a  ) for d in rslt]
					random.shuffle(rslt_  )
					logger.info(f'''Dump to {dp_file}'''  )
					with open(__a					,   '''wb'''  ) as handle:
										pickle.dump(rslt_					,   __a					,   protocol=pickle.HIGHEST_PROTOCOL  )


if __name__ == "__main__":
		main()



'''simple docstring'''
import inspect
import os
import unittest

import torch

import accelerate
from accelerate import debug_launcher
from accelerate.test_utils import (
    execute_subprocess_async,
    require_cpu,
    require_huggingface_suite,
    require_multi_gpu,
    require_single_gpu,
)
from accelerate.utils import patch_environment
@require_huggingface_suite
class   lowercase      (				unittest.TestCase   ):

		"""simple docstring"""

		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Any				   =					inspect.getfile(accelerate.test_utils				)
							UpperCamelCase__  :Any				   =					os.path.sep.join(
							    mod_file.split(os.path.sep				)[:-1] + ['''scripts''', '''external_deps''', '''test_metrics.py''']				)

							from accelerate.test_utils.scripts.external_deps import test_metrics  # noqa: F401

							UpperCamelCase__  :List[str]				   =					test_metrics




		@require_cpu
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							debug_launcher(self.test_metrics.main       ,  num_processes=1				)




		@require_cpu
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							debug_launcher(self.test_metrics.main				)




		@require_single_gpu
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							self.test_metrics.main()




		@require_multi_gpu
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							print(F'''Found {torch.cuda.device_count()} devices.'''				)
							UpperCamelCase__  :Optional[Any]				   =					['''torchrun''', F'''--nproc_per_node={torch.cuda.device_count()}''', self.test_file_path]
							with patch_environment(omp_num_threads=1				):
												execute_subprocess_async(UpperCamelCase_       ,  env=os.environ.copy()				)