





"""simple docstring"""


# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os
import subprocess

from packaging.version import Version, parse

from accelerate.commands.config.config_args import default_config_file, load_config_from_file


__magic_name__       :					Any						=			"""Run commands across TPU VMs for initial setup before running `accelerate launch`."""

def      UpperCamelCase							(SCREAMING_SNAKE_CASE=None     ):
						if subparsers is not None:
												UpperCamelCase				:		Optional[Any]				   =			subparsers.add_parser("""tpu-config"""							,     description=_description     )
						else:
												UpperCamelCase				:		Optional[Any]				   =			argparse.ArgumentParser("""Accelerate tpu-config command"""							,     description=_description     )
						# Core arguments
						UpperCamelCase				:		Any				   =			parser.add_argument_group(
						    """Config Arguments"""							,     """Arguments that can be configured through `accelerate config`."""     )
						config_args.add_argument(
						    """--config_file"""							,     type=SCREAMING_SNAKE_CASE							,     default=SCREAMING_SNAKE_CASE							,     help="""Path to the config file to use for accelerate."""							,     )
						config_args.add_argument(
						    """--tpu_name"""							,     default=SCREAMING_SNAKE_CASE							,     help="""The name of the TPU to use. If not specified, will use the TPU specified in the config file."""							,     )
						config_args.add_argument(
						    """--tpu_zone"""							,     default=SCREAMING_SNAKE_CASE							,     help="""The zone of the TPU to use. If not specified, will use the zone specified in the config file."""							,     )
						UpperCamelCase				:		int				   =			parser.add_argument_group("""TPU Arguments"""							,     """Arguments for options ran inside the TPU."""     )
						pod_args.add_argument(
						    """--use_alpha"""							,     action="""store_true"""							,     help="""Whether to use `gcloud alpha` when running the TPU training script instead of `gcloud`."""							,     )
						pod_args.add_argument(
						    """--command_file"""							,     default=SCREAMING_SNAKE_CASE							,     help="""The path to the file containing the commands to run on the pod on startup."""							,     )
						pod_args.add_argument(
						    """--command"""							,     action="""append"""							,     nargs="""+"""							,     help="""A command to run on the pod. Can be passed multiple times."""							,     )
						pod_args.add_argument(
						    """--install_accelerate"""							,     action="""store_true"""							,     help="""Whether to install accelerate on the pod. Defaults to False."""							,     )
						pod_args.add_argument(
						    """--accelerate_version"""							,     default="""latest"""							,     help="""The version of accelerate to install on the pod. If not specified, will use the latest pypi version. Specify 'dev' to install from GitHub."""							,     )
						pod_args.add_argument(
						    """--debug"""							,     action="""store_true"""							,     help="""If set, will print the command that would be run instead of running it."""     )

						if subparsers is not None:
												parser.set_defaults(func=SCREAMING_SNAKE_CASE     )
						return parser

def      UpperCamelCase							(SCREAMING_SNAKE_CASE     ):
						UpperCamelCase				:		Any				   =			None

						# Get the default from the config file if it exists.
						if args.config_file is not None or os.path.isfile(SCREAMING_SNAKE_CASE     ):
												UpperCamelCase				:		Union[str, Any]				   =			load_config_from_file(args.config_file     )
												if not args.command_file and defaults.command_file is not None and not args.command:
																		UpperCamelCase				:		Tuple				   =			defaults.command_file
												if not args.command and defaults.commands is not None:
																		UpperCamelCase				:		Optional[int]				   =			defaults.commands
												if not args.tpu_name:
																		UpperCamelCase				:		Dict				   =			defaults.tpu_name
												if not args.tpu_zone:
																		UpperCamelCase				:		Optional[Any]				   =			defaults.tpu_zone
						if args.accelerate_version == "dev":
												UpperCamelCase				:		int				   =			"""git+https://github.com/huggingface/accelerate.git"""
						elif args.accelerate_version == "latest":
												UpperCamelCase				:		Dict				   =			"""accelerate -U"""
						elif isinstance(parse(args.accelerate_version     )							,     SCREAMING_SNAKE_CASE     ):
												UpperCamelCase				:		int				   =			f"""accelerate=={args.accelerate_version}"""

						if not args.command_file and not args.command:
												raise ValueError("""You must specify either a command file or a command to run on the pod."""     )

						if args.command_file:
												with open(args.command_file							,     """r"""     ) as f:
																		UpperCamelCase				:		Optional[Any]				   =			[f.read().splitlines()]

    # To turn list of lists into list of strings
						if isinstance(args.command[0]							,     SCREAMING_SNAKE_CASE     ):
												UpperCamelCase				:		Union[str, Any]				   =			[line for cmd in args.command for line in cmd]
						# Default to the shared folder and install accelerate
						UpperCamelCase				:		Optional[int]				   =			["""cd /usr/share"""]
						if args.install_accelerate:
												new_cmd += [f"""pip install {args.accelerate_version}"""]
						new_cmd += args.command
						UpperCamelCase				:		Tuple				   =			"""; """.join(SCREAMING_SNAKE_CASE     )

						# Then send it to gcloud
						# Eventually try to use google-api-core to do this instead of subprocess
						UpperCamelCase				:		int				   =			["""gcloud"""]
						if args.use_alpha:
												cmd += ["alpha"]
						cmd += [
						    "compute",
						    "tpus",
						    "tpu-vm",
						    "ssh",
						    args.tpu_name,
						    "--zone",
						    args.tpu_zone,
						    "--command",
						    args.command,
						    "--worker",
						    "all",
						]
						if args.debug:
												print(f"""Running {" ".join(SCREAMING_SNAKE_CASE     )}"""     )
												return
						subprocess.run(SCREAMING_SNAKE_CASE     )
						print("""Successfully setup pod."""     )

def      UpperCamelCase							():
						UpperCamelCase				:		Any				   =			tpu_command_parser()
						UpperCamelCase				:		int				   =			parser.parse_args()

						tpu_command_launcher(SCREAMING_SNAKE_CASE     )










"""simple docstring"""
import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
__UpperCamelCase     :						str						       =  logging.getLogger(__name__)

def 		_SCREAMING_SNAKE_CASE			():
 lowerCAmelCase    				= argparse.ArgumentParser(
     description='Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'			)
 parser.add_argument('--file_path'  ,     type=_UpperCAmelCase  ,     default='data/dump.txt'  ,     help='The path to the data.'			)
 parser.add_argument('--tokenizer_type'  ,     type=_UpperCAmelCase  ,     default='bert'  ,     choices=['bert', 'roberta', 'gpt2']			)
 parser.add_argument('--tokenizer_name'  ,     type=_UpperCAmelCase  ,     default='bert-base-uncased'  ,     help='The tokenizer to use.'			)
 parser.add_argument('--dump_file'  ,     type=_UpperCAmelCase  ,     default='data/dump'  ,     help='The dump file prefix.'			)
 lowerCAmelCase    				= parser.parse_args()

 logger.info(F'Loading Tokenizer ({args.tokenizer_name})'			)
 if args.tokenizer_type == "bert":
  lowerCAmelCase    				= BertTokenizer.from_pretrained(args.tokenizer_name			)
  lowerCAmelCase    				= tokenizer.special_tokens_map['cls_token']  # `[CLS]`
  lowerCAmelCase    				= tokenizer.special_tokens_map['sep_token']  # `[SEP]`
 elif args.tokenizer_type == "roberta":
  lowerCAmelCase    				= RobertaTokenizer.from_pretrained(args.tokenizer_name			)
  lowerCAmelCase    				= tokenizer.special_tokens_map['cls_token']  # `<s>`
  lowerCAmelCase    				= tokenizer.special_tokens_map['sep_token']  # `</s>`
 elif args.tokenizer_type == "gpt2":
  lowerCAmelCase    				= GPTaTokenizer.from_pretrained(args.tokenizer_name			)
  lowerCAmelCase    				= tokenizer.special_tokens_map['bos_token']  # `<|endoftext|>`
  lowerCAmelCase    				= tokenizer.special_tokens_map['eos_token']  # `<|endoftext|>`

 logger.info(F'Loading text from {args.file_path}'			)
 with open(args.file_path  ,     'r'  ,     encoding='utf8'			) as fp:
  lowerCAmelCase    				= fp.readlines()

 logger.info('Start encoding'			)
 logger.info(F'{len(_UpperCAmelCase			)} examples to process.'			)

 lowerCAmelCase    				= []
 lowerCAmelCase    				= 0
 lowerCAmelCase    				= 1_0000
 lowerCAmelCase    				= time.time()
 for text in data:
  lowerCAmelCase    				= F'{bos} {text.strip()} {sep}'
  lowerCAmelCase    				= tokenizer.encode(_UpperCAmelCase  ,     add_special_tokens=_UpperCAmelCase			)
  rslt.append(_UpperCAmelCase			)

  iter += 1
  if iter % interval == 0:
   lowerCAmelCase    				= time.time()
   logger.info(F'{iter} examples processed. - {(end-start):.2f}s/{interval}expl'			)
   lowerCAmelCase    				= time.time()
 logger.info('Finished binarization'			)
 logger.info(F'{len(_UpperCAmelCase			)} examples processed.'			)

 lowerCAmelCase    				= F'{args.dump_file}.{args.tokenizer_name}.pickle'
 lowerCAmelCase    				= tokenizer.vocab_size
 if vocab_size < (1 << 16):
  lowerCAmelCase    				= [np.uintaa(_UpperCAmelCase			) for d in rslt]
 else:
  lowerCAmelCase    				= [np.intaa(_UpperCAmelCase			) for d in rslt]
 random.shuffle(rslt_			)
 logger.info(F'Dump to {dp_file}'			)
 with open(_UpperCAmelCase  ,     'wb'			) as handle:
  pickle.dump(rslt_  ,     _UpperCAmelCase  ,     protocol=pickle.HIGHEST_PROTOCOL			)


if __name__ == "__main__":
       main()





