





import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp





def    __UpperCamelCase      (    ) ->List[str]:
					lowerCamelCase__       =       ArgumentParser(
					    description=(
					        "PyTorch TPU distributed training launch "
					        "helper utility that will spawn up "
					        "multiple distributed processes"
					    ))

					# Optional arguments for the launch helper
					parser.add_argument("--num_cores",     type=a,     default=1,     help="Number of TPU cores to use (1 or 8).")

					# positional
					parser.add_argument(
					    "training_script",     type=a,     help=(
					        "The full path to the single TPU training "
					        "program/script to be launched in parallel, "
					        "followed by all the arguments for the "
					        "training script"
					    ),     )

					# rest from the training program
					parser.add_argument("training_script_args",     nargs=a)

					return parser.parse_args()





def    __UpperCamelCase      (    ) ->List[str]:
					lowerCamelCase__       =       parse_args()

					# Import training_script as a module.
					lowerCamelCase__       =       Path(args.training_script)
					sys.path.append(str(script_fpath.parent.resolve()))
					lowerCamelCase__       =       script_fpath.stem
					lowerCamelCase__       =       importlib.import_module(a)

					# Patch sys.argv
					lowerCamelCase__       =       [args.training_script] + args.training_script_args + ["--tpu_num_cores", str(args.num_cores)]
					xmp.spawn(mod._mp_fn,     args=(),     nprocs=args.num_cores)


if __name__ == "__main__":
				main()








from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig


A_												=     {
    "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/config.json",
    "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/config.json",
    "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/config.json",
    "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/config.json",
    "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/config.json",
    "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/config.json",
    "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/config.json",
    "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/config.json",
}

class SCREAMING_SNAKE_CASE_ (					lowercase_   ):





					"""simple docstring"""


					A__				 =    "albert"





					def __init__( self	,  _lowerCAmelCase=3_0000	,  _lowerCAmelCase=128	,  _lowerCAmelCase=4096	,  _lowerCAmelCase=12	,  _lowerCAmelCase=1	,  _lowerCAmelCase=64	,  _lowerCAmelCase=1_6384	,  _lowerCAmelCase=1	,  _lowerCAmelCase="gelu_new"	,  _lowerCAmelCase=0	,  _lowerCAmelCase=0	,  _lowerCAmelCase=512	,  _lowerCAmelCase=2	,  _lowerCAmelCase=0.02	,  _lowerCAmelCase=1E-12	,  _lowerCAmelCase=0.1	,  _lowerCAmelCase="absolute"	,  _lowerCAmelCase=0	,  _lowerCAmelCase=2	,  _lowerCAmelCase=3	,  **_lowerCAmelCase	,  ):
										super().__init__(pad_token_id=_lowerCAmelCase	,  bos_token_id=_lowerCAmelCase	,  eos_token_id=_lowerCAmelCase	,  **_lowerCAmelCase	)

										lowerCamelCase__       =       vocab_size
										lowerCamelCase__       =       embedding_size
										lowerCamelCase__       =       hidden_size
										lowerCamelCase__       =       num_hidden_layers
										lowerCamelCase__       =       num_hidden_groups
										lowerCamelCase__       =       num_attention_heads
										lowerCamelCase__       =       inner_group_num
										lowerCamelCase__       =       hidden_act
										lowerCamelCase__       =       intermediate_size
										lowerCamelCase__       =       hidden_dropout_prob
										lowerCamelCase__       =       attention_probs_dropout_prob
										lowerCamelCase__       =       max_position_embeddings
										lowerCamelCase__       =       type_vocab_size
										lowerCamelCase__       =       initializer_range
										lowerCamelCase__       =       layer_norm_eps
										lowerCamelCase__       =       classifier_dropout_prob
										lowerCamelCase__       =       position_embedding_type

class SCREAMING_SNAKE_CASE_ (					lowercase_   ):





					"""simple docstring"""


					@property
					def 					__magic_name__ ( self	):
										if self.task == "multiple-choice":
															lowerCamelCase__       =       {0: "batch", 1: "choice", 2: "sequence"}
										else:
															lowerCamelCase__       =       {0: "batch", 1: "sequence"}
										return OrderedDict(
										    [
										        ("input_ids", dynamic_axis),
										        ("attention_mask", dynamic_axis),
										        ("token_type_ids", dynamic_axis),
										    ]	)

