




import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp







def 			_a   (		):
					__lowerCAmelCase            = ArgumentParser(
					    description=(
					        "PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes"
					    )  )

					# Optional arguments for the launch helper
					parser.add_argument("--num_cores"   ,   type=SCREAMING_SNAKE_CASE_   ,   default=1   ,   help="Number of TPU cores to use (1 or 8)."  )

					# positional
					parser.add_argument(
					    "training_script"   ,   type=SCREAMING_SNAKE_CASE_   ,   help=(
					        "The full path to the single TPU training "
					        "program/script to be launched in parallel, "
					        "followed by all the arguments for the "
					        "training script"
					    )   ,   )

					# rest from the training program
					parser.add_argument("training_script_args"   ,   nargs=SCREAMING_SNAKE_CASE_  )

					return parser.parse_args()







def 			_a   (		):
					__lowerCAmelCase            = parse_args()

					# Import training_script as a module.
					__lowerCAmelCase            = Path(args.training_script  )
					sys.path.append(str(script_fpath.parent.resolve()  )  )
					__lowerCAmelCase            = script_fpath.stem
					__lowerCAmelCase            = importlib.import_module(SCREAMING_SNAKE_CASE_  )

					# Patch sys.argv
					__lowerCAmelCase            = [args.training_script] + args.training_script_args + ["--tpu_num_cores", str(args.num_cores  )]

					xmp.spawn(mod._mp_fn   ,   args=()   ,   nprocs=args.num_cores  )


if __name__ == "__main__":
					main()










from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


UpperCamelCase__					    =    {
    """configuration_ctrl""": ["""CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP""", """CTRLConfig"""],
    """tokenization_ctrl""": ["""CTRLTokenizer"""],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__					    =    [
					    """CTRL_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """CTRLForSequenceClassification""",
					    """CTRLLMHeadModel""",
					    """CTRLModel""",
					    """CTRLPreTrainedModel""",
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__					    =    [
					    """TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """TFCTRLForSequenceClassification""",
					    """TFCTRLLMHeadModel""",
					    """TFCTRLModel""",
					    """TFCTRLPreTrainedModel""",
					]


if TYPE_CHECKING:
					from .configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig
					from .tokenization_ctrl import CTRLTokenizer

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_ctrl import (
										    CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
										    CTRLForSequenceClassification,
										    CTRLLMHeadModel,
										    CTRLModel,
										    CTRLPreTrainedModel,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_ctrl import (
										    TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFCTRLForSequenceClassification,
										    TFCTRLLMHeadModel,
										    TFCTRLModel,
										    TFCTRLPreTrainedModel,
										)

else:
					import sys

					UpperCamelCase__					    =    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




