


'''simple docstring'''



import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


__A	=logging.get_logger(__name__)







def     _UpperCamelCase      (		):
			# Get the sagemaker specific mp parameters from smp_options variable.
			UpperCAmelCase__  :					Any        =     os.getenv("""SM_HP_MP_PARAMETERS""" ,      """{}"""					)
			try:
						# Parse it and check the field "partitions" is included, it is required for model parallel.
						UpperCAmelCase__  :					Tuple        =     json.loads(UpperCamelCase__					)
						if "partitions" not in smp_options:
									return False
			except json.JSONDecodeError:
						return False

			# Get the sagemaker specific framework parameters from mpi_options variable.
			UpperCAmelCase__  :					Optional[int]        =     os.getenv("""SM_FRAMEWORK_PARAMS""" ,      """{}"""					)
			try:
						# Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
						UpperCAmelCase__  :					List[str]        =     json.loads(UpperCamelCase__					)
						if not mpi_options.get("""sagemaker_mpi_enabled""" ,      UpperCamelCase__					):
									return False
			except json.JSONDecodeError:
						return False
			# Lastly, check if the `smdistributed` module is present.
			return importlib.util.find_spec("""smdistributed"""					) is not None


if is_sagemaker_model_parallel_available():
		import smdistributed.modelparallel.torch as smp

		smp.init()


@dataclass
class 		_snake_case					(     a__	):
					lowerCAmelCase      :str   =					field(
					    default=''''''       ,   metadata={'''help''': '''Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer'''}       ,   )




					def 				snake_case__						(      self):
								super().__post_init__()
								warnings.warn(
								    """`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use """
								    """`TrainingArguments` instead."""							,							_lowerCamelCase							,							)




					@cached_property
					def 				snake_case__						(      self):
								logger.info("""PyTorch: setting up devices""")
								if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
											logger.warning(
											    """torch.distributed process group is initialized, but local_rank == -1. """
											    """In order to use Torch DDP, launch your script with `python -m torch.distributed.launch""")
								if self.no_cuda:
											UpperCAmelCase__  :					List[Any]        =     torch.device("""cpu""")
											UpperCAmelCase__  :					Optional[int]        =     0
								elif is_sagemaker_model_parallel_available():
											UpperCAmelCase__  :					str        =     smp.local_rank()
											UpperCAmelCase__  :					Any        =     torch.device("""cuda"""							,							_lowerCamelCase)
											UpperCAmelCase__  :					Dict        =     1
								elif is_sagemaker_dp_enabled():
											import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

											torch.distributed.init_process_group(backend="""smddp"""							,							timeout=self.ddp_timeout_delta)
											UpperCAmelCase__  :					Union[str, Any]        =     int(os.getenv("""SMDATAPARALLEL_LOCAL_RANK"""))
											UpperCAmelCase__  :					int        =     torch.device("""cuda"""							,							self.local_rank)
											UpperCAmelCase__  :					Dict        =     1
								elif self.local_rank == -1:
											# if n_gpu is > 1 we'll use nn.DataParallel.
											# If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
											# Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
											# trigger an error that a device index is missing. Index 0 takes into account the
											# GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
											# will use the first GPU in that env, i.e. GPU#1
											UpperCAmelCase__  :					int        =     torch.device("""cuda:0""" if torch.cuda.is_available() else """cpu""")
											# Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
											# the default value.
											UpperCAmelCase__  :					int        =     torch.cuda.device_count()
								else:
											# Here, we'll use torch.distributed.
											# Initializes the distributed backend which will take care of synchronizing nodes/GPUs
											if not torch.distributed.is_initialized():
														torch.distributed.init_process_group(backend="""nccl"""							,							timeout=self.ddp_timeout_delta)
											UpperCAmelCase__  :					Dict        =     torch.device("""cuda"""							,							self.local_rank)
											UpperCAmelCase__  :					Tuple        =     1

								if device.type == "cuda":
											torch.cuda.set_device(_lowerCamelCase)

								return device




					@property
					def 				snake_case__						(      self):
								if is_sagemaker_model_parallel_available():
											return smp.dp_size()

								return super().world_size




					@property
					def 				snake_case__						(      self):
								return not is_sagemaker_model_parallel_available()




					@property
					def 				snake_case__						(      self):
								return False



'''simple docstring'''



from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__A	=logging.get_logger(__name__)

__A	={
    'xlm-mlm-en-2048': 'https://huggingface.co/xlm-mlm-en-2048/resolve/main/config.json',
    'xlm-mlm-ende-1024': 'https://huggingface.co/xlm-mlm-ende-1024/resolve/main/config.json',
    'xlm-mlm-enfr-1024': 'https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/config.json',
    'xlm-mlm-enro-1024': 'https://huggingface.co/xlm-mlm-enro-1024/resolve/main/config.json',
    'xlm-mlm-tlm-xnli15-1024': 'https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/config.json',
    'xlm-mlm-xnli15-1024': 'https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/config.json',
    'xlm-clm-enfr-1024': 'https://huggingface.co/xlm-clm-enfr-1024/resolve/main/config.json',
    'xlm-clm-ende-1024': 'https://huggingface.co/xlm-clm-ende-1024/resolve/main/config.json',
    'xlm-mlm-17-1280': 'https://huggingface.co/xlm-mlm-17-1280/resolve/main/config.json',
    'xlm-mlm-100-1280': 'https://huggingface.co/xlm-mlm-100-1280/resolve/main/config.json',
}


class 		_snake_case					(     a__	):
					lowerCAmelCase      :Any   =					'''xlm'''
					lowerCAmelCase      :Any   =					{
					    '''hidden_size''': '''emb_dim''',
					    '''num_attention_heads''': '''n_heads''',
					    '''num_hidden_layers''': '''n_layers''',
					    '''n_words''': '''vocab_size''',  # For backward compatibility
					}




					def __init__(      self							,							_lowerCamelCase=3_0145							,							_lowerCamelCase=2048							,							_lowerCamelCase=12							,							_lowerCamelCase=16							,							_lowerCamelCase=0.1							,							_lowerCamelCase=0.1							,							_lowerCamelCase=True							,							_lowerCamelCase=False							,							_lowerCamelCase=False							,							_lowerCamelCase=False							,							_lowerCamelCase=1							,							_lowerCamelCase=True							,							_lowerCamelCase=512							,							_lowerCamelCase=2048**-0.5							,							_lowerCamelCase=1e-1_2							,							_lowerCamelCase=0.02							,							_lowerCamelCase=0							,							_lowerCamelCase=1							,							_lowerCamelCase=2							,							_lowerCamelCase=3							,							_lowerCamelCase=5							,							_lowerCamelCase=True							,							_lowerCamelCase="first"							,							_lowerCamelCase=True							,							_lowerCamelCase=None							,							_lowerCamelCase=True							,							_lowerCamelCase=0.1							,							_lowerCamelCase=5							,							_lowerCamelCase=5							,							_lowerCamelCase=0							,							_lowerCamelCase=0							,							_lowerCamelCase=2							,							_lowerCamelCase=0							,							**_lowerCamelCase							,							):
								UpperCAmelCase__  :					Tuple        =     vocab_size
								UpperCAmelCase__  :					Tuple        =     emb_dim
								UpperCAmelCase__  :					Optional[Any]        =     n_layers
								UpperCAmelCase__  :					List[str]        =     n_heads
								UpperCAmelCase__  :					List[str]        =     dropout
								UpperCAmelCase__  :					Optional[int]        =     attention_dropout
								UpperCAmelCase__  :					Tuple        =     gelu_activation
								UpperCAmelCase__  :					Optional[Any]        =     sinusoidal_embeddings
								UpperCAmelCase__  :					int        =     causal
								UpperCAmelCase__  :					Union[str, Any]        =     asm
								UpperCAmelCase__  :					Optional[Any]        =     n_langs
								UpperCAmelCase__  :					List[Any]        =     use_lang_emb
								UpperCAmelCase__  :					Union[str, Any]        =     layer_norm_eps
								UpperCAmelCase__  :					List[str]        =     bos_index
								UpperCAmelCase__  :					List[Any]        =     eos_index
								UpperCAmelCase__  :					int        =     pad_index
								UpperCAmelCase__  :					str        =     unk_index
								UpperCAmelCase__  :					Dict        =     mask_index
								UpperCAmelCase__  :					str        =     is_encoder
								UpperCAmelCase__  :					Dict        =     max_position_embeddings
								UpperCAmelCase__  :					Any        =     embed_init_std
								UpperCAmelCase__  :					List[Any]        =     init_std
								UpperCAmelCase__  :					List[str]        =     summary_type
								UpperCAmelCase__  :					Union[str, Any]        =     summary_use_proj
								UpperCAmelCase__  :					Any        =     summary_activation
								UpperCAmelCase__  :					List[str]        =     summary_proj_to_labels
								UpperCAmelCase__  :					Union[str, Any]        =     summary_first_dropout
								UpperCAmelCase__  :					str        =     start_n_top
								UpperCAmelCase__  :					str        =     end_n_top
								UpperCAmelCase__  :					Tuple        =     mask_token_id
								UpperCAmelCase__  :					Union[str, Any]        =     lang_id

								if "n_words" in kwargs:
											UpperCAmelCase__  :					List[str]        =     kwargs["""n_words"""]

								super().__init__(pad_token_id=_lowerCamelCase							,							bos_token_id=_lowerCamelCase							,							**_lowerCamelCase)


class 		_snake_case					(     a__	):




					@property
					def 				snake_case__						(      self):
								if self.task == "multiple-choice":
											UpperCAmelCase__  :					Any        =     {0: """batch""", 1: """choice""", 2: """sequence"""}
								else:
											UpperCAmelCase__  :					List[Any]        =     {0: """batch""", 1: """sequence"""}
								return OrderedDict(
								    [
								        ("""input_ids""", dynamic_axis),
								        ("""attention_mask""", dynamic_axis),
								        ("""token_type_ids""", dynamic_axis),
								    ])