


'''simple docstring'''
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow


if is_torch_available():
		import torch

		from transformers import XLMRobertaModel
@require_sentencepiece
@require_tokenizers
@require_torch
class   lowercase      (				unittest.TestCase   ):

		"""simple docstring"""

		@slow
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Union[str, Any]				   =					XLMRobertaModel.from_pretrained('''xlm-roberta-base'''				)
							UpperCamelCase__  :Optional[Any]				   =					torch.tensor([[0, 581, 10269, 83, 99942, 136, 60742, 23, 70, 80583, 18276, 2]]				)
							# The dog is cute and lives in the garden house

							UpperCamelCase__  :List[str]				   =					torch.Size((1, 12, 768)				)  # batch_size, sequence_length, embedding_vector_dim
							UpperCamelCase__  :int				   =					torch.tensor(
							    [[-0.0101, 0.1218, -0.0803, 0.0801, 0.1327, 0.0776, -0.1215, 0.2383, 0.3338, 0.3106, 0.0300, 0.0252]]				)
							#  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')
							#  xlmr.eval()
							#  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
							with torch.no_grad():
												UpperCamelCase__  :Dict				   =					model(__A				)["""last_hidden_state"""].detach()
							self.assertEqual(output.shape       ,  __A				)
							# compare the actual values for a slice of last dim
							self.assertTrue(torch.allclose(output[:, :, -1]       ,  __A       ,  atol=1e-3				)				)




		@slow
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Any				   =					XLMRobertaModel.from_pretrained('''xlm-roberta-large'''				)
							UpperCamelCase__  :int				   =					torch.tensor([[0, 581, 10269, 83, 99942, 136, 60742, 23, 70, 80583, 18276, 2]]				)
							# The dog is cute and lives in the garden house

							UpperCamelCase__  :Optional[Any]				   =					torch.Size((1, 12, 1024)				)  # batch_size, sequence_length, embedding_vector_dim
							UpperCamelCase__  :Dict				   =					torch.tensor(
							    [[-0.0699, -0.0318, 0.0705, -0.1241, 0.0999, -0.0520, 0.1004, -0.1838, -0.4704, 0.1437, 0.0821, 0.0126]]				)
							#  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.large')
							#  xlmr.eval()
							#  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
							with torch.no_grad():
												UpperCamelCase__  :Dict				   =					model(__A				)["""last_hidden_state"""].detach()
							self.assertEqual(output.shape       ,  __A				)
							# compare the actual values for a slice of last dim
							self.assertTrue(torch.allclose(output[:, :, -1]       ,  __A       ,  atol=1e-3				)				)


import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp
def 							__SCREAMING_SNAKE_CASE     (   )					->				Tuple:
	__A :			List[Any]		    =			ArgumentParser(
	    description=(
	        """PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes"""
	    )       )

	# Optional arguments for the launch helper
	parser.add_argument("""--num_cores"""			,type=a__			,default=1			,help="""Number of TPU cores to use (1 or 8)."""       )

	# positional
	parser.add_argument(
	    """training_script"""			,type=a__			,help=(
	        """The full path to the single TPU training """
	        """program/script to be launched in parallel, """
	        """followed by all the arguments for the """
	        """training script"""
	    )			,)

	# rest from the training program
	parser.add_argument("""training_script_args"""			,nargs=a__       )

	return parser.parse_args()
def 							__SCREAMING_SNAKE_CASE     (   )					->				str:
	__A :			Union[str, Any]		    =			parse_args()

	# Import training_script as a module.
	__A :			List[Any]		    =			Path(args.training_script       )
	sys.path.append(str(script_fpath.parent.resolve()       )       )
	__A :			str		    =			script_fpath.stem
	__A :			int		    =			importlib.import_module(a__       )

	# Patch sys.argv
	__A :			List[str]		    =			[args.training_script] + args.training_script_args + ["""--tpu_num_cores""", str(args.num_cores       )]

	xmp.spawn(mod._mp_fn			,args=()			,nprocs=args.num_cores       )


if __name__ == "__main__":
			main()





