

'''simple docstring'''

import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class    snake_case     (      lowercase_,						unittest.TestCase		):






				"""simple docstring"""
				_a    	=     None
				_a    	=     BloomTokenizerFast
				_a    	=     BloomTokenizerFast
				_a    	=     True
				_a    	=     False
				_a    	=     """tokenizer_file"""
				_a    	=     {"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}



				def   a__					(    self       )     ->							int:
											super().setUp()
											SCREAMING_SNAKE_CASE_				=				BloomTokenizerFast.from_pretrained('bigscience/tokenizer'       )
											tokenizer.save_pretrained(self.tmpdirname       )



				def   a__					(    self,     **_lowercase       )     ->							List[Any]:
											kwargs.update(self.special_tokens_map       )
											return BloomTokenizerFast.from_pretrained(self.tmpdirname,     **_lowercase       )



				def   a__					(    self       )     ->							Union[str, Any]:
											SCREAMING_SNAKE_CASE_				=				self.get_rust_tokenizer()

											SCREAMING_SNAKE_CASE_				=				['The quick brown fox</s>', 'jumps over the lazy dog</s>']
											SCREAMING_SNAKE_CASE_				=				[[2175, 23714, 73173, 144252, 2], [77, 132619, 3478, 368, 109586, 35433, 2]]

											SCREAMING_SNAKE_CASE_				=				tokenizer.batch_encode_plus(_lowercase       )['input_ids']
											self.assertListEqual(_lowercase,     _lowercase       )

											SCREAMING_SNAKE_CASE_				=				tokenizer.batch_decode(_lowercase       )
											self.assertListEqual(_lowercase,     _lowercase       )



				def   a__					(    self,     _lowercase=6       )     ->							Dict:
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																									SCREAMING_SNAKE_CASE_				=				self.rust_tokenizer_class.from_pretrained(_lowercase,     **_lowercase       )
																									# tokenizer_r.pad_token = None # Hotfixing padding = None
																									# Simple input
																									SCREAMING_SNAKE_CASE_				=				'This is a simple input'
																									SCREAMING_SNAKE_CASE_				=				['This is a simple input 1', 'This is a simple input 2']
																									SCREAMING_SNAKE_CASE_				=				('This is a simple input', 'This is a pair')
																									SCREAMING_SNAKE_CASE_				=				[
																									    ('This is a simple input 1', 'This is a simple input 2'),
																									    ('This is a simple pair 1', 'This is a simple pair 2'),
																									]

																									# Simple input tests
																									try:
																																tokenizer_r.encode(_lowercase,     max_length=_lowercase       )
																																tokenizer_r.encode_plus(_lowercase,     max_length=_lowercase       )

																																tokenizer_r.batch_encode_plus(_lowercase,     max_length=_lowercase       )
																																tokenizer_r.encode(_lowercase,     max_length=_lowercase       )
																																tokenizer_r.batch_encode_plus(_lowercase,     max_length=_lowercase       )
																									except ValueError:
																																self.fail('Bloom Tokenizer should be able to deal with padding'       )

																									SCREAMING_SNAKE_CASE_				=				None  # Hotfixing padding = None
																									self.assertRaises(_lowercase,     tokenizer_r.encode,     _lowercase,     max_length=_lowercase,     padding='max_length'       )

																									# Simple input
																									self.assertRaises(_lowercase,     tokenizer_r.encode_plus,     _lowercase,     max_length=_lowercase,     padding='max_length'       )

																									# Simple input
																									self.assertRaises(
																									    _lowercase,     tokenizer_r.batch_encode_plus,     _lowercase,     max_length=_lowercase,     padding='max_length',     )

																									# Pair input
																									self.assertRaises(_lowercase,     tokenizer_r.encode,     _lowercase,     max_length=_lowercase,     padding='max_length'       )

																									# Pair input
																									self.assertRaises(_lowercase,     tokenizer_r.encode_plus,     _lowercase,     max_length=_lowercase,     padding='max_length'       )

																									# Pair input
																									self.assertRaises(
																									    _lowercase,     tokenizer_r.batch_encode_plus,     _lowercase,     max_length=_lowercase,     padding='max_length',     )



				def   a__					(    self       )     ->							str:
											SCREAMING_SNAKE_CASE_				=				self.get_rust_tokenizer()
											SCREAMING_SNAKE_CASE_				=				load_dataset('xnli',     'all_languages',     split='test',     streaming=_lowercase       )

											SCREAMING_SNAKE_CASE_				=				next(iter(_lowercase       )       )['premise']  # pick up one data
											SCREAMING_SNAKE_CASE_				=				list(sample_data.values()       )

											SCREAMING_SNAKE_CASE_				=				list(map(tokenizer.encode,     _lowercase       )       )
											SCREAMING_SNAKE_CASE_				=				[tokenizer.decode(_lowercase,     clean_up_tokenization_spaces=_lowercase       ) for x in output_tokens]
											self.assertListEqual(_lowercase,     _lowercase       )





				def   a__					(    self       )     ->							Tuple:
											# The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
											# any sequence length constraints. This test of the parent class will fail since it relies on the
											# maximum sequence length of the positoonal embeddings.
											self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map       ),     1       )
											self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()       )[0]       ),     1       )








'''simple docstring'''

import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


SCREAMING_SNAKE_CASE							:				Optional[Any]     	=			logging.get_logger(__name__)




def 							_UpperCamelCase				(   )      -> List[Any]:
							# Get the sagemaker specific mp parameters from smp_options variable.
							SCREAMING_SNAKE_CASE_				=				os.getenv('SM_HP_MP_PARAMETERS'		,'{}'   )
							try:
														# Parse it and check the field "partitions" is included, it is required for model parallel.
														SCREAMING_SNAKE_CASE_				=				json.loads(lowerCAmelCase__   )
														if "partitions" not in smp_options:
																					return False
							except json.JSONDecodeError:
														return False

							# Get the sagemaker specific framework parameters from mpi_options variable.
							SCREAMING_SNAKE_CASE_				=				os.getenv('SM_FRAMEWORK_PARAMS'		,'{}'   )
							try:
														# Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
														SCREAMING_SNAKE_CASE_				=				json.loads(lowerCAmelCase__   )
														if not mpi_options.get('sagemaker_mpi_enabled'		,lowerCAmelCase__   ):
																					return False
							except json.JSONDecodeError:
														return False
							# Lastly, check if the `smdistributed` module is present.
							return importlib.util.find_spec('smdistributed'   ) is not None


if is_sagemaker_model_parallel_available():
							import smdistributed.modelparallel.torch as smp

							smp.init()






@dataclass
class    snake_case     (      lowercase_		):






				"""simple docstring"""
				_a    	=     field(
				    default="""""",						metadata={"""help""": """Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer"""},						)



				def   a__					(    self       )     ->							Union[str, Any]:
											super().__post_init__()
											warnings.warn(
											    '`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use '
											    '`TrainingArguments` instead.',     _lowercase,     )



				@cached_property
				def   a__					(    self       )     ->							"torch.device":
											logger.info('PyTorch: setting up devices'       )
											if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
																		logger.warning(
																		    'torch.distributed process group is initialized, but local_rank == -1. '
																		    'In order to use Torch DDP, launch your script with `python -m torch.distributed.launch'       )
											if self.no_cuda:
																		SCREAMING_SNAKE_CASE_				=				torch.device('cpu'       )
																		SCREAMING_SNAKE_CASE_				=				0
											elif is_sagemaker_model_parallel_available():
																		SCREAMING_SNAKE_CASE_				=				smp.local_rank()
																		SCREAMING_SNAKE_CASE_				=				torch.device('cuda',     _lowercase       )
																		SCREAMING_SNAKE_CASE_				=				1
											elif is_sagemaker_dp_enabled():
																		import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

																		torch.distributed.init_process_group(backend='smddp',     timeout=self.ddp_timeout_delta       )
																		SCREAMING_SNAKE_CASE_				=				int(os.getenv('SMDATAPARALLEL_LOCAL_RANK'       )       )
																		SCREAMING_SNAKE_CASE_				=				torch.device('cuda',     self.local_rank       )
																		SCREAMING_SNAKE_CASE_				=				1
											elif self.local_rank == -1:
																		# if n_gpu is > 1 we'll use nn.DataParallel.
																		# If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
																		# Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
																		# trigger an error that a device index is missing. Index 0 takes into account the
																		# GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
																		# will use the first GPU in that env, i.e. GPU#1
																		SCREAMING_SNAKE_CASE_				=				torch.device('cuda:0' if torch.cuda.is_available() else 'cpu'       )
																		# Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
																		# the default value.
																		SCREAMING_SNAKE_CASE_				=				torch.cuda.device_count()
											else:
																		# Here, we'll use torch.distributed.
																		# Initializes the distributed backend which will take care of synchronizing nodes/GPUs
																		if not torch.distributed.is_initialized():
																									torch.distributed.init_process_group(backend='nccl',     timeout=self.ddp_timeout_delta       )
																		SCREAMING_SNAKE_CASE_				=				torch.device('cuda',     self.local_rank       )
																		SCREAMING_SNAKE_CASE_				=				1

											if device.type == "cuda":
																		torch.cuda.set_device(_lowercase       )

											return device



				@property
				def   a__					(    self       )     ->							Optional[Any]:
											if is_sagemaker_model_parallel_available():
																		return smp.dp_size()

											return super().world_size



				@property
				def   a__					(    self       )     ->							Optional[Any]:
											return not is_sagemaker_model_parallel_available()





				@property
				def   a__					(    self       )     ->							Tuple:
											return False





