
import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


UpperCAmelCase       =					logging.get_logger(__name__)






def 				UpperCAmelCase_						(				):
	# Get the sagemaker specific mp parameters from smp_options variable.
	lowercase	    =       os.getenv('SM_HP_MP_PARAMETERS'					,				'{}'				)
	try:
		# Parse it and check the field "partitions" is included, it is required for model parallel.
		lowercase	    =       json.loads(__SCREAMING_SNAKE_CASE				)
		if "partitions" not in smp_options:
			return False
	except json.JSONDecodeError:
		return False

	# Get the sagemaker specific framework parameters from mpi_options variable.
	lowercase	    =       os.getenv('SM_FRAMEWORK_PARAMS'					,				'{}'				)
	try:
		# Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
		lowercase	    =       json.loads(__SCREAMING_SNAKE_CASE				)
		if not mpi_options.get('sagemaker_mpi_enabled'					,				__SCREAMING_SNAKE_CASE				):
			return False
	except json.JSONDecodeError:
		return False
	# Lastly, check if the `smdistributed` module is present.
	return importlib.util.find_spec('smdistributed'				) is not None


if is_sagemaker_model_parallel_available():
	import smdistributed.modelparallel.torch as smp

	smp.init()







@dataclass
class 			A_  (     __lowerCamelCase  ):




	'''simple docstring'''


	_UpperCamelCase	:	str									=			field(
	    default="""""" ,      metadata={"""help""": """Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer"""} ,      )





	def       SCREAMING_SNAKE_CASE__		(			self			):
		super().__post_init__()
		warnings.warn(
		    '`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use '
		    '`TrainingArguments` instead.'						,       snake_case						,       )





	@cached_property
	def       SCREAMING_SNAKE_CASE__		(			self			):
		logger.info('PyTorch: setting up devices'			)
		if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
			logger.warning(
			    'torch.distributed process group is initialized, but local_rank == -1. '
			    'In order to use Torch DDP, launch your script with `python -m torch.distributed.launch'			)
		if self.no_cuda:
			lowercase	    =       torch.device('cpu'			)
			lowercase	    =       0
		elif is_sagemaker_model_parallel_available():
			lowercase	    =       smp.local_rank()
			lowercase	    =       torch.device('cuda'						,       snake_case			)
			lowercase	    =       1
		elif is_sagemaker_dp_enabled():
			import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

			torch.distributed.init_process_group(backend='smddp'						,       timeout=self.ddp_timeout_delta			)
			lowercase	    =       int(os.getenv('SMDATAPARALLEL_LOCAL_RANK'			)			)
			lowercase	    =       torch.device('cuda'						,       self.local_rank			)
			lowercase	    =       1
		elif self.local_rank == -1:
			# if n_gpu is > 1 we'll use nn.DataParallel.
			# If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
			# Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
			# trigger an error that a device index is missing. Index 0 takes into account the
			# GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
			# will use the first GPU in that env, i.e. GPU#1
			lowercase	    =       torch.device('cuda:0' if torch.cuda.is_available() else 'cpu'			)
			# Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
			# the default value.
			lowercase	    =       torch.cuda.device_count()
		else:
			# Here, we'll use torch.distributed.
			# Initializes the distributed backend which will take care of synchronizing nodes/GPUs
			if not torch.distributed.is_initialized():
				torch.distributed.init_process_group(backend='nccl'						,       timeout=self.ddp_timeout_delta			)
			lowercase	    =       torch.device('cuda'						,       self.local_rank			)
			lowercase	    =       1

		if device.type == "cuda":
			torch.cuda.set_device(snake_case			)

		return device





	@property
	def       SCREAMING_SNAKE_CASE__		(			self			):
		if is_sagemaker_model_parallel_available():
			return smp.dp_size()

		return super().world_size





	@property
	def       SCREAMING_SNAKE_CASE__		(			self			):
		return not is_sagemaker_model_parallel_available()





	@property
	def       SCREAMING_SNAKE_CASE__		(			self			):
		return False






'''simple docstring'''



import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFAutoModel, is_tensorflow_text_available, is_tf_available
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.testing_utils import require_tensorflow_text, require_tf, slow


if is_tf_available():
							import tensorflow as tf

if is_tensorflow_text_available():
							from transformers.models.bert import TFBertTokenizer


lowerCAmelCase_  :							List[str]             =    ['bert-base-uncased', 'bert-base-cased']
lowerCAmelCase_  :							int             =    'hf-internal-testing/tiny-bert-tf-only'

if is_tf_available():
							class 						lowerCamelCase_				(       tf.keras.Model      ):







													def __init__(	self :   Optional[int]					,     lowerCAmelCase__ :   Optional[Any]					):

																				"""simple docstring"""
																				super().__init__()
																				SCREAMING_SNAKE_CASE       :   List[str]         =      tokenizer
																				SCREAMING_SNAKE_CASE       :   str         =      AutoConfig.from_pretrained(lowerCAmelCase__					)
																				SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      TFAutoModel.from_config(lowerCAmelCase__					)







													def        __lowercase     (	self :   Tuple					,     lowerCAmelCase__ :   Tuple					):

																				"""simple docstring"""
																				SCREAMING_SNAKE_CASE       :   Optional[Any]         =      self.tokenizer(lowerCAmelCase__					)
																				SCREAMING_SNAKE_CASE       :   str         =      self.bert(**lowerCAmelCase__					)
																				return out["pooler_output"]
@require_tf
@require_tensorflow_text
class 						lowerCamelCase_				(       unittest.TestCase      ):







						def        __lowercase     (	self :   str					):

													"""simple docstring"""
													super().setUp()

													SCREAMING_SNAKE_CASE       :   int         =      [
													    BertTokenizer.from_pretrained(lowerCAmelCase__					) for checkpoint in (TOKENIZER_CHECKPOINTS * 2)
													]  # repeat for when fast_bert_tokenizer=false
													SCREAMING_SNAKE_CASE       :   Dict         =      [TFBertTokenizer.from_pretrained(lowerCAmelCase__					) for checkpoint in TOKENIZER_CHECKPOINTS] + [
													    TFBertTokenizer.from_pretrained(lowerCAmelCase__					,     use_fast_bert_tokenizer=lowerCAmelCase__					)
													    for checkpoint in TOKENIZER_CHECKPOINTS
													]
													assert len(self.tokenizers					) == len(self.tf_tokenizers					)

													SCREAMING_SNAKE_CASE       :   Any         =      [
													    '''This is a straightforward English test sentence.''',
													    '''This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.''',
													    '''Now we\'re going to add some Chinese: 一 二 三 一二三''',
													    '''And some much more rare Chinese: 齉 堃 齉堃''',
													    '''Je vais aussi écrire en français pour tester les accents''',
													    '''Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ''',
													]
													SCREAMING_SNAKE_CASE       :   int         =      list(zip(self.test_sentences					,     self.test_sentences[::-1]					)					)







						def        __lowercase     (	self :   Any					):

													"""simple docstring"""
													for tokenizer, tf_tokenizer in zip(self.tokenizers					,     self.tf_tokenizers					):
																				for test_inputs in (self.test_sentences, self.paired_sentences):
																											SCREAMING_SNAKE_CASE       :   Tuple         =      tokenizer(lowerCAmelCase__					,     return_tensors='''tf'''					,     padding='''longest'''					)
																											SCREAMING_SNAKE_CASE       :   List[Any]         =      tf_tokenizer(lowerCAmelCase__					)

																											for key in python_outputs.keys():
																																		self.assertTrue(tf.reduce_all(python_outputs[key].shape == tf_outputs[key].shape					)					)
																																		self.assertTrue(tf.reduce_all(tf.cast(python_outputs[key]					,     tf.intaa					) == tf_outputs[key]					)					)







						@slow
						def        __lowercase     (	self :   Dict					):

													"""simple docstring"""
													for tf_tokenizer in self.tf_tokenizers:
																				SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      tf_tokenizer(self.paired_sentences					)
																				SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      tf_tokenizer(
																				    text=[sentence[0] for sentence in self.paired_sentences]					,     text_pair=[sentence[1] for sentence in self.paired_sentences]					,     )
																				for key in merged_outputs.keys():
																											self.assertTrue(tf.reduce_all(tf.cast(merged_outputs[key]					,     tf.intaa					) == separated_outputs[key]					)					)







						@slow
						def        __lowercase     (	self :   Union[str, Any]					):

													"""simple docstring"""
													for tf_tokenizer in self.tf_tokenizers:
																				SCREAMING_SNAKE_CASE       :   Any         =      tf.function(lowerCAmelCase__					)
																				for test_inputs in (self.test_sentences, self.paired_sentences):
																											SCREAMING_SNAKE_CASE       :   int         =      tf.constant(lowerCAmelCase__					)
																											SCREAMING_SNAKE_CASE       :   Any         =      compiled_tokenizer(lowerCAmelCase__					)
																											SCREAMING_SNAKE_CASE       :   List[str]         =      tf_tokenizer(lowerCAmelCase__					)

																											for key in eager_outputs.keys():
																																		self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]					)					)







						@slow
						def        __lowercase     (	self :   int					):

													"""simple docstring"""
													for tf_tokenizer in self.tf_tokenizers:
																				SCREAMING_SNAKE_CASE       :   int         =      ModelToSave(tokenizer=lowerCAmelCase__					)
																				SCREAMING_SNAKE_CASE       :   Tuple         =      tf.convert_to_tensor(self.test_sentences					)
																				SCREAMING_SNAKE_CASE       :   Optional[Any]         =      model(lowerCAmelCase__					)  # Build model with some sample inputs
																				with TemporaryDirectory() as tempdir:
																											SCREAMING_SNAKE_CASE       :   Optional[int]         =      Path(lowerCAmelCase__					) / '''saved.model'''
																											model.save(lowerCAmelCase__					)
																											SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      tf.keras.models.load_model(lowerCAmelCase__					)
																				SCREAMING_SNAKE_CASE       :   Tuple         =      loaded_model(lowerCAmelCase__					)
																				# We may see small differences because the loaded model is compiled, so we need an epsilon for the test
																				self.assertLessEqual(tf.reduce_max(tf.abs(out - loaded_output					)					)					,     1e-5					)




