
'''simple docstring'''


import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowercase							:   List[str]					=     get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model''')

@require_sentencepiece
@require_tokenizers
class 	__lowercase				(  _lowercase      ,     unittest.TestCase    ):
			lowerCamelCase			:		Optional[int]	      =			GPTSwaTokenizer
			lowerCamelCase			:		Union[str, Any]	      =			False
			lowerCamelCase			:		List[str]	      =			True
			lowerCamelCase			:		Any	      =			False

			def        UpperCAmelCase__ (self      ):
								super().setUp()

								# We have a SentencePiece fixture for testing
								lowerCamelCase_					:      Dict     =		GPTSwaTokenizer(A			,   eos_token='''<unk>'''			,   bos_token='''<unk>'''			,   pad_token='''<unk>'''      )

								tokenizer.save_pretrained(self.tmpdirname      )

			def        UpperCAmelCase__ (self			,   A      ):
								lowerCamelCase_					:      Tuple     =		'''This is a test'''
								lowerCamelCase_					:      Dict     =		'''This is a test'''
								return input_text, output_text

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Union[str, Any]     =		'''<s>'''
								lowerCamelCase_					:      Tuple     =		1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(A      )			,   A      )
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(A      )			,   A      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Optional[int]     =		list(self.get_tokenizer().get_vocab().keys()      )

								self.assertEqual(vocab_keys[0]			,   '''<unk>'''      )
								self.assertEqual(vocab_keys[1]			,   '''<s>'''      )
								self.assertEqual(vocab_keys[-1]			,   '''j'''      )
								self.assertEqual(len(A      )			,   2_0_0_0      )

			def        UpperCAmelCase__ (self      ):
								self.assertEqual(self.get_tokenizer().vocab_size			,   2_0_0_0      )

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      int     =		GPTSwaTokenizer(A      )

								lowerCamelCase_					:      Dict     =		tokenizer.tokenize('''This is a test'''      )
								self.assertListEqual(A			,   ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

								self.assertListEqual(tokenizer.convert_tokens_to_ids(A      )			,   [4_6_5, 2_8_7, 2_6_5, 6_3_1, 8_4_2]      )

								lowerCamelCase_					:      Tuple     =		tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
								# fmt: off
								self.assertListEqual(
								    A			,   ['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.''']			,   )
								# fmt: on

								lowerCamelCase_					:      Union[str, Any]     =		tokenizer.convert_tokens_to_ids(A      )
								self.assertListEqual(
								    A			,   [2_6_2, 2_7_2, 1_5_2_5, 2_8_6, 2_7_1, 2_6_8, 6_0, 9_1_6, 6_3_3, 6_3_3, 6_3_3, 2_5_9, 2_6_6, 3_0_1, 2_8_7, 3_8_4, 3_6_7, 2_6_3, 1_9_8, 1_7_2, 2_6_0]			,   )

								lowerCamelCase_					:      Union[str, Any]     =		tokenizer.convert_ids_to_tokens(A      )
								# fmt: off
								self.assertListEqual(
								    A			,   ['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.''']      )
								# fmt: on

			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      Optional[int]     =		GPTSwaTokenizer(A      )
								lowerCamelCase_					:      Tuple     =		['''This is a test''', '''I was born in 92000, and this is falsé.''']
								lowerCamelCase_					:      List[str]     =		[
								    [4_6_5, 2_8_7, 2_6_5, 6_3_1, 8_4_2],
								    [2_6_2, 2_7_2, 1_5_2_5, 2_8_6, 2_7_1, 2_6_8, 6_0, 9_1_6, 6_3_3, 6_3_3, 6_3_3, 2_5_9, 2_6_6, 3_0_1, 2_8_7, 3_8_4, 3_6_7, 2_6_3, 1_9_8, 1_7_2, 2_6_0],
								]

								# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
								for text, expected_ids in zip(A			,   A      ):
													self.assertListEqual(tokenizer.encode_fast(A      )			,   A      )

								# Test that decode_fast returns the input text
								for text, token_ids in zip(A			,   A      ):
													self.assertEqual(tokenizer.decode_fast(A      )			,   A      )






			@slow
			def        UpperCAmelCase__ (self      ):
								lowerCamelCase_					:      List[str]     =		[
								    '''<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')''',
								    '''Hey there, how are you doing this fine day?''',
								    '''This is a text with a trailing spaces followed by a dot     .''',
								    '''Häj sväjs lillebrör! =)''',
								    '''Det är inget fel på Mr. Cool''',
								]

								# fmt: off
								lowerCamelCase_					:      Any     =		{'''input_ids''': [[6_3_4_2_3, 5, 6_8_1_1, 1_4_9_5_4, 2_8_2, 8_1_6, 3_8_2_1, 6_3_4_6_6, 6_3_4_2_5, 6_3_4_6_2, 1_8, 6_3_9_7_8, 6_7_8, 3_0_1, 1_3_2_0, 6_3_4_2_3, 6_3_4_5_5, 6_3_4_5_8, 1_8, 6_3_9_8_2, 4_2_4_6, 3_9_4_0, 1_9_0_1, 4_7_7_8_9, 5_5_4_7, 1_8_9_9_4], [1_9_6_3_0, 1_1_0_0, 6_3_4_4_6, 1_3_4_2, 6_3_3, 5_4_4, 4_4_8_8, 5_9_3, 5_1_0_2, 2_4_1_6, 6_3_4_9_5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_6_5_2, 4_2_8, 2_6_8, 1_9_3_6, 5_1_5, 2_6_8, 5_8_5_9_3, 2_2_4_1_3, 9_1_0_6, 5_4_6, 2_6_8, 3_3_2_1_3, 6_3_9_7_9, 6_9_8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5_1_3_0, 6_3_4_5_0, 9_2_4, 6_3_4_4_9, 2_2_4_9, 4_0_6_2, 1_5_5_8, 3_1_8, 6_3_5_0_4, 2_1_4_9_8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_0_9, 3_7_7, 2_8_2_7, 2_5_5_9, 3_3_2, 6_5_7_5, 6_3_4_4_3, 2_6_8_0_1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
								# fmt: on
								self.tokenizer_integration_test_util(
								    expected_encoding=A			,   model_name='''AI-Sweden/gpt-sw3-126m'''			,   sequences=A			,   )




'''simple docstring'''


import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


__lowercase							:   Union[str, Any]					=     logging.get_logger(__name__)




def lowercase_	(   )     ->							Union[str, Any]:
					'''simple docstring'''

					lowerCamelCase_					:      Optional[Any]     =		os.getenv('''SM_HP_MP_PARAMETERS'''			,       '''{}'''				)
					try:
										# Parse it and check the field "partitions" is included, it is required for model parallel.
										lowerCamelCase_					:      Optional[Any]     =		json.loads(_lowercase				)
										if "partitions" not in smp_options:
															return False
					except json.JSONDecodeError:
										return False

					# Get the sagemaker specific framework parameters from mpi_options variable.
					lowerCamelCase_					:      List[str]     =		os.getenv('''SM_FRAMEWORK_PARAMS'''			,       '''{}'''				)
					try:
										# Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
										lowerCamelCase_					:      List[str]     =		json.loads(_lowercase				)
										if not mpi_options.get('''sagemaker_mpi_enabled'''			,       _lowercase				):
															return False
					except json.JSONDecodeError:
										return False
					# Lastly, check if the `smdistributed` module is present.
					return importlib.util.find_spec('''smdistributed'''				) is not None


if is_sagemaker_model_parallel_available():
					import smdistributed.modelparallel.torch as smp

					smp.init()






@dataclass
class 	__lowercase				(  _lowercase    ):
			lowerCamelCase			:		str	      =			field(
			    default=""      ,     metadata={"help": "Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer"}      ,     )

			def        UpperCAmelCase__ (self      ):
								super().__post_init__()
								warnings.warn(
								    '''`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use '''
								    '''`TrainingArguments` instead.'''			,   A			,   )

			@cached_property
			def        UpperCAmelCase__ (self      ):
								logger.info('''PyTorch: setting up devices'''      )
								if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
													logger.warning(
													    '''torch.distributed process group is initialized, but local_rank == -1. '''
													    '''In order to use Torch DDP, launch your script with `python -m torch.distributed.launch'''      )
								if self.no_cuda:
													lowerCamelCase_					:      Optional[int]     =		torch.device('''cpu'''      )
													lowerCamelCase_					:      Dict     =		0
								elif is_sagemaker_model_parallel_available():
													lowerCamelCase_					:      int     =		smp.local_rank()
													lowerCamelCase_					:      str     =		torch.device('''cuda'''			,   A      )
													lowerCamelCase_					:      Tuple     =		1
								elif is_sagemaker_dp_enabled():
													import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

													torch.distributed.init_process_group(backend='''smddp'''			,   timeout=self.ddp_timeout_delta      )
													lowerCamelCase_					:      List[Any]     =		int(os.getenv('''SMDATAPARALLEL_LOCAL_RANK'''      )      )
													lowerCamelCase_					:      Union[str, Any]     =		torch.device('''cuda'''			,   self.local_rank      )
													lowerCamelCase_					:      List[str]     =		1
								elif self.local_rank == -1:
													# if n_gpu is > 1 we'll use nn.DataParallel.
													# If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
													# Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
													# trigger an error that a device index is missing. Index 0 takes into account the
													# GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
													# will use the first GPU in that env, i.e. GPU#1
													lowerCamelCase_					:      List[str]     =		torch.device('''cuda:0''' if torch.cuda.is_available() else '''cpu'''      )
													# Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
													# the default value.
													lowerCamelCase_					:      Any     =		torch.cuda.device_count()
								else:
													# Here, we'll use torch.distributed.
													# Initializes the distributed backend which will take care of synchronizing nodes/GPUs
													if not torch.distributed.is_initialized():
																		torch.distributed.init_process_group(backend='''nccl'''			,   timeout=self.ddp_timeout_delta      )
													lowerCamelCase_					:      Any     =		torch.device('''cuda'''			,   self.local_rank      )
													lowerCamelCase_					:      List[str]     =		1

								if device.type == "cuda":
													torch.cuda.set_device(A      )

								return device

			@property
			def        UpperCAmelCase__ (self      ):
								if is_sagemaker_model_parallel_available():
													return smp.dp_size()

								return super().world_size

			@property
			def        UpperCAmelCase__ (self      ):
								return not is_sagemaker_model_parallel_available()






			@property
			def        UpperCAmelCase__ (self      ):
								return False


