


import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCamelCase    		=       get_tests_dir('fixtures/test_sentencepiece.model')







@require_sentencepiece
@require_tokenizers
class        lowerCamelCase__	(							UpperCAmelCase					,	unittest.TestCase      ):






			"""simple docstring"""






			_UpperCamelCase		:					str    =		ReformerTokenizer
			_UpperCamelCase		:					Optional[Any]    =		ReformerTokenizerFast
			_UpperCamelCase		:					List[str]    =		True
			_UpperCamelCase		:					Any    =		False
			_UpperCamelCase		:					Tuple    =		True




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						super().setUp()

						UpperCamelCase__					=       ReformerTokenizer(snake_case      ,						keep_accents=snake_case     )
						tokenizer.save_pretrained(self.tmpdirname     )




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						UpperCamelCase__					=       "<s>"
						UpperCamelCase__					=       1

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case     )      ,						snake_case     )
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case     )      ,						snake_case     )




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						UpperCamelCase__					=       list(self.get_tokenizer().get_vocab().keys()     )

						self.assertEqual(vocab_keys[0]      ,						"<unk>"     )
						self.assertEqual(vocab_keys[1]      ,						"<s>"     )
						self.assertEqual(vocab_keys[-1]      ,						"j"     )
						self.assertEqual(len(snake_case     )      ,						1000     )




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						self.assertEqual(self.get_tokenizer().vocab_size      ,						1000     )




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						if not self.test_rust_tokenizer:
									return

						UpperCamelCase__					=       self.get_tokenizer()
						UpperCamelCase__					=       self.get_rust_tokenizer()

						UpperCamelCase__					=       "I was born in 92000, and this is falsé."

						UpperCamelCase__					=       tokenizer.tokenize(snake_case     )
						UpperCamelCase__					=       rust_tokenizer.tokenize(snake_case     )
						self.assertListEqual(snake_case      ,						snake_case     )

						UpperCamelCase__					=       tokenizer.encode(snake_case      ,						add_special_tokens=snake_case     )
						UpperCamelCase__					=       rust_tokenizer.encode(snake_case      ,						add_special_tokens=snake_case     )
						self.assertListEqual(snake_case      ,						snake_case     )

						UpperCamelCase__					=       self.get_rust_tokenizer()
						UpperCamelCase__					=       tokenizer.encode(snake_case     )
						UpperCamelCase__					=       rust_tokenizer.encode(snake_case     )
						self.assertListEqual(snake_case      ,						snake_case     )




			def 				snake_case__				(							self      ,						snake_case=15     ):

						'''simple docstring'''


						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
												UpperCamelCase__					=       self.rust_tokenizer_class.from_pretrained(snake_case      ,						**snake_case     )

												# Simple input
												UpperCamelCase__					=       "This is a simple input"
												UpperCamelCase__					=       ["This is a simple input 1", "This is a simple input 2"]
												UpperCamelCase__					=       ("This is a simple input", "This is a pair")
												UpperCamelCase__					=       [
												    ("This is a simple input 1", "This is a simple input 2"),
												    ("This is a simple pair 1", "This is a simple pair 2"),
												]

												# Simple input tests
												self.assertRaises(snake_case      ,						tokenizer_r.encode      ,						snake_case      ,						max_length=snake_case      ,						padding="max_length"     )

												# Simple input
												self.assertRaises(snake_case      ,						tokenizer_r.encode_plus      ,						snake_case      ,						max_length=snake_case      ,						padding="max_length"     )

												# Simple input
												self.assertRaises(
												    snake_case      ,						tokenizer_r.batch_encode_plus      ,						snake_case      ,						max_length=snake_case      ,						padding="max_length"      ,						)

												# Pair input
												self.assertRaises(snake_case      ,						tokenizer_r.encode      ,						snake_case      ,						max_length=snake_case      ,						padding="max_length"     )

												# Pair input
												self.assertRaises(snake_case      ,						tokenizer_r.encode_plus      ,						snake_case      ,						max_length=snake_case      ,						padding="max_length"     )

												# Pair input
												self.assertRaises(
												    snake_case      ,						tokenizer_r.batch_encode_plus      ,						snake_case      ,						max_length=snake_case      ,						padding="max_length"      ,						)




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						pass




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						UpperCamelCase__					=       ReformerTokenizer(snake_case      ,						keep_accents=snake_case     )

						UpperCamelCase__					=       tokenizer.tokenize("This is a test"     )
						self.assertListEqual(snake_case      ,						["▁This", "▁is", "▁a", "▁t", "est"]     )

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(snake_case     )      ,						[285, 46, 10, 170, 382]      ,						)

						UpperCamelCase__					=       tokenizer.tokenize("I was born in 92000, and this is falsé."     )
						self.assertListEqual(
						    snake_case      ,						[
						        SPIECE_UNDERLINE + "I",
						        SPIECE_UNDERLINE + "was",
						        SPIECE_UNDERLINE + "b",
						        "or",
						        "n",
						        SPIECE_UNDERLINE + "in",
						        SPIECE_UNDERLINE + "",
						        "9",
						        "2",
						        "0",
						        "0",
						        "0",
						        ",",
						        SPIECE_UNDERLINE + "and",
						        SPIECE_UNDERLINE + "this",
						        SPIECE_UNDERLINE + "is",
						        SPIECE_UNDERLINE + "f",
						        "al",
						        "s",
						        "é",
						        ".",
						    ]      ,						)
						UpperCamelCase__					=       tokenizer.convert_tokens_to_ids(snake_case     )
						self.assertListEqual(
						    snake_case      ,						[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]      ,						)

						UpperCamelCase__					=       tokenizer.convert_ids_to_tokens(snake_case     )
						self.assertListEqual(
						    snake_case      ,						[
						        SPIECE_UNDERLINE + "I",
						        SPIECE_UNDERLINE + "was",
						        SPIECE_UNDERLINE + "b",
						        "or",
						        "n",
						        SPIECE_UNDERLINE + "in",
						        SPIECE_UNDERLINE + "",
						        "<unk>",
						        "2",
						        "0",
						        "0",
						        "0",
						        ",",
						        SPIECE_UNDERLINE + "and",
						        SPIECE_UNDERLINE + "this",
						        SPIECE_UNDERLINE + "is",
						        SPIECE_UNDERLINE + "f",
						        "al",
						        "s",
						        "<unk>",
						        ".",
						    ]      ,						)




			@cached_property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						return ReformerTokenizer.from_pretrained("google/reformer-crime-and-punishment"     )




			@slow
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						UpperCamelCase__					=       "Hello World!"
						UpperCamelCase__					=       [126, 32, 262, 152, 38, 72, 287]

						self.assertListEqual(snake_case      ,						self.big_tokenizer.encode(snake_case     )     )




			@slow
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						UpperCamelCase__					=       (
						    "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
						    " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
						)
						UpperCamelCase__					=       [
						    108,
						    265,
						    24,
						    111,
						    4,
						    258,
						    156,
						    35,
						    28,
						    275,
						    3,
						    259,
						    297,
						    260,
						    84,
						    4,
						    35,
						    110,
						    44,
						    8,
						    259,
						    91,
						    268,
						    21,
						    11,
						    209,
						    274,
						    109,
						    266,
						    277,
						    117,
						    86,
						    93,
						    315,
						    258,
						    278,
						    258,
						    277,
						    258,
						    0,
						    258,
						    288,
						    258,
						    319,
						    258,
						    0,
						    258,
						    0,
						    258,
						    0,
						    258,
						    0,
						    258,
						    287,
						    258,
						    315,
						    258,
						    289,
						    258,
						    278,
						    99,
						    269,
						    266,
						    262,
						    8,
						    259,
						    241,
						    4,
						    217,
						    230,
						    268,
						    266,
						    55,
						    168,
						    106,
						    75,
						    193,
						    266,
						    223,
						    27,
						    49,
						    26,
						    282,
						    25,
						    264,
						    299,
						    19,
						    26,
						    0,
						    258,
						    277,
						    117,
						    86,
						    93,
						    176,
						    183,
						    270,
						    11,
						    262,
						    42,
						    61,
						    265,
						]

						self.assertListEqual(snake_case      ,						self.big_tokenizer.encode(snake_case     )     )




			@require_torch
			@slow
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						import torch

						from transformers import ReformerConfig, ReformerModel

						# Build sequence
						UpperCamelCase__					=       list(self.big_tokenizer.get_vocab().keys()     )[:10]
						UpperCamelCase__					=       " ".join(snake_case     )
						UpperCamelCase__					=       self.big_tokenizer.encode_plus(snake_case      ,						return_tensors="pt"     )
						UpperCamelCase__					=       self.big_tokenizer.batch_encode_plus([sequence, sequence]      ,						return_tensors="pt"     )

						UpperCamelCase__					=       ReformerConfig()
						# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
						UpperCamelCase__					=       encoded_sequence["input_ids"].shape
						UpperCamelCase__					=       ReformerModel(snake_case     )

						# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
						assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

						with torch.no_grad():
									model(**snake_case     )
									model(**snake_case     )







			@slow
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						UpperCamelCase__					=       {"input_ids": [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
						# fmt: on

						# This tokenizer does not know some characters like ")".
						# That is the reason why we use very simple texts here.
						# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
						UpperCamelCase__					=       [
						    "This is a very simple sentence.",
						    "The quick brown fox jumps over the lazy dog.",
						]

						self.tokenizer_integration_test_util(
						    expected_encoding=snake_case      ,						model_name="google/reformer-crime-and-punishment"      ,						revision="0e6c3decb8211d49bf881013425dc8b0448b3f5a"      ,						padding=snake_case      ,						sequences=snake_case      ,						)






import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


__UpperCamelCase    		=       logging.get_logger(__name__)
def        UpperCamelCase_(				)->		Any:
			# Get the sagemaker specific mp parameters from smp_options variable.
			UpperCamelCase__					=       os.getenv("SM_HP_MP_PARAMETERS"				,     "{}"  )
			try:
						# Parse it and check the field "partitions" is included, it is required for model parallel.
						UpperCamelCase__					=       json.loads(_A  )
						if "partitions" not in smp_options:
									return False
			except json.JSONDecodeError:
						return False

			# Get the sagemaker specific framework parameters from mpi_options variable.
			UpperCamelCase__					=       os.getenv("SM_FRAMEWORK_PARAMS"				,     "{}"  )
			try:
						# Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
						UpperCamelCase__					=       json.loads(_A  )
						if not mpi_options.get("sagemaker_mpi_enabled"				,     _A  ):
									return False
			except json.JSONDecodeError:
						return False
			# Lastly, check if the `smdistributed` module is present.
			return importlib.util.find_spec("smdistributed"  ) is not None


if is_sagemaker_model_parallel_available():
						import smdistributed.modelparallel.torch as smp

						smp.init()







@dataclass
class        lowerCamelCase__	(							UpperCAmelCase      ):






			"""simple docstring"""






			_UpperCamelCase		:					str    =		field(
			    default=''					,	metadata={'help': 'Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer'}					,	)




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						super().__post_init__()
						warnings.warn(
						    "`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use "
						    "`TrainingArguments` instead."      ,						snake_case      ,						)




			@cached_property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						logger.info("PyTorch: setting up devices"     )
						if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
									logger.warning(
									    "torch.distributed process group is initialized, but local_rank == -1. "
									    "In order to use Torch DDP, launch your script with `python -m torch.distributed.launch"     )
						if self.no_cuda:
									UpperCamelCase__					=       torch.device("cpu"     )
									UpperCamelCase__					=       0
						elif is_sagemaker_model_parallel_available():
									UpperCamelCase__					=       smp.local_rank()
									UpperCamelCase__					=       torch.device("cuda"      ,						snake_case     )
									UpperCamelCase__					=       1
						elif is_sagemaker_dp_enabled():
									import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

									torch.distributed.init_process_group(backend="smddp"      ,						timeout=self.ddp_timeout_delta     )
									UpperCamelCase__					=       int(os.getenv("SMDATAPARALLEL_LOCAL_RANK"     )     )
									UpperCamelCase__					=       torch.device("cuda"      ,						self.local_rank     )
									UpperCamelCase__					=       1
						elif self.local_rank == -1:
									# if n_gpu is > 1 we'll use nn.DataParallel.
									# If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
									# Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
									# trigger an error that a device index is missing. Index 0 takes into account the
									# GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
									# will use the first GPU in that env, i.e. GPU#1
									UpperCamelCase__					=       torch.device("cuda:0" if torch.cuda.is_available() else "cpu"     )
									# Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
									# the default value.
									UpperCamelCase__					=       torch.cuda.device_count()
						else:
									# Here, we'll use torch.distributed.
									# Initializes the distributed backend which will take care of synchronizing nodes/GPUs
									if not torch.distributed.is_initialized():
												torch.distributed.init_process_group(backend="nccl"      ,						timeout=self.ddp_timeout_delta     )
									UpperCamelCase__					=       torch.device("cuda"      ,						self.local_rank     )
									UpperCamelCase__					=       1

						if device.type == "cuda":
									torch.cuda.set_device(snake_case     )

						return device




			@property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						if is_sagemaker_model_parallel_available():
									return smp.dp_size()

						return super().world_size




			@property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						return not is_sagemaker_model_parallel_available()







			@property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						return False


