
"""simple docstring"""






import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin



class 			lowercase					( lowercase__   ,unittest.TestCase		):
			lowercase        =     DebertaTokenizer
			lowercase        =     True
			lowercase        =     DebertaTokenizerFast






			def 					UpperCAmelCase							(self  :	Any			)       ->	str:





				"""simple docstring"""


				super().setUp()

				# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
				lowerCAmelCase												=  [
				    '''l''',
				    '''o''',
				    '''w''',
				    '''e''',
				    '''r''',
				    '''s''',
				    '''t''',
				    '''i''',
				    '''d''',
				    '''n''',
				    '''\u0120''',
				    '''\u0120l''',
				    '''\u0120n''',
				    '''\u0120lo''',
				    '''\u0120low''',
				    '''er''',
				    '''\u0120lowest''',
				    '''\u0120newer''',
				    '''\u0120wider''',
				    '''[UNK]''',
				]
				lowerCAmelCase												=  dict(zip(SCREAMING_SNAKE_CASE_      ,range(len(SCREAMING_SNAKE_CASE_			)			)			)			)
				lowerCAmelCase												=  ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
				lowerCAmelCase												=  {'''unk_token''': '''[UNK]'''}

				lowerCAmelCase												=  os.path.join(self.tmpdirname      ,VOCAB_FILES_NAMES['''vocab_file''']			)
				lowerCAmelCase												=  os.path.join(self.tmpdirname      ,VOCAB_FILES_NAMES['''merges_file''']			)
				with open(self.vocab_file      ,'''w'''      ,encoding='''utf-8'''			) as fp:
					fp.write(json.dumps(SCREAMING_SNAKE_CASE_			) + '''\n'''			)
				with open(self.merges_file      ,'''w'''      ,encoding='''utf-8'''			) as fp:
					fp.write('''\n'''.join(SCREAMING_SNAKE_CASE_			)			)






			def 					UpperCAmelCase							(self  :	Tuple      ,**SCREAMING_SNAKE_CASE_  :	str			)       ->	List[str]:





				"""simple docstring"""


				kwargs.update(self.special_tokens_map			)
				return self.tokenizer_class.from_pretrained(self.tmpdirname      ,**SCREAMING_SNAKE_CASE_			)






			def 					UpperCAmelCase							(self  :	Tuple      ,SCREAMING_SNAKE_CASE_  :	Optional[Any]			)       ->	Optional[int]:





				"""simple docstring"""


				lowerCAmelCase												=  '''lower newer'''
				lowerCAmelCase												=  '''lower newer'''
				return input_text, output_text






			def 					UpperCAmelCase							(self  :	str			)       ->	Any:





				"""simple docstring"""


				lowerCAmelCase												=  self.get_tokenizer()
				lowerCAmelCase												=  '''lower newer'''
				lowerCAmelCase												=  ['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
				lowerCAmelCase												=  tokenizer.tokenize(SCREAMING_SNAKE_CASE_			)
				self.assertListEqual(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)

				lowerCAmelCase												=  tokens + [tokenizer.unk_token]
				lowerCAmelCase												=  [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
				self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_			)      ,SCREAMING_SNAKE_CASE_			)






			def 					UpperCAmelCase							(self  :	Dict			)       ->	Dict:





				"""simple docstring"""


				lowerCAmelCase												=  self.get_tokenizer()
				lowerCAmelCase												=  tokenizer('''Hello'''      ,'''World'''			)
				lowerCAmelCase												=  [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
				self.assertListEqual(tokd['''token_type_ids''']      ,SCREAMING_SNAKE_CASE_			)






			@slow
			def 					UpperCAmelCase							(self  :	List[str]			)       ->	Tuple:





				"""simple docstring"""


				lowerCAmelCase												=  self.tokenizer_class.from_pretrained('''microsoft/deberta-base'''			)

				lowerCAmelCase												=  tokenizer.encode('''sequence builders'''      ,add_special_tokens=SCREAMING_SNAKE_CASE_			)
				lowerCAmelCase												=  tokenizer.encode('''multi-sequence build'''      ,add_special_tokens=SCREAMING_SNAKE_CASE_			)

				lowerCAmelCase												=  tokenizer.encode(
				    '''sequence builders'''      ,add_special_tokens=SCREAMING_SNAKE_CASE_      ,add_prefix_space=SCREAMING_SNAKE_CASE_			)
				lowerCAmelCase												=  tokenizer.encode(
				    '''sequence builders'''      ,'''multi-sequence build'''      ,add_special_tokens=SCREAMING_SNAKE_CASE_      ,add_prefix_space=SCREAMING_SNAKE_CASE_			)

				lowerCAmelCase												=  tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_			)
				lowerCAmelCase												=  tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)

				assert encoded_sentence == encoded_text_from_decode
				assert encoded_pair == encoded_pair_from_decode






			@slow
			def 					UpperCAmelCase							(self  :	List[str]			)       ->	List[str]:





				"""simple docstring"""


				lowerCAmelCase												=  [self.tokenizer_class]
				if self.test_rust_tokenizer:
					tokenizer_classes.append(self.rust_tokenizer_class			)

				for tokenizer_class in tokenizer_classes:
					lowerCAmelCase												=  tokenizer_class.from_pretrained('''microsoft/deberta-base'''			)

					lowerCAmelCase												=  [
					    '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
					    '''ALBERT incorporates two parameter reduction techniques''',
					    '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
					    ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
					    ''' vocabulary embedding.''',
					]

					lowerCAmelCase												=  tokenizer(SCREAMING_SNAKE_CASE_      ,padding=SCREAMING_SNAKE_CASE_			)
					lowerCAmelCase												=  [tokenizer.decode(SCREAMING_SNAKE_CASE_      ,skip_special_tokens=SCREAMING_SNAKE_CASE_			) for seq in encoding['''input_ids''']]

					# fmt: off
					lowerCAmelCase												=  {
					    '''input_ids''': [
					        [1, 2_118, 11_126, 565, 35, 83, 25_191, 163, 18_854, 13, 12_156, 12, 16_101, 25_376, 13_807, 9, 22_205, 27_893, 1_635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
					        [1, 2_118, 11_126, 565, 24_536, 80, 43_797, 4_878, 7_373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
					        [1, 133, 78, 65, 16, 10, 3_724, 1_538, 33_183, 11_303, 43_797, 1_938, 4, 870, 24_165, 29_105, 5, 739, 32_644, 33_183, 11_303, 36_173, 88, 80, 650, 7_821, 45_940, 6, 52, 2_559, 5, 1_836, 9, 5, 7_397, 13_171, 31, 5, 1_836, 9, 32_644, 33_183, 11_303, 4, 2]
					    ],
					    '''token_type_ids''': [
					        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
					        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
					        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
					    ],
					    '''attention_mask''': [
					        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
					        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
					        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
					    ]
					}
					# fmt: on

					lowerCAmelCase												=  [
					    '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
					    '''ALBERT incorporates two parameter reduction techniques''',
					    '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
					    ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
					    ''' vocabulary embedding.''',
					]

					self.assertDictEqual(encoding.data      ,SCREAMING_SNAKE_CASE_			)

					for expected, decoded in zip(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			):
						self.assertEqual(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_			)




"""simple docstring"""






from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_torch_dist_unique_port,
    require_torch_multi_gpu,
    require_torch_neuroncore,
)
from transformers.training_args import ParallelMode
from transformers.utils import logging


UpperCAmelCase        =				logging.get_logger(__name__)


if is_torch_available():
					import torch
					from torch import nn
					from torch.utils.data import Dataset

					from transformers import Trainer



					class 			lowercase					( lowercase__		):






								def __init__(self  :	int      ,SCREAMING_SNAKE_CASE_  :	int = 101			)       ->	List[str]:





									"""simple docstring"""


									lowerCAmelCase												=  length






								def __len__(self  :	Any			)       ->	List[Any]:





									"""simple docstring"""


									return self.length






								def __getitem__(self  :	str      ,SCREAMING_SNAKE_CASE_  :	Union[str, Any]			)       ->	int:





									"""simple docstring"""


									return i





					class 			lowercase					:






								def __call__(self  :	List[Any]      ,SCREAMING_SNAKE_CASE_  :	List[Any]			)       ->	Optional[Any]:





									"""simple docstring"""


									return {"input_ids": torch.tensor(SCREAMING_SNAKE_CASE_			), "labels": torch.tensor(SCREAMING_SNAKE_CASE_			)}





					class 			lowercase					( nn.Module		):






								def __init__(self  :	List[str]			)       ->	List[Any]:





									"""simple docstring"""


									super().__init__()
									# Add some (unused) params otherwise DDP will complain.
									lowerCAmelCase												=  nn.Linear(120      ,80			)






								def 					UpperCAmelCase							(self  :	Optional[int]      ,SCREAMING_SNAKE_CASE_  :	str      ,SCREAMING_SNAKE_CASE_  :	Optional[int]=None			)       ->	Union[str, Any]:





									"""simple docstring"""


									if labels is not None:
										return torch.tensor(0.0      ,device=input_ids.device			), input_ids
									else:
										return input_ids



class 			lowercase					( lowercase__		):






			@require_torch_neuroncore
			def 					UpperCAmelCase							(self  :	Union[str, Any]			)       ->	Union[str, Any]:





				"""simple docstring"""


				lowerCAmelCase												=  F"""--nproc_per_node=2
            --master_port={get_torch_dist_unique_port()}
            {self.test_file_dir}/test_trainer_distributed.py
        """.split()
				lowerCAmelCase												=  self.get_auto_remove_tmp_dir()
				lowerCAmelCase												=  F"""--output_dir {output_dir}""".split()
				lowerCAmelCase												=  ['''torchrun'''] + distributed_args + args
				execute_subprocess_async(SCREAMING_SNAKE_CASE_      ,env=self.get_env()			)
				# successful return here == success - any errors would have caused an error in the sub-call





class 			lowercase					( lowercase__		):






			@require_torch_multi_gpu
			def 					UpperCAmelCase							(self  :	List[str]			)       ->	List[str]:





				"""simple docstring"""


				lowerCAmelCase												=  F"""--nproc_per_node={torch.cuda.device_count()}
            --master_port={get_torch_dist_unique_port()}
            {self.test_file_dir}/test_trainer_distributed.py
        """.split()
				lowerCAmelCase												=  self.get_auto_remove_tmp_dir()
				lowerCAmelCase												=  F"""--output_dir {output_dir}""".split()
				lowerCAmelCase												=  ['''torchrun'''] + distributed_args + args
				execute_subprocess_async(SCREAMING_SNAKE_CASE_      ,env=self.get_env()			)
				# successful return here == success - any errors would have caused an error in the sub-call


if __name__ == "__main__":
					# The script below is meant to be run under torch.distributed, on a machine with multiple GPUs:
					#
					# PYTHONPATH="src" python -m torch.distributed.run --nproc_per_node 2 --output_dir output_dir ./tests/test_trainer_distributed.py

					UpperCAmelCase        =				HfArgumentParser((TrainingArguments,))
					UpperCAmelCase        =				parser.parse_args_into_dataclasses()[0]

					logger.warning(
					    f"""Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, """
					    f"""distributed training: {training_args.parallel_mode != ParallelMode.NOT_DISTRIBUTED}"""
					)

					# Essentially, what we want to verify in the distributed case is that we get all samples back,
					# in the right order. (this is crucial for prediction for instance)
					for dataset_length in [1_0_1, 4_0, 7]:
										UpperCAmelCase        =				DummyDataset(dataset_length)




										def __magic_name__	(				_lowerCamelCase:		EvalPrediction			)					->						Dict:

											'''simple docstring'''




											lowerCAmelCase												=  list(range(len(_lowerCamelCase			)			)			)
											lowerCAmelCase												=  p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
											if not success and training_args.local_rank == 0:
												logger.warning(
												    '''Predictions and/or labels do not match expected results:\n  - predictions: '''
												    F"""{p.predictions.tolist()}\n  - labels: {p.label_ids.tolist()}\n  - expected: {sequential}"""			)
											return {"success": success}

										UpperCAmelCase        =				Trainer(
										    model=DummyModel(),
										    args=training_args,
										    data_collator=DummyDataCollator(),
										    eval_dataset=dataset,
										    compute_metrics=compute_metrics,
										)
										UpperCAmelCase        =				trainer.evaluate()
										logger.info(metrics)
										if metrics["eval_success"] is not True:
															logger.error(metrics)
															exit(1)

										UpperCAmelCase        =				trainer.predict(dataset)
										logger.info(p.metrics)
										if p.metrics["test_success"] is not True:
															logger.error(p.metrics)
															exit(1)

										UpperCAmelCase        =				2

										UpperCAmelCase        =				trainer.evaluate()
										logger.info(metrics)
										if metrics["eval_success"] is not True:
															logger.error(metrics)
															exit(1)

										UpperCAmelCase        =				trainer.predict(dataset)
										logger.info(p.metrics)
										if p.metrics["test_success"] is not True:
															logger.error(p.metrics)
															exit(1)



										UpperCAmelCase        =				None


