


'''simple docstring'''



import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin





class 							__UpperCamelCase    (		lowerCAmelCase_		,				unittest.TestCase ):
	A_           =   ProphetNetTokenizer
	A_           =   False




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						super().setUp()

						__a      :	Optional[Any]    =  [
						    '[UNK]',
						    '[CLS]',
						    '[SEP]',
						    '[PAD]',
						    '[MASK]',
						    'want',
						    '##want',
						    '##ed',
						    'wa',
						    'un',
						    'runn',
						    '##ing',
						    ',',
						    'low',
						    'lowest',
						]
						__a      :	Dict    =  os.path.join(self.tmpdirname						,    VOCAB_FILES_NAMES['vocab_file']							)
						with open(self.vocab_file						,    'w'						,    encoding='utf-8'							) as vocab_writer:
											vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]							)							)




	def 				__UpperCAmelCase							(       self						,    __a							):

						'''simple docstring'''





						__a      :	Optional[int]    =  'UNwant\u00E9d,running'
						__a      :	Optional[Any]    =  'unwanted, running'
						return input_text, output_text




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	List[Any]    =  self.tokenizer_class(self.vocab_file							)

						__a      :	Optional[int]    =  tokenizer.tokenize('UNwant\u00E9d,running'							)
						self.assertListEqual(__a						,    ['un', '##want', '##ed', ',', 'runn', '##ing']							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__a							)						,    [9, 6, 7, 12, 10, 11]							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Any    =  BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'							)						,    ['ah', '\u535A', '\u63A8', 'zz']							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Optional[int]    =  BasicTokenizer(do_lower_case=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '							)						,    ['hello', '!', 'how', 'are', 'you', '?']							)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)						,    ['hello']							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	str    =  BasicTokenizer(do_lower_case=__a						,    strip_accents=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)						,    ['hällo', '!', 'how', 'are', 'you', '?']							)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)						,    ['h\u00E9llo']							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	List[Any]    =  BasicTokenizer(do_lower_case=__a						,    strip_accents=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)						,    ['hallo', '!', 'how', 'are', 'you', '?']							)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)						,    ['hello']							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Any    =  BasicTokenizer(do_lower_case=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)						,    ['hallo', '!', 'how', 'are', 'you', '?']							)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'							)						,    ['hello']							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	List[Any]    =  BasicTokenizer(do_lower_case=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '							)						,    ['HeLLo', '!', 'how', 'Are', 'yoU', '?']							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Any    =  BasicTokenizer(do_lower_case=__a						,    strip_accents=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)						,    ['HäLLo', '!', 'how', 'Are', 'yoU', '?']							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Union[str, Any]    =  BasicTokenizer(do_lower_case=__a						,    strip_accents=__a							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '							)						,    ['HaLLo', '!', 'how', 'Are', 'yoU', '?']							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	str    =  BasicTokenizer(do_lower_case=__a						,    never_split=['[UNK]']							)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'							)						,    ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Optional[int]    =  ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

						__a      :	Any    =  {}
						for i, token in enumerate(__a							):
											__a      :	int    =  i
						__a      :	Tuple    =  WordpieceTokenizer(vocab=__a						,    unk_token='[UNK]'							)

						self.assertListEqual(tokenizer.tokenize(''							)						,    []							)

						self.assertListEqual(tokenizer.tokenize('unwanted running'							)						,    ['un', '##want', '##ed', 'runn', '##ing']							)

						self.assertListEqual(tokenizer.tokenize('unwantedX running'							)						,    ['[UNK]', 'runn', '##ing']							)




	@require_torch
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	List[str]    =  self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'							)

						__a      :	Optional[Any]    =  ['A long paragraph for summarization.', 'Another paragraph for summarization.']
						__a      :	List[str]    =  [1037, 2146, 2_0423, 2005, 7680, 7849, 3989, 1012, 102]
						__a      :	List[str]    =  tokenizer(__a						,    padding=__a						,    return_tensors='pt'							)
						self.assertIsInstance(__a						,    __a							)
						__a      :	Optional[int]    =  list(batch.input_ids.numpy()[0]							)
						self.assertListEqual(__a						,    __a							)

						self.assertEqual((2, 9)						,    batch.input_ids.shape							)
						self.assertEqual((2, 9)						,    batch.attention_mask.shape							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						self.assertTrue(_is_whitespace(' '							)							)
						self.assertTrue(_is_whitespace('\t'							)							)
						self.assertTrue(_is_whitespace('\r'							)							)
						self.assertTrue(_is_whitespace('\n'							)							)
						self.assertTrue(_is_whitespace('\u00A0'							)							)

						self.assertFalse(_is_whitespace('A'							)							)
						self.assertFalse(_is_whitespace('-'							)							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						self.assertTrue(_is_control('\u0005'							)							)

						self.assertFalse(_is_control('A'							)							)
						self.assertFalse(_is_control(' '							)							)
						self.assertFalse(_is_control('\t'							)							)
						self.assertFalse(_is_control('\r'							)							)




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						self.assertTrue(_is_punctuation('-'							)							)
						self.assertTrue(_is_punctuation('$'							)							)
						self.assertTrue(_is_punctuation('`'							)							)
						self.assertTrue(_is_punctuation('.'							)							)

						self.assertFalse(_is_punctuation('A'							)							)
						self.assertFalse(_is_punctuation(' '							)							)





	@slow
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	List[Any]    =  self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'							)

						__a      :	int    =  tokenizer.encode('sequence builders'						,    add_special_tokens=__a							)
						__a      :	Union[str, Any]    =  tokenizer.encode('multi-sequence build'						,    add_special_tokens=__a							)

						__a      :	List[Any]    =  tokenizer.build_inputs_with_special_tokens(__a							)
						__a      :	Dict    =  tokenizer.build_inputs_with_special_tokens(__a						,    __a							)

						assert encoded_sentence == text + [102]
						assert encoded_pair == text + [102] + text_a + [102]




'''simple docstring'''



import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
	from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
	import torch_xla.core.xla_model as xm






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       Optional[Any]       ):
					if is_torch_version('<'						,				'2.0.0'       ) or not hasattr(_SCREAMING_SNAKE_CASE						,				'_dynamo'       ):
										return False
					return isinstance(_SCREAMING_SNAKE_CASE						,				torch._dynamo.eval_frame.OptimizedModule       )






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       Optional[Any]						,				_SCREAMING_SNAKE_CASE				:       bool = True       ):
					__a      :	int    =  (torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

					__a      :	Any    =  is_compiled_module(_SCREAMING_SNAKE_CASE       )
					if is_compiled:
										__a      :	List[Any]    =  model
										__a      :	Union[str, Any]    =  model._orig_mod

					if is_deepspeed_available():
										options += (DeepSpeedEngine,)

					while isinstance(_SCREAMING_SNAKE_CASE						,				_SCREAMING_SNAKE_CASE       ):
										__a      :	Union[str, Any]    =  model.module

					if not keep_fpaa_wrapper:
										__a      :	Optional[Any]    =  getattr(_SCREAMING_SNAKE_CASE						,				'forward'       )
										__a      :	str    =  model.__dict__.pop('_original_forward'						,				_SCREAMING_SNAKE_CASE       )
										if original_forward is not None:
															while hasattr(_SCREAMING_SNAKE_CASE						,				'__wrapped__'       ):
																				__a      :	Any    =  forward.__wrapped__
																				if forward == original_forward:
																									break
															__a      :	str    =  forward
										if getattr(_SCREAMING_SNAKE_CASE						,				'_converted_to_transformer_engine'						,				_SCREAMING_SNAKE_CASE       ):
															convert_model(_SCREAMING_SNAKE_CASE						,				to_transformer_engine=_SCREAMING_SNAKE_CASE       )

					if is_compiled:
										__a      :	List[str]    =  model
										__a      :	Optional[int]    =  compiled_model

					return model






def   lowerCamelCase							():
					PartialState().wait_for_everyone()






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       int						,				_SCREAMING_SNAKE_CASE				:       Tuple       ):
					if PartialState().distributed_type == DistributedType.TPU:
										xm.save(_SCREAMING_SNAKE_CASE						,				_SCREAMING_SNAKE_CASE       )
					elif PartialState().local_process_index == 0:
										torch.save(_SCREAMING_SNAKE_CASE						,				_SCREAMING_SNAKE_CASE       )






@contextmanager
def   lowerCamelCase							(**_SCREAMING_SNAKE_CASE				:       Tuple       ):
					for key, value in kwargs.items():
										__a      :	Optional[int]    =  str(_SCREAMING_SNAKE_CASE       )

					yield

					for key in kwargs:
										if key.upper() in os.environ:
															del os.environ[key.upper()]






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       Dict       ):
					if not hasattr(_SCREAMING_SNAKE_CASE						,				'__qualname__'       ) and not hasattr(_SCREAMING_SNAKE_CASE						,				'__name__'       ):
										__a      :	List[Any]    =  getattr(_SCREAMING_SNAKE_CASE						,				'__class__'						,				_SCREAMING_SNAKE_CASE       )
					if hasattr(_SCREAMING_SNAKE_CASE						,				'__qualname__'       ):
										return obj.__qualname__
					if hasattr(_SCREAMING_SNAKE_CASE						,				'__name__'       ):
										return obj.__name__
					return str(_SCREAMING_SNAKE_CASE       )






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       int						,				_SCREAMING_SNAKE_CASE				:       List[str]       ):
					for key, value in source.items():
										if isinstance(_SCREAMING_SNAKE_CASE						,				_SCREAMING_SNAKE_CASE       ):
															__a      :	int    =  destination.setdefault(_SCREAMING_SNAKE_CASE						,				{}       )
															merge_dicts(_SCREAMING_SNAKE_CASE						,				_SCREAMING_SNAKE_CASE       )
										else:
															__a      :	Tuple    =  value

					return destination






def   lowerCamelCase							(_SCREAMING_SNAKE_CASE				:       int = None       ):
					if port is None:
										__a      :	List[str]    =  29_500
					with socket.socket(socket.AF_INET						,				socket.SOCK_STREAM       ) as s:
										return s.connect_ex(('localhost', port)       ) == 0



