


import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin

class 						A__							(		UpperCamelCase__   ,     unittest.TestCase  ):
						UpperCAmelCase             =						ProphetNetTokenizer
						UpperCAmelCase             =						False




						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     int:


										"""simple docstring"""





										super().setUp()

										_SCREAMING_SNAKE_CASE			  =[
										    '''[UNK]''',
										    '''[CLS]''',
										    '''[SEP]''',
										    '''[PAD]''',
										    '''[MASK]''',
										    '''want''',
										    '''##want''',
										    '''##ed''',
										    '''wa''',
										    '''un''',
										    '''runn''',
										    '''##ing''',
										    ''',''',
										    '''low''',
										    '''lowest''',
										]
										_SCREAMING_SNAKE_CASE			  =os.path.join(self.tmpdirname				,    VOCAB_FILES_NAMES['''vocab_file''']  )
										with open(self.vocab_file				,    '''w'''				,    encoding='''utf-8'''  ) as vocab_writer:
														vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )




						def 							__UpperCamelCase				(       self					:      str				,    _a					:      Union[str, Any]  )							->     List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''UNwant\u00E9d,running'''
										_SCREAMING_SNAKE_CASE			  ='''unwanted, running'''
										return input_text, output_text




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.tokenizer_class(self.vocab_file  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.tokenize('''UNwant\u00E9d,running'''  )
										self.assertListEqual(_a				,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(_a  )				,    [9, 6, 7, 12, 10, 11]  )




						def 							__UpperCamelCase				(       self					:      Tuple  )							->     Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''  )				,    ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )				,    ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )				,    ['''hello''']  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a				,    strip_accents=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )				,    ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )				,    ['''h\u00E9llo''']  )




						def 							__UpperCamelCase				(       self					:      List[str]  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a				,    strip_accents=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )				,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )				,    ['''hello''']  )




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )				,    ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )				,    ['''hello''']  )




						def 							__UpperCamelCase				(       self					:      Any  )							->     Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )				,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a				,    strip_accents=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )				,    ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )




						def 							__UpperCamelCase				(       self					:      Optional[Any]  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a				,    strip_accents=_a  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )				,    ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )




						def 							__UpperCamelCase				(       self					:      List[str]  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =BasicTokenizer(do_lower_case=_a				,    never_split=['''[UNK]''']  )

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''  )				,    ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']  )




						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

										_SCREAMING_SNAKE_CASE			  ={}
										for i, token in enumerate(_a  ):
														_SCREAMING_SNAKE_CASE			  =i
										_SCREAMING_SNAKE_CASE			  =WordpieceTokenizer(vocab=_a				,    unk_token='''[UNK]'''  )

										self.assertListEqual(tokenizer.tokenize(''''''  )				,    []  )

										self.assertListEqual(tokenizer.tokenize('''unwanted running'''  )				,    ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']  )

										self.assertListEqual(tokenizer.tokenize('''unwantedX running'''  )				,    ['''[UNK]''', '''runn''', '''##ing''']  )




						@require_torch
						def 							__UpperCamelCase				(       self					:      Any  )							->     Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''  )

										_SCREAMING_SNAKE_CASE			  =['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
										_SCREAMING_SNAKE_CASE			  =[1037, 2146, 2_0423, 2005, 7680, 7849, 3989, 1012, 102]
										_SCREAMING_SNAKE_CASE			  =tokenizer(_a				,    padding=_a				,    return_tensors='''pt'''  )
										self.assertIsInstance(_a				,    _a  )
										_SCREAMING_SNAKE_CASE			  =list(batch.input_ids.numpy()[0]  )
										self.assertListEqual(_a				,    _a  )

										self.assertEqual((2, 9)				,    batch.input_ids.shape  )
										self.assertEqual((2, 9)				,    batch.attention_mask.shape  )




						def 							__UpperCamelCase				(       self					:      List[str]  )							->     Optional[int]:


										"""simple docstring"""





										self.assertTrue(_is_whitespace(''' '''  )  )
										self.assertTrue(_is_whitespace('''\t'''  )  )
										self.assertTrue(_is_whitespace('''\r'''  )  )
										self.assertTrue(_is_whitespace('''\n'''  )  )
										self.assertTrue(_is_whitespace('''\u00A0'''  )  )

										self.assertFalse(_is_whitespace('''A'''  )  )
										self.assertFalse(_is_whitespace('''-'''  )  )




						def 							__UpperCamelCase				(       self					:      int  )							->     Dict:


										"""simple docstring"""





										self.assertTrue(_is_control('''\u0005'''  )  )

										self.assertFalse(_is_control('''A'''  )  )
										self.assertFalse(_is_control(''' '''  )  )
										self.assertFalse(_is_control('''\t'''  )  )
										self.assertFalse(_is_control('''\r'''  )  )




						def 							__UpperCamelCase				(       self					:      List[str]  )							->     List[str]:


										"""simple docstring"""





										self.assertTrue(_is_punctuation('''-'''  )  )
										self.assertTrue(_is_punctuation('''$'''  )  )
										self.assertTrue(_is_punctuation('''`'''  )  )
										self.assertTrue(_is_punctuation('''.'''  )  )

										self.assertFalse(_is_punctuation('''A'''  )  )
										self.assertFalse(_is_punctuation(''' '''  )  )




						@slow
						def 							__UpperCamelCase				(       self					:      Tuple  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.encode('''sequence builders'''				,    add_special_tokens=_a  )
										_SCREAMING_SNAKE_CASE			  =tokenizer.encode('''multi-sequence build'''				,    add_special_tokens=_a  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.build_inputs_with_special_tokens(_a  )
										_SCREAMING_SNAKE_CASE			  =tokenizer.build_inputs_with_special_tokens(_a				,    _a  )

										assert encoded_sentence == text + [102]
										assert encoded_pair == text + [102] + text_a + [102]



import unittest

import numpy as np
import requests

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
							import torch

							from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
							snake_case_						:			Any													=							False

if is_vision_available():
							from PIL import Image

							from transformers import PixaStructImageProcessor

class 						A__							(		unittest.TestCase  ):




						def __init__(       self					:      Tuple				,    _a					:      str				,    _a					:      List[str]=7				,    _a					:      Union[str, Any]=3				,    _a					:      List[str]=18				,    _a					:      List[Any]=30				,    _a					:      Optional[Any]=400				,    _a					:      Dict=None				,    _a					:      Union[str, Any]=True				,    _a					:      List[str]=True				,    _a					:      Optional[Any]=None				,    )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =size if size is not None else {'''height''': 20, '''width''': 20}
										_SCREAMING_SNAKE_CASE			  =parent
										_SCREAMING_SNAKE_CASE			  =batch_size
										_SCREAMING_SNAKE_CASE			  =num_channels
										_SCREAMING_SNAKE_CASE			  =image_size
										_SCREAMING_SNAKE_CASE			  =min_resolution
										_SCREAMING_SNAKE_CASE			  =max_resolution
										_SCREAMING_SNAKE_CASE			  =size
										_SCREAMING_SNAKE_CASE			  =do_normalize
										_SCREAMING_SNAKE_CASE			  =do_convert_rgb
										_SCREAMING_SNAKE_CASE			  =[512, 1024, 2048, 4096]
										_SCREAMING_SNAKE_CASE			  =patch_size if patch_size is not None else {'''height''': 16, '''width''': 16}




						def 							__UpperCamelCase				(       self					:      int  )							->     Any:


										"""simple docstring"""





										return {"do_normalize": self.do_normalize, "do_convert_rgb": self.do_convert_rgb}




						def 							__UpperCamelCase				(       self					:      str  )							->     Optional[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/australia.jpg'''
										_SCREAMING_SNAKE_CASE			  =Image.open(requests.get(_a				,    stream=_a  ).raw  ).convert('''RGB'''  )
										return raw_image

@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11   ,     reason="`Pix2StructImageProcessor` requires `torch>=1.11.0`."   ,     )
@require_torch
@require_vision
class 						A__							(		UpperCamelCase__   ,     unittest.TestCase  ):
						UpperCAmelCase             =						PixaStructImageProcessor if is_vision_available() else None




						def 							__UpperCamelCase				(       self					:      int  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =PixaStructImageProcessingTester(self  )




						@property
						def 							__UpperCamelCase				(       self					:      int  )							->     Any:


										"""simple docstring"""





										return self.image_processor_tester.prepare_image_processor_dict()




						def 							__UpperCamelCase				(       self					:      str  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.image_processing_class(**self.image_processor_dict  )
										self.assertTrue(hasattr(_a				,    '''do_normalize'''  )  )
										self.assertTrue(hasattr(_a				,    '''do_convert_rgb'''  )  )




						def 							__UpperCamelCase				(       self					:      int  )							->     Optional[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.image_processor_tester.prepare_dummy_image()

										_SCREAMING_SNAKE_CASE			  =self.image_processing_class(**self.image_processor_dict  )
										_SCREAMING_SNAKE_CASE			  =2048

										_SCREAMING_SNAKE_CASE			  =image_processor(_a				,    return_tensors='''pt'''				,    max_patches=_a  )
										self.assertTrue(torch.allclose(inputs.flattened_patches.mean()				,    torch.tensor(0.06_06  )				,    atol=1E-3				,    rtol=1E-3  )  )




						def 							__UpperCamelCase				(       self					:      Tuple  )							->     Optional[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.image_processing_class(**self.image_processor_dict  )
										# create random PIL images
										_SCREAMING_SNAKE_CASE			  =prepare_image_inputs(self.image_processor_tester				,    equal_resolution=_a  )
										for image in image_inputs:
														self.assertIsInstance(_a				,    Image.Image  )

										# Test not batched input
										_SCREAMING_SNAKE_CASE			  =(
										    (self.image_processor_tester.patch_size['''height'''] * self.image_processor_tester.patch_size['''width'''])
										    * self.image_processor_tester.num_channels
										) + 2

										for max_patch in self.image_processor_tester.max_patches:
														# Test not batched input
														_SCREAMING_SNAKE_CASE			  =image_processor(
														    image_inputs[0]				,    return_tensors='''pt'''				,    max_patches=_a  ).flattened_patches
														self.assertEqual(
														    encoded_images.shape				,    (1, max_patch, expected_hidden_dim)				,    )

														# Test batched
														_SCREAMING_SNAKE_CASE			  =image_processor(
														    _a				,    return_tensors='''pt'''				,    max_patches=_a  ).flattened_patches
														self.assertEqual(
														    encoded_images.shape				,    (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)				,    )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.image_processing_class(**self.image_processor_dict  )
										# create random PIL images
										_SCREAMING_SNAKE_CASE			  =prepare_image_inputs(self.image_processor_tester				,    equal_resolution=_a  )
										for image in image_inputs:
														self.assertIsInstance(_a				,    Image.Image  )

										# Test not batched input
										_SCREAMING_SNAKE_CASE			  =(
										    (self.image_processor_tester.patch_size['''height'''] * self.image_processor_tester.patch_size['''width'''])
										    * self.image_processor_tester.num_channels
										) + 2

										_SCREAMING_SNAKE_CASE			  =True

										for max_patch in self.image_processor_tester.max_patches:
														# Test not batched input
														with self.assertRaises(_a  ):
																		_SCREAMING_SNAKE_CASE			  =image_processor(
																		    image_inputs[0]				,    return_tensors='''pt'''				,    max_patches=_a  ).flattened_patches

														_SCREAMING_SNAKE_CASE			  ='''Hello'''

														_SCREAMING_SNAKE_CASE			  =image_processor(
														    image_inputs[0]				,    return_tensors='''pt'''				,    max_patches=_a				,    header_text=_a  ).flattened_patches
														self.assertEqual(
														    encoded_images.shape				,    (1, max_patch, expected_hidden_dim)				,    )

														# Test batched
														_SCREAMING_SNAKE_CASE			  =image_processor(
														    _a				,    return_tensors='''pt'''				,    max_patches=_a				,    header_text=_a  ).flattened_patches
														self.assertEqual(
														    encoded_images.shape				,    (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)				,    )




						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.image_processing_class(**self.image_processor_dict  )
										# create random numpy tensors
										_SCREAMING_SNAKE_CASE			  =prepare_image_inputs(self.image_processor_tester				,    equal_resolution=_a				,    numpify=_a  )
										for image in image_inputs:
														self.assertIsInstance(_a				,    np.ndarray  )

										_SCREAMING_SNAKE_CASE			  =(
										    (self.image_processor_tester.patch_size['''height'''] * self.image_processor_tester.patch_size['''width'''])
										    * self.image_processor_tester.num_channels
										) + 2

										for max_patch in self.image_processor_tester.max_patches:
														# Test not batched input
														_SCREAMING_SNAKE_CASE			  =image_processor(
														    image_inputs[0]				,    return_tensors='''pt'''				,    max_patches=_a  ).flattened_patches
														self.assertEqual(
														    encoded_images.shape				,    (1, max_patch, expected_hidden_dim)				,    )

														# Test batched
														_SCREAMING_SNAKE_CASE			  =image_processor(
														    _a				,    return_tensors='''pt'''				,    max_patches=_a  ).flattened_patches
														self.assertEqual(
														    encoded_images.shape				,    (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)				,    )




						def 							__UpperCamelCase				(       self					:      Union[str, Any]  )							->     List[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.image_processing_class(**self.image_processor_dict  )
										# create random PyTorch tensors
										_SCREAMING_SNAKE_CASE			  =prepare_image_inputs(self.image_processor_tester				,    equal_resolution=_a				,    torchify=_a  )
										for image in image_inputs:
														self.assertIsInstance(_a				,    torch.Tensor  )

										# Test not batched input
										_SCREAMING_SNAKE_CASE			  =(
										    (self.image_processor_tester.patch_size['''height'''] * self.image_processor_tester.patch_size['''width'''])
										    * self.image_processor_tester.num_channels
										) + 2

										for max_patch in self.image_processor_tester.max_patches:
														# Test not batched input
														_SCREAMING_SNAKE_CASE			  =image_processor(
														    image_inputs[0]				,    return_tensors='''pt'''				,    max_patches=_a  ).flattened_patches
														self.assertEqual(
														    encoded_images.shape				,    (1, max_patch, expected_hidden_dim)				,    )

														# Test batched
														_SCREAMING_SNAKE_CASE			  =image_processor(
														    _a				,    return_tensors='''pt'''				,    max_patches=_a  ).flattened_patches
														self.assertEqual(
														    encoded_images.shape				,    (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)				,    )

@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11   ,     reason="`Pix2StructImageProcessor` requires `torch>=1.11.0`."   ,     )
@require_torch
@require_vision
class 						A__							(		UpperCamelCase__   ,     unittest.TestCase  ):
						UpperCAmelCase             =						PixaStructImageProcessor if is_vision_available() else None




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =PixaStructImageProcessingTester(self				,    num_channels=4  )
										_SCREAMING_SNAKE_CASE			  =3




						@property
						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     List[str]:


										"""simple docstring"""





										return self.image_processor_tester.prepare_image_processor_dict()




						def 							__UpperCamelCase				(       self					:      Dict  )							->     Optional[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.image_processing_class(**self.image_processor_dict  )
										self.assertTrue(hasattr(_a				,    '''do_normalize'''  )  )
										self.assertTrue(hasattr(_a				,    '''do_convert_rgb'''  )  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.image_processing_class(**self.image_processor_dict  )
										# create random PIL images
										_SCREAMING_SNAKE_CASE			  =prepare_image_inputs(self.image_processor_tester				,    equal_resolution=_a  )
										for image in image_inputs:
														self.assertIsInstance(_a				,    Image.Image  )

										# Test not batched input
										_SCREAMING_SNAKE_CASE			  =(
										    (self.image_processor_tester.patch_size['''height'''] * self.image_processor_tester.patch_size['''width'''])
										    * (self.image_processor_tester.num_channels - 1)
										) + 2

										for max_patch in self.image_processor_tester.max_patches:
														# Test not batched input
														_SCREAMING_SNAKE_CASE			  =image_processor(
														    image_inputs[0]				,    return_tensors='''pt'''				,    max_patches=_a  ).flattened_patches
														self.assertEqual(
														    encoded_images.shape				,    (1, max_patch, expected_hidden_dim)				,    )

														# Test batched
														_SCREAMING_SNAKE_CASE			  =image_processor(
														    _a				,    return_tensors='''pt'''				,    max_patches=_a  ).flattened_patches
														self.assertEqual(
														    encoded_images.shape				,    (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)				,    )