


import unittest

import numpy as np
import requests

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
			import torch

			from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
			__snake_case            =   False

if is_vision_available():
			from PIL import Image

			from transformers import PixaStructImageProcessor




class    UpperCAmelCase					(       unittest.TestCase    ):




			def __init__(						self							:		Optional[int]       ,       __magic_name__							:		Union[str, Any]       ,       __magic_name__							:		str=7       ,       __magic_name__							:		List[str]=3       ,       __magic_name__							:		List[str]=1_8       ,       __magic_name__							:		List[Any]=3_0       ,       __magic_name__							:		Optional[int]=4_0_0       ,       __magic_name__							:		List[Any]=None       ,       __magic_name__							:		Optional[Any]=True       ,       __magic_name__							:		List[Any]=True       ,       __magic_name__							:		List[Any]=None       ,       ):







						"""simple docstring"""







						UpperCamelCase         =					size if size is not None else {"""height""": 2_0, """width""": 2_0}
						UpperCamelCase         =					parent
						UpperCamelCase         =					batch_size
						UpperCamelCase         =					num_channels
						UpperCamelCase         =					image_size
						UpperCamelCase         =					min_resolution
						UpperCamelCase         =					max_resolution
						UpperCamelCase         =					size
						UpperCamelCase         =					do_normalize
						UpperCamelCase         =					do_convert_rgb
						UpperCamelCase         =					[5_1_2, 1_0_2_4, 2_0_4_8, 4_0_9_6]
						UpperCamelCase         =					patch_size if patch_size is not None else {"""height""": 1_6, """width""": 1_6}




			def   lowerCamelCase_					(						self							:		Tuple						):







						"""simple docstring"""







						return {"do_normalize": self.do_normalize, "do_convert_rgb": self.do_convert_rgb}





			def   lowerCamelCase_					(						self							:		Optional[int]						):







						"""simple docstring"""







						UpperCamelCase         =					"""https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/australia.jpg"""
						UpperCamelCase         =					Image.open(requests.get(__magic_name__       ,       stream=__magic_name__						).raw						).convert("""RGB"""						)
						return raw_image




@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11      , reason="""`Pix2StructImageProcessor` requires `torch>=1.11.0`."""      , )
@require_torch
@require_vision
class    UpperCAmelCase					(       __snake_case      , unittest.TestCase    ):
			lowercase						= PixaStructImageProcessor if is_vision_available() else None




			def   lowerCamelCase_					(						self							:		Any						):







						"""simple docstring"""







						UpperCamelCase         =					PixaStructImageProcessingTester(self						)




			@property
			def   lowerCamelCase_					(						self							:		Union[str, Any]						):







						"""simple docstring"""







						return self.image_processor_tester.prepare_image_processor_dict()




			def   lowerCamelCase_					(						self							:		List[str]						):







						"""simple docstring"""







						UpperCamelCase         =					self.image_processing_class(**self.image_processor_dict						)
						self.assertTrue(hasattr(__magic_name__       ,       """do_normalize"""						)						)
						self.assertTrue(hasattr(__magic_name__       ,       """do_convert_rgb"""						)						)




			def   lowerCamelCase_					(						self							:		Optional[int]						):







						"""simple docstring"""







						UpperCamelCase         =					self.image_processor_tester.prepare_dummy_image()

						UpperCamelCase         =					self.image_processing_class(**self.image_processor_dict						)
						UpperCamelCase         =					2_0_4_8

						UpperCamelCase         =					image_processor(__magic_name__       ,       return_tensors="""pt"""       ,       max_patches=__magic_name__						)
						self.assertTrue(torch.allclose(inputs.flattened_patches.mean()       ,       torch.tensor(0.0_606						)       ,       atol=1e-3       ,       rtol=1e-3						)						)




			def   lowerCamelCase_					(						self							:		Optional[Any]						):







						"""simple docstring"""







						UpperCamelCase         =					self.image_processing_class(**self.image_processor_dict						)
						# create random PIL images
						UpperCamelCase         =					prepare_image_inputs(self.image_processor_tester       ,       equal_resolution=__magic_name__						)
						for image in image_inputs:
									self.assertIsInstance(__magic_name__       ,       Image.Image						)

						# Test not batched input
						UpperCamelCase         =					(
						    (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
						    * self.image_processor_tester.num_channels
						) + 2

						for max_patch in self.image_processor_tester.max_patches:
									# Test not batched input
									UpperCamelCase         =					image_processor(
									    image_inputs[0]       ,       return_tensors="""pt"""       ,       max_patches=__magic_name__						).flattened_patches
									self.assertEqual(
									    encoded_images.shape       ,       (1, max_patch, expected_hidden_dim)       ,       )

									# Test batched
									UpperCamelCase         =					image_processor(
									    __magic_name__       ,       return_tensors="""pt"""       ,       max_patches=__magic_name__						).flattened_patches
									self.assertEqual(
									    encoded_images.shape       ,       (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)       ,       )




			def   lowerCamelCase_					(						self							:		Union[str, Any]						):







						"""simple docstring"""







						UpperCamelCase         =					self.image_processing_class(**self.image_processor_dict						)
						# create random PIL images
						UpperCamelCase         =					prepare_image_inputs(self.image_processor_tester       ,       equal_resolution=__magic_name__						)
						for image in image_inputs:
									self.assertIsInstance(__magic_name__       ,       Image.Image						)

						# Test not batched input
						UpperCamelCase         =					(
						    (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
						    * self.image_processor_tester.num_channels
						) + 2

						UpperCamelCase         =					True

						for max_patch in self.image_processor_tester.max_patches:
									# Test not batched input
									with self.assertRaises(__magic_name__						):
												UpperCamelCase         =					image_processor(
												    image_inputs[0]       ,       return_tensors="""pt"""       ,       max_patches=__magic_name__						).flattened_patches

									UpperCamelCase         =					"""Hello"""

									UpperCamelCase         =					image_processor(
									    image_inputs[0]       ,       return_tensors="""pt"""       ,       max_patches=__magic_name__       ,       header_text=__magic_name__						).flattened_patches
									self.assertEqual(
									    encoded_images.shape       ,       (1, max_patch, expected_hidden_dim)       ,       )

									# Test batched
									UpperCamelCase         =					image_processor(
									    __magic_name__       ,       return_tensors="""pt"""       ,       max_patches=__magic_name__       ,       header_text=__magic_name__						).flattened_patches
									self.assertEqual(
									    encoded_images.shape       ,       (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)       ,       )




			def   lowerCamelCase_					(						self							:		Optional[int]						):







						"""simple docstring"""







						UpperCamelCase         =					self.image_processing_class(**self.image_processor_dict						)
						# create random numpy tensors
						UpperCamelCase         =					prepare_image_inputs(self.image_processor_tester       ,       equal_resolution=__magic_name__       ,       numpify=__magic_name__						)
						for image in image_inputs:
									self.assertIsInstance(__magic_name__       ,       np.ndarray						)

						UpperCamelCase         =					(
						    (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
						    * self.image_processor_tester.num_channels
						) + 2

						for max_patch in self.image_processor_tester.max_patches:
									# Test not batched input
									UpperCamelCase         =					image_processor(
									    image_inputs[0]       ,       return_tensors="""pt"""       ,       max_patches=__magic_name__						).flattened_patches
									self.assertEqual(
									    encoded_images.shape       ,       (1, max_patch, expected_hidden_dim)       ,       )

									# Test batched
									UpperCamelCase         =					image_processor(
									    __magic_name__       ,       return_tensors="""pt"""       ,       max_patches=__magic_name__						).flattened_patches
									self.assertEqual(
									    encoded_images.shape       ,       (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)       ,       )





			def   lowerCamelCase_					(						self							:		Any						):







						"""simple docstring"""







						UpperCamelCase         =					self.image_processing_class(**self.image_processor_dict						)
						# create random PyTorch tensors
						UpperCamelCase         =					prepare_image_inputs(self.image_processor_tester       ,       equal_resolution=__magic_name__       ,       torchify=__magic_name__						)
						for image in image_inputs:
									self.assertIsInstance(__magic_name__       ,       torch.Tensor						)

						# Test not batched input
						UpperCamelCase         =					(
						    (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
						    * self.image_processor_tester.num_channels
						) + 2

						for max_patch in self.image_processor_tester.max_patches:
									# Test not batched input
									UpperCamelCase         =					image_processor(
									    image_inputs[0]       ,       return_tensors="""pt"""       ,       max_patches=__magic_name__						).flattened_patches
									self.assertEqual(
									    encoded_images.shape       ,       (1, max_patch, expected_hidden_dim)       ,       )

									# Test batched
									UpperCamelCase         =					image_processor(
									    __magic_name__       ,       return_tensors="""pt"""       ,       max_patches=__magic_name__						).flattened_patches
									self.assertEqual(
									    encoded_images.shape       ,       (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)       ,       )




@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11      , reason="""`Pix2StructImageProcessor` requires `torch>=1.11.0`."""      , )
@require_torch
@require_vision
class    UpperCAmelCase					(       __snake_case      , unittest.TestCase    ):
			lowercase						= PixaStructImageProcessor if is_vision_available() else None




			def   lowerCamelCase_					(						self							:		str						):







						"""simple docstring"""







						UpperCamelCase         =					PixaStructImageProcessingTester(self       ,       num_channels=4						)
						UpperCamelCase         =					3




			@property
			def   lowerCamelCase_					(						self							:		str						):







						"""simple docstring"""







						return self.image_processor_tester.prepare_image_processor_dict()




			def   lowerCamelCase_					(						self							:		Union[str, Any]						):







						"""simple docstring"""







						UpperCamelCase         =					self.image_processing_class(**self.image_processor_dict						)
						self.assertTrue(hasattr(__magic_name__       ,       """do_normalize"""						)						)
						self.assertTrue(hasattr(__magic_name__       ,       """do_convert_rgb"""						)						)





			def   lowerCamelCase_					(						self							:		Dict						):







						"""simple docstring"""







						UpperCamelCase         =					self.image_processing_class(**self.image_processor_dict						)
						# create random PIL images
						UpperCamelCase         =					prepare_image_inputs(self.image_processor_tester       ,       equal_resolution=__magic_name__						)
						for image in image_inputs:
									self.assertIsInstance(__magic_name__       ,       Image.Image						)

						# Test not batched input
						UpperCamelCase         =					(
						    (self.image_processor_tester.patch_size["""height"""] * self.image_processor_tester.patch_size["""width"""])
						    * (self.image_processor_tester.num_channels - 1)
						) + 2

						for max_patch in self.image_processor_tester.max_patches:
									# Test not batched input
									UpperCamelCase         =					image_processor(
									    image_inputs[0]       ,       return_tensors="""pt"""       ,       max_patches=__magic_name__						).flattened_patches
									self.assertEqual(
									    encoded_images.shape       ,       (1, max_patch, expected_hidden_dim)       ,       )

									# Test batched
									UpperCamelCase         =					image_processor(
									    __magic_name__       ,       return_tensors="""pt"""       ,       max_patches=__magic_name__						).flattened_patches
									self.assertEqual(
									    encoded_images.shape       ,       (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)       ,       )








import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin




class    UpperCAmelCase					(       __snake_case      , unittest.TestCase    ):
			lowercase						= XLMTokenizer
			lowercase						= False




			def   lowerCamelCase_					(						self							:		Optional[int]						):







						"""simple docstring"""







						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						UpperCamelCase         =					[
						    """l""",
						    """o""",
						    """w""",
						    """e""",
						    """r""",
						    """s""",
						    """t""",
						    """i""",
						    """d""",
						    """n""",
						    """w</w>""",
						    """r</w>""",
						    """t</w>""",
						    """lo""",
						    """low""",
						    """er</w>""",
						    """low</w>""",
						    """lowest</w>""",
						    """newer</w>""",
						    """wider</w>""",
						    """<unk>""",
						]
						UpperCamelCase         =					dict(zip(__magic_name__       ,       range(len(__magic_name__						)						)						)						)
						UpperCamelCase         =					["""l o 123""", """lo w 1456""", """e r</w> 1789""", """"""]

						UpperCamelCase         =					os.path.join(self.tmpdirname       ,       VOCAB_FILES_NAMES["""vocab_file"""]						)
						UpperCamelCase         =					os.path.join(self.tmpdirname       ,       VOCAB_FILES_NAMES["""merges_file"""]						)
						with open(self.vocab_file       ,       """w"""						) as fp:
									fp.write(json.dumps(__magic_name__						)						)
						with open(self.merges_file       ,       """w"""						) as fp:
									fp.write("""\n""".join(__magic_name__						)						)




			def   lowerCamelCase_					(						self							:		Any       ,       __magic_name__							:		List[str]						):







						"""simple docstring"""







						UpperCamelCase         =					"""lower newer"""
						UpperCamelCase         =					"""lower newer"""
						return input_text, output_text




			def   lowerCamelCase_					(						self							:		List[str]						):







						"""simple docstring"""







						UpperCamelCase         =					XLMTokenizer(self.vocab_file       ,       self.merges_file						)

						UpperCamelCase         =					"""lower"""
						UpperCamelCase         =					["""low""", """er</w>"""]
						UpperCamelCase         =					tokenizer.tokenize(__magic_name__						)
						self.assertListEqual(__magic_name__       ,       __magic_name__						)

						UpperCamelCase         =					tokens + ["""<unk>"""]
						UpperCamelCase         =					[1_4, 1_5, 2_0]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__						)       ,       __magic_name__						)





			@slow
			def   lowerCamelCase_					(						self							:		Optional[int]						):







						"""simple docstring"""







						UpperCamelCase         =					XLMTokenizer.from_pretrained("""xlm-mlm-en-2048"""						)

						UpperCamelCase         =					tokenizer.encode("""sequence builders"""       ,       add_special_tokens=__magic_name__						)
						UpperCamelCase         =					tokenizer.encode("""multi-sequence build"""       ,       add_special_tokens=__magic_name__						)

						UpperCamelCase         =					tokenizer.build_inputs_with_special_tokens(__magic_name__						)
						UpperCamelCase         =					tokenizer.build_inputs_with_special_tokens(__magic_name__       ,       __magic_name__						)

						assert encoded_sentence == [0] + text + [1]
						assert encoded_pair == [0] + text + [1] + text_a + [1]




