





import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
     import torch

if is_vision_available():
     from PIL import Image

     from transformers import PoolFormerImageProcessor



class 							_UpperCamelCase					(	unittest.TestCase     ):

       def __init__(		self: int					,   _SCREAMING_SNAKE_CASE: Any					,   _SCREAMING_SNAKE_CASE: Optional[Any]=7					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=3					,   _SCREAMING_SNAKE_CASE: Dict=30					,   _SCREAMING_SNAKE_CASE: List[str]=400					,   _SCREAMING_SNAKE_CASE: List[str]=True					,   _SCREAMING_SNAKE_CASE: Tuple=None					,   _SCREAMING_SNAKE_CASE: Dict=0.9					,   _SCREAMING_SNAKE_CASE: int=None					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=True					,   _SCREAMING_SNAKE_CASE: str=[0.5, 0.5, 0.5]					,   _SCREAMING_SNAKE_CASE: Tuple=[0.5, 0.5, 0.5]					,   )       -> List[Any]:




              """simple docstring"""


              UpperCamelCase_          =       size if size is not None else {"shortest_edge": 30}
              UpperCamelCase_          =       crop_size if crop_size is not None else {"height": 30, "width": 30}
              UpperCamelCase_          =       parent
              UpperCamelCase_          =       batch_size
              UpperCamelCase_          =       num_channels
              UpperCamelCase_          =       min_resolution
              UpperCamelCase_          =       max_resolution
              UpperCamelCase_          =       do_resize_and_center_crop
              UpperCamelCase_          =       size
              UpperCamelCase_          =       crop_pct
              UpperCamelCase_          =       crop_size
              UpperCamelCase_          =       do_normalize
              UpperCamelCase_          =       image_mean
              UpperCamelCase_          =       image_std



       def lowercase (		self: Dict		)       -> List[str]:




              """simple docstring"""


              return {
                  "size": self.size,
                  "do_resize_and_center_crop": self.do_resize_and_center_crop,
                  "crop_pct": self.crop_pct,
                  "crop_size": self.crop_size,
                  "do_normalize": self.do_normalize,
                  "image_mean": self.image_mean,
                  "image_std": self.image_std,
              }



@require_torch
@require_vision
class 							_UpperCamelCase					(	lowerCAmelCase_  ,						unittest.TestCase     ):
       _UpperCamelCase				:					int         =   PoolFormerImageProcessor if is_vision_available() else None

       def lowercase (		self: Any		)       -> List[Any]:




              """simple docstring"""


              UpperCamelCase_          =       PoolFormerImageProcessingTester(self		)

       @property
       def lowercase (		self: Any		)       -> int:




              """simple docstring"""


              return self.image_processor_tester.prepare_image_processor_dict()

       def lowercase (		self: Dict		)       -> int:




              """simple docstring"""


              UpperCamelCase_          =       self.image_processing_class(**self.image_processor_dict		)
              self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE					,   "do_resize_and_center_crop"		)		)
              self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE					,   "size"		)		)
              self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE					,   "crop_pct"		)		)
              self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE					,   "do_normalize"		)		)
              self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE					,   "image_mean"		)		)
              self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE					,   "image_std"		)		)

       def lowercase (		self: Any		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =       self.image_processing_class.from_dict(self.image_processor_dict		)
              self.assertEqual(image_processor.size					,   {"shortest_edge": 30}		)
              self.assertEqual(image_processor.crop_size					,   {"height": 30, "width": 30}		)

              UpperCamelCase_          =       self.image_processing_class.from_dict(self.image_processor_dict					,   size=42					,   crop_size=84		)
              self.assertEqual(image_processor.size					,   {"shortest_edge": 42}		)
              self.assertEqual(image_processor.crop_size					,   {"height": 84, "width": 84}		)

       def lowercase (		self: Optional[Any]		)       -> Optional[int]:




              """simple docstring"""


              pass

       def lowercase (		self: Union[str, Any]		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =       self.image_processing_class(**self.image_processor_dict		)
              # create random PIL images
              UpperCamelCase_          =       prepare_image_inputs(self.image_processor_tester					,   equal_resolution=_SCREAMING_SNAKE_CASE		)
              for image in image_inputs:
                     self.assertIsInstance(_SCREAMING_SNAKE_CASE					,   Image.Image		)

              # Test not batched input
              UpperCamelCase_          =       image_processing(image_inputs[0]					,   return_tensors="pt"		).pixel_values

              self.assertEqual(
                  encoded_images.shape					,   (
                      1,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["height"],
                      self.image_processor_tester.crop_size["width"],
                  )					,   )

              # Test batched
              UpperCamelCase_          =       image_processing(_SCREAMING_SNAKE_CASE					,   return_tensors="pt"		).pixel_values
              self.assertEqual(
                  encoded_images.shape					,   (
                      self.image_processor_tester.batch_size,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["height"],
                      self.image_processor_tester.crop_size["width"],
                  )					,   )

       def lowercase (		self: Any		)       -> List[Any]:




              """simple docstring"""


              UpperCamelCase_          =       self.image_processing_class(**self.image_processor_dict		)
              # create random numpy tensors
              UpperCamelCase_          =       prepare_image_inputs(self.image_processor_tester					,   equal_resolution=_SCREAMING_SNAKE_CASE					,   numpify=_SCREAMING_SNAKE_CASE		)
              for image in image_inputs:
                     self.assertIsInstance(_SCREAMING_SNAKE_CASE					,   np.ndarray		)

              # Test not batched input
              UpperCamelCase_          =       image_processing(image_inputs[0]					,   return_tensors="pt"		).pixel_values
              self.assertEqual(
                  encoded_images.shape					,   (
                      1,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["height"],
                      self.image_processor_tester.crop_size["width"],
                  )					,   )

              # Test batched
              UpperCamelCase_          =       image_processing(_SCREAMING_SNAKE_CASE					,   return_tensors="pt"		).pixel_values
              self.assertEqual(
                  encoded_images.shape					,   (
                      self.image_processor_tester.batch_size,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["height"],
                      self.image_processor_tester.crop_size["width"],
                  )					,   )



       def lowercase (		self: List[Any]		)       -> Union[str, Any]:




              """simple docstring"""


              UpperCamelCase_          =       self.image_processing_class(**self.image_processor_dict		)
              # create random PyTorch tensors
              UpperCamelCase_          =       prepare_image_inputs(self.image_processor_tester					,   equal_resolution=_SCREAMING_SNAKE_CASE					,   torchify=_SCREAMING_SNAKE_CASE		)
              for image in image_inputs:
                     self.assertIsInstance(_SCREAMING_SNAKE_CASE					,   torch.Tensor		)

              # Test not batched input
              UpperCamelCase_          =       image_processing(image_inputs[0]					,   return_tensors="pt"		).pixel_values
              self.assertEqual(
                  encoded_images.shape					,   (
                      1,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["height"],
                      self.image_processor_tester.crop_size["width"],
                  )					,   )

              # Test batched
              UpperCamelCase_          =       image_processing(_SCREAMING_SNAKE_CASE					,   return_tensors="pt"		).pixel_values
              self.assertEqual(
                  encoded_images.shape					,   (
                      self.image_processor_tester.batch_size,
                      self.image_processor_tester.num_channels,
                      self.image_processor_tester.crop_size["height"],
                      self.image_processor_tester.crop_size["width"],
                  )					,   )








import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin



class 							_UpperCamelCase					(	lowerCAmelCase_  ,						unittest.TestCase     ):
       _UpperCamelCase				:					Union[str, Any]         =   XLMTokenizer
       _UpperCamelCase				:					Optional[int]         =   False

       def lowercase (		self: Dict		)       -> Union[str, Any]:




              """simple docstring"""


              super().setUp()

              # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
              UpperCamelCase_          =  [
                  "l",
                  "o",
                  "w",
                  "e",
                  "r",
                  "s",
                  "t",
                  "i",
                  "d",
                  "n",
                  "w</w>",
                  "r</w>",
                  "t</w>",
                  "lo",
                  "low",
                  "er</w>",
                  "low</w>",
                  "lowest</w>",
                  "newer</w>",
                  "wider</w>",
                  "<unk>",
              ]
              UpperCamelCase_          =  dict(zip(_SCREAMING_SNAKE_CASE					,   range(len(_SCREAMING_SNAKE_CASE		)		)		)		)
              UpperCamelCase_          =  ["l o 123", "lo w 1456", "e r</w> 1789", ""]

              UpperCamelCase_          =  os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES["vocab_file"]		)
              UpperCamelCase_          =  os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES["merges_file"]		)
              with open(self.vocab_file					,   "w"		) as fp:
                     fp.write(json.dumps(_SCREAMING_SNAKE_CASE		)		)
              with open(self.merges_file					,   "w"		) as fp:
                     fp.write("\n".join(_SCREAMING_SNAKE_CASE		)		)

       def lowercase (		self: Optional[Any]					,   _SCREAMING_SNAKE_CASE: List[str]		)       -> str:




              """simple docstring"""


              UpperCamelCase_          =  "lower newer"
              UpperCamelCase_          =  "lower newer"
              return input_text, output_text

       def lowercase (		self: int		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =  XLMTokenizer(self.vocab_file					,   self.merges_file		)

              UpperCamelCase_          =  "lower"
              UpperCamelCase_          =  ["low", "er</w>"]
              UpperCamelCase_          =  tokenizer.tokenize(_SCREAMING_SNAKE_CASE		)
              self.assertListEqual(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE		)

              UpperCamelCase_          =  tokens + ["<unk>"]
              UpperCamelCase_          =  [14, 15, 20]
              self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE		)					,   _SCREAMING_SNAKE_CASE		)



       @slow
       def lowercase (		self: Optional[int]		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =  XLMTokenizer.from_pretrained("xlm-mlm-en-2048"		)

              UpperCamelCase_          =  tokenizer.encode("sequence builders"					,   add_special_tokens=_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  tokenizer.encode("multi-sequence build"					,   add_special_tokens=_SCREAMING_SNAKE_CASE		)

              UpperCamelCase_          =  tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE					,   _SCREAMING_SNAKE_CASE		)

              assert encoded_sentence == [0] + text + [1]
              assert encoded_pair == [0] + text + [1] + text_a + [1]

