






import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_	:    Tuple            =      get_tests_dir('''fixtures/test_sentencepiece.model''')





@require_sentencepiece
@require_tokenizers
class   UpperCamelCase_ (       UpperCamelCase_   ,					unittest.TestCase  ):
  _A       :					Tuple				=	XLNetTokenizer
  _A       :					Dict				=	XLNetTokenizerFast
  _A       :					int				=	True
  _A       :					Tuple				=	True



  def 		UpperCamelCase_				(      self				)			->			List[Any]:



         """simple docstring"""





         super().setUp()

         # We have a SentencePiece fixture for testing
         UpperCAmelCase	    =			XLNetTokenizer(__a ,		keep_accents=__a				)
         tokenizer.sanitize_special_tokens()
         tokenizer.save_pretrained(self.tmpdirname				)



  def 		UpperCamelCase_				(      self				)			->			Optional[int]:



         """simple docstring"""





         UpperCAmelCase	    =			'<s>'
         UpperCAmelCase	    =			1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a				) ,		__a				)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a				) ,		__a				)



  def 		UpperCamelCase_				(      self				)			->			List[Any]:



         """simple docstring"""





         UpperCAmelCase	    =			list(self.get_tokenizer().get_vocab().keys()				)

         self.assertEqual(vocab_keys[0] ,		"""<unk>"""				)
         self.assertEqual(vocab_keys[1] ,		"""<s>"""				)
         self.assertEqual(vocab_keys[-1] ,		"""<eod>"""				)
         self.assertEqual(len(__a				) ,		10_06				)



  def 		UpperCamelCase_				(      self				)			->			Optional[int]:



         """simple docstring"""





         self.assertEqual(self.get_tokenizer().vocab_size ,		10_00				)



  def 		UpperCamelCase_				(      self				)			->			List[str]:



         """simple docstring"""





         UpperCAmelCase	    =			XLNetTokenizer(__a ,		keep_accents=__a				)

         UpperCAmelCase	    =			tokenizer.tokenize("""This is a test"""				)
         self.assertListEqual(__a ,		["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]				)

         self.assertListEqual(tokenizer.convert_tokens_to_ids(__a				) ,		[2_85, 46, 10, 1_70, 3_82]				)

         UpperCAmelCase	    =			tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
         self.assertListEqual(
             __a ,		[
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """9""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """é""",
                 """.""",
             ] ,		)
         UpperCAmelCase	    =			tokenizer.convert_tokens_to_ids(__a				)
         self.assertListEqual(__a ,		[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]				)

         UpperCAmelCase	    =			tokenizer.convert_ids_to_tokens(__a				)
         self.assertListEqual(
             __a ,		[
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """<unk>""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """s""",
                 """<unk>""",
                 """.""",
             ] ,		)



  def 		UpperCamelCase_				(      self				)			->			List[str]:



         """simple docstring"""





         UpperCAmelCase	    =			XLNetTokenizer(__a ,		do_lower_case=__a				)
         UpperCAmelCase	    =			tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
         self.assertListEqual(
             __a ,		[
                 SPIECE_UNDERLINE + """""",
                 """i""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """9""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """se""",
                 """.""",
             ] ,		)
         self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""				) ,		["""▁he""", """ll""", """o"""]				)



  def 		UpperCamelCase_				(      self				)			->			List[str]:



         """simple docstring"""





         UpperCAmelCase	    =			XLNetTokenizer(__a ,		do_lower_case=__a				)
         UpperCAmelCase	    =			tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
         self.assertListEqual(
             __a ,		[
                 SPIECE_UNDERLINE + """I""",
                 SPIECE_UNDERLINE + """was""",
                 SPIECE_UNDERLINE + """b""",
                 """or""",
                 """n""",
                 SPIECE_UNDERLINE + """in""",
                 SPIECE_UNDERLINE + """""",
                 """9""",
                 """2""",
                 """0""",
                 """0""",
                 """0""",
                 """,""",
                 SPIECE_UNDERLINE + """and""",
                 SPIECE_UNDERLINE + """this""",
                 SPIECE_UNDERLINE + """is""",
                 SPIECE_UNDERLINE + """f""",
                 """al""",
                 """se""",
                 """.""",
             ] ,		)



  @slow
  def 		UpperCamelCase_				(      self				)			->			List[str]:



         """simple docstring"""





         UpperCAmelCase	    =			XLNetTokenizer.from_pretrained("""xlnet-base-cased"""				)

         UpperCAmelCase	    =			tokenizer.encode("""sequence builders""" ,		add_special_tokens=__a				)
         UpperCAmelCase	    =			tokenizer.encode("""multi-sequence build""" ,		add_special_tokens=__a				)

         UpperCAmelCase	    =			tokenizer.build_inputs_with_special_tokens(__a				)
         UpperCAmelCase	    =			tokenizer.build_inputs_with_special_tokens(__a ,		__a				)

         assert encoded_sentence == text + [4, 3]
         assert encoded_pair == text + [4] + text_a + [4, 3]



  @slow
  def 		UpperCamelCase_				(      self				)			->			Optional[Any]:



         """simple docstring"""





         UpperCAmelCase	    =			{'input_ids': [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=__a ,		model_name="""xlnet-base-cased""" ,		revision="""c841166438c31ec7ca9a106dee7bb312b73ae511""" ,		)








"""simple docstring"""







from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
 import PIL


lowerCAmelCase_	:    Union[str, Any]            =      logging.get_logger(__name__)





class   UpperCamelCase_ (       a_  ):
  _A       :					List[str]				=	['pixel_values']



  def __init__(      self ,		snake_case__ = True ,		snake_case__ = None ,		snake_case__ = 0.9 ,		snake_case__ = PILImageResampling.BICUBIC ,		snake_case__ = True ,		snake_case__ = None ,		snake_case__ = 1 / 2_55 ,		snake_case__ = True ,		snake_case__ = True ,		snake_case__ = None ,		snake_case__ = None ,		**snake_case__ ,		)			->			None:



         """simple docstring"""





         super().__init__(**snake_case__				)
         UpperCAmelCase	    =			size if size is not None else {"""shortest_edge""": 2_24}
         UpperCAmelCase	    =			get_size_dict(snake_case__ ,		default_to_square=snake_case__				)
         UpperCAmelCase	    =			crop_size if crop_size is not None else {"""height""": 2_24, """width""": 2_24}
         UpperCAmelCase	    =			get_size_dict(snake_case__ ,		param_name="""crop_size"""				)

         UpperCAmelCase	    =			do_resize
         UpperCAmelCase	    =			size
         UpperCAmelCase	    =			crop_pct
         UpperCAmelCase	    =			resample
         UpperCAmelCase	    =			do_center_crop
         UpperCAmelCase	    =			crop_size
         UpperCAmelCase	    =			do_rescale
         UpperCAmelCase	    =			rescale_factor
         UpperCAmelCase	    =			do_normalize
         UpperCAmelCase	    =			image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
         UpperCAmelCase	    =			image_std if image_std is not None else IMAGENET_DEFAULT_STD



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ ,		snake_case__ = None ,		snake_case__ = PILImageResampling.BICUBIC ,		snake_case__ = None ,		**snake_case__ ,		)			->			np.ndarray:



         """simple docstring"""





         UpperCAmelCase	    =			get_size_dict(snake_case__ ,		default_to_square=snake_case__				)
         if "shortest_edge" not in size and ("height" not in size or "width" not in size):
                raise ValueError(f'''size must contain \'height\' and \'width\' or \'shortest_edge\' as keys. Got {size.keys()}'''				)
         if crop_pct is not None:
                if "shortest_edge" in size:
                       UpperCAmelCase	    =			int(size["""shortest_edge"""] / crop_pct				)
                elif "height" in size and "width" in size:
                       if size["height"] == size["width"]:
                              UpperCAmelCase	    =			int(size["""height"""] / crop_pct				)
                       else:
                              UpperCAmelCase	    =			(int(size["""height"""] / crop_pct				), int(size["""width"""] / crop_pct				))
                else:
                       raise ValueError("""Invalid size for resize: {}""".format(snake_case__				)				)

                UpperCAmelCase	    =			get_resize_output_image_size(snake_case__ ,		size=snake_case__ ,		default_to_square=snake_case__				)
         else:
                if "shortest_edge" in size:
                       UpperCAmelCase	    =			get_resize_output_image_size(snake_case__ ,		size=size["""shortest_edge"""] ,		default_to_square=snake_case__				)
                elif "height" in size and "width" in size:
                       UpperCAmelCase	    =			(size["""height"""], size["""width"""])
                else:
                       raise ValueError("""Invalid size for resize: {}""".format(snake_case__				)				)

         return resize(snake_case__ ,		size=snake_case__ ,		resample=snake_case__ ,		data_format=snake_case__ ,		**snake_case__				)



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ ,		snake_case__ = None ,		**snake_case__ ,		)			->			np.ndarray:



         """simple docstring"""





         UpperCAmelCase	    =			get_size_dict(snake_case__				)
         if "height" not in size or "width" not in size:
                raise ValueError(f'''size must contain \'height\' and \'width\' as keys. Got {size.keys()}'''				)
         return center_crop(snake_case__ ,		size=(size["""height"""], size["""width"""]) ,		data_format=snake_case__ ,		**snake_case__				)



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ ,		snake_case__ = None ,		**snake_case__ ,		)			->			Tuple:



         """simple docstring"""





         return rescale(snake_case__ ,		scale=snake_case__ ,		data_format=snake_case__ ,		**snake_case__				)



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ ,		snake_case__ ,		snake_case__ = None ,		**snake_case__ ,		)			->			np.ndarray:



         """simple docstring"""





         return normalize(snake_case__ ,		mean=snake_case__ ,		std=snake_case__ ,		data_format=snake_case__ ,		**snake_case__				)



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ = None ,		snake_case__ = None ,		snake_case__ = None ,		snake_case__ = None ,		snake_case__ = None ,		snake_case__ = None ,		snake_case__ = None ,		snake_case__ = None ,		snake_case__ = None ,		snake_case__ = None ,		snake_case__ = None ,		snake_case__ = None ,		snake_case__ = ChannelDimension.FIRST ,		**snake_case__ ,		)			->			PIL.Image.Image:



         """simple docstring"""





         UpperCAmelCase	    =			do_resize if do_resize is not None else self.do_resize
         UpperCAmelCase	    =			crop_pct if crop_pct is not None else self.crop_pct
         UpperCAmelCase	    =			resample if resample is not None else self.resample
         UpperCAmelCase	    =			do_center_crop if do_center_crop is not None else self.do_center_crop
         UpperCAmelCase	    =			do_rescale if do_rescale is not None else self.do_rescale
         UpperCAmelCase	    =			rescale_factor if rescale_factor is not None else self.rescale_factor
         UpperCAmelCase	    =			do_normalize if do_normalize is not None else self.do_normalize
         UpperCAmelCase	    =			image_mean if image_mean is not None else self.image_mean
         UpperCAmelCase	    =			image_std if image_std is not None else self.image_std

         UpperCAmelCase	    =			size if size is not None else self.size
         UpperCAmelCase	    =			get_size_dict(snake_case__ ,		default_to_square=snake_case__				)
         UpperCAmelCase	    =			crop_size if crop_size is not None else self.crop_size
         UpperCAmelCase	    =			get_size_dict(snake_case__ ,		param_name="""crop_size"""				)

         UpperCAmelCase	    =			make_list_of_images(snake_case__				)

         if not valid_images(snake_case__				):
                raise ValueError(
                    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
                    """torch.Tensor, tf.Tensor or jax.ndarray."""				)

         if do_resize and size is None or resample is None:
                raise ValueError("""Size and resample must be specified if do_resize is True."""				)

         if do_center_crop and crop_pct is None:
                raise ValueError("""Crop_pct must be specified if do_center_crop is True."""				)

         if do_rescale and rescale_factor is None:
                raise ValueError("""Rescale factor must be specified if do_rescale is True."""				)

         if do_normalize and (image_mean is None or image_std is None):
                raise ValueError("""Image mean and std must be specified if do_normalize is True."""				)

         # All transformations expect numpy arrays.
         UpperCAmelCase	    =			[to_numpy_array(snake_case__				) for image in images]

         if do_resize:
                UpperCAmelCase	    =			[self.resize(image=snake_case__ ,		size=snake_case__ ,		crop_pct=snake_case__ ,		resample=snake_case__				) for image in images]

         if do_center_crop:
                UpperCAmelCase	    =			[self.center_crop(image=snake_case__ ,		size=snake_case__				) for image in images]

         if do_rescale:
                UpperCAmelCase	    =			[self.rescale(image=snake_case__ ,		scale=snake_case__				) for image in images]

         if do_normalize:
                UpperCAmelCase	    =			[self.normalize(image=snake_case__ ,		mean=snake_case__ ,		std=snake_case__				) for image in images]

         UpperCAmelCase	    =			[to_channel_dimension_format(snake_case__ ,		snake_case__				) for image in images]

         UpperCAmelCase	    =			{"""pixel_values""": images}
         return BatchFeature(data=snake_case__ ,		tensor_type=snake_case__				)
