



import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a      :  Any			     =			get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model')







@require_sentencepiece
@require_tokenizers
class 				_a				(   _lowerCAmelCase ,		unittest.TestCase					):
			A      		=	GPTSwaTokenizer
			A      		=	False
			A      		=	True
			A      		=	False

			def      __snake_case       (self					)       ->   List[str]:
						super().setUp()

						# We have a SentencePiece fixture for testing
						UpperCAmelCase_:		Optional[int]							      =     GPTSwaTokenizer(SCREAMING_SNAKE_CASE_,   eos_token="""<unk>""",   bos_token="""<unk>""",   pad_token="""<unk>"""					)

						tokenizer.save_pretrained(self.tmpdirname					)

			def      __snake_case       (self,   SCREAMING_SNAKE_CASE_					)       ->   str:
						UpperCAmelCase_:		Optional[int]							      =     """This is a test"""
						UpperCAmelCase_:		Tuple							      =     """This is a test"""
						return input_text, output_text

			def      __snake_case       (self					)       ->   List[Any]:
						UpperCAmelCase_:		Tuple							      =     """<s>"""
						UpperCAmelCase_:		List[str]							      =     1

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE_					),   SCREAMING_SNAKE_CASE_					)
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE_					),   SCREAMING_SNAKE_CASE_					)

			def      __snake_case       (self					)       ->   Union[str, Any]:
						UpperCAmelCase_:		List[Any]							      =     list(self.get_tokenizer().get_vocab().keys()					)

						self.assertEqual(vocab_keys[0],   """<unk>"""					)
						self.assertEqual(vocab_keys[1],   """<s>"""					)
						self.assertEqual(vocab_keys[-1],   """j"""					)
						self.assertEqual(len(SCREAMING_SNAKE_CASE_					),   2000					)

			def      __snake_case       (self					)       ->   Union[str, Any]:
						self.assertEqual(self.get_tokenizer().vocab_size,   2000					)

			def      __snake_case       (self					)       ->   Union[str, Any]:
						UpperCAmelCase_:		Any							      =     GPTSwaTokenizer(SCREAMING_SNAKE_CASE_					)

						UpperCAmelCase_:		int							      =     tokenizer.tokenize("""This is a test"""					)
						self.assertListEqual(SCREAMING_SNAKE_CASE_,   ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]					)

						self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_					),   [465, 287, 265, 631, 842]					)

						UpperCAmelCase_:		Tuple							      =     tokenizer.tokenize("""I was born in 92000, and this is falsé."""					)
						# fmt: off
						self.assertListEqual(
						    SCREAMING_SNAKE_CASE_,   ["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""],   )
						# fmt: on

						UpperCAmelCase_:		Union[str, Any]							      =     tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_					)
						self.assertListEqual(
						    SCREAMING_SNAKE_CASE_,   [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],   )

						UpperCAmelCase_:		int							      =     tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_					)
						# fmt: off
						self.assertListEqual(
						    SCREAMING_SNAKE_CASE_,   ["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]					)
						# fmt: on

			def      __snake_case       (self					)       ->   Optional[Any]:
						UpperCAmelCase_:		int							      =     GPTSwaTokenizer(SCREAMING_SNAKE_CASE_					)
						UpperCAmelCase_:		str							      =     ["""This is a test""", """I was born in 92000, and this is falsé."""]
						UpperCAmelCase_:		Optional[int]							      =     [
						    [465, 287, 265, 631, 842],
						    [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
						]

						# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
						for text, expected_ids in zip(SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_					):
									self.assertListEqual(tokenizer.encode_fast(SCREAMING_SNAKE_CASE_					),   SCREAMING_SNAKE_CASE_					)

						# Test that decode_fast returns the input text
						for text, token_ids in zip(SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_					):
									self.assertEqual(tokenizer.decode_fast(SCREAMING_SNAKE_CASE_					),   SCREAMING_SNAKE_CASE_					)

			@slow
			def      __snake_case       (self					)       ->   Optional[int]:
						UpperCAmelCase_:		Optional[Any]							      =     [
						    """<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')""",
						    """Hey there, how are you doing this fine day?""",
						    """This is a text with a trailing spaces followed by a dot     .""",
						    """Häj sväjs lillebrör! =)""",
						    """Det är inget fel på Mr. Cool""",
						]

						# fmt: off
						UpperCAmelCase_:		Optional[int]							      =     {"""input_ids""": [[63423, 5, 6811, 14954, 282, 816, 3821, 63466, 63425, 63462, 18, 63978, 678, 301, 1320, 63423, 63455, 63458, 18, 63982, 4246, 3940, 1901, 47789, 5547, 18994], [19630, 1100, 63446, 1342, 633, 544, 4488, 593, 5102, 2416, 63495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 58593, 22413, 9106, 546, 268, 33213, 63979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [55130, 63450, 924, 63449, 2249, 4062, 1558, 318, 63504, 21498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 63443, 26801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
						# fmt: on
						self.tokenizer_integration_test_util(
						    expected_encoding=SCREAMING_SNAKE_CASE_,   model_name="""AI-Sweden/gpt-sw3-126m""",   sequences=SCREAMING_SNAKE_CASE_,   )











from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


a      :  List[str]			     =			logging.get_logger(__name__)


if is_vision_available():
	import PIL







class 				_a				(   _lowerCAmelCase					):
			A      		=	['''pixel_values''']

			def __init__(self,   SCREAMING_SNAKE_CASE_ = True,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = PILImageResampling.BICUBIC,   SCREAMING_SNAKE_CASE_ = True,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = True,   SCREAMING_SNAKE_CASE_ = 1 / 255,   SCREAMING_SNAKE_CASE_ = True,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = True,   **SCREAMING_SNAKE_CASE_,   )       ->   None:
						super().__init__(**SCREAMING_SNAKE_CASE_					)
						UpperCAmelCase_:		Optional[Any]							      =     size if size is not None else {"""shortest_edge""": 224}
						UpperCAmelCase_:		int							      =     get_size_dict(SCREAMING_SNAKE_CASE_,   default_to_square=SCREAMING_SNAKE_CASE_					)
						UpperCAmelCase_:		Dict							      =     crop_size if crop_size is not None else {"""height""": 224, """width""": 224}
						UpperCAmelCase_:		Union[str, Any]							      =     get_size_dict(SCREAMING_SNAKE_CASE_,   default_to_square=SCREAMING_SNAKE_CASE_,   param_name="""crop_size"""					)

						UpperCAmelCase_:		Optional[int]							      =     do_resize
						UpperCAmelCase_:		Optional[int]							      =     size
						UpperCAmelCase_:		Optional[int]							      =     resample
						UpperCAmelCase_:		Tuple							      =     do_center_crop
						UpperCAmelCase_:		Optional[int]							      =     crop_size
						UpperCAmelCase_:		str							      =     do_rescale
						UpperCAmelCase_:		Optional[int]							      =     rescale_factor
						UpperCAmelCase_:		List[str]							      =     do_normalize
						UpperCAmelCase_:		Any							      =     image_mean if image_mean is not None else OPENAI_CLIP_MEAN
						UpperCAmelCase_:		Dict							      =     image_std if image_std is not None else OPENAI_CLIP_STD
						UpperCAmelCase_:		Any							      =     do_convert_rgb

			def      __snake_case       (self,   SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_ = PILImageResampling.BICUBIC,   SCREAMING_SNAKE_CASE_ = None,   **SCREAMING_SNAKE_CASE_,   )       ->   np.ndarray:
						UpperCAmelCase_:		Optional[Any]							      =     get_size_dict(SCREAMING_SNAKE_CASE_,   default_to_square=SCREAMING_SNAKE_CASE_					)
						if "shortest_edge" not in size:
									raise ValueError(f'The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}'					)
						UpperCAmelCase_:		Optional[Any]							      =     get_resize_output_image_size(SCREAMING_SNAKE_CASE_,   size=size["""shortest_edge"""],   default_to_square=SCREAMING_SNAKE_CASE_					)
						return resize(SCREAMING_SNAKE_CASE_,   size=SCREAMING_SNAKE_CASE_,   resample=SCREAMING_SNAKE_CASE_,   data_format=SCREAMING_SNAKE_CASE_,   **SCREAMING_SNAKE_CASE_					)

			def      __snake_case       (self,   SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_ = None,   **SCREAMING_SNAKE_CASE_,   )       ->   np.ndarray:
						UpperCAmelCase_:		Any							      =     get_size_dict(SCREAMING_SNAKE_CASE_					)
						if "height" not in size or "width" not in size:
									raise ValueError(f'The `size` parameter must contain the keys (height, width). Got {size.keys()}'					)
						return center_crop(SCREAMING_SNAKE_CASE_,   size=(size["""height"""], size["""width"""]),   data_format=SCREAMING_SNAKE_CASE_,   **SCREAMING_SNAKE_CASE_					)

			def      __snake_case       (self,   SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_ = None,   **SCREAMING_SNAKE_CASE_,   )       ->   Union[str, Any]:
						return rescale(SCREAMING_SNAKE_CASE_,   scale=SCREAMING_SNAKE_CASE_,   data_format=SCREAMING_SNAKE_CASE_,   **SCREAMING_SNAKE_CASE_					)

			def      __snake_case       (self,   SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_ = None,   **SCREAMING_SNAKE_CASE_,   )       ->   np.ndarray:
						return normalize(SCREAMING_SNAKE_CASE_,   mean=SCREAMING_SNAKE_CASE_,   std=SCREAMING_SNAKE_CASE_,   data_format=SCREAMING_SNAKE_CASE_,   **SCREAMING_SNAKE_CASE_					)

			def      __snake_case       (self,   SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = None,   SCREAMING_SNAKE_CASE_ = ChannelDimension.FIRST,   **SCREAMING_SNAKE_CASE_,   )       ->   PIL.Image.Image:
						UpperCAmelCase_:		Tuple							      =     do_resize if do_resize is not None else self.do_resize
						UpperCAmelCase_:		Tuple							      =     size if size is not None else self.size
						UpperCAmelCase_:		str							      =     get_size_dict(SCREAMING_SNAKE_CASE_,   param_name="""size""",   default_to_square=SCREAMING_SNAKE_CASE_					)
						UpperCAmelCase_:		List[str]							      =     resample if resample is not None else self.resample
						UpperCAmelCase_:		Optional[int]							      =     do_center_crop if do_center_crop is not None else self.do_center_crop
						UpperCAmelCase_:		Tuple							      =     crop_size if crop_size is not None else self.crop_size
						UpperCAmelCase_:		int							      =     get_size_dict(SCREAMING_SNAKE_CASE_,   param_name="""crop_size""",   default_to_square=SCREAMING_SNAKE_CASE_					)
						UpperCAmelCase_:		Union[str, Any]							      =     do_rescale if do_rescale is not None else self.do_rescale
						UpperCAmelCase_:		int							      =     rescale_factor if rescale_factor is not None else self.rescale_factor
						UpperCAmelCase_:		Tuple							      =     do_normalize if do_normalize is not None else self.do_normalize
						UpperCAmelCase_:		Dict							      =     image_mean if image_mean is not None else self.image_mean
						UpperCAmelCase_:		int							      =     image_std if image_std is not None else self.image_std
						UpperCAmelCase_:		Union[str, Any]							      =     do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

						UpperCAmelCase_:		List[str]							      =     make_list_of_images(SCREAMING_SNAKE_CASE_					)

						if not valid_images(SCREAMING_SNAKE_CASE_					):
									raise ValueError(
									    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
									    """torch.Tensor, tf.Tensor or jax.ndarray."""					)

						if do_resize and size is None:
									raise ValueError("""Size must be specified if do_resize is True."""					)

						if do_center_crop and crop_size is None:
									raise ValueError("""Crop size must be specified if do_center_crop is True."""					)

						if do_rescale and rescale_factor is None:
									raise ValueError("""Rescale factor must be specified if do_rescale is True."""					)

						if do_normalize and (image_mean is None or image_std is None):
									raise ValueError("""Image mean and std must be specified if do_normalize is True."""					)

						# PIL RGBA images are converted to RGB
						if do_convert_rgb:
									UpperCAmelCase_:		Any							      =     [convert_to_rgb(SCREAMING_SNAKE_CASE_					) for image in images]

						# All transformations expect numpy arrays.
						UpperCAmelCase_:		List[str]							      =     [to_numpy_array(SCREAMING_SNAKE_CASE_					) for image in images]

						if do_resize:
									UpperCAmelCase_:		str							      =     [self.resize(image=SCREAMING_SNAKE_CASE_,   size=SCREAMING_SNAKE_CASE_,   resample=SCREAMING_SNAKE_CASE_					) for image in images]

						if do_center_crop:
									UpperCAmelCase_:		Dict							      =     [self.center_crop(image=SCREAMING_SNAKE_CASE_,   size=SCREAMING_SNAKE_CASE_					) for image in images]

						if do_rescale:
									UpperCAmelCase_:		Optional[Any]							      =     [self.rescale(image=SCREAMING_SNAKE_CASE_,   scale=SCREAMING_SNAKE_CASE_					) for image in images]

						if do_normalize:
									UpperCAmelCase_:		Dict							      =     [self.normalize(image=SCREAMING_SNAKE_CASE_,   mean=SCREAMING_SNAKE_CASE_,   std=SCREAMING_SNAKE_CASE_					) for image in images]

						UpperCAmelCase_:		Tuple							      =     [to_channel_dimension_format(SCREAMING_SNAKE_CASE_,   SCREAMING_SNAKE_CASE_					) for image in images]

						UpperCAmelCase_:		Union[str, Any]							      =     {"""pixel_values""": images}
						return BatchFeature(data=SCREAMING_SNAKE_CASE_,   tensor_type=SCREAMING_SNAKE_CASE_					)






