






import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 							_UpperCamelCase			(UpperCAmelCase_							,		unittest.TestCase			):
      snake_case_  =							GPTSanJapaneseTokenizer
      snake_case_  =							False
      snake_case_  =							{'do_clean_text': False, 'add_prefix_space': False}


      def  __UpperCAmelCase		(	self	)->	List[Any]:
         super().setUp()

         # fmt: off
         __lowerCAmelCase								=					['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
         # fmt: on
         __lowerCAmelCase								=					{'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
         __lowerCAmelCase								=					{'unk_token': '<unk>'}

         __lowerCAmelCase								=					os.path.join(self.tmpdirname					,						VOCAB_FILES_NAMES["vocab_file"]	)
         __lowerCAmelCase								=					os.path.join(self.tmpdirname					,						VOCAB_FILES_NAMES["emoji_file"]	)
         with open(self.vocab_file					,						"w"					,						encoding="utf-8"	) as vocab_writer:
            vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)
         with open(self.emoji_file					,						"w"	) as emoji_writer:
            emoji_writer.write(json.dumps(_lowercase	)	)


      def  __UpperCAmelCase		(	self					,						**__UpperCamelCase	)->	List[str]:
         kwargs.update(self.special_tokens_map	)
         return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname					,						**_lowercase	)


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase	)->	str:
         __lowerCAmelCase								=					'こんにちは、世界。 \nこんばんは、㔺界。😀'
         __lowerCAmelCase								=					'こんにちは、世界。 \nこんばんは、世界。😀'
         return input_text, output_text


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase	)->	Union[str, Any]:
         __lowerCAmelCase								=					self.get_input_output_texts(_lowercase	)
         __lowerCAmelCase								=					tokenizer.encode(_lowercase					,						add_special_tokens=_lowercase	)
         __lowerCAmelCase								=					tokenizer.decode(_lowercase					,						clean_up_tokenization_spaces=_lowercase	)
         return text, ids


      def  __UpperCAmelCase		(	self	)->	Tuple:
         pass  # TODO add if relevant


      def  __UpperCAmelCase		(	self	)->	int:
         pass  # TODO add if relevant


      def  __UpperCAmelCase		(	self	)->	int:
         pass  # TODO add if relevant


      def  __UpperCAmelCase		(	self	)->	List[str]:
         __lowerCAmelCase								=					self.get_tokenizer()

         # Testing tokenization
         __lowerCAmelCase								=					'こんにちは、世界。　こんばんは、㔺界。'
         __lowerCAmelCase								=					['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
         __lowerCAmelCase								=					tokenizer.tokenize(_lowercase	)
         self.assertListEqual(_lowercase					,						_lowercase	)

         # Testing conversion to ids without special tokens
         __lowerCAmelCase								=					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
         __lowerCAmelCase								=					tokenizer.convert_tokens_to_ids(_lowercase	)
         self.assertListEqual(_lowercase					,						_lowercase	)

         # Testing conversion to ids with special tokens
         __lowerCAmelCase								=					tokens + [tokenizer.unk_token]
         __lowerCAmelCase								=					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
         __lowerCAmelCase								=					tokenizer.convert_tokens_to_ids(_lowercase	)
         self.assertListEqual(_lowercase					,						_lowercase	)


      def  __UpperCAmelCase		(	self	)->	List[str]:
         __lowerCAmelCase								=					self.get_tokenizer()

         # Testing tokenization
         __lowerCAmelCase								=					'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
         __lowerCAmelCase								=					'こんにちは、、、、世界。こんばんは、、、、世界。'
         __lowerCAmelCase								=					tokenizer.encode(_lowercase	)
         __lowerCAmelCase								=					tokenizer.decode(_lowercase	)
         self.assertEqual(_lowercase					,						_lowercase	)


      @slow
      def  __UpperCAmelCase		(	self	)->	Optional[Any]:
         __lowerCAmelCase								=					self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"	)

         # Testing tokenization
         __lowerCAmelCase								=					'こんにちは、世界。'
         __lowerCAmelCase								=					'こんばんは、㔺界。😀'
         __lowerCAmelCase								=					'こんにちは、世界。こんばんは、世界。😀'
         __lowerCAmelCase								=					tokenizer.encode(prefix_text + input_text	)
         __lowerCAmelCase								=					tokenizer.encode(""					,						prefix_text=prefix_text + input_text	)
         __lowerCAmelCase								=					tokenizer.encode(_lowercase					,						prefix_text=_lowercase	)
         __lowerCAmelCase								=					tokenizer.decode(_lowercase	)
         __lowerCAmelCase								=					tokenizer.decode(_lowercase	)
         __lowerCAmelCase								=					tokenizer.decode(_lowercase	)
         self.assertEqual(_lowercase					,						_lowercase	)
         self.assertEqual(_lowercase					,						_lowercase	)
         self.assertEqual(_lowercase					,						_lowercase	)


      @slow
      def  __UpperCAmelCase		(	self	)->	Union[str, Any]:
         __lowerCAmelCase								=					self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"	)

         # Testing tokenization
         __lowerCAmelCase								=					'こんにちは、世界。'
         __lowerCAmelCase								=					'こんばんは、㔺界。😀'

         __lowerCAmelCase								=					len(tokenizer.encode(_lowercase	)	) - 2
         __lowerCAmelCase								=					len(tokenizer.encode(_lowercase	)	) - 2

         __lowerCAmelCase								=					[1] + [0] * (len_prefix + len_text + 1)
         __lowerCAmelCase								=					[1] * (len_prefix + len_text + 1) + [0]
         __lowerCAmelCase								=					[1] + [1] * (len_prefix) + [0] * (len_text + 1)

         __lowerCAmelCase								=					tokenizer(prefix_text + input_text	).token_type_ids
         __lowerCAmelCase								=					tokenizer(""					,						prefix_text=prefix_text + input_text	).token_type_ids
         __lowerCAmelCase								=					tokenizer(_lowercase					,						prefix_text=_lowercase	).token_type_ids
         self.assertListEqual(_lowercase					,						_lowercase	)
         self.assertListEqual(_lowercase					,						_lowercase	)
         self.assertListEqual(_lowercase					,						_lowercase	)


      @slow
      def  __UpperCAmelCase		(	self	)->	List[str]:
         __lowerCAmelCase								=					self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"	)

         __lowerCAmelCase								=					tokenizer.encode("あンいワ"	)
         __lowerCAmelCase								=					tokenizer.encode(""					,						prefix_text="あンいワ"	)
         __lowerCAmelCase								=					tokenizer.encode("いワ"					,						prefix_text="あン"	)

         self.assertEqual(tokenizer.decode(_lowercase	)					,						tokenizer.decode(_lowercase	)	)
         self.assertEqual(tokenizer.decode(_lowercase	)					,						tokenizer.decode(_lowercase	)	)
         self.assertNotEqual(_lowercase					,						_lowercase	)
         self.assertNotEqual(_lowercase					,						_lowercase	)
         self.assertEqual(x_token_a[1]					,						x_token_a[-1]	)  # SEG token
         self.assertEqual(x_token_a[1]					,						x_token_a[3]	)  # SEG token


      @slow
      def  __UpperCAmelCase		(	self	)->	str:
         __lowerCAmelCase								=					self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"	)

         __lowerCAmelCase								=					[['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
         __lowerCAmelCase								=					tokenizer(_lowercase					,						padding=_lowercase	)
         __lowerCAmelCase								=					tokenizer.batch_encode_plus(_lowercase					,						padding=_lowercase	)

         # fmt: off
         __lowerCAmelCase								=					[[3_5_9_9_3, 8_6_4_0, 2_5_9_4_8, 3_5_9_9_8, 3_0_6_4_7, 3_5_6_7_5, 3_5_9_9_9, 3_5_9_9_9], [3_5_9_9_3, 1_0_3_8_2, 9_8_6_8, 3_5_9_9_8, 3_0_6_4_6, 9_4_5_9, 3_0_6_4_6, 3_5_6_7_5]]
         __lowerCAmelCase								=					[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
         __lowerCAmelCase								=					[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
         # fmt: on
         self.assertListEqual(x_token.input_ids					,						_lowercase	)
         self.assertListEqual(x_token.token_type_ids					,						_lowercase	)
         self.assertListEqual(x_token.attention_mask					,						_lowercase	)
         self.assertListEqual(x_token_a.input_ids					,						_lowercase	)
         self.assertListEqual(x_token_a.token_type_ids					,						_lowercase	)
         self.assertListEqual(x_token_a.attention_mask					,						_lowercase	)


      def  __UpperCAmelCase		(	self	)->	int:
         # Intentionally convert some words to accommodate character fluctuations unique to Japanese
         pass


      def  __UpperCAmelCase		(	self	)->	Any:
         # tokenizer has no padding token
         pass














from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_batched,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


lowerCamelCase			:					Optional[Any]         =	logging.get_logger(__name__)



class 							_UpperCamelCase			(a_			):
      snake_case_  =							["""pixel_values"""]


      def __init__(	self					,						__UpperCamelCase = True					,						__UpperCamelCase = None					,						__UpperCamelCase = PILImageResampling.BICUBIC					,						__UpperCamelCase = True					,						__UpperCamelCase = True					,						__UpperCamelCase = 1 / 2_5_5					,						__UpperCamelCase = None					,						__UpperCamelCase = True					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						**__UpperCamelCase					,						)->	None:
           super().__init__(**__UpperCamelCase	)
           __lowerCAmelCase								=					size if size is not None else {"height": 2_2_4, "width": 2_2_4}
           __lowerCAmelCase								=					get_size_dict(__UpperCamelCase	)
           __lowerCAmelCase								=					crop_size if crop_size is not None else {"height": 2_2_4, "width": 2_2_4}
           __lowerCAmelCase								=					get_size_dict(__UpperCamelCase					,						default_to_square=__UpperCamelCase					,						param_name="crop_size"	)

           __lowerCAmelCase								=					do_resize
           __lowerCAmelCase								=					do_rescale
           __lowerCAmelCase								=					do_normalize
           __lowerCAmelCase								=					do_center_crop
           __lowerCAmelCase								=					crop_size
           __lowerCAmelCase								=					size
           __lowerCAmelCase								=					resample
           __lowerCAmelCase								=					rescale_factor
           __lowerCAmelCase								=					image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
           __lowerCAmelCase								=					image_std if image_std is not None else IMAGENET_DEFAULT_STD


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase					,						__UpperCamelCase					,						__UpperCamelCase = PILImageResampling.BILINEAR					,						__UpperCamelCase = None					,						**__UpperCamelCase					,						)->	np.ndarray:
           __lowerCAmelCase								=					get_size_dict(__UpperCamelCase	)

           if "shortest_edge" in size:
                __lowerCAmelCase								=					get_resize_output_image_size(__UpperCamelCase					,						size=size["shortest_edge"]					,						default_to_square=__UpperCamelCase	)
                # size = get_resize_output_image_size(image, size["shortest_edge"], size["longest_edge"])
           elif "height" in size and "width" in size:
                __lowerCAmelCase								=					(size["height"], size["width"])
           else:
                raise ValueError(F"""Size must contain 'height' and 'width' keys or 'shortest_edge' key. Got {size.keys()}"""	)
           return resize(__UpperCamelCase					,						size=__UpperCamelCase					,						resample=__UpperCamelCase					,						data_format=__UpperCamelCase					,						**__UpperCamelCase	)


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase					,						__UpperCamelCase					,						__UpperCamelCase = None					,						**__UpperCamelCase					,						)->	np.ndarray:
           __lowerCAmelCase								=					get_size_dict(__UpperCamelCase	)
           if "height" not in size or "width" not in size:
                raise ValueError(F"""The `size` parameter must contain the keys (height, width). Got {size.keys()}"""	)
           return center_crop(__UpperCamelCase					,						size=(size["height"], size["width"])					,						data_format=__UpperCamelCase					,						**__UpperCamelCase	)


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase					,						__UpperCamelCase					,						__UpperCamelCase = None					,						**__UpperCamelCase	)->	np.ndarray:
           return rescale(__UpperCamelCase					,						scale=__UpperCamelCase					,						data_format=__UpperCamelCase					,						**__UpperCamelCase	)


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase					,						__UpperCamelCase					,						__UpperCamelCase					,						__UpperCamelCase = None					,						**__UpperCamelCase					,						)->	np.ndarray:
           return normalize(__UpperCamelCase					,						mean=__UpperCamelCase					,						std=__UpperCamelCase					,						data_format=__UpperCamelCase					,						**__UpperCamelCase	)


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = ChannelDimension.FIRST					,						**__UpperCamelCase					,						)->	BatchFeature:
           __lowerCAmelCase								=					do_resize if do_resize is not None else self.do_resize
           __lowerCAmelCase								=					do_rescale if do_rescale is not None else self.do_rescale
           __lowerCAmelCase								=					do_normalize if do_normalize is not None else self.do_normalize
           __lowerCAmelCase								=					do_center_crop if do_center_crop is not None else self.do_center_crop
           __lowerCAmelCase								=					crop_size if crop_size is not None else self.crop_size
           __lowerCAmelCase								=					get_size_dict(__UpperCamelCase					,						param_name="crop_size"					,						default_to_square=__UpperCamelCase	)
           __lowerCAmelCase								=					resample if resample is not None else self.resample
           __lowerCAmelCase								=					rescale_factor if rescale_factor is not None else self.rescale_factor
           __lowerCAmelCase								=					image_mean if image_mean is not None else self.image_mean
           __lowerCAmelCase								=					image_std if image_std is not None else self.image_std

           __lowerCAmelCase								=					size if size is not None else self.size
           __lowerCAmelCase								=					get_size_dict(__UpperCamelCase	)

           if not is_batched(__UpperCamelCase	):
                __lowerCAmelCase								=					[images]

           if not valid_images(__UpperCamelCase	):
                raise ValueError(
                    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
                    "torch.Tensor, tf.Tensor or jax.ndarray."	)

           if do_resize and size is None:
                raise ValueError("Size must be specified if do_resize is True."	)

           if do_center_crop and crop_size is None:
                raise ValueError("Crop size must be specified if do_center_crop is True."	)

           if do_rescale and rescale_factor is None:
                raise ValueError("Rescale factor must be specified if do_rescale is True."	)

           # All transformations expect numpy arrays.
           __lowerCAmelCase								=					[to_numpy_array(__UpperCamelCase	) for image in images]

           if do_resize:
                __lowerCAmelCase								=					[self.resize(image=__UpperCamelCase					,						size=__UpperCamelCase					,						resample=__UpperCamelCase	) for image in images]

           if do_center_crop:
                __lowerCAmelCase								=					[self.center_crop(image=__UpperCamelCase					,						size=__UpperCamelCase	) for image in images]

           if do_rescale:
                __lowerCAmelCase								=					[self.rescale(image=__UpperCamelCase					,						scale=__UpperCamelCase	) for image in images]

           if do_normalize:
                __lowerCAmelCase								=					[self.normalize(image=__UpperCamelCase					,						mean=__UpperCamelCase					,						std=__UpperCamelCase	) for image in images]

           __lowerCAmelCase								=					[to_channel_dimension_format(__UpperCamelCase					,						__UpperCamelCase	) for image in images]

           __lowerCAmelCase								=					{"pixel_values": images}
           return BatchFeature(data=__UpperCamelCase					,						tensor_type=__UpperCamelCase	)






