



from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


__snake_case     :int			       =logging.get_logger(__name__)



class        lowerCAmelCase__					(  _lowerCamelCase       ):
				A_   :     str								=     ['pixel_values']


				def __init__(							self      :  Tuple						, __UpperCamelCase      :  bool = True						, __UpperCamelCase      :  Optional[Dict[str, int]] = None						, __UpperCamelCase      :  PILImageResampling = PILImageResampling.BILINEAR						, __UpperCamelCase      :  bool = True						, __UpperCamelCase      :  Dict[str, int] = None						, __UpperCamelCase      :  bool = True						, __UpperCamelCase      :  Union[int, float] = 1 / 255						, __UpperCamelCase      :  bool = True						, __UpperCamelCase      :  Optional[Union[float, List[float]]] = None						, __UpperCamelCase      :  Optional[Union[float, List[float]]] = None						, **__UpperCamelCase      :  List[str]						, )		->     None:
											super().__init__(**__UpperCamelCase       )
											A      						=    size if size is not None else {'shortest_edge': 256}
											A      						=    get_size_dict(__UpperCamelCase						, default_to_square=__UpperCamelCase       )
											A      						=    crop_size if crop_size is not None else {'height': 224, 'width': 224}
											A      						=    get_size_dict(__UpperCamelCase       )
											A      						=    do_resize
											A      						=    size
											A      						=    resample
											A      						=    do_center_crop
											A      						=    crop_size
											A      						=    do_rescale
											A      						=    rescale_factor
											A      						=    do_normalize
											A      						=    image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
											A      						=    image_std if image_std is not None else IMAGENET_STANDARD_STD


				def __UpperCamelCase						(							self      :  List[Any]						, __UpperCamelCase      :  np.ndarray						, __UpperCamelCase      :  Dict[str, int]						, __UpperCamelCase      :  PILImageResampling = PILImageResampling.BICUBIC						, __UpperCamelCase      :  Optional[Union[str, ChannelDimension]] = None						, **__UpperCamelCase      :  Any						, )		->     np.ndarray:
											A      						=    get_size_dict(__UpperCamelCase						, default_to_square=__UpperCamelCase       )
											if "shortest_edge" not in size:
																		raise ValueError(f'''The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}'''       )
											A      						=    get_resize_output_image_size(__UpperCamelCase						, size=size['shortest_edge']						, default_to_square=__UpperCamelCase       )
											return resize(__UpperCamelCase						, size=__UpperCamelCase						, resample=__UpperCamelCase						, data_format=__UpperCamelCase						, **__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Union[str, Any]						, __UpperCamelCase      :  np.ndarray						, __UpperCamelCase      :  Dict[str, int]						, __UpperCamelCase      :  Optional[Union[str, ChannelDimension]] = None						, **__UpperCamelCase      :  Optional[Any]						, )		->     np.ndarray:
											A      						=    get_size_dict(__UpperCamelCase       )
											return center_crop(__UpperCamelCase						, size=(size['height'], size['width'])						, data_format=__UpperCamelCase						, **__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  List[str]						, __UpperCamelCase      :  np.ndarray						, __UpperCamelCase      :  float						, __UpperCamelCase      :  Optional[Union[str, ChannelDimension]] = None						, **__UpperCamelCase      :  Optional[int]       )		->     np.ndarray:
											return rescale(__UpperCamelCase						, scale=__UpperCamelCase						, data_format=__UpperCamelCase						, **__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  List[Any]						, __UpperCamelCase      :  np.ndarray						, __UpperCamelCase      :  Union[float, List[float]]						, __UpperCamelCase      :  Union[float, List[float]]						, __UpperCamelCase      :  Optional[Union[str, ChannelDimension]] = None						, **__UpperCamelCase      :  List[Any]						, )		->     np.ndarray:
											return normalize(__UpperCamelCase						, mean=__UpperCamelCase						, std=__UpperCamelCase						, data_format=__UpperCamelCase						, **__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  List[str]						, __UpperCamelCase      :  ImageInput						, __UpperCamelCase      :  Optional[bool] = None						, __UpperCamelCase      :  Dict[str, int] = None						, __UpperCamelCase      :  PILImageResampling = None						, __UpperCamelCase      :  bool = None						, __UpperCamelCase      :  Dict[str, int] = None						, __UpperCamelCase      :  Optional[bool] = None						, __UpperCamelCase      :  Optional[float] = None						, __UpperCamelCase      :  Optional[bool] = None						, __UpperCamelCase      :  Optional[Union[float, List[float]]] = None						, __UpperCamelCase      :  Optional[Union[float, List[float]]] = None						, __UpperCamelCase      :  Optional[Union[str, TensorType]] = None						, __UpperCamelCase      :  Union[str, ChannelDimension] = ChannelDimension.FIRST						, **__UpperCamelCase      :  str						, )		->     Any:
											A      						=    do_resize if do_resize is not None else self.do_resize
											A      						=    size if size is not None else self.size
											A      						=    get_size_dict(__UpperCamelCase						, default_to_square=__UpperCamelCase       )
											A      						=    resample if resample is not None else self.resample
											A      						=    do_center_crop if do_center_crop is not None else self.do_center_crop
											A      						=    crop_size if crop_size is not None else self.crop_size
											A      						=    get_size_dict(__UpperCamelCase       )
											A      						=    do_rescale if do_rescale is not None else self.do_rescale
											A      						=    rescale_factor if rescale_factor is not None else self.rescale_factor
											A      						=    do_normalize if do_normalize is not None else self.do_normalize
											A      						=    image_mean if image_mean is not None else self.image_mean
											A      						=    image_std if image_std is not None else self.image_std

											A      						=    make_list_of_images(__UpperCamelCase       )

											if not valid_images(__UpperCamelCase       ):
																		raise ValueError(
																		    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
																		    'torch.Tensor, tf.Tensor or jax.ndarray.'       )

											if do_resize and size is None:
																		raise ValueError('Size must be specified if do_resize is True.'       )

											if do_center_crop and crop_size is None:
																		raise ValueError('Crop size must be specified if do_center_crop is True.'       )

											if do_rescale and rescale_factor is None:
																		raise ValueError('Rescale factor must be specified if do_rescale is True.'       )

											if do_normalize and (image_mean is None or image_std is None):
																		raise ValueError('Image mean and std must be specified if do_normalize is True.'       )

											# All transformations expect numpy arrays.
											A      						=    [to_numpy_array(__UpperCamelCase       ) for image in images]

											if do_resize:
																		A      						=    [self.resize(image=__UpperCamelCase						, size=__UpperCamelCase						, resample=__UpperCamelCase       ) for image in images]

											if do_center_crop:
																		A      						=    [self.center_crop(image=__UpperCamelCase						, size=__UpperCamelCase       ) for image in images]

											if do_rescale:
																		A      						=    [self.rescale(image=__UpperCamelCase						, scale=__UpperCamelCase       ) for image in images]

											if do_normalize:
																		A      						=    [self.normalize(image=__UpperCamelCase						, mean=__UpperCamelCase						, std=__UpperCamelCase       ) for image in images]

											A      						=    [to_channel_dimension_format(__UpperCamelCase						, __UpperCamelCase       ) for image in images]

											A      						=    {'pixel_values': images}
											return BatchFeature(data=__UpperCamelCase						, tensor_type=__UpperCamelCase       )




import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class        lowerCAmelCase__					(  _lowerCamelCase ,				unittest.TestCase       ):
				A_   :     Dict								=     GPTSanJapaneseTokenizer
				A_   :     Optional[int]								=     False
				A_   :     List[str]								=     {'do_clean_text': False, 'add_prefix_space': False}


				def __UpperCamelCase						(							self      :  Tuple       )		->     Any:
											super().setUp()

											# fmt: off
											A      						=    ['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
											# fmt: on
											A      						=    {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
											A      						=    {'unk_token': '<unk>'}

											A      						=    os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES['vocab_file']       )
											A      						=    os.path.join(self.tmpdirname						, VOCAB_FILES_NAMES['emoji_file']       )
											with open(self.vocab_file						, 'w'						, encoding='utf-8'       ) as vocab_writer:
																		vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )
											with open(self.emoji_file						, 'w'       ) as emoji_writer:
																		emoji_writer.write(json.dumps(__UpperCamelCase       )       )


				def __UpperCamelCase						(							self      :  Any						, **__UpperCamelCase      :  Any       )		->     Tuple:
											kwargs.update(self.special_tokens_map       )
											return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname						, **__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  str						, __UpperCamelCase      :  Dict       )		->     List[str]:
											A      						=    'こんにちは、世界。 \nこんばんは、㔺界。😀'
											A      						=    'こんにちは、世界。 \nこんばんは、世界。😀'
											return input_text, output_text


				def __UpperCamelCase						(							self      :  str						, __UpperCamelCase      :  Optional[Any]       )		->     List[str]:
											A						,   A      						=    self.get_input_output_texts(__UpperCamelCase       )
											A      						=    tokenizer.encode(__UpperCamelCase						, add_special_tokens=__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase						, clean_up_tokenization_spaces=__UpperCamelCase       )
											return text, ids


				def __UpperCamelCase						(							self      :  Any       )		->     int:
											pass  # TODO add if relevant


				def __UpperCamelCase						(							self      :  List[str]       )		->     Tuple:
											pass  # TODO add if relevant


				def __UpperCamelCase						(							self      :  List[str]       )		->     List[Any]:
											pass  # TODO add if relevant


				def __UpperCamelCase						(							self      :  List[str]       )		->     int:
											A      						=    self.get_tokenizer()

											# Testing tokenization
											A      						=    'こんにちは、世界。　こんばんは、㔺界。'
											A      						=    ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
											A      						=    tokenizer.tokenize(__UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )

											# Testing conversion to ids without special tokens
											A      						=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
											A      						=    tokenizer.convert_tokens_to_ids(__UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )

											# Testing conversion to ids with special tokens
											A      						=    tokens + [tokenizer.unk_token]
											A      						=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
											A      						=    tokenizer.convert_tokens_to_ids(__UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     List[str]:
											A      						=    self.get_tokenizer()

											# Testing tokenization
											A      						=    'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
											A      						=    'こんにちは、、、、世界。こんばんは、、、、世界。'
											A      						=    tokenizer.encode(__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     str:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											# Testing tokenization
											A      						=    'こんにちは、世界。'
											A      						=    'こんばんは、㔺界。😀'
											A      						=    'こんにちは、世界。こんばんは、世界。😀'
											A      						=    tokenizer.encode(prefix_text + input_text       )
											A      						=    tokenizer.encode(''						, prefix_text=prefix_text + input_text       )
											A      						=    tokenizer.encode(__UpperCamelCase						, prefix_text=__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											A      						=    tokenizer.decode(__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     List[Any]:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											# Testing tokenization
											A      						=    'こんにちは、世界。'
											A      						=    'こんばんは、㔺界。😀'

											A      						=    len(tokenizer.encode(__UpperCamelCase       )       ) - 2
											A      						=    len(tokenizer.encode(__UpperCamelCase       )       ) - 2

											A      						=    [1] + [0] * (len_prefix + len_text + 1)
											A      						=    [1] * (len_prefix + len_text + 1) + [0]
											A      						=    [1] + [1] * (len_prefix) + [0] * (len_text + 1)

											A      						=    tokenizer(prefix_text + input_text       ).token_type_ids
											A      						=    tokenizer(''						, prefix_text=prefix_text + input_text       ).token_type_ids
											A      						=    tokenizer(__UpperCamelCase						, prefix_text=__UpperCamelCase       ).token_type_ids
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )


				@slow
				def __UpperCamelCase						(							self      :  Any       )		->     str:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											A      						=    tokenizer.encode('あンいワ'       )
											A      						=    tokenizer.encode(''						, prefix_text='あンいワ'       )
											A      						=    tokenizer.encode('いワ'						, prefix_text='あン'       )

											self.assertEqual(tokenizer.decode(__UpperCamelCase       )						, tokenizer.decode(__UpperCamelCase       )       )
											self.assertEqual(tokenizer.decode(__UpperCamelCase       )						, tokenizer.decode(__UpperCamelCase       )       )
											self.assertNotEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertNotEqual(__UpperCamelCase						, __UpperCamelCase       )
											self.assertEqual(x_token_a[1]						, x_token_a[-1]       )  # SEG token
											self.assertEqual(x_token_a[1]						, x_token_a[3]       )  # SEG token


				@slow
				def __UpperCamelCase						(							self      :  Any       )		->     List[Any]:
											A      						=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'       )

											A      						=    [['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
											A      						=    tokenizer(__UpperCamelCase						, padding=__UpperCamelCase       )
											A      						=    tokenizer.batch_encode_plus(__UpperCamelCase						, padding=__UpperCamelCase       )

											# fmt: off
											A      						=    [[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
											A      						=    [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
											A      						=    [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
											# fmt: on
											self.assertListEqual(x_token.input_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token.token_type_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token.attention_mask						, __UpperCamelCase       )
											self.assertListEqual(x_token_a.input_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token_a.token_type_ids						, __UpperCamelCase       )
											self.assertListEqual(x_token_a.attention_mask						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Tuple       )		->     str:
											# Intentionally convert some words to accommodate character fluctuations unique to Japanese
											pass


				def __UpperCamelCase						(							self      :  Dict       )		->     int:
											# tokenizer has no padding token
											pass