






import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 					SCREAMING_SNAKE_CASE_  (       _a  ,						unittest.TestCase ):
       """simple docstring"""


       __lowerCAmelCase    :		List[Any]					=GPTSanJapaneseTokenizer
       __lowerCAmelCase    :		Optional[int]					=False
       __lowerCAmelCase    :		Optional[Any]					={'''do_clean_text''': False, '''add_prefix_space''': False}





       def 				UpperCamelCase__						(				self	:int):



          """simple docstring"""

          super().setUp()

          # fmt: off
          _lowercase              =['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
          # fmt: on
          _lowercase              ={'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
          _lowercase              ={'unk_token': '<unk>'}

          _lowercase              =os.path.join(self.tmpdirname,      VOCAB_FILES_NAMES['vocab_file'])
          _lowercase              =os.path.join(self.tmpdirname,      VOCAB_FILES_NAMES['emoji_file'])
          with open(self.vocab_file,      'w',      encoding='utf-8') as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))
          with open(self.emoji_file,      'w') as emoji_writer:
             emoji_writer.write(json.dumps(snake_case))





       def 				UpperCamelCase__						(				self	:int,      **snake_case	:Optional[int]):



          """simple docstring"""

          kwargs.update(self.special_tokens_map)
          return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname,      **snake_case)





       def 				UpperCamelCase__						(				self	:Optional[int],      snake_case	:int):



          """simple docstring"""

          _lowercase              ='こんにちは、世界。 \nこんばんは、㔺界。😀'
          _lowercase              ='こんにちは、世界。 \nこんばんは、世界。😀'
          return input_text, output_text





       def 				UpperCamelCase__						(				self	:int,      snake_case	:Any):



          """simple docstring"""

          _lowercase       ,       _lowercase              =self.get_input_output_texts(snake_case)
          _lowercase              =tokenizer.encode(snake_case,      add_special_tokens=snake_case)
          _lowercase              =tokenizer.decode(snake_case,      clean_up_tokenization_spaces=snake_case)
          return text, ids





       def 				UpperCamelCase__						(				self	:Dict):



          """simple docstring"""

          pass  # TODO add if relevant





       def 				UpperCamelCase__						(				self	:Tuple):



          """simple docstring"""

          pass  # TODO add if relevant





       def 				UpperCamelCase__						(				self	:Optional[int]):



          """simple docstring"""

          pass  # TODO add if relevant





       def 				UpperCamelCase__						(				self	:int):



          """simple docstring"""

          _lowercase              =self.get_tokenizer()

          # Testing tokenization
          _lowercase              ='こんにちは、世界。　こんばんは、㔺界。'
          _lowercase              =['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
          _lowercase              =tokenizer.tokenize(snake_case)
          self.assertListEqual(snake_case,      snake_case)

          # Testing conversion to ids without special tokens
          _lowercase              =[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
          _lowercase              =tokenizer.convert_tokens_to_ids(snake_case)
          self.assertListEqual(snake_case,      snake_case)

          # Testing conversion to ids with special tokens
          _lowercase              =tokens + [tokenizer.unk_token]
          _lowercase              =[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
          _lowercase              =tokenizer.convert_tokens_to_ids(snake_case)
          self.assertListEqual(snake_case,      snake_case)





       def 				UpperCamelCase__						(				self	:Dict):



          """simple docstring"""

          _lowercase              =self.get_tokenizer()

          # Testing tokenization
          _lowercase              ='こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
          _lowercase              ='こんにちは、、、、世界。こんばんは、、、、世界。'
          _lowercase              =tokenizer.encode(snake_case)
          _lowercase              =tokenizer.decode(snake_case)
          self.assertEqual(snake_case,      snake_case)





       @slow
       def 				UpperCamelCase__						(				self	:List[Any]):



          """simple docstring"""

          _lowercase              =self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese')

          # Testing tokenization
          _lowercase              ='こんにちは、世界。'
          _lowercase              ='こんばんは、㔺界。😀'
          _lowercase              ='こんにちは、世界。こんばんは、世界。😀'
          _lowercase              =tokenizer.encode(prefix_text + input_text)
          _lowercase              =tokenizer.encode('',      prefix_text=prefix_text + input_text)
          _lowercase              =tokenizer.encode(snake_case,      prefix_text=snake_case)
          _lowercase              =tokenizer.decode(snake_case)
          _lowercase              =tokenizer.decode(snake_case)
          _lowercase              =tokenizer.decode(snake_case)
          self.assertEqual(snake_case,      snake_case)
          self.assertEqual(snake_case,      snake_case)
          self.assertEqual(snake_case,      snake_case)





       @slow
       def 				UpperCamelCase__						(				self	:List[Any]):



          """simple docstring"""

          _lowercase              =self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese')

          # Testing tokenization
          _lowercase              ='こんにちは、世界。'
          _lowercase              ='こんばんは、㔺界。😀'

          _lowercase              =len(tokenizer.encode(snake_case)) - 2
          _lowercase              =len(tokenizer.encode(snake_case)) - 2

          _lowercase              =[1] + [0] * (len_prefix + len_text + 1)
          _lowercase              =[1] * (len_prefix + len_text + 1) + [0]
          _lowercase              =[1] + [1] * (len_prefix) + [0] * (len_text + 1)

          _lowercase              =tokenizer(prefix_text + input_text).token_type_ids
          _lowercase              =tokenizer('',      prefix_text=prefix_text + input_text).token_type_ids
          _lowercase              =tokenizer(snake_case,      prefix_text=snake_case).token_type_ids
          self.assertListEqual(snake_case,      snake_case)
          self.assertListEqual(snake_case,      snake_case)
          self.assertListEqual(snake_case,      snake_case)





       @slow
       def 				UpperCamelCase__						(				self	:List[str]):



          """simple docstring"""

          _lowercase              =self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese')

          _lowercase              =tokenizer.encode('あンいワ')
          _lowercase              =tokenizer.encode('',      prefix_text='あンいワ')
          _lowercase              =tokenizer.encode('いワ',      prefix_text='あン')

          self.assertEqual(tokenizer.decode(snake_case),      tokenizer.decode(snake_case))
          self.assertEqual(tokenizer.decode(snake_case),      tokenizer.decode(snake_case))
          self.assertNotEqual(snake_case,      snake_case)
          self.assertNotEqual(snake_case,      snake_case)
          self.assertEqual(x_token_a[1],      x_token_a[-1])  # SEG token
          self.assertEqual(x_token_a[1],      x_token_a[3])  # SEG token





       @slow
       def 				UpperCamelCase__						(				self	:Dict):



          """simple docstring"""

          _lowercase              =self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese')

          _lowercase              =[['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
          _lowercase              =tokenizer(snake_case,      padding=snake_case)
          _lowercase              =tokenizer.batch_encode_plus(snake_case,      padding=snake_case)

          # fmt: off
          _lowercase              =[[3_5993, 8640, 2_5948, 3_5998, 3_0647, 3_5675, 3_5999, 3_5999], [3_5993, 1_0382, 9868, 3_5998, 3_0646, 9459, 3_0646, 3_5675]]
          _lowercase              =[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
          _lowercase              =[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
          # fmt: on
          self.assertListEqual(x_token.input_ids,      snake_case)
          self.assertListEqual(x_token.token_type_ids,      snake_case)
          self.assertListEqual(x_token.attention_mask,      snake_case)
          self.assertListEqual(x_token_a.input_ids,      snake_case)
          self.assertListEqual(x_token_a.token_type_ids,      snake_case)
          self.assertListEqual(x_token_a.attention_mask,      snake_case)





       def 				UpperCamelCase__						(				self	:List[str]):



          """simple docstring"""

          pass





       def 				UpperCamelCase__						(				self	:Union[str, Any]):



          """simple docstring"""

          pass










import unittest

import numpy as np
import requests

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
    import torch

    from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
    _SCREAMING_SNAKE_CASE   =  False

if is_vision_available():
    from PIL import Image

    from transformers import PixaStructImageProcessor






class 					SCREAMING_SNAKE_CASE_  (       unittest.TestCase ):
       """simple docstring"""


       def __init__(				self	:str,      snake_case	:int,      snake_case	:Dict=7,      snake_case	:List[Any]=3,      snake_case	:int=18,      snake_case	:List[str]=30,      snake_case	:str=400,      snake_case	:int=None,      snake_case	:List[Any]=True,      snake_case	:Optional[int]=True,      snake_case	:Union[str, Any]=None,      ):



          """simple docstring"""

          _lowercase              =size if size is not None else {'height': 20, 'width': 20}
          _lowercase              =parent
          _lowercase              =batch_size
          _lowercase              =num_channels
          _lowercase              =image_size
          _lowercase              =min_resolution
          _lowercase              =max_resolution
          _lowercase              =size
          _lowercase              =do_normalize
          _lowercase              =do_convert_rgb
          _lowercase              =[512, 1024, 2048, 4096]
          _lowercase              =patch_size if patch_size is not None else {'height': 16, 'width': 16}





       def 				UpperCamelCase__						(				self	:Any):



          """simple docstring"""

          return {"do_normalize": self.do_normalize, "do_convert_rgb": self.do_convert_rgb}





       def 				UpperCamelCase__						(				self	:int):



          """simple docstring"""

          _lowercase              ='https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/australia.jpg'
          _lowercase              =Image.open(requests.get(snake_case,      stream=snake_case).raw).convert('RGB')
          return raw_image






@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11  ,						reason='''`Pix2StructImageProcessor` requires `torch>=1.11.0`.'''  ,						)
@require_torch
@require_vision
class 					SCREAMING_SNAKE_CASE_  (       _a  ,						unittest.TestCase ):
       """simple docstring"""


       __lowerCAmelCase    :		Tuple					=PixaStructImageProcessor if is_vision_available() else None





       def 				UpperCamelCase__						(				self	:str):



          """simple docstring"""

          _lowercase              =PixaStructImageProcessingTester(self)





       @property
       def 				UpperCamelCase__						(				self	:Optional[int]):



          """simple docstring"""

          return self.image_processor_tester.prepare_image_processor_dict()





       def 				UpperCamelCase__						(				self	:str):



          """simple docstring"""

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          self.assertTrue(hasattr(snake_case,      'do_normalize'))
          self.assertTrue(hasattr(snake_case,      'do_convert_rgb'))





       def 				UpperCamelCase__						(				self	:Any):



          """simple docstring"""

          _lowercase              =self.image_processor_tester.prepare_dummy_image()

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          _lowercase              =2048

          _lowercase              =image_processor(snake_case,      return_tensors='pt',      max_patches=snake_case)
          self.assertTrue(torch.allclose(inputs.flattened_patches.mean(),      torch.tensor(0.0_6_0_6),      atol=1e-3,      rtol=1e-3))





       def 				UpperCamelCase__						(				self	:int):



          """simple docstring"""

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          # create random PIL images
          _lowercase              =prepare_image_inputs(self.image_processor_tester,      equal_resolution=snake_case)
          for image in image_inputs:
             self.assertIsInstance(snake_case,      Image.Image)

          # Test not batched input
          _lowercase              =(
              (self.image_processor_tester.patch_size['height'] * self.image_processor_tester.patch_size['width'])
              * self.image_processor_tester.num_channels
          ) + 2

          for max_patch in self.image_processor_tester.max_patches:
             # Test not batched input
             _lowercase              =image_processor(
                 image_inputs[0],      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (1, max_patch, expected_hidden_dim),      )

             # Test batched
             _lowercase              =image_processor(
                 snake_case,      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim),      )





       def 				UpperCamelCase__						(				self	:Union[str, Any]):



          """simple docstring"""

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          # create random PIL images
          _lowercase              =prepare_image_inputs(self.image_processor_tester,      equal_resolution=snake_case)
          for image in image_inputs:
             self.assertIsInstance(snake_case,      Image.Image)

          # Test not batched input
          _lowercase              =(
              (self.image_processor_tester.patch_size['height'] * self.image_processor_tester.patch_size['width'])
              * self.image_processor_tester.num_channels
          ) + 2

          _lowercase              =True

          for max_patch in self.image_processor_tester.max_patches:
             # Test not batched input
             with self.assertRaises(snake_case):
                _lowercase              =image_processor(
                    image_inputs[0],      return_tensors='pt',      max_patches=snake_case).flattened_patches

             _lowercase              ='Hello'

             _lowercase              =image_processor(
                 image_inputs[0],      return_tensors='pt',      max_patches=snake_case,      header_text=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (1, max_patch, expected_hidden_dim),      )

             # Test batched
             _lowercase              =image_processor(
                 snake_case,      return_tensors='pt',      max_patches=snake_case,      header_text=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim),      )





       def 				UpperCamelCase__						(				self	:int):



          """simple docstring"""

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          # create random numpy tensors
          _lowercase              =prepare_image_inputs(self.image_processor_tester,      equal_resolution=snake_case,      numpify=snake_case)
          for image in image_inputs:
             self.assertIsInstance(snake_case,      np.ndarray)

          _lowercase              =(
              (self.image_processor_tester.patch_size['height'] * self.image_processor_tester.patch_size['width'])
              * self.image_processor_tester.num_channels
          ) + 2

          for max_patch in self.image_processor_tester.max_patches:
             # Test not batched input
             _lowercase              =image_processor(
                 image_inputs[0],      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (1, max_patch, expected_hidden_dim),      )

             # Test batched
             _lowercase              =image_processor(
                 snake_case,      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim),      )





       def 				UpperCamelCase__						(				self	:Any):



          """simple docstring"""

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          # create random PyTorch tensors
          _lowercase              =prepare_image_inputs(self.image_processor_tester,      equal_resolution=snake_case,      torchify=snake_case)
          for image in image_inputs:
             self.assertIsInstance(snake_case,      torch.Tensor)

          # Test not batched input
          _lowercase              =(
              (self.image_processor_tester.patch_size['height'] * self.image_processor_tester.patch_size['width'])
              * self.image_processor_tester.num_channels
          ) + 2

          for max_patch in self.image_processor_tester.max_patches:
             # Test not batched input
             _lowercase              =image_processor(
                 image_inputs[0],      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (1, max_patch, expected_hidden_dim),      )

             # Test batched
             _lowercase              =image_processor(
                 snake_case,      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim),      )






@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11  ,						reason='''`Pix2StructImageProcessor` requires `torch>=1.11.0`.'''  ,						)
@require_torch
@require_vision
class 					SCREAMING_SNAKE_CASE_  (       _a  ,						unittest.TestCase ):
       """simple docstring"""


       __lowerCAmelCase    :		Optional[Any]					=PixaStructImageProcessor if is_vision_available() else None





       def 				UpperCamelCase__						(				self	:Optional[int]):



          """simple docstring"""

          _lowercase              =PixaStructImageProcessingTester(self,      num_channels=4)
          _lowercase              =3





       @property
       def 				UpperCamelCase__						(				self	:str):



          """simple docstring"""

          return self.image_processor_tester.prepare_image_processor_dict()





       def 				UpperCamelCase__						(				self	:Optional[int]):



          """simple docstring"""

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          self.assertTrue(hasattr(snake_case,      'do_normalize'))
          self.assertTrue(hasattr(snake_case,      'do_convert_rgb'))





       def 				UpperCamelCase__						(				self	:Optional[Any]):



          """simple docstring"""

          _lowercase              =self.image_processing_class(**self.image_processor_dict)
          # create random PIL images
          _lowercase              =prepare_image_inputs(self.image_processor_tester,      equal_resolution=snake_case)
          for image in image_inputs:
             self.assertIsInstance(snake_case,      Image.Image)

          # Test not batched input
          _lowercase              =(
              (self.image_processor_tester.patch_size['height'] * self.image_processor_tester.patch_size['width'])
              * (self.image_processor_tester.num_channels - 1)
          ) + 2

          for max_patch in self.image_processor_tester.max_patches:
             # Test not batched input
             _lowercase              =image_processor(
                 image_inputs[0],      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (1, max_patch, expected_hidden_dim),      )

             # Test batched
             _lowercase              =image_processor(
                 snake_case,      return_tensors='pt',      max_patches=snake_case).flattened_patches
             self.assertEqual(
                 encoded_images.shape,      (self.image_processor_tester.batch_size, max_patch, expected_hidden_dim),      )


