





import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 							_UpperCamelCase					(	__snake_case  ,						unittest.TestCase     ):
       _UpperCamelCase				:					str         =   GPTSanJapaneseTokenizer
       _UpperCamelCase				:					Dict         =   False
       _UpperCamelCase				:					Union[str, Any]         =   {'''do_clean_text''': False, '''add_prefix_space''': False}

       def lowercase (		self: Optional[Any]		)       -> Dict:




              """simple docstring"""


              super().setUp()

              # fmt: off
              UpperCamelCase_          =  ["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
              # fmt: on
              UpperCamelCase_          =  {"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
              UpperCamelCase_          =  {"""unk_token""": """<unk>"""}

              UpperCamelCase_          =  os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES["vocab_file"]		)
              UpperCamelCase_          =  os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES["emoji_file"]		)
              with open(self.vocab_file					,   "w"					,   encoding="utf-8"		) as vocab_writer:
                     vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)
              with open(self.emoji_file					,   "w"		) as emoji_writer:
                     emoji_writer.write(json.dumps(_lowercase		)		)

       def lowercase (		self: Optional[int]					,   **_SCREAMING_SNAKE_CASE: Union[str, Any]		)       -> Tuple:




              """simple docstring"""


              kwargs.update(self.special_tokens_map		)
              return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname					,   **_lowercase		)

       def lowercase (		self: str					,   _SCREAMING_SNAKE_CASE: List[Any]		)       -> List[str]:




              """simple docstring"""


              UpperCamelCase_          =  """こんにちは、世界。 \nこんばんは、㔺界。😀"""
              UpperCamelCase_          =  """こんにちは、世界。 \nこんばんは、世界。😀"""
              return input_text, output_text

       def lowercase (		self: List[Any]					,   _SCREAMING_SNAKE_CASE: str		)       -> List[Any]:




              """simple docstring"""


              UpperCamelCase_          =  self.get_input_output_texts(_lowercase		)
              UpperCamelCase_          =  tokenizer.encode(_lowercase					,   add_special_tokens=_lowercase		)
              UpperCamelCase_          =  tokenizer.decode(_lowercase					,   clean_up_tokenization_spaces=_lowercase		)
              return text, ids

       def lowercase (		self: List[Any]		)       -> int:




              """simple docstring"""


              pass  # TODO add if relevant

       def lowercase (		self: Tuple		)       -> Tuple:




              """simple docstring"""


              pass  # TODO add if relevant

       def lowercase (		self: List[str]		)       -> List[str]:




              """simple docstring"""


              pass  # TODO add if relevant

       def lowercase (		self: List[str]		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =  self.get_tokenizer()

              # Testing tokenization
              UpperCamelCase_          =  """こんにちは、世界。　こんばんは、㔺界。"""
              UpperCamelCase_          =  ["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
              UpperCamelCase_          =  tokenizer.tokenize(_lowercase		)
              self.assertListEqual(_lowercase					,   _lowercase		)

              # Testing conversion to ids without special tokens
              UpperCamelCase_          =  [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
              UpperCamelCase_          =  tokenizer.convert_tokens_to_ids(_lowercase		)
              self.assertListEqual(_lowercase					,   _lowercase		)

              # Testing conversion to ids with special tokens
              UpperCamelCase_          =  tokens + [tokenizer.unk_token]
              UpperCamelCase_          =  [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
              UpperCamelCase_          =  tokenizer.convert_tokens_to_ids(_lowercase		)
              self.assertListEqual(_lowercase					,   _lowercase		)

       def lowercase (		self: str		)       -> Union[str, Any]:




              """simple docstring"""


              UpperCamelCase_          =  self.get_tokenizer()

              # Testing tokenization
              UpperCamelCase_          =  """こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
              UpperCamelCase_          =  """こんにちは、、、、世界。こんばんは、、、、世界。"""
              UpperCamelCase_          =  tokenizer.encode(_lowercase		)
              UpperCamelCase_          =  tokenizer.decode(_lowercase		)
              self.assertEqual(_lowercase					,   _lowercase		)

       @slow
       def lowercase (		self: int		)       -> int:




              """simple docstring"""


              UpperCamelCase_          =  self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"		)

              # Testing tokenization
              UpperCamelCase_          =  """こんにちは、世界。"""
              UpperCamelCase_          =  """こんばんは、㔺界。😀"""
              UpperCamelCase_          =  """こんにちは、世界。こんばんは、世界。😀"""
              UpperCamelCase_          =  tokenizer.encode(prefix_text + input_text		)
              UpperCamelCase_          =  tokenizer.encode(""					,   prefix_text=prefix_text + input_text		)
              UpperCamelCase_          =  tokenizer.encode(_lowercase					,   prefix_text=_lowercase		)
              UpperCamelCase_          =  tokenizer.decode(_lowercase		)
              UpperCamelCase_          =  tokenizer.decode(_lowercase		)
              UpperCamelCase_          =  tokenizer.decode(_lowercase		)
              self.assertEqual(_lowercase					,   _lowercase		)
              self.assertEqual(_lowercase					,   _lowercase		)
              self.assertEqual(_lowercase					,   _lowercase		)

       @slow
       def lowercase (		self: str		)       -> Optional[int]:




              """simple docstring"""


              UpperCamelCase_          =  self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"		)

              # Testing tokenization
              UpperCamelCase_          =  """こんにちは、世界。"""
              UpperCamelCase_          =  """こんばんは、㔺界。😀"""

              UpperCamelCase_          =  len(tokenizer.encode(_lowercase		)		) - 2
              UpperCamelCase_          =  len(tokenizer.encode(_lowercase		)		) - 2

              UpperCamelCase_          =  [1] + [0] * (len_prefix + len_text + 1)
              UpperCamelCase_          =  [1] * (len_prefix + len_text + 1) + [0]
              UpperCamelCase_          =  [1] + [1] * (len_prefix) + [0] * (len_text + 1)

              UpperCamelCase_          =  tokenizer(prefix_text + input_text		).token_type_ids
              UpperCamelCase_          =  tokenizer(""					,   prefix_text=prefix_text + input_text		).token_type_ids
              UpperCamelCase_          =  tokenizer(_lowercase					,   prefix_text=_lowercase		).token_type_ids
              self.assertListEqual(_lowercase					,   _lowercase		)
              self.assertListEqual(_lowercase					,   _lowercase		)
              self.assertListEqual(_lowercase					,   _lowercase		)

       @slow
       def lowercase (		self: Tuple		)       -> Any:




              """simple docstring"""


              UpperCamelCase_          =  self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"		)

              UpperCamelCase_          =  tokenizer.encode("あンいワ"		)
              UpperCamelCase_          =  tokenizer.encode(""					,   prefix_text="あンいワ"		)
              UpperCamelCase_          =  tokenizer.encode("いワ"					,   prefix_text="あン"		)

              self.assertEqual(tokenizer.decode(_lowercase		)					,   tokenizer.decode(_lowercase		)		)
              self.assertEqual(tokenizer.decode(_lowercase		)					,   tokenizer.decode(_lowercase		)		)
              self.assertNotEqual(_lowercase					,   _lowercase		)
              self.assertNotEqual(_lowercase					,   _lowercase		)
              self.assertEqual(x_token_a[1]					,   x_token_a[-1]		)  # SEG token
              self.assertEqual(x_token_a[1]					,   x_token_a[3]		)  # SEG token

       @slow
       def lowercase (		self: Optional[int]		)       -> Dict:




              """simple docstring"""


              UpperCamelCase_          =  self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"		)

              UpperCamelCase_          =  [["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
              UpperCamelCase_          =  tokenizer(_lowercase					,   padding=_lowercase		)
              UpperCamelCase_          =  tokenizer.batch_encode_plus(_lowercase					,   padding=_lowercase		)

              # fmt: off
              UpperCamelCase_          =  [[35993, 8640, 25948, 35998, 30647, 35675, 35999, 35999], [35993, 10382, 9868, 35998, 30646, 9459, 30646, 35675]]
              UpperCamelCase_          =  [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
              UpperCamelCase_          =  [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
              # fmt: on
              self.assertListEqual(x_token.input_ids					,   _lowercase		)
              self.assertListEqual(x_token.token_type_ids					,   _lowercase		)
              self.assertListEqual(x_token.attention_mask					,   _lowercase		)
              self.assertListEqual(x_token_a.input_ids					,   _lowercase		)
              self.assertListEqual(x_token_a.token_type_ids					,   _lowercase		)
              self.assertListEqual(x_token_a.attention_mask					,   _lowercase		)

       def lowercase (		self: Union[str, Any]		)       -> List[Any]:




              """simple docstring"""


              pass



       def lowercase (		self: List[Any]		)       -> Dict:




              """simple docstring"""


              pass








import unittest

from transformers import MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING, is_vision_available, pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
     from PIL import Image
else:



     class 							_UpperCamelCase					:

            @staticmethod
            def lowercase (		*_SCREAMING_SNAKE_CASE: Optional[int]					,   **_SCREAMING_SNAKE_CASE: Dict		)       -> Union[str, Any]:




                   """simple docstring"""


                   pass



@is_pipeline_test
@require_vision
@require_torch
class 							_UpperCamelCase					(	unittest.TestCase     ):
       _UpperCamelCase				:					str         =   MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING

       def lowercase (		self: int					,   _SCREAMING_SNAKE_CASE: str					,   _SCREAMING_SNAKE_CASE: int					,   _SCREAMING_SNAKE_CASE: Optional[Any]		)       -> Any:




              """simple docstring"""


              UpperCamelCase_          =  pipeline(
                  "zero-shot-object-detection"					,   model="hf-internal-testing/tiny-random-owlvit-object-detection"		)

              UpperCamelCase_          =  [
                  {
                      "image": "./tests/fixtures/tests_samples/COCO/000000039769.png",
                      "candidate_labels": ["cat", "remote", "couch"],
                  }
              ]
              return object_detector, examples

       def lowercase (		self: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Tuple					,   _SCREAMING_SNAKE_CASE: Tuple		)       -> Optional[int]:




              """simple docstring"""


              UpperCamelCase_          =  object_detector(examples[0]					,   threshold=0.0		)

              UpperCamelCase_          =  len(_SCREAMING_SNAKE_CASE		)
              self.assertGreater(_SCREAMING_SNAKE_CASE					,   0		)
              self.assertEqual(
                  _SCREAMING_SNAKE_CASE					,   [
                      {
                          "score": ANY(_SCREAMING_SNAKE_CASE		),
                          "label": ANY(_SCREAMING_SNAKE_CASE		),
                          "box": {"xmin": ANY(_SCREAMING_SNAKE_CASE		), "ymin": ANY(_SCREAMING_SNAKE_CASE		), "xmax": ANY(_SCREAMING_SNAKE_CASE		), "ymax": ANY(_SCREAMING_SNAKE_CASE		)},
                      }
                      for i in range(_SCREAMING_SNAKE_CASE		)
                  ]					,   )

       @require_tf
       @unittest.skip("Zero Shot Object Detection not implemented in TF"		)
       def lowercase (		self: Tuple		)       -> List[str]:




              """simple docstring"""


              pass

       @require_torch
       def lowercase (		self: Union[str, Any]		)       -> str:




              """simple docstring"""


              UpperCamelCase_          =  pipeline(
                  "zero-shot-object-detection"					,   model="hf-internal-testing/tiny-random-owlvit-object-detection"		)

              UpperCamelCase_          =  object_detector(
                  "./tests/fixtures/tests_samples/COCO/000000039769.png"					,   candidate_labels=["cat", "remote", "couch"]					,   threshold=0.64					,   )

              self.assertEqual(
                  nested_simplify(_SCREAMING_SNAKE_CASE					,   decimals=4		)					,   [
                      {"score": 0.72_35, "label": "cat", "box": {"xmin": 204, "ymin": 167, "xmax": 232, "ymax": 190}},
                      {"score": 0.72_18, "label": "remote", "box": {"xmin": 204, "ymin": 167, "xmax": 232, "ymax": 190}},
                      {"score": 0.71_84, "label": "couch", "box": {"xmin": 204, "ymin": 167, "xmax": 232, "ymax": 190}},
                      {"score": 0.67_48, "label": "remote", "box": {"xmin": 571, "ymin": 83, "xmax": 598, "ymax": 103}},
                      {"score": 0.66_56, "label": "cat", "box": {"xmin": 571, "ymin": 83, "xmax": 598, "ymax": 103}},
                      {"score": 0.66_14, "label": "couch", "box": {"xmin": 571, "ymin": 83, "xmax": 598, "ymax": 103}},
                      {"score": 0.64_56, "label": "remote", "box": {"xmin": 494, "ymin": 105, "xmax": 521, "ymax": 127}},
                      {"score": 0.6_42, "label": "remote", "box": {"xmin": 67, "ymin": 274, "xmax": 93, "ymax": 297}},
                      {"score": 0.64_19, "label": "cat", "box": {"xmin": 494, "ymin": 105, "xmax": 521, "ymax": 127}},
                  ]					,   )

              UpperCamelCase_          =  object_detector(
                  [
                      {
                          "image": "./tests/fixtures/tests_samples/COCO/000000039769.png",
                          "candidate_labels": ["cat", "remote", "couch"],
                      }
                  ]					,   threshold=0.64					,   )

              self.assertEqual(
                  nested_simplify(_SCREAMING_SNAKE_CASE					,   decimals=4		)					,   [
                      [
                          {"score": 0.72_35, "label": "cat", "box": {"xmin": 204, "ymin": 167, "xmax": 232, "ymax": 190}},
                          {"score": 0.72_18, "label": "remote", "box": {"xmin": 204, "ymin": 167, "xmax": 232, "ymax": 190}},
                          {"score": 0.71_84, "label": "couch", "box": {"xmin": 204, "ymin": 167, "xmax": 232, "ymax": 190}},
                          {"score": 0.67_48, "label": "remote", "box": {"xmin": 571, "ymin": 83, "xmax": 598, "ymax": 103}},
                          {"score": 0.66_56, "label": "cat", "box": {"xmin": 571, "ymin": 83, "xmax": 598, "ymax": 103}},
                          {"score": 0.66_14, "label": "couch", "box": {"xmin": 571, "ymin": 83, "xmax": 598, "ymax": 103}},
                          {"score": 0.64_56, "label": "remote", "box": {"xmin": 494, "ymin": 105, "xmax": 521, "ymax": 127}},
                          {"score": 0.6_42, "label": "remote", "box": {"xmin": 67, "ymin": 274, "xmax": 93, "ymax": 297}},
                          {"score": 0.64_19, "label": "cat", "box": {"xmin": 494, "ymin": 105, "xmax": 521, "ymax": 127}},
                      ]
                  ]					,   )

       @require_torch
       @slow
       def lowercase (		self: List[str]		)       -> List[str]:




              """simple docstring"""


              UpperCamelCase_          =  pipeline("zero-shot-object-detection"		)

              UpperCamelCase_          =  object_detector(
                  "http://images.cocodataset.org/val2017/000000039769.jpg"					,   candidate_labels=["cat", "remote", "couch"]					,   )
              self.assertEqual(
                  nested_simplify(_SCREAMING_SNAKE_CASE					,   decimals=4		)					,   [
                      {"score": 0.28_68, "label": "cat", "box": {"xmin": 324, "ymin": 20, "xmax": 640, "ymax": 373}},
                      {"score": 0.2_77, "label": "remote", "box": {"xmin": 40, "ymin": 72, "xmax": 177, "ymax": 115}},
                      {"score": 0.25_37, "label": "cat", "box": {"xmin": 1, "ymin": 55, "xmax": 315, "ymax": 472}},
                      {"score": 0.14_74, "label": "remote", "box": {"xmin": 335, "ymin": 74, "xmax": 371, "ymax": 187}},
                      {"score": 0.12_08, "label": "couch", "box": {"xmin": 4, "ymin": 0, "xmax": 642, "ymax": 476}},
                  ]					,   )

              UpperCamelCase_          =  object_detector(
                  [
                      {
                          "image": "http://images.cocodataset.org/val2017/000000039769.jpg",
                          "candidate_labels": ["cat", "remote", "couch"],
                      },
                      {
                          "image": "http://images.cocodataset.org/val2017/000000039769.jpg",
                          "candidate_labels": ["cat", "remote", "couch"],
                      },
                  ]					,   )
              self.assertEqual(
                  nested_simplify(_SCREAMING_SNAKE_CASE					,   decimals=4		)					,   [
                      [
                          {"score": 0.28_68, "label": "cat", "box": {"xmin": 324, "ymin": 20, "xmax": 640, "ymax": 373}},
                          {"score": 0.2_77, "label": "remote", "box": {"xmin": 40, "ymin": 72, "xmax": 177, "ymax": 115}},
                          {"score": 0.25_37, "label": "cat", "box": {"xmin": 1, "ymin": 55, "xmax": 315, "ymax": 472}},
                          {"score": 0.14_74, "label": "remote", "box": {"xmin": 335, "ymin": 74, "xmax": 371, "ymax": 187}},
                          {"score": 0.12_08, "label": "couch", "box": {"xmin": 4, "ymin": 0, "xmax": 642, "ymax": 476}},
                      ],
                      [
                          {"score": 0.28_68, "label": "cat", "box": {"xmin": 324, "ymin": 20, "xmax": 640, "ymax": 373}},
                          {"score": 0.2_77, "label": "remote", "box": {"xmin": 40, "ymin": 72, "xmax": 177, "ymax": 115}},
                          {"score": 0.25_37, "label": "cat", "box": {"xmin": 1, "ymin": 55, "xmax": 315, "ymax": 472}},
                          {"score": 0.14_74, "label": "remote", "box": {"xmin": 335, "ymin": 74, "xmax": 371, "ymax": 187}},
                          {"score": 0.12_08, "label": "couch", "box": {"xmin": 4, "ymin": 0, "xmax": 642, "ymax": 476}},
                      ],
                  ]					,   )

       @require_tf
       @unittest.skip("Zero Shot Object Detection not implemented in TF"		)
       def lowercase (		self: List[Any]		)       -> str:




              """simple docstring"""


              pass

       @require_torch
       @slow
       def lowercase (		self: Any		)       -> int:




              """simple docstring"""


              UpperCamelCase_          =  0.2
              UpperCamelCase_          =  pipeline("zero-shot-object-detection"		)

              UpperCamelCase_          =  object_detector(
                  "http://images.cocodataset.org/val2017/000000039769.jpg"					,   candidate_labels=["cat", "remote", "couch"]					,   threshold=_SCREAMING_SNAKE_CASE					,   )
              self.assertEqual(
                  nested_simplify(_SCREAMING_SNAKE_CASE					,   decimals=4		)					,   [
                      {"score": 0.28_68, "label": "cat", "box": {"xmin": 324, "ymin": 20, "xmax": 640, "ymax": 373}},
                      {"score": 0.2_77, "label": "remote", "box": {"xmin": 40, "ymin": 72, "xmax": 177, "ymax": 115}},
                      {"score": 0.25_37, "label": "cat", "box": {"xmin": 1, "ymin": 55, "xmax": 315, "ymax": 472}},
                  ]					,   )



       @require_torch
       @slow
       def lowercase (		self: Dict		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =  2
              UpperCamelCase_          =  pipeline("zero-shot-object-detection"		)

              UpperCamelCase_          =  object_detector(
                  "http://images.cocodataset.org/val2017/000000039769.jpg"					,   candidate_labels=["cat", "remote", "couch"]					,   top_k=_SCREAMING_SNAKE_CASE					,   )
              self.assertEqual(
                  nested_simplify(_SCREAMING_SNAKE_CASE					,   decimals=4		)					,   [
                      {"score": 0.28_68, "label": "cat", "box": {"xmin": 324, "ymin": 20, "xmax": 640, "ymax": 373}},
                      {"score": 0.2_77, "label": "remote", "box": {"xmin": 40, "ymin": 72, "xmax": 177, "ymax": 115}},
                  ]					,   )

