




import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class     a__					(      snake_case__				,       unittest.TestCase   ):
						_a							:					Tuple      =					GPTSanJapaneseTokenizer
						_a							:					Tuple      =					False
						_a							:					Optional[int]      =					{"""do_clean_text""": False, """add_prefix_space""": False}






						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											super().setUp()

											# fmt: off
											__lowerCAmelCase            = ["こん", "こんに", "にちは", "ばんは", "世界,㔺界", "、", "。", "<BR>", "<SP>", "<TAB>", "<URL>", "<EMAIL>", "<TEL>", "<DATE>", "<PRICE>", "<BLOCK>", "<KIGOU>", "<U2000U2BFF>", "<|emoji1|>", "<unk>", "<|bagoftoken|>", "<|endoftext|>"]
											# fmt: on
											__lowerCAmelCase            = {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
											__lowerCAmelCase            = {"unk_token": "<unk>"}

											__lowerCAmelCase            = os.path.join(self.tmpdirname	,    VOCAB_FILES_NAMES["vocab_file"]    )
											__lowerCAmelCase            = os.path.join(self.tmpdirname	,    VOCAB_FILES_NAMES["emoji_file"]    )
											with open(self.vocab_file	,    "w"	,    encoding="utf-8"    ) as vocab_writer:
																vocab_writer.write("".join([x + "\n" for x in vocab_tokens]    )    )
											with open(self.emoji_file	,    "w"    ) as emoji_writer:
																emoji_writer.write(json.dumps(_A    )    )






						def        __SCREAMING_SNAKE_CASE(     self	,    **_A    ):




											"""simple docstring"""



											kwargs.update(self.special_tokens_map    )
											return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname	,    **_A    )






						def        __SCREAMING_SNAKE_CASE(     self	,    _A    ):




											"""simple docstring"""



											__lowerCAmelCase            = "こんにちは、世界。 \nこんばんは、㔺界。😀"
											__lowerCAmelCase            = "こんにちは、世界。 \nこんばんは、世界。😀"
											return input_text, output_text






						def        __SCREAMING_SNAKE_CASE(     self	,    _A    ):




											"""simple docstring"""



											__lowerCAmelCase     ,							__lowerCAmelCase            = self.get_input_output_texts(_A    )
											__lowerCAmelCase            = tokenizer.encode(_A	,    add_special_tokens=_A    )
											__lowerCAmelCase            = tokenizer.decode(_A	,    clean_up_tokenization_spaces=_A    )
											return text, ids






						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											pass  # TODO add if relevant






						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											pass  # TODO add if relevant






						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											pass  # TODO add if relevant






						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											__lowerCAmelCase            = self.get_tokenizer()

											# Testing tokenization
											__lowerCAmelCase            = "こんにちは、世界。　こんばんは、㔺界。"
											__lowerCAmelCase            = ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
											__lowerCAmelCase            = tokenizer.tokenize(_A    )
											self.assertListEqual(_A	,    _A    )

											# Testing conversion to ids without special tokens
											__lowerCAmelCase            = [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
											__lowerCAmelCase            = tokenizer.convert_tokens_to_ids(_A    )
											self.assertListEqual(_A	,    _A    )

											# Testing conversion to ids with special tokens
											__lowerCAmelCase            = tokens + [tokenizer.unk_token]
											__lowerCAmelCase            = [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
											__lowerCAmelCase            = tokenizer.convert_tokens_to_ids(_A    )
											self.assertListEqual(_A	,    _A    )






						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											__lowerCAmelCase            = self.get_tokenizer()

											# Testing tokenization
											__lowerCAmelCase            = "こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"
											__lowerCAmelCase            = "こんにちは、、、、世界。こんばんは、、、、世界。"
											__lowerCAmelCase            = tokenizer.encode(_A    )
											__lowerCAmelCase            = tokenizer.decode(_A    )
											self.assertEqual(_A	,    _A    )






						@slow
						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											__lowerCAmelCase            = self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"    )

											# Testing tokenization
											__lowerCAmelCase            = "こんにちは、世界。"
											__lowerCAmelCase            = "こんばんは、㔺界。😀"
											__lowerCAmelCase            = "こんにちは、世界。こんばんは、世界。😀"
											__lowerCAmelCase            = tokenizer.encode(prefix_text + input_text    )
											__lowerCAmelCase            = tokenizer.encode(""	,    prefix_text=prefix_text + input_text    )
											__lowerCAmelCase            = tokenizer.encode(_A	,    prefix_text=_A    )
											__lowerCAmelCase            = tokenizer.decode(_A    )
											__lowerCAmelCase            = tokenizer.decode(_A    )
											__lowerCAmelCase            = tokenizer.decode(_A    )
											self.assertEqual(_A	,    _A    )
											self.assertEqual(_A	,    _A    )
											self.assertEqual(_A	,    _A    )






						@slow
						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											__lowerCAmelCase            = self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"    )

											# Testing tokenization
											__lowerCAmelCase            = "こんにちは、世界。"
											__lowerCAmelCase            = "こんばんは、㔺界。😀"

											__lowerCAmelCase            = len(tokenizer.encode(_A    )    ) - 2
											__lowerCAmelCase            = len(tokenizer.encode(_A    )    ) - 2

											__lowerCAmelCase            = [1] + [0] * (len_prefix + len_text + 1)
											__lowerCAmelCase            = [1] * (len_prefix + len_text + 1) + [0]
											__lowerCAmelCase            = [1] + [1] * (len_prefix) + [0] * (len_text + 1)

											__lowerCAmelCase            = tokenizer(prefix_text + input_text    ).token_type_ids
											__lowerCAmelCase            = tokenizer(""	,    prefix_text=prefix_text + input_text    ).token_type_ids
											__lowerCAmelCase            = tokenizer(_A	,    prefix_text=_A    ).token_type_ids
											self.assertListEqual(_A	,    _A    )
											self.assertListEqual(_A	,    _A    )
											self.assertListEqual(_A	,    _A    )






						@slow
						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											__lowerCAmelCase            = self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"    )

											__lowerCAmelCase            = tokenizer.encode("あンいワ"    )
											__lowerCAmelCase            = tokenizer.encode(""	,    prefix_text="あンいワ"    )
											__lowerCAmelCase            = tokenizer.encode("いワ"	,    prefix_text="あン"    )

											self.assertEqual(tokenizer.decode(_A    )	,    tokenizer.decode(_A    )    )
											self.assertEqual(tokenizer.decode(_A    )	,    tokenizer.decode(_A    )    )
											self.assertNotEqual(_A	,    _A    )
											self.assertNotEqual(_A	,    _A    )
											self.assertEqual(x_token_a[1]	,    x_token_a[-1]    )  # SEG token
											self.assertEqual(x_token_a[1]	,    x_token_a[3]    )  # SEG token






						@slow
						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											__lowerCAmelCase            = self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"    )

											__lowerCAmelCase            = [["武田信玄", "は、"], ["織田信長", "の配下の、"]]
											__lowerCAmelCase            = tokenizer(_A	,    padding=_A    )
											__lowerCAmelCase            = tokenizer.batch_encode_plus(_A	,    padding=_A    )

											# fmt: off
											__lowerCAmelCase            = [[3_5_9_9_3, 8_6_4_0, 2_5_9_4_8, 3_5_9_9_8, 3_0_6_4_7, 3_5_6_7_5, 3_5_9_9_9, 3_5_9_9_9], [3_5_9_9_3, 1_0_3_8_2, 9_8_6_8, 3_5_9_9_8, 3_0_6_4_6, 9_4_5_9, 3_0_6_4_6, 3_5_6_7_5]]
											__lowerCAmelCase            = [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
											__lowerCAmelCase            = [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
											# fmt: on
											self.assertListEqual(x_token.input_ids	,    _A    )
											self.assertListEqual(x_token.token_type_ids	,    _A    )
											self.assertListEqual(x_token.attention_mask	,    _A    )
											self.assertListEqual(x_token_a.input_ids	,    _A    )
											self.assertListEqual(x_token_a.token_type_ids	,    _A    )
											self.assertListEqual(x_token_a.attention_mask	,    _A    )






						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											pass






						def        __SCREAMING_SNAKE_CASE(     self    ):




											"""simple docstring"""



											pass










import torch
from transformers import PreTrainedModel, XLMRobertaConfig, XLMRobertaModel
class     a__					(      snake_case__   ):
						_a							:					Optional[Any]      =					"""M-CLIP"""






						def __init__(     self	,    _A=1_0_2_4	,    _A=7_6_8	,    **_A    ):




											"""simple docstring"""



											__lowerCAmelCase            = transformerDimSize
											__lowerCAmelCase            = imageDimSize
											super().__init__(**_A    )






class     a__					(      snake_case__   ):
						_a							:					Tuple      =					MCLIPConfig






						def __init__(     self	,    _A	,    *_A	,    **_A    ):




											"""simple docstring"""



											super().__init__(_A	,    *_A	,    **_A    )
											__lowerCAmelCase            = XLMRobertaModel(_A    )
											__lowerCAmelCase            = torch.nn.Linear(
											    in_features=config.transformerDimensions	,    out_features=config.numDims    )






						def        __SCREAMING_SNAKE_CASE(     self	,    _A	,    _A    ):




											"""simple docstring"""



											__lowerCAmelCase            = self.transformer(input_ids=_A	,    attention_mask=_A    )[0]
											__lowerCAmelCase            = (embs * attention_mask.unsqueeze(2    )).sum(dim=1    ) / attention_mask.sum(dim=1    )[:, None]
											return self.LinearTransformation(_A    ), embs




