
"""simple docstring"""





import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 				a ( SCREAMING_SNAKE_CASE_,    unittest.TestCase							):





       """simple docstring"""







       A__       :							int    				=				GPTSanJapaneseTokenizer
       A__       :							Dict    				=				False
       A__       :							Any    				=				{"do_clean_text": False, "add_prefix_space": False}






       def 				__A						(			self			)    ->				str:
          super().setUp()

          # fmt: off
          _UpperCAmelCase         =      ['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
          # fmt: on
          _UpperCAmelCase         =      {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
          _UpperCAmelCase         =      {'unk_token': '<unk>'}

          _UpperCAmelCase         =      os.path.join(self.tmpdirname							,	VOCAB_FILES_NAMES["vocab_file"]			)
          _UpperCAmelCase         =      os.path.join(self.tmpdirname							,	VOCAB_FILES_NAMES["emoji_file"]			)
          with open(self.vocab_file							,	"w"							,	encoding="utf-8"			) as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)
          with open(self.emoji_file							,	"w"			) as emoji_writer:
             emoji_writer.write(json.dumps(__a			)			)






       def 				__A						(			self							,	**snake_case_			)    ->				Dict:
          kwargs.update(self.special_tokens_map			)
          return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname							,	**__a			)






       def 				__A						(			self							,	snake_case_			)    ->				Tuple:
          _UpperCAmelCase         =      'こんにちは、世界。 \nこんばんは、㔺界。😀'
          _UpperCAmelCase         =      'こんにちは、世界。 \nこんばんは、世界。😀'
          return input_text, output_text






       def 				__A						(			self							,	snake_case_			)    ->				Tuple:
          _UpperCAmelCase         =      self.get_input_output_texts(__a			)
          _UpperCAmelCase         =      tokenizer.encode(__a							,	add_special_tokens=__a			)
          _UpperCAmelCase         =      tokenizer.decode(__a							,	clean_up_tokenization_spaces=__a			)
          return text, ids






       def 				__A						(			self			)    ->				List[str]:
          pass  # TODO add if relevant






       def 				__A						(			self			)    ->				str:
          pass  # TODO add if relevant






       def 				__A						(			self			)    ->				Optional[Any]:
          pass  # TODO add if relevant






       def 				__A						(			self			)    ->				Dict:
          _UpperCAmelCase         =      self.get_tokenizer()

          # Testing tokenization
          _UpperCAmelCase         =      'こんにちは、世界。　こんばんは、㔺界。'
          _UpperCAmelCase         =      ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
          _UpperCAmelCase         =      tokenizer.tokenize(__a			)
          self.assertListEqual(__a							,	__a			)

          # Testing conversion to ids without special tokens
          _UpperCAmelCase         =      [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
          _UpperCAmelCase         =      tokenizer.convert_tokens_to_ids(__a			)
          self.assertListEqual(__a							,	__a			)

          # Testing conversion to ids with special tokens
          _UpperCAmelCase         =      tokens + [tokenizer.unk_token]
          _UpperCAmelCase         =      [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
          _UpperCAmelCase         =      tokenizer.convert_tokens_to_ids(__a			)
          self.assertListEqual(__a							,	__a			)






       def 				__A						(			self			)    ->				Any:
          _UpperCAmelCase         =      self.get_tokenizer()

          # Testing tokenization
          _UpperCAmelCase         =      'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
          _UpperCAmelCase         =      'こんにちは、、、、世界。こんばんは、、、、世界。'
          _UpperCAmelCase         =      tokenizer.encode(__a			)
          _UpperCAmelCase         =      tokenizer.decode(__a			)
          self.assertEqual(__a							,	__a			)






       @slow
       def 				__A						(			self			)    ->				Optional[int]:
          _UpperCAmelCase         =      self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"			)

          # Testing tokenization
          _UpperCAmelCase         =      'こんにちは、世界。'
          _UpperCAmelCase         =      'こんばんは、㔺界。😀'
          _UpperCAmelCase         =      'こんにちは、世界。こんばんは、世界。😀'
          _UpperCAmelCase         =      tokenizer.encode(prefix_text + input_text			)
          _UpperCAmelCase         =      tokenizer.encode(""							,	prefix_text=prefix_text + input_text			)
          _UpperCAmelCase         =      tokenizer.encode(__a							,	prefix_text=__a			)
          _UpperCAmelCase         =      tokenizer.decode(__a			)
          _UpperCAmelCase         =      tokenizer.decode(__a			)
          _UpperCAmelCase         =      tokenizer.decode(__a			)
          self.assertEqual(__a							,	__a			)
          self.assertEqual(__a							,	__a			)
          self.assertEqual(__a							,	__a			)






       @slow
       def 				__A						(			self			)    ->				int:
          _UpperCAmelCase         =      self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"			)

          # Testing tokenization
          _UpperCAmelCase         =      'こんにちは、世界。'
          _UpperCAmelCase         =      'こんばんは、㔺界。😀'

          _UpperCAmelCase         =      len(tokenizer.encode(__a			)			) - 2
          _UpperCAmelCase         =      len(tokenizer.encode(__a			)			) - 2

          _UpperCAmelCase         =      [1] + [0] * (len_prefix + len_text + 1)
          _UpperCAmelCase         =      [1] * (len_prefix + len_text + 1) + [0]
          _UpperCAmelCase         =      [1] + [1] * (len_prefix) + [0] * (len_text + 1)

          _UpperCAmelCase         =      tokenizer(prefix_text + input_text			).token_type_ids
          _UpperCAmelCase         =      tokenizer(""							,	prefix_text=prefix_text + input_text			).token_type_ids
          _UpperCAmelCase         =      tokenizer(__a							,	prefix_text=__a			).token_type_ids
          self.assertListEqual(__a							,	__a			)
          self.assertListEqual(__a							,	__a			)
          self.assertListEqual(__a							,	__a			)






       @slow
       def 				__A						(			self			)    ->				Union[str, Any]:
          _UpperCAmelCase         =      self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"			)

          _UpperCAmelCase         =      tokenizer.encode("あンいワ"			)
          _UpperCAmelCase         =      tokenizer.encode(""							,	prefix_text="あンいワ"			)
          _UpperCAmelCase         =      tokenizer.encode("いワ"							,	prefix_text="あン"			)

          self.assertEqual(tokenizer.decode(__a			)							,	tokenizer.decode(__a			)			)
          self.assertEqual(tokenizer.decode(__a			)							,	tokenizer.decode(__a			)			)
          self.assertNotEqual(__a							,	__a			)
          self.assertNotEqual(__a							,	__a			)
          self.assertEqual(x_token_a[1]							,	x_token_a[-1]			)  # SEG token
          self.assertEqual(x_token_a[1]							,	x_token_a[3]			)  # SEG token






       @slow
       def 				__A						(			self			)    ->				Optional[Any]:
          _UpperCAmelCase         =      self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"			)

          _UpperCAmelCase         =      [['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
          _UpperCAmelCase         =      tokenizer(__a							,	padding=__a			)
          _UpperCAmelCase         =      tokenizer.batch_encode_plus(__a							,	padding=__a			)

          # fmt: off
          _UpperCAmelCase         =      [[35993, 8640, 25948, 35998, 30647, 35675, 35999, 35999], [35993, 10382, 9868, 35998, 30646, 9459, 30646, 35675]]
          _UpperCAmelCase         =      [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
          _UpperCAmelCase         =      [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
          # fmt: on
          self.assertListEqual(x_token.input_ids							,	__a			)
          self.assertListEqual(x_token.token_type_ids							,	__a			)
          self.assertListEqual(x_token.attention_mask							,	__a			)
          self.assertListEqual(x_token_a.input_ids							,	__a			)
          self.assertListEqual(x_token_a.token_type_ids							,	__a			)
          self.assertListEqual(x_token_a.attention_mask							,	__a			)






       def 				__A						(			self			)    ->				int:
          pass






       def 				__A						(			self			)    ->				List[Any]:
          pass






'''simple docstring'''





from __future__ import annotations





def   a_   (		_UpperCAmelCase    :      list[int]				)    ->  bool:
      return len(set(_UpperCAmelCase				)				) == len(_UpperCAmelCase				)


if __name__ == "__main__":
    import doctest

    doctest.testmod()



