


import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class       __magic_name__	(     __lowerCAmelCase					,				unittest.TestCase):
     A:		Union[str, Any]			=      GPTSanJapaneseTokenizer
     A:		Tuple			=      False
     A:		Any			=      {"do_clean_text": False, "add_prefix_space": False}


     def      UpperCAmelCase__	(      self							:     Any			)  ->      str:





      '''simple docstring'''






      super().setUp()

      # fmt: off
      UpperCamelCase__					:    int          =   ['''こん''', '''こんに''', '''にちは''', '''ばんは''', '''世界,㔺界''', '''、''', '''。''', '''<BR>''', '''<SP>''', '''<TAB>''', '''<URL>''', '''<EMAIL>''', '''<TEL>''', '''<DATE>''', '''<PRICE>''', '''<BLOCK>''', '''<KIGOU>''', '''<U2000U2BFF>''', '''<|emoji1|>''', '''<unk>''', '''<|bagoftoken|>''', '''<|endoftext|>''']
      # fmt: on
      UpperCamelCase__					:    Tuple          =   {'''emoji''': {'''\ud83d\ude00''': '''<|emoji1|>'''}, '''emoji_inv''': {'''<|emoji1|>''': '''\ud83d\ude00'''}}  # 😀
      UpperCamelCase__					:    Optional[int]          =   {'''unk_token''': '''<unk>'''}

      UpperCamelCase__					:    str          =   os.path.join(self.tmpdirname							,	VOCAB_FILES_NAMES['''vocab_file''']			)
      UpperCamelCase__					:    Optional[Any]          =   os.path.join(self.tmpdirname							,	VOCAB_FILES_NAMES['''emoji_file''']			)
      with open(self.vocab_file							,	'''w'''							,	encoding='''utf-8'''			) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]			)			)
      with open(self.emoji_file							,	'''w'''			) as emoji_writer:
       emoji_writer.write(json.dumps(lowerCamelCase__			)			)


     def      UpperCAmelCase__	(      self							:     Optional[int]							,	**lowerCamelCase__							:     Optional[int]			)  ->      Any:





      '''simple docstring'''






      kwargs.update(self.special_tokens_map			)
      return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname							,	**lowerCamelCase__			)


     def      UpperCAmelCase__	(      self							:     Optional[int]							,	lowerCamelCase__							:     Dict			)  ->      Optional[int]:





      '''simple docstring'''






      UpperCamelCase__					:    int          =   '''こんにちは、世界。 \nこんばんは、㔺界。😀'''
      UpperCamelCase__					:    List[Any]          =   '''こんにちは、世界。 \nこんばんは、世界。😀'''
      return input_text, output_text


     def      UpperCAmelCase__	(      self							:     Union[str, Any]							,	lowerCamelCase__							:     List[str]			)  ->      Union[str, Any]:





      '''simple docstring'''






      UpperCamelCase__       ,				UpperCamelCase__					:    Union[str, Any]          =   self.get_input_output_texts(lowerCamelCase__			)
      UpperCamelCase__					:    List[str]          =   tokenizer.encode(lowerCamelCase__							,	add_special_tokens=lowerCamelCase__			)
      UpperCamelCase__					:    str          =   tokenizer.decode(lowerCamelCase__							,	clean_up_tokenization_spaces=lowerCamelCase__			)
      return text, ids


     def      UpperCAmelCase__	(      self							:     Any			)  ->      List[str]:





      '''simple docstring'''






      pass  # TODO add if relevant


     def      UpperCAmelCase__	(      self							:     Optional[Any]			)  ->      Any:





      '''simple docstring'''






      pass  # TODO add if relevant


     def      UpperCAmelCase__	(      self							:     Any			)  ->      List[Any]:





      '''simple docstring'''






      pass  # TODO add if relevant


     def      UpperCAmelCase__	(      self							:     Any			)  ->      Any:





      '''simple docstring'''






      UpperCamelCase__					:    Optional[Any]          =   self.get_tokenizer()

      # Testing tokenization
      UpperCamelCase__					:    List[Any]          =   '''こんにちは、世界。　こんばんは、㔺界。'''
      UpperCamelCase__					:    Optional[Any]          =   ['''こん''', '''にちは''', '''、''', '''世界''', '''。''', '''<SP>''', '''こん''', '''ばんは''', '''、''', '''㔺界''', '''。''']
      UpperCamelCase__					:    int          =   tokenizer.tokenize(lowerCamelCase__			)
      self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)

      # Testing conversion to ids without special tokens
      UpperCamelCase__					:    List[str]          =   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
      UpperCamelCase__					:    Any          =   tokenizer.convert_tokens_to_ids(lowerCamelCase__			)
      self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)

      # Testing conversion to ids with special tokens
      UpperCamelCase__					:    str          =   tokens + [tokenizer.unk_token]
      UpperCamelCase__					:    Tuple          =   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
      UpperCamelCase__					:    Union[str, Any]          =   tokenizer.convert_tokens_to_ids(lowerCamelCase__			)
      self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)


     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    Optional[int]          =   self.get_tokenizer()

      # Testing tokenization
      UpperCamelCase__					:    Optional[int]          =   '''こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'''
      UpperCamelCase__					:    Optional[Any]          =   '''こんにちは、、、、世界。こんばんは、、、、世界。'''
      UpperCamelCase__					:    Tuple          =   tokenizer.encode(lowerCamelCase__			)
      UpperCamelCase__					:    Optional[Any]          =   tokenizer.decode(lowerCamelCase__			)
      self.assertEqual(lowerCamelCase__							,	lowerCamelCase__			)


     @slow
     def      UpperCAmelCase__	(      self							:     int			)  ->      Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    Dict          =   self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''			)

      # Testing tokenization
      UpperCamelCase__					:    str          =   '''こんにちは、世界。'''
      UpperCamelCase__					:    Optional[int]          =   '''こんばんは、㔺界。😀'''
      UpperCamelCase__					:    Optional[int]          =   '''こんにちは、世界。こんばんは、世界。😀'''
      UpperCamelCase__					:    List[str]          =   tokenizer.encode(prefix_text + input_text			)
      UpperCamelCase__					:    List[str]          =   tokenizer.encode(''''''							,	prefix_text=prefix_text + input_text			)
      UpperCamelCase__					:    str          =   tokenizer.encode(lowerCamelCase__							,	prefix_text=lowerCamelCase__			)
      UpperCamelCase__					:    Tuple          =   tokenizer.decode(lowerCamelCase__			)
      UpperCamelCase__					:    List[str]          =   tokenizer.decode(lowerCamelCase__			)
      UpperCamelCase__					:    Optional[int]          =   tokenizer.decode(lowerCamelCase__			)
      self.assertEqual(lowerCamelCase__							,	lowerCamelCase__			)
      self.assertEqual(lowerCamelCase__							,	lowerCamelCase__			)
      self.assertEqual(lowerCamelCase__							,	lowerCamelCase__			)


     @slow
     def      UpperCAmelCase__	(      self							:     Any			)  ->      List[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''			)

      # Testing tokenization
      UpperCamelCase__					:    Optional[int]          =   '''こんにちは、世界。'''
      UpperCamelCase__					:    Dict          =   '''こんばんは、㔺界。😀'''

      UpperCamelCase__					:    Optional[int]          =   len(tokenizer.encode(lowerCamelCase__			)			) - 2
      UpperCamelCase__					:    int          =   len(tokenizer.encode(lowerCamelCase__			)			) - 2

      UpperCamelCase__					:    Optional[Any]          =   [1] + [0] * (len_prefix + len_text + 1)
      UpperCamelCase__					:    Tuple          =   [1] * (len_prefix + len_text + 1) + [0]
      UpperCamelCase__					:    int          =   [1] + [1] * (len_prefix) + [0] * (len_text + 1)

      UpperCamelCase__					:    Tuple          =   tokenizer(prefix_text + input_text			).token_type_ids
      UpperCamelCase__					:    Tuple          =   tokenizer(''''''							,	prefix_text=prefix_text + input_text			).token_type_ids
      UpperCamelCase__					:    List[str]          =   tokenizer(lowerCamelCase__							,	prefix_text=lowerCamelCase__			).token_type_ids
      self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)
      self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)
      self.assertListEqual(lowerCamelCase__							,	lowerCamelCase__			)


     @slow
     def      UpperCAmelCase__	(      self							:     Optional[Any]			)  ->      Optional[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    Tuple          =   self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''			)

      UpperCamelCase__					:    str          =   tokenizer.encode('''あンいワ'''			)
      UpperCamelCase__					:    Any          =   tokenizer.encode(''''''							,	prefix_text='''あンいワ'''			)
      UpperCamelCase__					:    List[str]          =   tokenizer.encode('''いワ'''							,	prefix_text='''あン'''			)

      self.assertEqual(tokenizer.decode(lowerCamelCase__			)							,	tokenizer.decode(lowerCamelCase__			)			)
      self.assertEqual(tokenizer.decode(lowerCamelCase__			)							,	tokenizer.decode(lowerCamelCase__			)			)
      self.assertNotEqual(lowerCamelCase__							,	lowerCamelCase__			)
      self.assertNotEqual(lowerCamelCase__							,	lowerCamelCase__			)
      self.assertEqual(x_token_a[1]							,	x_token_a[-1]			)  # SEG token
      self.assertEqual(x_token_a[1]							,	x_token_a[3]			)  # SEG token


     @slow
     def      UpperCAmelCase__	(      self							:     List[str]			)  ->      List[Any]:





      '''simple docstring'''






      UpperCamelCase__					:    List[Any]          =   self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''			)

      UpperCamelCase__					:    Any          =   [['''武田信玄''', '''は、'''], ['''織田信長''', '''の配下の、''']]
      UpperCamelCase__					:    Union[str, Any]          =   tokenizer(lowerCamelCase__							,	padding=lowerCamelCase__			)
      UpperCamelCase__					:    Any          =   tokenizer.batch_encode_plus(lowerCamelCase__							,	padding=lowerCamelCase__			)

      # fmt: off
      UpperCamelCase__					:    Union[str, Any]          =   [[35993, 8640, 25948, 35998, 30647, 35675, 35999, 35999], [35993, 10382, 9868, 35998, 30646, 9459, 30646, 35675]]
      UpperCamelCase__					:    str          =   [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
      UpperCamelCase__					:    List[str]          =   [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
      # fmt: on
      self.assertListEqual(x_token.input_ids							,	lowerCamelCase__			)
      self.assertListEqual(x_token.token_type_ids							,	lowerCamelCase__			)
      self.assertListEqual(x_token.attention_mask							,	lowerCamelCase__			)
      self.assertListEqual(x_token_a.input_ids							,	lowerCamelCase__			)
      self.assertListEqual(x_token_a.token_type_ids							,	lowerCamelCase__			)
      self.assertListEqual(x_token_a.attention_mask							,	lowerCamelCase__			)


     def      UpperCAmelCase__	(      self							:     int			)  ->      Any:





      '''simple docstring'''






      pass


     def      UpperCAmelCase__	(      self							:     Dict			)  ->      Any:





      '''simple docstring'''






      pass






from typing import TYPE_CHECKING

from ..utils import _LazyModule


__UpperCamelCase       :       int						       =				{
    "config": [
        "EXTERNAL_DATA_FORMAT_SIZE_LIMIT",
        "OnnxConfig",
        "OnnxConfigWithPast",
        "OnnxSeq2SeqConfigWithPast",
        "PatchingSpec",
    ],
    "convert": ["export", "validate_model_outputs"],
    "features": ["FeaturesManager"],
    "utils": ["ParameterFormat", "compute_serialized_parameters_size"],
}


if TYPE_CHECKING:
       from .config import (
           EXTERNAL_DATA_FORMAT_SIZE_LIMIT,
           OnnxConfig,
           OnnxConfigWithPast,
           OnnxSeqaSeqConfigWithPast,
           PatchingSpec,
       )
       from .convert import export, validate_model_outputs
       from .features import FeaturesManager
       from .utils import ParameterFormat, compute_serialized_parameters_size

else:
       import sys

       __UpperCamelCase       :       List[str]						       =				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


