import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class __snake_case       (		lowerCamelCase_     ,	unittest.TestCase	):
   lowerCAmelCase_														=							GPTSanJapaneseTokenizer
   lowerCAmelCase_														=							False
   lowerCAmelCase_														=							{"do_clean_text": False, "add_prefix_space": False}




   def 			__a		(					self    :       Union[str, Any]   ):



         """simple docstring"""






         super().setUp()

         # fmt: off
         SCREAMING_SNAKE_CASE__    							=   ["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
         # fmt: on
         SCREAMING_SNAKE_CASE__    							=   {"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
         SCREAMING_SNAKE_CASE__    							=   {"""unk_token""": """<unk>"""}

         SCREAMING_SNAKE_CASE__    							=   os.path.join(self.tmpdirname ,			VOCAB_FILES_NAMES["""vocab_file"""]   )
         SCREAMING_SNAKE_CASE__    							=   os.path.join(self.tmpdirname ,			VOCAB_FILES_NAMES["""emoji_file"""]   )
         with open(self.vocab_file ,			"""w""" ,			encoding="""utf-8"""   ) as vocab_writer:
               vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]   )   )
         with open(self.emoji_file ,			"""w"""   ) as emoji_writer:
               emoji_writer.write(json.dumps(_lowercase   )   )




   def 			__a		(					self    :       Optional[int] ,			**_lowercase    :       Any   ):



         """simple docstring"""






         kwargs.update(self.special_tokens_map   )
         return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname ,			**_lowercase   )




   def 			__a		(					self    :       int ,			_lowercase    :       Tuple   ):



         """simple docstring"""






         SCREAMING_SNAKE_CASE__    							=   """こんにちは、世界。 \nこんばんは、㔺界。😀"""
         SCREAMING_SNAKE_CASE__    							=   """こんにちは、世界。 \nこんばんは、世界。😀"""
         return input_text, output_text




   def 			__a		(					self    :       List[Any] ,			_lowercase    :       Union[str, Any]   ):



         """simple docstring"""






         SCREAMING_SNAKE_CASE__							,  SCREAMING_SNAKE_CASE__    							=   self.get_input_output_texts(_lowercase   )
         SCREAMING_SNAKE_CASE__    							=   tokenizer.encode(_lowercase ,			add_special_tokens=_lowercase   )
         SCREAMING_SNAKE_CASE__    							=   tokenizer.decode(_lowercase ,			clean_up_tokenization_spaces=_lowercase   )
         return text, ids




   def 			__a		(					self    :       Optional[Any]   ):



         """simple docstring"""






         pass  # TODO add if relevant




   def 			__a		(					self    :       List[str]   ):



         """simple docstring"""






         pass  # TODO add if relevant




   def 			__a		(					self    :       Optional[Any]   ):



         """simple docstring"""






         pass  # TODO add if relevant




   def 			__a		(					self    :       int   ):



         """simple docstring"""






         SCREAMING_SNAKE_CASE__    							=   self.get_tokenizer()

         # Testing tokenization
         SCREAMING_SNAKE_CASE__    							=   """こんにちは、世界。　こんばんは、㔺界。"""
         SCREAMING_SNAKE_CASE__    							=   ["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
         SCREAMING_SNAKE_CASE__    							=   tokenizer.tokenize(_lowercase   )
         self.assertListEqual(_lowercase ,			_lowercase   )

         # Testing conversion to ids without special tokens
         SCREAMING_SNAKE_CASE__    							=   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
         SCREAMING_SNAKE_CASE__    							=   tokenizer.convert_tokens_to_ids(_lowercase   )
         self.assertListEqual(_lowercase ,			_lowercase   )

         # Testing conversion to ids with special tokens
         SCREAMING_SNAKE_CASE__    							=   tokens + [tokenizer.unk_token]
         SCREAMING_SNAKE_CASE__    							=   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
         SCREAMING_SNAKE_CASE__    							=   tokenizer.convert_tokens_to_ids(_lowercase   )
         self.assertListEqual(_lowercase ,			_lowercase   )




   def 			__a		(					self    :       Any   ):



         """simple docstring"""






         SCREAMING_SNAKE_CASE__    							=   self.get_tokenizer()

         # Testing tokenization
         SCREAMING_SNAKE_CASE__    							=   """こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
         SCREAMING_SNAKE_CASE__    							=   """こんにちは、、、、世界。こんばんは、、、、世界。"""
         SCREAMING_SNAKE_CASE__    							=   tokenizer.encode(_lowercase   )
         SCREAMING_SNAKE_CASE__    							=   tokenizer.decode(_lowercase   )
         self.assertEqual(_lowercase ,			_lowercase   )




   @slow
   def 			__a		(					self    :       List[Any]   ):



         """simple docstring"""






         SCREAMING_SNAKE_CASE__    							=   self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""   )

         # Testing tokenization
         SCREAMING_SNAKE_CASE__    							=   """こんにちは、世界。"""
         SCREAMING_SNAKE_CASE__    							=   """こんばんは、㔺界。😀"""
         SCREAMING_SNAKE_CASE__    							=   """こんにちは、世界。こんばんは、世界。😀"""
         SCREAMING_SNAKE_CASE__    							=   tokenizer.encode(prefix_text + input_text   )
         SCREAMING_SNAKE_CASE__    							=   tokenizer.encode("""""" ,			prefix_text=prefix_text + input_text   )
         SCREAMING_SNAKE_CASE__    							=   tokenizer.encode(_lowercase ,			prefix_text=_lowercase   )
         SCREAMING_SNAKE_CASE__    							=   tokenizer.decode(_lowercase   )
         SCREAMING_SNAKE_CASE__    							=   tokenizer.decode(_lowercase   )
         SCREAMING_SNAKE_CASE__    							=   tokenizer.decode(_lowercase   )
         self.assertEqual(_lowercase ,			_lowercase   )
         self.assertEqual(_lowercase ,			_lowercase   )
         self.assertEqual(_lowercase ,			_lowercase   )




   @slow
   def 			__a		(					self    :       Tuple   ):



         """simple docstring"""






         SCREAMING_SNAKE_CASE__    							=   self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""   )

         # Testing tokenization
         SCREAMING_SNAKE_CASE__    							=   """こんにちは、世界。"""
         SCREAMING_SNAKE_CASE__    							=   """こんばんは、㔺界。😀"""

         SCREAMING_SNAKE_CASE__    							=   len(tokenizer.encode(_lowercase   )   ) - 2
         SCREAMING_SNAKE_CASE__    							=   len(tokenizer.encode(_lowercase   )   ) - 2

         SCREAMING_SNAKE_CASE__    							=   [1] + [0] * (len_prefix + len_text + 1)
         SCREAMING_SNAKE_CASE__    							=   [1] * (len_prefix + len_text + 1) + [0]
         SCREAMING_SNAKE_CASE__    							=   [1] + [1] * (len_prefix) + [0] * (len_text + 1)

         SCREAMING_SNAKE_CASE__    							=   tokenizer(prefix_text + input_text   ).token_type_ids
         SCREAMING_SNAKE_CASE__    							=   tokenizer("""""" ,			prefix_text=prefix_text + input_text   ).token_type_ids
         SCREAMING_SNAKE_CASE__    							=   tokenizer(_lowercase ,			prefix_text=_lowercase   ).token_type_ids
         self.assertListEqual(_lowercase ,			_lowercase   )
         self.assertListEqual(_lowercase ,			_lowercase   )
         self.assertListEqual(_lowercase ,			_lowercase   )




   @slow
   def 			__a		(					self    :       Union[str, Any]   ):



         """simple docstring"""






         SCREAMING_SNAKE_CASE__    							=   self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""   )

         SCREAMING_SNAKE_CASE__    							=   tokenizer.encode("""あンいワ"""   )
         SCREAMING_SNAKE_CASE__    							=   tokenizer.encode("""""" ,			prefix_text="""あンいワ"""   )
         SCREAMING_SNAKE_CASE__    							=   tokenizer.encode("""いワ""" ,			prefix_text="""あン"""   )

         self.assertEqual(tokenizer.decode(_lowercase   ) ,			tokenizer.decode(_lowercase   )   )
         self.assertEqual(tokenizer.decode(_lowercase   ) ,			tokenizer.decode(_lowercase   )   )
         self.assertNotEqual(_lowercase ,			_lowercase   )
         self.assertNotEqual(_lowercase ,			_lowercase   )
         self.assertEqual(x_token_a[1] ,			x_token_a[-1]   )  # SEG token
         self.assertEqual(x_token_a[1] ,			x_token_a[3]   )  # SEG token




   @slow
   def 			__a		(					self    :       List[str]   ):



         """simple docstring"""






         SCREAMING_SNAKE_CASE__    							=   self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""   )

         SCREAMING_SNAKE_CASE__    							=   [["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
         SCREAMING_SNAKE_CASE__    							=   tokenizer(_lowercase ,			padding=_lowercase   )
         SCREAMING_SNAKE_CASE__    							=   tokenizer.batch_encode_plus(_lowercase ,			padding=_lowercase   )

         # fmt: off
         SCREAMING_SNAKE_CASE__    							=   [[3_59_93, 86_40, 2_59_48, 3_59_98, 3_06_47, 3_56_75, 3_59_99, 3_59_99], [3_59_93, 1_03_82, 98_68, 3_59_98, 3_06_46, 94_59, 3_06_46, 3_56_75]]
         SCREAMING_SNAKE_CASE__    							=   [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
         SCREAMING_SNAKE_CASE__    							=   [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
         # fmt: on
         self.assertListEqual(x_token.input_ids ,			_lowercase   )
         self.assertListEqual(x_token.token_type_ids ,			_lowercase   )
         self.assertListEqual(x_token.attention_mask ,			_lowercase   )
         self.assertListEqual(x_token_a.input_ids ,			_lowercase   )
         self.assertListEqual(x_token_a.token_type_ids ,			_lowercase   )
         self.assertListEqual(x_token_a.attention_mask ,			_lowercase   )




   def 			__a		(					self    :       Union[str, Any]   ):



         """simple docstring"""






         pass




   def 			__a		(					self    :       Optional[int]   ):



         """simple docstring"""






         pass

from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__lowerCamelCase    :				List[Any]							     =	{
    '''configuration_informer''': [
        '''INFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''InformerConfig''',
    ],
}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __lowerCamelCase    :				Union[str, Any]							     =	[
       '''INFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''InformerForPrediction''',
       '''InformerModel''',
       '''InformerPreTrainedModel''',
   ]


if TYPE_CHECKING:
   from .configuration_informer import INFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, InformerConfig

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_informer import (
          INFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          InformerForPrediction,
          InformerModel,
          InformerPreTrainedModel,
      )

else:
   import sys

   __lowerCamelCase    :				Dict							     =	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
