





import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 	__lowerCAmelCase      (       _UpperCamelCase		,unittest.TestCase	):
   _UpperCamelCase			:    Tuple  		=					GPTSanJapaneseTokenizer
   _UpperCamelCase			:    Tuple  		=					False
   _UpperCamelCase			:    Union[str, Any]  		=					{"""do_clean_text""": False, """add_prefix_space""": False}





   def 						_snake_case			(			self   )					->	Optional[int]:
      """simple docstring"""





      super().setUp()

      # fmt: off
      a__						:    Optional[Any]   		=	["こん", "こんに", "にちは", "ばんは", "世界,㔺界", "、", "。", "<BR>", "<SP>", "<TAB>", "<URL>", "<EMAIL>", "<TEL>", "<DATE>", "<PRICE>", "<BLOCK>", "<KIGOU>", "<U2000U2BFF>", "<|emoji1|>", "<unk>", "<|bagoftoken|>", "<|endoftext|>"]
      # fmt: on
      a__						:    int   		=	{"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
      a__						:    Optional[int]   		=	{"unk_token": "<unk>"}

      a__						:    int   		=	os.path.join(self.tmpdirname       ,      VOCAB_FILES_NAMES["vocab_file"]   )
      a__						:    Any   		=	os.path.join(self.tmpdirname       ,      VOCAB_FILES_NAMES["emoji_file"]   )
      with open(self.vocab_file       ,      "w"       ,      encoding="utf-8"   ) as vocab_writer:
         vocab_writer.write("".join([x + "\n" for x in vocab_tokens]   )   )
      with open(self.emoji_file       ,      "w"   ) as emoji_writer:
         emoji_writer.write(json.dumps(snake_case   )   )





   def 						_snake_case			(			self       ,      **snake_case   )					->	List[str]:
      """simple docstring"""





      kwargs.update(self.special_tokens_map   )
      return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname       ,      **snake_case   )





   def 						_snake_case			(			self       ,      snake_case   )					->	int:
      """simple docstring"""





      a__						:    Optional[int]   		=	"こんにちは、世界。 \nこんばんは、㔺界。😀"
      a__						:    List[Any]   		=	"こんにちは、世界。 \nこんばんは、世界。😀"
      return input_text, output_text





   def 						_snake_case			(			self       ,      snake_case   )					->	Any:
      """simple docstring"""





      a__		,  a__						:    Dict   		=	self.get_input_output_texts(snake_case   )
      a__						:    Tuple   		=	tokenizer.encode(snake_case       ,      add_special_tokens=snake_case   )
      a__						:    List[str]   		=	tokenizer.decode(snake_case       ,      clean_up_tokenization_spaces=snake_case   )
      return text, ids





   def 						_snake_case			(			self   )					->	List[Any]:
      """simple docstring"""





      pass  # TODO add if relevant





   def 						_snake_case			(			self   )					->	str:
      """simple docstring"""





      pass  # TODO add if relevant





   def 						_snake_case			(			self   )					->	str:
      """simple docstring"""





      pass  # TODO add if relevant





   def 						_snake_case			(			self   )					->	int:
      """simple docstring"""





      a__						:    Any   		=	self.get_tokenizer()

      # Testing tokenization
      a__						:    Optional[int]   		=	"こんにちは、世界。　こんばんは、㔺界。"
      a__						:    List[Any]   		=	["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
      a__						:    List[Any]   		=	tokenizer.tokenize(snake_case   )
      self.assertListEqual(snake_case       ,      snake_case   )

      # Testing conversion to ids without special tokens
      a__						:    Optional[int]   		=	[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
      a__						:    Optional[int]   		=	tokenizer.convert_tokens_to_ids(snake_case   )
      self.assertListEqual(snake_case       ,      snake_case   )

      # Testing conversion to ids with special tokens
      a__						:    List[Any]   		=	tokens + [tokenizer.unk_token]
      a__						:    Optional[int]   		=	[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
      a__						:    Any   		=	tokenizer.convert_tokens_to_ids(snake_case   )
      self.assertListEqual(snake_case       ,      snake_case   )





   def 						_snake_case			(			self   )					->	Optional[int]:
      """simple docstring"""





      a__						:    Dict   		=	self.get_tokenizer()

      # Testing tokenization
      a__						:    Optional[Any]   		=	"こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"
      a__						:    str   		=	"こんにちは、、、、世界。こんばんは、、、、世界。"
      a__						:    Union[str, Any]   		=	tokenizer.encode(snake_case   )
      a__						:    int   		=	tokenizer.decode(snake_case   )
      self.assertEqual(snake_case       ,      snake_case   )





   @slow
   def 						_snake_case			(			self   )					->	Optional[Any]:
      """simple docstring"""





      a__						:    Optional[Any]   		=	self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"   )

      # Testing tokenization
      a__						:    int   		=	"こんにちは、世界。"
      a__						:    List[Any]   		=	"こんばんは、㔺界。😀"
      a__						:    List[str]   		=	"こんにちは、世界。こんばんは、世界。😀"
      a__						:    List[Any]   		=	tokenizer.encode(prefix_text + input_text   )
      a__						:    List[Any]   		=	tokenizer.encode(""       ,      prefix_text=prefix_text + input_text   )
      a__						:    List[Any]   		=	tokenizer.encode(snake_case       ,      prefix_text=snake_case   )
      a__						:    Optional[Any]   		=	tokenizer.decode(snake_case   )
      a__						:    List[str]   		=	tokenizer.decode(snake_case   )
      a__						:    List[str]   		=	tokenizer.decode(snake_case   )
      self.assertEqual(snake_case       ,      snake_case   )
      self.assertEqual(snake_case       ,      snake_case   )
      self.assertEqual(snake_case       ,      snake_case   )





   @slow
   def 						_snake_case			(			self   )					->	Dict:
      """simple docstring"""





      a__						:    str   		=	self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"   )

      # Testing tokenization
      a__						:    Dict   		=	"こんにちは、世界。"
      a__						:    List[Any]   		=	"こんばんは、㔺界。😀"

      a__						:    List[Any]   		=	len(tokenizer.encode(snake_case   )   ) - 2
      a__						:    Union[str, Any]   		=	len(tokenizer.encode(snake_case   )   ) - 2

      a__						:    str   		=	[1] + [0] * (len_prefix + len_text + 1)
      a__						:    Union[str, Any]   		=	[1] * (len_prefix + len_text + 1) + [0]
      a__						:    Union[str, Any]   		=	[1] + [1] * (len_prefix) + [0] * (len_text + 1)

      a__						:    Union[str, Any]   		=	tokenizer(prefix_text + input_text   ).token_type_ids
      a__						:    Tuple   		=	tokenizer(""       ,      prefix_text=prefix_text + input_text   ).token_type_ids
      a__						:    List[Any]   		=	tokenizer(snake_case       ,      prefix_text=snake_case   ).token_type_ids
      self.assertListEqual(snake_case       ,      snake_case   )
      self.assertListEqual(snake_case       ,      snake_case   )
      self.assertListEqual(snake_case       ,      snake_case   )





   @slow
   def 						_snake_case			(			self   )					->	Optional[Any]:
      """simple docstring"""





      a__						:    Dict   		=	self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"   )

      a__						:    List[Any]   		=	tokenizer.encode("あンいワ"   )
      a__						:    Optional[Any]   		=	tokenizer.encode(""       ,      prefix_text="あンいワ"   )
      a__						:    List[str]   		=	tokenizer.encode("いワ"       ,      prefix_text="あン"   )

      self.assertEqual(tokenizer.decode(snake_case   )       ,      tokenizer.decode(snake_case   )   )
      self.assertEqual(tokenizer.decode(snake_case   )       ,      tokenizer.decode(snake_case   )   )
      self.assertNotEqual(snake_case       ,      snake_case   )
      self.assertNotEqual(snake_case       ,      snake_case   )
      self.assertEqual(x_token_a[1]       ,      x_token_a[-1]   )  # SEG token
      self.assertEqual(x_token_a[1]       ,      x_token_a[3]   )  # SEG token





   @slow
   def 						_snake_case			(			self   )					->	Tuple:
      """simple docstring"""





      a__						:    List[str]   		=	self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"   )

      a__						:    Union[str, Any]   		=	[["武田信玄", "は、"], ["織田信長", "の配下の、"]]
      a__						:    List[str]   		=	tokenizer(snake_case       ,      padding=snake_case   )
      a__						:    str   		=	tokenizer.batch_encode_plus(snake_case       ,      padding=snake_case   )

      # fmt: off
      a__						:    List[Any]   		=	[[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
      a__						:    Any   		=	[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
      a__						:    List[str]   		=	[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
      # fmt: on
      self.assertListEqual(x_token.input_ids       ,      snake_case   )
      self.assertListEqual(x_token.token_type_ids       ,      snake_case   )
      self.assertListEqual(x_token.attention_mask       ,      snake_case   )
      self.assertListEqual(x_token_a.input_ids       ,      snake_case   )
      self.assertListEqual(x_token_a.token_type_ids       ,      snake_case   )
      self.assertListEqual(x_token_a.attention_mask       ,      snake_case   )





   def 						_snake_case			(			self   )					->	Optional[int]:
      """simple docstring"""





      pass





   def 						_snake_case			(			self   )					->	Union[str, Any]:
      """simple docstring"""





      pass








from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


SCREAMING_SNAKE_CASE__							:      Union[str, Any]         =    {
    """configuration_autoformer""": [
        """AUTOFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """AutoformerConfig""",
    ],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  SCREAMING_SNAKE_CASE__							:      Tuple         =    [
      """AUTOFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """AutoformerForPrediction""",
      """AutoformerModel""",
      """AutoformerPreTrainedModel""",
  ]


if TYPE_CHECKING:
  from .configuration_autoformer import (
      AUTOFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
      AutoformerConfig,
  )

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_autoformer import (
        AUTOFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
        AutoformerForPrediction,
        AutoformerModel,
        AutoformerPreTrainedModel,
    )

else:
  import sys

  SCREAMING_SNAKE_CASE__							:      Any         =    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

