





'''simple docstring'''






import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 					UpperCAmelCase	(     snake_case_				,						unittest.TestCase  ):
      _lowercase:     List[Any]			=  GPTSanJapaneseTokenizer
      _lowercase:     Optional[Any]			=  False
      _lowercase:     str			=  {'''do_clean_text''': False, '''add_prefix_space''': False}




      def 							lowercase__       (     self     :					Optional[int]    )							-> str:
            super().setUp()

            # fmt: off
            _lowerCAmelCase					=				["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
            # fmt: on
            _lowerCAmelCase					=				{"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
            _lowerCAmelCase					=				{"""unk_token""": """<unk>"""}

            _lowerCAmelCase					=				os.path.join(self.tmpdirname     ,     VOCAB_FILES_NAMES["""vocab_file"""]    )
            _lowerCAmelCase					=				os.path.join(self.tmpdirname     ,     VOCAB_FILES_NAMES["""emoji_file"""]    )
            with open(self.vocab_file     ,     """w"""     ,     encoding="""utf-8"""    ) as vocab_writer:
                  vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )
            with open(self.emoji_file     ,     """w"""    ) as emoji_writer:
                  emoji_writer.write(json.dumps(__snake_case    )    )




      def 							lowercase__       (     self     :					Optional[int]     ,     **__snake_case     :					Optional[Any]    )							-> str:
            kwargs.update(self.special_tokens_map    )
            return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname     ,     **__snake_case    )




      def 							lowercase__       (     self     :					Dict     ,     __snake_case     :					Union[str, Any]    )							-> str:
            _lowerCAmelCase					=				"""こんにちは、世界。 \nこんばんは、㔺界。😀"""
            _lowerCAmelCase					=				"""こんにちは、世界。 \nこんばんは、世界。😀"""
            return input_text, output_text




      def 							lowercase__       (     self     :					Tuple     ,     __snake_case     :					str    )							-> Any:
            _lowerCAmelCase	,				_lowerCAmelCase					=				self.get_input_output_texts(__snake_case    )
            _lowerCAmelCase					=				tokenizer.encode(__snake_case     ,     add_special_tokens=__snake_case    )
            _lowerCAmelCase					=				tokenizer.decode(__snake_case     ,     clean_up_tokenization_spaces=__snake_case    )
            return text, ids




      def 							lowercase__       (     self     :					Optional[int]    )							-> List[Any]:
            pass  # TODO add if relevant




      def 							lowercase__       (     self     :					Optional[int]    )							-> Dict:
            pass  # TODO add if relevant




      def 							lowercase__       (     self     :					Dict    )							-> str:
            pass  # TODO add if relevant




      def 							lowercase__       (     self     :					Optional[int]    )							-> Any:
            _lowerCAmelCase					=				self.get_tokenizer()

            # Testing tokenization
            _lowerCAmelCase					=				"""こんにちは、世界。　こんばんは、㔺界。"""
            _lowerCAmelCase					=				["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
            _lowerCAmelCase					=				tokenizer.tokenize(__snake_case    )
            self.assertListEqual(__snake_case     ,     __snake_case    )

            # Testing conversion to ids without special tokens
            _lowerCAmelCase					=				[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
            _lowerCAmelCase					=				tokenizer.convert_tokens_to_ids(__snake_case    )
            self.assertListEqual(__snake_case     ,     __snake_case    )

            # Testing conversion to ids with special tokens
            _lowerCAmelCase					=				tokens + [tokenizer.unk_token]
            _lowerCAmelCase					=				[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
            _lowerCAmelCase					=				tokenizer.convert_tokens_to_ids(__snake_case    )
            self.assertListEqual(__snake_case     ,     __snake_case    )




      def 							lowercase__       (     self     :					int    )							-> Optional[int]:
            _lowerCAmelCase					=				self.get_tokenizer()

            # Testing tokenization
            _lowerCAmelCase					=				"""こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
            _lowerCAmelCase					=				"""こんにちは、、、、世界。こんばんは、、、、世界。"""
            _lowerCAmelCase					=				tokenizer.encode(__snake_case    )
            _lowerCAmelCase					=				tokenizer.decode(__snake_case    )
            self.assertEqual(__snake_case     ,     __snake_case    )




      @slow
      def 							lowercase__       (     self     :					int    )							-> int:
            _lowerCAmelCase					=				self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""    )

            # Testing tokenization
            _lowerCAmelCase					=				"""こんにちは、世界。"""
            _lowerCAmelCase					=				"""こんばんは、㔺界。😀"""
            _lowerCAmelCase					=				"""こんにちは、世界。こんばんは、世界。😀"""
            _lowerCAmelCase					=				tokenizer.encode(prefix_text + input_text    )
            _lowerCAmelCase					=				tokenizer.encode(""""""     ,     prefix_text=prefix_text + input_text    )
            _lowerCAmelCase					=				tokenizer.encode(__snake_case     ,     prefix_text=__snake_case    )
            _lowerCAmelCase					=				tokenizer.decode(__snake_case    )
            _lowerCAmelCase					=				tokenizer.decode(__snake_case    )
            _lowerCAmelCase					=				tokenizer.decode(__snake_case    )
            self.assertEqual(__snake_case     ,     __snake_case    )
            self.assertEqual(__snake_case     ,     __snake_case    )
            self.assertEqual(__snake_case     ,     __snake_case    )




      @slow
      def 							lowercase__       (     self     :					List[Any]    )							-> List[str]:
            _lowerCAmelCase					=				self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""    )

            # Testing tokenization
            _lowerCAmelCase					=				"""こんにちは、世界。"""
            _lowerCAmelCase					=				"""こんばんは、㔺界。😀"""

            _lowerCAmelCase					=				len(tokenizer.encode(__snake_case    )    ) - 2
            _lowerCAmelCase					=				len(tokenizer.encode(__snake_case    )    ) - 2

            _lowerCAmelCase					=				[1] + [0] * (len_prefix + len_text + 1)
            _lowerCAmelCase					=				[1] * (len_prefix + len_text + 1) + [0]
            _lowerCAmelCase					=				[1] + [1] * (len_prefix) + [0] * (len_text + 1)

            _lowerCAmelCase					=				tokenizer(prefix_text + input_text    ).token_type_ids
            _lowerCAmelCase					=				tokenizer(""""""     ,     prefix_text=prefix_text + input_text    ).token_type_ids
            _lowerCAmelCase					=				tokenizer(__snake_case     ,     prefix_text=__snake_case    ).token_type_ids
            self.assertListEqual(__snake_case     ,     __snake_case    )
            self.assertListEqual(__snake_case     ,     __snake_case    )
            self.assertListEqual(__snake_case     ,     __snake_case    )




      @slow
      def 							lowercase__       (     self     :					List[str]    )							-> Dict:
            _lowerCAmelCase					=				self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""    )

            _lowerCAmelCase					=				tokenizer.encode("""あンいワ"""    )
            _lowerCAmelCase					=				tokenizer.encode(""""""     ,     prefix_text="""あンいワ"""    )
            _lowerCAmelCase					=				tokenizer.encode("""いワ"""     ,     prefix_text="""あン"""    )

            self.assertEqual(tokenizer.decode(__snake_case    )     ,     tokenizer.decode(__snake_case    )    )
            self.assertEqual(tokenizer.decode(__snake_case    )     ,     tokenizer.decode(__snake_case    )    )
            self.assertNotEqual(__snake_case     ,     __snake_case    )
            self.assertNotEqual(__snake_case     ,     __snake_case    )
            self.assertEqual(x_token_a[1]     ,     x_token_a[-1]    )  # SEG token
            self.assertEqual(x_token_a[1]     ,     x_token_a[3]    )  # SEG token




      @slow
      def 							lowercase__       (     self     :					Optional[Any]    )							-> Any:
            _lowerCAmelCase					=				self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""    )

            _lowerCAmelCase					=				[["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
            _lowerCAmelCase					=				tokenizer(__snake_case     ,     padding=__snake_case    )
            _lowerCAmelCase					=				tokenizer.batch_encode_plus(__snake_case     ,     padding=__snake_case    )

            # fmt: off
            _lowerCAmelCase					=				[[3_59_93, 86_40, 2_59_48, 3_59_98, 3_06_47, 3_56_75, 3_59_99, 3_59_99], [3_59_93, 1_03_82, 98_68, 3_59_98, 3_06_46, 94_59, 3_06_46, 3_56_75]]
            _lowerCAmelCase					=				[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
            _lowerCAmelCase					=				[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
            # fmt: on
            self.assertListEqual(x_token.input_ids     ,     __snake_case    )
            self.assertListEqual(x_token.token_type_ids     ,     __snake_case    )
            self.assertListEqual(x_token.attention_mask     ,     __snake_case    )
            self.assertListEqual(x_token_a.input_ids     ,     __snake_case    )
            self.assertListEqual(x_token_a.token_type_ids     ,     __snake_case    )
            self.assertListEqual(x_token_a.attention_mask     ,     __snake_case    )




      def 							lowercase__       (     self     :					Tuple    )							-> Dict:
            # Intentionally convert some words to accommodate character fluctuations unique to Japanese
            pass




      def 							lowercase__       (     self     :					List[str]    )							-> Optional[Any]:
            # tokenizer has no padding token
            pass













'''simple docstring'''






from typing import Callable, Optional

from .. import Features
from ..packaged_modules.generator.generator import Generator
from .abc import AbstractDatasetInputStream




class 					UpperCAmelCase	(     snake_case_  ):




      def __init__(     self     :					Union[str, Any]     ,     __snake_case     :					Callable     ,     __snake_case     :					Optional[Features] = None     ,     __snake_case     :					str = None     ,     __snake_case     :					bool = False     ,     __snake_case     :					bool = False     ,     __snake_case     :					Optional[dict] = None     ,     __snake_case     :					Optional[int] = None     ,     **__snake_case     :					Optional[Any]     ,     )							-> Union[str, Any]:
            super().__init__(
                features=__snake_case     ,     cache_dir=__snake_case     ,     keep_in_memory=__snake_case     ,     streaming=__snake_case     ,     num_proc=__snake_case     ,     **__snake_case     ,     )
            _lowerCAmelCase					=				Generator(
                cache_dir=__snake_case     ,     features=__snake_case     ,     generator=__snake_case     ,     gen_kwargs=__snake_case     ,     **__snake_case     ,     )




      def 							lowercase__       (     self     :					str    )							-> Dict:
            # Build iterable dataset
            if self.streaming:
                  _lowerCAmelCase					=				self.builder.as_streaming_dataset(split="""train"""    )
            # Build regular (map-style) dataset
            else:
                  _lowerCAmelCase					=				None
                  _lowerCAmelCase					=				None
                  _lowerCAmelCase					=				None
                  _lowerCAmelCase					=				None

                  self.builder.download_and_prepare(
                      download_config=__snake_case     ,     download_mode=__snake_case     ,     verification_mode=__snake_case     ,     base_path=__snake_case     ,     num_proc=self.num_proc     ,     )
                  _lowerCAmelCase					=				self.builder.as_dataset(
                      split="""train"""     ,     verification_mode=__snake_case     ,     in_memory=self.keep_in_memory    )
            return dataset






