
"""simple docstring"""





import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class __lowerCAmelCase					(       snake_case_  ,					unittest.TestCase   ):




			"""simple docstring"""







			snake_case						   =					GPTSanJapaneseTokenizer
			snake_case						   =					False
			snake_case						   =					{'''do_clean_text''': False, '''add_prefix_space''': False}




			def   lowerCamelCase__  (					self    :       Optional[int]						)   ->			str:







								"""simple docstring"""
								super().setUp()

								# fmt: off
								A_								=    ["こん", "こんに", "にちは", "ばんは", "世界,㔺界", "、", "。", "<BR>", "<SP>", "<TAB>", "<URL>", "<EMAIL>", "<TEL>", "<DATE>", "<PRICE>", "<BLOCK>", "<KIGOU>", "<U2000U2BFF>", "<|emoji1|>", "<unk>", "<|bagoftoken|>", "<|endoftext|>"]
								# fmt: on
								A_								=    {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
								A_								=    {"unk_token": "<unk>"}

								A_								=    os.path.join(self.tmpdirname							, VOCAB_FILES_NAMES["vocab_file"]						)
								A_								=    os.path.join(self.tmpdirname							, VOCAB_FILES_NAMES["emoji_file"]						)
								with open(self.vocab_file							, "w"							, encoding="utf-8"						) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)
								with open(self.emoji_file							, "w"						) as emoji_writer:
													emoji_writer.write(json.dumps(__snake_case						)						)




			def   lowerCamelCase__  (					self    :       Optional[int]							, **_snake_case    :       Optional[Any]						)   ->			str:







								"""simple docstring"""
								kwargs.update(self.special_tokens_map						)
								return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname							, **__snake_case						)




			def   lowerCamelCase__  (					self    :       Dict							, _snake_case    :       Union[str, Any]						)   ->			str:







								"""simple docstring"""
								A_								=    "こんにちは、世界。 \nこんばんは、㔺界。😀"
								A_								=    "こんにちは、世界。 \nこんばんは、世界。😀"
								return input_text, output_text




			def   lowerCamelCase__  (					self    :       Tuple							, _snake_case    :       str						)   ->			Any:







								"""simple docstring"""
								A_			,     A_								=    self.get_input_output_texts(__snake_case						)
								A_								=    tokenizer.encode(__snake_case							, add_special_tokens=__snake_case						)
								A_								=    tokenizer.decode(__snake_case							, clean_up_tokenization_spaces=__snake_case						)
								return text, ids




			def   lowerCamelCase__  (					self    :       Optional[int]						)   ->			List[Any]:







								"""simple docstring"""
								pass  # TODO add if relevant




			def   lowerCamelCase__  (					self    :       Optional[int]						)   ->			Dict:







								"""simple docstring"""
								pass  # TODO add if relevant




			def   lowerCamelCase__  (					self    :       Dict						)   ->			str:







								"""simple docstring"""
								pass  # TODO add if relevant




			def   lowerCamelCase__  (					self    :       Optional[int]						)   ->			Any:







								"""simple docstring"""
								A_								=    self.get_tokenizer()

								# Testing tokenization
								A_								=    "こんにちは、世界。　こんばんは、㔺界。"
								A_								=    ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
								A_								=    tokenizer.tokenize(__snake_case						)
								self.assertListEqual(__snake_case							, __snake_case						)

								# Testing conversion to ids without special tokens
								A_								=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
								A_								=    tokenizer.convert_tokens_to_ids(__snake_case						)
								self.assertListEqual(__snake_case							, __snake_case						)

								# Testing conversion to ids with special tokens
								A_								=    tokens + [tokenizer.unk_token]
								A_								=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
								A_								=    tokenizer.convert_tokens_to_ids(__snake_case						)
								self.assertListEqual(__snake_case							, __snake_case						)




			def   lowerCamelCase__  (					self    :       int						)   ->			Optional[int]:







								"""simple docstring"""
								A_								=    self.get_tokenizer()

								# Testing tokenization
								A_								=    "こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"
								A_								=    "こんにちは、、、、世界。こんばんは、、、、世界。"
								A_								=    tokenizer.encode(__snake_case						)
								A_								=    tokenizer.decode(__snake_case						)
								self.assertEqual(__snake_case							, __snake_case						)




			@slow
			def   lowerCamelCase__  (					self    :       int						)   ->			int:







								"""simple docstring"""
								A_								=    self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"						)

								# Testing tokenization
								A_								=    "こんにちは、世界。"
								A_								=    "こんばんは、㔺界。😀"
								A_								=    "こんにちは、世界。こんばんは、世界。😀"
								A_								=    tokenizer.encode(prefix_text + input_text						)
								A_								=    tokenizer.encode(""							, prefix_text=prefix_text + input_text						)
								A_								=    tokenizer.encode(__snake_case							, prefix_text=__snake_case						)
								A_								=    tokenizer.decode(__snake_case						)
								A_								=    tokenizer.decode(__snake_case						)
								A_								=    tokenizer.decode(__snake_case						)
								self.assertEqual(__snake_case							, __snake_case						)
								self.assertEqual(__snake_case							, __snake_case						)
								self.assertEqual(__snake_case							, __snake_case						)




			@slow
			def   lowerCamelCase__  (					self    :       List[Any]						)   ->			List[str]:







								"""simple docstring"""
								A_								=    self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"						)

								# Testing tokenization
								A_								=    "こんにちは、世界。"
								A_								=    "こんばんは、㔺界。😀"

								A_								=    len(tokenizer.encode(__snake_case						)						) - 2
								A_								=    len(tokenizer.encode(__snake_case						)						) - 2

								A_								=    [1] + [0] * (len_prefix + len_text + 1)
								A_								=    [1] * (len_prefix + len_text + 1) + [0]
								A_								=    [1] + [1] * (len_prefix) + [0] * (len_text + 1)

								A_								=    tokenizer(prefix_text + input_text						).token_type_ids
								A_								=    tokenizer(""							, prefix_text=prefix_text + input_text						).token_type_ids
								A_								=    tokenizer(__snake_case							, prefix_text=__snake_case						).token_type_ids
								self.assertListEqual(__snake_case							, __snake_case						)
								self.assertListEqual(__snake_case							, __snake_case						)
								self.assertListEqual(__snake_case							, __snake_case						)




			@slow
			def   lowerCamelCase__  (					self    :       List[str]						)   ->			Dict:







								"""simple docstring"""
								A_								=    self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"						)

								A_								=    tokenizer.encode("あンいワ"						)
								A_								=    tokenizer.encode(""							, prefix_text="あンいワ"						)
								A_								=    tokenizer.encode("いワ"							, prefix_text="あン"						)

								self.assertEqual(tokenizer.decode(__snake_case						)							, tokenizer.decode(__snake_case						)						)
								self.assertEqual(tokenizer.decode(__snake_case						)							, tokenizer.decode(__snake_case						)						)
								self.assertNotEqual(__snake_case							, __snake_case						)
								self.assertNotEqual(__snake_case							, __snake_case						)
								self.assertEqual(x_token_a[1]							, x_token_a[-1]						)  # SEG token
								self.assertEqual(x_token_a[1]							, x_token_a[3]						)  # SEG token




			@slow
			def   lowerCamelCase__  (					self    :       Optional[Any]						)   ->			Any:







								"""simple docstring"""
								A_								=    self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"						)

								A_								=    [["武田信玄", "は、"], ["織田信長", "の配下の、"]]
								A_								=    tokenizer(__snake_case							, padding=__snake_case						)
								A_								=    tokenizer.batch_encode_plus(__snake_case							, padding=__snake_case						)

								# fmt: off
								A_								=    [[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
								A_								=    [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
								A_								=    [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
								# fmt: on
								self.assertListEqual(x_token.input_ids							, __snake_case						)
								self.assertListEqual(x_token.token_type_ids							, __snake_case						)
								self.assertListEqual(x_token.attention_mask							, __snake_case						)
								self.assertListEqual(x_token_a.input_ids							, __snake_case						)
								self.assertListEqual(x_token_a.token_type_ids							, __snake_case						)
								self.assertListEqual(x_token_a.attention_mask							, __snake_case						)




			def   lowerCamelCase__  (					self    :       Tuple						)   ->			Dict:







								"""simple docstring"""
								# Intentionally convert some words to accommodate character fluctuations unique to Japanese
								pass






			def   lowerCamelCase__  (					self    :       List[str]						)   ->			Optional[Any]:







								"""simple docstring"""
								# tokenizer has no padding token
								pass











'''simple docstring'''






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_xlnet import XLNetTokenizer
else:
      A__							:  Any    =None


A__							:  Optional[int]    =logging.get_logger(__name__)

A__							:  Union[str, Any]    ={'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

A__							:  List[str]    ={
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json''',
    },
}

A__							:  List[str]    ={
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

A__							:  List[str]    ='''▁'''

# Segments (not really needed)
A__							:  str    =0
A__							:  str    =1
A__							:  List[Any]    =2
A__							:  str    =3
A__							:  Optional[Any]    =4




class 					UpperCAmelCase	(     snake_case_  ):
      _lowercase:     Optional[int]			=  VOCAB_FILES_NAMES
      _lowercase:     Optional[int]			=  PRETRAINED_VOCAB_FILES_MAP
      _lowercase:     Dict			=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _lowercase:     Optional[Any]			=  '''left'''
      _lowercase:     Dict			=  XLNetTokenizer




      def __init__(     self     :					List[str]     ,     __snake_case     :					Optional[Any]=None     ,     __snake_case     :					str=None     ,     __snake_case     :					Union[str, Any]=False     ,     __snake_case     :					str=True     ,     __snake_case     :					Union[str, Any]=False     ,     __snake_case     :					List[Any]="<s>"     ,     __snake_case     :					List[Any]="</s>"     ,     __snake_case     :					str="<unk>"     ,     __snake_case     :					int="<sep>"     ,     __snake_case     :					int="<pad>"     ,     __snake_case     :					Dict="<cls>"     ,     __snake_case     :					int="<mask>"     ,     __snake_case     :					Optional[int]=["<eop>", "<eod>"]     ,     **__snake_case     :					List[str]     ,     )							-> List[str]:
            # Mask token behave like a normal word, i.e. include the space before it
            _lowerCAmelCase					=				AddedToken(__snake_case     ,     lstrip=__snake_case     ,     rstrip=__snake_case    ) if isinstance(__snake_case     ,     __snake_case    ) else mask_token

            super().__init__(
                vocab_file=__snake_case     ,     tokenizer_file=__snake_case     ,     do_lower_case=__snake_case     ,     remove_space=__snake_case     ,     keep_accents=__snake_case     ,     bos_token=__snake_case     ,     eos_token=__snake_case     ,     unk_token=__snake_case     ,     sep_token=__snake_case     ,     pad_token=__snake_case     ,     cls_token=__snake_case     ,     mask_token=__snake_case     ,     additional_special_tokens=__snake_case     ,     **__snake_case     ,     )

            _lowerCAmelCase					=				3
            _lowerCAmelCase					=				do_lower_case
            _lowerCAmelCase					=				remove_space
            _lowerCAmelCase					=				keep_accents
            _lowerCAmelCase					=				vocab_file
            _lowerCAmelCase					=				False if not self.vocab_file else True




      def 							lowercase__       (     self     :					Optional[int]     ,     __snake_case     :					List[int]     ,     __snake_case     :					Optional[List[int]] = None    )							-> List[int]:
            _lowerCAmelCase					=				[self.sep_token_id]
            _lowerCAmelCase					=				[self.cls_token_id]
            if token_ids_a is None:
                  return token_ids_a + sep + cls
            return token_ids_a + sep + token_ids_a + sep + cls




      def 							lowercase__       (     self     :					str     ,     __snake_case     :					List[int]     ,     __snake_case     :					Optional[List[int]] = None    )							-> List[int]:
            _lowerCAmelCase					=				[self.sep_token_id]
            _lowerCAmelCase					=				[2]

            if token_ids_a is None:
                  return len(token_ids_a + sep    ) * [0] + cls_segment_id
            return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1] + cls_segment_id




      def 							lowercase__       (     self     :					Optional[Any]     ,     __snake_case     :					str     ,     __snake_case     :					Optional[str] = None    )							-> Tuple[str]:
            if not self.can_save_slow_tokenizer:
                  raise ValueError(
                      """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                      """tokenizer."""    )

            if not os.path.isdir(__snake_case    ):
                  logger.error(f"Vocabulary path ({save_directory}) should be a directory"    )
                  return
            _lowerCAmelCase					=				os.path.join(
                __snake_case     ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

            if os.path.abspath(self.vocab_file    ) != os.path.abspath(__snake_case    ):
                  copyfile(self.vocab_file     ,     __snake_case    )

            return (out_vocab_file,)






