





import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class   _SCREAMING_SNAKE_CASE		(      _lowerCamelCase       ,				unittest.TestCase       ):







 '''simple docstring'''
 lowercase_						=     GPTSanJapaneseTokenizer
 lowercase_						=     False
 lowercase_						=     {'do_clean_text': False, 'add_prefix_space': False}

 def 				SCREAMING_SNAKE_CASE_   (self							:  Tuple)			->Any:



   '''simple docstring'''



   super().setUp()

   # fmt: off
   lowerCamelCase__: Optional[Any]         =["こん", "こんに", "にちは", "ばんは", "世界,㔺界", "、", "。", "<BR>", "<SP>", "<TAB>", "<URL>", "<EMAIL>", "<TEL>", "<DATE>", "<PRICE>", "<BLOCK>", "<KIGOU>", "<U2000U2BFF>", "<|emoji1|>", "<unk>", "<|bagoftoken|>", "<|endoftext|>"]
   # fmt: on
   lowerCamelCase__: Tuple         ={"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
   lowerCamelCase__: List[str]         ={"unk_token": "<unk>"}

   lowerCamelCase__: Optional[int]         =os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES["vocab_file"])
   lowerCamelCase__: int         =os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES["emoji_file"])
   with open(self.vocab_file			,					"w"			,					encoding="utf-8") as vocab_writer:
     vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
   with open(self.emoji_file			,					"w") as emoji_writer:
     emoji_writer.write(json.dumps(__UpperCamelCase))

 def 				SCREAMING_SNAKE_CASE_   (self							:  Any			,					**UpperCAmelCase_							:  Any)			->Tuple:



   '''simple docstring'''



   kwargs.update(self.special_tokens_map)
   return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname			,					**__UpperCamelCase)

 def 				SCREAMING_SNAKE_CASE_   (self							:  str			,					UpperCAmelCase_							:  Dict)			->List[str]:



   '''simple docstring'''



   lowerCamelCase__: List[Any]         ="こんにちは、世界。 \nこんばんは、㔺界。😀"
   lowerCamelCase__: Any         ="こんにちは、世界。 \nこんばんは、世界。😀"
   return input_text, output_text

 def 				SCREAMING_SNAKE_CASE_   (self							:  str			,					UpperCAmelCase_							:  Optional[Any])			->List[str]:



   '''simple docstring'''



   lowerCamelCase__    ,   lowerCamelCase__: Optional[Any]         =self.get_input_output_texts(__UpperCamelCase)
   lowerCamelCase__: str         =tokenizer.encode(__UpperCamelCase			,					add_special_tokens=__UpperCamelCase)
   lowerCamelCase__: str         =tokenizer.decode(__UpperCamelCase			,					clean_up_tokenization_spaces=__UpperCamelCase)
   return text, ids

 def 				SCREAMING_SNAKE_CASE_   (self							:  Any)			->int:



   '''simple docstring'''



   pass  # TODO add if relevant

 def 				SCREAMING_SNAKE_CASE_   (self							:  List[str])			->Tuple:



   '''simple docstring'''



   pass  # TODO add if relevant

 def 				SCREAMING_SNAKE_CASE_   (self							:  List[str])			->List[Any]:



   '''simple docstring'''



   pass  # TODO add if relevant

 def 				SCREAMING_SNAKE_CASE_   (self							:  List[str])			->int:



   '''simple docstring'''



   lowerCamelCase__: int         =self.get_tokenizer()

   # Testing tokenization
   lowerCamelCase__: List[Any]         ="こんにちは、世界。　こんばんは、㔺界。"
   lowerCamelCase__: List[str]         =["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
   lowerCamelCase__: int         =tokenizer.tokenize(__UpperCamelCase)
   self.assertListEqual(__UpperCamelCase			,					__UpperCamelCase)

   # Testing conversion to ids without special tokens
   lowerCamelCase__: Union[str, Any]         =[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
   lowerCamelCase__: Tuple         =tokenizer.convert_tokens_to_ids(__UpperCamelCase)
   self.assertListEqual(__UpperCamelCase			,					__UpperCamelCase)

   # Testing conversion to ids with special tokens
   lowerCamelCase__: Dict         =tokens + [tokenizer.unk_token]
   lowerCamelCase__: Union[str, Any]         =[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
   lowerCamelCase__: Union[str, Any]         =tokenizer.convert_tokens_to_ids(__UpperCamelCase)
   self.assertListEqual(__UpperCamelCase			,					__UpperCamelCase)

 def 				SCREAMING_SNAKE_CASE_   (self							:  Union[str, Any])			->List[str]:



   '''simple docstring'''



   lowerCamelCase__: Union[str, Any]         =self.get_tokenizer()

   # Testing tokenization
   lowerCamelCase__: int         ="こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"
   lowerCamelCase__: List[Any]         ="こんにちは、、、、世界。こんばんは、、、、世界。"
   lowerCamelCase__: Optional[Any]         =tokenizer.encode(__UpperCamelCase)
   lowerCamelCase__: List[Any]         =tokenizer.decode(__UpperCamelCase)
   self.assertEqual(__UpperCamelCase			,					__UpperCamelCase)

 @slow
 def 				SCREAMING_SNAKE_CASE_   (self							:  Union[str, Any])			->str:



   '''simple docstring'''



   lowerCamelCase__: Optional[Any]         =self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese")

   # Testing tokenization
   lowerCamelCase__: Any         ="こんにちは、世界。"
   lowerCamelCase__: Tuple         ="こんばんは、㔺界。😀"
   lowerCamelCase__: List[Any]         ="こんにちは、世界。こんばんは、世界。😀"
   lowerCamelCase__: Optional[int]         =tokenizer.encode(prefix_text + input_text)
   lowerCamelCase__: Optional[Any]         =tokenizer.encode(""			,					prefix_text=prefix_text + input_text)
   lowerCamelCase__: str         =tokenizer.encode(__UpperCamelCase			,					prefix_text=__UpperCamelCase)
   lowerCamelCase__: Dict         =tokenizer.decode(__UpperCamelCase)
   lowerCamelCase__: int         =tokenizer.decode(__UpperCamelCase)
   lowerCamelCase__: int         =tokenizer.decode(__UpperCamelCase)
   self.assertEqual(__UpperCamelCase			,					__UpperCamelCase)
   self.assertEqual(__UpperCamelCase			,					__UpperCamelCase)
   self.assertEqual(__UpperCamelCase			,					__UpperCamelCase)

 @slow
 def 				SCREAMING_SNAKE_CASE_   (self							:  Union[str, Any])			->List[Any]:



   '''simple docstring'''



   lowerCamelCase__: Any         =self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese")

   # Testing tokenization
   lowerCamelCase__: str         ="こんにちは、世界。"
   lowerCamelCase__: Union[str, Any]         ="こんばんは、㔺界。😀"

   lowerCamelCase__: Optional[Any]         =len(tokenizer.encode(__UpperCamelCase)) - 2
   lowerCamelCase__: Optional[int]         =len(tokenizer.encode(__UpperCamelCase)) - 2

   lowerCamelCase__: Tuple         =[1] + [0] * (len_prefix + len_text + 1)
   lowerCamelCase__: int         =[1] * (len_prefix + len_text + 1) + [0]
   lowerCamelCase__: Tuple         =[1] + [1] * (len_prefix) + [0] * (len_text + 1)

   lowerCamelCase__: Optional[int]         =tokenizer(prefix_text + input_text).token_type_ids
   lowerCamelCase__: Union[str, Any]         =tokenizer(""			,					prefix_text=prefix_text + input_text).token_type_ids
   lowerCamelCase__: int         =tokenizer(__UpperCamelCase			,					prefix_text=__UpperCamelCase).token_type_ids
   self.assertListEqual(__UpperCamelCase			,					__UpperCamelCase)
   self.assertListEqual(__UpperCamelCase			,					__UpperCamelCase)
   self.assertListEqual(__UpperCamelCase			,					__UpperCamelCase)

 @slow
 def 				SCREAMING_SNAKE_CASE_   (self							:  Any)			->str:



   '''simple docstring'''



   lowerCamelCase__: Dict         =self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese")

   lowerCamelCase__: List[Any]         =tokenizer.encode("あンいワ")
   lowerCamelCase__: Dict         =tokenizer.encode(""			,					prefix_text="あンいワ")
   lowerCamelCase__: Any         =tokenizer.encode("いワ"			,					prefix_text="あン")

   self.assertEqual(tokenizer.decode(__UpperCamelCase)			,					tokenizer.decode(__UpperCamelCase))
   self.assertEqual(tokenizer.decode(__UpperCamelCase)			,					tokenizer.decode(__UpperCamelCase))
   self.assertNotEqual(__UpperCamelCase			,					__UpperCamelCase)
   self.assertNotEqual(__UpperCamelCase			,					__UpperCamelCase)
   self.assertEqual(x_token_a[1]			,					x_token_a[-1])  # SEG token
   self.assertEqual(x_token_a[1]			,					x_token_a[3])  # SEG token

 @slow
 def 				SCREAMING_SNAKE_CASE_   (self							:  Any)			->List[Any]:



   '''simple docstring'''



   lowerCamelCase__: Dict         =self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese")

   lowerCamelCase__: Any         =[["武田信玄", "は、"], ["織田信長", "の配下の、"]]
   lowerCamelCase__: List[Any]         =tokenizer(__UpperCamelCase			,					padding=__UpperCamelCase)
   lowerCamelCase__: Tuple         =tokenizer.batch_encode_plus(__UpperCamelCase			,					padding=__UpperCamelCase)

   # fmt: off
   lowerCamelCase__: Tuple         =[[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
   lowerCamelCase__: Union[str, Any]         =[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
   lowerCamelCase__: Any         =[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
   # fmt: on
   self.assertListEqual(x_token.input_ids			,					__UpperCamelCase)
   self.assertListEqual(x_token.token_type_ids			,					__UpperCamelCase)
   self.assertListEqual(x_token.attention_mask			,					__UpperCamelCase)
   self.assertListEqual(x_token_a.input_ids			,					__UpperCamelCase)
   self.assertListEqual(x_token_a.token_type_ids			,					__UpperCamelCase)
   self.assertListEqual(x_token_a.attention_mask			,					__UpperCamelCase)

 def 				SCREAMING_SNAKE_CASE_   (self							:  Tuple)			->str:



   '''simple docstring'''



   pass







 def 				SCREAMING_SNAKE_CASE_   (self							:  Dict)			->int:



   '''simple docstring'''



   pass






from typing import List

import numpy as np






def  lowerCamelCase_	(      lowerCAmelCase__	:	dict				)       ->							int:

							'''simple docstring'''






							A      						=    {key: len(lowerCAmelCase__				) for key, value in gen_kwargs.items() if isinstance(lowerCAmelCase__			,	lowerCAmelCase__				)}
							if len(set(lists_lengths.values()				)				) > 1:
														raise RuntimeError(
														    (
														        'Sharding is ambiguous for this dataset: '
														        + 'we found several data sources lists of different lengths, and we don\'t know over which list we should parallelize:\n'
														        + '\n'.join(F'''\t- key {key} has length {length}''' for key, length in lists_lengths.items()				)
														        + '\nTo fix this, check the \'gen_kwargs\' and make sure to use lists only for data sources, '
														        + 'and use tuples otherwise. In the end there should only be one single list, or several lists with the same length.'
														    )				)
							A      						=    max(lists_lengths.values()			,	default=0				)
							return max(1			,	lowerCAmelCase__				)






def  lowerCamelCase_	(      lowerCAmelCase__	:	int			,	lowerCAmelCase__	:	int				)       ->							List[range]:

							'''simple docstring'''






							A      						=    []
							for group_idx in range(lowerCAmelCase__				):
														A      						=    num_shards // max_num_jobs + (group_idx < (num_shards % max_num_jobs))
														if num_shards_to_add == 0:
																					break
														A      						=    shards_indices_per_group[-1].stop if shards_indices_per_group else 0
														A      						=    range(lowerCAmelCase__			,	start + num_shards_to_add				)
														shards_indices_per_group.append(lowerCAmelCase__				)
							return shards_indices_per_group






def  lowerCamelCase_	(      lowerCAmelCase__	:	dict			,	lowerCAmelCase__	:	int				)       ->							List[dict]:

							'''simple docstring'''






							A      						=    _number_of_shards_in_gen_kwargs(lowerCAmelCase__				)
							if num_shards == 1:
														return [dict(lowerCAmelCase__				)]
							else:
														A      						=    _distribute_shards(num_shards=lowerCAmelCase__			,	max_num_jobs=lowerCAmelCase__				)
														return [
														    {
														        key: [value[shard_idx] for shard_idx in shard_indices_per_group[group_idx]]
														        if isinstance(lowerCAmelCase__			,	lowerCAmelCase__				)
														        else value
														        for key, value in gen_kwargs.items()
														    }
														    for group_idx in range(len(lowerCAmelCase__				)				)
														]






def  lowerCamelCase_	(      lowerCAmelCase__	:	List[dict]				)       ->							dict:

							'''simple docstring'''






							return {
							    key: [value for gen_kwargs in gen_kwargs_list for value in gen_kwargs[key]]
							    if isinstance(gen_kwargs_list[0][key]			,	lowerCAmelCase__				)
							    else gen_kwargs_list[0][key]
							    for key in gen_kwargs_list[0]
							}






def  lowerCamelCase_	(      lowerCAmelCase__	:	np.random.Generator			,	lowerCAmelCase__	:	dict				)       ->							dict:

							'''simple docstring'''






							A      						=    {len(lowerCAmelCase__				) for value in gen_kwargs.values() if isinstance(lowerCAmelCase__			,	lowerCAmelCase__				)}
							A      						=    {}
							for size in list_sizes:
														A      						=    list(range(lowerCAmelCase__				)				)
														rng.shuffle(indices_per_size[size]				)
							# Now let's copy the gen_kwargs and shuffle the lists based on their sizes
							A      						=    dict(lowerCAmelCase__				)
							for key, value in shuffled_kwargs.items():
														if isinstance(lowerCAmelCase__			,	lowerCAmelCase__				):
																					A      						=    [value[i] for i in indices_per_size[len(lowerCAmelCase__				)]]
							return shuffled_kwargs