



'''simple docstring'''





import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class  snake_case__						(    SCREAMING_SNAKE_CASE_  ,     unittest.TestCase				):
 A__				= GPTSanJapaneseTokenizer
 A__				= False
 A__				= {'''do_clean_text''': False, '''add_prefix_space''': False}
 def A_   (   self						:							Tuple  )      ->   str:







       '''simple docstring'''
       super().setUp()

       # fmt: off
       __snake_case :   Optional[Any]													=    ['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
       # fmt: on
       __snake_case :   int													=    {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
       __snake_case :   List[Any]													=    {'unk_token': '<unk>'}

       __snake_case :   Optional[Any]													=    os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES['vocab_file']  )
       __snake_case :   Optional[int]													=    os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES['emoji_file']  )
       with open(self.vocab_file      ,     'w'      ,     encoding='utf-8'  ) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
       with open(self.emoji_file      ,     'w'  ) as emoji_writer:
             emoji_writer.write(json.dumps(__a  )  )
 def A_   (   self						:							List[str]      ,     **__a						:							List[Any]  )      ->   Dict:







       '''simple docstring'''
       kwargs.update(self.special_tokens_map  )
       return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname      ,     **__a  )
 def A_   (   self						:							Optional[Any]      ,     __a						:							Tuple  )      ->   Tuple:







       '''simple docstring'''
       __snake_case :   Any													=    'こんにちは、世界。 \nこんばんは、㔺界。😀'
       __snake_case :   List[Any]													=    'こんにちは、世界。 \nこんばんは、世界。😀'
       return input_text, output_text
 def A_   (   self						:							Optional[Any]      ,     __a						:							Any  )      ->   Tuple:







       '''simple docstring'''
       __snake_case							,     __snake_case :   List[Any]													=    self.get_input_output_texts(__a  )
       __snake_case :   List[str]													=    tokenizer.encode(__a      ,     add_special_tokens=__a  )
       __snake_case :   str													=    tokenizer.decode(__a      ,     clean_up_tokenization_spaces=__a  )
       return text, ids
 def A_   (   self						:							Union[str, Any]  )      ->   List[str]:







       '''simple docstring'''
       pass  # TODO add if relevant
 def A_   (   self						:							Tuple  )      ->   str:







       '''simple docstring'''
       pass  # TODO add if relevant
 def A_   (   self						:							List[Any]  )      ->   Optional[Any]:







       '''simple docstring'''
       pass  # TODO add if relevant
 def A_   (   self						:							List[str]  )      ->   Dict:







       '''simple docstring'''
       __snake_case :   Any													=    self.get_tokenizer()

       # Testing tokenization
       __snake_case :   Optional[Any]													=    'こんにちは、世界。　こんばんは、㔺界。'
       __snake_case :   List[str]													=    ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
       __snake_case :   Tuple													=    tokenizer.tokenize(__a  )
       self.assertListEqual(__a      ,     __a  )

       # Testing conversion to ids without special tokens
       __snake_case :   str													=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
       __snake_case :   str													=    tokenizer.convert_tokens_to_ids(__a  )
       self.assertListEqual(__a      ,     __a  )

       # Testing conversion to ids with special tokens
       __snake_case :   List[str]													=    tokens + [tokenizer.unk_token]
       __snake_case :   Union[str, Any]													=    [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
       __snake_case :   List[Any]													=    tokenizer.convert_tokens_to_ids(__a  )
       self.assertListEqual(__a      ,     __a  )
 def A_   (   self						:							Any  )      ->   Any:







       '''simple docstring'''
       __snake_case :   Optional[Any]													=    self.get_tokenizer()

       # Testing tokenization
       __snake_case :   Optional[int]													=    'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
       __snake_case :   Dict													=    'こんにちは、、、、世界。こんばんは、、、、世界。'
       __snake_case :   Optional[int]													=    tokenizer.encode(__a  )
       __snake_case :   Union[str, Any]													=    tokenizer.decode(__a  )
       self.assertEqual(__a      ,     __a  )
 @slow
 def A_   (   self						:							List[Any]  )      ->   Optional[int]:







       '''simple docstring'''
       __snake_case :   int													=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

       # Testing tokenization
       __snake_case :   Optional[int]													=    'こんにちは、世界。'
       __snake_case :   Tuple													=    'こんばんは、㔺界。😀'
       __snake_case :   Any													=    'こんにちは、世界。こんばんは、世界。😀'
       __snake_case :   Tuple													=    tokenizer.encode(prefix_text + input_text  )
       __snake_case :   List[str]													=    tokenizer.encode(''      ,     prefix_text=prefix_text + input_text  )
       __snake_case :   Tuple													=    tokenizer.encode(__a      ,     prefix_text=__a  )
       __snake_case :   Any													=    tokenizer.decode(__a  )
       __snake_case :   Dict													=    tokenizer.decode(__a  )
       __snake_case :   Dict													=    tokenizer.decode(__a  )
       self.assertEqual(__a      ,     __a  )
       self.assertEqual(__a      ,     __a  )
       self.assertEqual(__a      ,     __a  )
 @slow
 def A_   (   self						:							int  )      ->   int:







       '''simple docstring'''
       __snake_case :   Tuple													=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

       # Testing tokenization
       __snake_case :   Optional[Any]													=    'こんにちは、世界。'
       __snake_case :   Tuple													=    'こんばんは、㔺界。😀'

       __snake_case :   Union[str, Any]													=    len(tokenizer.encode(__a  )  ) - 2
       __snake_case :   Any													=    len(tokenizer.encode(__a  )  ) - 2

       __snake_case :   Optional[Any]													=    [1] + [0] * (len_prefix + len_text + 1)
       __snake_case :   Union[str, Any]													=    [1] * (len_prefix + len_text + 1) + [0]
       __snake_case :   Optional[Any]													=    [1] + [1] * (len_prefix) + [0] * (len_text + 1)

       __snake_case :   Optional[int]													=    tokenizer(prefix_text + input_text  ).token_type_ids
       __snake_case :   Dict													=    tokenizer(''      ,     prefix_text=prefix_text + input_text  ).token_type_ids
       __snake_case :   Union[str, Any]													=    tokenizer(__a      ,     prefix_text=__a  ).token_type_ids
       self.assertListEqual(__a      ,     __a  )
       self.assertListEqual(__a      ,     __a  )
       self.assertListEqual(__a      ,     __a  )
 @slow
 def A_   (   self						:							Any  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case :   List[str]													=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

       __snake_case :   Union[str, Any]													=    tokenizer.encode('あンいワ'  )
       __snake_case :   Optional[Any]													=    tokenizer.encode(''      ,     prefix_text='あンいワ'  )
       __snake_case :   int													=    tokenizer.encode('いワ'      ,     prefix_text='あン'  )

       self.assertEqual(tokenizer.decode(__a  )      ,     tokenizer.decode(__a  )  )
       self.assertEqual(tokenizer.decode(__a  )      ,     tokenizer.decode(__a  )  )
       self.assertNotEqual(__a      ,     __a  )
       self.assertNotEqual(__a      ,     __a  )
       self.assertEqual(x_token_a[1]      ,     x_token_a[-1]  )  # SEG token
       self.assertEqual(x_token_a[1]      ,     x_token_a[3]  )  # SEG token
 @slow
 def A_   (   self						:							Dict  )      ->   Optional[Any]:







       '''simple docstring'''
       __snake_case :   Dict													=    self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

       __snake_case :   List[str]													=    [['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
       __snake_case :   Optional[Any]													=    tokenizer(__a      ,     padding=__a  )
       __snake_case :   str													=    tokenizer.batch_encode_plus(__a      ,     padding=__a  )

       # fmt: off
       __snake_case :   Optional[int]													=    [[35993, 8640, 25948, 35998, 30647, 35675, 35999, 35999], [35993, 10382, 9868, 35998, 30646, 9459, 30646, 35675]]
       __snake_case :   List[Any]													=    [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
       __snake_case :   Dict													=    [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
       # fmt: on
       self.assertListEqual(x_token.input_ids      ,     __a  )
       self.assertListEqual(x_token.token_type_ids      ,     __a  )
       self.assertListEqual(x_token.attention_mask      ,     __a  )
       self.assertListEqual(x_token_a.input_ids      ,     __a  )
       self.assertListEqual(x_token_a.token_type_ids      ,     __a  )
       self.assertListEqual(x_token_a.attention_mask      ,     __a  )
 def A_   (   self						:							List[Any]  )      ->   int:







       '''simple docstring'''
       # Intentionally convert some words to accommodate character fluctuations unique to Japanese
       pass





 def A_   (   self						:							Optional[Any]  )      ->   List[Any]:







       '''simple docstring'''
       # tokenizer has no padding token
       pass








'''simple docstring'''





import json
import os
import unittest

from transformers.models.blenderbot_small.tokenization_blenderbot_small import (
    VOCAB_FILES_NAMES,
    BlenderbotSmallTokenizer,
)

from ...test_tokenization_common import TokenizerTesterMixin



class  snake_case__						(    SCREAMING_SNAKE_CASE_  ,     unittest.TestCase				):
 A__				= BlenderbotSmallTokenizer
 A__				= False
 def A_   (   self						:							Any  )      ->   str:







       '''simple docstring'''
       super().setUp()

       __snake_case :   Any													=    ['__start__', 'adapt', 'act', 'ap@@', 'te', '__end__', '__unk__']
       __snake_case :   str													=    dict(zip(__a      ,     range(len(__a  )  )  )  )

       __snake_case :   List[Any]													=    ['#version: 0.2', 'a p', 't e</w>', 'ap t</w>', 'a d', 'ad apt</w>', 'a c', 'ac t</w>', '']
       __snake_case :   Tuple													=    {'unk_token': '__unk__', 'bos_token': '__start__', 'eos_token': '__end__'}

       __snake_case :   str													=    os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES['vocab_file']  )
       __snake_case :   Union[str, Any]													=    os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES['merges_file']  )
       with open(self.vocab_file      ,     'w'      ,     encoding='utf-8'  ) as fp:
             fp.write(json.dumps(__a  ) + '\n'  )
       with open(self.merges_file      ,     'w'      ,     encoding='utf-8'  ) as fp:
             fp.write('\n'.join(__a  )  )
 def A_   (   self						:							Tuple      ,     **__a						:							Optional[int]  )      ->   List[str]:







       '''simple docstring'''
       kwargs.update(self.special_tokens_map  )
       return BlenderbotSmallTokenizer.from_pretrained(self.tmpdirname      ,     **__a  )
 def A_   (   self						:							int      ,     __a						:							int  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   List[str]													=    'adapt act apte'
       __snake_case :   List[str]													=    'adapt act apte'
       return input_text, output_text
 def A_   (   self						:							Dict  )      ->   str:







       '''simple docstring'''
       __snake_case :   Dict													=    BlenderbotSmallTokenizer(self.vocab_file      ,     self.merges_file      ,     **self.special_tokens_map  )
       __snake_case :   List[str]													=    'adapt act apte'
       __snake_case :   List[Any]													=    ['adapt', 'act', 'ap@@', 'te']
       __snake_case :   Optional[int]													=    tokenizer.tokenize(__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   List[str]													=    [tokenizer.bos_token] + tokens + [tokenizer.eos_token]

       __snake_case :   List[str]													=    [0, 1, 2, 3, 4, 5]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__a  )      ,     __a  )
 def A_   (   self						:							int  )      ->   str:







       '''simple docstring'''
       __snake_case :   int													=    BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot-90M'  )
       assert tok('sam'  ).input_ids == [1384]
       __snake_case :   List[str]													=    'I am a small frog.'
       __snake_case :   List[Any]													=    tok([src_text]      ,     padding=__a      ,     truncation=__a  )['input_ids']
       __snake_case :   int													=    tok.batch_decode(__a      ,     skip_special_tokens=__a      ,     clean_up_tokenization_spaces=__a  )[0]
       assert src_text != decoded  # I wish it did!
       assert decoded == "i am a small frog ."





 def A_   (   self						:							List[str]  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case :   Tuple													=    BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot-90M'  )
       __snake_case :   Optional[Any]													=    'I am a small frog .'
       __snake_case :   List[str]													=    '.'
       __snake_case :   List[str]													=    tok(__a  )['input_ids']
       __snake_case :   str													=    tok(__a  )['input_ids']

       assert encoded[-1] == encoded_dot[0]



