




import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class    lowercase_		(_lowerCAmelCase							,	unittest.TestCase ):
						lowerCAmelCase__			      =GPTSanJapaneseTokenizer
						lowerCAmelCase__			      =False
						lowerCAmelCase__			      ={"do_clean_text": False, "add_prefix_space": False}



						def 	__a					(			self	:   int  ):

								"""simple docstring"""

								super().setUp()

								# fmt: off
								SCREAMING_SNAKE_CASE_					      =							['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
								# fmt: on
								SCREAMING_SNAKE_CASE_					      =							{'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
								SCREAMING_SNAKE_CASE_					      =							{'unk_token': '<unk>'}

								SCREAMING_SNAKE_CASE_					      =							os.path.join(self.tmpdirname					,							VOCAB_FILES_NAMES['vocab_file']  )
								SCREAMING_SNAKE_CASE_					      =							os.path.join(self.tmpdirname					,							VOCAB_FILES_NAMES['emoji_file']  )
								with open(self.vocab_file					,							'w'					,							encoding='utf-8'  ) as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
								with open(self.emoji_file					,							'w'  ) as emoji_writer:
										emoji_writer.write(json.dumps(snake_case__  )  )



						def 	__a					(			self	:   int					,							**snake_case__	:   Dict  ):

								"""simple docstring"""

								kwargs.update(self.special_tokens_map  )
								return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname					,							**snake_case__  )



						def 	__a					(			self	:   Any					,							snake_case__	:   Any  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							'こんにちは、世界。 \nこんばんは、㔺界。😀'
								SCREAMING_SNAKE_CASE_					      =							'こんにちは、世界。 \nこんばんは、世界。😀'
								return input_text, output_text



						def 	__a					(			self	:   Dict					,							snake_case__	:   Any  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_      ,				SCREAMING_SNAKE_CASE_					      =							self.get_input_output_texts(snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.encode(snake_case__					,							add_special_tokens=snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.decode(snake_case__					,							clean_up_tokenization_spaces=snake_case__  )
								return text, ids



						def 	__a					(			self	:   int  ):

								"""simple docstring"""

								pass  # TODO add if relevant



						def 	__a					(			self	:   int  ):

								"""simple docstring"""

								pass  # TODO add if relevant



						def 	__a					(			self	:   int  ):

								"""simple docstring"""

								pass  # TODO add if relevant



						def 	__a					(			self	:   List[str]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.get_tokenizer()

								# Testing tokenization
								SCREAMING_SNAKE_CASE_					      =							'こんにちは、世界。　こんばんは、㔺界。'
								SCREAMING_SNAKE_CASE_					      =							['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
								SCREAMING_SNAKE_CASE_					      =							tokenizer.tokenize(snake_case__  )
								self.assertListEqual(snake_case__					,							snake_case__  )

								# Testing conversion to ids without special tokens
								SCREAMING_SNAKE_CASE_					      =							[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
								SCREAMING_SNAKE_CASE_					      =							tokenizer.convert_tokens_to_ids(snake_case__  )
								self.assertListEqual(snake_case__					,							snake_case__  )

								# Testing conversion to ids with special tokens
								SCREAMING_SNAKE_CASE_					      =							tokens + [tokenizer.unk_token]
								SCREAMING_SNAKE_CASE_					      =							[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
								SCREAMING_SNAKE_CASE_					      =							tokenizer.convert_tokens_to_ids(snake_case__  )
								self.assertListEqual(snake_case__					,							snake_case__  )



						def 	__a					(			self	:   Union[str, Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.get_tokenizer()

								# Testing tokenization
								SCREAMING_SNAKE_CASE_					      =							'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
								SCREAMING_SNAKE_CASE_					      =							'こんにちは、、、、世界。こんばんは、、、、世界。'
								SCREAMING_SNAKE_CASE_					      =							tokenizer.encode(snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.decode(snake_case__  )
								self.assertEqual(snake_case__					,							snake_case__  )



						@slow
						def 	__a					(			self	:   List[Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

								# Testing tokenization
								SCREAMING_SNAKE_CASE_					      =							'こんにちは、世界。'
								SCREAMING_SNAKE_CASE_					      =							'こんばんは、㔺界。😀'
								SCREAMING_SNAKE_CASE_					      =							'こんにちは、世界。こんばんは、世界。😀'
								SCREAMING_SNAKE_CASE_					      =							tokenizer.encode(prefix_text + input_text  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.encode(''					,							prefix_text=prefix_text + input_text  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.encode(snake_case__					,							prefix_text=snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.decode(snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.decode(snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.decode(snake_case__  )
								self.assertEqual(snake_case__					,							snake_case__  )
								self.assertEqual(snake_case__					,							snake_case__  )
								self.assertEqual(snake_case__					,							snake_case__  )



						@slow
						def 	__a					(			self	:   int  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

								# Testing tokenization
								SCREAMING_SNAKE_CASE_					      =							'こんにちは、世界。'
								SCREAMING_SNAKE_CASE_					      =							'こんばんは、㔺界。😀'

								SCREAMING_SNAKE_CASE_					      =							len(tokenizer.encode(snake_case__  )  ) - 2
								SCREAMING_SNAKE_CASE_					      =							len(tokenizer.encode(snake_case__  )  ) - 2

								SCREAMING_SNAKE_CASE_					      =							[1] + [0] * (len_prefix + len_text + 1)
								SCREAMING_SNAKE_CASE_					      =							[1] * (len_prefix + len_text + 1) + [0]
								SCREAMING_SNAKE_CASE_					      =							[1] + [1] * (len_prefix) + [0] * (len_text + 1)

								SCREAMING_SNAKE_CASE_					      =							tokenizer(prefix_text + input_text  ).token_type_ids
								SCREAMING_SNAKE_CASE_					      =							tokenizer(''					,							prefix_text=prefix_text + input_text  ).token_type_ids
								SCREAMING_SNAKE_CASE_					      =							tokenizer(snake_case__					,							prefix_text=snake_case__  ).token_type_ids
								self.assertListEqual(snake_case__					,							snake_case__  )
								self.assertListEqual(snake_case__					,							snake_case__  )
								self.assertListEqual(snake_case__					,							snake_case__  )



						@slow
						def 	__a					(			self	:   int  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

								SCREAMING_SNAKE_CASE_					      =							tokenizer.encode('あンいワ'  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.encode(''					,							prefix_text='あンいワ'  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.encode('いワ'					,							prefix_text='あン'  )

								self.assertEqual(tokenizer.decode(snake_case__  )					,							tokenizer.decode(snake_case__  )  )
								self.assertEqual(tokenizer.decode(snake_case__  )					,							tokenizer.decode(snake_case__  )  )
								self.assertNotEqual(snake_case__					,							snake_case__  )
								self.assertNotEqual(snake_case__					,							snake_case__  )
								self.assertEqual(x_token_a[1]					,							x_token_a[-1]  )  # SEG token
								self.assertEqual(x_token_a[1]					,							x_token_a[3]  )  # SEG token



						@slow
						def 	__a					(			self	:   Union[str, Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'  )

								SCREAMING_SNAKE_CASE_					      =							[['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
								SCREAMING_SNAKE_CASE_					      =							tokenizer(snake_case__					,							padding=snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							tokenizer.batch_encode_plus(snake_case__					,							padding=snake_case__  )

								# fmt: off
								SCREAMING_SNAKE_CASE_					      =							[[3_59_93, 86_40, 2_59_48, 3_59_98, 3_06_47, 3_56_75, 3_59_99, 3_59_99], [3_59_93, 1_03_82, 98_68, 3_59_98, 3_06_46, 94_59, 3_06_46, 3_56_75]]
								SCREAMING_SNAKE_CASE_					      =							[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
								SCREAMING_SNAKE_CASE_					      =							[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
								# fmt: on
								self.assertListEqual(x_token.input_ids					,							snake_case__  )
								self.assertListEqual(x_token.token_type_ids					,							snake_case__  )
								self.assertListEqual(x_token.attention_mask					,							snake_case__  )
								self.assertListEqual(x_token_a.input_ids					,							snake_case__  )
								self.assertListEqual(x_token_a.token_type_ids					,							snake_case__  )
								self.assertListEqual(x_token_a.attention_mask					,							snake_case__  )



						def 	__a					(			self	:   Union[str, Any]  ):

								"""simple docstring"""

								pass






						def 	__a					(			self	:   List[Any]  ):

								"""simple docstring"""

								pass







'''simple docstring'''



from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
 from ...feature_extraction_utils import FeatureExtractionMixin
 from ...tokenization_utils_base import PreTrainedTokenizerBase
 from ...utils import TensorType

__UpperCamelCase   :							List[str]         =   logging.get_logger(__name__)

__UpperCamelCase   :							Optional[int]         =   {
    """openai/whisper-base""": """https://huggingface.co/openai/whisper-base/resolve/main/config.json""",
}

# fmt: off
__UpperCamelCase   :							Optional[Any]         =   [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 357, 366, 438, 532, 685,
    705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377,
    1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211,
    4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786,
    11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791,
    17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409,
    34949, 40283, 40493, 40549, 47282, 49146, 50257, 50359, 50360, 50361
]
__UpperCamelCase   :							List[str]         =   [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 359, 503, 522, 542, 873,
    893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627,
    3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647,
    7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793,
    14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675,
    22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865,
    42863, 47425, 49870, 50254, 50258, 50360, 50361, 50362
]
class 				__SCREAMING_SNAKE_CASE							(				_lowerCAmelCase  ):
      __a	       ="whisper"
      __a	       =["past_key_values"]
      __a	       ={"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}




      def __init__(	self			, lowerCamelCase=5_1865			, lowerCamelCase=80			, lowerCamelCase=6			, lowerCamelCase=4			, lowerCamelCase=6			, lowerCamelCase=4			, lowerCamelCase=1536			, lowerCamelCase=1536			, lowerCamelCase=0.0			, lowerCamelCase=0.0			, lowerCamelCase=5_0257			, lowerCamelCase=True			, lowerCamelCase=True			, lowerCamelCase="gelu"			, lowerCamelCase=256			, lowerCamelCase=0.0			, lowerCamelCase=0.0			, lowerCamelCase=0.0			, lowerCamelCase=0.02			, lowerCamelCase=False			, lowerCamelCase=1500			, lowerCamelCase=448			, lowerCamelCase=5_0256			, lowerCamelCase=5_0256			, lowerCamelCase=5_0256			, lowerCamelCase=None			, lowerCamelCase=[220, 5_0256]			, lowerCamelCase=False			, lowerCamelCase=256			, lowerCamelCase=False			, lowerCamelCase=0.05			, lowerCamelCase=10			, lowerCamelCase=2			, lowerCamelCase=0.0			, lowerCamelCase=10			, lowerCamelCase=0			, lowerCamelCase=7			, **lowerCamelCase			, )				->Any:



         '''simple docstring'''
         __a					  =     vocab_size
         __a					  =     num_mel_bins
         __a					  =     d_model
         __a					  =     encoder_layers
         __a					  =     encoder_attention_heads
         __a					  =     decoder_layers
         __a					  =     decoder_attention_heads
         __a					  =     decoder_ffn_dim
         __a					  =     encoder_ffn_dim
         __a					  =     dropout
         __a					  =     attention_dropout
         __a					  =     activation_dropout
         __a					  =     activation_function
         __a					  =     init_std
         __a					  =     encoder_layerdrop
         __a					  =     decoder_layerdrop
         __a					  =     use_cache
         __a					  =     encoder_layers
         __a					  =     scale_embedding  # scale factor will be sqrt(d_model) if True
         __a					  =     max_source_positions
         __a					  =     max_target_positions

         # Audio Classification-specific parameters. Feel free to ignore for other classes.
         __a					  =     classifier_proj_size
         __a					  =     use_weighted_layer_sum

         # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
         __a					  =     apply_spec_augment
         __a					  =     mask_time_prob
         __a					  =     mask_time_length
         __a					  =     mask_time_min_masks
         __a					  =     mask_feature_prob
         __a					  =     mask_feature_length
         __a					  =     mask_feature_min_masks

         __a					  =     median_filter_width

         super().__init__(
             pad_token_id=lowerCamelCase			, bos_token_id=lowerCamelCase			, eos_token_id=lowerCamelCase			, is_encoder_decoder=lowerCamelCase			, decoder_start_token_id=lowerCamelCase			, suppress_tokens=lowerCamelCase			, begin_suppress_tokens=lowerCamelCase			, **lowerCamelCase			, )







class 				__SCREAMING_SNAKE_CASE							(				_lowerCAmelCase  ):




      @property
      def 			__UpperCamelCase				(	self   )				->Mapping[str, Mapping[int, str]]:



         '''simple docstring'''
         __a					  =     OrderedDict(
             [
                 ('input_features', {0: 'batch', 1: 'feature_size', 2: 'encoder_sequence'}),
             ]   )
         if self.use_past:
            __a					  =     {0: 'batch'}
         else:
            __a					  =     {0: 'batch', 1: 'decoder_sequence'}

         if self.use_past:
            self.fill_with_past_key_values_(lowerCamelCase			, direction='inputs'   )

         return common_inputs




      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase = -1			, lowerCamelCase = -1			, lowerCamelCase = False			, lowerCamelCase = None			, lowerCamelCase = 2_2050			, lowerCamelCase = 5.0			, lowerCamelCase = 220			, )				->Mapping[str, Any]:



         '''simple docstring'''
         __a					  =     OrderedDict()
         __a					  =     OnnxConfig.generate_dummy_inputs(
             self			, preprocessor=preprocessor.feature_extractor			, batch_size=lowerCamelCase			, framework=lowerCamelCase			, sampling_rate=lowerCamelCase			, time_duration=lowerCamelCase			, frequency=lowerCamelCase			, )
         __a					  =     encoder_inputs['input_features'].shape[2]
         __a					  =     encoder_sequence_length // 2 if self.use_past else seq_length

         __a					  =     super().generate_dummy_inputs(
             preprocessor.tokenizer			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase   )

         __a					  =     encoder_inputs.pop('input_features'   )
         __a					  =     decoder_inputs.pop('decoder_input_ids'   )

         if "past_key_values" in decoder_inputs:
            __a					  =     decoder_inputs.pop('past_key_values'   )

         return dummy_inputs






      @property
      def 			__UpperCamelCase				(	self   )				->float:



         '''simple docstring'''
         return 1e-3