




"""simple docstring"""




from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
					from ...feature_extraction_utils import FeatureExtractionMixin
					from ...tokenization_utils_base import PreTrainedTokenizerBase
					from ...utils import TensorType

lowerCAmelCase				:     str					=				logging.get_logger(__name__)

lowerCAmelCase				:     Optional[Any]					=				{
    """openai/whisper-base""": """https://huggingface.co/openai/whisper-base/resolve/main/config.json""",
}

# fmt: off
lowerCAmelCase				:     Dict					=				[
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 357, 366, 438, 532, 685,
    705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377,
    1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211,
    4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 1_0563, 1_0786,
    1_1420, 1_1709, 1_1907, 1_3163, 1_3697, 1_3700, 1_4808, 1_5306, 1_6410, 1_6791,
    1_7992, 1_9203, 1_9510, 2_0724, 2_2305, 2_2935, 2_7007, 3_0109, 3_0420, 3_3409,
    3_4949, 4_0283, 4_0493, 4_0549, 4_7282, 4_9146, 5_0257, 5_0359, 5_0360, 5_0361
]
lowerCAmelCase				:     Tuple					=				[
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 359, 503, 522, 542, 873,
    893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627,
    3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647,
    7273, 9061, 9383, 1_0428, 1_0929, 1_1938, 1_2033, 1_2331, 1_2562, 1_3793,
    1_4157, 1_4635, 1_5265, 1_5618, 1_6553, 1_6604, 1_8362, 1_8956, 2_0075, 2_1675,
    2_2520, 2_6130, 2_6161, 2_6435, 2_8279, 2_9464, 3_1650, 3_2302, 3_2470, 3_6865,
    4_2863, 4_7425, 4_9870, 5_0254, 5_0258, 5_0360, 5_0361, 5_0362
]






class        __magic_name__				(			UpperCAmelCase__							):






				'''simple docstring'''
				__UpperCamelCase         =		"whisper"
				__UpperCamelCase         =		["past_key_values"]
				__UpperCamelCase         =		{"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}





				def __init__(			self						,					_a=51_865						,					_a=80						,					_a=6						,					_a=4						,					_a=6						,					_a=4						,					_a=1_536						,					_a=1_536						,					_a=0.0						,					_a=0.0						,					_a=50_257						,					_a=True						,					_a=True						,					_a="gelu"						,					_a=256						,					_a=0.0						,					_a=0.0						,					_a=0.0						,					_a=0.02						,					_a=False						,					_a=1_500						,					_a=448						,					_a=50_256						,					_a=50_256						,					_a=50_256						,					_a=None						,					_a=[220, 50_256]						,					_a=False						,					_a=256						,					_a=False						,					_a=0.05						,					_a=10						,					_a=2						,					_a=0.0						,					_a=10						,					_a=0						,					_a=7						,					**_a						,					):





							"""simple docstring"""
							lowerCamelCase      =   vocab_size
							lowerCamelCase      =   num_mel_bins
							lowerCamelCase      =   d_model
							lowerCamelCase      =   encoder_layers
							lowerCamelCase      =   encoder_attention_heads
							lowerCamelCase      =   decoder_layers
							lowerCamelCase      =   decoder_attention_heads
							lowerCamelCase      =   decoder_ffn_dim
							lowerCamelCase      =   encoder_ffn_dim
							lowerCamelCase      =   dropout
							lowerCamelCase      =   attention_dropout
							lowerCamelCase      =   activation_dropout
							lowerCamelCase      =   activation_function
							lowerCamelCase      =   init_std
							lowerCamelCase      =   encoder_layerdrop
							lowerCamelCase      =   decoder_layerdrop
							lowerCamelCase      =   use_cache
							lowerCamelCase      =   encoder_layers
							lowerCamelCase      =   scale_embedding  # scale factor will be sqrt(d_model) if True
							lowerCamelCase      =   max_source_positions
							lowerCamelCase      =   max_target_positions

							# Audio Classification-specific parameters. Feel free to ignore for other classes.
							lowerCamelCase      =   classifier_proj_size
							lowerCamelCase      =   use_weighted_layer_sum

							# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
							lowerCamelCase      =   apply_spec_augment
							lowerCamelCase      =   mask_time_prob
							lowerCamelCase      =   mask_time_length
							lowerCamelCase      =   mask_time_min_masks
							lowerCamelCase      =   mask_feature_prob
							lowerCamelCase      =   mask_feature_length
							lowerCamelCase      =   mask_feature_min_masks

							lowerCamelCase      =   median_filter_width

							super().__init__(
							    pad_token_id=_a						,					bos_token_id=_a						,					eos_token_id=_a						,					is_encoder_decoder=_a						,					decoder_start_token_id=_a						,					suppress_tokens=_a						,					begin_suppress_tokens=_a						,					**_a						,					)






class        __magic_name__				(			UpperCAmelCase__							):






				'''simple docstring'''





				@property
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							lowerCamelCase      =   OrderedDict(
							    [
							        ("""input_features""", {0: """batch""", 1: """feature_size""", 2: """encoder_sequence"""}),
							    ]					)
							if self.use_past:
										lowerCamelCase      =   {0: """batch"""}
							else:
										lowerCamelCase      =   {0: """batch""", 1: """decoder_sequence"""}

							if self.use_past:
										self.fill_with_past_key_values_(_a						,					direction="""inputs"""					)

							return common_inputs





				def   _lowerCAmelCase							(			self						,					_a						,					_a = -1						,					_a = -1						,					_a = False						,					_a = None						,					_a = 22_050						,					_a = 5.0						,					_a = 220						,					):





							"""simple docstring"""
							lowerCamelCase      =   OrderedDict()
							lowerCamelCase      =   OnnxConfig.generate_dummy_inputs(
							    self						,					preprocessor=preprocessor.feature_extractor						,					batch_size=_a						,					framework=_a						,					sampling_rate=_a						,					time_duration=_a						,					frequency=_a						,					)
							lowerCamelCase      =   encoder_inputs["""input_features"""].shape[2]
							lowerCamelCase      =   encoder_sequence_length // 2 if self.use_past else seq_length

							lowerCamelCase      =   super().generate_dummy_inputs(
							    preprocessor.tokenizer						,					_a						,					_a						,					_a						,					_a					)

							lowerCamelCase      =   encoder_inputs.pop("""input_features"""					)
							lowerCamelCase      =   decoder_inputs.pop("""decoder_input_ids"""					)

							if "past_key_values" in decoder_inputs:
										lowerCamelCase      =   decoder_inputs.pop("""past_key_values"""					)

							return dummy_inputs





				@property
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							return 1e-3









"""simple docstring"""




import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase				:     Union[str, Any]					=				logging.get_logger(__name__)

lowerCAmelCase				:     Optional[int]					=				{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

lowerCAmelCase				:     Optional[Any]					=				{
    """vocab_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"""},
    """merges_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"""},
}

lowerCAmelCase				:     Optional[Any]					=				{
    """ctrl""": 256,
}

lowerCAmelCase				:     int					=				{
    """Pregnancy""": 16_8629,
    """Christianity""": 7675,
    """Explain""": 10_6423,
    """Fitness""": 6_3440,
    """Saving""": 6_3163,
    """Ask""": 2_7171,
    """Ass""": 9_5985,
    """Joke""": 16_3509,
    """Questions""": 4_5622,
    """Thoughts""": 4_9605,
    """Retail""": 5_2342,
    """Feminism""": 16_4338,
    """Writing""": 1_1992,
    """Atheism""": 19_2263,
    """Netflix""": 4_8616,
    """Computing""": 3_9639,
    """Opinion""": 4_3213,
    """Alone""": 4_4967,
    """Funny""": 5_8917,
    """Gaming""": 4_0358,
    """Human""": 4088,
    """India""": 1331,
    """Joker""": 7_7138,
    """Diet""": 3_6206,
    """Legal""": 1_1859,
    """Norman""": 4939,
    """Tip""": 7_2689,
    """Weight""": 5_2343,
    """Movies""": 4_6273,
    """Running""": 2_3425,
    """Science""": 2090,
    """Horror""": 3_7793,
    """Confession""": 6_0572,
    """Finance""": 1_2250,
    """Politics""": 1_6360,
    """Scary""": 19_1985,
    """Support""": 1_2654,
    """Technologies""": 3_2516,
    """Teenage""": 6_6160,
    """Event""": 3_2769,
    """Learned""": 6_7460,
    """Notion""": 18_2770,
    """Wikipedia""": 3_7583,
    """Books""": 6665,
    """Extract""": 7_6050,
    """Confessions""": 10_2701,
    """Conspiracy""": 7_5932,
    """Links""": 6_3674,
    """Narcissus""": 15_0425,
    """Relationship""": 5_4766,
    """Relationships""": 13_4796,
    """Reviews""": 4_1671,
    """News""": 4256,
    """Translation""": 2_6820,
    """multilingual""": 12_8406,
}



def        a__					(							snake_case__							)		->   Optional[Any]:
			lowerCamelCase      =   set()
			lowerCamelCase      =   word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)							)
						lowerCamelCase      =   char

			lowerCamelCase      =   set(snake_case__							)
			return pairs






class        __magic_name__				(			UpperCAmelCase__							):






				'''simple docstring'''
				__UpperCamelCase         =		VOCAB_FILES_NAMES
				__UpperCamelCase         =		PRETRAINED_VOCAB_FILES_MAP
				__UpperCamelCase         =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__UpperCamelCase         =		CONTROL_CODES





				def __init__(			self						,					_a						,					_a						,					_a="<unk>"						,					**_a					):





							"""simple docstring"""
							super().__init__(unk_token=_a						,					**_a					)

							with open(_a						,					encoding="""utf-8"""					) as vocab_handle:
										lowerCamelCase      =   json.load(_a					)
							lowerCamelCase      =   {v: k for k, v in self.encoder.items()}
							with open(_a						,					encoding="""utf-8"""					) as merges_handle:
										lowerCamelCase      =   merges_handle.read().split("""\n"""					)[1:-1]
							lowerCamelCase      =   [tuple(merge.split()					) for merge in merges]
							lowerCamelCase      =   dict(zip(_a						,					range(len(_a					)					)					)					)
							lowerCamelCase      =   {}





				@property
				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							return len(self.encoder					)





				def   _lowerCAmelCase							(			self					):





							"""simple docstring"""
							return dict(self.encoder						,					**self.added_tokens_encoder					)





				def   _lowerCAmelCase							(			self						,					_a					):





							"""simple docstring"""
							if token in self.cache:
										return self.cache[token]
							lowerCamelCase      =   tuple(_a					)
							lowerCamelCase      =   tuple(list(word[:-1]					) + [word[-1] + """</w>"""]					)
							lowerCamelCase      =   get_pairs(_a					)

							if not pairs:
										return token

							while True:
										lowerCamelCase      =   min(_a						,					key=lambda _a					: self.bpe_ranks.get(_a						,					float("""inf"""					)					)					)
										if bigram not in self.bpe_ranks:
													break
										lowerCamelCase ,  lowerCamelCase      =   bigram
										lowerCamelCase      =   []
										lowerCamelCase      =   0
										while i < len(_a					):
													try:
																lowerCamelCase      =   word.index(_a						,					_a					)
													except ValueError:
																new_word.extend(word[i:]					)
																break
													else:
																new_word.extend(word[i:j]					)
																lowerCamelCase      =   j

													if word[i] == first and i < len(_a					) - 1 and word[i + 1] == second:
																new_word.append(first + second					)
																i += 2
													else:
																new_word.append(word[i]					)
																i += 1
										lowerCamelCase      =   tuple(_a					)
										lowerCamelCase      =   new_word
										if len(_a					) == 1:
													break
										else:
													lowerCamelCase      =   get_pairs(_a					)
							lowerCamelCase      =   """@@ """.join(_a					)
							lowerCamelCase      =   word[:-4]
							lowerCamelCase      =   word
							return word





				def   _lowerCAmelCase							(			self						,					_a					):





							"""simple docstring"""
							lowerCamelCase      =   []

							lowerCamelCase      =   re.findall(r"""\S+\n?"""						,					_a					)

							for token in words:
										split_tokens.extend(list(self.bpe(_a					).split(""" """					)					)					)
							return split_tokens





				def   _lowerCAmelCase							(			self						,					_a					):





							"""simple docstring"""
							return self.encoder.get(_a						,					self.encoder.get(self.unk_token					)					)





				def   _lowerCAmelCase							(			self						,					_a					):





							"""simple docstring"""
							return self.decoder.get(_a						,					self.unk_token					)





				def   _lowerCAmelCase							(			self						,					_a					):





							"""simple docstring"""
							lowerCamelCase      =   """ """.join(_a					).replace("""@@ """						,					""""""					).strip()
							return out_string





				def   _lowerCAmelCase							(			self						,					_a						,					_a = None					):





							"""simple docstring"""
							if not os.path.isdir(_a					):
										logger.error(f'Vocabulary path ({save_directory}) should be a directory'					)
										return
							lowerCamelCase      =   os.path.join(
							    _a						,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)
							lowerCamelCase      =   os.path.join(
							    _a						,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]					)

							with open(_a						,					"""w"""						,					encoding="""utf-8"""					) as f:
										f.write(json.dumps(self.encoder						,					indent=2						,					sort_keys=_a						,					ensure_ascii=_a					) + """\n"""					)

							lowerCamelCase      =   0
							with open(_a						,					"""w"""						,					encoding="""utf-8"""					) as writer:
										writer.write("""#version: 0.2\n"""					)
										for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,					key=lambda _a					: kv[1]					):
													if index != token_index:
																logger.warning(
																    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																    """ Please check that the tokenizer is not corrupted!"""					)
																lowerCamelCase      =   token_index
													writer.write(""" """.join(_a					) + """\n"""					)
													index += 1

							return vocab_file, merge_file

				# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
				#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
				#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
				#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
				#     return ''.join(tokens_generated_so_far)



