
import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__a         =   logging.get_logger(__name__)

__a         =   {
    'microsoft/unispeech-sat-base-100h-libri-ft': (
        'https://huggingface.co/microsoft/unispeech-sat-base-100h-libri-ft/resolve/main/config.json'
    ),
    # See all UniSpeechSat models at https://huggingface.co/models?filter=unispeech_sat
}
class 					lowercase__(							UpperCAmelCase				):



	"""simple docstring"""




	a     :Any       	=      'unispeech-sat'
	def __init__(			self    :     Any			,				SCREAMING_SNAKE_CASE_    :     Any=3_2			,				SCREAMING_SNAKE_CASE_    :     Any=7_6_8			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]=1_2			,				SCREAMING_SNAKE_CASE_    :     int=1_2			,				SCREAMING_SNAKE_CASE_    :     str=3_0_7_2			,				SCREAMING_SNAKE_CASE_    :     Dict="gelu"			,				SCREAMING_SNAKE_CASE_    :     int=0.1			,				SCREAMING_SNAKE_CASE_    :     Any=0.1			,				SCREAMING_SNAKE_CASE_    :     Tuple=0.1			,				SCREAMING_SNAKE_CASE_    :     Tuple=0.0			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]=0.0			,				SCREAMING_SNAKE_CASE_    :     Tuple=0.1			,				SCREAMING_SNAKE_CASE_    :     Dict=0.1			,				SCREAMING_SNAKE_CASE_    :     List[str]=0.02			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]=1e-5			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]="group"			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]="gelu"			,				SCREAMING_SNAKE_CASE_    :     List[str]=(5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2)			,				SCREAMING_SNAKE_CASE_    :     Tuple=(5, 2, 2, 2, 2, 2, 2)			,				SCREAMING_SNAKE_CASE_    :     List[str]=(1_0, 3, 3, 3, 3, 2, 2)			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]=False			,				SCREAMING_SNAKE_CASE_    :     Any=1_2_8			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]=1_6			,				SCREAMING_SNAKE_CASE_    :     Dict=False			,				SCREAMING_SNAKE_CASE_    :     List[Any]=True			,				SCREAMING_SNAKE_CASE_    :     Tuple=0.05			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]=1_0			,				SCREAMING_SNAKE_CASE_    :     List[Any]=2			,				SCREAMING_SNAKE_CASE_    :     Tuple=0.0			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]=1_0			,				SCREAMING_SNAKE_CASE_    :     int=0			,				SCREAMING_SNAKE_CASE_    :     Tuple=3_2_0			,				SCREAMING_SNAKE_CASE_    :     Dict=2			,				SCREAMING_SNAKE_CASE_    :     Dict=0.1			,				SCREAMING_SNAKE_CASE_    :     List[str]=1_0_0			,				SCREAMING_SNAKE_CASE_    :     str=2_5_6			,				SCREAMING_SNAKE_CASE_    :     List[str]=2_5_6			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]=0.1			,				SCREAMING_SNAKE_CASE_    :     Dict="mean"			,				SCREAMING_SNAKE_CASE_    :     Dict=False			,				SCREAMING_SNAKE_CASE_    :     Dict=False			,				SCREAMING_SNAKE_CASE_    :     Tuple=2_5_6			,				SCREAMING_SNAKE_CASE_    :     Optional[int]=(5_1_2, 5_1_2, 5_1_2, 5_1_2, 1_5_0_0)			,				SCREAMING_SNAKE_CASE_    :     List[Any]=(5, 3, 3, 1, 1)			,				SCREAMING_SNAKE_CASE_    :     Dict=(1, 2, 3, 1, 1)			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]=5_1_2			,				SCREAMING_SNAKE_CASE_    :     List[Any]=0			,				SCREAMING_SNAKE_CASE_    :     List[Any]=1			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]=2			,				SCREAMING_SNAKE_CASE_    :     List[str]=5_0_4			,				**SCREAMING_SNAKE_CASE_    :     Dict			,				)			->	Any:
					super().__init__(**SCREAMING_SNAKE_CASE_			,				pad_token_id=SCREAMING_SNAKE_CASE_			,				bos_token_id=SCREAMING_SNAKE_CASE_			,				eos_token_id=SCREAMING_SNAKE_CASE_			)
					lowercase_	    =   hidden_size
					lowercase_	    =   feat_extract_norm
					lowercase_	    =   feat_extract_activation
					lowercase_	    =   list(SCREAMING_SNAKE_CASE_			)
					lowercase_	    =   list(SCREAMING_SNAKE_CASE_			)
					lowercase_	    =   list(SCREAMING_SNAKE_CASE_			)
					lowercase_	    =   conv_bias
					lowercase_	    =   num_conv_pos_embeddings
					lowercase_	    =   num_conv_pos_embedding_groups
					lowercase_	    =   len(self.conv_dim			)
					lowercase_	    =   num_hidden_layers
					lowercase_	    =   intermediate_size
					lowercase_	    =   hidden_act
					lowercase_	    =   num_attention_heads
					lowercase_	    =   hidden_dropout
					lowercase_	    =   attention_dropout
					lowercase_	    =   activation_dropout
					lowercase_	    =   feat_proj_dropout
					lowercase_	    =   final_dropout
					lowercase_	    =   layerdrop
					lowercase_	    =   layer_norm_eps
					lowercase_	    =   initializer_range
					lowercase_	    =   vocab_size
					lowercase_	    =   num_clusters
					lowercase_	    =   do_stable_layer_norm
					lowercase_	    =   use_weighted_layer_sum

					if (
					    (len(self.conv_stride			) != self.num_feat_extract_layers)
					    or (len(self.conv_kernel			) != self.num_feat_extract_layers)
					    or (len(self.conv_dim			) != self.num_feat_extract_layers)
					):
									raise ValueError(
									    '''Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='''
									    ''' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='''
									    f''' {len(self.conv_dim			)}`, `len(config.conv_stride) = {len(self.conv_stride			)}`,'''
									    f''' `len(config.conv_kernel) = {len(self.conv_kernel			)}`.'''			)

					# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
					lowercase_	    =   apply_spec_augment
					lowercase_	    =   mask_time_prob
					lowercase_	    =   mask_time_length
					lowercase_	    =   mask_time_min_masks
					lowercase_	    =   mask_feature_prob
					lowercase_	    =   mask_feature_length
					lowercase_	    =   mask_feature_min_masks

					# parameters for pretraining with codevector quantized representations
					lowercase_	    =   num_codevectors_per_group
					lowercase_	    =   num_codevector_groups
					lowercase_	    =   contrastive_logits_temperature
					lowercase_	    =   feat_quantizer_dropout
					lowercase_	    =   num_negatives
					lowercase_	    =   codevector_dim
					lowercase_	    =   proj_codevector_dim
					lowercase_	    =   diversity_loss_weight

					# ctc loss
					lowercase_	    =   ctc_loss_reduction
					lowercase_	    =   ctc_zero_infinity

					# SequenceClassification-specific parameter. Feel free to ignore for other classes.
					lowercase_	    =   classifier_proj_size

					# XVector-specific parameters. Feel free to ignore for other classes.
					lowercase_	    =   list(SCREAMING_SNAKE_CASE_			)
					lowercase_	    =   list(SCREAMING_SNAKE_CASE_			)
					lowercase_	    =   list(SCREAMING_SNAKE_CASE_			)
					lowercase_	    =   xvector_output_dim




	@property
	def 		_lowercase    (			self    :     Optional[Any]			)			->	Any:
					return functools.reduce(operator.mul			,				self.conv_stride			,				1			)












"""simple docstring"""







import json
import os
from typing import Dict, List, Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase            =	logging.get_logger(__name__)


_lowerCAmelCase            =	{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
    'tokenizer_config_file': 'tokenizer_config.json',
}

_lowerCAmelCase            =	{
    'vocab_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json'
    },
    'merges_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt'
    },
    'tokenizer_config_file': {
        'facebook/blenderbot_small-90M': (
            'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json'
        )
    },
}

_lowerCAmelCase            =	{'facebook/blenderbot_small-90M': 5_12}


def      UpperCamelCase       ( _A )     ->       Dict:
				lowercase    :      Union[str, Any]       = set()
				lowercase    :      Union[str, Any]       = word[0]
				for char in word[1:]:
								pairs.add((prev_char, char) )
								lowercase    :      List[str]       = char

				lowercase    :      Optional[Any]       = set(_A )
				return pairs


class      UpperCamelCase		(__snake_case     ):
						_SCREAMING_SNAKE_CASE  : Optional[int]												=		VOCAB_FILES_NAMES
						_SCREAMING_SNAKE_CASE  : str												=		PRETRAINED_VOCAB_FILES_MAP
						_SCREAMING_SNAKE_CASE  : Union[str, Any]												=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_SCREAMING_SNAKE_CASE  : Any												=		["""input_ids""", """attention_mask"""]






						def __init__(		self  :Any			, __magic_name__  :Union[str, Any]			, __magic_name__  :str			, __magic_name__  :Union[str, Any]="__start__"			, __magic_name__  :Optional[int]="__end__"			, __magic_name__  :Tuple="__unk__"			, __magic_name__  :Optional[int]="__null__"			, **__magic_name__  :Tuple			, )      ->Tuple:
										super().__init__(unk_token=__magic_name__			, bos_token=__magic_name__			, eos_token=__magic_name__			, pad_token=__magic_name__			, **__magic_name__				)

										with open(__magic_name__			, encoding="""utf-8"""				) as vocab_handle:
														lowercase    :      int       = json.load(__magic_name__				)
										lowercase    :      int       = {v: k for k, v in self.encoder.items()}
										with open(__magic_name__			, encoding="""utf-8"""				) as merges_handle:
														lowercase    :      Any       = merges_handle.read().split("""\n"""				)[1:-1]
										lowercase    :      Union[str, Any]       = [tuple(merge.split()				) for merge in merges]
										lowercase    :      Any       = dict(zip(__magic_name__			, range(len(__magic_name__				)				)				)				)
										lowercase    :      Union[str, Any]       = {}






						@property
						def      __snake_case      (		self  :int				)      ->int:
										return len(self.encoder				)






						def      __snake_case      (		self  :Optional[Any]				)      ->Dict:
										return dict(self.encoder			, **self.added_tokens_encoder				)






						def      __snake_case      (		self  :List[str]			, __magic_name__  :str				)      ->str:
										if token in self.cache:
														return self.cache[token]
										lowercase    :      Dict       = re.sub("""([.,!?()])"""			, r""" \1"""			, __magic_name__				)
										lowercase    :      Tuple       = re.sub("""(')"""			, r""" \1 """			, __magic_name__				)
										lowercase    :      List[str]       = re.sub(r"""\s{2,}"""			, """ """			, __magic_name__				)
										if "\n" in token:
														lowercase    :      int       = token.replace("""\n"""			, """ __newln__"""				)

										lowercase    :      Tuple       = token.split(""" """				)
										lowercase    :      Dict       = []
										for token in tokens:
														if not len(__magic_name__				):
																		continue

														lowercase    :      List[Any]       = token.lower()
														lowercase    :      int       = tuple(__magic_name__				)
														lowercase    :      List[str]       = tuple(list(word[:-1]				) + [word[-1] + """</w>"""]				)
														lowercase    :      List[str]       = get_pairs(__magic_name__				)

														if not pairs:
																		words.append(__magic_name__				)
																		continue

														while True:
																		lowercase    :      Optional[Any]       = min(__magic_name__			, key=lambda __magic_name__				: self.bpe_ranks.get(__magic_name__			, float("""inf"""				)				)				)
																		if bigram not in self.bpe_ranks:
																						break
																		lowercase      ,      lowercase    :      List[str]       = bigram
																		lowercase    :      Dict       = []
																		lowercase    :      Optional[int]       = 0

																		while i < len(__magic_name__				):
																						try:
																										lowercase    :      Optional[Any]       = word.index(__magic_name__			, __magic_name__				)
																										new_word.extend(word[i:j]				)
																										lowercase    :      Union[str, Any]       = j
																						except ValueError:
																										new_word.extend(word[i:]				)
																										break

																						if word[i] == first and i < len(__magic_name__				) - 1 and word[i + 1] == second:
																										new_word.append(first + second				)
																										i += 2
																						else:
																										new_word.append(word[i]				)
																										i += 1
																		lowercase    :      str       = tuple(__magic_name__				)
																		lowercase    :      Optional[Any]       = new_word
																		if len(__magic_name__				) == 1:
																						break
																		else:
																						lowercase    :      List[Any]       = get_pairs(__magic_name__				)
														lowercase    :      int       = """@@ """.join(__magic_name__				)
														lowercase    :      int       = word[:-4]

														lowercase    :      List[str]       = word
														words.append(__magic_name__				)
										return " ".join(__magic_name__				)






						def      __snake_case      (		self  :List[str]			, __magic_name__  :str				)      ->List[str]:
										lowercase    :      str       = []

										lowercase    :      Dict       = re.findall(r"""\S+\n?"""			, __magic_name__				)

										for token in words:
														split_tokens.extend(list(self.bpe(__magic_name__				).split(""" """				)				)				)
										return split_tokens






						def      __snake_case      (		self  :int			, __magic_name__  :str				)      ->int:
										lowercase    :      Tuple       = token.lower()
										return self.encoder.get(__magic_name__			, self.encoder.get(self.unk_token				)				)






						def      __snake_case      (		self  :Tuple			, __magic_name__  :int				)      ->str:
										return self.decoder.get(__magic_name__			, self.unk_token				)






						def      __snake_case      (		self  :Tuple			, __magic_name__  :List[str]				)      ->str:
										lowercase    :      Tuple       = """ """.join(__magic_name__				).replace("""@@ """			, """"""				).strip()
										return out_string






						def      __snake_case      (		self  :List[Any]			, __magic_name__  :str			, __magic_name__  :Optional[str] = None				)      ->Tuple[str]:
										if not os.path.isdir(__magic_name__				):
														logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
														return
										lowercase    :      str       = os.path.join(
										    __magic_name__			, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)
										lowercase    :      List[str]       = os.path.join(
										    __magic_name__			, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]				)

										with open(__magic_name__			, """w"""			, encoding="""utf-8"""				) as f:
														f.write(json.dumps(self.encoder			, indent=2			, sort_keys=__magic_name__			, ensure_ascii=__magic_name__				) + """\n"""				)

										lowercase    :      Any       = 0
										with open(__magic_name__			, """w"""			, encoding="""utf-8"""				) as writer:
														writer.write("""#version: 0.2\n"""				)
														for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			, key=lambda __magic_name__				: kv[1]				):
																		if index != token_index:
																						logger.warning(
																						    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																						    """ Please check that the tokenizer is not corrupted!"""				)
																						lowercase    :      Any       = token_index
																		writer.write(""" """.join(__magic_name__				) + """\n"""				)
																		index += 1

										return vocab_file, merge_file
