




import json
from typing import Dict, List, Optional, Tuple, Union

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding, EncodedInput
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_led import LEDTokenizer


__SCREAMING_SNAKE_CASE			    =logging.get_logger(__name__)


__SCREAMING_SNAKE_CASE			    ={'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__SCREAMING_SNAKE_CASE			    ={
    '''vocab_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''allenai/led-base-16384''': '''https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json''',
    },
}

__SCREAMING_SNAKE_CASE			    ={
    '''allenai/led-base-16384''': 16_384,
}






class 			__magic_name__    (	__UpperCAmelCase):
				'''simple docstring'''







				SCREAMING_SNAKE_CASE__       :				Dict						   =       VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__       :				Dict						   =       PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__       :				Tuple						   =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__       :				List[Any]						   =       LEDTokenizer
				SCREAMING_SNAKE_CASE__       :				Dict						   =       ["input_ids", "attention_mask"]


				def __init__(				self: Union[str, Any]		,       _lowerCamelCase: Optional[Any]=None		,       _lowerCamelCase: str=None		,       _lowerCamelCase: Dict=None		,       _lowerCamelCase: List[Any]="replace"		,       _lowerCamelCase: int="<s>"		,       _lowerCamelCase: Optional[int]="</s>"		,       _lowerCamelCase: Union[str, Any]="</s>"		,       _lowerCamelCase: List[str]="<s>"		,       _lowerCamelCase: str="<unk>"		,       _lowerCamelCase: Union[str, Any]="<pad>"		,       _lowerCamelCase: Union[str, Any]="<mask>"		,       _lowerCamelCase: Any=False		,       _lowerCamelCase: Dict=True		,       **_lowerCamelCase: str		,       ):
											super().__init__(
											    _lowerCamelCase		,       _lowerCamelCase		,       tokenizer_file=_lowerCamelCase		,       errors=_lowerCamelCase		,       bos_token=_lowerCamelCase		,       eos_token=_lowerCamelCase		,       sep_token=_lowerCamelCase		,       cls_token=_lowerCamelCase		,       unk_token=_lowerCamelCase		,       pad_token=_lowerCamelCase		,       mask_token=_lowerCamelCase		,       add_prefix_space=_lowerCamelCase		,       trim_offsets=_lowerCamelCase		,       **_lowerCamelCase		,       )

											SCREAMING_SNAKE_CASE_ 						=     json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
											if pre_tok_state.get('''add_prefix_space'''		,       _lowerCamelCase		) != add_prefix_space:
																		SCREAMING_SNAKE_CASE_ 						=     getattr(_lowerCamelCase		,       pre_tok_state.pop('''type'''		)		)
																		SCREAMING_SNAKE_CASE_ 						=     add_prefix_space
																		SCREAMING_SNAKE_CASE_ 						=     pre_tok_class(**_lowerCamelCase		)

											SCREAMING_SNAKE_CASE_ 						=     add_prefix_space

											# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
											SCREAMING_SNAKE_CASE_ 						=     '''post_processor'''
											SCREAMING_SNAKE_CASE_ 						=     getattr(self.backend_tokenizer		,       _lowerCamelCase		,       _lowerCamelCase		)
											if tokenizer_component_instance:
																		SCREAMING_SNAKE_CASE_ 						=     json.loads(tokenizer_component_instance.__getstate__()		)

																		# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																		if "sep" in state:
																									SCREAMING_SNAKE_CASE_ 						=     tuple(state['''sep''']		)
																		if "cls" in state:
																									SCREAMING_SNAKE_CASE_ 						=     tuple(state['''cls''']		)

																		SCREAMING_SNAKE_CASE_ 						=     False

																		if state.get('''add_prefix_space'''		,       _lowerCamelCase		) != add_prefix_space:
																									SCREAMING_SNAKE_CASE_ 						=     add_prefix_space
																									SCREAMING_SNAKE_CASE_ 						=     True

																		if state.get('''trim_offsets'''		,       _lowerCamelCase		) != trim_offsets:
																									SCREAMING_SNAKE_CASE_ 						=     trim_offsets
																									SCREAMING_SNAKE_CASE_ 						=     True

																		if changes_to_apply:
																									SCREAMING_SNAKE_CASE_ 						=     getattr(_lowerCamelCase		,       state.pop('''type'''		)		)
																									SCREAMING_SNAKE_CASE_ 						=     component_class(**_lowerCamelCase		)
																									setattr(self.backend_tokenizer		,       _lowerCamelCase		,       _lowerCamelCase		)


				@property
				# Copied from transformers.models.bart.tokenization_bart_fast.BartTokenizerFast.mask_token with BART->LED
				def      _A				(				self: Tuple		):
											if self._mask_token is None:
																		if self.verbose:
																									logger.error('''Using mask_token, but it is not set yet.'''		)
																		return None
											return str(self._mask_token		)


				@mask_token.setter
				def      _A				(				self: Dict		,       _lowerCamelCase: Optional[int]		):
											SCREAMING_SNAKE_CASE_ 						=     AddedToken(_lowerCamelCase		,       lstrip=_lowerCamelCase		,       rstrip=_lowerCamelCase		) if isinstance(_lowerCamelCase		,       _lowerCamelCase		) else value
											SCREAMING_SNAKE_CASE_ 						=     value


				def      _A				(				self: Optional[int]		,       *_lowerCamelCase: str		,       **_lowerCamelCase: Optional[int]		):
											SCREAMING_SNAKE_CASE_ 						=     kwargs.get('''is_split_into_words'''		,       _lowerCamelCase		)

											if is_split_into_words and not self.add_prefix_space:
																		raise ValueError(
																		    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
																		    '''to use it with pretokenized inputs.'''		)

											return super()._batch_encode_plus(*_lowerCamelCase		,       **_lowerCamelCase		)


				def      _A				(				self: Union[str, Any]		,       *_lowerCamelCase: int		,       **_lowerCamelCase: Union[str, Any]		):
											SCREAMING_SNAKE_CASE_ 						=     kwargs.get('''is_split_into_words'''		,       _lowerCamelCase		)

											if is_split_into_words and not self.add_prefix_space:
																		raise ValueError(
																		    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
																		    '''to use it with pretokenized inputs.'''		)

											return super()._encode_plus(*_lowerCamelCase		,       **_lowerCamelCase		)


				def      _A				(				self: List[str]		,       _lowerCamelCase: str		,       _lowerCamelCase: Optional[str] = None		):
											SCREAMING_SNAKE_CASE_ 						=     self._tokenizer.model.save(_lowerCamelCase		,       name=_lowerCamelCase		)
											return tuple(_lowerCamelCase		)


				def      _A				(				self: Optional[int]		,       _lowerCamelCase: str		,       _lowerCamelCase: Optional[int]=None		):
											SCREAMING_SNAKE_CASE_ 						=     [self.bos_token_id] + token_ids_a + [self.eos_token_id]
											if token_ids_a is None:
																		return output

											return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]


				def      _A				(				self: Optional[int]		,       _lowerCamelCase: List[int]		,       _lowerCamelCase: Optional[List[int]] = None		):
											SCREAMING_SNAKE_CASE_ 						=     [self.sep_token_id]
											SCREAMING_SNAKE_CASE_ 						=     [self.cls_token_id]

											if token_ids_a is None:
																		return len(cls + token_ids_a + sep		) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]





				def      _A				(				self: List[str]		,       _lowerCamelCase: Union[Dict[str, EncodedInput], BatchEncoding]		,       _lowerCamelCase: Optional[int] = None		,       _lowerCamelCase: PaddingStrategy = PaddingStrategy.DO_NOT_PAD		,       _lowerCamelCase: Optional[int] = None		,       _lowerCamelCase: Optional[bool] = None		,       ):
											SCREAMING_SNAKE_CASE_ 						=     super()._pad(
											    encoded_inputs=_lowerCamelCase		,       max_length=_lowerCamelCase		,       padding_strategy=_lowerCamelCase		,       pad_to_multiple_of=_lowerCamelCase		,       return_attention_mask=_lowerCamelCase		,       )

											# Load from model defaults
											if return_attention_mask is None:
																		SCREAMING_SNAKE_CASE_ 						=     '''attention_mask''' in self.model_input_names

											if return_attention_mask and "global_attention_mask" in encoded_inputs:
																		SCREAMING_SNAKE_CASE_ 						=     encoded_inputs[self.model_input_names[0]]
																		# `global_attention_mask` need to have the same length as other (sequential) inputs.
																		SCREAMING_SNAKE_CASE_ 						=     len(encoded_inputs['''global_attention_mask''']		) != len(_lowerCamelCase		)

																		if needs_to_be_padded:
																									SCREAMING_SNAKE_CASE_ 						=     len(_lowerCamelCase		) - len(encoded_inputs['''global_attention_mask''']		)

																									if self.padding_side == "right":
																																# Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
																																SCREAMING_SNAKE_CASE_ 						=     (
																																    encoded_inputs['''global_attention_mask'''] + [-1] * difference
																																)
																									elif self.padding_side == "left":
																																SCREAMING_SNAKE_CASE_ 						=     [-1] * difference + encoded_inputs[
																																    '''global_attention_mask'''
																																]
																									else:
																																raise ValueError('''Invalid padding strategy:''' + str(self.padding_side		)		)

											return encoded_inputs






'''simple docstring'''
import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__SCREAMING_SNAKE_CASE					:				Dict									=		logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE					:				int									=		{
    '''facebook/wav2vec2-base-960h''': '''https://huggingface.co/facebook/wav2vec2-base-960h/resolve/main/config.json''',
    # See all Wav2Vec2 models at https://huggingface.co/models?filter=wav2vec2
}






class 					__lowerCAmelCase   (   lowercase						):

						"""simple docstring"""






						_UpperCAmelCase		:       Dict   						="wav2vec2"






						def __init__(  self							:      str ,  lowerCAmelCase							:      Dict=32 ,  lowerCAmelCase							:      List[Any]=7_68 ,  lowerCAmelCase							:      List[str]=12 ,  lowerCAmelCase							:      List[str]=12 ,  lowerCAmelCase							:      List[Any]=30_72 ,  lowerCAmelCase							:      Optional[Any]="gelu" ,  lowerCAmelCase							:      Optional[int]=0.1 ,  lowerCAmelCase							:      Optional[int]=0.1 ,  lowerCAmelCase							:      List[Any]=0.1 ,  lowerCAmelCase							:      Dict=0.0 ,  lowerCAmelCase							:      Union[str, Any]=0.0 ,  lowerCAmelCase							:      str=0.1 ,  lowerCAmelCase							:      List[str]=0.1 ,  lowerCAmelCase							:      Dict=0.0_2 ,  lowerCAmelCase							:      List[str]=1e-5 ,  lowerCAmelCase							:      Optional[Any]="group" ,  lowerCAmelCase							:      List[Any]="gelu" ,  lowerCAmelCase							:      int=(5_12, 5_12, 5_12, 5_12, 5_12, 5_12, 5_12) ,  lowerCAmelCase							:      Union[str, Any]=(5, 2, 2, 2, 2, 2, 2) ,  lowerCAmelCase							:      Any=(10, 3, 3, 3, 3, 2, 2) ,  lowerCAmelCase							:      str=False ,  lowerCAmelCase							:      Tuple=1_28 ,  lowerCAmelCase							:      Union[str, Any]=16 ,  lowerCAmelCase							:      List[str]=False ,  lowerCAmelCase							:      Union[str, Any]=True ,  lowerCAmelCase							:      Any=0.0_5 ,  lowerCAmelCase							:      Any=10 ,  lowerCAmelCase							:      Dict=2 ,  lowerCAmelCase							:      List[Any]=0.0 ,  lowerCAmelCase							:      int=10 ,  lowerCAmelCase							:      List[str]=0 ,  lowerCAmelCase							:      Tuple=3_20 ,  lowerCAmelCase							:      List[str]=2 ,  lowerCAmelCase							:      int=0.1 ,  lowerCAmelCase							:      List[Any]=1_00 ,  lowerCAmelCase							:      List[str]=2_56 ,  lowerCAmelCase							:      int=2_56 ,  lowerCAmelCase							:      Optional[Any]=0.1 ,  lowerCAmelCase							:      str="sum" ,  lowerCAmelCase							:      List[Any]=False ,  lowerCAmelCase							:      List[str]=False ,  lowerCAmelCase							:      Tuple=2_56 ,  lowerCAmelCase							:      str=(5_12, 5_12, 5_12, 5_12, 15_00) ,  lowerCAmelCase							:      List[str]=(5, 3, 3, 1, 1) ,  lowerCAmelCase							:      Tuple=(1, 2, 3, 1, 1) ,  lowerCAmelCase							:      Optional[Any]=5_12 ,  lowerCAmelCase							:      Any=0 ,  lowerCAmelCase							:      str=1 ,  lowerCAmelCase							:      Any=2 ,  lowerCAmelCase							:      Any=False ,  lowerCAmelCase							:      List[Any]=3 ,  lowerCAmelCase							:      Any=2 ,  lowerCAmelCase							:      List[str]=3 ,  lowerCAmelCase							:      int=None ,  lowerCAmelCase							:      Optional[int]=None ,  **lowerCAmelCase							:      Union[str, Any] ,  ):
													super().__init__(**lowerCAmelCase ,  pad_token_id=lowerCAmelCase ,  bos_token_id=lowerCAmelCase ,  eos_token_id=lowerCAmelCase	)
													A_			    =		hidden_size
													A_			    =		feat_extract_norm
													A_			    =		feat_extract_activation
													A_			    =		list(lowerCAmelCase	)
													A_			    =		list(lowerCAmelCase	)
													A_			    =		list(lowerCAmelCase	)
													A_			    =		conv_bias
													A_			    =		num_conv_pos_embeddings
													A_			    =		num_conv_pos_embedding_groups
													A_			    =		len(self.conv_dim	)
													A_			    =		num_hidden_layers
													A_			    =		intermediate_size
													A_			    =		hidden_act
													A_			    =		num_attention_heads
													A_			    =		hidden_dropout
													A_			    =		attention_dropout
													A_			    =		activation_dropout
													A_			    =		feat_proj_dropout
													A_			    =		final_dropout
													A_			    =		layerdrop
													A_			    =		layer_norm_eps
													A_			    =		initializer_range
													A_			    =		vocab_size
													A_			    =		do_stable_layer_norm
													A_			    =		use_weighted_layer_sum

													if (
													    (len(self.conv_stride	) != self.num_feat_extract_layers)
													    or (len(self.conv_kernel	) != self.num_feat_extract_layers)
													    or (len(self.conv_dim	) != self.num_feat_extract_layers)
													):
																				raise ValueError(
																				    "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
																				    " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
																				    F" {len(self.conv_dim	)}`, `len(config.conv_stride) = {len(self.conv_stride	)}`,"
																				    F" `len(config.conv_kernel) = {len(self.conv_kernel	)}`."	)

													# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
													A_			    =		apply_spec_augment
													A_			    =		mask_time_prob
													A_			    =		mask_time_length
													A_			    =		mask_time_min_masks
													A_			    =		mask_feature_prob
													A_			    =		mask_feature_length
													A_			    =		mask_feature_min_masks

													# parameters for pretraining with codevector quantized representations
													A_			    =		num_codevectors_per_group
													A_			    =		num_codevector_groups
													A_			    =		contrastive_logits_temperature
													A_			    =		feat_quantizer_dropout
													A_			    =		num_negatives
													A_			    =		codevector_dim
													A_			    =		proj_codevector_dim
													A_			    =		diversity_loss_weight

													# ctc loss
													A_			    =		ctc_loss_reduction
													A_			    =		ctc_zero_infinity

													# adapter
													A_			    =		add_adapter
													A_			    =		adapter_kernel_size
													A_			    =		adapter_stride
													A_			    =		num_adapter_layers
													A_			    =		output_hidden_size or hidden_size
													A_			    =		adapter_attn_dim

													# SequenceClassification-specific parameter. Feel free to ignore for other classes.
													A_			    =		classifier_proj_size

													# XVector-specific parameters. Feel free to ignore for other classes.
													A_			    =		list(lowerCAmelCase	)
													A_			    =		list(lowerCAmelCase	)
													A_			    =		list(lowerCAmelCase	)
													A_			    =		xvector_output_dim






						@property
						def 				_UpperCAmelCase       (  self							:      Tuple	):
													return functools.reduce(operator.mul ,  self.conv_stride ,  1	)





