
"""simple docstring"""






from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
 from ...feature_extraction_utils import FeatureExtractionMixin
 from ...tokenization_utils_base import PreTrainedTokenizerBase
 from ...utils import TensorType

__SCREAMING_SNAKE_CASE							=     logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							=     {
    """openai/whisper-base""": """https://huggingface.co/openai/whisper-base/resolve/main/config.json""",
}

# fmt: off
__SCREAMING_SNAKE_CASE							=     [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 357, 366, 438, 532, 685,
    705, 796, 930, 1_058, 1_220, 1_267, 1_279, 1_303, 1_343, 1_377,
    1_391, 1_635, 1_782, 1_875, 2_162, 2_361, 2_488, 3_467, 4_008, 4_211,
    4_600, 4_808, 5_299, 5_855, 6_329, 7_203, 9_609, 9_959, 10_563, 10_786,
    11_420, 11_709, 11_907, 13_163, 13_697, 13_700, 14_808, 15_306, 16_410, 16_791,
    17_992, 19_203, 19_510, 20_724, 22_305, 22_935, 27_007, 30_109, 30_420, 33_409,
    34_949, 40_283, 40_493, 40_549, 47_282, 49_146, 50_257, 50_359, 50_360, 50_361
]
__SCREAMING_SNAKE_CASE							=     [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 359, 503, 522, 542, 873,
    893, 902, 918, 922, 931, 1_350, 1_853, 1_982, 2_460, 2_627,
    3_246, 3_253, 3_268, 3_536, 3_846, 3_961, 4_183, 4_667, 6_585, 6_647,
    7_273, 9_061, 9_383, 10_428, 10_929, 11_938, 12_033, 12_331, 12_562, 13_793,
    14_157, 14_635, 15_265, 15_618, 16_553, 16_604, 18_362, 18_956, 20_075, 21_675,
    22_520, 26_130, 26_161, 26_435, 28_279, 29_464, 31_650, 32_302, 32_470, 36_865,
    42_863, 47_425, 49_870, 50_254, 50_258, 50_360, 50_361, 50_362
]





class 				a__					(						A__       ):
     UpperCAmelCase__         =      '''whisper'''
     UpperCAmelCase__         =      ['''past_key_values''']
     UpperCAmelCase__         =      {'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}



     def __init__(							self :Optional[Any]      ,  _lowerCamelCase :List[Any]=51_865      ,  _lowerCamelCase :Dict=80      ,  _lowerCamelCase :Tuple=6      ,  _lowerCamelCase :List[str]=4      ,  _lowerCamelCase :Optional[int]=6      ,  _lowerCamelCase :Tuple=4      ,  _lowerCamelCase :List[Any]=1_536      ,  _lowerCamelCase :Tuple=1_536      ,  _lowerCamelCase :Optional[Any]=0.0      ,  _lowerCamelCase :Union[str, Any]=0.0      ,  _lowerCamelCase :List[Any]=50_257      ,  _lowerCamelCase :List[str]=True      ,  _lowerCamelCase :Optional[Any]=True      ,  _lowerCamelCase :Union[str, Any]="gelu"      ,  _lowerCamelCase :int=256      ,  _lowerCamelCase :Optional[Any]=0.0      ,  _lowerCamelCase :List[Any]=0.0      ,  _lowerCamelCase :int=0.0      ,  _lowerCamelCase :Dict=0.02      ,  _lowerCamelCase :Dict=False      ,  _lowerCamelCase :List[Any]=1_500      ,  _lowerCamelCase :str=448      ,  _lowerCamelCase :Any=50_256      ,  _lowerCamelCase :Tuple=50_256      ,  _lowerCamelCase :Union[str, Any]=50_256      ,  _lowerCamelCase :Optional[int]=None      ,  _lowerCamelCase :str=[220, 50_256]      ,  _lowerCamelCase :int=False      ,  _lowerCamelCase :Optional[int]=256      ,  _lowerCamelCase :Tuple=False      ,  _lowerCamelCase :List[Any]=0.05      ,  _lowerCamelCase :List[Any]=10      ,  _lowerCamelCase :int=2      ,  _lowerCamelCase :List[Any]=0.0      ,  _lowerCamelCase :Optional[int]=10      ,  _lowerCamelCase :Tuple=0      ,  _lowerCamelCase :Optional[int]=7      ,  **_lowerCamelCase :List[Any]      ,  ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=vocab_size
      UpperCamelCase_     :					List[Any]   	=num_mel_bins
      UpperCamelCase_     :					List[str]   	=d_model
      UpperCamelCase_     :					str   	=encoder_layers
      UpperCamelCase_     :					int   	=encoder_attention_heads
      UpperCamelCase_     :					Tuple   	=decoder_layers
      UpperCamelCase_     :					Dict   	=decoder_attention_heads
      UpperCamelCase_     :					Tuple   	=decoder_ffn_dim
      UpperCamelCase_     :					Dict   	=encoder_ffn_dim
      UpperCamelCase_     :					List[Any]   	=dropout
      UpperCamelCase_     :					List[Any]   	=attention_dropout
      UpperCamelCase_     :					List[str]   	=activation_dropout
      UpperCamelCase_     :					List[str]   	=activation_function
      UpperCamelCase_     :					List[Any]   	=init_std
      UpperCamelCase_     :					List[Any]   	=encoder_layerdrop
      UpperCamelCase_     :					Optional[int]   	=decoder_layerdrop
      UpperCamelCase_     :					Any   	=use_cache
      UpperCamelCase_     :					int   	=encoder_layers
      UpperCamelCase_     :					Any   	=scale_embedding  # scale factor will be sqrt(d_model) if True
      UpperCamelCase_     :					Tuple   	=max_source_positions
      UpperCamelCase_     :					Optional[Any]   	=max_target_positions

      # Audio Classification-specific parameters. Feel free to ignore for other classes.
      UpperCamelCase_     :					Any   	=classifier_proj_size
      UpperCamelCase_     :					List[Any]   	=use_weighted_layer_sum

      # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
      UpperCamelCase_     :					Dict   	=apply_spec_augment
      UpperCamelCase_     :					Optional[int]   	=mask_time_prob
      UpperCamelCase_     :					Tuple   	=mask_time_length
      UpperCamelCase_     :					Dict   	=mask_time_min_masks
      UpperCamelCase_     :					List[str]   	=mask_feature_prob
      UpperCamelCase_     :					Any   	=mask_feature_length
      UpperCamelCase_     :					str   	=mask_feature_min_masks

      UpperCamelCase_     :					Union[str, Any]   	=median_filter_width

      super().__init__(
          pad_token_id=snake_case__      ,  bos_token_id=snake_case__      ,  eos_token_id=snake_case__      ,  is_encoder_decoder=snake_case__      ,  decoder_start_token_id=snake_case__      ,  suppress_tokens=snake_case__      ,  begin_suppress_tokens=snake_case__      ,  **snake_case__      ,  )





class 				a__					(						A__       ):



     @property
     def  lowerCamelCase_						(							self :List[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=OrderedDict(
          [
              ('input_features', {0: 'batch', 1: 'feature_size', 2: 'encoder_sequence'}),
          ]   )
      if self.use_past:
       UpperCamelCase_     :					Dict   	={0: "batch"}
      else:
       UpperCamelCase_     :					Optional[int]   	={0: "batch", 1: "decoder_sequence"}

      if self.use_past:
       self.fill_with_past_key_values_(snake_case__      ,  direction='inputs'   )

      return common_inputs



     def  lowerCamelCase_						(							self :Optional[Any]      ,  _lowerCamelCase :Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"]      ,  _lowerCamelCase :int = -1      ,  _lowerCamelCase :int = -1      ,  _lowerCamelCase :bool = False      ,  _lowerCamelCase :Optional["TensorType"] = None      ,  _lowerCamelCase :int = 22_050      ,  _lowerCamelCase :float = 5.0      ,  _lowerCamelCase :int = 220      ,  ):







      '''simple docstring'''



      UpperCamelCase_     :					List[Any]   	=OrderedDict()
      UpperCamelCase_     :					Any   	=OnnxConfig.generate_dummy_inputs(
          self      ,  preprocessor=preprocessor.feature_extractor      ,  batch_size=snake_case__      ,  framework=snake_case__      ,  sampling_rate=snake_case__      ,  time_duration=snake_case__      ,  frequency=snake_case__      ,  )
      UpperCamelCase_     :					int   	=encoder_inputs["input_features"].shape[2]
      UpperCamelCase_     :					Union[str, Any]   	=encoder_sequence_length // 2 if self.use_past else seq_length

      UpperCamelCase_     :					Tuple   	=super().generate_dummy_inputs(
          preprocessor.tokenizer      ,  snake_case__      ,  snake_case__      ,  snake_case__      ,  snake_case__   )

      UpperCamelCase_     :					Dict   	=encoder_inputs.pop('input_features'   )
      UpperCamelCase_     :					str   	=decoder_inputs.pop('decoder_input_ids'   )

      if "past_key_values" in decoder_inputs:
       UpperCamelCase_     :					Any   	=decoder_inputs.pop('past_key_values'   )

      return dummy_inputs




     @property
     def  lowerCamelCase_						(							self :List[Any]   ):







      '''simple docstring'''



      return 1E-3








"""simple docstring"""



import json
from typing import Dict, List, Optional, Tuple, Union

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding, EncodedInput
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_led import LEDTokenizer


_lowerCAmelCase	:	str				=      logging.get_logger(__name__)


_lowerCAmelCase	:	int				=      {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowerCAmelCase	:	List[Any]				=      {
    """vocab_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json""",
    },
}

_lowerCAmelCase	:	List[str]				=      {
    """allenai/led-base-16384""": 16_384,
}




class 			lowerCAmelCase__			(    __magic_name__   ):
   SCREAMING_SNAKE_CASE_    =VOCAB_FILES_NAMES
   SCREAMING_SNAKE_CASE_    =PRETRAINED_VOCAB_FILES_MAP
   SCREAMING_SNAKE_CASE_    =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   SCREAMING_SNAKE_CASE_    =LEDTokenizer
   SCREAMING_SNAKE_CASE_    =['''input_ids''', '''attention_mask''']
   def __init__(		self      :	Optional[Any]							,			snake_case__      :	str=None							,			snake_case__      :	List[Any]=None							,			snake_case__      :	Dict=None							,			snake_case__      :	List[str]="replace"							,			snake_case__      :	Optional[int]="<s>"							,			snake_case__      :	List[str]="</s>"							,			snake_case__      :	Union[str, Any]="</s>"							,			snake_case__      :	Dict="<s>"							,			snake_case__      :	Tuple="<unk>"							,			snake_case__      :	Any="<pad>"							,			snake_case__      :	Dict="<mask>"							,			snake_case__      :	int=False							,			snake_case__      :	Optional[int]=True							,			**snake_case__      :	List[Any]							,			):


        '''simple docstring'''
        super().__init__(
            snake_case__							,			snake_case__							,			tokenizer_file=snake_case__							,			errors=snake_case__							,			bos_token=snake_case__							,			eos_token=snake_case__							,			sep_token=snake_case__							,			cls_token=snake_case__							,			unk_token=snake_case__							,			pad_token=snake_case__							,			mask_token=snake_case__							,			add_prefix_space=snake_case__							,			trim_offsets=snake_case__							,			**snake_case__							,			)

        UpperCAmelCase__						:    str							=							json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
        if pre_tok_state.get("add_prefix_space"							,			snake_case__							) != add_prefix_space:
             UpperCAmelCase__						:    Dict							=							getattr(snake_case__							,			pre_tok_state.pop("type"							)							)
             UpperCAmelCase__						:    str							=							add_prefix_space
             UpperCAmelCase__						:    Any							=							pre_tok_class(**snake_case__							)

        UpperCAmelCase__						:    Dict							=							add_prefix_space

        # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
        UpperCAmelCase__						:    List[str]							=							"post_processor"
        UpperCAmelCase__						:    List[Any]							=							getattr(self.backend_tokenizer							,			snake_case__							,			snake_case__							)
        if tokenizer_component_instance:
             UpperCAmelCase__						:    int							=							json.loads(tokenizer_component_instance.__getstate__()							)

             # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
             if "sep" in state:
                  UpperCAmelCase__						:    Optional[Any]							=							tuple(state["sep"]							)
             if "cls" in state:
                  UpperCAmelCase__						:    Any							=							tuple(state["cls"]							)

             UpperCAmelCase__						:    Any							=							False

             if state.get("add_prefix_space"							,			snake_case__							) != add_prefix_space:
                  UpperCAmelCase__						:    Union[str, Any]							=							add_prefix_space
                  UpperCAmelCase__						:    List[Any]							=							True

             if state.get("trim_offsets"							,			snake_case__							) != trim_offsets:
                  UpperCAmelCase__						:    Optional[int]							=							trim_offsets
                  UpperCAmelCase__						:    List[Any]							=							True

             if changes_to_apply:
                  UpperCAmelCase__						:    List[str]							=							getattr(snake_case__							,			state.pop("type"							)							)
                  UpperCAmelCase__						:    Optional[int]							=							component_class(**snake_case__							)
                  setattr(self.backend_tokenizer							,			snake_case__							,			snake_case__							)
   @property
   # Copied from transformers.models.bart.tokenization_bart_fast.BartTokenizerFast.mask_token with BART->LED
   def       __a							(		self      :	Any							):


        '''simple docstring'''
        if self._mask_token is None:
             if self.verbose:
                  logger.error("Using mask_token, but it is not set yet."							)
             return None
        return str(self._mask_token							)
   @mask_token.setter
   def       __a							(		self      :	Any							,			snake_case__      :	Dict							):


        '''simple docstring'''
        UpperCAmelCase__						:    str							=							AddedToken(snake_case__							,			lstrip=snake_case__							,			rstrip=snake_case__							) if isinstance(snake_case__							,			snake_case__							) else value
        UpperCAmelCase__						:    Dict							=							value
   def       __a							(		self      :	str							,			*snake_case__      :	Any							,			**snake_case__      :	Optional[int]							):


        '''simple docstring'''
        UpperCAmelCase__						:    Any							=							kwargs.get("is_split_into_words"							,			snake_case__							)

        if is_split_into_words and not self.add_prefix_space:
             raise ValueError(
                 f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
                 "to use it with pretokenized inputs."							)

        return super()._batch_encode_plus(*snake_case__							,			**snake_case__							)
   def       __a							(		self      :	List[str]							,			*snake_case__      :	Union[str, Any]							,			**snake_case__      :	List[Any]							):


        '''simple docstring'''
        UpperCAmelCase__						:    Optional[Any]							=							kwargs.get("is_split_into_words"							,			snake_case__							)

        if is_split_into_words and not self.add_prefix_space:
             raise ValueError(
                 f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
                 "to use it with pretokenized inputs."							)

        return super()._encode_plus(*snake_case__							,			**snake_case__							)
   def       __a							(		self      :	Tuple							,			snake_case__      :	str							,			snake_case__      :	Optional[str] = None							):


        '''simple docstring'''
        UpperCAmelCase__						:    Union[str, Any]							=							self._tokenizer.model.save(snake_case__							,			name=snake_case__							)
        return tuple(snake_case__							)
   def       __a							(		self      :	str							,			snake_case__      :	List[Any]							,			snake_case__      :	str=None							):


        '''simple docstring'''
        UpperCAmelCase__						:    Optional[Any]							=							[self.bos_token_id] + token_ids_a + [self.eos_token_id]
        if token_ids_a is None:
             return output

        return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]
   def       __a							(		self      :	str							,			snake_case__      :	List[int]							,			snake_case__      :	Optional[List[int]] = None							):


        '''simple docstring'''
        UpperCAmelCase__						:    Optional[int]							=							[self.sep_token_id]
        UpperCAmelCase__						:    Optional[Any]							=							[self.cls_token_id]

        if token_ids_a is None:
             return len(cls + token_ids_a + sep							) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]




   def       __a							(		self      :	Any							,			snake_case__      :	Union[Dict[str, EncodedInput], BatchEncoding]							,			snake_case__      :	Optional[int] = None							,			snake_case__      :	PaddingStrategy = PaddingStrategy.DO_NOT_PAD							,			snake_case__      :	Optional[int] = None							,			snake_case__      :	Optional[bool] = None							,			):


        '''simple docstring'''
        UpperCAmelCase__						:    str							=							super()._pad(
            encoded_inputs=snake_case__							,			max_length=snake_case__							,			padding_strategy=snake_case__							,			pad_to_multiple_of=snake_case__							,			return_attention_mask=snake_case__							,			)

        # Load from model defaults
        if return_attention_mask is None:
             UpperCAmelCase__						:    Optional[int]							=							"attention_mask" in self.model_input_names

        if return_attention_mask and "global_attention_mask" in encoded_inputs:
             UpperCAmelCase__						:    List[Any]							=							encoded_inputs[self.model_input_names[0]]
             # `global_attention_mask` need to have the same length as other (sequential) inputs.
             UpperCAmelCase__						:    Any							=							len(encoded_inputs["global_attention_mask"]							) != len(snake_case__							)

             if needs_to_be_padded:
                  UpperCAmelCase__						:    List[str]							=							len(snake_case__							) - len(encoded_inputs["global_attention_mask"]							)

                  if self.padding_side == "right":
                       # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
                       UpperCAmelCase__						:    Dict							=							(
                           encoded_inputs["global_attention_mask"] + [-1] * difference
                       )
                  elif self.padding_side == "left":
                       UpperCAmelCase__						:    Dict							=							[-1] * difference + encoded_inputs[
                           "global_attention_mask"
                       ]
                  else:
                       raise ValueError("Invalid padding strategy:" + str(self.padding_side							)							)

        return encoded_inputs


