




"""simple docstring"""







import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_ta import TaTokenizer
else:
  SCREAMING_SNAKE_CASE_	     =       None


SCREAMING_SNAKE_CASE_	     =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_	     =       {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE_	     =       {
    'vocab_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/spiece.model',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/spiece.model',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/spiece.model',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/spiece.model',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/tokenizer.json',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/tokenizer.json',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/tokenizer.json',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/tokenizer.json',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/tokenizer.json',
    },
}


# TODO(PVP) - this should be removed in Transformers v5
SCREAMING_SNAKE_CASE_	     =       {
    't5-small': 512,
    't5-base': 512,
    't5-large': 512,
    't5-3b': 512,
    't5-11b': 512,
}

class 				snake_case_			(					lowerCamelCase_			):







   """simple docstring"""




   A_				      =       VOCAB_FILES_NAMES
   A_				      =       PRETRAINED_VOCAB_FILES_MAP
   A_				      =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A_				      =       ['''input_ids''', '''attention_mask''']
   A_				      =       TaTokenizer

   A_				      =       []







   def __init__(		self       ,					lowerCamelCase_=None       ,					lowerCamelCase_=None       ,					lowerCamelCase_="</s>"       ,					lowerCamelCase_="<unk>"       ,					lowerCamelCase_="<pad>"       ,					lowerCamelCase_=1_0_0       ,					lowerCamelCase_=None       ,					**lowerCamelCase_       ,					)				->   List[Any]:
       # Add extra_ids to the special token list
       if extra_ids > 0 and additional_special_tokens is None:
           UpperCamelCase              =	[F'<extra_id_{i}>' for i in range(lowerCamelCase_)]
       elif extra_ids > 0 and additional_special_tokens is not None:
           # Check that we have the right number of extra special tokens
           UpperCamelCase              =	len(set(filter(lambda lowerCamelCase_: bool('''extra_id_''' in str(lowerCamelCase_))       ,					lowerCamelCase_)))
           if extra_tokens != extra_ids:
               raise ValueError(
                   F'Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'
                   ''' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'''
                   ''' tokens''')

       super().__init__(
           lowerCamelCase_       ,					tokenizer_file=lowerCamelCase_       ,					eos_token=lowerCamelCase_       ,					unk_token=lowerCamelCase_       ,					pad_token=lowerCamelCase_       ,					extra_ids=lowerCamelCase_       ,					additional_special_tokens=lowerCamelCase_       ,					**lowerCamelCase_       ,					)

       UpperCamelCase              =	vocab_file
       UpperCamelCase              =	False if not self.vocab_file else True
       UpperCamelCase              =	extra_ids







   @staticmethod
   def   UpperCAmelCase__			(		lowerCamelCase_       ,					lowerCamelCase_       ,					lowerCamelCase_)				->   List[str]:
       if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
           UpperCamelCase              =	TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
           if init_max_model_length is not None and init_max_model_length != max_model_length:
               return init_max_model_length
           elif init_max_model_length is None:
               warnings.warn(
                   '''This tokenizer was incorrectly instantiated with a model max length of'''
                   F' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'
                   ''' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'''
                   ''' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'''
                   F' {pretrained_model_name_or_path} automatically truncating your input to'
                   F' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'
                   F' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'
                   ''' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'''
                   ''' instantiate this tokenizer with `model_max_length` set to your preferred value.'''       ,					lowerCamelCase_       ,					)

       return max_model_length







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None)				->   Tuple[str]:
       if not self.can_save_slow_tokenizer:
           raise ValueError(
               '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
               '''tokenizer.''')

       if not os.path.isdir(lowerCamelCase_):
           logger.error(F'Vocabulary path ({save_directory}) should be a directory')
           return
       UpperCamelCase              =	os.path.join(
           lowerCamelCase_       ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

       if os.path.abspath(self.vocab_file) != os.path.abspath(lowerCamelCase_):
           copyfile(self.vocab_file       ,					lowerCamelCase_)
           logger.info(F'Copy vocab file to {out_vocab_file}')

       return (out_vocab_file,)







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None)				->   List[int]:
       UpperCamelCase              =	token_ids_a + [self.eos_token_id]
       if token_ids_a is None:
           return self.prefix_tokens + token_ids_a
       else:
           UpperCamelCase              =	token_ids_a + [self.eos_token_id]
           return self.prefix_tokens + token_ids_a + token_ids_a







   def   UpperCAmelCase__			(		self       ,					lowerCamelCase_       ,					lowerCamelCase_ = None)				->   List[int]:
       UpperCamelCase              =	[self.eos_token_id]

       if token_ids_a is None:
           return len(token_ids_a + eos) * [0]
       return len(token_ids_a + eos + token_ids_a + eos) * [0]







   def   UpperCAmelCase__			(		self)				->   Optional[Any]:
       return list(
           set(filter(lambda lowerCamelCase_: bool(re.search(R'''<extra_id_\d+>'''       ,					lowerCamelCase_)) is not None       ,					self.additional_special_tokens)))







   def   UpperCAmelCase__			(		self)				->   List[str]:
       return [self.convert_tokens_to_ids(lowerCamelCase_) for token in self.get_sentinel_tokens()]



import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE						:List[str]			=  logging.get_logger(__name__)

SCREAMING_SNAKE_CASE						:Dict			=  {
    '''facebook/wav2vec2-base-960h''': '''https://huggingface.co/facebook/wav2vec2-base-960h/resolve/main/config.json''',
    # See all Wav2Vec2 models at https://huggingface.co/models?filter=wav2vec2
}


class      __lowerCAmelCase  (			a       ):






  """simple docstring"""





  _SCREAMING_SNAKE_CASE		     =							'wav2vec2'



  def __init__(  self							:							Any     ,					_lowerCAmelCase							:							Optional[Any]=3_2     ,					_lowerCAmelCase							:							List[Any]=7_6_8     ,					_lowerCAmelCase							:							Dict=1_2     ,					_lowerCAmelCase							:							Tuple=1_2     ,					_lowerCAmelCase							:							Optional[int]=3_0_7_2     ,					_lowerCAmelCase							:							Any="gelu"     ,					_lowerCAmelCase							:							List[str]=0.1     ,					_lowerCAmelCase							:							Dict=0.1     ,					_lowerCAmelCase							:							Union[str, Any]=0.1     ,					_lowerCAmelCase							:							Dict=0.0     ,					_lowerCAmelCase							:							Optional[int]=0.0     ,					_lowerCAmelCase							:							List[Any]=0.1     ,					_lowerCAmelCase							:							List[Any]=0.1     ,					_lowerCAmelCase							:							Dict=0.02     ,					_lowerCAmelCase							:							Any=1e-5     ,					_lowerCAmelCase							:							Any="group"     ,					_lowerCAmelCase							:							Union[str, Any]="gelu"     ,					_lowerCAmelCase							:							str=(5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2)     ,					_lowerCAmelCase							:							Optional[Any]=(5, 2, 2, 2, 2, 2, 2)     ,					_lowerCAmelCase							:							Optional[Any]=(1_0, 3, 3, 3, 3, 2, 2)     ,					_lowerCAmelCase							:							int=False     ,					_lowerCAmelCase							:							int=1_2_8     ,					_lowerCAmelCase							:							Union[str, Any]=1_6     ,					_lowerCAmelCase							:							Union[str, Any]=False     ,					_lowerCAmelCase							:							List[Any]=True     ,					_lowerCAmelCase							:							Dict=0.05     ,					_lowerCAmelCase							:							List[str]=1_0     ,					_lowerCAmelCase							:							Tuple=2     ,					_lowerCAmelCase							:							str=0.0     ,					_lowerCAmelCase							:							Union[str, Any]=1_0     ,					_lowerCAmelCase							:							int=0     ,					_lowerCAmelCase							:							Optional[int]=3_2_0     ,					_lowerCAmelCase							:							Optional[Any]=2     ,					_lowerCAmelCase							:							str=0.1     ,					_lowerCAmelCase							:							Optional[Any]=1_0_0     ,					_lowerCAmelCase							:							Tuple=2_5_6     ,					_lowerCAmelCase							:							Union[str, Any]=2_5_6     ,					_lowerCAmelCase							:							Union[str, Any]=0.1     ,					_lowerCAmelCase							:							Optional[Any]="sum"     ,					_lowerCAmelCase							:							List[str]=False     ,					_lowerCAmelCase							:							List[Any]=False     ,					_lowerCAmelCase							:							List[str]=2_5_6     ,					_lowerCAmelCase							:							List[Any]=(5_1_2, 5_1_2, 5_1_2, 5_1_2, 1_5_0_0)     ,					_lowerCAmelCase							:							List[str]=(5, 3, 3, 1, 1)     ,					_lowerCAmelCase							:							Union[str, Any]=(1, 2, 3, 1, 1)     ,					_lowerCAmelCase							:							Optional[Any]=5_1_2     ,					_lowerCAmelCase							:							Any=0     ,					_lowerCAmelCase							:							Optional[int]=1     ,					_lowerCAmelCase							:							List[str]=2     ,					_lowerCAmelCase							:							Tuple=False     ,					_lowerCAmelCase							:							Any=3     ,					_lowerCAmelCase							:							List[Any]=2     ,					_lowerCAmelCase							:							Optional[Any]=3     ,					_lowerCAmelCase							:							List[str]=None     ,					_lowerCAmelCase							:							str=None     ,					**_lowerCAmelCase							:							Tuple     ,					) ->			Optional[int]:




     """simple docstring"""
     super().__init__(**_lowerCAmelCase     ,					pad_token_id=_lowerCAmelCase     ,					bos_token_id=_lowerCAmelCase     ,					eos_token_id=_lowerCAmelCase	)
     snake_case_            =       hidden_size
     snake_case_            =       feat_extract_norm
     snake_case_            =       feat_extract_activation
     snake_case_            =       list(_lowerCAmelCase	)
     snake_case_            =       list(_lowerCAmelCase	)
     snake_case_            =       list(_lowerCAmelCase	)
     snake_case_            =       conv_bias
     snake_case_            =       num_conv_pos_embeddings
     snake_case_            =       num_conv_pos_embedding_groups
     snake_case_            =       len(self.conv_dim	)
     snake_case_            =       num_hidden_layers
     snake_case_            =       intermediate_size
     snake_case_            =       hidden_act
     snake_case_            =       num_attention_heads
     snake_case_            =       hidden_dropout
     snake_case_            =       attention_dropout
     snake_case_            =       activation_dropout
     snake_case_            =       feat_proj_dropout
     snake_case_            =       final_dropout
     snake_case_            =       layerdrop
     snake_case_            =       layer_norm_eps
     snake_case_            =       initializer_range
     snake_case_            =       vocab_size
     snake_case_            =       do_stable_layer_norm
     snake_case_            =       use_weighted_layer_sum

     if (
         (len(self.conv_stride	) != self.num_feat_extract_layers)
         or (len(self.conv_kernel	) != self.num_feat_extract_layers)
         or (len(self.conv_dim	) != self.num_feat_extract_layers)
     ):
        raise ValueError(
            "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
            " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
            F''' {len(self.conv_dim	)}`, `len(config.conv_stride) = {len(self.conv_stride	)}`,'''
            F''' `len(config.conv_kernel) = {len(self.conv_kernel	)}`.'''	)

     # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
     snake_case_            =       apply_spec_augment
     snake_case_            =       mask_time_prob
     snake_case_            =       mask_time_length
     snake_case_            =       mask_time_min_masks
     snake_case_            =       mask_feature_prob
     snake_case_            =       mask_feature_length
     snake_case_            =       mask_feature_min_masks

     # parameters for pretraining with codevector quantized representations
     snake_case_            =       num_codevectors_per_group
     snake_case_            =       num_codevector_groups
     snake_case_            =       contrastive_logits_temperature
     snake_case_            =       feat_quantizer_dropout
     snake_case_            =       num_negatives
     snake_case_            =       codevector_dim
     snake_case_            =       proj_codevector_dim
     snake_case_            =       diversity_loss_weight

     # ctc loss
     snake_case_            =       ctc_loss_reduction
     snake_case_            =       ctc_zero_infinity

     # adapter
     snake_case_            =       add_adapter
     snake_case_            =       adapter_kernel_size
     snake_case_            =       adapter_stride
     snake_case_            =       num_adapter_layers
     snake_case_            =       output_hidden_size or hidden_size
     snake_case_            =       adapter_attn_dim

     # SequenceClassification-specific parameter. Feel free to ignore for other classes.
     snake_case_            =       classifier_proj_size

     # XVector-specific parameters. Feel free to ignore for other classes.
     snake_case_            =       list(_lowerCAmelCase	)
     snake_case_            =       list(_lowerCAmelCase	)
     snake_case_            =       list(_lowerCAmelCase	)
     snake_case_            =       xvector_output_dim



  @property
  def        lowerCAmelCase__	(  self							:							str	) ->			int:




     """simple docstring"""
     return functools.reduce(operator.mul     ,					self.conv_stride     ,					1	)




