




'''simple docstring'''





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_pegasus import PegasusTokenizer
else:
   UpperCamelCase_		:     List[str]          =		None


UpperCamelCase_		:     str          =		logging.get_logger(__name__)


UpperCamelCase_		:     Optional[int]          =		'''▁'''

UpperCamelCase_		:     int          =		{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase_		:     Tuple          =		{
    '''vocab_file''': {'''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'''},
    '''tokenizer_file''': {
        '''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json'''
    },
}

UpperCamelCase_		:     str          =		{
    '''google/pegasus-xsum''': 512,
}

class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       Optional[int]       							=		VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE_				:       Tuple       							=		PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE_				:       int       							=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE_				:       Optional[Any]       							=		PegasusTokenizer
     SCREAMING_SNAKE_CASE_				:       Union[str, Any]       							=		["""input_ids""", """attention_mask"""]






     def __init__(					self   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE="<pad>"   ,_SCREAMING_SNAKE_CASE="</s>"   ,_SCREAMING_SNAKE_CASE="<unk>"   ,_SCREAMING_SNAKE_CASE="<mask_2>"   ,_SCREAMING_SNAKE_CASE="<mask_1>"   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=103   ,**_SCREAMING_SNAKE_CASE   ,)				->     int:
          _snake_case				= offset

          if additional_special_tokens is not None:
               if not isinstance(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       ):
                    raise TypeError(
                        f"""additional_special_tokens should be of type {type(_SCREAMING_SNAKE_CASE       )}, but is"""
                        f""" {type(_SCREAMING_SNAKE_CASE       )}"""       )

               _snake_case				= (
                   ([mask_token_sent] + additional_special_tokens)
                   if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
                   else additional_special_tokens
               )
               # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
               additional_special_tokens_extended += [
                   f"""<unk_{i}>""" for i in range(len(_SCREAMING_SNAKE_CASE       )   ,self.offset - 1       )
               ]

               if len(set(_SCREAMING_SNAKE_CASE       )       ) != len(_SCREAMING_SNAKE_CASE       ):
                    raise ValueError(
                        "Please make sure that the provided additional_special_tokens do not contain an incorrectly"
                        f""" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."""       )
               _snake_case				= additional_special_tokens_extended
          else:
               _snake_case				= [mask_token_sent] if mask_token_sent is not None else []
               additional_special_tokens += [f"""<unk_{i}>""" for i in range(2   ,self.offset       )]

          super().__init__(
              _SCREAMING_SNAKE_CASE   ,tokenizer_file=_SCREAMING_SNAKE_CASE   ,pad_token=_SCREAMING_SNAKE_CASE   ,eos_token=_SCREAMING_SNAKE_CASE   ,unk_token=_SCREAMING_SNAKE_CASE   ,mask_token=_SCREAMING_SNAKE_CASE   ,mask_token_sent=_SCREAMING_SNAKE_CASE   ,offset=_SCREAMING_SNAKE_CASE   ,additional_special_tokens=_SCREAMING_SNAKE_CASE   ,**_SCREAMING_SNAKE_CASE   ,)
          _snake_case				= vocab_file
          _snake_case				= False if not self.vocab_file else True






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     Union[str, Any]:
          _snake_case				= set(self.all_special_ids       )  # call it once instead of inside list comp
          all_special_ids.remove(self.unk_token_id       )  # <unk> is only sometimes special

          if all_special_ids != set(range(len(self.additional_special_tokens       ) + 3       )       ):
               raise ValueError(
                   "There should be 3 special tokens: mask_token, pad_token, and eos_token +"
                   f""" {len(self.additional_special_tokens       )} additional_special_tokens, but got {all_special_ids}"""       )

          return [1 if x in all_special_ids else 0 for x in seq]






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE = None   ,_SCREAMING_SNAKE_CASE = False       )				->     List[int]:
          if already_has_special_tokens:
               return self._special_token_mask(_SCREAMING_SNAKE_CASE       )
          elif token_ids_a is None:
               return self._special_token_mask(_SCREAMING_SNAKE_CASE       ) + [1]
          else:
               return self._special_token_mask(token_ids_a + token_ids_a       ) + [1]






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE=None       )				->     List[int]:
          if token_ids_a is None:
               return token_ids_a + [self.eos_token_id]
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return token_ids_a + token_ids_a + [self.eos_token_id]






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE = None       )				->     Tuple[str]:
          if not self.can_save_slow_tokenizer:
               raise ValueError(
                   "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                   "tokenizer."       )

          if not os.path.isdir(_SCREAMING_SNAKE_CASE       ):
               logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""       )
               return
          _snake_case				= os.path.join(
              _SCREAMING_SNAKE_CASE   ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )

          if os.path.abspath(self.vocab_file       ) != os.path.abspath(_SCREAMING_SNAKE_CASE       ):
               copyfile(self.vocab_file   ,_SCREAMING_SNAKE_CASE       )

          return (out_vocab_file,)








'''simple docstring'''





import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase_		:     Optional[Any]          =		logging.get_logger(__name__)

UpperCamelCase_		:     Any          =		{
    '''microsoft/unispeech-sat-base-100h-libri-ft''': (
        '''https://huggingface.co/microsoft/unispeech-sat-base-100h-libri-ft/resolve/main/config.json'''
    ),
    # See all UniSpeechSat models at https://huggingface.co/models?filter=unispeech_sat
}

class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       str       							=		"""unispeech-sat"""






     def __init__(					self   ,_SCREAMING_SNAKE_CASE=32   ,_SCREAMING_SNAKE_CASE=768   ,_SCREAMING_SNAKE_CASE=12   ,_SCREAMING_SNAKE_CASE=12   ,_SCREAMING_SNAKE_CASE=3_072   ,_SCREAMING_SNAKE_CASE="gelu"   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.0_2   ,_SCREAMING_SNAKE_CASE=1e-5   ,_SCREAMING_SNAKE_CASE="group"   ,_SCREAMING_SNAKE_CASE="gelu"   ,_SCREAMING_SNAKE_CASE=(512, 512, 512, 512, 512, 512, 512)   ,_SCREAMING_SNAKE_CASE=(5, 2, 2, 2, 2, 2, 2)   ,_SCREAMING_SNAKE_CASE=(10, 3, 3, 3, 3, 2, 2)   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=128   ,_SCREAMING_SNAKE_CASE=16   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=0.0_5   ,_SCREAMING_SNAKE_CASE=10   ,_SCREAMING_SNAKE_CASE=2   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=10   ,_SCREAMING_SNAKE_CASE=0   ,_SCREAMING_SNAKE_CASE=320   ,_SCREAMING_SNAKE_CASE=2   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=100   ,_SCREAMING_SNAKE_CASE=256   ,_SCREAMING_SNAKE_CASE=256   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE="mean"   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=256   ,_SCREAMING_SNAKE_CASE=(512, 512, 512, 512, 1_500)   ,_SCREAMING_SNAKE_CASE=(5, 3, 3, 1, 1)   ,_SCREAMING_SNAKE_CASE=(1, 2, 3, 1, 1)   ,_SCREAMING_SNAKE_CASE=512   ,_SCREAMING_SNAKE_CASE=0   ,_SCREAMING_SNAKE_CASE=1   ,_SCREAMING_SNAKE_CASE=2   ,_SCREAMING_SNAKE_CASE=504   ,**_SCREAMING_SNAKE_CASE   ,)				->     Any:
          super().__init__(**_SCREAMING_SNAKE_CASE   ,pad_token_id=_SCREAMING_SNAKE_CASE   ,bos_token_id=_SCREAMING_SNAKE_CASE   ,eos_token_id=_SCREAMING_SNAKE_CASE       )
          _snake_case				= hidden_size
          _snake_case				= feat_extract_norm
          _snake_case				= feat_extract_activation
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= conv_bias
          _snake_case				= num_conv_pos_embeddings
          _snake_case				= num_conv_pos_embedding_groups
          _snake_case				= len(self.conv_dim       )
          _snake_case				= num_hidden_layers
          _snake_case				= intermediate_size
          _snake_case				= hidden_act
          _snake_case				= num_attention_heads
          _snake_case				= hidden_dropout
          _snake_case				= attention_dropout
          _snake_case				= activation_dropout
          _snake_case				= feat_proj_dropout
          _snake_case				= final_dropout
          _snake_case				= layerdrop
          _snake_case				= layer_norm_eps
          _snake_case				= initializer_range
          _snake_case				= vocab_size
          _snake_case				= num_clusters
          _snake_case				= do_stable_layer_norm
          _snake_case				= use_weighted_layer_sum

          if (
              (len(self.conv_stride       ) != self.num_feat_extract_layers)
              or (len(self.conv_kernel       ) != self.num_feat_extract_layers)
              or (len(self.conv_dim       ) != self.num_feat_extract_layers)
          ):
               raise ValueError(
                   "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
                   " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
                   f""" {len(self.conv_dim       )}`, `len(config.conv_stride) = {len(self.conv_stride       )}`,"""
                   f""" `len(config.conv_kernel) = {len(self.conv_kernel       )}`."""       )

          # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
          _snake_case				= apply_spec_augment
          _snake_case				= mask_time_prob
          _snake_case				= mask_time_length
          _snake_case				= mask_time_min_masks
          _snake_case				= mask_feature_prob
          _snake_case				= mask_feature_length
          _snake_case				= mask_feature_min_masks

          # parameters for pretraining with codevector quantized representations
          _snake_case				= num_codevectors_per_group
          _snake_case				= num_codevector_groups
          _snake_case				= contrastive_logits_temperature
          _snake_case				= feat_quantizer_dropout
          _snake_case				= num_negatives
          _snake_case				= codevector_dim
          _snake_case				= proj_codevector_dim
          _snake_case				= diversity_loss_weight

          # ctc loss
          _snake_case				= ctc_loss_reduction
          _snake_case				= ctc_zero_infinity

          # SequenceClassification-specific parameter. Feel free to ignore for other classes.
          _snake_case				= classifier_proj_size

          # XVector-specific parameters. Feel free to ignore for other classes.
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= xvector_output_dim






     @property
     def 							_lowercase      (					self       )				->     List[str]:
          return functools.reduce(operator.mul   ,self.conv_stride   ,1       )


