




'''simple docstring'''





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase_		:     Optional[int]          =		logging.get_logger(__name__)

UpperCamelCase_		:     int          =		'''▁'''

UpperCamelCase_		:     Optional[int]          =		{'''vocab_file''': '''sentencepiece.bpe.model'''}

UpperCamelCase_		:     Union[str, Any]          =		{
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

UpperCamelCase_		:     int          =		{
    '''facebook/mbart-large-50-one-to-many-mmt''': 1024,
}

# fmt: off
UpperCamelCase_		:     Any          =		['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']

class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       List[str]       							=		VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE_				:       Any       							=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE_				:       str       							=		PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE_				:       Union[str, Any]       							=		["""input_ids""", """attention_mask"""]

     SCREAMING_SNAKE_CASE_				:       List[int]       							=		[]
     SCREAMING_SNAKE_CASE_				:       List[int]       							=		[]






     def __init__(					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE="</s>"   ,_SCREAMING_SNAKE_CASE="</s>"   ,_SCREAMING_SNAKE_CASE="<s>"   ,_SCREAMING_SNAKE_CASE="<unk>"   ,_SCREAMING_SNAKE_CASE="<pad>"   ,_SCREAMING_SNAKE_CASE="<mask>"   ,_SCREAMING_SNAKE_CASE = None   ,**_SCREAMING_SNAKE_CASE   ,)				->     None:
          # Mask token behave like a normal word, i.e. include the space before it
          _snake_case				= AddedToken(_SCREAMING_SNAKE_CASE   ,lstrip=_SCREAMING_SNAKE_CASE   ,rstrip=_SCREAMING_SNAKE_CASE       ) if isinstance(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       ) else mask_token

          _snake_case				= {} if sp_model_kwargs is None else sp_model_kwargs

          _snake_case				= kwargs.get("additional_special_tokens"   ,[]       )
          kwargs["additional_special_tokens"] += [
              code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
          ]

          super().__init__(
              src_lang=_SCREAMING_SNAKE_CASE   ,tgt_lang=_SCREAMING_SNAKE_CASE   ,eos_token=_SCREAMING_SNAKE_CASE   ,unk_token=_SCREAMING_SNAKE_CASE   ,sep_token=_SCREAMING_SNAKE_CASE   ,cls_token=_SCREAMING_SNAKE_CASE   ,pad_token=_SCREAMING_SNAKE_CASE   ,mask_token=_SCREAMING_SNAKE_CASE   ,sp_model_kwargs=self.sp_model_kwargs   ,**_SCREAMING_SNAKE_CASE   ,)

          _snake_case				= spm.SentencePieceProcessor(**self.sp_model_kwargs       )
          self.sp_model.Load(str(_SCREAMING_SNAKE_CASE       )       )
          _snake_case				= vocab_file

          # Original fairseq vocab and spm vocab must be "aligned":
          # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
          # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
          # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
          # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

          # Mimic fairseq token-to-id alignment for the first 4 token
          _snake_case				= {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

          # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
          _snake_case				= 1

          _snake_case				= len(self.sp_model       )
          _snake_case				= {
              code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_SCREAMING_SNAKE_CASE       )
          }
          _snake_case				= {v: k for k, v in self.lang_code_to_id.items()}
          _snake_case				= len(self.sp_model       ) + len(self.lang_code_to_id       ) + self.fairseq_offset

          self.fairseq_tokens_to_ids.update(self.lang_code_to_id       )
          _snake_case				= {v: k for k, v in self.fairseq_tokens_to_ids.items()}

          _snake_case				= src_lang if src_lang is not None else "en_XX"
          _snake_case				= self.lang_code_to_id[self._src_lang]
          _snake_case				= tgt_lang
          self.set_src_lang_special_tokens(self._src_lang       )






     @property
     def 							_lowercase      (					self       )				->     int:
          return len(self.sp_model       ) + len(self.lang_code_to_id       ) + self.fairseq_offset + 1  # Plus 1 for the mask token






     @property
     def 							_lowercase      (					self       )				->     str:
          return self._src_lang






     @src_lang.setter
     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     None:
          _snake_case				= new_src_lang
          self.set_src_lang_special_tokens(self._src_lang       )






     def __getstate__(					self       )				->     Dict:
          _snake_case				= self.__dict__.copy()
          _snake_case				= None
          return state






     def __setstate__(					self   ,_SCREAMING_SNAKE_CASE       )				->     None:
          _snake_case				= d

          # for backward compatibility
          if not hasattr(self   ,"sp_model_kwargs"       ):
               _snake_case				= {}

          _snake_case				= spm.SentencePieceProcessor(**self.sp_model_kwargs       )
          self.sp_model.Load(self.vocab_file       )






     def 							_lowercase      (					self       )				->     Dict:
          _snake_case				= {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE       ): i for i in range(self.vocab_size       )}
          vocab.update(self.added_tokens_encoder       )
          return vocab






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     List[str]:
          return self.sp_model.encode(_SCREAMING_SNAKE_CASE   ,out_type=_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     int:
          if token in self.fairseq_tokens_to_ids:
               return self.fairseq_tokens_to_ids[token]
          _snake_case				= self.sp_model.PieceToId(_SCREAMING_SNAKE_CASE       )

          # Need to return unknown token if the SP model returned 0
          return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     str:
          if index in self.fairseq_ids_to_tokens:
               return self.fairseq_ids_to_tokens[index]
          return self.sp_model.IdToPiece(index - self.fairseq_offset       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     List[Any]:
          _snake_case				= []
          _snake_case				= ""
          _snake_case				= False
          for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                    if not prev_is_special:
                         out_string += " "
                    out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE       ) + token
                    _snake_case				= True
                    _snake_case				= []
               else:
                    current_sub_tokens.append(_SCREAMING_SNAKE_CASE       )
                    _snake_case				= False
          out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE       )
          return out_string.strip()






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE = None       )				->     Tuple[str]:
          if not os.path.isdir(_SCREAMING_SNAKE_CASE       ):
               logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""       )
               return
          _snake_case				= os.path.join(
              _SCREAMING_SNAKE_CASE   ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )

          if os.path.abspath(self.vocab_file       ) != os.path.abspath(_SCREAMING_SNAKE_CASE       ) and os.path.isfile(self.vocab_file       ):
               copyfile(self.vocab_file   ,_SCREAMING_SNAKE_CASE       )
          elif not os.path.isfile(self.vocab_file       ):
               with open(_SCREAMING_SNAKE_CASE   ,"wb"       ) as fi:
                    _snake_case				= self.sp_model.serialized_model_proto()
                    fi.write(_SCREAMING_SNAKE_CASE       )

          return (out_vocab_file,)






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE = None   ,_SCREAMING_SNAKE_CASE = False       )				->     List[int]:

          if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=_SCREAMING_SNAKE_CASE   ,token_ids_a=_SCREAMING_SNAKE_CASE   ,already_has_special_tokens=_SCREAMING_SNAKE_CASE       )

          _snake_case				= [1] * len(self.prefix_tokens       )
          _snake_case				= [1] * len(self.suffix_tokens       )
          if token_ids_a is None:
               return prefix_ones + ([0] * len(_SCREAMING_SNAKE_CASE       )) + suffix_ones
          return prefix_ones + ([0] * len(_SCREAMING_SNAKE_CASE       )) + ([0] * len(_SCREAMING_SNAKE_CASE       )) + suffix_ones






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE = None       )				->     List[int]:
          if token_ids_a is None:
               return self.prefix_tokens + token_ids_a + self.suffix_tokens
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,**_SCREAMING_SNAKE_CASE       )				->     Any:
          if src_lang is None or tgt_lang is None:
               raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"       )
          _snake_case				= src_lang
          _snake_case				= self(_SCREAMING_SNAKE_CASE   ,add_special_tokens=_SCREAMING_SNAKE_CASE   ,return_tensors=_SCREAMING_SNAKE_CASE   ,**_SCREAMING_SNAKE_CASE       )
          _snake_case				= self.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE       )
          _snake_case				= tgt_lang_id
          return inputs






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE = "en_XX"   ,_SCREAMING_SNAKE_CASE = None   ,_SCREAMING_SNAKE_CASE = "ro_RO"   ,**_SCREAMING_SNAKE_CASE   ,)				->     BatchEncoding:
          _snake_case				= src_lang
          _snake_case				= tgt_lang
          return super().prepare_seqaseq_batch(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE   ,**_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self       )				->     Optional[int]:
          return self.set_src_lang_special_tokens(self.src_lang       )






     def 							_lowercase      (					self       )				->     Tuple:
          return self.set_tgt_lang_special_tokens(self.tgt_lang       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     None:
          _snake_case				= self.lang_code_to_id[src_lang]
          _snake_case				= [self.cur_lang_code_id]
          _snake_case				= [self.eos_token_id]






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     None:
          _snake_case				= self.lang_code_to_id[tgt_lang]
          _snake_case				= [self.cur_lang_code_id]
          _snake_case				= [self.eos_token_id]








'''simple docstring'''





import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase_		:     Optional[Any]          =		logging.get_logger(__name__)

UpperCamelCase_		:     Any          =		{
    '''microsoft/unispeech-sat-base-100h-libri-ft''': (
        '''https://huggingface.co/microsoft/unispeech-sat-base-100h-libri-ft/resolve/main/config.json'''
    ),
    # See all UniSpeechSat models at https://huggingface.co/models?filter=unispeech_sat
}

class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       str       							=		"""unispeech-sat"""






     def __init__(					self   ,_SCREAMING_SNAKE_CASE=32   ,_SCREAMING_SNAKE_CASE=768   ,_SCREAMING_SNAKE_CASE=12   ,_SCREAMING_SNAKE_CASE=12   ,_SCREAMING_SNAKE_CASE=3_072   ,_SCREAMING_SNAKE_CASE="gelu"   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=0.0_2   ,_SCREAMING_SNAKE_CASE=1e-5   ,_SCREAMING_SNAKE_CASE="group"   ,_SCREAMING_SNAKE_CASE="gelu"   ,_SCREAMING_SNAKE_CASE=(512, 512, 512, 512, 512, 512, 512)   ,_SCREAMING_SNAKE_CASE=(5, 2, 2, 2, 2, 2, 2)   ,_SCREAMING_SNAKE_CASE=(10, 3, 3, 3, 3, 2, 2)   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=128   ,_SCREAMING_SNAKE_CASE=16   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=0.0_5   ,_SCREAMING_SNAKE_CASE=10   ,_SCREAMING_SNAKE_CASE=2   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=10   ,_SCREAMING_SNAKE_CASE=0   ,_SCREAMING_SNAKE_CASE=320   ,_SCREAMING_SNAKE_CASE=2   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE=100   ,_SCREAMING_SNAKE_CASE=256   ,_SCREAMING_SNAKE_CASE=256   ,_SCREAMING_SNAKE_CASE=0.1   ,_SCREAMING_SNAKE_CASE="mean"   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=256   ,_SCREAMING_SNAKE_CASE=(512, 512, 512, 512, 1_500)   ,_SCREAMING_SNAKE_CASE=(5, 3, 3, 1, 1)   ,_SCREAMING_SNAKE_CASE=(1, 2, 3, 1, 1)   ,_SCREAMING_SNAKE_CASE=512   ,_SCREAMING_SNAKE_CASE=0   ,_SCREAMING_SNAKE_CASE=1   ,_SCREAMING_SNAKE_CASE=2   ,_SCREAMING_SNAKE_CASE=504   ,**_SCREAMING_SNAKE_CASE   ,)				->     Any:
          super().__init__(**_SCREAMING_SNAKE_CASE   ,pad_token_id=_SCREAMING_SNAKE_CASE   ,bos_token_id=_SCREAMING_SNAKE_CASE   ,eos_token_id=_SCREAMING_SNAKE_CASE       )
          _snake_case				= hidden_size
          _snake_case				= feat_extract_norm
          _snake_case				= feat_extract_activation
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= conv_bias
          _snake_case				= num_conv_pos_embeddings
          _snake_case				= num_conv_pos_embedding_groups
          _snake_case				= len(self.conv_dim       )
          _snake_case				= num_hidden_layers
          _snake_case				= intermediate_size
          _snake_case				= hidden_act
          _snake_case				= num_attention_heads
          _snake_case				= hidden_dropout
          _snake_case				= attention_dropout
          _snake_case				= activation_dropout
          _snake_case				= feat_proj_dropout
          _snake_case				= final_dropout
          _snake_case				= layerdrop
          _snake_case				= layer_norm_eps
          _snake_case				= initializer_range
          _snake_case				= vocab_size
          _snake_case				= num_clusters
          _snake_case				= do_stable_layer_norm
          _snake_case				= use_weighted_layer_sum

          if (
              (len(self.conv_stride       ) != self.num_feat_extract_layers)
              or (len(self.conv_kernel       ) != self.num_feat_extract_layers)
              or (len(self.conv_dim       ) != self.num_feat_extract_layers)
          ):
               raise ValueError(
                   "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
                   " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
                   f""" {len(self.conv_dim       )}`, `len(config.conv_stride) = {len(self.conv_stride       )}`,"""
                   f""" `len(config.conv_kernel) = {len(self.conv_kernel       )}`."""       )

          # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
          _snake_case				= apply_spec_augment
          _snake_case				= mask_time_prob
          _snake_case				= mask_time_length
          _snake_case				= mask_time_min_masks
          _snake_case				= mask_feature_prob
          _snake_case				= mask_feature_length
          _snake_case				= mask_feature_min_masks

          # parameters for pretraining with codevector quantized representations
          _snake_case				= num_codevectors_per_group
          _snake_case				= num_codevector_groups
          _snake_case				= contrastive_logits_temperature
          _snake_case				= feat_quantizer_dropout
          _snake_case				= num_negatives
          _snake_case				= codevector_dim
          _snake_case				= proj_codevector_dim
          _snake_case				= diversity_loss_weight

          # ctc loss
          _snake_case				= ctc_loss_reduction
          _snake_case				= ctc_zero_infinity

          # SequenceClassification-specific parameter. Feel free to ignore for other classes.
          _snake_case				= classifier_proj_size

          # XVector-specific parameters. Feel free to ignore for other classes.
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= list(_SCREAMING_SNAKE_CASE       )
          _snake_case				= xvector_output_dim






     @property
     def 							_lowercase      (					self       )				->     List[str]:
          return functools.reduce(operator.mul   ,self.conv_stride   ,1       )


