


import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


snake_case							:       Any										=	logging.get_logger(__name__)


snake_case							:       Any										=	{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

# See all MVP models at https://huggingface.co/models?filter=mvp
snake_case							:       Tuple										=	{
    '''vocab_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json''',
    },
    '''added_tokens.json''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json''',
    },
    '''merges_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json''',
    },
}

snake_case							:       str										=	{
    '''RUCAIBox/mvp''': 10_24,
}







class 						_snake_case    ( _snake_case				):
 SCREAMING_SNAKE_CASE__ =			VOCAB_FILES_NAMES
 SCREAMING_SNAKE_CASE__ =			PRETRAINED_VOCAB_FILES_MAP
 SCREAMING_SNAKE_CASE__ =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 SCREAMING_SNAKE_CASE__ =			['input_ids', 'attention_mask']
 SCREAMING_SNAKE_CASE__ =			MvpTokenizer




 def __init__(						self   ,    _lowerCamelCase=None   ,    _lowerCamelCase=None   ,    _lowerCamelCase=None   ,    _lowerCamelCase="replace"   ,    _lowerCamelCase="<s>"   ,    _lowerCamelCase="</s>"   ,    _lowerCamelCase="</s>"   ,    _lowerCamelCase="<s>"   ,    _lowerCamelCase="<unk>"   ,    _lowerCamelCase="<pad>"   ,    _lowerCamelCase="<mask>"   ,    _lowerCamelCase=False   ,    _lowerCamelCase=True   ,    **_lowerCamelCase   ,    ):
       super().__init__(
           _lowerCamelCase   ,    _lowerCamelCase   ,    tokenizer_file=_lowerCamelCase   ,    errors=_lowerCamelCase   ,    bos_token=_lowerCamelCase   ,    eos_token=_lowerCamelCase   ,    sep_token=_lowerCamelCase   ,    cls_token=_lowerCamelCase   ,    unk_token=_lowerCamelCase   ,    pad_token=_lowerCamelCase   ,    mask_token=_lowerCamelCase   ,    add_prefix_space=_lowerCamelCase   ,    trim_offsets=_lowerCamelCase   ,    **_lowerCamelCase   ,    )

       a		:Union[str, Any]									=			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()       )
       if pre_tok_state.get('''add_prefix_space'''   ,    _lowerCamelCase       ) != add_prefix_space:
             a		:Dict									=			getattr(_lowerCamelCase   ,    pre_tok_state.pop('''type'''       )       )
             a		:str									=			add_prefix_space
             a		:Union[str, Any]									=			pre_tok_class(**_lowerCamelCase       )

       a		:Any									=			add_prefix_space

       # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
       a		:Dict									=			'''post_processor'''
       a		:List[str]									=			getattr(self.backend_tokenizer   ,    _lowerCamelCase   ,    _lowerCamelCase       )
       if tokenizer_component_instance:
             a		:List[str]									=			json.loads(tokenizer_component_instance.__getstate__()       )

             # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
             if "sep" in state:
                   a		:Dict									=			tuple(state['''sep''']       )
             if "cls" in state:
                   a		:Optional[Any]									=			tuple(state['''cls''']       )

             a		:Dict									=			False

             if state.get('''add_prefix_space'''   ,    _lowerCamelCase       ) != add_prefix_space:
                   a		:Tuple									=			add_prefix_space
                   a		:Dict									=			True

             if state.get('''trim_offsets'''   ,    _lowerCamelCase       ) != trim_offsets:
                   a		:Any									=			trim_offsets
                   a		:Union[str, Any]									=			True

             if changes_to_apply:
                   a		:List[Any]									=			getattr(_lowerCamelCase   ,    state.pop('''type'''       )       )
                   a		:Optional[int]									=			component_class(**_lowerCamelCase       )
                   setattr(self.backend_tokenizer   ,    _lowerCamelCase   ,    _lowerCamelCase       )




 @property
 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       if self._mask_token is None:
             if self.verbose:
                   logger.error('''Using mask_token, but it is not set yet.'''       )
             return None
       return str(self._mask_token       )




 @mask_token.setter
 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase       ):
       a		:Optional[Any]									=			AddedToken(_lowerCamelCase   ,    lstrip=_lowerCamelCase   ,    rstrip=_lowerCamelCase       ) if isinstance(_lowerCamelCase   ,    _lowerCamelCase       ) else value
       a		:Dict									=			value




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    *_lowerCamelCase   ,    **_lowerCamelCase       ):
       a		:List[Any]									=			kwargs.get('''is_split_into_words'''   ,    _lowerCamelCase       )

       if is_split_into_words and not self.add_prefix_space:
             raise ValueError(
                 F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                 '''to use it with pretokenized inputs.'''       )

       return super()._batch_encode_plus(*_lowerCamelCase   ,    **_lowerCamelCase       )




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    *_lowerCamelCase   ,    **_lowerCamelCase       ):
       a		:Dict									=			kwargs.get('''is_split_into_words'''   ,    _lowerCamelCase       )

       if is_split_into_words and not self.add_prefix_space:
             raise ValueError(
                 F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                 '''to use it with pretokenized inputs.'''       )

       return super()._encode_plus(*_lowerCamelCase   ,    **_lowerCamelCase       )




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase = None       ):
       a		:str									=			self._tokenizer.model.save(_lowerCamelCase   ,    name=_lowerCamelCase       )
       return tuple(_lowerCamelCase       )




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase=None       ):
       a		:Tuple									=			[self.bos_token_id] + token_ids_a + [self.eos_token_id]
       if token_ids_a is None:
             return output

       return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase = None       ):
       a		:Optional[int]									=			[self.sep_token_id]
       a		:Optional[int]									=			[self.cls_token_id]

       if token_ids_a is None:
             return len(cls + token_ids_a + sep       ) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]









import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case							:       Optional[int]										=	logging.get_logger(__name__)

snake_case							:       Optional[Any]										=	{
    '''microsoft/wavlm-base''': '''https://huggingface.co/microsoft/wavlm-base/resolve/main/config.json''',
    # See all WavLM models at https://huggingface.co/models?filter=wavlm
}







class 						_snake_case    ( _snake_case				):
 SCREAMING_SNAKE_CASE__ =			'wavlm'




 def __init__(						self   ,    _lowerCamelCase=32   ,    _lowerCamelCase=768   ,    _lowerCamelCase=12   ,    _lowerCamelCase=12   ,    _lowerCamelCase=3072   ,    _lowerCamelCase="gelu"   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.0   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.02   ,    _lowerCamelCase=1e-5   ,    _lowerCamelCase="group"   ,    _lowerCamelCase="gelu"   ,    _lowerCamelCase=(512, 512, 512, 512, 512, 512, 512)   ,    _lowerCamelCase=(5, 2, 2, 2, 2, 2, 2)   ,    _lowerCamelCase=(10, 3, 3, 3, 3, 2, 2)   ,    _lowerCamelCase=False   ,    _lowerCamelCase=128   ,    _lowerCamelCase=16   ,    _lowerCamelCase=320   ,    _lowerCamelCase=800   ,    _lowerCamelCase=False   ,    _lowerCamelCase=True   ,    _lowerCamelCase=0.05   ,    _lowerCamelCase=10   ,    _lowerCamelCase=2   ,    _lowerCamelCase=0.0   ,    _lowerCamelCase=10   ,    _lowerCamelCase=320   ,    _lowerCamelCase=2   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=100   ,    _lowerCamelCase=256   ,    _lowerCamelCase=256   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase="mean"   ,    _lowerCamelCase=False   ,    _lowerCamelCase=False   ,    _lowerCamelCase=256   ,    _lowerCamelCase=(512, 512, 512, 512, 1500)   ,    _lowerCamelCase=(5, 3, 3, 1, 1)   ,    _lowerCamelCase=(1, 2, 3, 1, 1)   ,    _lowerCamelCase=512   ,    _lowerCamelCase=80   ,    _lowerCamelCase=0   ,    _lowerCamelCase=1   ,    _lowerCamelCase=2   ,    _lowerCamelCase=False   ,    _lowerCamelCase=3   ,    _lowerCamelCase=2   ,    _lowerCamelCase=3   ,    _lowerCamelCase=None   ,    **_lowerCamelCase   ,    ):
       super().__init__(**_lowerCamelCase   ,    pad_token_id=_lowerCamelCase   ,    bos_token_id=_lowerCamelCase   ,    eos_token_id=_lowerCamelCase       )
       a		:Union[str, Any]									=			hidden_size
       a		:List[Any]									=			feat_extract_norm
       a		:Tuple									=			feat_extract_activation
       a		:Tuple									=			list(_lowerCamelCase       )
       a		:Optional[Any]									=			list(_lowerCamelCase       )
       a		:Optional[Any]									=			list(_lowerCamelCase       )
       a		:int									=			conv_bias
       a		:Tuple									=			num_buckets
       a		:Tuple									=			max_bucket_distance
       a		:int									=			num_conv_pos_embeddings
       a		:List[str]									=			num_conv_pos_embedding_groups
       a		:Tuple									=			len(self.conv_dim       )
       a		:Any									=			num_hidden_layers
       a		:Tuple									=			intermediate_size
       a		:Any									=			hidden_act
       a		:List[str]									=			num_attention_heads
       a		:int									=			hidden_dropout
       a		:Optional[Any]									=			attention_dropout
       a		:Dict									=			activation_dropout
       a		:Any									=			feat_proj_dropout
       a		:Dict									=			final_dropout
       a		:int									=			layerdrop
       a		:Tuple									=			layer_norm_eps
       a		:Union[str, Any]									=			initializer_range
       a		:Union[str, Any]									=			num_ctc_classes
       a		:Optional[Any]									=			vocab_size
       a		:str									=			do_stable_layer_norm
       a		:Dict									=			use_weighted_layer_sum
       a		:Any									=			classifier_proj_size

       if (
           (len(self.conv_stride       ) != self.num_feat_extract_layers)
           or (len(self.conv_kernel       ) != self.num_feat_extract_layers)
           or (len(self.conv_dim       ) != self.num_feat_extract_layers)
       ):
             raise ValueError(
                 '''Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='''
                 ''' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='''
                 F''' {len(self.conv_dim       )}`, `len(config.conv_stride) = {len(self.conv_stride       )}`,'''
                 F''' `len(config.conv_kernel) = {len(self.conv_kernel       )}`.'''       )

       # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
       a		:List[Any]									=			apply_spec_augment
       a		:Optional[int]									=			mask_time_prob
       a		:Optional[Any]									=			mask_time_length
       a		:Tuple									=			mask_time_min_masks
       a		:Dict									=			mask_feature_prob
       a		:List[Any]									=			mask_feature_length

       # parameters for pretraining with codevector quantized representations
       a		:Tuple									=			num_codevectors_per_group
       a		:List[str]									=			num_codevector_groups
       a		:List[Any]									=			contrastive_logits_temperature
       a		:Union[str, Any]									=			num_negatives
       a		:Union[str, Any]									=			codevector_dim
       a		:Dict									=			proj_codevector_dim
       a		:Union[str, Any]									=			diversity_loss_weight

       # ctc loss
       a		:Dict									=			ctc_loss_reduction
       a		:Optional[int]									=			ctc_zero_infinity

       # adapter
       a		:Any									=			add_adapter
       a		:str									=			adapter_kernel_size
       a		:Tuple									=			adapter_stride
       a		:Dict									=			num_adapter_layers
       a		:Any									=			output_hidden_size or hidden_size

       # SequenceClassification-specific parameter. Feel free to ignore for other classes.
       a		:Tuple									=			classifier_proj_size

       # XVector-specific parameters. Feel free to ignore for other classes.
       a		:List[Any]									=			list(_lowerCamelCase       )
       a		:Optional[int]									=			list(_lowerCamelCase       )
       a		:List[str]									=			list(_lowerCamelCase       )
       a		:Optional[int]									=			xvector_output_dim




 @property
 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       return functools.reduce(operator.mul   ,    self.conv_stride   ,    1       )





