


from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy
@add_start_docstrings(
    'The RoBERTa Model transformer with early exiting (DeeRoBERTa). '   ,snake_case_   ,)
class 		__magic_name__						(snake_case_				):





     '''simple docstring'''



     __lowercase				:							Tuple	   =       RobertaConfig
     __lowercase				:							List[str]	   =       'roberta'

     def __init__(					self:List[str]     ,		_a:Union[str, Any]   ):
       super().__init__(_a   )

       snake_case__						= RobertaEmbeddings(_a   )
       self.init_weights()




@add_start_docstrings(
    'RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. '   ,snake_case_   ,)
class 		__magic_name__						(snake_case_				):





     '''simple docstring'''



     __lowercase				:							List[str]	   =       RobertaConfig
     __lowercase				:							int	   =       'roberta'

     def __init__(					self:str     ,		_a:Optional[Any]   ):
       super().__init__(_a   )
       snake_case__						= config.num_labels
       snake_case__						= config.num_hidden_layers

       snake_case__						= DeeRobertaModel(_a   )
       snake_case__						= nn.Dropout(config.hidden_dropout_prob   )
       snake_case__						= nn.Linear(config.hidden_size     ,		self.config.num_labels   )







     @add_start_docstrings_to_model_forward(_a   )
     def 	SCREAMING_SNAKE_CASE__						(					self:str     ,		_a:Any=None     ,		_a:Any=None     ,		_a:str=None     ,		_a:Optional[Any]=None     ,		_a:Union[str, Any]=None     ,		_a:Optional[Any]=None     ,		_a:Dict=None     ,		_a:str=-1     ,		_a:Optional[int]=False     ,		):
       snake_case__						= self.num_layers
       try:
         snake_case__						= self.roberta(
             _a     ,		attention_mask=_a     ,		token_type_ids=_a     ,		position_ids=_a     ,		head_mask=_a     ,		inputs_embeds=_a     ,		)

         snake_case__						= outputs[1]

         snake_case__						= self.dropout(_a   )
         snake_case__						= self.classifier(_a   )
         snake_case__						= (logits,) + outputs[2:]  # add hidden states and attention if they are here
       except HighwayException as e:
         snake_case__						= e.message
         snake_case__						= e.exit_layer
         snake_case__						= outputs[0]

       if not self.training:
         snake_case__						= entropy(_a   )
         snake_case__						= []
         snake_case__						= []
       if labels is not None:
         if self.num_labels == 1:
           #  We are doing regression
           snake_case__						= MSELoss()
           snake_case__						= loss_fct(logits.view(-1   )     ,		labels.view(-1   )   )
         else:
           snake_case__						= CrossEntropyLoss()
           snake_case__						= loss_fct(logits.view(-1     ,		self.num_labels   )     ,		labels.view(-1   )   )

         # work with highway exits
         snake_case__						= []
         for highway_exit in outputs[-1]:
           snake_case__						= highway_exit[0]
           if not self.training:
             highway_logits_all.append(_a   )
             highway_entropy.append(highway_exit[2]   )
           if self.num_labels == 1:
             #  We are doing regression
             snake_case__						= MSELoss()
             snake_case__						= loss_fct(highway_logits.view(-1   )     ,		labels.view(-1   )   )
           else:
             snake_case__						= CrossEntropyLoss()
             snake_case__						= loss_fct(highway_logits.view(-1     ,		self.num_labels   )     ,		labels.view(-1   )   )
           highway_losses.append(_a   )

         if train_highway:
           snake_case__						= (sum(highway_losses[:-1]   ),) + outputs
           # exclude the final highway, of course
         else:
           snake_case__						= (loss,) + outputs
       if not self.training:
         snake_case__						= outputs + ((original_entropy, highway_entropy), exit_layer)
         if output_layer >= 0:
           snake_case__						= (
               (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
           )  # use the highway of the last layer

       return outputs  # (loss), logits, (hidden_states), (attentions), entropy



"""simple docstring"""





import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_a								=						logging.get_logger(__name__)

_a								=						{
    """microsoft/wavlm-base""": """https://huggingface.co/microsoft/wavlm-base/resolve/main/config.json""",
    # See all WavLM models at https://huggingface.co/models?filter=wavlm
}






class        _UpperCAmelCase(       lowerCamelCase      ):
		lowercase__         = 'wavlm'



		def __init__(   self							,						__a=32							,						__a=7_68							,						__a=12							,						__a=12							,						__a=30_72							,						__a="gelu"							,						__a=0.1							,						__a=0.1							,						__a=0.1							,						__a=0.0							,						__a=0.1							,						__a=0.1							,						__a=0.02							,						__a=1e-5							,						__a="group"							,						__a="gelu"							,						__a=(5_12, 5_12, 5_12, 5_12, 5_12, 5_12, 5_12)							,						__a=(5, 2, 2, 2, 2, 2, 2)							,						__a=(10, 3, 3, 3, 3, 2, 2)							,						__a=False							,						__a=1_28							,						__a=16							,						__a=3_20							,						__a=8_00							,						__a=False							,						__a=True							,						__a=0.05							,						__a=10							,						__a=2							,						__a=0.0							,						__a=10							,						__a=3_20							,						__a=2							,						__a=0.1							,						__a=1_00							,						__a=2_56							,						__a=2_56							,						__a=0.1							,						__a="mean"							,						__a=False							,						__a=False							,						__a=2_56							,						__a=(5_12, 5_12, 5_12, 5_12, 15_00)							,						__a=(5, 3, 3, 1, 1)							,						__a=(1, 2, 3, 1, 1)							,						__a=5_12							,						__a=80							,						__a=0							,						__a=1							,						__a=2							,						__a=False							,						__a=3							,						__a=2							,						__a=3							,						__a=None							,						**__a							,						)			->			Union[str, Any]:






							'''simple docstring'''
							super().__init__(**__a							,						pad_token_id=__a							,						bos_token_id=__a							,						eos_token_id=__a)
							_UpperCamelCase									=    hidden_size
							_UpperCamelCase									=    feat_extract_norm
							_UpperCamelCase									=    feat_extract_activation
							_UpperCamelCase									=    list(__a)
							_UpperCamelCase									=    list(__a)
							_UpperCamelCase									=    list(__a)
							_UpperCamelCase									=    conv_bias
							_UpperCamelCase									=    num_buckets
							_UpperCamelCase									=    max_bucket_distance
							_UpperCamelCase									=    num_conv_pos_embeddings
							_UpperCamelCase									=    num_conv_pos_embedding_groups
							_UpperCamelCase									=    len(self.conv_dim)
							_UpperCamelCase									=    num_hidden_layers
							_UpperCamelCase									=    intermediate_size
							_UpperCamelCase									=    hidden_act
							_UpperCamelCase									=    num_attention_heads
							_UpperCamelCase									=    hidden_dropout
							_UpperCamelCase									=    attention_dropout
							_UpperCamelCase									=    activation_dropout
							_UpperCamelCase									=    feat_proj_dropout
							_UpperCamelCase									=    final_dropout
							_UpperCamelCase									=    layerdrop
							_UpperCamelCase									=    layer_norm_eps
							_UpperCamelCase									=    initializer_range
							_UpperCamelCase									=    num_ctc_classes
							_UpperCamelCase									=    vocab_size
							_UpperCamelCase									=    do_stable_layer_norm
							_UpperCamelCase									=    use_weighted_layer_sum
							_UpperCamelCase									=    classifier_proj_size

							if (
							    (len(self.conv_stride) != self.num_feat_extract_layers)
							    or (len(self.conv_kernel) != self.num_feat_extract_layers)
							    or (len(self.conv_dim) != self.num_feat_extract_layers)
							):
												raise ValueError(
												    '''Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='''
												    ''' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='''
												    F''' {len(self.conv_dim)}`, `len(config.conv_stride) = {len(self.conv_stride)}`,'''
												    F''' `len(config.conv_kernel) = {len(self.conv_kernel)}`.''')

							# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
							_UpperCamelCase									=    apply_spec_augment
							_UpperCamelCase									=    mask_time_prob
							_UpperCamelCase									=    mask_time_length
							_UpperCamelCase									=    mask_time_min_masks
							_UpperCamelCase									=    mask_feature_prob
							_UpperCamelCase									=    mask_feature_length

							# parameters for pretraining with codevector quantized representations
							_UpperCamelCase									=    num_codevectors_per_group
							_UpperCamelCase									=    num_codevector_groups
							_UpperCamelCase									=    contrastive_logits_temperature
							_UpperCamelCase									=    num_negatives
							_UpperCamelCase									=    codevector_dim
							_UpperCamelCase									=    proj_codevector_dim
							_UpperCamelCase									=    diversity_loss_weight

							# ctc loss
							_UpperCamelCase									=    ctc_loss_reduction
							_UpperCamelCase									=    ctc_zero_infinity

							# adapter
							_UpperCamelCase									=    add_adapter
							_UpperCamelCase									=    adapter_kernel_size
							_UpperCamelCase									=    adapter_stride
							_UpperCamelCase									=    num_adapter_layers
							_UpperCamelCase									=    output_hidden_size or hidden_size

							# SequenceClassification-specific parameter. Feel free to ignore for other classes.
							_UpperCamelCase									=    classifier_proj_size

							# XVector-specific parameters. Feel free to ignore for other classes.
							_UpperCamelCase									=    list(__a)
							_UpperCamelCase									=    list(__a)
							_UpperCamelCase									=    list(__a)
							_UpperCamelCase									=    xvector_output_dim




		@property
		def 		UpperCAmelCase				(   self)			->			Optional[Any]:






							'''simple docstring'''
							return functools.reduce(operator.mul							,						self.conv_stride							,						1)






