
'''simple docstring'''





import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


SCREAMING_SNAKE_CASE     =       logging.getLogger(__name__)






class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''







 def 							UpperCamelCase  (					self		:  Optional[int]			,				snake_case__		:  Tuple			,				snake_case__		:  Dict			,				snake_case__		:  List[str]=None			,				snake_case__		:  Dict=None ):



   '''simple docstring'''




   UpperCAmelCase__      :					Any				       =							self.layer[current_layer](snake_case__			,				snake_case__			,				head_mask[current_layer] )

   UpperCAmelCase__      :					List[str]				       =							layer_outputs[0]

   return hidden_states






@add_start_docstrings(
    "The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top."					,				A					,				)
class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''







 def __init__(					self		:  Dict			,				snake_case__		:  Any ):



   '''simple docstring'''




   super().__init__(snake_case__ )

   UpperCAmelCase__      :					Optional[int]				       =							BertEncoderWithPabee(snake_case__ )

   self.init_weights()
   UpperCAmelCase__      :					List[Any]				       =							0
   UpperCAmelCase__      :					Any				       =							0
   UpperCAmelCase__      :					List[Any]				       =							0

   UpperCAmelCase__      :					str				       =							0







 def 							UpperCamelCase  (					self		:  Optional[int]			,				snake_case__		:  str ):



   '''simple docstring'''




   UpperCAmelCase__      :					Tuple				       =							threshold







 def 							UpperCamelCase  (					self		:  Optional[Any]			,				snake_case__		:  Dict ):



   '''simple docstring'''




   UpperCAmelCase__      :					Union[str, Any]				       =							patience







 def 							UpperCamelCase  (					self		:  List[str] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Dict				       =							0
   UpperCAmelCase__      :					Union[str, Any]				       =							0







 def 							UpperCamelCase  (					self		:  List[Any] ):



   '''simple docstring'''




   UpperCAmelCase__      :					Any				       =							self.inference_layers_num / self.inference_instances_num
   UpperCAmelCase__      :					List[str]				       =							(
       F"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
       F""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
   )
   print(snake_case__ )







 @add_start_docstrings_to_model_forward(snake_case__ )
 def 							UpperCamelCase  (					self		:  List[str]			,				snake_case__		:  int=None			,				snake_case__		:  List[str]=None			,				snake_case__		:  int=None			,				snake_case__		:  Optional[int]=None			,				snake_case__		:  Dict=None			,				snake_case__		:  List[str]=None			,				snake_case__		:  Tuple=None			,				snake_case__		:  Optional[Any]=None			,				snake_case__		:  Optional[int]=None			,				snake_case__		:  Dict=None			,				snake_case__		:  int=False			,				):



   '''simple docstring'''




   if input_ids is not None and inputs_embeds is not None:
     raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time" )
   elif input_ids is not None:
     UpperCAmelCase__      :					List[str]				       =							input_ids.size()
   elif inputs_embeds is not None:
     UpperCAmelCase__      :					Optional[int]				       =							inputs_embeds.size()[:-1]
   else:
     raise ValueError("You have to specify either input_ids or inputs_embeds" )

   UpperCAmelCase__      :					List[Any]				       =							input_ids.device if input_ids is not None else inputs_embeds.device

   if attention_mask is None:
     UpperCAmelCase__      :					Optional[Any]				       =							torch.ones(snake_case__			,				device=snake_case__ )
   if token_type_ids is None:
     UpperCAmelCase__      :					int				       =							torch.zeros(snake_case__			,				dtype=torch.long			,				device=snake_case__ )

   # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
   # ourselves in which case we just need to make it broadcastable to all heads.
   UpperCAmelCase__      :					torch.Tensor				       =							self.get_extended_attention_mask(snake_case__			,				snake_case__			,				snake_case__ )

   # If a 2D ou 3D attention mask is provided for the cross-attention
   # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
   if self.config.is_decoder and encoder_hidden_states is not None:
     UpperCAmelCase__       ,			UpperCAmelCase__       ,			UpperCAmelCase__      :					int				       =							encoder_hidden_states.size()
     UpperCAmelCase__      :					Optional[int]				       =							(encoder_batch_size, encoder_sequence_length)
     if encoder_attention_mask is None:
       UpperCAmelCase__      :					Tuple				       =							torch.ones(snake_case__			,				device=snake_case__ )
     UpperCAmelCase__      :					Optional[int]				       =							self.invert_attention_mask(snake_case__ )
   else:
     UpperCAmelCase__      :					int				       =							None

   # Prepare head mask if needed
   # 1.0 in head_mask indicate we keep the head
   # attention_probs has shape bsz x n_heads x N x N
   # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
   # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
   UpperCAmelCase__      :					Optional[Any]				       =							self.get_head_mask(snake_case__			,				self.config.num_hidden_layers )

   UpperCAmelCase__      :					Optional[Any]				       =							self.embeddings(
       input_ids=snake_case__			,				position_ids=snake_case__			,				token_type_ids=snake_case__			,				inputs_embeds=snake_case__ )
   UpperCAmelCase__      :					Union[str, Any]				       =							embedding_output

   if self.training:
     UpperCAmelCase__      :					Any				       =							[]
     for i in range(self.config.num_hidden_layers ):
       UpperCAmelCase__      :					Union[str, Any]				       =							self.encoder.adaptive_forward(
           snake_case__			,				current_layer=snake_case__			,				attention_mask=snake_case__			,				head_mask=snake_case__ )

       UpperCAmelCase__      :					str				       =							self.pooler(snake_case__ )
       UpperCAmelCase__      :					Dict				       =							output_layers[i](output_dropout(snake_case__ ) )
       res.append(snake_case__ )
   elif self.patience == 0:  # Use all layers for inference
     UpperCAmelCase__      :					Dict				       =							self.encoder(
         snake_case__			,				attention_mask=snake_case__			,				head_mask=snake_case__			,				encoder_hidden_states=snake_case__			,				encoder_attention_mask=snake_case__			,				)
     UpperCAmelCase__      :					str				       =							self.pooler(encoder_outputs[0] )
     UpperCAmelCase__      :					str				       =							[output_layers[self.config.num_hidden_layers - 1](snake_case__ )]
   else:
     UpperCAmelCase__      :					Tuple				       =							0
     UpperCAmelCase__      :					List[str]				       =							None
     UpperCAmelCase__      :					Any				       =							0
     for i in range(self.config.num_hidden_layers ):
       calculated_layer_num += 1
       UpperCAmelCase__      :					str				       =							self.encoder.adaptive_forward(
           snake_case__			,				current_layer=snake_case__			,				attention_mask=snake_case__			,				head_mask=snake_case__ )

       UpperCAmelCase__      :					List[str]				       =							self.pooler(snake_case__ )
       UpperCAmelCase__      :					List[Any]				       =							output_layers[i](snake_case__ )
       if regression:
         UpperCAmelCase__      :					Optional[Any]				       =							logits.detach()
         if patient_result is not None:
           UpperCAmelCase__      :					Any				       =							patient_result.detach()
         if (patient_result is not None) and torch.abs(patient_result - labels ) < self.regression_threshold:
           patient_counter += 1
         else:
           UpperCAmelCase__      :					int				       =							0
       else:
         UpperCAmelCase__      :					Dict				       =							logits.detach().argmax(dim=1 )
         if patient_result is not None:
           UpperCAmelCase__      :					Tuple				       =							patient_result.detach().argmax(dim=1 )
         if (patient_result is not None) and torch.all(labels.eq(snake_case__ ) ):
           patient_counter += 1
         else:
           UpperCAmelCase__      :					Optional[Any]				       =							0

       UpperCAmelCase__      :					Optional[int]				       =							logits
       if patient_counter == self.patience:
         break
     UpperCAmelCase__      :					int				       =							[patient_result]
     self.inference_layers_num += calculated_layer_num
     self.inference_instances_num += 1

   return res






@add_start_docstrings(
    "Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. "					,				A					,				)
class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''







 def __init__(					self		:  List[Any]			,				snake_case__		:  Union[str, Any] ):



   '''simple docstring'''




   super().__init__(snake_case__ )
   UpperCAmelCase__      :					List[str]				       =							config.num_labels

   UpperCAmelCase__      :					Any				       =							BertModelWithPabee(snake_case__ )
   UpperCAmelCase__      :					Union[str, Any]				       =							nn.Dropout(config.hidden_dropout_prob )
   UpperCAmelCase__      :					Optional[Any]				       =							nn.ModuleList(
       [nn.Linear(config.hidden_size			,				self.config.num_labels ) for _ in range(config.num_hidden_layers )] )

   self.init_weights()







 @add_start_docstrings_to_model_forward(snake_case__ )
 def 							UpperCamelCase  (					self		:  Tuple			,				snake_case__		:  Tuple=None			,				snake_case__		:  Any=None			,				snake_case__		:  Any=None			,				snake_case__		:  Optional[Any]=None			,				snake_case__		:  Tuple=None			,				snake_case__		:  Any=None			,				snake_case__		:  str=None			,				):



   '''simple docstring'''




   UpperCAmelCase__      :					Dict				       =							self.bert(
       input_ids=snake_case__			,				attention_mask=snake_case__			,				token_type_ids=snake_case__			,				position_ids=snake_case__			,				head_mask=snake_case__			,				inputs_embeds=snake_case__			,				output_dropout=self.dropout			,				output_layers=self.classifiers			,				regression=self.num_labels == 1			,				)

   UpperCAmelCase__      :					Tuple				       =							(logits[-1],)

   if labels is not None:
     UpperCAmelCase__      :					Any				       =							None
     UpperCAmelCase__      :					Dict				       =							0
     for ix, logits_item in enumerate(snake_case__ ):
       if self.num_labels == 1:
         #  We are doing regression
         UpperCAmelCase__      :					Optional[Any]				       =							MSELoss()
         UpperCAmelCase__      :					int				       =							loss_fct(logits_item.view(-1 )			,				labels.view(-1 ) )
       else:
         UpperCAmelCase__      :					List[str]				       =							CrossEntropyLoss()
         UpperCAmelCase__      :					Optional[Any]				       =							loss_fct(logits_item.view(-1			,				self.num_labels )			,				labels.view(-1 ) )
       if total_loss is None:
         UpperCAmelCase__      :					int				       =							loss
       else:
         total_loss += loss * (ix + 1)
       total_weights += ix + 1
     UpperCAmelCase__      :					Union[str, Any]				       =							(total_loss / total_weights,) + outputs

   return outputs





'''simple docstring'''





import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE     =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE     =       {
    """microsoft/wavlm-base""": """https://huggingface.co/microsoft/wavlm-base/resolve/main/config.json""",
    # See all WavLM models at https://huggingface.co/models?filter=wavlm
}






class 							UpperCAmelCase_							(    A				):

 '''simple docstring'''
 lowercase_					:    List[str]				=							"wavlm"







 def __init__(					self		:  Optional[int]			,				snake_case__		:  Dict=32			,				snake_case__		:  Optional[Any]=7_68			,				snake_case__		:  Tuple=12			,				snake_case__		:  List[Any]=12			,				snake_case__		:  List[str]=30_72			,				snake_case__		:  Tuple="gelu"			,				snake_case__		:  Optional[Any]=0.1			,				snake_case__		:  List[str]=0.1			,				snake_case__		:  Any=0.1			,				snake_case__		:  List[Any]=0.0			,				snake_case__		:  Dict=0.1			,				snake_case__		:  List[Any]=0.1			,				snake_case__		:  Union[str, Any]=0.02			,				snake_case__		:  int=1e-5			,				snake_case__		:  Dict="group"			,				snake_case__		:  Any="gelu"			,				snake_case__		:  List[Any]=(5_12, 5_12, 5_12, 5_12, 5_12, 5_12, 5_12)			,				snake_case__		:  int=(5, 2, 2, 2, 2, 2, 2)			,				snake_case__		:  Dict=(10, 3, 3, 3, 3, 2, 2)			,				snake_case__		:  Tuple=False			,				snake_case__		:  int=1_28			,				snake_case__		:  Dict=16			,				snake_case__		:  int=3_20			,				snake_case__		:  Dict=8_00			,				snake_case__		:  List[str]=False			,				snake_case__		:  Union[str, Any]=True			,				snake_case__		:  str=0.05			,				snake_case__		:  List[Any]=10			,				snake_case__		:  Tuple=2			,				snake_case__		:  Optional[int]=0.0			,				snake_case__		:  int=10			,				snake_case__		:  Any=3_20			,				snake_case__		:  Dict=2			,				snake_case__		:  Optional[int]=0.1			,				snake_case__		:  Union[str, Any]=1_00			,				snake_case__		:  Dict=2_56			,				snake_case__		:  List[Any]=2_56			,				snake_case__		:  List[str]=0.1			,				snake_case__		:  List[Any]="mean"			,				snake_case__		:  int=False			,				snake_case__		:  Any=False			,				snake_case__		:  Any=2_56			,				snake_case__		:  Union[str, Any]=(5_12, 5_12, 5_12, 5_12, 15_00)			,				snake_case__		:  Union[str, Any]=(5, 3, 3, 1, 1)			,				snake_case__		:  Optional[Any]=(1, 2, 3, 1, 1)			,				snake_case__		:  List[Any]=5_12			,				snake_case__		:  str=80			,				snake_case__		:  Any=0			,				snake_case__		:  str=1			,				snake_case__		:  Union[str, Any]=2			,				snake_case__		:  int=False			,				snake_case__		:  Union[str, Any]=3			,				snake_case__		:  Union[str, Any]=2			,				snake_case__		:  List[str]=3			,				snake_case__		:  List[Any]=None			,				**snake_case__		:  int			,				):



   '''simple docstring'''




   super().__init__(**snake_case__			,				pad_token_id=snake_case__			,				bos_token_id=snake_case__			,				eos_token_id=snake_case__ )
   UpperCAmelCase__      :					Union[str, Any]				       =							hidden_size
   UpperCAmelCase__      :					List[str]				       =							feat_extract_norm
   UpperCAmelCase__      :					int				       =							feat_extract_activation
   UpperCAmelCase__      :					Optional[int]				       =							list(snake_case__ )
   UpperCAmelCase__      :					Tuple				       =							list(snake_case__ )
   UpperCAmelCase__      :					Tuple				       =							list(snake_case__ )
   UpperCAmelCase__      :					Tuple				       =							conv_bias
   UpperCAmelCase__      :					List[Any]				       =							num_buckets
   UpperCAmelCase__      :					Union[str, Any]				       =							max_bucket_distance
   UpperCAmelCase__      :					Tuple				       =							num_conv_pos_embeddings
   UpperCAmelCase__      :					Union[str, Any]				       =							num_conv_pos_embedding_groups
   UpperCAmelCase__      :					Optional[Any]				       =							len(self.conv_dim )
   UpperCAmelCase__      :					Optional[int]				       =							num_hidden_layers
   UpperCAmelCase__      :					Tuple				       =							intermediate_size
   UpperCAmelCase__      :					List[str]				       =							hidden_act
   UpperCAmelCase__      :					List[str]				       =							num_attention_heads
   UpperCAmelCase__      :					Any				       =							hidden_dropout
   UpperCAmelCase__      :					List[Any]				       =							attention_dropout
   UpperCAmelCase__      :					Optional[int]				       =							activation_dropout
   UpperCAmelCase__      :					Optional[Any]				       =							feat_proj_dropout
   UpperCAmelCase__      :					List[str]				       =							final_dropout
   UpperCAmelCase__      :					int				       =							layerdrop
   UpperCAmelCase__      :					int				       =							layer_norm_eps
   UpperCAmelCase__      :					Tuple				       =							initializer_range
   UpperCAmelCase__      :					Union[str, Any]				       =							num_ctc_classes
   UpperCAmelCase__      :					Any				       =							vocab_size
   UpperCAmelCase__      :					Dict				       =							do_stable_layer_norm
   UpperCAmelCase__      :					List[Any]				       =							use_weighted_layer_sum
   UpperCAmelCase__      :					Optional[Any]				       =							classifier_proj_size

   if (
       (len(self.conv_stride ) != self.num_feat_extract_layers)
       or (len(self.conv_kernel ) != self.num_feat_extract_layers)
       or (len(self.conv_dim ) != self.num_feat_extract_layers)
   ):
     raise ValueError(
         "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
         " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
         F""" {len(self.conv_dim )}`, `len(config.conv_stride) = {len(self.conv_stride )}`,"""
         F""" `len(config.conv_kernel) = {len(self.conv_kernel )}`.""" )

   # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
   UpperCAmelCase__      :					List[str]				       =							apply_spec_augment
   UpperCAmelCase__      :					Union[str, Any]				       =							mask_time_prob
   UpperCAmelCase__      :					List[Any]				       =							mask_time_length
   UpperCAmelCase__      :					List[str]				       =							mask_time_min_masks
   UpperCAmelCase__      :					List[str]				       =							mask_feature_prob
   UpperCAmelCase__      :					Dict				       =							mask_feature_length

   # parameters for pretraining with codevector quantized representations
   UpperCAmelCase__      :					Dict				       =							num_codevectors_per_group
   UpperCAmelCase__      :					Union[str, Any]				       =							num_codevector_groups
   UpperCAmelCase__      :					int				       =							contrastive_logits_temperature
   UpperCAmelCase__      :					List[str]				       =							num_negatives
   UpperCAmelCase__      :					List[str]				       =							codevector_dim
   UpperCAmelCase__      :					List[Any]				       =							proj_codevector_dim
   UpperCAmelCase__      :					List[Any]				       =							diversity_loss_weight

   # ctc loss
   UpperCAmelCase__      :					Tuple				       =							ctc_loss_reduction
   UpperCAmelCase__      :					Any				       =							ctc_zero_infinity

   # adapter
   UpperCAmelCase__      :					int				       =							add_adapter
   UpperCAmelCase__      :					List[Any]				       =							adapter_kernel_size
   UpperCAmelCase__      :					Dict				       =							adapter_stride
   UpperCAmelCase__      :					Dict				       =							num_adapter_layers
   UpperCAmelCase__      :					Union[str, Any]				       =							output_hidden_size or hidden_size

   # SequenceClassification-specific parameter. Feel free to ignore for other classes.
   UpperCAmelCase__      :					str				       =							classifier_proj_size

   # XVector-specific parameters. Feel free to ignore for other classes.
   UpperCAmelCase__      :					Any				       =							list(snake_case__ )
   UpperCAmelCase__      :					List[Any]				       =							list(snake_case__ )
   UpperCAmelCase__      :					List[str]				       =							list(snake_case__ )
   UpperCAmelCase__      :					Dict				       =							xvector_output_dim







 @property
 def 							UpperCamelCase  (					self		:  int ):



   '''simple docstring'''




   return functools.reduce(operator.mul			,				self.conv_stride			,				1 )



