






import inspect
import os
import re

from transformers.configuration_utils import PretrainedConfig
from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
UpperCamelCase    							=      'src/transformers'


# This is to make sure the transformers module imported is the one in the repo.
UpperCamelCase    							=      direct_transformers_import(PATH_TO_TRANSFORMERS)

UpperCamelCase    							=      transformers.models.auto.configuration_auto.CONFIG_MAPPING

UpperCamelCase    							=      {
    # used to compute the property `self.chunk_length`
    'EncodecConfig': ['overlap'],
    # used as `self.bert_model = BertModel(config, ...)`
    'DPRConfig': True,
    # not used in modeling files, but it's an important information
    'FSMTConfig': ['langs'],
    # used internally in the configuration class file
    'GPTNeoConfig': ['attention_types'],
    # used internally in the configuration class file
    'EsmConfig': ['is_folding_model'],
    # used during training (despite we don't have training script for these models yet)
    'Mask2FormerConfig': ['ignore_value'],
    # `ignore_value` used during training (despite we don't have training script for these models yet)
    # `norm` used in conversion script (despite not using in the modeling file)
    'OneFormerConfig': ['ignore_value', 'norm'],
    # used during preprocessing and collation, see `collating_graphormer.py`
    'GraphormerConfig': ['spatial_pos_max'],
    # used internally in the configuration class file
    'T5Config': ['feed_forward_proj'],
    # used internally in the configuration class file
    # `tokenizer_class` get default value `T5Tokenizer` intentionally
    'MT5Config': ['feed_forward_proj', 'tokenizer_class'],
    'UMT5Config': ['feed_forward_proj', 'tokenizer_class'],
    # used internally in the configuration class file
    'LongT5Config': ['feed_forward_proj'],
    # used internally in the configuration class file
    'SwitchTransformersConfig': ['feed_forward_proj'],
    # having default values other than `1e-5` - we can't fix them without breaking
    'BioGptConfig': ['layer_norm_eps'],
    # having default values other than `1e-5` - we can't fix them without breaking
    'GLPNConfig': ['layer_norm_eps'],
    # having default values other than `1e-5` - we can't fix them without breaking
    'SegformerConfig': ['layer_norm_eps'],
    # having default values other than `1e-5` - we can't fix them without breaking
    'CvtConfig': ['layer_norm_eps'],
    # having default values other than `1e-5` - we can't fix them without breaking
    'PerceiverConfig': ['layer_norm_eps'],
    # used internally to calculate the feature size
    'InformerConfig': ['num_static_real_features', 'num_time_features'],
    # used internally to calculate the feature size
    'TimeSeriesTransformerConfig': ['num_static_real_features', 'num_time_features'],
    # used internally to calculate the feature size
    'AutoformerConfig': ['num_static_real_features', 'num_time_features'],
    # used internally to calculate `mlp_dim`
    'SamVisionConfig': ['mlp_ratio'],
    # For (head) training, but so far not implemented
    'ClapAudioConfig': ['num_classes'],
    # Not used, but providing useful information to users
    'SpeechT5HifiGanConfig': ['sampling_rate'],
}


# TODO (ydshieh): Check the failing cases, try to fix them or move some cases to the above block once we are sure
SPECIAL_CASES_TO_ALLOW.update(
    {
        'CLIPSegConfig': True,
        'DeformableDetrConfig': True,
        'DetaConfig': True,
        'DinatConfig': True,
        'DonutSwinConfig': True,
        'EfficientFormerConfig': True,
        'FSMTConfig': True,
        'JukeboxConfig': True,
        'LayoutLMv2Config': True,
        'MaskFormerSwinConfig': True,
        'MT5Config': True,
        'NatConfig': True,
        'OneFormerConfig': True,
        'PerceiverConfig': True,
        'RagConfig': True,
        'SpeechT5Config': True,
        'SwinConfig': True,
        'Swin2SRConfig': True,
        'Swinv2Config': True,
        'SwitchTransformersConfig': True,
        'TableTransformerConfig': True,
        'TapasConfig': True,
        'TransfoXLConfig': True,
        'UniSpeechConfig': True,
        'UniSpeechSatConfig': True,
        'WavLMConfig': True,
        'WhisperConfig': True,
        # TODO: @Arthur (for `alignment_head` and `alignment_layer`)
        'JukeboxPriorConfig': True,
        # TODO: @Younes (for `is_decoder`)
        'Pix2StructTextConfig': True,
    }
)


def 				__lowerCamelCase     (							__lowerCAmelCase							:					Any					,							__lowerCAmelCase							:					str					,							__lowerCAmelCase							:					Tuple					,							__lowerCAmelCase							:					Optional[Any]							)     ->  Dict:
   __UpperCamelCase	:			Dict					=					False
   for attribute in attributes:
      for modeling_source in source_strings:
         # check if we can find `config.xxx`, `getattr(config, "xxx", ...)` or `getattr(self.config, "xxx", ...)`
         if (
             f'config.{attribute}' in modeling_source
             or f'getattr(config, "{attribute}"' in modeling_source
             or f'getattr(self.config, "{attribute}"' in modeling_source
         ):
            __UpperCamelCase	:			int					=					True
         # Deal with multi-line cases
         elif (
             re.search(
                 Rf'getattr[ \t\v\n\r\f]*\([ \t\v\n\r\f]*(self\.)?config,[ \t\v\n\r\f]*"{attribute}"'					,							__lowerCAmelCase					,							)
             is not None
         ):
            __UpperCamelCase	:			Dict					=					True
         # `SequenceSummary` is called with `SequenceSummary(config)`
         elif attribute in [
             "summary_type",
             "summary_use_proj",
             "summary_activation",
             "summary_last_dropout",
             "summary_proj_to_labels",
             "summary_first_dropout",
         ]:
            if "SequenceSummary" in modeling_source:
               __UpperCamelCase	:			str					=					True
         if attribute_used:
            break
      if attribute_used:
         break

    # common and important attributes, even if they do not always appear in the modeling files
   __UpperCamelCase	:			List[Any]					=					[
       """bos_index""",
       """eos_index""",
       """pad_index""",
       """unk_index""",
       """mask_index""",
       """image_size""",
       """use_cache""",
       """out_features""",
       """out_indices""",
   ]
   __UpperCamelCase	:			Dict					=					["""encoder_no_repeat_ngram_size"""]

   # Special cases to be allowed
   __UpperCamelCase	:			Any					=					True
   if not attribute_used:
      __UpperCamelCase	:			List[Any]					=					False
      for attribute in attributes:
         # Allow if the default value in the configuration class is different from the one in `PretrainedConfig`
         if attribute in ["is_encoder_decoder"] and default_value is True:
            __UpperCamelCase	:			Optional[Any]					=					True
         elif attribute in ["tie_word_embeddings"] and default_value is False:
            __UpperCamelCase	:			str					=					True

         # Allow cases without checking the default value in the configuration class
         elif attribute in attributes_to_allow + attributes_used_in_generation:
            __UpperCamelCase	:			List[Any]					=					True
         elif attribute.endswith("""_token_id"""							):
            __UpperCamelCase	:			List[str]					=					True

         # configuration class specific cases
         if not case_allowed:
            __UpperCamelCase	:			List[str]					=					SPECIAL_CASES_TO_ALLOW.get(config_class.__name__					,							[]							)
            __UpperCamelCase	:			Optional[Any]					=					allowed_cases is True or attribute in allowed_cases

   return attribute_used or case_allowed


def 				__lowerCamelCase     (							__lowerCAmelCase							:					int							)     ->  int:
   __UpperCamelCase	:			Union[str, Any]					=					dict(inspect.signature(config_class.__init__							).parameters							)
   __UpperCamelCase	:			int					=					[x for x in list(signature.keys()							) if x not in ["""self""", """kwargs"""]]
   __UpperCamelCase	:			Tuple					=					[signature[param].default for param in parameter_names]

   # If `attribute_map` exists, an attribute can have different names to be used in the modeling files, and as long
   # as one variant is used, the test should pass
   __UpperCamelCase	:			List[str]					=					{}
   if len(config_class.attribute_map							) > 0:
      __UpperCamelCase	:			Dict					=					{v: k for k, v in config_class.attribute_map.items()}

   # Get the path to modeling source files
   __UpperCamelCase	:			int					=					inspect.getsourcefile(__lowerCAmelCase							)
   __UpperCamelCase	:			Optional[int]					=					os.path.dirname(__lowerCAmelCase							)
   # Let's check against all frameworks: as long as one framework uses an attribute, we are good.
   __UpperCamelCase	:			List[str]					=					[os.path.join(__lowerCAmelCase					,							__lowerCAmelCase							) for fn in os.listdir(__lowerCAmelCase							) if fn.startswith("""modeling_"""							)]

   # Get the source code strings
   __UpperCamelCase	:			List[Any]					=					[]
   for path in modeling_paths:
      if os.path.isfile(__lowerCAmelCase							):
         with open(__lowerCAmelCase							) as fp:
            modeling_sources.append(fp.read()							)

   __UpperCamelCase	:			Optional[Any]					=					[]
   for config_param, default_value in zip(__lowerCAmelCase					,							__lowerCAmelCase							):
      # `attributes` here is all the variant names for `config_param`
      __UpperCamelCase	:			str					=					[config_param]
      # some configuration classes have non-empty `attribute_map`, and both names could be used in the
      # corresponding modeling files. As long as one of them appears, it is fine.
      if config_param in reversed_attribute_map:
         attributes.append(reversed_attribute_map[config_param]							)

      if not check_attribute_being_used(__lowerCAmelCase					,							__lowerCAmelCase					,							__lowerCAmelCase					,							__lowerCAmelCase							):
         unused_attributes.append(attributes[0]							)

   return sorted(__lowerCAmelCase							)


def 				__lowerCamelCase     (							)     ->  Union[str, Any]:
   __UpperCamelCase	:			Optional[int]					=					{}
   for _config_class in list(CONFIG_MAPPING.values()							):
      # Skip deprecated models
      if "models.deprecated" in _config_class.__module__:
         continue
      # Some config classes are not in `CONFIG_MAPPING` (e.g. `CLIPVisionConfig`, `Blip2VisionConfig`, etc.)
      __UpperCamelCase	:			Optional[Any]					=					[
          cls
          for name, cls in inspect.getmembers(
              inspect.getmodule(_config_class							)					,							lambda __lowerCAmelCase							: inspect.isclass(__lowerCAmelCase							)
              and issubclass(__lowerCAmelCase					,							__lowerCAmelCase							)
              and inspect.getmodule(__lowerCAmelCase							) == inspect.getmodule(_config_class							)					,							)
      ]
      for config_class in config_classes_in_module:
         __UpperCamelCase	:			Any					=					check_config_attributes_being_used(__lowerCAmelCase							)
         if len(__lowerCAmelCase							) > 0:
            __UpperCamelCase	:			Dict					=					unused_attributes

   if len(__lowerCAmelCase							) > 0:
      __UpperCamelCase	:			Optional[int]					=					"""The following configuration classes contain unused attributes in the corresponding modeling files:\n"""
      for name, attributes in configs_with_unused_attributes.items():
         error += f'{name}: {attributes}\n'

      raise ValueError(__lowerCAmelCase							)


if __name__ == "__main__":
       check_config_attributes()











import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


UpperCamelCase    							=      logging.getLogger(__name__)







class 		_A						( UpperCAmelCase_ ):

  def    a					(    self						:	Tuple   ,				lowerCamelCase__						:	Optional[int]   ,				lowerCamelCase__						:	Union[str, Any]   ,				lowerCamelCase__						:	str=None   ,				lowerCamelCase__						:	int=None			):
     """simple docstring"""

     __UpperCamelCase	:			Optional[Any]					=					self.layer[current_layer](lowerCamelCase__   ,				lowerCamelCase__   ,				head_mask[current_layer]			)

     __UpperCamelCase	:			Optional[Any]					=					layer_outputs[0]

     return hidden_states







@add_start_docstrings(
    '''The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''	,				UpperCAmelCase_	,				)
class 		_A						( UpperCAmelCase_ ):

  def __init__(    self						:	Tuple   ,				lowerCamelCase__						:	Tuple			):
     """simple docstring"""

     super().__init__(lowerCamelCase__			)

     __UpperCamelCase	:			List[str]					=					BertEncoderWithPabee(lowerCamelCase__			)

     self.init_weights()
     __UpperCamelCase	:			Optional[int]					=					0
     __UpperCamelCase	:			Tuple					=					0
     __UpperCamelCase	:			Any					=					0

     __UpperCamelCase	:			Optional[Any]					=					0

  def    a					(    self						:	Union[str, Any]   ,				lowerCamelCase__						:	Any			):
     """simple docstring"""

     __UpperCamelCase	:			List[Any]					=					threshold

  def    a					(    self						:	Optional[int]   ,				lowerCamelCase__						:	Union[str, Any]			):
     """simple docstring"""

     __UpperCamelCase	:			int					=					patience

  def    a					(    self						:	Optional[int]			):
     """simple docstring"""

     __UpperCamelCase	:			int					=					0
     __UpperCamelCase	:			Tuple					=					0

  def    a					(    self						:	int			):
     """simple docstring"""

     __UpperCamelCase	:			int					=					self.inference_layers_num / self.inference_instances_num
     __UpperCamelCase	:			str					=					(
         f'*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='
         f' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'
     )
     print(lowerCamelCase__			)

  @add_start_docstrings_to_model_forward(lowerCamelCase__			)
  def    a					(    self						:	Any   ,				lowerCamelCase__						:	List[Any]=None   ,				lowerCamelCase__						:	List[str]=None   ,				lowerCamelCase__						:	List[Any]=None   ,				lowerCamelCase__						:	Optional[Any]=None   ,				lowerCamelCase__						:	List[str]=None   ,				lowerCamelCase__						:	str=None   ,				lowerCamelCase__						:	List[Any]=None   ,				lowerCamelCase__						:	Any=None   ,				lowerCamelCase__						:	List[str]=None   ,				lowerCamelCase__						:	List[str]=None   ,				lowerCamelCase__						:	str=False   ,				):
     """simple docstring"""

     if input_ids is not None and inputs_embeds is not None:
        raise ValueError("""You cannot specify both input_ids and inputs_embeds at the same time"""			)
     elif input_ids is not None:
        __UpperCamelCase	:			Optional[Any]					=					input_ids.size()
     elif inputs_embeds is not None:
        __UpperCamelCase	:			Tuple					=					inputs_embeds.size()[:-1]
     else:
        raise ValueError("""You have to specify either input_ids or inputs_embeds"""			)

     __UpperCamelCase	:			Tuple					=					input_ids.device if input_ids is not None else inputs_embeds.device

     if attention_mask is None:
        __UpperCamelCase	:			Optional[Any]					=					torch.ones(lowerCamelCase__   ,				device=lowerCamelCase__			)
     if token_type_ids is None:
        __UpperCamelCase	:			str					=					torch.zeros(lowerCamelCase__   ,				dtype=torch.long   ,				device=lowerCamelCase__			)

     # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
     # ourselves in which case we just need to make it broadcastable to all heads.
     __UpperCamelCase	:			torch.Tensor					=					self.get_extended_attention_mask(lowerCamelCase__   ,				lowerCamelCase__   ,				lowerCamelCase__			)

     # If a 2D ou 3D attention mask is provided for the cross-attention
     # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
     if self.config.is_decoder and encoder_hidden_states is not None:
        __UpperCamelCase	,     __UpperCamelCase	,     __UpperCamelCase	:			Optional[Any]					=					encoder_hidden_states.size()
        __UpperCamelCase	:			Union[str, Any]					=					(encoder_batch_size, encoder_sequence_length)
        if encoder_attention_mask is None:
           __UpperCamelCase	:			Union[str, Any]					=					torch.ones(lowerCamelCase__   ,				device=lowerCamelCase__			)
        __UpperCamelCase	:			Optional[int]					=					self.invert_attention_mask(lowerCamelCase__			)
     else:
        __UpperCamelCase	:			str					=					None

     # Prepare head mask if needed
     # 1.0 in head_mask indicate we keep the head
     # attention_probs has shape bsz x n_heads x N x N
     # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
     # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
     __UpperCamelCase	:			int					=					self.get_head_mask(lowerCamelCase__   ,				self.config.num_hidden_layers			)

     __UpperCamelCase	:			Any					=					self.embeddings(
         input_ids=lowerCamelCase__   ,				position_ids=lowerCamelCase__   ,				token_type_ids=lowerCamelCase__   ,				inputs_embeds=lowerCamelCase__			)
     __UpperCamelCase	:			Optional[int]					=					embedding_output

     if self.training:
        __UpperCamelCase	:			List[Any]					=					[]
        for i in range(self.config.num_hidden_layers			):
           __UpperCamelCase	:			List[Any]					=					self.encoder.adaptive_forward(
               lowerCamelCase__   ,				current_layer=lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				head_mask=lowerCamelCase__			)

           __UpperCamelCase	:			Tuple					=					self.pooler(lowerCamelCase__			)
           __UpperCamelCase	:			str					=					output_layers[i](output_dropout(lowerCamelCase__			)			)
           res.append(lowerCamelCase__			)
     elif self.patience == 0:  # Use all layers for inference
        __UpperCamelCase	:			Union[str, Any]					=					self.encoder(
            lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				head_mask=lowerCamelCase__   ,				encoder_hidden_states=lowerCamelCase__   ,				encoder_attention_mask=lowerCamelCase__   ,				)
        __UpperCamelCase	:			Optional[int]					=					self.pooler(encoder_outputs[0]			)
        __UpperCamelCase	:			Union[str, Any]					=					[output_layers[self.config.num_hidden_layers - 1](lowerCamelCase__			)]
     else:
        __UpperCamelCase	:			Union[str, Any]					=					0
        __UpperCamelCase	:			Optional[Any]					=					None
        __UpperCamelCase	:			List[Any]					=					0
        for i in range(self.config.num_hidden_layers			):
           calculated_layer_num += 1
           __UpperCamelCase	:			Union[str, Any]					=					self.encoder.adaptive_forward(
               lowerCamelCase__   ,				current_layer=lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				head_mask=lowerCamelCase__			)

           __UpperCamelCase	:			str					=					self.pooler(lowerCamelCase__			)
           __UpperCamelCase	:			Any					=					output_layers[i](lowerCamelCase__			)
           if regression:
              __UpperCamelCase	:			Optional[int]					=					logits.detach()
              if patient_result is not None:
                 __UpperCamelCase	:			Dict					=					patient_result.detach()
              if (patient_result is not None) and torch.abs(patient_result - labels			) < self.regression_threshold:
                 patient_counter += 1
              else:
                 __UpperCamelCase	:			List[str]					=					0
           else:
              __UpperCamelCase	:			Optional[int]					=					logits.detach().argmax(dim=1			)
              if patient_result is not None:
                 __UpperCamelCase	:			Union[str, Any]					=					patient_result.detach().argmax(dim=1			)
              if (patient_result is not None) and torch.all(labels.eq(lowerCamelCase__			)			):
                 patient_counter += 1
              else:
                 __UpperCamelCase	:			str					=					0

           __UpperCamelCase	:			Union[str, Any]					=					logits
           if patient_counter == self.patience:
              break
        __UpperCamelCase	:			List[str]					=					[patient_result]
        self.inference_layers_num += calculated_layer_num
        self.inference_instances_num += 1

     return res







@add_start_docstrings(
    '''Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. '''	,				UpperCAmelCase_	,				)
class 		_A						( UpperCAmelCase_ ):

  def __init__(    self						:	Any   ,				lowerCamelCase__						:	Union[str, Any]			):
     """simple docstring"""

     super().__init__(lowerCamelCase__			)
     __UpperCamelCase	:			Dict					=					config.num_labels

     __UpperCamelCase	:			List[Any]					=					BertModelWithPabee(lowerCamelCase__			)
     __UpperCamelCase	:			Optional[int]					=					nn.Dropout(config.hidden_dropout_prob			)
     __UpperCamelCase	:			List[Any]					=					nn.ModuleList(
         [nn.Linear(config.hidden_size   ,				self.config.num_labels			) for _ in range(config.num_hidden_layers			)]			)

     self.init_weights()

  @add_start_docstrings_to_model_forward(lowerCamelCase__			)
  def    a					(    self						:	List[Any]   ,				lowerCamelCase__						:	Union[str, Any]=None   ,				lowerCamelCase__						:	Dict=None   ,				lowerCamelCase__						:	Union[str, Any]=None   ,				lowerCamelCase__						:	List[Any]=None   ,				lowerCamelCase__						:	Optional[Any]=None   ,				lowerCamelCase__						:	Optional[int]=None   ,				lowerCamelCase__						:	Dict=None   ,				):
     """simple docstring"""

     __UpperCamelCase	:			str					=					self.bert(
         input_ids=lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				token_type_ids=lowerCamelCase__   ,				position_ids=lowerCamelCase__   ,				head_mask=lowerCamelCase__   ,				inputs_embeds=lowerCamelCase__   ,				output_dropout=self.dropout   ,				output_layers=self.classifiers   ,				regression=self.num_labels == 1   ,				)

     __UpperCamelCase	:			List[str]					=					(logits[-1],)

     if labels is not None:
        __UpperCamelCase	:			Tuple					=					None
        __UpperCamelCase	:			Union[str, Any]					=					0
        for ix, logits_item in enumerate(lowerCamelCase__			):
           if self.num_labels == 1:
              #  We are doing regression
              __UpperCamelCase	:			Union[str, Any]					=					MSELoss()
              __UpperCamelCase	:			Optional[Any]					=					loss_fct(logits_item.view(-1			)   ,				labels.view(-1			)			)
           else:
              __UpperCamelCase	:			Dict					=					CrossEntropyLoss()
              __UpperCamelCase	:			Dict					=					loss_fct(logits_item.view(-1   ,				self.num_labels			)   ,				labels.view(-1			)			)
           if total_loss is None:
              __UpperCamelCase	:			Dict					=					loss
           else:
              total_loss += loss * (ix + 1)
           total_weights += ix + 1
        __UpperCamelCase	:			Tuple					=					(total_loss / total_weights,) + outputs

     return outputs



