

import collections

from .utils import ExplicitEnum, is_torch_available, logging


if is_torch_available():
    import torch


lowerCamelCase_    =					logging.get_logger(__name__)







class  __lowerCamelCase				:







      def __init__(     self							,  lowerCamelCase							,  lowerCamelCase=21							,  lowerCamelCase=[]							,  lowerCamelCase=None   )						->       Optional[Any]:
        snake_case_       						=      model
        snake_case_       						=      trace_batch_nums
        snake_case_       						=      abort_after_batch_num

        # keep a LIFO buffer of frames to dump as soon as inf/nan is encountered to give context to the problem emergence
        snake_case_       						=      collections.deque([]							,  lowerCamelCase   )
        snake_case_       						=      []
        snake_case_       						=      0
        snake_case_       						=      0
        snake_case_       						=      False
        snake_case_       						=      """                 """

        self.analyse_model()

        self.register_forward_hook()







      def 	lowerCAmelCase_    (     self							,  lowerCamelCase=None   )						->       Dict:
        if frame is not None:
          self.expand_frame(lowerCamelCase   )
        self.frames.append("""\n""".join(self.frame   )   )
        snake_case_       						=      []  # start a new frame







      def 	lowerCAmelCase_    (     self							,  lowerCamelCase   )						->       Optional[Any]:
        self.frame.append(lowerCamelCase   )







      def 	lowerCAmelCase_    (     self   )						->       str:
        print("""\n""".join(self.frames   )   )
        snake_case_       						=      []







      def 	lowerCAmelCase_    (     self   )						->       str:
        snake_case_       						=      []







      def 	lowerCAmelCase_    (     self   )						->       Optional[Any]:
        print(f'''\nDetected inf/nan during batch_number={self.batch_number}'''   )
        print(f'''Last {len(self.frames   )} forward frames:'''   )
        print(f'''{"abs min":8} {"abs max":8} metadata'''   )
        print("""\n""".join(self.frames   )   )
        print("""\n\n"""   )
        snake_case_       						=      []







      def 	lowerCAmelCase_    (     self   )						->       List[Any]:
        # extract the fully qualified module names, to be able to report at run time. e.g.:
        # encoder.block.2.layer.0.SelfAttention.o
        #
        # for shared weights only the first shared module name will be registered
        snake_case_       						=      {m: name for name, m in self.model.named_modules()}
        # self.longest_module_name = max(len(v) for v in self.module_names.values())







      def 	lowerCAmelCase_    (     self							,  lowerCamelCase							,  lowerCamelCase   )						->       Optional[int]:
        if torch.is_tensor(lowerCamelCase   ):
          self.expand_frame(get_abs_min_max(lowerCamelCase							,  lowerCamelCase   )   )
          if detect_overflow(lowerCamelCase							,  lowerCamelCase   ):
            snake_case_       						=      True
        elif var is None:
          self.expand_frame(f'''{"None":>17} {ctx}'''   )
        else:
          self.expand_frame(f'''{"not a tensor":>17} {ctx}'''   )







      def 	lowerCAmelCase_    (     self   )						->       Union[str, Any]:
        self.expand_frame(f'''\n\n{self.prefix} *** Starting batch number={self.batch_number} ***'''   )
        self.expand_frame(f'''{"abs min":8} {"abs max":8} metadata'''   )







      def 	lowerCAmelCase_    (     self   )						->       Optional[int]:
        self.expand_frame(f'''{self.prefix} *** Finished batch number={self.batch_number-1} ***\n\n'''   )







      def 	lowerCAmelCase_    (     self							,  lowerCamelCase							,  lowerCamelCase							,  lowerCamelCase   )						->       Optional[int]:
        self.expand_frame(f'''{self.prefix} {self.module_names[module]} {module.__class__.__name__}'''   )

        # params
        for name, p in module.named_parameters(recurse=lowerCamelCase   ):
          self.analyse_variable(lowerCamelCase							,  lowerCamelCase   )

        # inputs
        if isinstance(lowerCamelCase							,  lowerCamelCase   ):
          for i, x in enumerate(lowerCamelCase   ):
            self.analyse_variable(lowerCamelCase							,  f'''input[{i}]'''   )
        else:
          self.analyse_variable(lowerCamelCase							,  """input"""   )

        # outputs
        if isinstance(lowerCamelCase							,  lowerCamelCase   ):
          for i, x in enumerate(lowerCamelCase   ):
            # possibly a tuple of tuples
            if isinstance(lowerCamelCase							,  lowerCamelCase   ):
              for j, y in enumerate(lowerCamelCase   ):
                self.analyse_variable(lowerCamelCase							,  f'''output[{i}][{j}]'''   )
            else:
              self.analyse_variable(lowerCamelCase							,  f'''output[{i}]'''   )
        else:
          self.analyse_variable(lowerCamelCase							,  """output"""   )

        self.save_frame()







      def 	lowerCAmelCase_    (     self   )						->       str:
        self.model.apply(self._register_forward_hook   )







      def 	lowerCAmelCase_    (     self							,  lowerCamelCase   )						->       Dict:
        module.register_forward_hook(self.forward_hook   )







      def 	lowerCAmelCase_    (     self							,  lowerCamelCase							,  lowerCamelCase							,  lowerCamelCase   )						->       List[Any]:
        # - input is a tuple of packed inputs (could be non-Tensors)
        # - output could be a Tensor or a tuple of Tensors and non-Tensors

        snake_case_       						=      False

        snake_case_       						=      True if self.batch_number in self.trace_batch_nums else False
        if trace_mode:
          self.reset_saved_frames()

        if self.total_calls == 0:
          self.batch_start_frame()
        self.total_calls += 1

        # count batch numbers - the very first forward hook of the batch will be called when the
        # batch completes - i.e. it gets called very last - we know this batch has finished
        if module == self.model:
          self.batch_number += 1
          snake_case_       						=      True

        self.create_frame(lowerCamelCase							,  lowerCamelCase							,  lowerCamelCase   )

        # if last_frame_of_batch:
        #     self.batch_end_frame()

        if trace_mode:
          self.trace_frames()

        if last_frame_of_batch:
          self.batch_start_frame()

        if self.detected_overflow and not trace_mode:
          self.dump_saved_frames()

          # now we can abort, as it's pointless to continue running
          raise ValueError(
              """DebugUnderflowOverflow: inf/nan detected, aborting as there is no point running further. """
              """Please scroll up above this traceback to see the activation values prior to this event."""   )

        # abort after certain batch if requested to do so
        if self.abort_after_batch_num is not None and self.batch_number > self.abort_after_batch_num:
          raise ValueError(
              f'''DebugUnderflowOverflow: aborting after {self.batch_number} batches due to'''
              f''' `abort_after_batch_num={self.abort_after_batch_num}` arg'''   )



def    UpperCamelCase(					lowercase_			,					lowercase_							)	->				Dict:





  '''simple docstring'''





  snake_case_       						=      var.abs()
  return f'''{abs_var.min():8.2e} {abs_var.max():8.2e} {ctx}'''



def    UpperCamelCase(					lowercase_			,					lowercase_							)	->				Union[str, Any]:





  '''simple docstring'''





  snake_case_       						=      False
  if torch.isnan(lowercase_							).any().item():
    snake_case_       						=      True
    print(f'''{ctx} has nans'''							)
  if torch.isinf(lowercase_							).any().item():
    snake_case_       						=      True
    print(f'''{ctx} has infs'''							)

  # if needed to monitor large elements can enable the following
  if 0:  # and detected:
    snake_case_       						=      var[torch.ge(var.abs()			,					100							)]
    if naaa.numel() > 0:
      print(f'''{ctx}:  n100={naaa.numel()}'''							)
    snake_case_       						=      var[torch.ge(var.abs()			,					1000							)]
    if naaaa.numel() > 0:
      print(f'''{ctx}: n1000={naaaa.numel()}'''							)
    snake_case_       						=      var[torch.ge(var.abs()			,					10000							)]
    if naaaaa.numel() > 0:
      print(f'''{ctx}: n10000={naaaaa.numel()}'''							)

  if 0:
    print(f'''min={var.min():9.2e} max={var.max():9.2e}'''							)

  if 0:
    print(f'''min={var.min():9.2e} max={var.max():9.2e} var={var.var():9.2e} mean={var.mean():9.2e} ({ctx})'''							)

  return detected







class  __lowerCamelCase				(	__snake_case	):
      lowerCamelCase_	:      Optional[int]          =					'underflow_overflow'
      lowerCamelCase_	:      Dict          =					'tpu_metrics_debug'


from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCamelCase_    =					logging.get_logger(__name__)

lowerCamelCase_    =					{
    '''google/rembert''': '''https://huggingface.co/google/rembert/resolve/main/config.json''',
    # See all RemBERT models at https://huggingface.co/models?filter=rembert
}







class  __lowerCamelCase				(	__snake_case	):
      lowerCamelCase_	:      Tuple          =					'rembert'







      def __init__(     self							,  lowerCamelCase=250300							,  lowerCamelCase=1152							,  lowerCamelCase=32							,  lowerCamelCase=18							,  lowerCamelCase=256							,  lowerCamelCase=1664							,  lowerCamelCase=4608							,  lowerCamelCase="gelu"							,  lowerCamelCase=0.0							,  lowerCamelCase=0.0							,  lowerCamelCase=0.1							,  lowerCamelCase=512							,  lowerCamelCase=2							,  lowerCamelCase=0.02							,  lowerCamelCase=1e-12							,  lowerCamelCase=True							,  lowerCamelCase=0							,  lowerCamelCase=312							,  lowerCamelCase=313							,  **lowerCamelCase							,  )						->       Union[str, Any]:
        super().__init__(pad_token_id=lowerCamelCase							,  bos_token_id=lowerCamelCase							,  eos_token_id=lowerCamelCase							,  **lowerCamelCase   )

        snake_case_       						=      vocab_size
        snake_case_       						=      input_embedding_size
        snake_case_       						=      output_embedding_size
        snake_case_       						=      max_position_embeddings
        snake_case_       						=      hidden_size
        snake_case_       						=      num_hidden_layers
        snake_case_       						=      num_attention_heads
        snake_case_       						=      intermediate_size
        snake_case_       						=      hidden_act
        snake_case_       						=      hidden_dropout_prob
        snake_case_       						=      attention_probs_dropout_prob
        snake_case_       						=      classifier_dropout_prob
        snake_case_       						=      initializer_range
        snake_case_       						=      type_vocab_size
        snake_case_       						=      layer_norm_eps
        snake_case_       						=      use_cache
        snake_case_       						=      False







class  __lowerCamelCase				(	__snake_case	):







      @property
      def 	lowerCAmelCase_    (     self   )						->       Mapping[str, Mapping[int, str]]:
        if self.task == "multiple-choice":
          snake_case_       						=      {0: """batch""", 1: """choice""", 2: """sequence"""}
        else:
          snake_case_       						=      {0: """batch""", 1: """sequence"""}
        return OrderedDict(
            [
                ("""input_ids""", dynamic_axis),
                ("""attention_mask""", dynamic_axis),
                ("""token_type_ids""", dynamic_axis),
            ]   )







      @property
      def 	lowerCAmelCase_    (     self   )						->       float:
        return 1e-4