



"""simple docstring"""







import functools
import gc
import inspect

import torch

from .imports import is_npu_available, is_xpu_available

def     UpperCamelCase_   (   *lowerCAmelCase__				:    str						)						->       Union[str, Any]:

  """simple docstring"""




  if not isinstance(lowerCAmelCase__   ,       lowerCAmelCase__						):
    lowerCAmelCase_       :    str								=			list(lowerCAmelCase__						)
  for i in range(len(lowerCAmelCase__						)						):
    lowerCAmelCase_       :    Tuple								=			None
  gc.collect()
  if is_xpu_available():
    torch.xpu.empty_cache()
  elif is_npu_available():
    torch.npu.empty_cache()
  else:
    torch.cuda.empty_cache()
  return objects

def     UpperCamelCase_   (   lowerCAmelCase__				:    Exception						)						->       bool:

  """simple docstring"""




  lowerCAmelCase_       :    Union[str, Any]								=			[
      'CUDA out of memory.',  # CUDA OOM
      'cuDNN error: CUDNN_STATUS_NOT_SUPPORTED.',  # CUDNN SNAFU
      'DefaultCPUAllocator: can\'t allocate memory',  # CPU OOM
  ]
  if isinstance(lowerCAmelCase__   ,       lowerCAmelCase__						) and len(exception.args						) == 1:
    return any(err in exception.args[0] for err in _statements						)
  return False

def     UpperCamelCase_   (   lowerCAmelCase__				:    callable = None   ,       lowerCAmelCase__				:    int = 128						)						->       Tuple:

  """simple docstring"""




  if function is None:
    return functools.partial(lowerCAmelCase__   ,       starting_batch_size=lowerCAmelCase__						)

  lowerCAmelCase_       :    str								=			starting_batch_size

  def decorator(*lowerCAmelCase__				:    Optional[Any]   ,       **lowerCAmelCase__				:    int						):
    nonlocal batch_size
    gc.collect()
    if is_xpu_available():
      torch.xpu.empty_cache()
    elif is_npu_available():
      torch.npu.empty_cache()
    else:
      torch.cuda.empty_cache()
    lowerCAmelCase_       :    Any								=			list(inspect.signature(lowerCAmelCase__						).parameters.keys()						)
    # Guard against user error
    if len(lowerCAmelCase__						) < (len(lowerCAmelCase__						) + 1):
      lowerCAmelCase_       :    Union[str, Any]								=			', '.join([f"{arg}={value}" for arg, value in zip(params[1:]   ,       args[1:]						)]						)
      raise TypeError(
          f"Batch size was passed into `{function.__name__}` as the first argument when called."
          f"Remove this as the decorator already does so: `{function.__name__}({arg_str})`"						)
    while True:
      if batch_size == 0:
        raise RuntimeError('No executable batch size found, reached zero.'						)
      try:
        return function(lowerCAmelCase__   ,       *lowerCAmelCase__   ,       **lowerCAmelCase__						)
      except Exception as e:
        if should_reduce_batch_size(lowerCAmelCase__						):
          gc.collect()
          if is_xpu_available():
            torch.xpu.empty_cache()
          elif is_npu_available():
            torch.npu.empty_cache()
          else:
            torch.cuda.empty_cache()
          batch_size //= 2
        else:
          raise

  return decorator










"""simple docstring"""







import logging

from transformers.configuration_utils import PretrainedConfig


lowercase__		:    List[Any]				   =     logging.getLogger(__name__)







class 				UpperCamelCase__       (     lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE      =					"""masked_bert"""



      def __init__(				self    :				Any   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]=3_0_5_2_2   ,      SCREAMING_SNAKE_CASE_    :				int=7_6_8   ,      SCREAMING_SNAKE_CASE_    :				Tuple=1_2   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=1_2   ,      SCREAMING_SNAKE_CASE_    :				Tuple=3_0_7_2   ,      SCREAMING_SNAKE_CASE_    :				List[str]="gelu"   ,      SCREAMING_SNAKE_CASE_    :				Tuple=0.1   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=0.1   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=5_1_2   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]=2   ,      SCREAMING_SNAKE_CASE_    :				Dict=0.02   ,      SCREAMING_SNAKE_CASE_    :				Any=1E-12   ,      SCREAMING_SNAKE_CASE_    :				List[str]=0   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]="topK"   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]="constant"   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]=0.0   ,      **SCREAMING_SNAKE_CASE_    :				Any   ,      ):
        super().__init__(pad_token_id=SCREAMING_SNAKE_CASE_   ,      **SCREAMING_SNAKE_CASE_	)

        lowerCAmelCase_       :    List[Any]								=			vocab_size
        lowerCAmelCase_       :    str								=			hidden_size
        lowerCAmelCase_       :    Optional[int]								=			num_hidden_layers
        lowerCAmelCase_       :    Dict								=			num_attention_heads
        lowerCAmelCase_       :    List[str]								=			hidden_act
        lowerCAmelCase_       :    List[Any]								=			intermediate_size
        lowerCAmelCase_       :    Any								=			hidden_dropout_prob
        lowerCAmelCase_       :    str								=			attention_probs_dropout_prob
        lowerCAmelCase_       :    Any								=			max_position_embeddings
        lowerCAmelCase_       :    Dict								=			type_vocab_size
        lowerCAmelCase_       :    Tuple								=			initializer_range
        lowerCAmelCase_       :    List[Any]								=			layer_norm_eps
        lowerCAmelCase_       :    str								=			pruning_method
        lowerCAmelCase_       :    Optional[Any]								=			mask_init
        lowerCAmelCase_       :    int								=			mask_scale





