





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_UpperCAmelCase        =				logging.get_logger(__name__)

_UpperCAmelCase        =				{
    'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/config.json',
    'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/config.json',
    'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/config.json',
    'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/config.json',
    'bert-base-multilingual-uncased': 'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/config.json',
    'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/config.json',
    'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/config.json',
    'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/config.json',
    'bert-large-uncased-whole-word-masking': (
        'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/config.json'
    ),
    'bert-large-cased-whole-word-masking': (
        'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/config.json'
    ),
    'bert-large-uncased-whole-word-masking-finetuned-squad': (
        'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/config.json'
    ),
    'bert-large-cased-whole-word-masking-finetuned-squad': (
        'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/config.json'
    ),
    'bert-base-cased-finetuned-mrpc': 'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/config.json',
    'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/config.json',
    'bert-base-german-dbmdz-uncased': 'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/config.json',
    'cl-tohoku/bert-base-japanese': 'https://huggingface.co/cl-tohoku/bert-base-japanese/resolve/main/config.json',
    'cl-tohoku/bert-base-japanese-whole-word-masking': (
        'https://huggingface.co/cl-tohoku/bert-base-japanese-whole-word-masking/resolve/main/config.json'
    ),
    'cl-tohoku/bert-base-japanese-char': (
        'https://huggingface.co/cl-tohoku/bert-base-japanese-char/resolve/main/config.json'
    ),
    'cl-tohoku/bert-base-japanese-char-whole-word-masking': (
        'https://huggingface.co/cl-tohoku/bert-base-japanese-char-whole-word-masking/resolve/main/config.json'
    ),
    'TurkuNLP/bert-base-finnish-cased-v1': (
        'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/config.json'
    ),
    'TurkuNLP/bert-base-finnish-uncased-v1': (
        'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/config.json'
    ),
    'wietsedv/bert-base-dutch-cased': 'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/config.json',
    # See all BERT models at https://huggingface.co/models?filter=bert
}



class 							_UpperCamelCase					(	lowerCAmelCase_     ):
       _UpperCamelCase				:					Union[str, Any]         =   '''bert'''

       def __init__(		self: Union[str, Any]					,   _SCREAMING_SNAKE_CASE: str=30522					,   _SCREAMING_SNAKE_CASE: Dict=768					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=12					,   _SCREAMING_SNAKE_CASE: int=12					,   _SCREAMING_SNAKE_CASE: int=3072					,   _SCREAMING_SNAKE_CASE: str="gelu"					,   _SCREAMING_SNAKE_CASE: Optional[Any]=0.1					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=0.1					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=512					,   _SCREAMING_SNAKE_CASE: Any=2					,   _SCREAMING_SNAKE_CASE: str=0.02					,   _SCREAMING_SNAKE_CASE: List[str]=1e-12					,   _SCREAMING_SNAKE_CASE: Any=0					,   _SCREAMING_SNAKE_CASE: List[str]="absolute"					,   _SCREAMING_SNAKE_CASE: List[Any]=True					,   _SCREAMING_SNAKE_CASE: Optional[int]=None					,   **_SCREAMING_SNAKE_CASE: Any					,   )       -> str:




              """simple docstring"""


              super().__init__(pad_token_id=_SCREAMING_SNAKE_CASE					,   **_SCREAMING_SNAKE_CASE		)

              UpperCamelCase_          =  vocab_size
              UpperCamelCase_          =  hidden_size
              UpperCamelCase_          =  num_hidden_layers
              UpperCamelCase_          =  num_attention_heads
              UpperCamelCase_          =  hidden_act
              UpperCamelCase_          =  intermediate_size
              UpperCamelCase_          =  hidden_dropout_prob
              UpperCamelCase_          =  attention_probs_dropout_prob
              UpperCamelCase_          =  max_position_embeddings
              UpperCamelCase_          =  type_vocab_size
              UpperCamelCase_          =  initializer_range
              UpperCamelCase_          =  layer_norm_eps
              UpperCamelCase_          =  position_embedding_type
              UpperCamelCase_          =  use_cache
              UpperCamelCase_          =  classifier_dropout



class 							_UpperCamelCase					(	lowerCAmelCase_     ):

       @property
       def lowercase (		self: Dict		)       -> Mapping[str, Mapping[int, str]]:




              """simple docstring"""


              if self.task == "multiple-choice":
                     UpperCamelCase_          =  {0: "batch", 1: "choice", 2: "sequence"}
              else:
                     UpperCamelCase_          =  {0: "batch", 1: "sequence"}
              return OrderedDict(
                  [
                      ("input_ids", dynamic_axis),
                      ("attention_mask", dynamic_axis),
                      ("token_type_ids", dynamic_axis),
                  ]		)








import unittest

import torch
from torch import nn

from accelerate.test_utils import require_cuda
from accelerate.utils.memory import find_executable_batch_size, release_memory




def  lowerCAmelCase_ (							)     ->		Dict:
       raise RuntimeError("CUDA out of memory."  )



class 							_UpperCamelCase					(	nn.Module     ):

       def __init__(		self: int		)       -> List[Any]:




              """simple docstring"""


              super().__init__()
              UpperCamelCase_          =  nn.Linear(3					,   4		)
              UpperCamelCase_          =  nn.BatchNormad(4		)
              UpperCamelCase_          =  nn.Linear(4					,   5		)



       def lowercase (		self: Any					,   _SCREAMING_SNAKE_CASE: Optional[int]		)       -> Optional[Any]:




              """simple docstring"""


              return self.lineara(self.batchnorm(self.lineara(_SCREAMING_SNAKE_CASE		)		)		)



class 							_UpperCamelCase					(	unittest.TestCase     ):

       def lowercase (		self: List[Any]		)       -> Any:




              """simple docstring"""


              UpperCamelCase_          =  []

              @find_executable_batch_size(starting_batch_size=128		)
              def mock_training_loop_function(_SCREAMING_SNAKE_CASE: str		):
                     nonlocal batch_sizes
                     batch_sizes.append(_SCREAMING_SNAKE_CASE		)
                     if batch_size != 8:
                            raise_fake_out_of_memory()

              mock_training_loop_function()
              self.assertListEqual(_SCREAMING_SNAKE_CASE					,   [128, 64, 32, 16, 8]		)

       def lowercase (		self: Optional[int]		)       -> List[Any]:




              """simple docstring"""


              UpperCamelCase_          =  []

              @find_executable_batch_size(starting_batch_size=128		)
              def mock_training_loop_function(_SCREAMING_SNAKE_CASE: List[str]					,   _SCREAMING_SNAKE_CASE: Optional[Any]		):
                     nonlocal batch_sizes
                     batch_sizes.append(_SCREAMING_SNAKE_CASE		)
                     if batch_size != 8:
                            raise_fake_out_of_memory()
                     return batch_size, arga

              UpperCamelCase_       , UpperCamelCase_          =  mock_training_loop_function("hello"		)
              self.assertListEqual(_SCREAMING_SNAKE_CASE					,   [128, 64, 32, 16, 8]		)
              self.assertListEqual([bs, arga]					,   [8, "hello"]		)

       def lowercase (		self: Any		)       -> Optional[int]:




              """simple docstring"""


              @find_executable_batch_size(starting_batch_size=0		)
              def mock_training_loop_function(_SCREAMING_SNAKE_CASE: Union[str, Any]		):
                     pass

              with self.assertRaises(_SCREAMING_SNAKE_CASE		) as cm:
                     mock_training_loop_function()
                     self.assertIn("No executable batch size found, reached zero."					,   cm.exception.args[0]		)

       def lowercase (		self: Dict		)       -> Optional[int]:




              """simple docstring"""


              @find_executable_batch_size(starting_batch_size=16		)
              def mock_training_loop_function(_SCREAMING_SNAKE_CASE: Optional[int]		):
                     if batch_size > 0:
                            raise_fake_out_of_memory()
                     pass

              with self.assertRaises(_SCREAMING_SNAKE_CASE		) as cm:
                     mock_training_loop_function()
                     self.assertIn("No executable batch size found, reached zero."					,   cm.exception.args[0]		)

       def lowercase (		self: Optional[int]		)       -> Dict:




              """simple docstring"""


              @find_executable_batch_size(starting_batch_size=128		)
              def mock_training_loop_function(_SCREAMING_SNAKE_CASE: Any					,   _SCREAMING_SNAKE_CASE: int					,   _SCREAMING_SNAKE_CASE: Tuple		):
                     if batch_size != 8:
                            raise raise_fake_out_of_memory()

              with self.assertRaises(_SCREAMING_SNAKE_CASE		) as cm:
                     mock_training_loop_function(128					,   "hello"					,   "world"		)
                     self.assertIn("Batch size was passed into `f`"					,   cm.exception.args[0]		)
                     self.assertIn("`f(arg1='hello', arg2='world')"					,   cm.exception.args[0]		)

       def lowercase (		self: List[str]		)       -> Optional[Any]:




              """simple docstring"""


              @find_executable_batch_size(starting_batch_size=16		)
              def mock_training_loop_function(_SCREAMING_SNAKE_CASE: List[Any]		):
                     raise ValueError("Oops, we had an error!"		)

              with self.assertRaises(_SCREAMING_SNAKE_CASE		) as cm:
                     mock_training_loop_function()
                     self.assertIn("Oops, we had an error!"					,   cm.exception.args[0]		)



       @require_cuda
       def lowercase (		self: Optional[int]		)       -> Union[str, Any]:




              """simple docstring"""


              UpperCamelCase_          =  torch.cuda.memory_allocated()
              UpperCamelCase_          =  ModelForTest()
              model.cuda()
              self.assertGreater(torch.cuda.memory_allocated()					,   _SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  release_memory(_SCREAMING_SNAKE_CASE		)
              self.assertEqual(torch.cuda.memory_allocated()					,   _SCREAMING_SNAKE_CASE		)

