

"""simple docstring"""
from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional

from packaging import version


if TYPE_CHECKING:
    from ... import PreTrainedTokenizer, TensorType

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import is_torch_available, logging


_A							=  logging.get_logger(__name__)

_A							=  {
    """bigscience/bloom""": """https://huggingface.co/bigscience/bloom/resolve/main/config.json""",
    """bigscience/bloom-560m""": """https://huggingface.co/bigscience/bloom-560m/blob/main/config.json""",
    """bigscience/bloom-1b1""": """https://huggingface.co/bigscience/bloom-1b1/blob/main/config.json""",
    """bigscience/bloom-1b7""": """https://huggingface.co/bigscience/bloom-1b7/blob/main/config.json""",
    """bigscience/bloom-3b""": """https://huggingface.co/bigscience/bloom-3b/blob/main/config.json""",
    """bigscience/bloom-7b1""": """https://huggingface.co/bigscience/bloom-7b1/blob/main/config.json""",
}

class       lowerCamelCase	(       lowerCAmelCase__   ):


  '''simple docstring'''







  SCREAMING_SNAKE_CASE					      =  'bloom'
  SCREAMING_SNAKE_CASE					      =  ['past_key_values']
  SCREAMING_SNAKE_CASE					      =  {
      'num_hidden_layers': 'n_layer',
      'num_attention_heads': 'n_head',
  }



  def __init__(self							,       _lowerCamelCase=250880							,       _lowerCamelCase=64							,       _lowerCamelCase=2							,       _lowerCamelCase=8							,       _lowerCamelCase=1e-5							,       _lowerCamelCase=0.02							,       _lowerCamelCase=True							,       _lowerCamelCase=1							,       _lowerCamelCase=2							,       _lowerCamelCase=False							,       _lowerCamelCase=0.0							,       _lowerCamelCase=0.0							,       _lowerCamelCase=1							,       _lowerCamelCase=False							,       **_lowerCamelCase							,       ):


        """simple docstring"""

        UpperCAmelCase__			:							str							=    vocab_size
        # Backward compatibility with n_embed kwarg
        UpperCAmelCase__			:							str							=    kwargs.pop("""n_embed"""							,       _lowerCamelCase	)
        UpperCAmelCase__			:							Tuple							=    hidden_size if n_embed is None else n_embed
        UpperCAmelCase__			:							str							=    n_layer
        UpperCAmelCase__			:							int							=    n_head
        UpperCAmelCase__			:							str							=    layer_norm_epsilon
        UpperCAmelCase__			:							int							=    initializer_range
        UpperCAmelCase__			:							str							=    use_cache
        UpperCAmelCase__			:							List[str]							=    pretraining_tp
        UpperCAmelCase__			:							Any							=    apply_residual_connection_post_layernorm
        UpperCAmelCase__			:							int							=    hidden_dropout
        UpperCAmelCase__			:							Optional[Any]							=    attention_dropout

        UpperCAmelCase__			:							Optional[Any]							=    bos_token_id
        UpperCAmelCase__			:							Union[str, Any]							=    eos_token_id
        UpperCAmelCase__			:							List[str]							=    slow_but_exact

        super().__init__(bos_token_id=_lowerCamelCase							,       eos_token_id=_lowerCamelCase							,       **_lowerCamelCase	)




class       lowerCamelCase	(       lowerCAmelCase__   ):


  '''simple docstring'''







  SCREAMING_SNAKE_CASE					      =  version.parse('1.12'   )



  def __init__(self							,       _lowerCamelCase							,       _lowerCamelCase = "default"							,       _lowerCamelCase = None							,       _lowerCamelCase = False							,       ):


        """simple docstring"""

        super().__init__(_lowerCamelCase							,       task=_lowerCamelCase							,       patching_specs=_lowerCamelCase							,       use_past=_lowerCamelCase	)
        if not getattr(self._config							,       """pad_token_id"""							,       _lowerCamelCase	):
              # TODO: how to do that better?
              UpperCAmelCase__			:							Tuple							=    0



  @property
  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							List[str]							=    OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}}	)
        if self.use_past:
              # BLOOM stores values on dynamic axis 2. For more details see: https://github.com/huggingface/transformers/pull/18344
              self.fill_with_past_key_values_(_lowerCamelCase							,       direction="""inputs"""							,       inverted_values_shape=_lowerCamelCase	)
              UpperCAmelCase__			:							Dict							=    {0: """batch""", 1: """past_sequence + sequence"""}
        else:
              UpperCAmelCase__			:							List[Any]							=    {0: """batch""", 1: """sequence"""}

        return common_inputs



  @property
  def  _a      (self	):


        """simple docstring"""

        return self._config.n_layer



  @property
  def  _a      (self	):


        """simple docstring"""

        return self._config.n_head



  @property
  def  _a      (self	):


        """simple docstring"""

        return 1e-3



  def  _a      (self							,       _lowerCamelCase							,       _lowerCamelCase = -1							,       _lowerCamelCase = -1							,       _lowerCamelCase = False							,       _lowerCamelCase = None							,       ):


        """simple docstring"""

        UpperCAmelCase__			:							str							=    super(_lowerCamelCase							,       self	).generate_dummy_inputs(
            _lowerCamelCase							,       batch_size=_lowerCamelCase							,       seq_length=_lowerCamelCase							,       is_pair=_lowerCamelCase							,       framework=_lowerCamelCase	)

        # We need to order the input in the way they appears in the forward()
        UpperCAmelCase__			:							List[str]							=    OrderedDict({"""input_ids""": common_inputs["""input_ids"""]}	)

        # Need to add the past_keys
        if self.use_past:
              if not is_torch_available():
                    raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""	)
              else:
                    import torch

                    UpperCAmelCase__					,		UpperCAmelCase__			:							str							=    common_inputs["""input_ids"""].shape
                    # Not using the same length for past_key_values
                    UpperCAmelCase__			:							Optional[int]							=    seqlen + 2
                    UpperCAmelCase__			:							Any							=    self._config.hidden_size // self.num_attention_heads
                    UpperCAmelCase__			:							List[Any]							=    (
                        batch * self.num_attention_heads,
                        head_dim,
                        past_key_values_length,
                    )
                    UpperCAmelCase__			:							Any							=    (
                        batch * self.num_attention_heads,
                        past_key_values_length,
                        head_dim,
                    )
                    UpperCAmelCase__			:							Union[str, Any]							=    [
                        (torch.zeros(_lowerCamelCase	), torch.zeros(_lowerCamelCase	)) for _ in range(self.num_layers	)
                    ]

        UpperCAmelCase__			:							str							=    common_inputs["""attention_mask"""]
        if self.use_past:
              UpperCAmelCase__			:							Tuple							=    ordered_inputs["""attention_mask"""].dtype
              UpperCAmelCase__			:							List[Any]							=    torch.cat(
                  [ordered_inputs["""attention_mask"""], torch.ones(_lowerCamelCase							,       _lowerCamelCase							,       dtype=_lowerCamelCase	)]							,       dim=1	)

        return ordered_inputs




  @property
  def  _a      (self	):


        """simple docstring"""

        return 13




"""simple docstring"""
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
    from transformers import AutoModelForSeqaSeqLM, AutoTokenizer

@require_torch
@require_sentencepiece
@require_tokenizers
class       lowerCamelCase	(       unittest.TestCase   ):


  '''simple docstring'''







  @slow
  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Dict							=    AutoModelForSeqaSeqLM.from_pretrained("""google/mt5-small"""							,       return_dict=_lowerCamelCase	).to(_lowerCamelCase	)
        UpperCAmelCase__			:							Optional[Any]							=    AutoTokenizer.from_pretrained("""google/mt5-small"""	)

        UpperCAmelCase__			:							Any							=    tokenizer("""Hello there"""							,       return_tensors="""pt"""	).input_ids
        UpperCAmelCase__			:							str							=    tokenizer("""Hi I am"""							,       return_tensors="""pt"""	).input_ids

        UpperCAmelCase__			:							Tuple							=    model(input_ids.to(_lowerCamelCase	)							,       labels=labels.to(_lowerCamelCase	)	).loss
        UpperCAmelCase__			:							Union[str, Any]							=    -(labels.shape[-1] * loss.item())

        UpperCAmelCase__			:							List[str]							=    -84.9_127
        self.assertTrue(abs(mtf_score - EXPECTED_SCORE	) < 1e-4	)

