


'''simple docstring'''




import unittest

from transformers import AlbertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MODEL_FOR_PRETRAINING_MAPPING,
     AlbertForMaskedLM,
     AlbertForMultipleChoice,
     AlbertForPreTraining,
     AlbertForQuestionAnswering,
     AlbertForSequenceClassification,
     AlbertForTokenClassification,
     AlbertModel,
 )
 from transformers.models.albert.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class 					a			:
 '''simple docstring'''




 def __init__(    self		,      lowerCamelCase_		,      lowerCamelCase_=1_3		,      lowerCamelCase_=7		,      lowerCamelCase_=True		,      lowerCamelCase_=True		,      lowerCamelCase_=True		,      lowerCamelCase_=True		,      lowerCamelCase_=9_9		,      lowerCamelCase_=1_6		,      lowerCamelCase_=3_6		,      lowerCamelCase_=6		,      lowerCamelCase_=6		,      lowerCamelCase_=6		,      lowerCamelCase_=3_7		,      lowerCamelCase_="gelu"		,      lowerCamelCase_=0.1		,      lowerCamelCase_=0.1		,      lowerCamelCase_=5_1_2		,      lowerCamelCase_=1_6		,      lowerCamelCase_=2		,      lowerCamelCase_=0.02		,      lowerCamelCase_=3		,      lowerCamelCase_=4		,      lowerCamelCase_=None		,      )      ->    Dict:
    _a					:  Dict						=  parent
    _a					:  Tuple						=  batch_size
    _a					:  Optional[int]						=  seq_length
    _a					:  List[str]						=  is_training
    _a					:  Tuple						=  use_input_mask
    _a					:  Dict						=  use_token_type_ids
    _a					:  List[str]						=  use_labels
    _a					:  Optional[int]						=  vocab_size
    _a					:  int						=  embedding_size
    _a					:  Tuple						=  hidden_size
    _a					:  str						=  num_hidden_layers
    _a					:  List[str]						=  num_hidden_groups
    _a					:  Union[str, Any]						=  num_attention_heads
    _a					:  Dict						=  intermediate_size
    _a					:  List[str]						=  hidden_act
    _a					:  Tuple						=  hidden_dropout_prob
    _a					:  Any						=  attention_probs_dropout_prob
    _a					:  Tuple						=  max_position_embeddings
    _a					:  Dict						=  type_vocab_size
    _a					:  Tuple						=  type_sequence_label_size
    _a					:  Dict						=  initializer_range
    _a					:  str						=  num_labels
    _a					:  Optional[Any]						=  num_choices
    _a					:  Optional[int]						=  scope




 def   __UpperCamelCase   (    self      )      ->    Optional[int]:
    _a					:  Optional[int]						=  ids_tensor([self.batch_size, self.seq_length]		,      self.vocab_size      )

    _a					:  List[str]						=  None
    if self.use_input_mask:
       _a					:  Tuple						=  random_attention_mask([self.batch_size, self.seq_length]      )

    _a					:  Dict						=  None
    if self.use_token_type_ids:
       _a					:  Any						=  ids_tensor([self.batch_size, self.seq_length]		,      self.type_vocab_size      )

    _a					:  List[str]						=  None
    _a					:  Optional[int]						=  None
    _a					:  List[Any]						=  None
    if self.use_labels:
       _a					:  Optional[int]						=  ids_tensor([self.batch_size]		,      self.type_sequence_label_size      )
       _a					:  Any						=  ids_tensor([self.batch_size, self.seq_length]		,      self.num_labels      )
       _a					:  str						=  ids_tensor([self.batch_size]		,      self.num_choices      )

    _a					:  int						=  self.get_config()

    return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




 def   __UpperCamelCase   (    self      )      ->    Any:
    return AlbertConfig(
        vocab_size=self.vocab_size		,      hidden_size=self.hidden_size		,      num_hidden_layers=self.num_hidden_layers		,      num_attention_heads=self.num_attention_heads		,      intermediate_size=self.intermediate_size		,      hidden_act=self.hidden_act		,      hidden_dropout_prob=self.hidden_dropout_prob		,      attention_probs_dropout_prob=self.attention_probs_dropout_prob		,      max_position_embeddings=self.max_position_embeddings		,      type_vocab_size=self.type_vocab_size		,      initializer_range=self.initializer_range		,      num_hidden_groups=self.num_hidden_groups		,      )




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    List[str]:
    _a					:  str						=  AlbertModel(config=lowerCamelCase_      )
    model.to(lowerCamelCase_      )
    model.eval()
    _a					:  Tuple						=  model(lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      token_type_ids=lowerCamelCase_      )
    _a					:  Dict						=  model(lowerCamelCase_		,      token_type_ids=lowerCamelCase_      )
    _a					:  List[str]						=  model(lowerCamelCase_      )
    self.parent.assertEqual(result.last_hidden_state.shape		,      (self.batch_size, self.seq_length, self.hidden_size)      )
    self.parent.assertEqual(result.pooler_output.shape		,      (self.batch_size, self.hidden_size)      )




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    Any:
    _a					:  Optional[int]						=  AlbertForPreTraining(config=lowerCamelCase_      )
    model.to(lowerCamelCase_      )
    model.eval()
    _a					:  Optional[Any]						=  model(
        lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      token_type_ids=lowerCamelCase_		,      labels=lowerCamelCase_		,      sentence_order_label=lowerCamelCase_		,      )
    self.parent.assertEqual(result.prediction_logits.shape		,      (self.batch_size, self.seq_length, self.vocab_size)      )
    self.parent.assertEqual(result.sop_logits.shape		,      (self.batch_size, config.num_labels)      )




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    Tuple:
    _a					:  Union[str, Any]						=  AlbertForMaskedLM(config=lowerCamelCase_      )
    model.to(lowerCamelCase_      )
    model.eval()
    _a					:  int						=  model(lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      token_type_ids=lowerCamelCase_		,      labels=lowerCamelCase_      )
    self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.seq_length, self.vocab_size)      )




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    Tuple:
    _a					:  Union[str, Any]						=  AlbertForQuestionAnswering(config=lowerCamelCase_      )
    model.to(lowerCamelCase_      )
    model.eval()
    _a					:  Tuple						=  model(
        lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      token_type_ids=lowerCamelCase_		,      start_positions=lowerCamelCase_		,      end_positions=lowerCamelCase_		,      )
    self.parent.assertEqual(result.start_logits.shape		,      (self.batch_size, self.seq_length)      )
    self.parent.assertEqual(result.end_logits.shape		,      (self.batch_size, self.seq_length)      )




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    int:
    _a					:  str						=  self.num_labels
    _a					:  str						=  AlbertForSequenceClassification(lowerCamelCase_      )
    model.to(lowerCamelCase_      )
    model.eval()
    _a					:  str						=  model(lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      token_type_ids=lowerCamelCase_		,      labels=lowerCamelCase_      )
    self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.num_labels)      )




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    Optional[int]:
    _a					:  Optional[int]						=  self.num_labels
    _a					:  Any						=  AlbertForTokenClassification(config=lowerCamelCase_      )
    model.to(lowerCamelCase_      )
    model.eval()
    _a					:  List[Any]						=  model(lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      token_type_ids=lowerCamelCase_		,      labels=lowerCamelCase_      )
    self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.seq_length, self.num_labels)      )




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    Union[str, Any]:
    _a					:  Optional[int]						=  self.num_choices
    _a					:  Any						=  AlbertForMultipleChoice(config=lowerCamelCase_      )
    model.to(lowerCamelCase_      )
    model.eval()
    _a					:  str						=  input_ids.unsqueeze(1      ).expand(-1		,      self.num_choices		,      -1      ).contiguous()
    _a					:  Dict						=  token_type_ids.unsqueeze(1      ).expand(-1		,      self.num_choices		,      -1      ).contiguous()
    _a					:  List[Any]						=  input_mask.unsqueeze(1      ).expand(-1		,      self.num_choices		,      -1      ).contiguous()
    _a					:  Optional[Any]						=  model(
        lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      token_type_ids=lowerCamelCase_		,      labels=lowerCamelCase_		,      )
    self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.num_choices)      )




 def   __UpperCamelCase   (    self      )      ->    Optional[Any]:
    _a					:  Optional[int]						=  self.prepare_config_and_inputs()
    (
        (
        _a
    )		,						(
        _a
    )		,						(
        _a
    )		,						(
        _a
    )		,						(
        _a
    )		,						(
        _a
    )		,						(
        _a
    )		,						
    )					:  Optional[int]						=  config_and_inputs
    _a					:  Optional[Any]						=  {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
    return config, inputs_dict







@require_torch
class 					a			(				snake_case__			,							snake_case__			,							unittest.TestCase  ):
 '''simple docstring'''




 __lowerCAmelCase						: str							  =		(
     (
         AlbertModel,
         AlbertForPreTraining,
         AlbertForMaskedLM,
         AlbertForMultipleChoice,
         AlbertForSequenceClassification,
         AlbertForTokenClassification,
         AlbertForQuestionAnswering,
     )
     if is_torch_available()
     else ()
 )
 __lowerCAmelCase						: List[Any]							  =		(
     {
         """feature-extraction""": AlbertModel,
         """fill-mask""": AlbertForMaskedLM,
         """question-answering""": AlbertForQuestionAnswering,
         """text-classification""": AlbertForSequenceClassification,
         """token-classification""": AlbertForTokenClassification,
         """zero-shot""": AlbertForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 __lowerCAmelCase						: Optional[int]							  =		True




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_		,      lowerCamelCase_=False      )      ->    List[str]:
    _a					:  Optional[Any]						=  super()._prepare_for_class(lowerCamelCase_		,      lowerCamelCase_		,      return_labels=lowerCamelCase_      )

    if return_labels:
       if model_class in get_values(lowerCamelCase_      ):
          _a					:  str						=  torch.zeros(
              (self.model_tester.batch_size, self.model_tester.seq_length)		,      dtype=torch.long		,      device=lowerCamelCase_      )
          _a					:  int						=  torch.zeros(
              self.model_tester.batch_size		,      dtype=torch.long		,      device=lowerCamelCase_      )
    return inputs_dict




 def   __UpperCamelCase   (    self      )      ->    List[Any]:
    _a					:  int						=  AlbertModelTester(self      )
    _a					:  Union[str, Any]						=  ConfigTester(self		,      config_class=lowerCamelCase_		,      hidden_size=3_7      )




 def   __UpperCamelCase   (    self      )      ->    Tuple:
    self.config_tester.run_common_tests()




 def   __UpperCamelCase   (    self      )      ->    Optional[int]:
    _a					:  Tuple						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_model(*lowerCamelCase_      )




 def   __UpperCamelCase   (    self      )      ->    Tuple:
    _a					:  str						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_pretraining(*lowerCamelCase_      )




 def   __UpperCamelCase   (    self      )      ->    Any:
    _a					:  Optional[int]						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase_      )




 def   __UpperCamelCase   (    self      )      ->    Union[str, Any]:
    _a					:  Any						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase_      )




 def   __UpperCamelCase   (    self      )      ->    Tuple:
    _a					:  Union[str, Any]						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_question_answering(*lowerCamelCase_      )




 def   __UpperCamelCase   (    self      )      ->    Union[str, Any]:
    _a					:  Optional[int]						=  self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase_      )




 def   __UpperCamelCase   (    self      )      ->    Dict:
    _a					:  str						=  self.model_tester.prepare_config_and_inputs()
    for type in ["absolute", "relative_key", "relative_key_query"]:
       _a					:  Optional[Any]						=  type
       self.model_tester.create_and_check_model(*lowerCamelCase_      )




 @slow
 def   __UpperCamelCase   (    self      )      ->    Optional[Any]:
    for model_name in ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       _a					:  int						=  AlbertModel.from_pretrained(lowerCamelCase_      )
       self.assertIsNotNone(lowerCamelCase_      )







@require_torch
class 					a			(				unittest.TestCase  ):
 '''simple docstring'''




 @slow
 def   __UpperCamelCase   (    self      )      ->    Union[str, Any]:
    _a					:  Dict						=  AlbertModel.from_pretrained('albert-base-v2'      )
    _a					:  Optional[Any]						=  torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]      )
    _a					:  List[str]						=  torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]      )
    with torch.no_grad():
       _a					:  int						=  model(lowerCamelCase_		,      attention_mask=lowerCamelCase_      )[0]
    _a					:  str						=  torch.Size((1, 1_1, 7_6_8)      )
    self.assertEqual(output.shape		,      lowerCamelCase_      )
    _a					:  int						=  torch.tensor(
        [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]      )

    self.assertTrue(torch.allclose(output[:, 1:4, 1:4]		,      lowerCamelCase_		,      atol=1e-4      )      )






'''simple docstring'''




from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel


def    UpperCAmelCase_  (       A		,	A		,	A=None		,	A=None      ):
   '''simple docstring'''





   if attention_mask is None:
      _a					:  List[Any]						=  tf.cast(tf.math.not_equal(A		,	config.pad_token_id      )		,	tf.inta      )
   return {"input_ids": input_ids, "attention_mask": attention_mask}







@require_tf
class 					a			:
 '''simple docstring'''




 __lowerCAmelCase						: List[str]							  =		OPTConfig
 __lowerCAmelCase						: Union[str, Any]							  =		{}
 __lowerCAmelCase						: Any							  =		"""gelu"""




 def __init__(    self		,      lowerCamelCase_		,      lowerCamelCase_=1_3		,      lowerCamelCase_=7		,      lowerCamelCase_=True		,      lowerCamelCase_=False		,      lowerCamelCase_=9_9		,      lowerCamelCase_=1_6		,      lowerCamelCase_=2		,      lowerCamelCase_=4		,      lowerCamelCase_=4		,      lowerCamelCase_="gelu"		,      lowerCamelCase_=0.1		,      lowerCamelCase_=0.1		,      lowerCamelCase_=2_0		,      lowerCamelCase_=2		,      lowerCamelCase_=1		,      lowerCamelCase_=0		,      lowerCamelCase_=1_6		,      lowerCamelCase_=1_6		,      )      ->    List[Any]:
    _a					:  Tuple						=  parent
    _a					:  List[str]						=  batch_size
    _a					:  str						=  seq_length
    _a					:  Optional[Any]						=  is_training
    _a					:  List[str]						=  use_labels
    _a					:  Optional[int]						=  vocab_size
    _a					:  int						=  hidden_size
    _a					:  List[Any]						=  num_hidden_layers
    _a					:  Optional[int]						=  num_attention_heads
    _a					:  int						=  intermediate_size
    _a					:  Union[str, Any]						=  hidden_act
    _a					:  Any						=  hidden_dropout_prob
    _a					:  List[Any]						=  attention_probs_dropout_prob
    _a					:  int						=  max_position_embeddings
    _a					:  Optional[Any]						=  eos_token_id
    _a					:  int						=  pad_token_id
    _a					:  Optional[int]						=  bos_token_id
    _a					:  Dict						=  embed_dim
    _a					:  Union[str, Any]						=  word_embed_proj_dim
    _a					:  str						=  False




 def   __UpperCamelCase   (    self      )      ->    Union[str, Any]:
    _a					:  Optional[int]						=  ids_tensor([self.batch_size, self.seq_length - 1]		,      self.vocab_size      )
    _a					:  Optional[int]						=  tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size      )		,      1      )
    _a					:  Union[str, Any]						=  tf.concat([input_ids, eos_tensor]		,      axis=1      )

    _a					:  Optional[int]						=  self.config_cls(
        vocab_size=self.vocab_size		,      hidden_size=self.hidden_size		,      num_hidden_layers=self.num_hidden_layers		,      num_attention_heads=self.num_attention_heads		,      ffn_dim=self.intermediate_size		,      dropout=self.hidden_dropout_prob		,      attention_dropout=self.attention_probs_dropout_prob		,      max_position_embeddings=self.max_position_embeddings		,      eos_token_id=self.eos_token_id		,      bos_token_id=self.bos_token_id		,      pad_token_id=self.pad_token_id		,      embed_dim=self.embed_dim		,      word_embed_proj_dim=self.word_embed_proj_dim		,      is_encoder_decoder=lowerCamelCase_		,      **self.config_updates		,      )
    _a					:  Any						=  prepare_opt_inputs_dict(lowerCamelCase_		,      lowerCamelCase_      )
    return config, inputs_dict




 def   __UpperCamelCase   (    self		,      lowerCamelCase_		,      lowerCamelCase_      )      ->    str:
    _a					:  List[str]						=  TFOPTModel(config=lowerCamelCase_      )
    _a					:  Any						=  inputs_dict['input_ids']

    _a					:  Union[str, Any]						=  input_ids[:1, :]
    _a					:  Any						=  inputs_dict['attention_mask'][:1, :]
    _a					:  str						=  1

    # first forward pass
    _a					:  str						=  model(lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      use_cache=lowerCamelCase_      )

    _a		,						_a					:  int						=  outputs.to_tuple()

    # create hypothetical next token and extent to next_input_ids
    _a					:  List[Any]						=  ids_tensor((self.batch_size, 3)		,      config.vocab_size      )
    _a					:  List[Any]						=  tf.cast(ids_tensor((self.batch_size, 3)		,      2      )		,      tf.inta      )

    # append to next input_ids and
    _a					:  List[str]						=  tf.concat([input_ids, next_tokens]		,      axis=-1      )
    _a					:  Dict						=  tf.concat([attention_mask, next_attn_mask]		,      axis=-1      )

    _a					:  Union[str, Any]						=  model(lowerCamelCase_		,      attention_mask=lowerCamelCase_      )[0]
    _a					:  int						=  model(lowerCamelCase_		,      attention_mask=lowerCamelCase_		,      past_key_values=lowerCamelCase_      )[0]

    self.parent.assertEqual(next_tokens.shape[1]		,      output_from_past.shape[1]      )

    # select random slice
    _a					:  Dict						=  int(ids_tensor((1,)		,      output_from_past.shape[-1]      )      )
    _a					:  Optional[Any]						=  output_from_no_past[:, -3:, random_slice_idx]
    _a					:  Dict						=  output_from_past[:, :, random_slice_idx]

    # test that outputs are equal for slice
    tf.debugging.assert_near(lowerCamelCase_		,      lowerCamelCase_		,      rtol=1e-3      )







@require_tf
class 					a			(				snake_case__			,							snake_case__			,							unittest.TestCase  ):
 '''simple docstring'''




 __lowerCAmelCase						: List[Any]							  =		(TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
 __lowerCAmelCase						: int							  =		(TFOPTForCausalLM,) if is_tf_available() else ()
 __lowerCAmelCase						: str							  =		(
     {"""feature-extraction""": TFOPTModel, """text-generation""": TFOPTForCausalLM} if is_tf_available() else {}
 )
 __lowerCAmelCase						: List[Any]							  =		False
 __lowerCAmelCase						: List[str]							  =		False
 __lowerCAmelCase						: Optional[Any]							  =		False
 __lowerCAmelCase						: List[str]							  =		10




 def   __UpperCamelCase   (    self      )      ->    str:
    _a					:  str						=  TFOPTModelTester(self      )
    _a					:  Optional[int]						=  ConfigTester(self		,      config_class=lowerCamelCase_      )




 def   __UpperCamelCase   (    self      )      ->    Optional[Any]:
    self.config_tester.run_common_tests()




 def   __UpperCamelCase   (    self      )      ->    List[str]:
    _a					:  List[Any]						=  self.model_tester.prepare_config_and_inputs_for_common()
    self.model_tester.check_decoder_model_past_large_inputs(*lowerCamelCase_      )




 def   __UpperCamelCase   (    self      )      ->    Any:
    _a		,						_a					:  Optional[int]						=  self.model_tester.prepare_config_and_inputs_for_common()

    def _get_word_embedding_weight(lowerCamelCase_		,      lowerCamelCase_      ):
       if hasattr(lowerCamelCase_		,      'weight'      ):
          return embedding_layer.weight
       else:
          # Here we build the word embeddings weights if not exists.
          # And then we retry to get the attribute once built.
          model.build()
          if hasattr(lowerCamelCase_		,      'weight'      ):
             return embedding_layer.weight
          else:
             return None

    for model_class in self.all_model_classes:
       for size in [config.vocab_size - 1_0, config.vocab_size + 1_0]:
          # build the embeddings
          _a					:  Union[str, Any]						=  model_class(config=lowerCamelCase_      )
          _a					:  Optional[int]						=  _get_word_embedding_weight(lowerCamelCase_		,      model.get_input_embeddings()      )
          _a					:  Optional[int]						=  _get_word_embedding_weight(lowerCamelCase_		,      model.get_output_embeddings()      )

          # reshape the embeddings
          model.resize_token_embeddings(lowerCamelCase_      )
          _a					:  str						=  _get_word_embedding_weight(lowerCamelCase_		,      model.get_input_embeddings()      )
          _a					:  Union[str, Any]						=  _get_word_embedding_weight(lowerCamelCase_		,      model.get_output_embeddings()      )

          # check that the resized embeddings size matches the desired size.
          _a					:  Optional[int]						=  size if size is not None else config.vocab_size

          self.assertEqual(new_input_embeddings.shape[0]		,      lowerCamelCase_      )

          # check that weights remain the same after resizing
          _a					:  Optional[int]						=  True
          for pa, pa in zip(old_input_embeddings.value()		,      new_input_embeddings.value()      ):
             if tf.math.reduce_sum(tf.math.abs(pa - pa      )      ) > 0:
                _a					:  List[str]						=  False
          self.assertTrue(lowerCamelCase_      )

          if old_output_embeddings is not None and new_output_embeddings is not None:
             self.assertEqual(new_output_embeddings.shape[0]		,      lowerCamelCase_      )

             _a					:  Optional[Any]						=  True
             for pa, pa in zip(old_output_embeddings.value()		,      new_output_embeddings.value()      ):
                if tf.math.reduce_sum(tf.math.abs(pa - pa      )      ) > 0:
                   _a					:  Optional[Any]						=  False
             self.assertTrue(lowerCamelCase_      )





def    UpperCAmelCase_  (       A      ):
   '''simple docstring'''





   return tf.constant(A		,	dtype=tf.intaa      )







@require_tf
class 					a			(				unittest.TestCase  ):
 '''simple docstring'''




 __lowerCAmelCase						: Optional[Any]							  =		99




 def   __UpperCamelCase   (    self      )      ->    List[str]:
    _a					:  List[str]						=  tf.ones((4, 1)		,      dtype=tf.intaa      ) * 2
    _a					:  Any						=  tf.concat([ids_tensor((4, 6)		,      self.vocab_size - 3      ) + 3, eos_column_vector]		,      axis=1      )
    _a					:  Tuple						=  input_ids.shape[0]
    _a					:  Any						=  OPTConfig(
        vocab_size=self.vocab_size		,      hidden_size=2_4		,      num_hidden_layers=2		,      num_attention_heads=2		,      ffn_dim=3_2		,      max_position_embeddings=4_8		,      eos_token_id=2		,      pad_token_id=1		,      bos_token_id=0		,      )
    return config, input_ids, batch_size







@require_sentencepiece
@require_tf
class 					a			(				unittest.TestCase  ):
 '''simple docstring'''




 @slow
 def   __UpperCamelCase   (    self      )      ->    Union[str, Any]:
    _a					:  Any						=  TFOPTModel.from_pretrained('facebook/opt-350m'      )
    _a					:  Union[str, Any]						=  _long_tensor([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]      )
    _a					:  Any						=  tf.not_equal(lowerCamelCase_		,      model.config.pad_token_id      )
    with tf.GradientTape():
       _a					:  Dict						=  model(input_ids=lowerCamelCase_		,      attention_mask=lowerCamelCase_      ).last_hidden_state
    _a					:  List[Any]						=  (1, 1_1, 5_1_2)
    self.assertEqual(output.shape		,      lowerCamelCase_      )
    _a					:  int						=  tf.constant(
        [[-0.2873, -1.9218, -0.3033], [-1.2710, -0.1338, -0.1902], [0.4095, 0.1214, -1.3121]]      )
    self.assertTrue(np.allclose(output[:, :3, :3]		,      lowerCamelCase_		,      atol=4e-3      )      )

    _a					:  Union[str, Any]						=  tf.function(lowerCamelCase_		,      jit_compile=lowerCamelCase_      )
    _a					:  List[str]						=  xla_generate(lowerCamelCase_		,      lowerCamelCase_      )[0]
    self.assertTrue(np.allclose(output[:, :3, :3]		,      lowerCamelCase_		,      atol=4e-2      )      )







@require_tf
@slow
class 					a			(				unittest.TestCase  ):
 '''simple docstring'''




 def   __UpperCamelCase   (    self      )      ->    Any:
    super().setUp()
    _a					:  Any						=  'facebook/opt-350m'




 def   __UpperCamelCase   (    self      )      ->    str:
    _a					:  List[Any]						=  TFOPTForCausalLM.from_pretrained(self.path_model      )
    _a					:  List[str]						=  GPTaTokenizer.from_pretrained(self.path_model      )

    _a					:  Dict						=  [
        'Today is a beautiful day and I want to',
        'In the city of',
        'Paris is the capital of France and',
        'Computers and mobile phones have taken',
    ]
    # verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
    _a					:  Union[str, Any]						=  tokenizer(lowerCamelCase_		,      return_tensors='tf'		,      padding=lowerCamelCase_		,      add_special_tokens=lowerCamelCase_      )
    _a					:  List[Any]						=  tf.math.reduce_mean(model(inputs.input_ids		,      attention_mask=inputs.attention_mask      )[0]		,      axis=-1      )
    _a					:  int						=  tf.constant(
        [
            [1.3851, -13.8923, -10.5229, -10.7533, -0.2309, -10.2384, -0.5365, -9.0947, -5.1670],
            [-4.7073, -10.6276, -3.9415, -21.5242, -0.2822, -0.2822, -0.2822, -0.2822, -0.2822],
            [0.6247, -3.4229, -8.9179, -1.4297, -14.1650, 1.4146, -9.0218, -0.2703, -0.2703],
            [6.4783, -1.9913, -10.7926, -2.3336, 1.5092, -0.9974, -6.8213, 1.3477, 1.3477],
        ]      )
    self.assertTrue(np.allclose(lowerCamelCase_		,      lowerCamelCase_		,      atol=1e-4      )      )

    _a					:  Any						=  tf.function(lowerCamelCase_		,      jit_compile=lowerCamelCase_      )
    _a					:  List[Any]						=  tf.math.reduce_mean(xla_generate(inputs.input_ids		,      attention_mask=inputs.attention_mask      )[0]		,      axis=-1      )
    self.assertTrue(np.allclose(lowerCamelCase_		,      lowerCamelCase_		,      atol=1e-4      )      )







@require_tf
@slow
class 					a			(				unittest.TestCase  ):
 '''simple docstring'''




 @property
 def   __UpperCamelCase   (    self      )      ->    Optional[int]:
    return [
        "Today is a beautiful day and I want",
        "In the city of",
        "Paris is the capital of France and",
        "Computers and mobile phones have taken",
    ]




 def   __UpperCamelCase   (    self      )      ->    List[Any]:
    _a					:  Any						=  'facebook/opt-125m'

    _a					:  List[str]						=  [
        'Today is a beautiful day and I want to',
        'In the city of New York, the city',
        'Paris is the capital of France and the capital',
        'Computers and mobile phones have taken over the',
    ]

    _a					:  str						=  []
    _a					:  Optional[Any]						=  GPTaTokenizer.from_pretrained(lowerCamelCase_      )
    _a					:  Tuple						=  TFOPTForCausalLM.from_pretrained(lowerCamelCase_      )

    for prompt in self.prompts:
       _a					:  Tuple						=  tokenizer(lowerCamelCase_		,      return_tensors='tf'      ).input_ids

       _a					:  Union[str, Any]						=  model.generate(lowerCamelCase_		,      max_length=1_0      )

       _a					:  Any						=  tokenizer.batch_decode(lowerCamelCase_		,      skip_special_tokens=lowerCamelCase_      )
       predicted_outputs += generated_string

    self.assertListEqual(lowerCamelCase_		,      lowerCamelCase_      )




 def   __UpperCamelCase   (    self      )      ->    Tuple:
    _a					:  int						=  'facebook/opt-350m'

    _a					:  Dict						=  GPTaTokenizer.from_pretrained(lowerCamelCase_      )
    _a					:  Any						=  TFOPTForCausalLM.from_pretrained(lowerCamelCase_      )

    _a					:  Any						=  'left'

    # use different length sentences to test batching
    _a					:  Optional[Any]						=  [
        'Hello, my dog is a little',
        'Today, I',
    ]

    _a					:  List[Any]						=  tokenizer(lowerCamelCase_		,      return_tensors='tf'		,      padding=lowerCamelCase_      )
    _a					:  Dict						=  inputs['input_ids']

    _a					:  Tuple						=  model.generate(input_ids=lowerCamelCase_		,      attention_mask=inputs['attention_mask']      )

    _a					:  Dict						=  tokenizer(sentences[0]		,      return_tensors='tf'      ).input_ids
    _a					:  Union[str, Any]						=  model.generate(input_ids=lowerCamelCase_      )

    _a					:  Any						=  inputs_non_padded.shape[-1] - tf.math.reduce_sum(
        tf.cast(inputs['attention_mask'][-1]		,      tf.intaa      )      )
    _a					:  Union[str, Any]						=  tokenizer(sentences[1]		,      return_tensors='tf'      ).input_ids
    _a					:  str						=  model.generate(input_ids=lowerCamelCase_		,      max_length=model.config.max_length - num_paddings      )

    _a					:  str						=  tokenizer.batch_decode(lowerCamelCase_		,      skip_special_tokens=lowerCamelCase_      )
    _a					:  Optional[int]						=  tokenizer.decode(output_non_padded[0]		,      skip_special_tokens=lowerCamelCase_      )
    _a					:  Any						=  tokenizer.decode(output_padded[0]		,      skip_special_tokens=lowerCamelCase_      )

    _a					:  Tuple						=  [
        'Hello, my dog is a little bit of a dork.\nI\'m a little bit',
        'Today, I was in the middle of a conversation with a friend about the',
    ]
    self.assertListEqual(lowerCamelCase_		,      lowerCamelCase_      )
    self.assertListEqual(lowerCamelCase_		,      [non_padded_sentence, padded_sentence]      )




 def   __UpperCamelCase   (    self      )      ->    Union[str, Any]:
    _a					:  str						=  'facebook/opt-350m'

    _a					:  List[str]						=  [
        'Today is a beautiful day and I want to',
        'In the city of San Francisco, the city',
        'Paris is the capital of France and the capital',
        'Computers and mobile phones have taken over the',
    ]

    _a					:  List[str]						=  []
    _a					:  Union[str, Any]						=  GPTaTokenizer.from_pretrained(lowerCamelCase_      )
    _a					:  int						=  TFOPTForCausalLM.from_pretrained(lowerCamelCase_      )

    for prompt in self.prompts:
       _a					:  List[str]						=  tokenizer(lowerCamelCase_		,      return_tensors='tf'      ).input_ids

       _a					:  List[Any]						=  model.generate(lowerCamelCase_		,      max_length=1_0      )

       _a					:  Any						=  tokenizer.batch_decode(lowerCamelCase_		,      skip_special_tokens=lowerCamelCase_      )
       predicted_outputs += generated_string

    self.assertListEqual(lowerCamelCase_		,      lowerCamelCase_      )


