

'''simple docstring'''

import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel




class 	A_			:






     '''simple docstring'''



     def __init__(		self			:					Tuple					,      lowercase_			:					str					,      lowercase_			:					int=13					,      lowercase_			:					Optional[Any]=7					,      lowercase_			:					List[Any]=True					,      lowercase_			:					int=True					,      lowercase_			:					str=False					,      lowercase_			:					Optional[int]=True					,      lowercase_			:					List[Any]=99					,      lowercase_			:					Optional[int]=32					,      lowercase_			:					str=5					,      lowercase_			:					Tuple=4					,      lowercase_			:					Optional[Any]=37					,      lowercase_			:					Tuple="gelu"					,      lowercase_			:					Dict=0.1					,      lowercase_			:					Dict=0.1					,      lowercase_			:					int=512					,      lowercase_			:					str=16					,      lowercase_			:					Optional[Any]=2					,      lowercase_			:					List[Any]=0.02					,      lowercase_			:					str=3					,      lowercase_			:					Union[str, Any]=4					,      lowercase_			:					Tuple=None					,      )						->     Optional[Any]:
      UpperCAmelCase	:     Any						 =			parent
      UpperCAmelCase	:     Union[str, Any]						 =			batch_size
      UpperCAmelCase	:     Union[str, Any]						 =			seq_length
      UpperCAmelCase	:     Union[str, Any]						 =			is_training
      UpperCAmelCase	:     Optional[Any]						 =			use_input_mask
      UpperCAmelCase	:     Dict						 =			use_token_type_ids
      UpperCAmelCase	:     Any						 =			use_labels
      UpperCAmelCase	:     Optional[int]						 =			vocab_size
      UpperCAmelCase	:     Dict						 =			hidden_size
      UpperCAmelCase	:     Any						 =			num_hidden_layers
      UpperCAmelCase	:     Tuple						 =			num_attention_heads
      UpperCAmelCase	:     Any						 =			intermediate_size
      UpperCAmelCase	:     Any						 =			hidden_act
      UpperCAmelCase	:     Tuple						 =			hidden_dropout_prob
      UpperCAmelCase	:     Union[str, Any]						 =			attention_probs_dropout_prob
      UpperCAmelCase	:     Dict						 =			max_position_embeddings
      UpperCAmelCase	:     Optional[int]						 =			type_vocab_size
      UpperCAmelCase	:     Union[str, Any]						 =			type_sequence_label_size
      UpperCAmelCase	:     Tuple						 =			initializer_range
      UpperCAmelCase	:     Optional[Any]						 =			num_labels
      UpperCAmelCase	:     Optional[Any]						 =			num_choices
      UpperCAmelCase	:     Any						 =			scope


     def        UpperCAmelCase_	(		self			:					List[str]	)						->     int:
      UpperCAmelCase	:     str						 =			ids_tensor([self.batch_size, self.seq_length]					,      self.vocab_size	)

      UpperCAmelCase	:     Optional[Any]						 =			None
      if self.use_input_mask:
       UpperCAmelCase	:     Any						 =			random_attention_mask([self.batch_size, self.seq_length]	)

      UpperCAmelCase	:     Any						 =			None
      if self.use_token_type_ids:
       UpperCAmelCase	:     Dict						 =			ids_tensor([self.batch_size, self.seq_length]					,      self.type_vocab_size	)

      UpperCAmelCase	:     Union[str, Any]						 =			None
      UpperCAmelCase	:     Dict						 =			None
      UpperCAmelCase	:     Optional[int]						 =			None
      if self.use_labels:
       UpperCAmelCase	:     Optional[int]						 =			ids_tensor([self.batch_size]					,      self.type_sequence_label_size	)
       UpperCAmelCase	:     int						 =			ids_tensor([self.batch_size, self.seq_length]					,      self.num_labels	)
       UpperCAmelCase	:     List[str]						 =			ids_tensor([self.batch_size]					,      self.num_choices	)

      UpperCAmelCase	:     Tuple						 =			self.get_config()

      return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


     def        UpperCAmelCase_	(		self			:					Union[str, Any]	)						->     Tuple:
      return OpenLlamaConfig(
          vocab_size=self.vocab_size					,      hidden_size=self.hidden_size					,      num_hidden_layers=self.num_hidden_layers					,      num_attention_heads=self.num_attention_heads					,      intermediate_size=self.intermediate_size					,      hidden_act=self.hidden_act					,      hidden_dropout_prob=self.hidden_dropout_prob					,      attention_probs_dropout_prob=self.attention_probs_dropout_prob					,      max_position_embeddings=self.max_position_embeddings					,      type_vocab_size=self.type_vocab_size					,      is_decoder=lowercase_					,      initializer_range=self.initializer_range					,      use_stable_embedding=lowercase_					,      )


     def        UpperCAmelCase_	(		self			:					List[Any]					,      lowercase_			:					str					,      lowercase_			:					Optional[Any]					,      lowercase_			:					str					,      lowercase_			:					str					,      lowercase_			:					List[str]					,      lowercase_			:					List[Any]					,      lowercase_			:					Union[str, Any]	)						->     List[Any]:
      UpperCAmelCase	:     List[Any]						 =			OpenLlamaModel(config=lowercase_	)
      model.to(lowercase_	)
      model.eval()
      UpperCAmelCase	:     Dict						 =			model(lowercase_					,      attention_mask=lowercase_	)
      UpperCAmelCase	:     List[str]						 =			model(lowercase_	)
      self.parent.assertEqual(result.last_hidden_state.shape					,      (self.batch_size, self.seq_length, self.hidden_size)	)


     def        UpperCAmelCase_	(		self			:					Optional[Any]					,      lowercase_			:					Optional[Any]					,      lowercase_			:					Optional[Any]					,      lowercase_			:					List[Any]					,      lowercase_			:					Optional[int]					,      lowercase_			:					Union[str, Any]					,      lowercase_			:					Optional[Any]					,      lowercase_			:					Union[str, Any]					,      lowercase_			:					List[Any]					,      lowercase_			:					Union[str, Any]					,      )						->     List[Any]:
      UpperCAmelCase	:     Optional[int]						 =			True
      UpperCAmelCase	:     Union[str, Any]						 =			OpenLlamaModel(lowercase_	)
      model.to(lowercase_	)
      model.eval()
      UpperCAmelCase	:     Union[str, Any]						 =			model(
          lowercase_					,      attention_mask=lowercase_					,      encoder_hidden_states=lowercase_					,      encoder_attention_mask=lowercase_					,      )
      UpperCAmelCase	:     Tuple						 =			model(
          lowercase_					,      attention_mask=lowercase_					,      encoder_hidden_states=lowercase_					,      )
      UpperCAmelCase	:     Tuple						 =			model(lowercase_					,      attention_mask=lowercase_	)
      self.parent.assertEqual(result.last_hidden_state.shape					,      (self.batch_size, self.seq_length, self.hidden_size)	)


     def        UpperCAmelCase_	(		self			:					str					,      lowercase_			:					str					,      lowercase_			:					List[Any]					,      lowercase_			:					Dict					,      lowercase_			:					List[Any]					,      lowercase_			:					List[Any]					,      lowercase_			:					str					,      lowercase_			:					List[Any]					,      lowercase_			:					Optional[Any]					,      lowercase_			:					str					,      )						->     Optional[int]:
      UpperCAmelCase	:     Optional[Any]						 =			OpenLlamaForCausalLM(config=lowercase_	)
      model.to(lowercase_	)
      model.eval()
      UpperCAmelCase	:     Optional[Any]						 =			model(lowercase_					,      attention_mask=lowercase_					,      labels=lowercase_	)
      self.parent.assertEqual(result.logits.shape					,      (self.batch_size, self.seq_length, self.vocab_size)	)


     def        UpperCAmelCase_	(		self			:					Dict					,      lowercase_			:					Any					,      lowercase_			:					Any					,      lowercase_			:					Dict					,      lowercase_			:					List[str]					,      lowercase_			:					Any					,      lowercase_			:					int					,      lowercase_			:					Tuple					,      lowercase_			:					Tuple					,      lowercase_			:					Union[str, Any]					,      )						->     Optional[Any]:
      UpperCAmelCase	:     int						 =			True
      UpperCAmelCase	:     Union[str, Any]						 =			True
      UpperCAmelCase	:     Dict						 =			OpenLlamaForCausalLM(config=lowercase_	)
      model.to(lowercase_	)
      model.eval()

      # first forward pass
      UpperCAmelCase	:     str						 =			model(
          lowercase_					,      attention_mask=lowercase_					,      encoder_hidden_states=lowercase_					,      encoder_attention_mask=lowercase_					,      use_cache=lowercase_					,      )
      UpperCAmelCase	:     int						 =			outputs.past_key_values

      # create hypothetical multiple next token and extent to next_input_ids
      UpperCAmelCase	:     str						 =			ids_tensor((self.batch_size, 3)					,      config.vocab_size	)
      UpperCAmelCase	:     Union[str, Any]						 =			ids_tensor((self.batch_size, 3)					,      vocab_size=2	)

      # append to next input_ids and
      UpperCAmelCase	:     Dict						 =			torch.cat([input_ids, next_tokens]					,      dim=-1	)
      UpperCAmelCase	:     Any						 =			torch.cat([input_mask, next_mask]					,      dim=-1	)

      UpperCAmelCase	:     Optional[int]						 =			model(
          lowercase_					,      attention_mask=lowercase_					,      encoder_hidden_states=lowercase_					,      encoder_attention_mask=lowercase_					,      output_hidden_states=lowercase_					,      )['hidden_states'][0]
      UpperCAmelCase	:     Dict						 =			model(
          lowercase_					,      attention_mask=lowercase_					,      encoder_hidden_states=lowercase_					,      encoder_attention_mask=lowercase_					,      past_key_values=lowercase_					,      output_hidden_states=lowercase_					,      )['hidden_states'][0]

      # select random slice
      UpperCAmelCase	:     Tuple						 =			ids_tensor((1,)					,      output_from_past.shape[-1]	).item()
      UpperCAmelCase	:     Union[str, Any]						 =			output_from_no_past[:, -3:, random_slice_idx].detach()
      UpperCAmelCase	:     Optional[int]						 =			output_from_past[:, :, random_slice_idx].detach()

      self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]	)

      # test that outputs are equal for slice
      self.parent.assertTrue(torch.allclose(lowercase_					,      lowercase_					,      atol=1E-3	)	)




     def        UpperCAmelCase_	(		self			:					List[str]	)						->     List[str]:
      UpperCAmelCase	:     List[Any]						 =			self.prepare_config_and_inputs()
      (
          (
          UpperCAmelCase			
      ) ,			(
          UpperCAmelCase			
      ) ,			(
          UpperCAmelCase			
      ) ,			(
          UpperCAmelCase			
      ) ,			(
          UpperCAmelCase			
      ) ,			(
          UpperCAmelCase			
      ) ,			(
          UpperCAmelCase			
      ) ,						
      )	:     List[str]						 =			config_and_inputs
      UpperCAmelCase	:     Union[str, Any]						 =			{'input_ids': input_ids, 'attention_mask': input_mask}
      return config, inputs_dict




@require_torch
class 	A_			(    _lowerCamelCase	,       _lowerCamelCase	,       _lowerCamelCase	,       unittest.TestCase							):






     '''simple docstring'''



     UpperCAmelCase_						:     int  		=  (
         (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
     )
     UpperCAmelCase_						:     Tuple  		=  (OpenLlamaForCausalLM,) if is_torch_available() else ()
     UpperCAmelCase_						:     Dict  		=  (
         {
             """feature-extraction""": OpenLlamaModel,
             """text-classification""": OpenLlamaForSequenceClassification,
             """text-generation""": OpenLlamaForCausalLM,
             """zero-shot""": OpenLlamaForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     UpperCAmelCase_						:     Optional[Any]  		=  False
     UpperCAmelCase_						:     Any  		=  False


     def        UpperCAmelCase_	(		self			:					Optional[int]	)						->     Union[str, Any]:
      UpperCAmelCase	:     int						 =			OpenLlamaModelTester(self	)
      UpperCAmelCase	:     str						 =			ConfigTester(self					,      config_class=lowercase_					,      hidden_size=37	)


     def        UpperCAmelCase_	(		self			:					Tuple	)						->     str:
      self.config_tester.run_common_tests()


     def        UpperCAmelCase_	(		self			:					Tuple	)						->     List[str]:
      UpperCAmelCase	:     Optional[Any]						 =			self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*lowercase_	)


     def        UpperCAmelCase_	(		self			:					Any	)						->     Any:
      UpperCAmelCase	:     List[str]						 =			self.model_tester.prepare_config_and_inputs()
      for type in ["absolute", "relative_key", "relative_key_query"]:
       UpperCAmelCase	:     Optional[int]						 =			type
       self.model_tester.create_and_check_model(*lowercase_	)


     def        UpperCAmelCase_	(		self			:					Dict	)						->     Optional[Any]:
      UpperCAmelCase ,			UpperCAmelCase	:     Tuple						 =			self.model_tester.prepare_config_and_inputs_for_common()
      UpperCAmelCase	:     int						 =			3
      UpperCAmelCase	:     List[Any]						 =			input_dict['input_ids']
      UpperCAmelCase	:     Optional[int]						 =			input_ids.ne(1	).to(lowercase_	)
      UpperCAmelCase	:     Dict						 =			ids_tensor([self.model_tester.batch_size]					,      self.model_tester.type_sequence_label_size	)
      UpperCAmelCase	:     Optional[Any]						 =			OpenLlamaForSequenceClassification(lowercase_	)
      model.to(lowercase_	)
      model.eval()
      UpperCAmelCase	:     List[Any]						 =			model(lowercase_					,      attention_mask=lowercase_					,      labels=lowercase_	)
      self.assertEqual(result.logits.shape					,      (self.model_tester.batch_size, self.model_tester.num_labels)	)


     def        UpperCAmelCase_	(		self			:					List[Any]	)						->     str:
      UpperCAmelCase ,			UpperCAmelCase	:     int						 =			self.model_tester.prepare_config_and_inputs_for_common()
      UpperCAmelCase	:     Tuple						 =			3
      UpperCAmelCase	:     List[str]						 =			'single_label_classification'
      UpperCAmelCase	:     Union[str, Any]						 =			input_dict['input_ids']
      UpperCAmelCase	:     Union[str, Any]						 =			input_ids.ne(1	).to(lowercase_	)
      UpperCAmelCase	:     List[str]						 =			ids_tensor([self.model_tester.batch_size]					,      self.model_tester.type_sequence_label_size	)
      UpperCAmelCase	:     Tuple						 =			OpenLlamaForSequenceClassification(lowercase_	)
      model.to(lowercase_	)
      model.eval()
      UpperCAmelCase	:     int						 =			model(lowercase_					,      attention_mask=lowercase_					,      labels=lowercase_	)
      self.assertEqual(result.logits.shape					,      (self.model_tester.batch_size, self.model_tester.num_labels)	)


     def        UpperCAmelCase_	(		self			:					Any	)						->     Optional[Any]:
      UpperCAmelCase ,			UpperCAmelCase	:     Any						 =			self.model_tester.prepare_config_and_inputs_for_common()
      UpperCAmelCase	:     Optional[int]						 =			3
      UpperCAmelCase	:     List[Any]						 =			'multi_label_classification'
      UpperCAmelCase	:     Union[str, Any]						 =			input_dict['input_ids']
      UpperCAmelCase	:     List[Any]						 =			input_ids.ne(1	).to(lowercase_	)
      UpperCAmelCase	:     Optional[Any]						 =			ids_tensor(
          [self.model_tester.batch_size, config.num_labels]					,      self.model_tester.type_sequence_label_size	).to(torch.float	)
      UpperCAmelCase	:     int						 =			OpenLlamaForSequenceClassification(lowercase_	)
      model.to(lowercase_	)
      model.eval()
      UpperCAmelCase	:     str						 =			model(lowercase_					,      attention_mask=lowercase_					,      labels=lowercase_	)
      self.assertEqual(result.logits.shape					,      (self.model_tester.batch_size, self.model_tester.num_labels)	)


     @unittest.skip('Open-Llama buffers include complex numbers, which breaks this test'	)
     def        UpperCAmelCase_	(		self			:					List[str]	)						->     str:
      pass




     @parameterized.expand([('linear',), ('dynamic',)]	)
     def        UpperCAmelCase_	(		self			:					Optional[int]					,      lowercase_			:					Optional[int]	)						->     Union[str, Any]:
      UpperCAmelCase ,			UpperCAmelCase	:     str						 =			self.model_tester.prepare_config_and_inputs_for_common()
      UpperCAmelCase	:     Union[str, Any]						 =			ids_tensor([1, 10]					,      config.vocab_size	)
      UpperCAmelCase	:     Optional[Any]						 =			ids_tensor([1, int(config.max_position_embeddings * 1.5	)]					,      config.vocab_size	)

      set_seed(42	)  # Fixed seed at init time so the two models get the same random weights
      UpperCAmelCase	:     Tuple						 =			OpenLlamaModel(lowercase_	)
      original_model.to(lowercase_	)
      original_model.eval()
      UpperCAmelCase	:     int						 =			original_model(lowercase_	).last_hidden_state
      UpperCAmelCase	:     Optional[int]						 =			original_model(lowercase_	).last_hidden_state

      set_seed(42	)  # Fixed seed at init time so the two models get the same random weights
      UpperCAmelCase	:     Dict						 =			{'type': scaling_type, 'factor': 10.0}
      UpperCAmelCase	:     Union[str, Any]						 =			OpenLlamaModel(lowercase_	)
      scaled_model.to(lowercase_	)
      scaled_model.eval()
      UpperCAmelCase	:     List[Any]						 =			scaled_model(lowercase_	).last_hidden_state
      UpperCAmelCase	:     Optional[Any]						 =			scaled_model(lowercase_	).last_hidden_state

      # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
      # maximum sequence length, so the outputs for the short input should match.
      if scaling_type == "dynamic":
       self.assertTrue(torch.allclose(lowercase_					,      lowercase_					,      atol=1E-5	)	)
      else:
       self.assertFalse(torch.allclose(lowercase_					,      lowercase_					,      atol=1E-5	)	)

      # The output should be different for long inputs
      self.assertFalse(torch.allclose(lowercase_					,      lowercase_					,      atol=1E-5	)	)



'''simple docstring'''

from __future__ import annotations




def 				UpperCamelCase(					UpperCAmelCase_  ,	UpperCAmelCase_  ,	UpperCAmelCase_     ):
 if len(UpperCAmelCase_     ) == 0:
  raise ValueError('find_max() arg is an empty sequence'     )
 if (
     left >= len(UpperCAmelCase_     )
     or left < -len(UpperCAmelCase_     )
     or right >= len(UpperCAmelCase_     )
     or right < -len(UpperCAmelCase_     )
 ):
  raise IndexError('list index out of range'     )
 if left == right:
  return nums[left]
 UpperCAmelCase	:     Optional[int]						 =			(left + right) >> 1  # the middle
 UpperCAmelCase	:     Any						 =			find_max(UpperCAmelCase_  ,	UpperCAmelCase_  ,	UpperCAmelCase_     )  # find max in range[left, mid]
 UpperCAmelCase	:     Union[str, Any]						 =			find_max(UpperCAmelCase_  ,	mid + 1  ,	UpperCAmelCase_     )  # find max in range[mid + 1, right]

 return left_max if left_max >= right_max else right_max


if __name__ == "__main__":
 import doctest

 doctest.testmod(verbose=True)




