
import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel







class   lowerCamelCase	:





					def __init__(self   :					Any       ,					_A   :					Union[str, Any]       ,					_A   :					str=1_3       ,					_A   :					Dict=7       ,					_A   :					Optional[Any]=True       ,					_A   :					Optional[int]=True       ,					_A   :					List[Any]=False       ,					_A   :					Dict=True       ,					_A   :					List[str]=9_9       ,					_A   :					Union[str, Any]=3_2       ,					_A   :					Optional[Any]=5       ,					_A   :					Dict=4       ,					_A   :					str=3_7       ,					_A   :					Tuple="gelu"       ,					_A   :					Any=0.1       ,					_A   :					str=0.1       ,					_A   :					List[str]=5_1_2       ,					_A   :					List[str]=1_6       ,					_A   :					int=2       ,					_A   :					Optional[int]=0.02       ,					_A   :					Optional[Any]=3       ,					_A   :					Any=4       ,					_A   :					Any=None       ,					)		->		List[Any]:
									snake_case				  =      parent
									snake_case				  =      batch_size
									snake_case				  =      seq_length
									snake_case				  =      is_training
									snake_case				  =      use_input_mask
									snake_case				  =      use_token_type_ids
									snake_case				  =      use_labels
									snake_case				  =      vocab_size
									snake_case				  =      hidden_size
									snake_case				  =      num_hidden_layers
									snake_case				  =      num_attention_heads
									snake_case				  =      intermediate_size
									snake_case				  =      hidden_act
									snake_case				  =      hidden_dropout_prob
									snake_case				  =      attention_probs_dropout_prob
									snake_case				  =      max_position_embeddings
									snake_case				  =      type_vocab_size
									snake_case				  =      type_sequence_label_size
									snake_case				  =      initializer_range
									snake_case				  =      num_labels
									snake_case				  =      num_choices
									snake_case				  =      scope





					def 	UpperCAmelCase(self   :					Any     )		->		List[str]:
									snake_case				  =      ids_tensor([self.batch_size, self.seq_length]       ,					self.vocab_size     )

									snake_case				  =      None
									if self.use_input_mask:
													snake_case				  =      random_attention_mask([self.batch_size, self.seq_length]     )

									snake_case				  =      None
									if self.use_token_type_ids:
													snake_case				  =      ids_tensor([self.batch_size, self.seq_length]       ,					self.type_vocab_size     )

									snake_case				  =      None
									snake_case				  =      None
									snake_case				  =      None
									if self.use_labels:
													snake_case				  =      ids_tensor([self.batch_size]       ,					self.type_sequence_label_size     )
													snake_case				  =      ids_tensor([self.batch_size, self.seq_length]       ,					self.num_labels     )
													snake_case				  =      ids_tensor([self.batch_size]       ,					self.num_choices     )

									snake_case				  =      self.get_config()

									return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





					def 	UpperCAmelCase(self   :					str     )		->		Optional[Any]:
									return OpenLlamaConfig(
									    vocab_size=self.vocab_size       ,					hidden_size=self.hidden_size       ,					num_hidden_layers=self.num_hidden_layers       ,					num_attention_heads=self.num_attention_heads       ,					intermediate_size=self.intermediate_size       ,					hidden_act=self.hidden_act       ,					hidden_dropout_prob=self.hidden_dropout_prob       ,					attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,					max_position_embeddings=self.max_position_embeddings       ,					type_vocab_size=self.type_vocab_size       ,					is_decoder=_A       ,					initializer_range=self.initializer_range       ,					use_stable_embedding=_A       ,					)





					def 	UpperCAmelCase(self   :					Dict       ,					_A   :					Optional[int]       ,					_A   :					str       ,					_A   :					str       ,					_A   :					Union[str, Any]       ,					_A   :					str       ,					_A   :					Optional[Any]       ,					_A   :					Optional[Any]     )		->		List[str]:
									snake_case				  =      OpenLlamaModel(config=_A     )
									model.to(_A     )
									model.eval()
									snake_case				  =      model(_A       ,					attention_mask=_A     )
									snake_case				  =      model(_A     )
									self.parent.assertEqual(result.last_hidden_state.shape       ,					(self.batch_size, self.seq_length, self.hidden_size)     )





					def 	UpperCAmelCase(self   :					int       ,					_A   :					Dict       ,					_A   :					Any       ,					_A   :					int       ,					_A   :					Tuple       ,					_A   :					List[Any]       ,					_A   :					Tuple       ,					_A   :					int       ,					_A   :					str       ,					_A   :					Union[str, Any]       ,					)		->		List[str]:
									snake_case				  =      True
									snake_case				  =      OpenLlamaModel(_A     )
									model.to(_A     )
									model.eval()
									snake_case				  =      model(
									    _A       ,					attention_mask=_A       ,					encoder_hidden_states=_A       ,					encoder_attention_mask=_A       ,					)
									snake_case				  =      model(
									    _A       ,					attention_mask=_A       ,					encoder_hidden_states=_A       ,					)
									snake_case				  =      model(_A       ,					attention_mask=_A     )
									self.parent.assertEqual(result.last_hidden_state.shape       ,					(self.batch_size, self.seq_length, self.hidden_size)     )





					def 	UpperCAmelCase(self   :					Optional[int]       ,					_A   :					List[str]       ,					_A   :					int       ,					_A   :					int       ,					_A   :					int       ,					_A   :					List[str]       ,					_A   :					Tuple       ,					_A   :					List[str]       ,					_A   :					Optional[int]       ,					_A   :					Tuple       ,					)		->		str:
									snake_case				  =      OpenLlamaForCausalLM(config=_A     )
									model.to(_A     )
									model.eval()
									snake_case				  =      model(_A       ,					attention_mask=_A       ,					labels=_A     )
									self.parent.assertEqual(result.logits.shape       ,					(self.batch_size, self.seq_length, self.vocab_size)     )





					def 	UpperCAmelCase(self   :					List[Any]       ,					_A   :					Dict       ,					_A   :					str       ,					_A   :					Union[str, Any]       ,					_A   :					int       ,					_A   :					Any       ,					_A   :					Optional[Any]       ,					_A   :					Optional[Any]       ,					_A   :					List[str]       ,					_A   :					Optional[int]       ,					)		->		Any:
									snake_case				  =      True
									snake_case				  =      True
									snake_case				  =      OpenLlamaForCausalLM(config=_A     )
									model.to(_A     )
									model.eval()

									# first forward pass
									snake_case				  =      model(
									    _A       ,					attention_mask=_A       ,					encoder_hidden_states=_A       ,					encoder_attention_mask=_A       ,					use_cache=_A       ,					)
									snake_case				  =      outputs.past_key_values

									# create hypothetical multiple next token and extent to next_input_ids
									snake_case				  =      ids_tensor((self.batch_size, 3)       ,					config.vocab_size     )
									snake_case				  =      ids_tensor((self.batch_size, 3)       ,					vocab_size=2     )

									# append to next input_ids and
									snake_case				  =      torch.cat([input_ids, next_tokens]       ,					dim=-1     )
									snake_case				  =      torch.cat([input_mask, next_mask]       ,					dim=-1     )

									snake_case				  =      model(
									    _A       ,					attention_mask=_A       ,					encoder_hidden_states=_A       ,					encoder_attention_mask=_A       ,					output_hidden_states=_A       ,					)['''hidden_states'''][0]
									snake_case				  =      model(
									    _A       ,					attention_mask=_A       ,					encoder_hidden_states=_A       ,					encoder_attention_mask=_A       ,					past_key_values=_A       ,					output_hidden_states=_A       ,					)['''hidden_states'''][0]

									# select random slice
									snake_case				  =      ids_tensor((1,)       ,					output_from_past.shape[-1]     ).item()
									snake_case				  =      output_from_no_past[:, -3:, random_slice_idx].detach()
									snake_case				  =      output_from_past[:, :, random_slice_idx].detach()

									self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]     )

									# test that outputs are equal for slice
									self.parent.assertTrue(torch.allclose(_A       ,					_A       ,					atol=1E-3     )     )





					def 	UpperCAmelCase(self   :					Dict     )		->		Any:
									snake_case				  =      self.prepare_config_and_inputs()
									(
									    snake_case	
									)				  =      config_and_inputs
									snake_case				  =      {'''input_ids''': input_ids, '''attention_mask''': input_mask}
									return config, inputs_dict







@require_torch
class   lowerCamelCase	(						UpperCamelCase_							,				UpperCamelCase_							,				UpperCamelCase_							,				unittest.TestCase						):
					UpperCAmelCase__       :							Optional[Any]     =	(
					    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
					)
					UpperCAmelCase__       :							str     =	(OpenLlamaForCausalLM,) if is_torch_available() else ()
					UpperCAmelCase__       :							Dict     =	(
					    {
					        "feature-extraction": OpenLlamaModel,
					        "text-classification": OpenLlamaForSequenceClassification,
					        "text-generation": OpenLlamaForCausalLM,
					        "zero-shot": OpenLlamaForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					UpperCAmelCase__       :							str     =	False
					UpperCAmelCase__       :							str     =	False





					def 	UpperCAmelCase(self   :					str     )		->		Union[str, Any]:
									snake_case				  =      OpenLlamaModelTester(self     )
									snake_case				  =      ConfigTester(self       ,					config_class=_A       ,					hidden_size=3_7     )





					def 	UpperCAmelCase(self   :					Any     )		->		Dict:
									self.config_tester.run_common_tests()





					def 	UpperCAmelCase(self   :					Optional[Any]     )		->		List[Any]:
									snake_case				  =      self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*_A     )





					def 	UpperCAmelCase(self   :					List[Any]     )		->		int:
									snake_case				  =      self.model_tester.prepare_config_and_inputs()
									for type in ["absolute", "relative_key", "relative_key_query"]:
													snake_case				  =      type
													self.model_tester.create_and_check_model(*_A     )





					def 	UpperCAmelCase(self   :					Union[str, Any]     )		->		Any:
									snake_case				  =      self.model_tester.prepare_config_and_inputs_for_common()
									snake_case				  =      3
									snake_case				  =      input_dict['''input_ids''']
									snake_case				  =      input_ids.ne(1     ).to(_A     )
									snake_case				  =      ids_tensor([self.model_tester.batch_size]       ,					self.model_tester.type_sequence_label_size     )
									snake_case				  =      OpenLlamaForSequenceClassification(_A     )
									model.to(_A     )
									model.eval()
									snake_case				  =      model(_A       ,					attention_mask=_A       ,					labels=_A     )
									self.assertEqual(result.logits.shape       ,					(self.model_tester.batch_size, self.model_tester.num_labels)     )





					def 	UpperCAmelCase(self   :					Tuple     )		->		Optional[Any]:
									snake_case				  =      self.model_tester.prepare_config_and_inputs_for_common()
									snake_case				  =      3
									snake_case				  =      '''single_label_classification'''
									snake_case				  =      input_dict['''input_ids''']
									snake_case				  =      input_ids.ne(1     ).to(_A     )
									snake_case				  =      ids_tensor([self.model_tester.batch_size]       ,					self.model_tester.type_sequence_label_size     )
									snake_case				  =      OpenLlamaForSequenceClassification(_A     )
									model.to(_A     )
									model.eval()
									snake_case				  =      model(_A       ,					attention_mask=_A       ,					labels=_A     )
									self.assertEqual(result.logits.shape       ,					(self.model_tester.batch_size, self.model_tester.num_labels)     )





					def 	UpperCAmelCase(self   :					Dict     )		->		int:
									snake_case				  =      self.model_tester.prepare_config_and_inputs_for_common()
									snake_case				  =      3
									snake_case				  =      '''multi_label_classification'''
									snake_case				  =      input_dict['''input_ids''']
									snake_case				  =      input_ids.ne(1     ).to(_A     )
									snake_case				  =      ids_tensor(
									    [self.model_tester.batch_size, config.num_labels]       ,					self.model_tester.type_sequence_label_size     ).to(torch.float     )
									snake_case				  =      OpenLlamaForSequenceClassification(_A     )
									model.to(_A     )
									model.eval()
									snake_case				  =      model(_A       ,					attention_mask=_A       ,					labels=_A     )
									self.assertEqual(result.logits.shape       ,					(self.model_tester.batch_size, self.model_tester.num_labels)     )





					@unittest.skip("Open-Llama buffers include complex numbers, which breaks this test"     )
					def 	UpperCAmelCase(self   :					str     )		->		str:
									pass





					@parameterized.expand([("linear",), ("dynamic",)]     )
					def 	UpperCAmelCase(self   :					int       ,					_A   :					Optional[int]     )		->		List[str]:
									snake_case				  =      self.model_tester.prepare_config_and_inputs_for_common()
									snake_case				  =      ids_tensor([1, 1_0]       ,					config.vocab_size     )
									snake_case				  =      ids_tensor([1, int(config.max_position_embeddings * 1.5     )]       ,					config.vocab_size     )

									set_seed(4_2     )  # Fixed seed at init time so the two models get the same random weights
									snake_case				  =      OpenLlamaModel(_A     )
									original_model.to(_A     )
									original_model.eval()
									snake_case				  =      original_model(_A     ).last_hidden_state
									snake_case				  =      original_model(_A     ).last_hidden_state

									set_seed(4_2     )  # Fixed seed at init time so the two models get the same random weights
									snake_case				  =      {'''type''': scaling_type, '''factor''': 10.0}
									snake_case				  =      OpenLlamaModel(_A     )
									scaled_model.to(_A     )
									scaled_model.eval()
									snake_case				  =      scaled_model(_A     ).last_hidden_state
									snake_case				  =      scaled_model(_A     ).last_hidden_state

									# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
									# maximum sequence length, so the outputs for the short input should match.
									if scaling_type == "dynamic":
													self.assertTrue(torch.allclose(_A       ,					_A       ,					atol=1E-5     )     )
									else:
													self.assertFalse(torch.allclose(_A       ,					_A       ,					atol=1E-5     )     )

									# The output should be different for long inputs
									self.assertFalse(torch.allclose(_A       ,					_A       ,					atol=1E-5     )     )



def 		lowercase_    (     A__				,	A__	)							->							List[Any]:
    """simple docstring"""

    snake_case				  =      [0 for i in range(r + 1	)]
    # nc0 = 1
    snake_case				  =      1
    for i in range(1				,	n + 1	):
        # to compute current row from previous row.
        snake_case				  =      min(A__				,	A__	)
        while j > 0:
            c[j] += c[j - 1]
            j -= 1
    return c[r]





print(binomial_coefficient(n=10, r=5))

