



'''simple docstring'''





import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel



class  snake_case__						:
 def __init__(   self						:							Dict      ,     __a						:							Dict      ,     __a						:							Tuple=13      ,     __a						:							Optional[Any]=7      ,     __a						:							str=True      ,     __a						:							Optional[Any]=True      ,     __a						:							Optional[int]=False      ,     __a						:							Optional[int]=True      ,     __a						:							Tuple=99      ,     __a						:							str=32      ,     __a						:							Optional[int]=5      ,     __a						:							List[str]=4      ,     __a						:							Tuple=37      ,     __a						:							str="gelu"      ,     __a						:							Tuple=0.1      ,     __a						:							Union[str, Any]=0.1      ,     __a						:							List[str]=512      ,     __a						:							List[Any]=16      ,     __a						:							Optional[int]=2      ,     __a						:							int=0.0_2      ,     __a						:							List[str]=3      ,     __a						:							List[str]=4      ,     __a						:							List[str]=None      ,     )      ->   Optional[Any]:







       '''simple docstring'''
       __snake_case :   Optional[Any]													=    parent
       __snake_case :   Dict													=    batch_size
       __snake_case :   List[Any]													=    seq_length
       __snake_case :   Dict													=    is_training
       __snake_case :   List[Any]													=    use_input_mask
       __snake_case :   Any													=    use_token_type_ids
       __snake_case :   int													=    use_labels
       __snake_case :   int													=    vocab_size
       __snake_case :   Tuple													=    hidden_size
       __snake_case :   int													=    num_hidden_layers
       __snake_case :   Optional[int]													=    num_attention_heads
       __snake_case :   Any													=    intermediate_size
       __snake_case :   Dict													=    hidden_act
       __snake_case :   Any													=    hidden_dropout_prob
       __snake_case :   List[str]													=    attention_probs_dropout_prob
       __snake_case :   Any													=    max_position_embeddings
       __snake_case :   Dict													=    type_vocab_size
       __snake_case :   List[Any]													=    type_sequence_label_size
       __snake_case :   Optional[int]													=    initializer_range
       __snake_case :   Any													=    num_labels
       __snake_case :   Any													=    num_choices
       __snake_case :   Dict													=    scope
 def A_   (   self						:							Union[str, Any]  )      ->   int:







       '''simple docstring'''
       __snake_case :   str													=    ids_tensor([self.batch_size, self.seq_length]      ,     self.vocab_size  )

       __snake_case :   Any													=    None
       if self.use_input_mask:
             __snake_case :   Optional[int]													=    random_attention_mask([self.batch_size, self.seq_length]  )

       __snake_case :   Optional[Any]													=    None
       if self.use_token_type_ids:
             __snake_case :   int													=    ids_tensor([self.batch_size, self.seq_length]      ,     self.type_vocab_size  )

       __snake_case :   Tuple													=    None
       __snake_case :   Optional[int]													=    None
       __snake_case :   Optional[Any]													=    None
       if self.use_labels:
             __snake_case :   Dict													=    ids_tensor([self.batch_size]      ,     self.type_sequence_label_size  )
             __snake_case :   Tuple													=    ids_tensor([self.batch_size, self.seq_length]      ,     self.num_labels  )
             __snake_case :   Dict													=    ids_tensor([self.batch_size]      ,     self.num_choices  )

       __snake_case :   int													=    self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
 def A_   (   self						:							int  )      ->   Dict:







       '''simple docstring'''
       return OpenLlamaConfig(
           vocab_size=self.vocab_size      ,     hidden_size=self.hidden_size      ,     num_hidden_layers=self.num_hidden_layers      ,     num_attention_heads=self.num_attention_heads      ,     intermediate_size=self.intermediate_size      ,     hidden_act=self.hidden_act      ,     hidden_dropout_prob=self.hidden_dropout_prob      ,     attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,     max_position_embeddings=self.max_position_embeddings      ,     type_vocab_size=self.type_vocab_size      ,     is_decoder=A_      ,     initializer_range=self.initializer_range      ,     use_stable_embedding=A_      ,     )
 def A_   (   self						:							Any      ,     __a						:							Any      ,     __a						:							List[Any]      ,     __a						:							Union[str, Any]      ,     __a						:							str      ,     __a						:							Optional[Any]      ,     __a						:							str      ,     __a						:							Union[str, Any]  )      ->   str:







       '''simple docstring'''
       __snake_case :   Dict													=    OpenLlamaModel(config=A_  )
       model.to(A_  )
       model.eval()
       __snake_case :   Union[str, Any]													=    model(A_      ,     attention_mask=A_  )
       __snake_case :   Union[str, Any]													=    model(A_  )
       self.parent.assertEqual(result.last_hidden_state.shape      ,     (self.batch_size, self.seq_length, self.hidden_size)  )
 def A_   (   self						:							Union[str, Any]      ,     __a						:							int      ,     __a						:							List[str]      ,     __a						:							Tuple      ,     __a						:							Any      ,     __a						:							List[Any]      ,     __a						:							List[Any]      ,     __a						:							List[Any]      ,     __a						:							str      ,     __a						:							List[Any]      ,     )      ->   Tuple:







       '''simple docstring'''
       __snake_case :   List[str]													=    True
       __snake_case :   Tuple													=    OpenLlamaModel(A_  )
       model.to(A_  )
       model.eval()
       __snake_case :   Tuple													=    model(
           A_      ,     attention_mask=A_      ,     encoder_hidden_states=A_      ,     encoder_attention_mask=A_      ,     )
       __snake_case :   str													=    model(
           A_      ,     attention_mask=A_      ,     encoder_hidden_states=A_      ,     )
       __snake_case :   Optional[int]													=    model(A_      ,     attention_mask=A_  )
       self.parent.assertEqual(result.last_hidden_state.shape      ,     (self.batch_size, self.seq_length, self.hidden_size)  )
 def A_   (   self						:							Any      ,     __a						:							str      ,     __a						:							Optional[int]      ,     __a						:							Tuple      ,     __a						:							Tuple      ,     __a						:							Optional[int]      ,     __a						:							Optional[Any]      ,     __a						:							List[Any]      ,     __a						:							str      ,     __a						:							str      ,     )      ->   str:







       '''simple docstring'''
       __snake_case :   List[Any]													=    OpenLlamaForCausalLM(config=A_  )
       model.to(A_  )
       model.eval()
       __snake_case :   int													=    model(A_      ,     attention_mask=A_      ,     labels=A_  )
       self.parent.assertEqual(result.logits.shape      ,     (self.batch_size, self.seq_length, self.vocab_size)  )
 def A_   (   self						:							str      ,     __a						:							Union[str, Any]      ,     __a						:							int      ,     __a						:							int      ,     __a						:							int      ,     __a						:							str      ,     __a						:							Tuple      ,     __a						:							int      ,     __a						:							str      ,     __a						:							int      ,     )      ->   Any:







       '''simple docstring'''
       __snake_case :   Union[str, Any]													=    True
       __snake_case :   List[str]													=    True
       __snake_case :   Any													=    OpenLlamaForCausalLM(config=A_  )
       model.to(A_  )
       model.eval()

       # first forward pass
       __snake_case :   List[str]													=    model(
           A_      ,     attention_mask=A_      ,     encoder_hidden_states=A_      ,     encoder_attention_mask=A_      ,     use_cache=A_      ,     )
       __snake_case :   Union[str, Any]													=    outputs.past_key_values

       # create hypothetical multiple next token and extent to next_input_ids
       __snake_case :   Any													=    ids_tensor((self.batch_size, 3)      ,     config.vocab_size  )
       __snake_case :   str													=    ids_tensor((self.batch_size, 3)      ,     vocab_size=2  )

       # append to next input_ids and
       __snake_case :   int													=    torch.cat([input_ids, next_tokens]      ,     dim=-1  )
       __snake_case :   List[Any]													=    torch.cat([input_mask, next_mask]      ,     dim=-1  )

       __snake_case :   Dict													=    model(
           A_      ,     attention_mask=A_      ,     encoder_hidden_states=A_      ,     encoder_attention_mask=A_      ,     output_hidden_states=A_      ,     )['hidden_states'][0]
       __snake_case :   Optional[int]													=    model(
           A_      ,     attention_mask=A_      ,     encoder_hidden_states=A_      ,     encoder_attention_mask=A_      ,     past_key_values=A_      ,     output_hidden_states=A_      ,     )['hidden_states'][0]

       # select random slice
       __snake_case :   List[Any]													=    ids_tensor((1,)      ,     output_from_past.shape[-1]  ).item()
       __snake_case :   List[str]													=    output_from_no_past[:, -3:, random_slice_idx].detach()
       __snake_case :   int													=    output_from_past[:, :, random_slice_idx].detach()

       self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]  )

       # test that outputs are equal for slice
       self.parent.assertTrue(torch.allclose(A_      ,     A_      ,     atol=1e-3  )  )





 def A_   (   self						:							Any  )      ->   Optional[int]:







       '''simple docstring'''
       __snake_case :   Union[str, Any]													=    self.prepare_config_and_inputs()
       (
           (
           __snake_case	
       )							,     (
           __snake_case	
       )							,     (
           __snake_case	
       )							,     (
           __snake_case	
       )							,     (
           __snake_case	
       )							,     (
           __snake_case	
       )							,     (
           __snake_case	
       )							,     	
       ) :   List[str]													=    config_and_inputs
       __snake_case :   Tuple													=    {'input_ids': input_ids, 'attention_mask': input_mask}
       return config, inputs_dict



@require_torch
class  snake_case__						(    snake_case_  ,     snake_case_  ,     snake_case_  ,     unittest.TestCase				):
 A__				= (
     (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
 )
 A__				= (OpenLlamaForCausalLM,) if is_torch_available() else ()
 A__				= (
     {
         '''feature-extraction''': OpenLlamaModel,
         '''text-classification''': OpenLlamaForSequenceClassification,
         '''text-generation''': OpenLlamaForCausalLM,
         '''zero-shot''': OpenLlamaForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 A__				= False
 A__				= False
 def A_   (   self						:							List[str]  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case :   Union[str, Any]													=    OpenLlamaModelTester(self  )
       __snake_case :   int													=    ConfigTester(self      ,     config_class=A_      ,     hidden_size=37  )
 def A_   (   self						:							List[Any]  )      ->   Tuple:







       '''simple docstring'''
       self.config_tester.run_common_tests()
 def A_   (   self						:							List[Any]  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   List[str]													=    self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*A_  )
 def A_   (   self						:							Union[str, Any]  )      ->   Optional[int]:







       '''simple docstring'''
       __snake_case :   List[Any]													=    self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
             __snake_case :   Dict													=    type
             self.model_tester.create_and_check_model(*A_  )
 def A_   (   self						:							Dict  )      ->   Tuple:







       '''simple docstring'''
       __snake_case							,     __snake_case :   Union[str, Any]													=    self.model_tester.prepare_config_and_inputs_for_common()
       __snake_case :   Tuple													=    3
       __snake_case :   int													=    input_dict['input_ids']
       __snake_case :   int													=    input_ids.ne(1  ).to(A_  )
       __snake_case :   Any													=    ids_tensor([self.model_tester.batch_size]      ,     self.model_tester.type_sequence_label_size  )
       __snake_case :   str													=    OpenLlamaForSequenceClassification(A_  )
       model.to(A_  )
       model.eval()
       __snake_case :   str													=    model(A_      ,     attention_mask=A_      ,     labels=A_  )
       self.assertEqual(result.logits.shape      ,     (self.model_tester.batch_size, self.model_tester.num_labels)  )
 def A_   (   self						:							Union[str, Any]  )      ->   Tuple:







       '''simple docstring'''
       __snake_case							,     __snake_case :   List[Any]													=    self.model_tester.prepare_config_and_inputs_for_common()
       __snake_case :   Optional[int]													=    3
       __snake_case :   Optional[Any]													=    'single_label_classification'
       __snake_case :   Dict													=    input_dict['input_ids']
       __snake_case :   Optional[Any]													=    input_ids.ne(1  ).to(A_  )
       __snake_case :   Optional[int]													=    ids_tensor([self.model_tester.batch_size]      ,     self.model_tester.type_sequence_label_size  )
       __snake_case :   Any													=    OpenLlamaForSequenceClassification(A_  )
       model.to(A_  )
       model.eval()
       __snake_case :   Optional[int]													=    model(A_      ,     attention_mask=A_      ,     labels=A_  )
       self.assertEqual(result.logits.shape      ,     (self.model_tester.batch_size, self.model_tester.num_labels)  )
 def A_   (   self						:							List[str]  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case							,     __snake_case :   Any													=    self.model_tester.prepare_config_and_inputs_for_common()
       __snake_case :   Optional[int]													=    3
       __snake_case :   Tuple													=    'multi_label_classification'
       __snake_case :   Optional[int]													=    input_dict['input_ids']
       __snake_case :   str													=    input_ids.ne(1  ).to(A_  )
       __snake_case :   Dict													=    ids_tensor(
           [self.model_tester.batch_size, config.num_labels]      ,     self.model_tester.type_sequence_label_size  ).to(torch.float  )
       __snake_case :   Dict													=    OpenLlamaForSequenceClassification(A_  )
       model.to(A_  )
       model.eval()
       __snake_case :   Union[str, Any]													=    model(A_      ,     attention_mask=A_      ,     labels=A_  )
       self.assertEqual(result.logits.shape      ,     (self.model_tester.batch_size, self.model_tester.num_labels)  )
 @unittest.skip('Open-Llama buffers include complex numbers, which breaks this test'  )
 def A_   (   self						:							Union[str, Any]  )      ->   Any:







       '''simple docstring'''
       pass





 @parameterized.expand([('linear',), ('dynamic',)]  )
 def A_   (   self						:							Optional[int]      ,     __a						:							Optional[Any]  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case							,     __snake_case :   Optional[int]													=    self.model_tester.prepare_config_and_inputs_for_common()
       __snake_case :   Optional[int]													=    ids_tensor([1, 10]      ,     config.vocab_size  )
       __snake_case :   str													=    ids_tensor([1, int(config.max_position_embeddings * 1.5  )]      ,     config.vocab_size  )

       set_seed(42  )  # Fixed seed at init time so the two models get the same random weights
       __snake_case :   Optional[Any]													=    OpenLlamaModel(A_  )
       original_model.to(A_  )
       original_model.eval()
       __snake_case :   List[str]													=    original_model(A_  ).last_hidden_state
       __snake_case :   Union[str, Any]													=    original_model(A_  ).last_hidden_state

       set_seed(42  )  # Fixed seed at init time so the two models get the same random weights
       __snake_case :   Any													=    {'type': scaling_type, 'factor': 1_0.0}
       __snake_case :   Any													=    OpenLlamaModel(A_  )
       scaled_model.to(A_  )
       scaled_model.eval()
       __snake_case :   Dict													=    scaled_model(A_  ).last_hidden_state
       __snake_case :   Union[str, Any]													=    scaled_model(A_  ).last_hidden_state

       # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
       # maximum sequence length, so the outputs for the short input should match.
       if scaling_type == "dynamic":
             self.assertTrue(torch.allclose(A_      ,     A_      ,     atol=1e-5  )  )
       else:
             self.assertFalse(torch.allclose(A_      ,     A_      ,     atol=1e-5  )  )

       # The output should be different for long inputs
       self.assertFalse(torch.allclose(A_      ,     A_      ,     atol=1e-5  )  )





'''simple docstring'''
import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase					:    int           = logging.get_logger(__name__)

lowerCAmelCase					:    Tuple           = {
    'microsoft/unispeech-sat-base-100h-libri-ft': (
        'https://huggingface.co/microsoft/unispeech-sat-base-100h-libri-ft/resolve/main/config.json'
    ),
    # See all UniSpeechSat models at https://huggingface.co/models?filter=unispeech_sat
}


class 			SCREAMING_SNAKE_CASE__      (     snake_case_):
				lowerCAmelCase_     			=							"""unispeech-sat"""







				def __init__(       self						,				A_=32						,				A_=768						,				A_=12						,				A_=12						,				A_=3072						,				A_="gelu"						,				A_=0.1						,				A_=0.1						,				A_=0.1						,				A_=0.0						,				A_=0.0						,				A_=0.1						,				A_=0.1						,				A_=0.02						,				A_=1e-5						,				A_="group"						,				A_="gelu"						,				A_=(512, 512, 512, 512, 512, 512, 512)						,				A_=(5, 2, 2, 2, 2, 2, 2)						,				A_=(10, 3, 3, 3, 3, 2, 2)						,				A_=False						,				A_=128						,				A_=16						,				A_=False						,				A_=True						,				A_=0.05						,				A_=10						,				A_=2						,				A_=0.0						,				A_=10						,				A_=0						,				A_=320						,				A_=2						,				A_=0.1						,				A_=100						,				A_=256						,				A_=256						,				A_=0.1						,				A_="mean"						,				A_=False						,				A_=False						,				A_=256						,				A_=(512, 512, 512, 512, 1500)						,				A_=(5, 3, 3, 1, 1)						,				A_=(1, 2, 3, 1, 1)						,				A_=512						,				A_=0						,				A_=1						,				A_=2						,				A_=504						,				**A_						,				)->		Tuple:







											'''simple docstring'''






											super().__init__(**A_						,				pad_token_id=A_						,				bos_token_id=A_						,				eos_token_id=A_     )
											UpperCamelCase   			=     hidden_size
											UpperCamelCase   			=     feat_extract_norm
											UpperCamelCase   			=     feat_extract_activation
											UpperCamelCase   			=     list(A_     )
											UpperCamelCase   			=     list(A_     )
											UpperCamelCase   			=     list(A_     )
											UpperCamelCase   			=     conv_bias
											UpperCamelCase   			=     num_conv_pos_embeddings
											UpperCamelCase   			=     num_conv_pos_embedding_groups
											UpperCamelCase   			=     len(self.conv_dim     )
											UpperCamelCase   			=     num_hidden_layers
											UpperCamelCase   			=     intermediate_size
											UpperCamelCase   			=     hidden_act
											UpperCamelCase   			=     num_attention_heads
											UpperCamelCase   			=     hidden_dropout
											UpperCamelCase   			=     attention_dropout
											UpperCamelCase   			=     activation_dropout
											UpperCamelCase   			=     feat_proj_dropout
											UpperCamelCase   			=     final_dropout
											UpperCamelCase   			=     layerdrop
											UpperCamelCase   			=     layer_norm_eps
											UpperCamelCase   			=     initializer_range
											UpperCamelCase   			=     vocab_size
											UpperCamelCase   			=     num_clusters
											UpperCamelCase   			=     do_stable_layer_norm
											UpperCamelCase   			=     use_weighted_layer_sum

											if (
											    (len(self.conv_stride     ) != self.num_feat_extract_layers)
											    or (len(self.conv_kernel     ) != self.num_feat_extract_layers)
											    or (len(self.conv_dim     ) != self.num_feat_extract_layers)
											):
																		raise ValueError(
																		    'Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='
																		    ' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='
																		    F''' {len(self.conv_dim     )}`, `len(config.conv_stride) = {len(self.conv_stride     )}`,'''
																		    F''' `len(config.conv_kernel) = {len(self.conv_kernel     )}`.'''     )

											# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
											UpperCamelCase   			=     apply_spec_augment
											UpperCamelCase   			=     mask_time_prob
											UpperCamelCase   			=     mask_time_length
											UpperCamelCase   			=     mask_time_min_masks
											UpperCamelCase   			=     mask_feature_prob
											UpperCamelCase   			=     mask_feature_length
											UpperCamelCase   			=     mask_feature_min_masks

											# parameters for pretraining with codevector quantized representations
											UpperCamelCase   			=     num_codevectors_per_group
											UpperCamelCase   			=     num_codevector_groups
											UpperCamelCase   			=     contrastive_logits_temperature
											UpperCamelCase   			=     feat_quantizer_dropout
											UpperCamelCase   			=     num_negatives
											UpperCamelCase   			=     codevector_dim
											UpperCamelCase   			=     proj_codevector_dim
											UpperCamelCase   			=     diversity_loss_weight

											# ctc loss
											UpperCamelCase   			=     ctc_loss_reduction
											UpperCamelCase   			=     ctc_zero_infinity

											# SequenceClassification-specific parameter. Feel free to ignore for other classes.
											UpperCamelCase   			=     classifier_proj_size

											# XVector-specific parameters. Feel free to ignore for other classes.
											UpperCamelCase   			=     list(A_     )
											UpperCamelCase   			=     list(A_     )
											UpperCamelCase   			=     list(A_     )
											UpperCamelCase   			=     xvector_output_dim







				@property
				def 						UpperCAmelCase_    (       self     )->		Optional[Any]:







											'''simple docstring'''






											return functools.reduce(operator.mul						,				self.conv_stride						,				1     )




