




import unittest

from transformers import MraConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_torch_available():
 import torch

 from transformers import (
     MraForMaskedLM,
     MraForMultipleChoice,
     MraForQuestionAnswering,
     MraForSequenceClassification,
     MraForTokenClassification,
     MraModel,
 )
 from transformers.models.mra.modeling_mra import MRA_PRETRAINED_MODEL_ARCHIVE_LIST

class  _lowerCamelCase						:

      """simple docstring"""

      def __init__(     self					: Union[str, Any] ,       snake_case					: Any ,       snake_case					: Optional[Any]=2 ,       snake_case					: Dict=8 ,       snake_case					: Any=True ,       snake_case					: Dict=True ,       snake_case					: List[Any]=True ,       snake_case					: List[Any]=True ,       snake_case					: Tuple=99 ,       snake_case					: int=16 ,       snake_case					: Optional[int]=5 ,       snake_case					: Optional[int]=2 ,       snake_case					: Dict=36 ,       snake_case					: List[Any]="gelu" ,       snake_case					: Any=0.0 ,       snake_case					: Union[str, Any]=0.0 ,       snake_case					: Any=512 ,       snake_case					: Any=16 ,       snake_case					: Dict=2 ,       snake_case					: Dict=0.02 ,       snake_case					: str=3 ,       snake_case					: Any=4 ,       snake_case					: List[str]=None ,       ):
          __UpperCamelCase  		=						parent
          __UpperCamelCase  		=						batch_size
          __UpperCamelCase  		=						seq_length
          __UpperCamelCase  		=						is_training
          __UpperCamelCase  		=						use_input_mask
          __UpperCamelCase  		=						use_token_type_ids
          __UpperCamelCase  		=						use_labels
          __UpperCamelCase  		=						vocab_size
          __UpperCamelCase  		=						hidden_size
          __UpperCamelCase  		=						num_hidden_layers
          __UpperCamelCase  		=						num_attention_heads
          __UpperCamelCase  		=						intermediate_size
          __UpperCamelCase  		=						hidden_act
          __UpperCamelCase  		=						hidden_dropout_prob
          __UpperCamelCase  		=						attention_probs_dropout_prob
          __UpperCamelCase  		=						max_position_embeddings
          __UpperCamelCase  		=						type_vocab_size
          __UpperCamelCase  		=						type_sequence_label_size
          __UpperCamelCase  		=						initializer_range
          __UpperCamelCase  		=						num_labels
          __UpperCamelCase  		=						num_choices
          __UpperCamelCase  		=						scope







      def       snake_case	(     self					: Union[str, Any]				):
          __UpperCamelCase  		=						ids_tensor([self.batch_size, self.seq_length] ,       self.vocab_size				)

          __UpperCamelCase  		=						None
          if self.use_input_mask:
              __UpperCamelCase  		=						random_attention_mask([self.batch_size, self.seq_length]				)

          __UpperCamelCase  		=						None
          if self.use_token_type_ids:
              __UpperCamelCase  		=						ids_tensor([self.batch_size, self.seq_length] ,       self.type_vocab_size				)

          __UpperCamelCase  		=						None
          __UpperCamelCase  		=						None
          __UpperCamelCase  		=						None
          if self.use_labels:
              __UpperCamelCase  		=						ids_tensor([self.batch_size] ,       self.type_sequence_label_size				)
              __UpperCamelCase  		=						ids_tensor([self.batch_size, self.seq_length] ,       self.num_labels				)
              __UpperCamelCase  		=						ids_tensor([self.batch_size] ,       self.num_choices				)

          __UpperCamelCase  		=						self.get_config()

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







      def       snake_case	(     self					: Optional[Any]				):
          return MraConfig(
              vocab_size=self.vocab_size ,       hidden_size=self.hidden_size ,       num_hidden_layers=self.num_hidden_layers ,       num_attention_heads=self.num_attention_heads ,       intermediate_size=self.intermediate_size ,       hidden_act=self.hidden_act ,       hidden_dropout_prob=self.hidden_dropout_prob ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob ,       max_position_embeddings=self.max_position_embeddings ,       type_vocab_size=self.type_vocab_size ,       is_decoder=snake_case ,       initializer_range=self.initializer_range ,       )







      def       snake_case	(     self					: Any				):
          __UpperCamelCase  		=						self.get_config()
          __UpperCamelCase  		=						300
          return config







      def       snake_case	(     self					: List[Any]				):
          (
              (
              __UpperCamelCase      
          )		,      (
              __UpperCamelCase      
          )		,      (
              __UpperCamelCase      
          )		,      (
              __UpperCamelCase      
          )		,      (
              __UpperCamelCase      
          )		,      (
              __UpperCamelCase      
          )		,      (
              __UpperCamelCase      
          )		,            
          )  		=						self.prepare_config_and_inputs()

          __UpperCamelCase  		=						True
          __UpperCamelCase  		=						floats_tensor([self.batch_size, self.seq_length, self.hidden_size]				)
          __UpperCamelCase  		=						ids_tensor([self.batch_size, self.seq_length] ,       vocab_size=2				)

          return (
              config,
              input_ids,
              token_type_ids,
              input_mask,
              sequence_labels,
              token_labels,
              choice_labels,
              encoder_hidden_states,
              encoder_attention_mask,
          )







      def       snake_case	(     self					: Dict ,       snake_case					: Optional[int] ,       snake_case					: List[Any] ,       snake_case					: Any ,       snake_case					: Tuple ,       snake_case					: Optional[int] ,       snake_case					: int ,       snake_case					: List[Any]				):
          __UpperCamelCase  		=						MraModel(config=snake_case				)
          model.to(snake_case				)
          model.eval()
          __UpperCamelCase  		=						model(snake_case ,       attention_mask=snake_case ,       token_type_ids=snake_case				)
          __UpperCamelCase  		=						model(snake_case ,       token_type_ids=snake_case				)
          __UpperCamelCase  		=						model(snake_case				)
          self.parent.assertEqual(result.last_hidden_state.shape ,       (self.batch_size, self.seq_length, self.hidden_size)				)







      def       snake_case	(     self					: List[str] ,       snake_case					: Optional[int] ,       snake_case					: List[str] ,       snake_case					: Union[str, Any] ,       snake_case					: List[Any] ,       snake_case					: List[Any] ,       snake_case					: Optional[int] ,       snake_case					: Any ,       snake_case					: List[Any] ,       snake_case					: Optional[int] ,       ):
          __UpperCamelCase  		=						True
          __UpperCamelCase  		=						MraModel(snake_case				)
          model.to(snake_case				)
          model.eval()
          __UpperCamelCase  		=						model(
              snake_case ,       attention_mask=snake_case ,       token_type_ids=snake_case ,       encoder_hidden_states=snake_case ,       encoder_attention_mask=snake_case ,       )
          __UpperCamelCase  		=						model(
              snake_case ,       attention_mask=snake_case ,       token_type_ids=snake_case ,       encoder_hidden_states=snake_case ,       )
          __UpperCamelCase  		=						model(snake_case ,       attention_mask=snake_case ,       token_type_ids=snake_case				)
          self.parent.assertEqual(result.last_hidden_state.shape ,       (self.batch_size, self.seq_length, self.hidden_size)				)







      def       snake_case	(     self					: Dict ,       snake_case					: str ,       snake_case					: Optional[Any] ,       snake_case					: List[Any] ,       snake_case					: List[Any] ,       snake_case					: List[str] ,       snake_case					: Any ,       snake_case					: Optional[int]				):
          __UpperCamelCase  		=						MraForMaskedLM(config=snake_case				)
          model.to(snake_case				)
          model.eval()
          __UpperCamelCase  		=						model(snake_case ,       attention_mask=snake_case ,       token_type_ids=snake_case ,       labels=snake_case				)
          self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.seq_length, self.vocab_size)				)







      def       snake_case	(     self					: Union[str, Any] ,       snake_case					: str ,       snake_case					: Dict ,       snake_case					: List[str] ,       snake_case					: Tuple ,       snake_case					: Any ,       snake_case					: List[Any] ,       snake_case					: Any				):
          __UpperCamelCase  		=						MraForQuestionAnswering(config=snake_case				)
          model.to(snake_case				)
          model.eval()
          __UpperCamelCase  		=						model(
              snake_case ,       attention_mask=snake_case ,       token_type_ids=snake_case ,       start_positions=snake_case ,       end_positions=snake_case ,       )
          self.parent.assertEqual(result.start_logits.shape ,       (self.batch_size, self.seq_length)				)
          self.parent.assertEqual(result.end_logits.shape ,       (self.batch_size, self.seq_length)				)







      def       snake_case	(     self					: Any ,       snake_case					: int ,       snake_case					: Tuple ,       snake_case					: int ,       snake_case					: List[Any] ,       snake_case					: Union[str, Any] ,       snake_case					: int ,       snake_case					: Optional[int]				):
          __UpperCamelCase  		=						self.num_labels
          __UpperCamelCase  		=						MraForSequenceClassification(snake_case				)
          model.to(snake_case				)
          model.eval()
          __UpperCamelCase  		=						model(snake_case ,       attention_mask=snake_case ,       token_type_ids=snake_case ,       labels=snake_case				)
          self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.num_labels)				)







      def       snake_case	(     self					: Tuple ,       snake_case					: Union[str, Any] ,       snake_case					: Optional[int] ,       snake_case					: Dict ,       snake_case					: List[Any] ,       snake_case					: Any ,       snake_case					: Tuple ,       snake_case					: Union[str, Any]				):
          __UpperCamelCase  		=						self.num_labels
          __UpperCamelCase  		=						MraForTokenClassification(config=snake_case				)
          model.to(snake_case				)
          model.eval()
          __UpperCamelCase  		=						model(snake_case ,       attention_mask=snake_case ,       token_type_ids=snake_case ,       labels=snake_case				)
          self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.seq_length, self.num_labels)				)







      def       snake_case	(     self					: List[Any] ,       snake_case					: Optional[Any] ,       snake_case					: List[str] ,       snake_case					: int ,       snake_case					: List[Any] ,       snake_case					: int ,       snake_case					: Optional[Any] ,       snake_case					: Dict				):
          __UpperCamelCase  		=						self.num_choices
          __UpperCamelCase  		=						MraForMultipleChoice(config=snake_case				)
          model.to(snake_case				)
          model.eval()
          __UpperCamelCase  		=						input_ids.unsqueeze(1				).expand(-1 ,       self.num_choices ,       -1				).contiguous()
          __UpperCamelCase  		=						token_type_ids.unsqueeze(1				).expand(-1 ,       self.num_choices ,       -1				).contiguous()
          __UpperCamelCase  		=						input_mask.unsqueeze(1				).expand(-1 ,       self.num_choices ,       -1				).contiguous()
          __UpperCamelCase  		=						model(
              snake_case ,       attention_mask=snake_case ,       token_type_ids=snake_case ,       labels=snake_case ,       )
          self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.num_choices)				)







      def       snake_case	(     self					: Dict				):
          __UpperCamelCase  		=						self.prepare_config_and_inputs()
          (
              (
              __UpperCamelCase      
          )		,      (
              __UpperCamelCase      
          )		,      (
              __UpperCamelCase      
          )		,      (
              __UpperCamelCase      
          )		,      (
              __UpperCamelCase      
          )		,      (
              __UpperCamelCase      
          )		,      (
              __UpperCamelCase      
          )		,            
          )  		=						config_and_inputs
          __UpperCamelCase  		=						{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
          return config, inputs_dict







@require_torch
class  _lowerCamelCase						(			_SCREAMING_SNAKE_CASE							,			unittest.TestCase   ):

      """simple docstring"""

      lowerCAmelCase__			: Optional[Any]    =					(
          (
              MraModel,
              MraForMaskedLM,
              MraForMultipleChoice,
              MraForQuestionAnswering,
              MraForSequenceClassification,
              MraForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      lowerCAmelCase__			: List[Any]    =					False
      lowerCAmelCase__			: Union[str, Any]    =					False
      lowerCAmelCase__			: List[str]    =					False
      lowerCAmelCase__			: Union[str, Any]    =					False

      lowerCAmelCase__			: str    =					()







      def       snake_case	(     self					: Any				):
          __UpperCamelCase  		=						MraModelTester(self				)
          __UpperCamelCase  		=						ConfigTester(self ,       config_class=snake_case ,       hidden_size=37				)







      def       snake_case	(     self					: int				):
          self.config_tester.run_common_tests()







      def       snake_case	(     self					: Optional[Any]				):
          __UpperCamelCase  		=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*snake_case				)







      def       snake_case	(     self					: Dict				):
          __UpperCamelCase  		=						self.model_tester.prepare_config_and_inputs()
          for type in ["absolute", "relative_key", "relative_key_query"]:
              __UpperCamelCase  		=						type
              self.model_tester.create_and_check_model(*snake_case				)







      def       snake_case	(     self					: Optional[Any]				):
          __UpperCamelCase  		=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*snake_case				)







      def       snake_case	(     self					: Optional[Any]				):
          __UpperCamelCase  		=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*snake_case				)







      def       snake_case	(     self					: Dict				):
          __UpperCamelCase  		=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*snake_case				)







      def       snake_case	(     self					: List[Any]				):
          __UpperCamelCase  		=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*snake_case				)







      def       snake_case	(     self					: Any				):
          __UpperCamelCase  		=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*snake_case				)







      @slow
      def       snake_case	(     self					: List[Any]				):
          for model_name in MRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              __UpperCamelCase  		=						MraModel.from_pretrained(snake_case				)
              self.assertIsNotNone(snake_case				)







      @unittest.skip(reason='''MRA does not output attentions'''				)
      def       snake_case	(     self					: List[str]				):
          return







@require_torch
class  _lowerCamelCase						(			unittest.TestCase   ):

      """simple docstring"""

      @slow
      def       snake_case	(     self					: int				):
          __UpperCamelCase  		=						MraModel.from_pretrained('''uw-madison/mra-base-512-4'''				)
          __UpperCamelCase  		=						torch.arange(256				).unsqueeze(0				)

          with torch.no_grad():
              __UpperCamelCase  		=						model(snake_case				)[0]

          __UpperCamelCase  		=						torch.Size((1, 256, 768)				)
          self.assertEqual(output.shape ,       snake_case				)

          __UpperCamelCase  		=						torch.tensor(
              [[[-0.01_40, 0.08_30, -0.03_81], [0.15_46, 0.14_02, 0.02_20], [0.11_62, 0.08_51, 0.01_65]]]				)

          self.assertTrue(torch.allclose(output[:, :3, :3] ,       snake_case ,       atol=1E-4				)				)







      @slow
      def       snake_case	(     self					: Optional[int]				):
          __UpperCamelCase  		=						MraForMaskedLM.from_pretrained('''uw-madison/mra-base-512-4'''				)
          __UpperCamelCase  		=						torch.arange(256				).unsqueeze(0				)

          with torch.no_grad():
              __UpperCamelCase  		=						model(snake_case				)[0]

          __UpperCamelCase  		=						50265

          __UpperCamelCase  		=						torch.Size((1, 256, vocab_size)				)
          self.assertEqual(output.shape ,       snake_case				)

          __UpperCamelCase  		=						torch.tensor(
              [[[9.25_95, -3.60_38, 11.88_19], [9.38_69, -3.26_93, 11.09_56], [11.85_24, -3.49_38, 13.12_10]]]				)

          self.assertTrue(torch.allclose(output[:, :3, :3] ,       snake_case ,       atol=1E-4				)				)







      @slow
      def       snake_case	(     self					: Union[str, Any]				):
          __UpperCamelCase  		=						MraForMaskedLM.from_pretrained('''uw-madison/mra-base-4096-8-d3'''				)
          __UpperCamelCase  		=						torch.arange(4096				).unsqueeze(0				)

          with torch.no_grad():
              __UpperCamelCase  		=						model(snake_case				)[0]

          __UpperCamelCase  		=						50265

          __UpperCamelCase  		=						torch.Size((1, 4096, vocab_size)				)
          self.assertEqual(output.shape ,       snake_case				)

          __UpperCamelCase  		=						torch.tensor(
              [[[5.47_89, -2.35_64, 7.50_64], [7.90_67, -1.33_69, 9.96_68], [9.07_12, -1.81_06, 7.03_80]]]				)

          self.assertTrue(torch.allclose(output[:, :3, :3] ,       snake_case ,       atol=1E-4				)				)








import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_   		=     logging.get_logger(__name__)

a_   		=     {
    "microsoft/unispeech-large-1500h-cv": (
        "https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json"
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}

class  _lowerCamelCase						(			_SCREAMING_SNAKE_CASE   ):

      """simple docstring"""

      lowerCAmelCase__			: Dict    =					"unispeech"







      def __init__(     self					: str ,       snake_case					: Union[str, Any]=32 ,       snake_case					: Optional[Any]=768 ,       snake_case					: Dict=12 ,       snake_case					: Tuple=12 ,       snake_case					: Optional[Any]=3072 ,       snake_case					: Any="gelu" ,       snake_case					: Dict=0.1 ,       snake_case					: Tuple=0.1 ,       snake_case					: str=0.1 ,       snake_case					: Optional[int]=0.0 ,       snake_case					: Any=0.0 ,       snake_case					: Optional[Any]=0.1 ,       snake_case					: List[Any]=0.1 ,       snake_case					: Optional[int]=0.02 ,       snake_case					: List[str]=1E-5 ,       snake_case					: str="group" ,       snake_case					: List[Any]="gelu" ,       snake_case					: Union[str, Any]=(512, 512, 512, 512, 512, 512, 512) ,       snake_case					: List[Any]=(5, 2, 2, 2, 2, 2, 2) ,       snake_case					: Union[str, Any]=(10, 3, 3, 3, 3, 2, 2) ,       snake_case					: Tuple=False ,       snake_case					: Optional[int]=128 ,       snake_case					: List[str]=16 ,       snake_case					: List[str]=False ,       snake_case					: Dict=True ,       snake_case					: Optional[Any]=0.05 ,       snake_case					: Optional[Any]=10 ,       snake_case					: Union[str, Any]=2 ,       snake_case					: List[str]=0.0 ,       snake_case					: str=10 ,       snake_case					: int=0 ,       snake_case					: Tuple=320 ,       snake_case					: Any=2 ,       snake_case					: List[str]=0.1 ,       snake_case					: Optional[Any]=100 ,       snake_case					: List[Any]=256 ,       snake_case					: Union[str, Any]=256 ,       snake_case					: Any=0.1 ,       snake_case					: str="mean" ,       snake_case					: Union[str, Any]=False ,       snake_case					: str=False ,       snake_case					: Union[str, Any]=256 ,       snake_case					: Optional[Any]=80 ,       snake_case					: str=0 ,       snake_case					: int=1 ,       snake_case					: int=2 ,       snake_case					: Dict=0.5 ,       **snake_case					: Optional[int] ,       ):
          super().__init__(**snake_case ,       pad_token_id=snake_case ,       bos_token_id=snake_case ,       eos_token_id=snake_case				)
          __UpperCamelCase  		=						hidden_size
          __UpperCamelCase  		=						feat_extract_norm
          __UpperCamelCase  		=						feat_extract_activation
          __UpperCamelCase  		=						list(snake_case				)
          __UpperCamelCase  		=						list(snake_case				)
          __UpperCamelCase  		=						list(snake_case				)
          __UpperCamelCase  		=						conv_bias
          __UpperCamelCase  		=						num_conv_pos_embeddings
          __UpperCamelCase  		=						num_conv_pos_embedding_groups
          __UpperCamelCase  		=						len(self.conv_dim				)
          __UpperCamelCase  		=						num_hidden_layers
          __UpperCamelCase  		=						intermediate_size
          __UpperCamelCase  		=						hidden_act
          __UpperCamelCase  		=						num_attention_heads
          __UpperCamelCase  		=						hidden_dropout
          __UpperCamelCase  		=						attention_dropout
          __UpperCamelCase  		=						activation_dropout
          __UpperCamelCase  		=						feat_proj_dropout
          __UpperCamelCase  		=						final_dropout
          __UpperCamelCase  		=						layerdrop
          __UpperCamelCase  		=						layer_norm_eps
          __UpperCamelCase  		=						initializer_range
          __UpperCamelCase  		=						num_ctc_classes
          __UpperCamelCase  		=						vocab_size
          __UpperCamelCase  		=						do_stable_layer_norm
          __UpperCamelCase  		=						use_weighted_layer_sum
          __UpperCamelCase  		=						classifier_proj_size

          if (
              (len(self.conv_stride				) != self.num_feat_extract_layers)
              or (len(self.conv_kernel				) != self.num_feat_extract_layers)
              or (len(self.conv_dim				) != self.num_feat_extract_layers)
          ):
              raise ValueError(
                  '''Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='''
                  ''' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='''
                  F" {len(self.conv_dim				)}`, `len(config.conv_stride) = {len(self.conv_stride				)}`,"
                  F" `len(config.conv_kernel) = {len(self.conv_kernel				)}`."				)

          # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
          __UpperCamelCase  		=						apply_spec_augment
          __UpperCamelCase  		=						mask_time_prob
          __UpperCamelCase  		=						mask_time_length
          __UpperCamelCase  		=						mask_time_min_masks
          __UpperCamelCase  		=						mask_feature_prob
          __UpperCamelCase  		=						mask_feature_length
          __UpperCamelCase  		=						mask_feature_min_masks

          # parameters for pretraining with codevector quantized representations
          __UpperCamelCase  		=						num_codevectors_per_group
          __UpperCamelCase  		=						num_codevector_groups
          __UpperCamelCase  		=						contrastive_logits_temperature
          __UpperCamelCase  		=						feat_quantizer_dropout
          __UpperCamelCase  		=						num_negatives
          __UpperCamelCase  		=						codevector_dim
          __UpperCamelCase  		=						proj_codevector_dim
          __UpperCamelCase  		=						diversity_loss_weight

          # ctc loss
          __UpperCamelCase  		=						ctc_loss_reduction
          __UpperCamelCase  		=						ctc_zero_infinity

          # pretraining loss
          __UpperCamelCase  		=						replace_prob







      @property
      def       snake_case	(     self					: Dict				):
          return functools.reduce(operator.mul ,       self.conv_stride ,       1				)






