



import argparse
import json
import os

import fairseq
import torch
from fairseq.data import Dictionary

from transformers import (
    WavaVecaConformerConfig,
    WavaVecaConformerForCTC,
    WavaVecaConformerForPreTraining,
    WavaVecaCTCTokenizer,
    WavaVecaFeatureExtractor,
    WavaVecaProcessor,
    logging,
)


logging.set_verbosity_info()
a_								=       logging.get_logger(__name__)

a_								=       {
    '''post_extract_proj''': '''feature_projection.projection''',
    '''encoder.pos_conv.0''': '''encoder.pos_conv_embed.conv''',
    '''self_attn.linear_k''': '''encoder.layers.*.self_attn.linear_k''',
    '''self_attn.linear_v''': '''encoder.layers.*.self_attn.linear_v''',
    '''self_attn.linear_q''': '''encoder.layers.*.self_attn.linear_q''',
    '''self_attn.pos_bias_u''': '''encoder.layers.*.self_attn.pos_bias_u''',
    '''self_attn.pos_bias_v''': '''encoder.layers.*.self_attn.pos_bias_v''',
    '''self_attn.linear_out''': '''encoder.layers.*.self_attn.linear_out''',
    '''self_attn.linear_pos''': '''encoder.layers.*.self_attn.linear_pos''',
    '''self_attn.rotary_emb''': '''encoder.embed_positions''',
    '''self_attn_layer_norm''': '''encoder.layers.*.self_attn_layer_norm''',
    '''conv_module.pointwise_conv1''': '''encoder.layers.*.conv_module.pointwise_conv1''',
    '''conv_module.pointwise_conv2''': '''encoder.layers.*.conv_module.pointwise_conv2''',
    '''conv_module.depthwise_conv''': '''encoder.layers.*.conv_module.depthwise_conv''',
    '''conv_module.batch_norm''': '''encoder.layers.*.conv_module.batch_norm''',
    '''conv_module.layer_norm''': '''encoder.layers.*.conv_module.layer_norm''',
    '''ffn1.w_1''': '''encoder.layers.*.ffn1.intermediate_dense''',
    '''ffn1.w_2''': '''encoder.layers.*.ffn1.output_dense''',
    '''ffn1.layer_norm''': '''encoder.layers.*.ffn1_layer_norm''',
    '''ffn2.w_1''': '''encoder.layers.*.ffn2.intermediate_dense''',
    '''ffn2.w_2''': '''encoder.layers.*.ffn2.output_dense''',
    '''ffn2.layer_norm''': '''encoder.layers.*.ffn2_layer_norm''',
    '''final_layer_norm''': '''encoder.layers.*.final_layer_norm''',
    '''encoder.layer_norm''': '''encoder.layer_norm''',
    '''w2v_model.layer_norm''': '''feature_projection.layer_norm''',
    '''quantizer.weight_proj''': '''quantizer.weight_proj''',
    '''quantizer.vars''': '''quantizer.codevectors''',
    '''project_q''': '''project_q''',
    '''final_proj''': '''project_hid''',
    '''w2v_encoder.proj''': '''lm_head''',
    '''mask_emb''': '''masked_spec_embed''',
}
a_								=       [
    '''lm_head''',
    '''quantizer.weight_proj''',
    '''quantizer.codevectors''',
    '''project_q''',
    '''project_hid''',
]





def 		a__      (		_UpperCamelCase      :			Optional[int]			,_UpperCamelCase      :			Optional[int]			,_UpperCamelCase      :			str			,_UpperCamelCase      :			Any			,_UpperCamelCase      :			List[Any]				):





   for attribute in key.split('''.'''				):
      __lowerCamelCase    				=     getattr(UpperCAmelCase_			,UpperCAmelCase_				)

   if weight_type is not None:
      __lowerCamelCase    				=     getattr(UpperCAmelCase_			,UpperCAmelCase_				).shape
   else:
      __lowerCamelCase    				=     hf_pointer.shape

   if hf_shape != value.shape:
      raise ValueError(
          F"""Shape of hf {key + "." + weight_type if weight_type is not None else ""} is {hf_shape}, but should be"""
          F""" {value.shape} for {full_name}"""				)

   if weight_type == "weight":
      __lowerCamelCase    				=     value
   elif weight_type == "weight_g":
      __lowerCamelCase    				=     value
   elif weight_type == "weight_v":
      __lowerCamelCase    				=     value
   elif weight_type == "bias":
      __lowerCamelCase    				=     value
   elif weight_type == "running_mean":
      __lowerCamelCase    				=     value
   elif weight_type == "running_var":
      __lowerCamelCase    				=     value
   elif weight_type == "num_batches_tracked":
      __lowerCamelCase    				=     value
   elif weight_type == "inv_freq":
      __lowerCamelCase    				=     value
   else:
      __lowerCamelCase    				=     value

   logger.info(F"""{key + "." + weight_type if weight_type is not None else ""} was initialized from {full_name}."""				)





def 		a__      (		_UpperCamelCase      :			List[Any]			,_UpperCamelCase      :			int			,_UpperCamelCase      :			List[str]				):
   __lowerCamelCase    				=     []
   __lowerCamelCase    				=     fairseq_model.state_dict()

   __lowerCamelCase    				=     hf_model.wavaveca_conformer.feature_extractor

   for name, value in fairseq_dict.items():
      __lowerCamelCase    				=     False
      if "conv_layers" in name:
         load_conv_layer(
             UpperCAmelCase_			,UpperCAmelCase_			,UpperCAmelCase_			,UpperCAmelCase_			,hf_model.config.feat_extract_norm == '''group'''			,)
         __lowerCamelCase    				=     True
      else:
         for key, mapped_key in MAPPING.items():
            __lowerCamelCase    				=     '''wav2vec2_conformer.''' + mapped_key if mapped_key not in TOP_LEVEL_KEYS else mapped_key
            if key in name or key.split('''w2v_model.'''				)[-1] == name.split('''.'''				)[0]:
               __lowerCamelCase    				=     True
               if "*" in mapped_key:
                  __lowerCamelCase    				=     name.split(UpperCAmelCase_				)[0].split('''.'''				)[-2]
                  __lowerCamelCase    				=     mapped_key.replace('''*'''			,UpperCAmelCase_				)
               if "pos_bias_u" in name:
                  __lowerCamelCase    				=     None
               elif "pos_bias_v" in name:
                  __lowerCamelCase    				=     None
               elif "weight_g" in name:
                  __lowerCamelCase    				=     '''weight_g'''
               elif "weight_v" in name:
                  __lowerCamelCase    				=     '''weight_v'''
               elif "bias" in name:
                  __lowerCamelCase    				=     '''bias'''
               elif "weight" in name:
                  # TODO: don't match quantizer.weight_proj
                  __lowerCamelCase    				=     '''weight'''
               elif "running_mean" in name:
                  __lowerCamelCase    				=     '''running_mean'''
               elif "inv_freq" in name:
                  __lowerCamelCase    				=     '''inv_freq'''
               elif "running_var" in name:
                  __lowerCamelCase    				=     '''running_var'''
               elif "num_batches_tracked" in name:
                  __lowerCamelCase    				=     '''num_batches_tracked'''
               else:
                  __lowerCamelCase    				=     None
               set_recursively(UpperCAmelCase_			,UpperCAmelCase_			,UpperCAmelCase_			,UpperCAmelCase_			,UpperCAmelCase_				)
            continue
      if not is_used:
         unused_weights.append(UpperCAmelCase_				)

   logger.warning(F"""Unused weights: {unused_weights}"""				)





def 		a__      (		_UpperCamelCase      :			List[Any]			,_UpperCamelCase      :			Dict			,_UpperCamelCase      :			int			,_UpperCamelCase      :			Tuple			,_UpperCamelCase      :			Dict				):
   __lowerCamelCase    				=     full_name.split('''conv_layers.'''				)[-1]
   __lowerCamelCase    				=     name.split('''.'''				)
   __lowerCamelCase    				=     int(items[0]				)
   __lowerCamelCase    				=     int(items[1]				)

   if type_id == 0:
      if "bias" in name:
         if value.shape != feature_extractor.conv_layers[layer_id].conv.bias.data.shape:
            raise ValueError(
                F"""{full_name} has size {value.shape}, but"""
                F""" {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found."""				)
         __lowerCamelCase    				=     value
         logger.info(F"""Feat extract conv layer {layer_id} was initialized from {full_name}."""				)
      elif "weight" in name:
         if value.shape != feature_extractor.conv_layers[layer_id].conv.weight.data.shape:
            raise ValueError(
                F"""{full_name} has size {value.shape}, but"""
                F""" {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found."""				)
         __lowerCamelCase    				=     value
         logger.info(F"""Feat extract conv layer {layer_id} was initialized from {full_name}."""				)
   elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
      if "bias" in name:
         if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape:
            raise ValueError(
                F"""{full_name} has size {value.shape}, but"""
                F""" {feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape} was found."""				)
         __lowerCamelCase    				=     value
         logger.info(F"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}."""				)
      elif "weight" in name:
         if value.shape != feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape:
            raise ValueError(
                F"""{full_name} has size {value.shape}, but"""
                F""" {feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape} was found."""				)
         __lowerCamelCase    				=     value
         logger.info(F"""Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}."""				)
   else:
      unused_weights.append(UpperCAmelCase_				)





@torch.no_grad()
def 		a__      (		_UpperCamelCase      :			Any			,_UpperCamelCase      :			Tuple			,_UpperCamelCase      :			Union[str, Any]=None			,_UpperCamelCase      :			List[str]=None			,_UpperCamelCase      :			Tuple=True				):





   if config_path is not None:
      __lowerCamelCase    				=     WavaVecaConformerConfig.from_pretrained(UpperCAmelCase_			,hidden_act='''swish'''				)
   else:
      __lowerCamelCase    				=     WavaVecaConformerConfig()

   if "rope" in checkpoint_path:
      __lowerCamelCase    				=     '''rotary'''

   if is_finetuned:
      if dict_path:
         __lowerCamelCase    				=     Dictionary.load(UpperCAmelCase_				)

         # important change bos & pad token id since CTC symbol is <pad> and
         # not <s> as in fairseq
         __lowerCamelCase    				=     target_dict.pad_index
         __lowerCamelCase    				=     target_dict.bos_index
         __lowerCamelCase    				=     target_dict.eos_index
         __lowerCamelCase    				=     len(target_dict.symbols				)
         __lowerCamelCase    				=     os.path.join(UpperCAmelCase_			,'''vocab.json'''				)
         if not os.path.isdir(UpperCAmelCase_				):
            logger.error('''--pytorch_dump_folder_path ({}) should be a directory'''.format(UpperCAmelCase_				)				)
            return
         os.makedirs(UpperCAmelCase_			,exist_ok=UpperCAmelCase_				)
         __lowerCamelCase    				=     target_dict.indices

         # fairseq has the <pad> and <s> switched
         __lowerCamelCase    				=     0
         __lowerCamelCase    				=     1
         with open(UpperCAmelCase_			,'''w'''			,encoding='''utf-8'''				) as vocab_handle:
            json.dump(UpperCAmelCase_			,UpperCAmelCase_				)
         __lowerCamelCase    				=     WavaVecaCTCTokenizer(
             UpperCAmelCase_			,unk_token=target_dict.unk_word			,pad_token=target_dict.pad_word			,bos_token=target_dict.bos_word			,eos_token=target_dict.eos_word			,word_delimiter_token='''|'''			,do_lower_case=UpperCAmelCase_			,)
         __lowerCamelCase    				=     True if config.feat_extract_norm == '''layer''' else False
         __lowerCamelCase    				=     WavaVecaFeatureExtractor(
             feature_size=1			,sampling_rate=1_60_00			,padding_value=0			,do_normalize=UpperCAmelCase_			,return_attention_mask=UpperCAmelCase_			,)
         __lowerCamelCase    				=     WavaVecaProcessor(feature_extractor=UpperCAmelCase_			,tokenizer=UpperCAmelCase_				)
         processor.save_pretrained(UpperCAmelCase_				)

      __lowerCamelCase    				=     WavaVecaConformerForCTC(UpperCAmelCase_				)
   else:
      __lowerCamelCase    				=     WavaVecaConformerForPreTraining(UpperCAmelCase_				)

   if is_finetuned:
      __lowerCamelCase    				=     fairseq.checkpoint_utils.load_model_ensemble_and_task(
          [checkpoint_path]			,arg_overrides={'''data''': '''/'''.join(dict_path.split('''/'''				)[:-1]				)}				)
   else:
      __lowerCamelCase    				=     argparse.Namespace(task='''audio_pretraining'''				)
      __lowerCamelCase    				=     fairseq.tasks.setup_task(UpperCAmelCase_				)

      __lowerCamelCase    				=     fairseq.checkpoint_utils.load_model_ensemble_and_task([checkpoint_path]			,task=UpperCAmelCase_				)

   __lowerCamelCase    				=     model[0].eval()

   recursively_load_weights(UpperCAmelCase_			,UpperCAmelCase_			,not is_finetuned				)

   hf_wavavec.save_pretrained(UpperCAmelCase_				)


if __name__ == "__main__":
  a_								=       argparse.ArgumentParser()
  parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
  parser.add_argument("""--checkpoint_path""", default=None, type=str, help="""Path to fairseq checkpoint""")
  parser.add_argument("""--dict_path""", default=None, type=str, help="""Path to dict of fine-tuned model""")
  parser.add_argument("""--config_path""", default=None, type=str, help="""Path to hf config.json of model to convert""")
  parser.add_argument(
      """--not_finetuned""", action="""store_true""", help="""Whether the model to convert is a fine-tuned model or not"""
  )
  a_								=       parser.parse_args()
  convert_wavaveca_conformer_checkpoint(
      args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path, args.dict_path, not args.not_finetuned
  )










import unittest

from transformers import AutoTokenizer, NystromformerConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         NystromformerForMaskedLM,
         NystromformerForMultipleChoice,
         NystromformerForQuestionAnswering,
         NystromformerForSequenceClassification,
         NystromformerForTokenClassification,
         NystromformerModel,
     )
     from transformers.models.nystromformer.modeling_nystromformer import NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST







class 						_snake_case    :




 def __init__(						self   ,    _lowerCamelCase   ,    _lowerCamelCase=13   ,    _lowerCamelCase=7   ,    _lowerCamelCase=True   ,    _lowerCamelCase=True   ,    _lowerCamelCase=True   ,    _lowerCamelCase=True   ,    _lowerCamelCase=99   ,    _lowerCamelCase=32   ,    _lowerCamelCase=5   ,    _lowerCamelCase=4   ,    _lowerCamelCase=37   ,    _lowerCamelCase="gelu"   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=512   ,    _lowerCamelCase=16   ,    _lowerCamelCase=2   ,    _lowerCamelCase=0.02   ,    _lowerCamelCase=3   ,    _lowerCamelCase=4   ,    _lowerCamelCase=None   ,    ):
       a		:int									=			parent
       a		:List[Any]									=			batch_size
       a		:Optional[int]									=			seq_length
       a		:List[Any]									=			is_training
       a		:Union[str, Any]									=			use_input_mask
       a		:Tuple									=			use_token_type_ids
       a		:Tuple									=			use_labels
       a		:int									=			vocab_size
       a		:int									=			hidden_size
       a		:Optional[Any]									=			num_hidden_layers
       a		:List[str]									=			num_attention_heads
       a		:str									=			intermediate_size
       a		:Union[str, Any]									=			hidden_act
       a		:Tuple									=			hidden_dropout_prob
       a		:Dict									=			attention_probs_dropout_prob
       a		:Tuple									=			max_position_embeddings
       a		:int									=			type_vocab_size
       a		:Union[str, Any]									=			type_sequence_label_size
       a		:List[Any]									=			initializer_range
       a		:Any									=			num_labels
       a		:int									=			num_choices
       a		:Optional[Any]									=			scope




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:str									=			ids_tensor([self.batch_size, self.seq_length]   ,    self.vocab_size       )

       a		:Dict									=			None
       if self.use_input_mask:
             a		:Optional[int]									=			random_attention_mask([self.batch_size, self.seq_length]       )

       a		:Dict									=			None
       if self.use_token_type_ids:
             a		:Dict									=			ids_tensor([self.batch_size, self.seq_length]   ,    self.type_vocab_size       )

       a		:Tuple									=			None
       a		:List[Any]									=			None
       a		:Any									=			None
       if self.use_labels:
             a		:Tuple									=			ids_tensor([self.batch_size]   ,    self.type_sequence_label_size       )
             a		:str									=			ids_tensor([self.batch_size, self.seq_length]   ,    self.num_labels       )
             a		:Any									=			ids_tensor([self.batch_size]   ,    self.num_choices       )

       a		:List[str]									=			self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       return NystromformerConfig(
           vocab_size=self.vocab_size   ,    hidden_size=self.hidden_size   ,    num_hidden_layers=self.num_hidden_layers   ,    num_attention_heads=self.num_attention_heads   ,    intermediate_size=self.intermediate_size   ,    hidden_act=self.hidden_act   ,    hidden_dropout_prob=self.hidden_dropout_prob   ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,    max_position_embeddings=self.max_position_embeddings   ,    type_vocab_size=self.type_vocab_size   ,    is_decoder=_lowerCamelCase   ,    initializer_range=self.initializer_range   ,    )




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase       ):
       a		:List[Any]									=			NystromformerModel(config=_lowerCamelCase       )
       model.to(_lowerCamelCase       )
       model.eval()
       a		:List[Any]									=			model(_lowerCamelCase   ,    attention_mask=_lowerCamelCase   ,    token_type_ids=_lowerCamelCase       )
       a		:Union[str, Any]									=			model(_lowerCamelCase   ,    token_type_ids=_lowerCamelCase       )
       a		:Any									=			model(_lowerCamelCase       )
       self.parent.assertEqual(result.last_hidden_state.shape   ,    (self.batch_size, self.seq_length, self.hidden_size)       )




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase       ):
       a		:Optional[int]									=			NystromformerForMaskedLM(config=_lowerCamelCase       )
       model.to(_lowerCamelCase       )
       model.eval()
       a		:Tuple									=			model(_lowerCamelCase   ,    attention_mask=_lowerCamelCase   ,    token_type_ids=_lowerCamelCase   ,    labels=_lowerCamelCase       )
       self.parent.assertEqual(result.logits.shape   ,    (self.batch_size, self.seq_length, self.vocab_size)       )




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase       ):
       a		:Dict									=			NystromformerForQuestionAnswering(config=_lowerCamelCase       )
       model.to(_lowerCamelCase       )
       model.eval()
       a		:Optional[Any]									=			model(
           _lowerCamelCase   ,    attention_mask=_lowerCamelCase   ,    token_type_ids=_lowerCamelCase   ,    start_positions=_lowerCamelCase   ,    end_positions=_lowerCamelCase   ,    )
       self.parent.assertEqual(result.start_logits.shape   ,    (self.batch_size, self.seq_length)       )
       self.parent.assertEqual(result.end_logits.shape   ,    (self.batch_size, self.seq_length)       )




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase       ):
       a		:List[str]									=			self.num_labels
       a		:Any									=			NystromformerForSequenceClassification(_lowerCamelCase       )
       model.to(_lowerCamelCase       )
       model.eval()
       a		:Optional[int]									=			model(_lowerCamelCase   ,    attention_mask=_lowerCamelCase   ,    token_type_ids=_lowerCamelCase   ,    labels=_lowerCamelCase       )
       self.parent.assertEqual(result.logits.shape   ,    (self.batch_size, self.num_labels)       )




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase       ):
       a		:List[str]									=			self.num_labels
       a		:str									=			NystromformerForTokenClassification(config=_lowerCamelCase       )
       model.to(_lowerCamelCase       )
       model.eval()
       a		:Union[str, Any]									=			model(_lowerCamelCase   ,    attention_mask=_lowerCamelCase   ,    token_type_ids=_lowerCamelCase   ,    labels=_lowerCamelCase       )
       self.parent.assertEqual(result.logits.shape   ,    (self.batch_size, self.seq_length, self.num_labels)       )




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase       ):
       a		:Optional[Any]									=			self.num_choices
       a		:Optional[Any]									=			NystromformerForMultipleChoice(config=_lowerCamelCase       )
       model.to(_lowerCamelCase       )
       model.eval()
       a		:Optional[Any]									=			input_ids.unsqueeze(1       ).expand(-1   ,    self.num_choices   ,    -1       ).contiguous()
       a		:List[Any]									=			token_type_ids.unsqueeze(1       ).expand(-1   ,    self.num_choices   ,    -1       ).contiguous()
       a		:int									=			input_mask.unsqueeze(1       ).expand(-1   ,    self.num_choices   ,    -1       ).contiguous()
       a		:List[Any]									=			model(
           _lowerCamelCase   ,    attention_mask=_lowerCamelCase   ,    token_type_ids=_lowerCamelCase   ,    labels=_lowerCamelCase   ,    )
       self.parent.assertEqual(result.logits.shape   ,    (self.batch_size, self.num_choices)       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:List[Any]									=			self.prepare_config_and_inputs()
       (
           (
           a
       )				,							(
           a
       )				,							(
           a
       )				,							(
           a
       )				,							(
           a
       )				,							(
           a
       )				,							(
           a
       )				,							
       )		:Any									=			config_and_inputs
       a		:Tuple									=			{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
       return config, inputs_dict







@require_torch
class 						_snake_case    ( _snake_case		,       _snake_case		,       unittest.TestCase				):
 SCREAMING_SNAKE_CASE__ =			(
     (
         NystromformerModel,
         NystromformerForMaskedLM,
         NystromformerForMultipleChoice,
         NystromformerForQuestionAnswering,
         NystromformerForSequenceClassification,
         NystromformerForTokenClassification,
     )
     if is_torch_available()
     else ()
 )
 SCREAMING_SNAKE_CASE__ =			(
     {
         'feature-extraction': NystromformerModel,
         'fill-mask': NystromformerForMaskedLM,
         'question-answering': NystromformerForQuestionAnswering,
         'text-classification': NystromformerForSequenceClassification,
         'token-classification': NystromformerForTokenClassification,
         'zero-shot': NystromformerForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 SCREAMING_SNAKE_CASE__ =			False
 SCREAMING_SNAKE_CASE__ =			False




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:Any									=			NystromformerModelTester(self       )
       a		:Tuple									=			ConfigTester(self   ,    config_class=_lowerCamelCase   ,    hidden_size=37       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       self.config_tester.run_common_tests()




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:Tuple									=			self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*_lowerCamelCase       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:Dict									=			self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
             a		:Union[str, Any]									=			type
             self.model_tester.create_and_check_model(*_lowerCamelCase       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:Tuple									=			self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*_lowerCamelCase       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:List[Any]									=			self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*_lowerCamelCase       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:Any									=			self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*_lowerCamelCase       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:Optional[int]									=			self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*_lowerCamelCase       )




 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:Optional[Any]									=			self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*_lowerCamelCase       )




 @slow
 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       for model_name in NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             a		:str									=			NystromformerModel.from_pretrained(_lowerCamelCase       )
             self.assertIsNotNone(_lowerCamelCase       )







@require_torch
class 						_snake_case    ( unittest.TestCase				):




 @slow
 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:Dict									=			NystromformerModel.from_pretrained('''uw-madison/nystromformer-512'''       )
       a		:Any									=			torch.tensor([[0, 1, 2, 3, 4, 5]]       )

       with torch.no_grad():
             a		:Optional[int]									=			model(_lowerCamelCase       )[0]

       a		:Optional[Any]									=			torch.Size((1, 6, 768)       )
       self.assertEqual(output.shape   ,    _lowerCamelCase       )

       a		:Dict									=			torch.tensor(
           [[[-0.4532, -0.0936, 0.5137], [-0.2676, 0.0628, 0.6186], [-0.3629, -0.1726, 0.4716]]]       )

       self.assertTrue(torch.allclose(output[:, :3, :3]   ,    _lowerCamelCase   ,    atol=1e-4       )       )




 @slow
 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       a		:Optional[int]									=			'''the [MASK] of Belgium is Brussels'''

       a		:Dict									=			AutoTokenizer.from_pretrained('''uw-madison/nystromformer-512'''       )
       a		:str									=			NystromformerForMaskedLM.from_pretrained('''uw-madison/nystromformer-512'''       )

       a		:Dict									=			tokenizer(_lowerCamelCase   ,    return_tensors='''pt'''       )

       with torch.no_grad():
             a		:Union[str, Any]									=			model(encoding.input_ids       ).logits

       a		:Optional[Any]									=			token_logits[:, 2, :].argmax(-1       )[0]

       self.assertEqual(tokenizer.decode(_lowerCamelCase       )   ,    '''capital'''       )





