code: def test_headmasking(self):
      if not self.test_head_masking:
          return
  
      global_rng.seed(42)
      (
          config,
          inputs_dict,
      ) = self.model_tester.prepare_config_and_inputs_for_common()
      global_rng.seed()
  
      inputs_dict["output_attentions"] = True
      config.output_hidden_states = True
      configs_no_init = _config_zero_init(
          config
      )  # To be sure we have no Nan
      for model_class in self.all_model_classes:
          model = model_class(config=configs_no_init)
          model.to(torch_device)
          model.eval()
  
          # Prepare head_mask
          # Set require_grad after having prepared the tensor to avoid error (leaf variable has been moved into the graph interior)
          head_mask = torch.ones(
              self.model_tester.num_hidden_layers,
              self.model_tester.num_attention_heads,
              device=torch_device,
          )
          head_mask[0, 0] = 0
          head_mask[-1, :-1] = 0
          head_mask.requires_grad_(requires_grad=True)
          inputs = self._prepare_for_class(
              inputs_dict, model_class
          ).copy()
          inputs["head_mask"] = head_mask
  
          outputs = model(**inputs, return_dict=True)
  
          # Test that we can get a gradient back for importance score computation
          output = sum(t.sum() for t in outputs[0])
          output = output.sum()
          output.backward()
          multihead_outputs = head_mask.grad
  
          self.assertIsNotNone(multihead_outputs)
          self.assertEqual(
              len(multihead_outputs),
              self.model_tester.num_hidden_layers,
          )
  
          def check_attentions_validity(attentions):
              # Remove Nan
              for t in attentions:
                  self.assertLess(
                      torch.sum(torch.isnan(t)), t.numel() / 4
                  )  # Check we don't have more than 25% nans (arbitrary)
              attentions = [
                  t.masked_fill(torch.isnan(t), 0.0) for t in attentions
              ]  # remove them (the test is less complete)
  
              self.assertAlmostEqual(
                  attentions[1][..., 0, :, :].flatten().sum().item(),
                  0.0,
              )
              self.assertNotEqual(
                  attentions[1][..., -1, :, :].flatten().sum().item(),
                  0.0,
              )
              self.assertAlmostEqual(
                  attentions[-2][..., -2, :, :].flatten().sum().item(),
                  0.0,
              )
              self.assertNotEqual(
                  attentions[-2][..., -1, :, :].flatten().sum().item(),
                  0.0,
              )
  
          check_attentions_validity(outputs.attentions)
  
imports: import unittest
  from typing import List, Tuple
  from transformers import CanineConfig, is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, _config_zero_init, global_rng, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
label_desc: Use image node_id 11 for calling the CanineModelTest obj's underlying member method code with example usage: obj.test_headmasking() without return types