code: def test_layer_attn_probs(self):
      model = LongformerModel.from_pretrained(
          "patrickvonplaten/longformer-random-tiny"
      )
      model.eval()
      layer = model.encoder.layer[0].attention.self.to(torch_device)
      hidden_states = torch.cat(
          [self._get_hidden_states(), self._get_hidden_states() - 0.5],
          dim=0,
      )
      batch_size, seq_length, hidden_size = hidden_states.size()
      attention_mask = torch.zeros(
          (batch_size, seq_length),
          dtype=torch.float32,
          device=torch_device,
      )
  
      # create attn mask
      attention_mask[0, -2:] = 10000.0
      attention_mask[0, -1:] = -10000.0
      attention_mask[1, 1:] = 10000.0
  
      is_index_masked = attention_mask < 0
      is_index_global_attn = attention_mask > 0
      is_global_attn = is_index_global_attn.flatten().any().item()
  
      output_hidden_states, local_attentions, global_attentions = layer(
          hidden_states,
          attention_mask=attention_mask,
          is_index_masked=is_index_masked,
          is_index_global_attn=is_index_global_attn,
          is_global_attn=is_global_attn,
          output_attentions=True,
      )
  
      self.assertEqual(local_attentions.shape, (2, 4, 2, 8))
      self.assertEqual(global_attentions.shape, (2, 2, 3, 4))
  
      # All tokens with global attention have weight 0 in local attentions.
      self.assertTrue(torch.all(local_attentions[0, 2:4, :, :] == 0))
      self.assertTrue(torch.all(local_attentions[1, 1:4, :, :] == 0))
  
      # The weight of all tokens with local attention must sum to 1.
      self.assertTrue(
          torch.all(
              torch.abs(global_attentions[0, :, :2, :].sum(dim=-1) - 1)
              < 1e-6
          )
      )
      self.assertTrue(
          torch.all(
              torch.abs(global_attentions[1, :, :1, :].sum(dim=-1) - 1)
              < 1e-6
          )
      )
  
      self.assertTrue(
          torch.allclose(
              local_attentions[0, 0, 0, :],
              torch.tensor(
                  [
                      0.3328,
                      0.0000,
                      0.0000,
                      0.0000,
                      0.0000,
                      0.3355,
                      0.3318,
                      0.0000,
                  ],
                  dtype=torch.float32,
                  device=torch_device,
              ),
              atol=1e-3,
          )
      )
  
      self.assertTrue(
          torch.allclose(
              local_attentions[1, 0, 0, :],
              torch.tensor(
                  [
                      0.2492,
                      0.2502,
                      0.2502,
                      0.0000,
                      0.0000,
                      0.2505,
                      0.0000,
                      0.0000,
                  ],
                  dtype=torch.float32,
                  device=torch_device,
              ),
              atol=1e-3,
          )
      )
  
      # All the global attention weights must sum to 1.
      self.assertTrue(
          torch.all(torch.abs(global_attentions.sum(dim=-1) - 1) < 1e-6)
      )
  
      self.assertTrue(
          torch.allclose(
              global_attentions[0, 0, 1, :],
              torch.tensor(
                  [0.2500, 0.2500, 0.2500, 0.2500],
                  dtype=torch.float32,
                  device=torch_device,
              ),
              atol=1e-3,
          )
      )
  
      self.assertTrue(
          torch.allclose(
              global_attentions[1, 0, 0, :],
              torch.tensor(
                  [0.2497, 0.2500, 0.2499, 0.2504],
                  dtype=torch.float32,
                  device=torch_device,
              ),
              atol=1e-3,
          )
      )
  
imports: import unittest
  from transformers import LongformerConfig, is_torch_available
  from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
label_desc: Use image node_id 8 for calling the LongformerModelIntegrationTest obj's underlying member method code with example usage: obj.test_layer_attn_probs() without return types