code: def test_apply_rotary_position_embeddings(self):
      # 2,12,16,64
      query_layer = (
          torch.arange(
              2 * 12 * 16 * 64, dtype=torch.float, device=torch_device
          ).reshape(2, 12, 16, 64)
          / 100
      ).to(torch_device)
      key_layer = (
          -torch.arange(
              2 * 12 * 16 * 64, dtype=torch.float, device=torch_device
          ).reshape(2, 12, 16, 64)
          / 100
      ).to(torch_device)
      embed_positions = RoFormerSinusoidalPositionalEmbedding(
          num_positions=32, embedding_dim=64
      ).to(torch_device)
      sinusoidal_pos = embed_positions([2, 16, 768])[None, None, :, :]
  
      (
          query_layer,
          key_layer,
      ) = RoFormerSelfAttention.apply_rotary_position_embeddings(
          sinusoidal_pos, query_layer, key_layer
      )
  
      desired_query_layer = torch.tensor(
          [
              [
                  0.0000,
                  0.0100,
                  0.0200,
                  0.0300,
                  0.0400,
                  0.0500,
                  0.0600,
                  0.0700,
              ],
              [
                  -0.2012,
                  0.8897,
                  0.0263,
                  0.9401,
                  0.2074,
                  0.9463,
                  0.3481,
                  0.9343,
              ],
              [
                  -1.7057,
                  0.6271,
                  -1.2145,
                  1.3897,
                  -0.6303,
                  1.7647,
                  -0.1173,
                  1.8985,
              ],
              [
                  -2.1731,
                  -1.6397,
                  -2.7358,
                  0.2854,
                  -2.1840,
                  1.7183,
                  -1.3018,
                  2.4871,
              ],
              [
                  0.2717,
                  -3.6173,
                  -2.9206,
                  -2.1988,
                  -3.6638,
                  0.3858,
                  -2.9155,
                  2.2980,
              ],
              [
                  3.9859,
                  -2.1580,
                  -0.7984,
                  -4.4904,
                  -4.1181,
                  -2.0252,
                  -4.4782,
                  1.1253,
              ],
          ]
      ).to(torch_device)
      desired_key_layer = torch.tensor(
          [
              [
                  0.0000,
                  -0.0100,
                  -0.0200,
                  -0.0300,
                  -0.0400,
                  -0.0500,
                  -0.0600,
                  -0.0700,
              ],
              [
                  0.2012,
                  -0.8897,
                  -0.0263,
                  -0.9401,
                  -0.2074,
                  -0.9463,
                  -0.3481,
                  -0.9343,
              ],
              [
                  1.7057,
                  -0.6271,
                  1.2145,
                  -1.3897,
                  0.6303,
                  -1.7647,
                  0.1173,
                  -1.8985,
              ],
              [
                  2.1731,
                  1.6397,
                  2.7358,
                  -0.2854,
                  2.1840,
                  -1.7183,
                  1.3018,
                  -2.4871,
              ],
              [
                  -0.2717,
                  3.6173,
                  2.9206,
                  2.1988,
                  3.6638,
                  -0.3858,
                  2.9155,
                  -2.2980,
              ],
              [
                  -3.9859,
                  2.1580,
                  0.7984,
                  4.4904,
                  4.1181,
                  2.0252,
                  4.4782,
                  -1.1253,
              ],
          ]
      ).to(torch_device)
  
      self.assertTrue(
          torch.allclose(
              query_layer[0, 0, :6, :8],
              desired_query_layer,
              atol=self.tolerance,
          ),
          msg=f"\nexp:\n{desired_query_layer}\ngot:\n{query_layer}\n",
      )
      self.assertTrue(
          torch.allclose(
              key_layer[0, 0, :6, :8],
              desired_key_layer,
              atol=self.tolerance,
          ),
          msg=f"\nexp:\n{desired_key_layer}\ngot:\n{key_layer}\n",
      )
  
imports: import unittest
  from transformers import RoFormerConfig, is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
label_desc: Use image node_id 1 for calling the RoFormerSelfAttentionRotaryPositionEmbeddingTest obj's underlying member method code with example usage: obj.test_apply_rotary_position_embeddings() without return types