title: Using mutex to synchronize two kernel threads causes a system panic
body: <p>I used the mutex to synchronize the two kernel threads. After running, panic occurred in the system and abnormal memory was found in mutex by kdump.</p>
  <p>Here is a simplified code example, You can run it directly to reproduce the problem.
  I changed the memory allocation method to use kmalloc instead of vmalloc, and then it worked, Who knows why？</p>
  <pre class="lang-c prettyprint-override"><code>#include &lt;linux/module.h&gt;
  #include &lt;linux/kernel.h&gt;
  #include &lt;linux/init.h&gt;
  #include &lt;linux/delay.h&gt;
  #include &lt;linux/spinlock.h&gt;
  #include &lt;linux/mutex.h&gt;
  #include &lt;linux/kthread.h&gt;
  #include &lt;linux/slab.h&gt;
  
  struct product {
      struct list_head list;
      struct mutex lock;
      bool finish;
  };
  
  struct task_struct *task1;
  struct task_struct *task2;
  
  spinlock_t spin;
  struct list_head products;
  
  struct product *create_product(void)
  {
      struct product *p_prod;
  
      p_prod = vmalloc(sizeof(struct product));
  //    p_prod = kmalloc(sizeof(struct product), GFP_KERNEL);
      if(!p_prod)
          return NULL;
  
      INIT_LIST_HEAD(&amp;p_prod-&gt;list);
      mutex_init(&amp;p_prod-&gt;lock);
      p_prod-&gt;finish = false;
  
      return p_prod;
  }
  
  void remove_product(struct product **pp_prod)
  {
      vfree(*pp_prod);
  //    kfree(*pp_prod);
      *pp_prod = NULL;
  }
  
  int producer(void *data)
  {
      while(!kthread_should_stop())
      {
          struct product *p_prod = create_product();
          if(!p_prod)
              continue;
  
          spin_lock(&amp;spin);
          list_add_tail(&amp;p_prod-&gt;list, &amp;products);
          spin_unlock(&amp;spin);
  
          while (true)
          {
              mutex_lock(&amp;p_prod-&gt;lock);
              if(p_prod-&gt;finish)
              {
                  mutex_unlock(&amp;p_prod-&gt;lock);
                  schedule();
                  break;
              }
              mutex_unlock(&amp;p_prod-&gt;lock);
          }
          remove_product(&amp;p_prod);
      }
  
      do_exit(0);
  }
  
  int consumer(void *data)
  {
      while(!kthread_should_stop())
      {
          struct product *p_prod;
  
          spin_lock(&amp;spin);
          if(list_empty(&amp;products))
          {
              spin_unlock(&amp;spin);
              schedule();
              continue;
          }
          p_prod = list_first_entry(&amp;products, struct product, list);
          list_del(&amp;p_prod-&gt;list);
          spin_unlock(&amp;spin);
  
          mutex_lock(&amp;p_prod-&gt;lock);
          p_prod-&gt;finish = true;
          mutex_unlock(&amp;p_prod-&gt;lock);
      }
  
      do_exit(0);
  }
  
  static int __init kdemo_init(void) {
      printk(&quot;&gt;&gt;&gt; demo driver begin!\n&quot;);
  
      spin_lock_init(&amp;spin);
      INIT_LIST_HEAD(&amp;products);
  
      task1 = kthread_run(producer, NULL, &quot;hdz-producer&quot;);
      task2 = kthread_run(consumer, NULL, &quot;hdz-consumer&quot;);
      return 0;
  }
  
  static void __exit kdemo_exit(void) {
      kthread_stop(task1);
      kthread_stop(task2);
  
      printk(&quot;&gt;&gt;&gt; demo driver exit!\n&quot;);
  }
  
  module_init(kdemo_init);
  module_exit(kdemo_exit);
  MODULE_LICENSE(&quot;GPL&quot;);
  MODULE_AUTHOR(&quot;xxxx@xxx.com&quot;);
  MODULE_VERSION(&quot;1.0&quot;);
  
  </code></pre>
  <p>dmesg log and consumer stack</p>
  <pre><code>[  176.599116] &gt;&gt;&gt; demo driver begin!
  [  177.167659] BUG: unable to handle kernel NULL pointer dereference at 0000000000000fb0
  [  177.167695] IP: [&lt;ffffffff9e0caa47&gt;] wake_q_add+0x17/0x50
  [  177.167719] PGD 0
  [  177.167729] Oops: 0002 [#1] SMP
  [  177.167743] Modules linked in: kdemo(OE) mpt3sas mptctl mptbase nvmet_rdma nvmet nvme_rdma nvme_fabrics nvme nvme_core drbd(OE) dell_rbu kvdo(OE) uds(OE) bonding sha512_ssse3 sha512_generic qat_api(OE) usdm_drv(OE) intel_qat(OE) authenc uio ib_isert iscsi_target_mod ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ucm rpcrdma sunrpc rdma_ucm ib_umad ib_uverbs ib_iser rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi ib_cm mlx5_ib ib_core intelcas(OE) inteldisk(OE) iTCO_wdt iTCO_vendor_support dell_smbios sparse_keymap dcdbas skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg joydev pcspkr ipmi_si i2c_i801 lpc_ich shpchp ipmi_devintf ipmi_msghandler mei_me acpi_power_meter
  [  177.168071]  mei acpi_pad wmi nfit libnvdimm dm_multipath binfmt_misc ip_tables xfs libcrc32c mgag200 drm_kms_helper crc32c_intel syscopyarea sysfillrect sysimgblt mlx5_core fb_sys_fops ttm ixgbe drm igb mlxfw devlink mdio ptp i2c_algo_bit pps_core i2c_core dca sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common ahci libahci libata mpt2sas raid_class scsi_transport_sas megaraid_sas dm_mirror dm_region_hash dm_log dm_mod
  [  177.168263] CPU: 24 PID: 5412 Comm: hdz-consumer Kdump: loaded Tainted: G           OE  ------------   3.10.0-862.el7.x86_64 #1
  [  177.168297] Hardware name: Dell Inc. PowerEdge R740/08D89F, BIOS 2.10.2 02/24/2021
  [  177.168320] task: ffff93db22af3f40 ti: ffff93dc89354000 task.ti: ffff93dc89354000
  [  177.168344] RIP: 0010:[&lt;ffffffff9e0caa47&gt;]  [&lt;ffffffff9e0caa47&gt;] wake_q_add+0x17/0x50
  [  177.168372] RSP: 0018:ffff93dc89357e48  EFLAGS: 00010246
  [  177.168389] RAX: 0000000000000000 RBX: ffffbe2ce6533018 RCX: 0000000000000fb0
  [  177.168410] RDX: 0000000000000001 RSI: 0000000000000000 RDI: ffff93dc89357e58
  [  177.168432] RBP: ffff93dc89357e48 R08: ffffbe2ce6533000 R09: 0000000000000000
  [  177.168453] R10: 0000000000000001 R11: 0000000000000001 R12: ffffbe2ce6533014
  [  177.168475] R13: ffff93dc89357e58 R14: 0000000000000000 R15: 0000000000000000
  [  177.168497] FS:  0000000000000000(0000) GS:ffff93dca9400000(0000) knlGS:0000000000000000
  [  177.168540] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
  [  177.168560] CR2: 0000000000000fb0 CR3: 00000002b7a0e000 CR4: 00000000007607e0
  [  177.168583] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
  [  177.168606] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
  [  177.168629] PKRU: 00000000
  [  177.168640] Call Trace:
  [  177.168656]  [&lt;ffffffff9e711b2e&gt;] __mutex_unlock_slowpath+0x5e/0x90
  [  177.168679]  [&lt;ffffffff9e710fab&gt;] mutex_unlock+0x1b/0x20
  [  177.168699]  [&lt;ffffffffc0637064&gt;] consumer+0x64/0x90 [kdemo]
  [  177.168723]  [&lt;ffffffffc0637000&gt;] ? 0xffffffffc0636fff
  [  177.168746]  [&lt;ffffffff9e0bae31&gt;] kthread+0xd1/0xe0
  [  177.168765]  [&lt;ffffffff9e0bad60&gt;] ? insert_kthread_work+0x40/0x40
  [  177.168788]  [&lt;ffffffff9e71f61d&gt;] ret_from_fork_nospec_begin+0x7/0x21
  [  177.168811]  [&lt;ffffffff9e0bad60&gt;] ? insert_kthread_work+0x40/0x40
  [  177.168831] Code: 09 00 00 31 c9 31 d2 e8 18 41 ff ff eb e4 66 0f 1f 44 00 00 0f 1f 44 00 00 55 48 8d 8e b0 0f 00 00 31 c0 ba 01 00 00 00 48 89 e5 &lt;f0&gt; 48 0f b1 96 b0 0f 00 00 48 85 c0 74 0b 5d c3 66 0f 1f 84 00
  [  177.168996] RIP  [&lt;ffffffff9e0caa47&gt;] wake_q_add+0x17/0x50
  [  177.169017]  RSP &lt;ffff93dc89357e48&gt;
  [  177.169959] CR2: 0000000000000fb0
  </code></pre>
  <p>producer stask</p>
  <pre><code>crash&gt; bt 5411
  PID: 5411   TASK: ffff93db22af4f10  CPU: 1   COMMAND: &quot;hdz-producer&quot;
  bt: page excluded: kernel virtual address: ffffffffffffffff  type: &quot;cpu_online_map&quot;
   #0 [ffff93dca8e48e48] crash_nmi_callback at ffffffff9e0533b7
   #1 [ffff93dca8e48e58] nmi_handle at ffffffff9e71790c
   #2 [ffff93dca8e48eb0] do_nmi at ffffffff9e717b2d
   #3 [ffff93dca8e48ef0] end_repeat_nmi at ffffffff9e716d79
   #4 [ffff93dca8e48f28] __vmalloc_node_range at ffffffff9e1d7518
      [exception RIP: mutex_unlock+20]
      RIP: ffffffff9e710fa4  RSP: ffff93ddafd73e98  RFLAGS: 00000202
      RAX: 0000000000000010  RBX: 0000000000000010  RCX: 0000000000000202
      RDX: ffff93ddafd73e98  RSI: 0000000000000018  RDI: 0000000000000001
      RBP: ffffffff9e710fa4   R8: ffffffff9e710fa4   R9: 0000000000000018
      R10: ffff93ddafd73e98  R11: 0000000000000202  R12: ffffffffffffffff
      R13: ffffbe2ce6535010  R14: ffffffffc0639240  R15: 0000000000000000
      ORIG_RAX: ffffffffc0639240  CS: 0010  SS: 0018
  --- &lt;(unknown) exception stack&gt; ---
   #5 [ffff93ddafd73e98] mutex_unlock at ffffffff9e710fa4
   #6 [ffff93ddafd73ea0] producer at ffffffffc0637145 [kdemo]
   #7 [ffff93ddafd73ec8] kthread at ffffffff9e0bae31
   #8 [ffff93ddafd73f50] ret_from_fork_nospec_begin at ffffffff9e71f61d
  </code></pre>