title: cudaMemGetInfo returns same amount of free memory on both devices of GTX 690
body: <p>I have run into problems with Geforce GTX 690 while trying to track down the memory usage. A simple test program:</p>
  
  <pre><code>BOOST_AUTO_TEST_CASE(cudaMemoryTest) {
  
  size_t mem_tot_0 = 0;
  size_t mem_free_0 = 0;
  size_t mem_tot_1 = 0;
  size_t mem_free_1 = 0;
  
  unsigned int mem_size = 100*1000000;
  
  float* h_P = new float[mem_size];
  for(size_t i = 0; i &lt; mem_size; i++) {
      h_P[i] = 0.f;
  }
  
  cudaSetDevice(0);
  cudaDeviceReset();
  cudaMemGetInfo  (&amp;mem_free_0, &amp; mem_tot_0);
  std::cout&lt;&lt;"Free memory before copy dev 0: "&lt;&lt;mem_free_0&lt;&lt;std::endl;
  
  cudaSetDevice(1);
  cudaDeviceReset();
  cudaMemGetInfo  (&amp;mem_free_1, &amp;mem_tot_1);
  std::cout&lt;&lt;"Free memory before copy dev 1: "&lt;&lt;mem_free_1&lt;&lt;std::endl;
  
  cudaSetDevice(0);
  float* P;
  cudaMalloc((void**)&amp;P, mem_size*sizeof(float));
  cudaMemcpy((void*)P, h_P,  mem_size*sizeof(float), cudaMemcpyHostToDevice);
  
  cudaSetDevice(0);
  cudaMemGetInfo(&amp;mem_free_0, &amp; mem_tot_0);
  std::cout&lt;&lt;"Free memory after copy  dev 0: "&lt;&lt;mem_free_0&lt;&lt;std::endl;
  
  cudaSetDevice(1);
  cudaMemGetInfo(&amp;mem_free_1, &amp;mem_tot_1);
  std::cout&lt;&lt;"Free memory after copy dev 1: "&lt;&lt;mem_free_1&lt;&lt;std::endl;
  
  BOOST_CHECK(mem_free_0 != mem_free_1);
  
  cudaError_t err;
  err = cudaGetLastError();
  if(err!=cudaSuccess)
      std::cout&lt;&lt;"an error occurred"&lt;&lt;std::endl;
  
  cudaSetDevice(0);
  destroyMem(P);
  delete [] h_P;
  }
  </code></pre>
  
  <p>The test prints out:</p>
  
  <pre><code>1&gt;  Free memory before copy dev 0: 1733173248
  1&gt;  Free memory before copy dev 1: 1688424448
  1&gt;  Free memory after copy  dev 0: 1289940992
  1&gt;  Free memory after copy dev 1: 1289940992     
  CudaUtilsTest.cpp(47): error in "cudaMemoryTest": check mem_free_0 != mem_free_1 failed
  </code></pre>
  
  <p>The problem is that after the allocation the amount of free memory on device 1 is exactly the same as on device 0, which shouldn't be the case, hence the problem has to be in cudaMemGetInfo and/or cudaSetDevice. Anyone run on the same problem, or is there something else fundamentally wrong in the test that someone can point out?</p>
  
  <p>Running the code on Windows 7, Visual studio 2010, Cuda SDK 5.0, compiling with code generation: compute_30,sm_30</p>
  
  <p>EDIT 22.4.2013</p>
  
  <p>I continued experimenting with this issue and it seems that cudaSetDevice works fine as can be verified from the result of cudaGetDevice calls. I added a reset of device 0 after the memory allocation test and it seems that the size of free memory returned by cudaMemGetInfo is again same for both devices. I have checked all return values of cuda_error_t in my own code and all function calls return cudaSuccess. Have anyone run into similar problems with GTX 690 with the setup descibed above?</p>
  
  <p>Most resent test code:</p>
  
  <pre><code>BOOST_AUTO_TEST_CASE(cudaMemoryTest) {
  size_t mem_tot_0 = 0;
  size_t mem_free_0 = 0;
  size_t mem_tot_1 = 0;
  size_t mem_free_1 = 0;
  
  int device_num = 0;
  
  unsigned int mem_size = 100*1000000;
  
  float* h_P = new float[mem_size];
  for(size_t i = 0; i &lt; mem_size; i++) {
      h_P[i] = 0.f;
  }
  
  cudaSetDevice(0);
  cudaGetDevice(&amp;device_num);
  cudaDeviceReset();
  cudaMemGetInfo  (&amp;mem_free_0, &amp; mem_tot_0);
  std::cout&lt;&lt;"Free memory before copy dev 0: "&lt;&lt;mem_free_0&lt;&lt;" Device: "&lt;&lt;device_num&lt;&lt;std::endl;
  cudaDeviceSynchronize();
  
  cudaSetDevice(1);
  cudaGetDevice(&amp;device_num);
  cudaDeviceReset();
  cudaMemGetInfo  (&amp;mem_free_1, &amp; mem_tot_1);
  std::cout&lt;&lt;"Free memory before copy dev 1: "&lt;&lt;mem_free_1&lt;&lt;" Device: "&lt;&lt;device_num&lt;&lt;std::endl;
  cudaDeviceSynchronize();
  
  cudaSetDevice(0);
  cudaGetDevice(&amp;device_num);
  float* P;
  cudaMalloc((void**)&amp;P, mem_size*sizeof(float));
  cudaMemcpy((void*)P, h_P,  mem_size*sizeof(float), cudaMemcpyHostToDevice);
  cudaMemGetInfo(&amp;mem_free_0, &amp; mem_tot_0);
  std::cout&lt;&lt;"Free memory after copy  dev 0: "&lt;&lt;mem_free_0&lt;&lt;" Device: "&lt;&lt;device_num&lt;&lt;std::endl;
  cudaDeviceSynchronize();
  
  cudaSetDevice(1);
  cudaGetDevice(&amp;device_num);
  cudaMemGetInfo(&amp;mem_free_1, &amp;mem_tot_1);
  std::cout&lt;&lt;"Free memory after copy dev 1: "&lt;&lt;mem_free_1&lt;&lt;" Device: "&lt;&lt;device_num&lt;&lt;std::endl;
  cudaDeviceSynchronize();
  
  BOOST_CHECK(mem_free_0 != mem_free_1);
  
  cudaError_t err;
  err = cudaGetLastError();
  if(err!=cudaSuccess)
      std::cout&lt;&lt;"an error occurred"&lt;&lt;std::endl;
  
  // Reset only device 0 and check both
  cudaSetDevice(0);
  cudaGetDevice(&amp;device_num);
  cudaDeviceReset();
  cudaMemGetInfo  (&amp;mem_free_0, &amp; mem_tot_0);
  std::cout&lt;&lt;"Free memory after second reset of device 0, dev 0: "&lt;&lt;mem_free_0&lt;&lt;" Device: "&lt;&lt;device_num&lt;&lt;std::endl;
  cudaDeviceSynchronize();
  
  cudaSetDevice(1);
  cudaGetDevice(&amp;device_num);
  cudaMemGetInfo  (&amp;mem_free_1, &amp; mem_tot_1);
  std::cout&lt;&lt;"Free memory after second device reset of device 0, dev 1: "&lt;&lt;mem_free_1&lt;&lt;" Device: "&lt;&lt;device_num&lt;&lt;std::endl;
  cudaDeviceSynchronize();
  
  delete [] h_P;
      }
  </code></pre>
  
  <p>Test output:</p>
  
  <pre><code>1&gt;  Free memory before copy dev 0: 1794379776 Device: 0
  1&gt;  Free memory before copy dev 1: 1751728128 Device: 1
  1&gt;  Free memory after copy  dev 0: 1351696384 Device: 0
  1&gt;  Free memory after copy dev 1: 1351696384 Device: 1
  1&gt;  CudaUtilsTest.cpp(353): error in "cudaMemoryTest": check mem_free_0 != mem_free_1 failed
  1&gt;  Free memory after second reset of device 0, dev 0: 1751728128 Device: 0
  1&gt;  Free memory after second device reset of device 0, dev 1: 1751728128 Device: 1
  </code></pre>