title: Can't restore my GPU after bad_alloc with `cudaDeviceReset()` from the CUDA library
body: <p>The following program does 3 successive calls to the "test" function where some thrust operations are executed. Each one of these 3 calls provides a different size to the problem: </p>
  
  <ul>
  <li>3,000 for the first call; </li>
  <li>300,000,000 for the second call;</li>
  <li>3,000 again for the third call. </li>
  </ul>
  
  <p>The second call is expected to fail, but the third one should be successful (as is the first one) if I properly cleaned up the status of my GPU. Unfortunately, it also fails. Moreover, successive calls would also result in a failure until I quit my process and start again.</p>
  
  <pre><code>#include "cuda_runtime.h"
  #include "device_launch_parameters.h"
  #include &lt;stdio.h&gt;
  #include &lt;cuda.h&gt;
  #include &lt;thrust/system_error.h&gt;
  #include &lt;thrust/device_vector.h&gt;
  #include &lt;thrust/sort.h&gt;
  #include &lt;thrust/execution_policy.h&gt;
  
  #define CUDA_CALL(x)do { if((x) != cudaSuccess) { return -11;}} while(0)
  
  typedef typename thrust::device_vector&lt;size_t&gt;  tDevVecInt;
  typedef typename thrust::device_vector&lt;float&gt;   tDevVecFlt;
  
  struct modSim : public thrust::unary_function&lt;int, int&gt;  
  {
      int szMat;
      int p;
  
      modSim(int in1, int in2)
      {
          this-&gt;p = in1;
          this-&gt;szMat = in2;
      }
      __host__ __device__ int operator()(const int &amp;x) 
      {
          return (x/szMat)*p+(x%p);
      }
  };
  
   int test(size_t szData)
  {
  
      modSim moduloCol(3, 33);
  
      CUDA_CALL(cudaSetDevice(0));
  
      try
      {
  
          tDevVecFlt devRand(szData);
          tDevVecInt devIndices(szData);
          tDevVecFlt devData(szData);
  
          thrust::sequence(devRand.begin(), devRand.end());
          thrust::tabulate(devIndices.begin(), devIndices.end(), moduloCol); 
          thrust::sort_by_key(devIndices.begin(), devIndices.end(), devRand.begin()); 
  
      }
      catch(std::bad_alloc &amp;e)
      {
          std::cout &lt;&lt; e.what() &lt;&lt; std::endl;
          CUDA_CALL(cudaDeviceReset());
          CUDA_CALL(cudaSetDevice(0));
          return -3;
      }
      catch(thrust::system_error &amp;e)
      {
          std::cout &lt;&lt; e.what() &lt;&lt; std::endl;
          CUDA_CALL(cudaDeviceReset());
          CUDA_CALL(cudaSetDevice(0));
          return -2;
      }
  
      CUDA_CALL(cudaDeviceReset());
      return 0;   
  }
  
  
  int main(void)
  {
  
      size_t n;
      int retVal;
  
      n = 3000;
      retVal = test(n);
      std::cout &lt;&lt; retVal &lt;&lt; std::endl;
  
      n = 300000000;
      retVal = test(n);
      std::cout &lt;&lt; retVal &lt;&lt; std::endl;
  
      n = 3000;
      retVal = test(n);
      std::cout &lt;&lt; retVal &lt;&lt; std::endl;
  
  
      return(0);
  }
  </code></pre>
  
  <p>On my setup (Windows 8, NVIDIA GeForce 820m with 2GB dedicated VRAM, CUDA 7.0 compiled with nvcc, the command line is "$nvcc -arch=compute_20 test.cu -run" ), I get this: </p>
  
  <ul>
  <li>first call with N = 3,000 succeeds;</li>
  <li>second call with N = 300,000,000 fails with the exception <code>bad allocation: out of memory</code>;</li>
  <li>third call with N = 3,000 fails with a <code>thrust::system error : after cub_::DeviceRadixSort::SortPairs(1): out of memory</code>.</li>
  </ul>
  
  <p>So the output looks like this:</p>
  
  <pre><code>0
  bad allocation: out of memory
  -3
  after cub_::DeviceRadixSort::SortPairs(1): out of memory
  -2
  </code></pre>
  
  <p>As mentioned above, the third call shouldn't have failed as it is identical to the successful first call. </p>
  
  <p>This failure seems to be a consequence of the previous call (the one that issued a <code>bad alloc</code>) but I cleaned everything up after the <code>bad alloc</code> with a <code>cudaDeviceReset()</code> and a <code>cudaSetDevice()</code>. </p>
  
  <p>Despite the cleaning instructions, the device is not back into a functional state and I don't understand why. </p>
  
  <p>If I did something wrong, what would by the proper way to restore the GPU to a functional state after the first failure without ending my process?</p>
  
  <p>Does anyone reproduce this?</p>