title: XGBoostError: [10:10:03] /workspace/src/tree/updater_gpu_hist.cu:1407: Exception in gpu_hist: NCCL failure
body: <p><strong>PROJECT</strong></p>
  <ul>
  <li><a href="https://developer.nvidia.com/blog/gradient-boosting-decision-trees-xgboost-cuda/" rel="nofollow noreferrer">Nvidia Developer project</a></li>
  <li>in a Google Collab environment</li>
  </ul>
  <p><strong>MY CODE</strong></p>
  <pre><code>import csv
  import numpy as np
  import os.path
  import pandas
  import time
  import xgboost as xgb
  import sys
  if sys.version_info[0] &gt;= 3:
      from urllib.request import urlretrieve
  else:
      from urllib import urlretrieve
  
  data_url = &quot;https://archive.ics.uci.edu/ml/machine-learning-databases/00280/HIGGS.csv.gz&quot;
  dmatrix_train_filename = &quot;higgs_train.dmatrix&quot;
  dmatrix_test_filename = &quot;higgs_test.dmatrix&quot;
  csv_filename = &quot;HIGGS.csv.gz&quot;
  train_rows = 10500000
  test_rows = 500000
  num_round = 1000
  
  plot = True
  
  # return xgboost dmatrix
  def load_higgs():
      if os.path.isfile(dmatrix_train_filename) and os.path.isfile(dmatrix_test_filename):           
          dtrain = xgb.DMatrix(dmatrix_train_filename)
          dtest = xgb.DMatrix(dmatrix_test_filename)
          if dtrain.num_row() == train_rows and dtest.num_row() == test_rows:
              print(&quot;Loading cached dmatrix...&quot;)
              return dtrain, dtest
  
      if not os.path.isfile(csv_filename):
          print(&quot;Downloading higgs file...&quot;)
          urlretrieve(data_url, csv_filename)
  
      df_higgs_train = pandas.read_csv(csv_filename, dtype=np.float32, 
                                       nrows=train_rows, header=None)
      dtrain = xgb.DMatrix(df_higgs_train.loc[:, 1:29], df_higgs_train[0])
      dtrain.save_binary(dmatrix_train_filename)
      df_higgs_test = pandas.read_csv(csv_filename, dtype=np.float32, 
                                      skiprows=train_rows, nrows=test_rows, 
                                      header=None)
      dtest = xgb.DMatrix(df_higgs_test.loc[:, 1:29], df_higgs_test[0])
      dtest.save_binary(dmatrix_test_filename)
  
      return dtrain, dtest
  
  
  dtrain, dtest = load_higgs()
  param = {}
  param['objective'] = 'binary:logitraw'
  param['eval_metric'] = 'error'
  param['tree_method'] = 'gpu_hist'
  param['silent'] = 1
  
  print(&quot;Training with GPU ...&quot;)
  tmp = time.time()
  gpu_res = {}
  xgb.train(param, dtrain, num_round, evals=[(dtest, &quot;test&quot;)], 
            evals_result=gpu_res)
  gpu_time = time.time() - tmp
  print(&quot;GPU Training Time: %s seconds&quot; % (str(gpu_time)))
  
  print(&quot;Training with CPU ...&quot;)
  param['tree_method'] = 'hist'
  tmp = time.time()
  cpu_res = {}
  xgb.train(param, dtrain, num_round, evals=[(dtest, &quot;test&quot;)], 
            evals_result=cpu_res)
  cpu_time = time.time() - tmp
  print(&quot;CPU Training Time: %s seconds&quot; % (str(cpu_time)))
  
  if plot:
      import matplotlib.pyplot as plt
      min_error = min(min(gpu_res[&quot;test&quot;][param['eval_metric']]), 
                      min(cpu_res[&quot;test&quot;][param['eval_metric']]))
      gpu_iteration_time = [x / (num_round * 1.0) * gpu_time for x in range(0, num_round)]
      cpu_iteration_time = [x / (num_round * 1.0) * cpu_time for x in range(0, num_round)]
      plt.plot(gpu_iteration_time, gpu_res['test'][param['eval_metric']], 
               label='Tesla P100')
      plt.plot(cpu_iteration_time, cpu_res['test'][param['eval_metric']], 
               label='2x Haswell E5-2698 v3 (32 cores)')
      plt.legend()
      plt.xlabel('Time (s)')
      plt.ylabel('Test error')
      plt.axhline(y=min_error, color='r', linestyle='dashed')
      plt.margins(x=0)
      plt.ylim((0.23, 0.35))
      plt.show()
  </code></pre>
  <p><strong>ERROR</strong></p>
  <pre><code>Training with GPU ...
  ---------------------------------------------------------------------------
  XGBoostError                              Traceback (most recent call last)
  &lt;ipython-input-4-edcaeafd94a7&gt; in &lt;module&gt;()
       58 gpu_res = {}
       59 xgb.train(param, dtrain, num_round, evals=[(dtest, &quot;test&quot;)], 
  ---&gt; 60           evals_result=gpu_res)
       61 gpu_time = time.time() - tmp
       62 print(&quot;GPU Training Time: %s seconds&quot; % (str(gpu_time)))
  
  3 frames
  /usr/local/lib/python3.6/dist-packages/xgboost/training.py in train(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, xgb_model, callbacks, learning_rates)
      214                            evals=evals,
      215                            obj=obj, feval=feval,
  --&gt; 216                            xgb_model=xgb_model, callbacks=callbacks)
      217 
      218 
  
  /usr/local/lib/python3.6/dist-packages/xgboost/training.py in _train_internal(params, dtrain, num_boost_round, evals, obj, feval, xgb_model, callbacks)
       72         # Skip the first update if it is a recovery step.
       73         if version % 2 == 0:
  ---&gt; 74             bst.update(dtrain, i, obj)
       75             bst.save_rabit_checkpoint()
       76             version += 1
  
  /usr/local/lib/python3.6/dist-packages/xgboost/core.py in update(self, dtrain, iteration, fobj)
     1107         if fobj is None:
     1108             _check_call(_LIB.XGBoosterUpdateOneIter(self.handle, ctypes.c_int(iteration),
  -&gt; 1109                                                     dtrain.handle))
     1110         else:
     1111             pred = self.predict(dtrain)
  
  /usr/local/lib/python3.6/dist-packages/xgboost/core.py in _check_call(ret)
      174     &quot;&quot;&quot;
      175     if ret != 0:
  --&gt; 176         raise XGBoostError(py_str(_LIB.XGBGetLastError()))
      177 
      178 
  
  XGBoostError: [10:10:03] /workspace/src/tree/updater_gpu_hist.cu:1407: Exception in gpu_hist: NCCL failure :unhandled cuda error /workspace/src/tree/../common/device_helpers.cuh(896)
  
  Stack trace:
    [bt] (0) /usr/local/lib/python3.6/dist-packages/xgboost/./lib/libxgboost.so(dmlc::LogMessageFatal::~LogMessageFatal()+0x24) [0x7fb070340cb4]
    [bt] (1) /usr/local/lib/python3.6/dist-packages/xgboost/./lib/libxgboost.so(xgboost::tree::GPUHistMakerSpecialised&lt;xgboost::detail::GradientPairInternal&lt;double&gt; &gt;::Update(xgboost::HostDeviceVector&lt;xgboost::detail::GradientPairInternal&lt;float&gt; &gt;*, xgboost::DMatrix*, std::vector&lt;xgboost::RegTree*, std::allocator&lt;xgboost::RegTree*&gt; &gt; const&amp;)+0x1270) [0x7fb07057c7f0]
    [bt] (2) /usr/local/lib/python3.6/dist-packages/xgboost/./lib/libxgboost.so(xgboost::gbm::GBTree::BoostNewTrees(xgboost::HostDeviceVector&lt;xgboost::detail::GradientPairInternal&lt;float&gt; &gt;*, xgboost::DMatrix*, int, std::vector&lt;std::unique_ptr&lt;xgboost::RegTree, std::default_delete&lt;xgboost::RegTree&gt; &gt;, std::allocator&lt;std::unique_ptr&lt;xgboost::RegTree, std::default_delete&lt;xgboost::RegTree&gt; &gt; &gt; &gt;*)+0xa81) [0x7fb0703c6791]
    [bt] (3) /usr/local/lib/python3.6/dist-packages/xgboost/./lib/libxgboost.so(xgboost::gbm::GBTree::DoBoost(xgboost::DMatrix*, xgboost::HostDeviceVector&lt;xgboost::detail::GradientPairInternal&lt;float&gt; &gt;*, xgboost::ObjFunction*)+0xd65) [0x7fb0703c7c95]
    [bt] (4) /usr/local/lib/python3.6/dist-packages/xgboost/./lib/libxgboost.so(xgboost::LearnerImpl::UpdateOneIter(int, xgboost::DMatrix*)+0x396) [0x7fb0703da556]
    [bt] (5) /usr/local/lib/python3.6/dist-packages/xgboost/./lib/libxgboost.so(XGBoosterUpdateOneIter+0x35) [0x7fb07033daa5]
    [bt] (6) /usr/lib/x86_64-linux-gnu/libffi.so.6(ffi_call_unix64+0x4c) [0x7fb09b874dae]
    [bt] (7) /usr/lib/x86_64-linux-gnu/libffi.so.6(ffi_call+0x22f) [0x7fb09b87471f]
    [bt] (8) /usr/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so(_ctypes_callproc+0x2b4) [0x7fb09ba885c4]
  
  </code></pre>
  <p><strong>SOLUTIONS I have tried</strong></p>
  <ul>
  <li>I have the corect installs - <a href="https://stackoverflow.com/questions/51760173/xgboost-with-gpu-support-on-google-colab">XGBoost with GPU support on Google Colab</a></li>
  <li>I have corrected this - <a href="https://stackoverflow.com/questions/51351982/xgboost-error-dmatrix-object-does-not-support-indexing">XGBoost error &#39;DMatrix&#39; object does not support indexing</a></li>
  </ul>