<h3 dir="auto">System information</h3>
<ul dir="auto">
<li><strong>Have I written custom code (as opposed to using a stock example script provided in TensorFlow)</strong>: N/A</li>
<li><strong>OS Platform and Distribution (e.g., Linux Ubuntu 16.04)</strong>: Linux</li>
<li><strong>TensorFlow installed from (source or binary)</strong>: binary</li>
<li><strong>TensorFlow version (use command below)</strong>: v1.9.0-rc0-35-g17d6639b55 1.9.0-rc1</li>
<li><strong>Python version</strong>: 3.6</li>
<li><strong>Bazel version (if compiling from source)</strong>: N/A</li>
<li><strong>GCC/Compiler version (if compiling from source)</strong>: N/A</li>
<li><strong>CUDA/cuDNN version</strong>: N/A</li>
<li><strong>GPU model and memory</strong>: N/A</li>
<li><strong>Exact command to reproduce</strong>:</li>
</ul>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="HOROVOD_WITH_TENSORFLOW=1 HOROVOD_CUDA_HOME=/xx/cuda/9.0/ HOROVOD_NCCL_HOME=/xx/NCCL/2.2.13/ HOROVOD_GPU_ALLREDUCE=NCCL pip install --no-cache-dir horovod --user -U"><pre class="notranslate"><code class="notranslate">HOROVOD_WITH_TENSORFLOW=1 HOROVOD_CUDA_HOME=/xx/cuda/9.0/ HOROVOD_NCCL_HOME=/xx/NCCL/2.2.13/ HOROVOD_GPU_ALLREDUCE=NCCL pip install --no-cache-dir horovod --user -U
</code></pre></div>
<p dir="auto">Logs:</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content=" In file included from /xx/.local/lib/python3.6/site-packages/tensorflow/include/tensorflow/stream_executor/dnn.h:33:0,
                     from /xx/.local/lib/python3.6/site-packages/tensorflow/include/tensorflow/stream_executor/stream.h:30,
                     from horovod/tensorflow/mpi_ops.cc:29:
    /xx/.local/lib/python3.6/site-packages/tensorflow/include/tensorflow/stream_executor/lib/statusor.h:21:46: fatal error: tensorflow/compiler/xla/statusor.h: No such file or di
rectory
    compilation terminated."><pre class="notranslate"><code class="notranslate"> In file included from /xx/.local/lib/python3.6/site-packages/tensorflow/include/tensorflow/stream_executor/dnn.h:33:0,
                     from /xx/.local/lib/python3.6/site-packages/tensorflow/include/tensorflow/stream_executor/stream.h:30,
                     from horovod/tensorflow/mpi_ops.cc:29:
    /xx/.local/lib/python3.6/site-packages/tensorflow/include/tensorflow/stream_executor/lib/statusor.h:21:46: fatal error: tensorflow/compiler/xla/statusor.h: No such file or di
rectory
    compilation terminated.
</code></pre></div>
<p dir="auto">Looks like some necessary headers are missing in this release.</p>
<h3 dir="auto">System information</h3>
<ul dir="auto">
<li><strong>Have I written custom code (as opposed to using a stock example script provided in TensorFlow)</strong>: Yes</li>
<li><strong>OS Platform and Distribution (e.g., Linux Ubuntu 16.04)</strong>: Fedora 27</li>
<li><strong>TensorFlow installed from (source or binary)</strong>: Source</li>
<li><strong>TensorFlow version (use command below)</strong>: v1.8.0-rc1-909-g968addadfd 1.8.0-rc1</li>
<li><strong>Python version</strong>: 3.6m</li>
<li><strong>Bazel version (if compiling from source)</strong>: 0.12.0</li>
<li><strong>GCC/Compiler version (if compiling from source)</strong>: 6.4</li>
<li><strong>CUDA/cuDNN version</strong>: 9.1/7.1</li>
<li><strong>GPU model and memory</strong>: GTX 1080 8GB</li>
<li><strong>Exact command to reproduce</strong>:</li>
</ul>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="include Makefile.config
TF_INC = `$(PYTHON) -W ignore -c 'import tensorflow as tf; print(tf.sysconfig.get_include())'`
TF_CFLAGS = `$(PYTHON) -W ignore -c 'import tensorflow as tf; print(&quot; &quot;.join(tf.sysconfig.get_compile_flags()))'`
TF_LFLAGS = `$(PYTHON) -W ignore -c 'import tensorflow as tf; print(&quot; &quot;.join(tf.sysconfig.get_link_flags()))'`
CUDA_INC = $(CUDA_HOME)/../
GPUCC     = nvcc -ccbin=$(CXX)
CFLAGS    = -std=c++11 $(TF_CFLAGS) -I. -I$(CUDA_INC) -I$(TF_INC)
GPUCFLAGS = -c
LFLAGS    = $(ODFLAGS) -pthread -fopenmp -shared -fPIC $(TF_LFLAGS)
GPULFLAGS = -x cu -Xcompiler &quot;$(CFLAGS) $(LFLAGS)&quot; --expt-relaxed-constexpr
GPUDEF    = -DGOOGLE_CUDA=1
CGPUFLAGS = -lcuda
SRC       = cart_hex_interpol.cc
GPUSRC    = cart_hex_interpol.cu.cc
SRC_O	  = cart_hex_interpol.o
GPUSRC_O  = cart_hex_interpol.cu.o
LIB       = cart_hex_interpol.so
all: gpu
default: gpu
cpu:
	$(CXX) $(CFLAGS) $(SRC) $(LFLAGS) -o $(LIB)
gpu:
	$(GPUCC) $(GPUDEF) $(CFLAGS) $(GPUCFLAGS) $(GPUSRC) $(GPULFLAGS) -o $(GPUSRC_O)
	$(CXX) $(GPUDEF) $(CFLAGS) $(SRC) $(GPUSRC_O) $(LFLAGS) $(CGPUFLAGS)  -o $(LIB)
clean:
	rm -f $(SRC_O) $(GPUSRC_O) $(LIB)"><pre class="notranslate"><code class="notranslate">include Makefile.config
TF_INC = `$(PYTHON) -W ignore -c 'import tensorflow as tf; print(tf.sysconfig.get_include())'`
TF_CFLAGS = `$(PYTHON) -W ignore -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))'`
TF_LFLAGS = `$(PYTHON) -W ignore -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))'`
CUDA_INC = $(CUDA_HOME)/../
GPUCC     = nvcc -ccbin=$(CXX)
CFLAGS    = -std=c++11 $(TF_CFLAGS) -I. -I$(CUDA_INC) -I$(TF_INC)
GPUCFLAGS = -c
LFLAGS    = $(ODFLAGS) -pthread -fopenmp -shared -fPIC $(TF_LFLAGS)
GPULFLAGS = -x cu -Xcompiler "$(CFLAGS) $(LFLAGS)" --expt-relaxed-constexpr
GPUDEF    = -DGOOGLE_CUDA=1
CGPUFLAGS = -lcuda
SRC       = cart_hex_interpol.cc
GPUSRC    = cart_hex_interpol.cu.cc
SRC_O	  = cart_hex_interpol.o
GPUSRC_O  = cart_hex_interpol.cu.o
LIB       = cart_hex_interpol.so
all: gpu
default: gpu
cpu:
	$(CXX) $(CFLAGS) $(SRC) $(LFLAGS) -o $(LIB)
gpu:
	$(GPUCC) $(GPUDEF) $(CFLAGS) $(GPUCFLAGS) $(GPUSRC) $(GPULFLAGS) -o $(GPUSRC_O)
	$(CXX) $(GPUDEF) $(CFLAGS) $(SRC) $(GPUSRC_O) $(LFLAGS) $(CGPUFLAGS)  -o $(LIB)
clean:
	rm -f $(SRC_O) $(GPUSRC_O) $(LIB)
</code></pre></div>
<h3 dir="auto">Describe the problem</h3>
<p dir="auto">When compiling a custom TensorFlow operator using a Makefile and including CUDA/GPU code, the compilation fails due to missing <code class="notranslate">fatal error: tensorflow/compiler/xla/statusor.h: No such file or directory</code>. The files are missing in <code class="notranslate">/usr/local/lib/python3.6/site-packages/tensorflow/</code> because the headers do not get installed during installation of the wheel package for TensorFlow.<br>
A temporary fix is to copy the headers from source:</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="sudo mkdir /usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/compiler/
sudo mkdir /usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/compiler/xla
sudo cp tensorflow/compiler/xla/*.h /usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/compiler/xla/"><pre class="notranslate"><code class="notranslate">sudo mkdir /usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/compiler/
sudo mkdir /usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/compiler/xla
sudo cp tensorflow/compiler/xla/*.h /usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/compiler/xla/
</code></pre></div>
<p dir="auto">This issue was not present in TensorFlow 1.5, but must have been introduced since then. Compiling with or without XLA makes no difference.</p>
<h3 dir="auto">Source code / logs</h3>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="make gpu -j2
nvcc -ccbin=/usr/local/gcc-6.4/bin/g++-6.4 -DGOOGLE_CUDA=1 -std=c++11 `python3 -W ignore -c 'import tensorflow as tf; print(&quot; &quot;.join(tf.sysconfig.get_compile_flags()))'` -I. -I/usr/local/cuda/../ -I`python3 -W ignore -c 'import tensorflow as tf; print(tf.sysconfig.get_include())'` -c cart_hex_interpol.cu.cc -x cu -Xcompiler &quot;-std=c++11 `python3 -W ignore -c 'import tensorflow as tf; print(&quot; &quot;.join(tf.sysconfig.get_compile_flags()))'` -I. -I/usr/local/cuda/../ -I`python3 -W ignore -c 'import tensorflow as tf; print(tf.sysconfig.get_include())'` -O3 -pthread -fopenmp -shared -fPIC `python3 -W ignore -c 'import tensorflow as tf; print(&quot; &quot;.join(tf.sysconfig.get_link_flags()))'`&quot; --expt-relaxed-constexpr -o cart_hex_interpol.cu.o
In file included from /usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/stream_executor/dso_loader.h:26:0,
                 from /usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/core/platform/stream_executor.h:26,
                 from /usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/core/util/cuda_launch_config.h:27,
                 from /usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/core/util/cuda_kernel_helper.h:22,
                 from cart_hex_interpol.cu.cc:5:
/usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/stream_executor/lib/statusor.h:21:46: fatal error: tensorflow/compiler/xla/statusor.h: No such file or directory
 #include &quot;tensorflow/compiler/xla/statusor.h&quot;
"><pre class="notranslate"><code class="notranslate">make gpu -j2
nvcc -ccbin=/usr/local/gcc-6.4/bin/g++-6.4 -DGOOGLE_CUDA=1 -std=c++11 `python3 -W ignore -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))'` -I. -I/usr/local/cuda/../ -I`python3 -W ignore -c 'import tensorflow as tf; print(tf.sysconfig.get_include())'` -c cart_hex_interpol.cu.cc -x cu -Xcompiler "-std=c++11 `python3 -W ignore -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))'` -I. -I/usr/local/cuda/../ -I`python3 -W ignore -c 'import tensorflow as tf; print(tf.sysconfig.get_include())'` -O3 -pthread -fopenmp -shared -fPIC `python3 -W ignore -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))'`" --expt-relaxed-constexpr -o cart_hex_interpol.cu.o
In file included from /usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/stream_executor/dso_loader.h:26:0,
                 from /usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/core/platform/stream_executor.h:26,
                 from /usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/core/util/cuda_launch_config.h:27,
                 from /usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/core/util/cuda_kernel_helper.h:22,
                 from cart_hex_interpol.cu.cc:5:
/usr/local/lib/python3.6/site-packages/tensorflow/include/tensorflow/stream_executor/lib/statusor.h:21:46: fatal error: tensorflow/compiler/xla/statusor.h: No such file or directory
 #include "tensorflow/compiler/xla/statusor.h"

</code></pre></div>