text: /* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
  
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at
  
      http://www.apache.org/licenses/LICENSE-2.0
  
  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
  ==============================================================================*/
  
  // Implements C wrappers around the CUDA library for easy linking in ORC jit.
  // Also adds some debugging helpers that are helpful when writing MLIR code to
  // run on GPUs.
  
  #include <cassert>
  #include <numeric>
  
  #include "llvm/ADT/ArrayRef.h"
  #include "llvm/Support/raw_ostream.h"
  #include "mlir/ExecutionEngine/CRunnerUtils.h"  // from @llvm-project
  
  #if GOOGLE_CUDA
  #include "third_party/gpus/cuda/include/cuda.h"
  
  #define CUDA_REPORT_IF_ERROR(expr)                                             \
    [](CUresult result) {                                                        \
      if (!result)                                                               \
        return;                                                                  \
      const char *name = nullptr;                                                \
      cuGetErrorName(result, &name);                                             \
      if (!name)                                                                 \
        name = "<unknown>";                                                      \
      llvm::errs() << "'" << #expr << "' failed with '" << name << "'\n";        \
    }(expr)
  
  extern "C" CUmodule mgpuModuleLoad(void *data) {
    CUmodule module = nullptr;
    CUDA_REPORT_IF_ERROR(cuModuleLoadData(&module, data));
    return module;
  }
  
  extern "C" void mgpuModuleUnload(CUmodule module) {
    CUDA_REPORT_IF_ERROR(cuModuleUnload(module));
  }
  
  extern "C" CUfunction mgpuModuleGetFunction(CUmodule module, const char *name) {
    CUfunction function = nullptr;
    CUDA_REPORT_IF_ERROR(cuModuleGetFunction(&function, module, name));
    return function;
  }
  
  // The wrapper uses intptr_t instead of CUDA's unsigned int to match
  // the type of MLIR's index type. This avoids the need for casts in the
  // generated MLIR code.
  extern "C" void mgpuLaunchKernel(CUfunction function, intptr_t gridX,
                                   intptr_t gridY, intptr_t gridZ,
                                   intptr_t blockX, intptr_t blockY,
                                   intptr_t blockZ, int32_t smem, CUstream stream,
                                   void **params, void **extra) {
    CUDA_REPORT_IF_ERROR(cuLaunchKernel(function, gridX, gridY, gridZ, blockX,
                                        blockY, blockZ, smem, stream, params,
                                        extra));
  }
  
  extern "C" CUstream mgpuStreamCreate() {
    CUstream stream = nullptr;
    CUDA_REPORT_IF_ERROR(cuStreamCreate(&stream, CU_STREAM_NON_BLOCKING));
    return stream;
  }
  
  extern "C" void mgpuStreamDestroy(CUstream stream) {
    CUDA_REPORT_IF_ERROR(cuStreamDestroy(stream));
  }
  
  extern "C" void mgpuStreamSynchronize(CUstream stream) {
    CUDA_REPORT_IF_ERROR(cuStreamSynchronize(stream));
  }
  
  /// Helper functions for writing mlir example code
  
  // Allows to register byte array with the CUDA runtime. Helpful until we have
  // transfer functions implemented.
  extern "C" void mgpuMemHostRegister(void *ptr, uint64_t sizeBytes) {
    CUDA_REPORT_IF_ERROR(cuMemHostRegister(ptr, sizeBytes, /*flags=*/0));
  }
  
  // Allows to register a MemRef with the CUDA runtime. Helpful until we have
  // transfer functions implemented.
  extern "C" void
  mgpuMemHostRegisterMemRef(int64_t rank, StridedMemRefType<char, 1> *descriptor,
                            int64_t elementSizeBytes) {
  
    llvm::SmallVector<int64_t, 4> denseStrides(rank);
    llvm::ArrayRef<int64_t> sizes(descriptor->sizes, rank);
    llvm::ArrayRef<int64_t> strides(sizes.end(), rank);
  
    std::partial_sum(sizes.rbegin(), sizes.rend(), denseStrides.rbegin(),
                     std::multiplies<int64_t>());
    auto sizeBytes = denseStrides.front() * elementSizeBytes;
  
    // Only densely packed tensors are currently supported.
    std::rotate(denseStrides.begin(), denseStrides.begin() + 1,
                denseStrides.end());
    denseStrides.back() = 1;
    assert(strides == llvm::makeArrayRef(denseStrides));
  
    auto ptr = descriptor->data + descriptor->offset * elementSizeBytes;
    mgpuMemHostRegister(ptr, sizeBytes);
  }
  
  #endif
  