text: //===- AMDGPUOpenCLEnqueuedBlockLowering.cpp - Lower enqueued block -------===//
  //
  // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  // See https://llvm.org/LICENSE.txt for license information.
  // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  //
  //===----------------------------------------------------------------------===//
  //
  // \file
  // This post-linking pass replaces the function pointer of enqueued
  // block kernel with a global variable (runtime handle) and adds
  // "runtime-handle" attribute to the enqueued block kernel.
  //
  // In LLVM CodeGen the runtime-handle metadata will be translated to
  // RuntimeHandle metadata in code object. Runtime allocates a global buffer
  // for each kernel with RuntimeHandel metadata and saves the kernel address
  // required for the AQL packet into the buffer. __enqueue_kernel function
  // in device library knows that the invoke function pointer in the block
  // literal is actually runtime handle and loads the kernel address from it
  // and put it into AQL packet for dispatching.
  //
  // This cannot be done in FE since FE cannot create a unique global variable
  // with external linkage across LLVM modules. The global variable with internal
  // linkage does not work since optimization passes will try to replace loads
  // of the global variable with its initialization value.
  //
  // It also identifies the kernels directly or indirectly enqueues kernels
  // and adds "calls-enqueue-kernel" function attribute to them, which will
  // be used to determine whether to emit runtime metadata for the kernel
  // enqueue related hidden kernel arguments.
  //
  //===----------------------------------------------------------------------===//
  
  #include "AMDGPU.h"
  #include "llvm/ADT/DenseSet.h"
  #include "llvm/ADT/SmallString.h"
  #include "llvm/IR/Instructions.h"
  #include "llvm/IR/Mangler.h"
  #include "llvm/IR/Module.h"
  #include "llvm/Pass.h"
  #include "llvm/Support/Debug.h"
  
  #define DEBUG_TYPE "amdgpu-lower-enqueued-block"
  
  using namespace llvm;
  
  namespace {
  
  /// Lower enqueued blocks.
  class AMDGPUOpenCLEnqueuedBlockLowering : public ModulePass {
  public:
    static char ID;
  
    explicit AMDGPUOpenCLEnqueuedBlockLowering() : ModulePass(ID) {}
  
  private:
    bool runOnModule(Module &M) override;
  };
  
  } // end anonymous namespace
  
  char AMDGPUOpenCLEnqueuedBlockLowering::ID = 0;
  
  char &llvm::AMDGPUOpenCLEnqueuedBlockLoweringID =
      AMDGPUOpenCLEnqueuedBlockLowering::ID;
  
  INITIALIZE_PASS(AMDGPUOpenCLEnqueuedBlockLowering, DEBUG_TYPE,
                  "Lower OpenCL enqueued blocks", false, false)
  
  ModulePass* llvm::createAMDGPUOpenCLEnqueuedBlockLoweringPass() {
    return new AMDGPUOpenCLEnqueuedBlockLowering();
  }
  
  /// Collect direct or indrect callers of \p F and save them
  /// to \p Callers.
  static void collectCallers(Function *F, DenseSet<Function *> &Callers) {
    for (auto U : F->users()) {
      if (auto *CI = dyn_cast<CallInst>(&*U)) {
        auto *Caller = CI->getParent()->getParent();
        if (Callers.insert(Caller).second)
          collectCallers(Caller, Callers);
      }
    }
  }
  
  /// If \p U is instruction or constant, collect functions which directly or
  /// indirectly use it.
  static void collectFunctionUsers(User *U, DenseSet<Function *> &Funcs) {
    if (auto *I = dyn_cast<Instruction>(U)) {
      auto *F = I->getParent()->getParent();
      if (Funcs.insert(F).second)
        collectCallers(F, Funcs);
      return;
    }
    if (!isa<Constant>(U))
      return;
    for (auto UU : U->users())
      collectFunctionUsers(&*UU, Funcs);
  }
  
  bool AMDGPUOpenCLEnqueuedBlockLowering::runOnModule(Module &M) {
    DenseSet<Function *> Callers;
    auto &C = M.getContext();
    bool Changed = false;
    for (auto &F : M.functions()) {
      if (F.hasFnAttribute("enqueued-block")) {
        if (!F.hasName()) {
          SmallString<64> Name;
          Mangler::getNameWithPrefix(Name, "__amdgpu_enqueued_kernel",
                                     M.getDataLayout());
          F.setName(Name);
        }
        LLVM_DEBUG(dbgs() << "found enqueued kernel: " << F.getName() << '\n');
        auto RuntimeHandle = (F.getName() + ".runtime_handle").str();
        auto T = ArrayType::get(Type::getInt64Ty(C), 2);
        auto *GV = new GlobalVariable(
            M, T,
            /*isConstant=*/false, GlobalValue::ExternalLinkage,
            /*Initializer=*/Constant::getNullValue(T), RuntimeHandle,
            /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
            AMDGPUAS::GLOBAL_ADDRESS,
            /*isExternallyInitialized=*/false);
        LLVM_DEBUG(dbgs() << "runtime handle created: " << *GV << '\n');
  
        for (auto U : F.users()) {
          auto *UU = &*U;
          if (!isa<ConstantExpr>(UU))
            continue;
          collectFunctionUsers(UU, Callers);
          auto *BitCast = cast<ConstantExpr>(UU);
          auto *NewPtr = ConstantExpr::getPointerCast(GV, BitCast->getType());
          BitCast->replaceAllUsesWith(NewPtr);
          F.addFnAttr("runtime-handle", RuntimeHandle);
          F.setLinkage(GlobalValue::ExternalLinkage);
          Changed = true;
        }
      }
    }
  
    for (auto F : Callers) {
      if (F->getCallingConv() != CallingConv::AMDGPU_KERNEL)
        continue;
      F->addFnAttr("calls-enqueue-kernel");
      LLVM_DEBUG(dbgs() << "mark enqueue_kernel caller:" << F->getName() << '\n');
    }
    return Changed;
  }
  