text: //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
  //
  // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  // See https://llvm.org/LICENSE.txt for license information.
  // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  //
  //===----------------------------------------------------------------------===//
  //
  // This file implements the auto-upgrade helper functions.
  // This is where deprecated IR intrinsics and other IR features are updated to
  // current specifications.
  //
  //===----------------------------------------------------------------------===//
  
  #include "llvm/IR/AutoUpgrade.h"
  #include "llvm/ADT/StringSwitch.h"
  #include "llvm/IR/Constants.h"
  #include "llvm/IR/DIBuilder.h"
  #include "llvm/IR/DebugInfo.h"
  #include "llvm/IR/DiagnosticInfo.h"
  #include "llvm/IR/Function.h"
  #include "llvm/IR/IRBuilder.h"
  #include "llvm/IR/Instruction.h"
  #include "llvm/IR/InstVisitor.h"
  #include "llvm/IR/IntrinsicInst.h"
  #include "llvm/IR/IntrinsicsAArch64.h"
  #include "llvm/IR/IntrinsicsARM.h"
  #include "llvm/IR/IntrinsicsX86.h"
  #include "llvm/IR/LLVMContext.h"
  #include "llvm/IR/Module.h"
  #include "llvm/IR/Verifier.h"
  #include "llvm/Support/ErrorHandling.h"
  #include "llvm/Support/Regex.h"
  #include <cstring>
  using namespace llvm;
  
  static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
  
  // Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
  // changed their type from v4f32 to v2i64.
  static bool UpgradePTESTIntrinsic(Function* F, Intrinsic::ID IID,
                                    Function *&NewFn) {
    // Check whether this is an old version of the function, which received
    // v4f32 arguments.
    Type *Arg0Type = F->getFunctionType()->getParamType(0);
    if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
      return false;
  
    // Yes, it's old, replace it with new version.
    rename(F);
    NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
    return true;
  }
  
  // Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
  // arguments have changed their type from i32 to i8.
  static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
                                               Function *&NewFn) {
    // Check that the last argument is an i32.
    Type *LastArgType = F->getFunctionType()->getParamType(
       F->getFunctionType()->getNumParams() - 1);
    if (!LastArgType->isIntegerTy(32))
      return false;
  
    // Move this function aside and map down.
    rename(F);
    NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
    return true;
  }
  
  // Upgrade the declaration of fp compare intrinsics that change return type
  // from scalar to vXi1 mask.
  static bool UpgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
                                        Function *&NewFn) {
    // Check if the return type is a vector.
    if (F->getReturnType()->isVectorTy())
      return false;
  
    rename(F);
    NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
    return true;
  }
  
  static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
    // All of the intrinsics matches below should be marked with which llvm
    // version started autoupgrading them. At some point in the future we would
    // like to use this information to remove upgrade code for some older
    // intrinsics. It is currently undecided how we will determine that future
    // point.
    if (Name == "addcarryx.u32" || // Added in 8.0
        Name == "addcarryx.u64" || // Added in 8.0
        Name == "addcarry.u32" || // Added in 8.0
        Name == "addcarry.u64" || // Added in 8.0
        Name == "subborrow.u32" || // Added in 8.0
        Name == "subborrow.u64" || // Added in 8.0
        Name.startswith("sse2.padds.") || // Added in 8.0
        Name.startswith("sse2.psubs.") || // Added in 8.0
        Name.startswith("sse2.paddus.") || // Added in 8.0
        Name.startswith("sse2.psubus.") || // Added in 8.0
        Name.startswith("avx2.padds.") || // Added in 8.0
        Name.startswith("avx2.psubs.") || // Added in 8.0
        Name.startswith("avx2.paddus.") || // Added in 8.0
        Name.startswith("avx2.psubus.") || // Added in 8.0
        Name.startswith("avx512.padds.") || // Added in 8.0
        Name.startswith("avx512.psubs.") || // Added in 8.0
        Name.startswith("avx512.mask.padds.") || // Added in 8.0
        Name.startswith("avx512.mask.psubs.") || // Added in 8.0
        Name.startswith("avx512.mask.paddus.") || // Added in 8.0
        Name.startswith("avx512.mask.psubus.") || // Added in 8.0
        Name=="ssse3.pabs.b.128" || // Added in 6.0
        Name=="ssse3.pabs.w.128" || // Added in 6.0
        Name=="ssse3.pabs.d.128" || // Added in 6.0
        Name.startswith("fma4.vfmadd.s") || // Added in 7.0
        Name.startswith("fma.vfmadd.") || // Added in 7.0
        Name.startswith("fma.vfmsub.") || // Added in 7.0
        Name.startswith("fma.vfmsubadd.") || // Added in 7.0
        Name.startswith("fma.vfnmadd.") || // Added in 7.0
        Name.startswith("fma.vfnmsub.") || // Added in 7.0
        Name.startswith("avx512.mask.vfmadd.") || // Added in 7.0
        Name.startswith("avx512.mask.vfnmadd.") || // Added in 7.0
        Name.startswith("avx512.mask.vfnmsub.") || // Added in 7.0
        Name.startswith("avx512.mask3.vfmadd.") || // Added in 7.0
        Name.startswith("avx512.maskz.vfmadd.") || // Added in 7.0
        Name.startswith("avx512.mask3.vfmsub.") || // Added in 7.0
        Name.startswith("avx512.mask3.vfnmsub.") || // Added in 7.0
        Name.startswith("avx512.mask.vfmaddsub.") || // Added in 7.0
        Name.startswith("avx512.maskz.vfmaddsub.") || // Added in 7.0
        Name.startswith("avx512.mask3.vfmaddsub.") || // Added in 7.0
        Name.startswith("avx512.mask3.vfmsubadd.") || // Added in 7.0
        Name.startswith("avx512.mask.shuf.i") || // Added in 6.0
        Name.startswith("avx512.mask.shuf.f") || // Added in 6.0
        Name.startswith("avx512.kunpck") || //added in 6.0
        Name.startswith("avx2.pabs.") || // Added in 6.0
        Name.startswith("avx512.mask.pabs.") || // Added in 6.0
        Name.startswith("avx512.broadcastm") || // Added in 6.0
        Name == "sse.sqrt.ss" || // Added in 7.0
        Name == "sse2.sqrt.sd" || // Added in 7.0
        Name.startswith("avx512.mask.sqrt.p") || // Added in 7.0
        Name.startswith("avx.sqrt.p") || // Added in 7.0
        Name.startswith("sse2.sqrt.p") || // Added in 7.0
        Name.startswith("sse.sqrt.p") || // Added in 7.0
        Name.startswith("avx512.mask.pbroadcast") || // Added in 6.0
        Name.startswith("sse2.pcmpeq.") || // Added in 3.1
        Name.startswith("sse2.pcmpgt.") || // Added in 3.1
        Name.startswith("avx2.pcmpeq.") || // Added in 3.1
        Name.startswith("avx2.pcmpgt.") || // Added in 3.1
        Name.startswith("avx512.mask.pcmpeq.") || // Added in 3.9
        Name.startswith("avx512.mask.pcmpgt.") || // Added in 3.9
        Name.startswith("avx.vperm2f128.") || // Added in 6.0
        Name == "avx2.vperm2i128" || // Added in 6.0
        Name == "sse.add.ss" || // Added in 4.0
        Name == "sse2.add.sd" || // Added in 4.0
        Name == "sse.sub.ss" || // Added in 4.0
        Name == "sse2.sub.sd" || // Added in 4.0
        Name == "sse.mul.ss" || // Added in 4.0
        Name == "sse2.mul.sd" || // Added in 4.0
        Name == "sse.div.ss" || // Added in 4.0
        Name == "sse2.div.sd" || // Added in 4.0
        Name == "sse41.pmaxsb" || // Added in 3.9
        Name == "sse2.pmaxs.w" || // Added in 3.9
        Name == "sse41.pmaxsd" || // Added in 3.9
        Name == "sse2.pmaxu.b" || // Added in 3.9
        Name == "sse41.pmaxuw" || // Added in 3.9
        Name == "sse41.pmaxud" || // Added in 3.9
        Name == "sse41.pminsb" || // Added in 3.9
        Name == "sse2.pmins.w" || // Added in 3.9
        Name == "sse41.pminsd" || // Added in 3.9
        Name == "sse2.pminu.b" || // Added in 3.9
        Name == "sse41.pminuw" || // Added in 3.9
        Name == "sse41.pminud" || // Added in 3.9
        Name == "avx512.kand.w" || // Added in 7.0
        Name == "avx512.kandn.w" || // Added in 7.0
        Name == "avx512.knot.w" || // Added in 7.0
        Name == "avx512.kor.w" || // Added in 7.0
        Name == "avx512.kxor.w" || // Added in 7.0
        Name == "avx512.kxnor.w" || // Added in 7.0
        Name == "avx512.kortestc.w" || // Added in 7.0
        Name == "avx512.kortestz.w" || // Added in 7.0
        Name.startswith("avx512.mask.pshuf.b.") || // Added in 4.0
        Name.startswith("avx2.pmax") || // Added in 3.9
        Name.startswith("avx2.pmin") || // Added in 3.9
        Name.startswith("avx512.mask.pmax") || // Added in 4.0
        Name.startswith("avx512.mask.pmin") || // Added in 4.0
        Name.startswith("avx2.vbroadcast") || // Added in 3.8
        Name.startswith("avx2.pbroadcast") || // Added in 3.8
        Name.startswith("avx.vpermil.") || // Added in 3.1
        Name.startswith("sse2.pshuf") || // Added in 3.9
        Name.startswith("avx512.pbroadcast") || // Added in 3.9
        Name.startswith("avx512.mask.broadcast.s") || // Added in 3.9
        Name.startswith("avx512.mask.movddup") || // Added in 3.9
        Name.startswith("avx512.mask.movshdup") || // Added in 3.9
        Name.startswith("avx512.mask.movsldup") || // Added in 3.9
        Name.startswith("avx512.mask.pshuf.d.") || // Added in 3.9
        Name.startswith("avx512.mask.pshufl.w.") || // Added in 3.9
        Name.startswith("avx512.mask.pshufh.w.") || // Added in 3.9
        Name.startswith("avx512.mask.shuf.p") || // Added in 4.0
        Name.startswith("avx512.mask.vpermil.p") || // Added in 3.9
        Name.startswith("avx512.mask.perm.df.") || // Added in 3.9
        Name.startswith("avx512.mask.perm.di.") || // Added in 3.9
        Name.startswith("avx512.mask.punpckl") || // Added in 3.9
        Name.startswith("avx512.mask.punpckh") || // Added in 3.9
        Name.startswith("avx512.mask.unpckl.") || // Added in 3.9
        Name.startswith("avx512.mask.unpckh.") || // Added in 3.9
        Name.startswith("avx512.mask.pand.") || // Added in 3.9
        Name.startswith("avx512.mask.pandn.") || // Added in 3.9
        Name.startswith("avx512.mask.por.") || // Added in 3.9
        Name.startswith("avx512.mask.pxor.") || // Added in 3.9
        Name.startswith("avx512.mask.and.") || // Added in 3.9
        Name.startswith("avx512.mask.andn.") || // Added in 3.9
        Name.startswith("avx512.mask.or.") || // Added in 3.9
        Name.startswith("avx512.mask.xor.") || // Added in 3.9
        Name.startswith("avx512.mask.padd.") || // Added in 4.0
        Name.startswith("avx512.mask.psub.") || // Added in 4.0
        Name.startswith("avx512.mask.pmull.") || // Added in 4.0
        Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0
        Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0
        Name.startswith("avx512.mask.cvtudq2ps.") || // Added in 7.0 updated 9.0
        Name.startswith("avx512.mask.cvtqq2pd.") || // Added in 7.0 updated 9.0
        Name.startswith("avx512.mask.cvtuqq2pd.") || // Added in 7.0 updated 9.0
        Name.startswith("avx512.mask.cvtdq2ps.") || // Added in 7.0 updated 9.0
        Name == "avx512.mask.vcvtph2ps.128" || // Added in 11.0
        Name == "avx512.mask.vcvtph2ps.256" || // Added in 11.0
        Name == "avx512.mask.cvtqq2ps.256" || // Added in 9.0
        Name == "avx512.mask.cvtqq2ps.512" || // Added in 9.0
        Name == "avx512.mask.cvtuqq2ps.256" || // Added in 9.0
        Name == "avx512.mask.cvtuqq2ps.512" || // Added in 9.0
        Name == "avx512.mask.cvtpd2dq.256" || // Added in 7.0
        Name == "avx512.mask.cvtpd2ps.256" || // Added in 7.0
        Name == "avx512.mask.cvttpd2dq.256" || // Added in 7.0
        Name == "avx512.mask.cvttps2dq.128" || // Added in 7.0
        Name == "avx512.mask.cvttps2dq.256" || // Added in 7.0
        Name == "avx512.mask.cvtps2pd.128" || // Added in 7.0
        Name == "avx512.mask.cvtps2pd.256" || // Added in 7.0
        Name == "avx512.cvtusi2sd" || // Added in 7.0
        Name.startswith("avx512.mask.permvar.") || // Added in 7.0
        Name == "sse2.pmulu.dq" || // Added in 7.0
        Name == "sse41.pmuldq" || // Added in 7.0
        Name == "avx2.pmulu.dq" || // Added in 7.0
        Name == "avx2.pmul.dq" || // Added in 7.0
        Name == "avx512.pmulu.dq.512" || // Added in 7.0
        Name == "avx512.pmul.dq.512" || // Added in 7.0
        Name.startswith("avx512.mask.pmul.dq.") || // Added in 4.0
        Name.startswith("avx512.mask.pmulu.dq.") || // Added in 4.0
        Name.startswith("avx512.mask.pmul.hr.sw.") || // Added in 7.0
        Name.startswith("avx512.mask.pmulh.w.") || // Added in 7.0
        Name.startswith("avx512.mask.pmulhu.w.") || // Added in 7.0
        Name.startswith("avx512.mask.pmaddw.d.") || // Added in 7.0
        Name.startswith("avx512.mask.pmaddubs.w.") || // Added in 7.0
        Name.startswith("avx512.mask.packsswb.") || // Added in 5.0
        Name.startswith("avx512.mask.packssdw.") || // Added in 5.0
        Name.startswith("avx512.mask.packuswb.") || // Added in 5.0
        Name.startswith("avx512.mask.packusdw.") || // Added in 5.0
        Name.startswith("avx512.mask.cmp.b") || // Added in 5.0
        Name.startswith("avx512.mask.cmp.d") || // Added in 5.0
        Name.startswith("avx512.mask.cmp.q") || // Added in 5.0
        Name.startswith("avx512.mask.cmp.w") || // Added in 5.0
        Name.startswith("avx512.cmp.p") || // Added in 12.0
        Name.startswith("avx512.mask.ucmp.") || // Added in 5.0
        Name.startswith("avx512.cvtb2mask.") || // Added in 7.0
        Name.startswith("avx512.cvtw2mask.") || // Added in 7.0
        Name.startswith("avx512.cvtd2mask.") || // Added in 7.0
        Name.startswith("avx512.cvtq2mask.") || // Added in 7.0
        Name.startswith("avx512.mask.vpermilvar.") || // Added in 4.0
        Name.startswith("avx512.mask.psll.d") || // Added in 4.0
        Name.startswith("avx512.mask.psll.q") || // Added in 4.0
        Name.startswith("avx512.mask.psll.w") || // Added in 4.0
        Name.startswith("avx512.mask.psra.d") || // Added in 4.0
        Name.startswith("avx512.mask.psra.q") || // Added in 4.0
        Name.startswith("avx512.mask.psra.w") || // Added in 4.0
        Name.startswith("avx512.mask.psrl.d") || // Added in 4.0
        Name.startswith("avx512.mask.psrl.q") || // Added in 4.0
        Name.startswith("avx512.mask.psrl.w") || // Added in 4.0
        Name.startswith("avx512.mask.pslli") || // Added in 4.0
        Name.startswith("avx512.mask.psrai") || // Added in 4.0
        Name.startswith("avx512.mask.psrli") || // Added in 4.0
        Name.startswith("avx512.mask.psllv") || // Added in 4.0
        Name.startswith("avx512.mask.psrav") || // Added in 4.0
        Name.startswith("avx512.mask.psrlv") || // Added in 4.0
        Name.startswith("sse41.pmovsx") || // Added in 3.8
        Name.startswith("sse41.pmovzx") || // Added in 3.9
        Name.startswith("avx2.pmovsx") || // Added in 3.9
        Name.startswith("avx2.pmovzx") || // Added in 3.9
        Name.startswith("avx512.mask.pmovsx") || // Added in 4.0
        Name.startswith("avx512.mask.pmovzx") || // Added in 4.0
        Name.startswith("avx512.mask.lzcnt.") || // Added in 5.0
        Name.startswith("avx512.mask.pternlog.") || // Added in 7.0
        Name.startswith("avx512.maskz.pternlog.") || // Added in 7.0
        Name.startswith("avx512.mask.vpmadd52") || // Added in 7.0
        Name.startswith("avx512.maskz.vpmadd52") || // Added in 7.0
        Name.startswith("avx512.mask.vpermi2var.") || // Added in 7.0
        Name.startswith("avx512.mask.vpermt2var.") || // Added in 7.0
        Name.startswith("avx512.maskz.vpermt2var.") || // Added in 7.0
        Name.startswith("avx512.mask.vpdpbusd.") || // Added in 7.0
        Name.startswith("avx512.maskz.vpdpbusd.") || // Added in 7.0
        Name.startswith("avx512.mask.vpdpbusds.") || // Added in 7.0
        Name.startswith("avx512.maskz.vpdpbusds.") || // Added in 7.0
        Name.startswith("avx512.mask.vpdpwssd.") || // Added in 7.0
        Name.startswith("avx512.maskz.vpdpwssd.") || // Added in 7.0
        Name.startswith("avx512.mask.vpdpwssds.") || // Added in 7.0
        Name.startswith("avx512.maskz.vpdpwssds.") || // Added in 7.0
        Name.startswith("avx512.mask.dbpsadbw.") || // Added in 7.0
        Name.startswith("avx512.mask.vpshld.") || // Added in 7.0
        Name.startswith("avx512.mask.vpshrd.") || // Added in 7.0
        Name.startswith("avx512.mask.vpshldv.") || // Added in 8.0
        Name.startswith("avx512.mask.vpshrdv.") || // Added in 8.0
        Name.startswith("avx512.maskz.vpshldv.") || // Added in 8.0
        Name.startswith("avx512.maskz.vpshrdv.") || // Added in 8.0
        Name.startswith("avx512.vpshld.") || // Added in 8.0
        Name.startswith("avx512.vpshrd.") || // Added in 8.0
        Name.startswith("avx512.mask.add.p") || // Added in 7.0. 128/256 in 4.0
        Name.startswith("avx512.mask.sub.p") || // Added in 7.0. 128/256 in 4.0
        Name.startswith("avx512.mask.mul.p") || // Added in 7.0. 128/256 in 4.0
        Name.startswith("avx512.mask.div.p") || // Added in 7.0. 128/256 in 4.0
        Name.startswith("avx512.mask.max.p") || // Added in 7.0. 128/256 in 5.0
        Name.startswith("avx512.mask.min.p") || // Added in 7.0. 128/256 in 5.0
        Name.startswith("avx512.mask.fpclass.p") || // Added in 7.0
        Name.startswith("avx512.mask.vpshufbitqmb.") || // Added in 8.0
        Name.startswith("avx512.mask.pmultishift.qb.") || // Added in 8.0
        Name.startswith("avx512.mask.conflict.") || // Added in 9.0
        Name == "avx512.mask.pmov.qd.256" || // Added in 9.0
        Name == "avx512.mask.pmov.qd.512" || // Added in 9.0
        Name == "avx512.mask.pmov.wb.256" || // Added in 9.0
        Name == "avx512.mask.pmov.wb.512" || // Added in 9.0
        Name == "sse.cvtsi2ss" || // Added in 7.0
        Name == "sse.cvtsi642ss" || // Added in 7.0
        Name == "sse2.cvtsi2sd" || // Added in 7.0
        Name == "sse2.cvtsi642sd" || // Added in 7.0
        Name == "sse2.cvtss2sd" || // Added in 7.0
        Name == "sse2.cvtdq2pd" || // Added in 3.9
        Name == "sse2.cvtdq2ps" || // Added in 7.0
        Name == "sse2.cvtps2pd" || // Added in 3.9
        Name == "avx.cvtdq2.pd.256" || // Added in 3.9
        Name == "avx.cvtdq2.ps.256" || // Added in 7.0
        Name == "avx.cvt.ps2.pd.256" || // Added in 3.9
        Name.startswith("vcvtph2ps.") || // Added in 11.0
        Name.startswith("avx.vinsertf128.") || // Added in 3.7
        Name == "avx2.vinserti128" || // Added in 3.7
        Name.startswith("avx512.mask.insert") || // Added in 4.0
        Name.startswith("avx.vextractf128.") || // Added in 3.7
        Name == "avx2.vextracti128" || // Added in 3.7
        Name.startswith("avx512.mask.vextract") || // Added in 4.0
        Name.startswith("sse4a.movnt.") || // Added in 3.9
        Name.startswith("avx.movnt.") || // Added in 3.2
        Name.startswith("avx512.storent.") || // Added in 3.9
        Name == "sse41.movntdqa" || // Added in 5.0
        Name == "avx2.movntdqa" || // Added in 5.0
        Name == "avx512.movntdqa" || // Added in 5.0
        Name == "sse2.storel.dq" || // Added in 3.9
        Name.startswith("sse.storeu.") || // Added in 3.9
        Name.startswith("sse2.storeu.") || // Added in 3.9
        Name.startswith("avx.storeu.") || // Added in 3.9
        Name.startswith("avx512.mask.storeu.") || // Added in 3.9
        Name.startswith("avx512.mask.store.p") || // Added in 3.9
        Name.startswith("avx512.mask.store.b.") || // Added in 3.9
        Name.startswith("avx512.mask.store.w.") || // Added in 3.9
        Name.startswith("avx512.mask.store.d.") || // Added in 3.9
        Name.startswith("avx512.mask.store.q.") || // Added in 3.9
        Name == "avx512.mask.store.ss" || // Added in 7.0
        Name.startswith("avx512.mask.loadu.") || // Added in 3.9
        Name.startswith("avx512.mask.load.") || // Added in 3.9
        Name.startswith("avx512.mask.expand.load.") || // Added in 7.0
        Name.startswith("avx512.mask.compress.store.") || // Added in 7.0
        Name.startswith("avx512.mask.expand.b") || // Added in 9.0
        Name.startswith("avx512.mask.expand.w") || // Added in 9.0
        Name.startswith("avx512.mask.expand.d") || // Added in 9.0
        Name.startswith("avx512.mask.expand.q") || // Added in 9.0
        Name.startswith("avx512.mask.expand.p") || // Added in 9.0
        Name.startswith("avx512.mask.compress.b") || // Added in 9.0
        Name.startswith("avx512.mask.compress.w") || // Added in 9.0
        Name.startswith("avx512.mask.compress.d") || // Added in 9.0
        Name.startswith("avx512.mask.compress.q") || // Added in 9.0
        Name.startswith("avx512.mask.compress.p") || // Added in 9.0
        Name == "sse42.crc32.64.8" || // Added in 3.4
        Name.startswith("avx.vbroadcast.s") || // Added in 3.5
        Name.startswith("avx512.vbroadcast.s") || // Added in 7.0
        Name.startswith("avx512.mask.palignr.") || // Added in 3.9
        Name.startswith("avx512.mask.valign.") || // Added in 4.0
        Name.startswith("sse2.psll.dq") || // Added in 3.7
        Name.startswith("sse2.psrl.dq") || // Added in 3.7
        Name.startswith("avx2.psll.dq") || // Added in 3.7
        Name.startswith("avx2.psrl.dq") || // Added in 3.7
        Name.startswith("avx512.psll.dq") || // Added in 3.9
        Name.startswith("avx512.psrl.dq") || // Added in 3.9
        Name == "sse41.pblendw" || // Added in 3.7
        Name.startswith("sse41.blendp") || // Added in 3.7
        Name.startswith("avx.blend.p") || // Added in 3.7
        Name == "avx2.pblendw" || // Added in 3.7
        Name.startswith("avx2.pblendd.") || // Added in 3.7
        Name.startswith("avx.vbroadcastf128") || // Added in 4.0
        Name == "avx2.vbroadcasti128" || // Added in 3.7
        Name.startswith("avx512.mask.broadcastf32x4.") || // Added in 6.0
        Name.startswith("avx512.mask.broadcastf64x2.") || // Added in 6.0
        Name.startswith("avx512.mask.broadcastf32x8.") || // Added in 6.0
        Name.startswith("avx512.mask.broadcastf64x4.") || // Added in 6.0
        Name.startswith("avx512.mask.broadcasti32x4.") || // Added in 6.0
        Name.startswith("avx512.mask.broadcasti64x2.") || // Added in 6.0
        Name.startswith("avx512.mask.broadcasti32x8.") || // Added in 6.0
        Name.startswith("avx512.mask.broadcasti64x4.") || // Added in 6.0
        Name == "xop.vpcmov" || // Added in 3.8
        Name == "xop.vpcmov.256" || // Added in 5.0
        Name.startswith("avx512.mask.move.s") || // Added in 4.0
        Name.startswith("avx512.cvtmask2") || // Added in 5.0
        Name.startswith("xop.vpcom") || // Added in 3.2, Updated in 9.0
        Name.startswith("xop.vprot") || // Added in 8.0
        Name.startswith("avx512.prol") || // Added in 8.0
        Name.startswith("avx512.pror") || // Added in 8.0
        Name.startswith("avx512.mask.prorv.") || // Added in 8.0
        Name.startswith("avx512.mask.pror.") ||  // Added in 8.0
        Name.startswith("avx512.mask.prolv.") || // Added in 8.0
        Name.startswith("avx512.mask.prol.") ||  // Added in 8.0
        Name.startswith("avx512.ptestm") || //Added in 6.0
        Name.startswith("avx512.ptestnm") || //Added in 6.0
        Name.startswith("avx512.mask.pavg")) // Added in 6.0
      return true;
  
    return false;
  }
  
  static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name,
                                          Function *&NewFn) {
    // Only handle intrinsics that start with "x86.".
    if (!Name.startswith("x86."))
      return false;
    // Remove "x86." prefix.
    Name = Name.substr(4);
  
    if (ShouldUpgradeX86Intrinsic(F, Name)) {
      NewFn = nullptr;
      return true;
    }
  
    if (Name == "rdtscp") { // Added in 8.0
      // If this intrinsic has 0 operands, it's the new version.
      if (F->getFunctionType()->getNumParams() == 0)
        return false;
  
      rename(F);
      NewFn = Intrinsic::getDeclaration(F->getParent(),
                                        Intrinsic::x86_rdtscp);
      return true;
    }
  
    // SSE4.1 ptest functions may have an old signature.
    if (Name.startswith("sse41.ptest")) { // Added in 3.2
      if (Name.substr(11) == "c")
        return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestc, NewFn);
      if (Name.substr(11) == "z")
        return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestz, NewFn);
      if (Name.substr(11) == "nzc")
        return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
    }
    // Several blend and other instructions with masks used the wrong number of
    // bits.
    if (Name == "sse41.insertps") // Added in 3.6
      return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,
                                              NewFn);
    if (Name == "sse41.dppd") // Added in 3.6
      return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,
                                              NewFn);
    if (Name == "sse41.dpps") // Added in 3.6
      return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,
                                              NewFn);
    if (Name == "sse41.mpsadbw") // Added in 3.6
      return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,
                                              NewFn);
    if (Name == "avx.dp.ps.256") // Added in 3.6
      return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,
                                              NewFn);
    if (Name == "avx2.mpsadbw") // Added in 3.6
      return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
                                              NewFn);
    if (Name == "avx512.mask.cmp.pd.128") // Added in 7.0
      return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_128,
                                       NewFn);
    if (Name == "avx512.mask.cmp.pd.256") // Added in 7.0
      return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_256,
                                       NewFn);
    if (Name == "avx512.mask.cmp.pd.512") // Added in 7.0
      return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_512,
                                       NewFn);
    if (Name == "avx512.mask.cmp.ps.128") // Added in 7.0
      return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_128,
                                       NewFn);
    if (Name == "avx512.mask.cmp.ps.256") // Added in 7.0
      return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_256,
                                       NewFn);
    if (Name == "avx512.mask.cmp.ps.512") // Added in 7.0
      return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
                                       NewFn);
  
    // frcz.ss/sd may need to have an argument dropped. Added in 3.2
    if (Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) {
      rename(F);
      NewFn = Intrinsic::getDeclaration(F->getParent(),
                                        Intrinsic::x86_xop_vfrcz_ss);
      return true;
    }
    if (Name.startswith("xop.vfrcz.sd") && F->arg_size() == 2) {
      rename(F);
      NewFn = Intrinsic::getDeclaration(F->getParent(),
                                        Intrinsic::x86_xop_vfrcz_sd);
      return true;
    }
    // Upgrade any XOP PERMIL2 index operand still using a float/double vector.
    if (Name.startswith("xop.vpermil2")) { // Added in 3.9
      auto Idx = F->getFunctionType()->getParamType(2);
      if (Idx->isFPOrFPVectorTy()) {
        rename(F);
        unsigned IdxSize = Idx->getPrimitiveSizeInBits();
        unsigned EltSize = Idx->getScalarSizeInBits();
        Intrinsic::ID Permil2ID;
        if (EltSize == 64 && IdxSize == 128)
          Permil2ID = Intrinsic::x86_xop_vpermil2pd;
        else if (EltSize == 32 && IdxSize == 128)
          Permil2ID = Intrinsic::x86_xop_vpermil2ps;
        else if (EltSize == 64 && IdxSize == 256)
          Permil2ID = Intrinsic::x86_xop_vpermil2pd_256;
        else
          Permil2ID = Intrinsic::x86_xop_vpermil2ps_256;
        NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID);
        return true;
      }
    }
  
    if (Name == "seh.recoverfp") {
      NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp);
      return true;
    }
  
    return false;
  }
  
  static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
    assert(F && "Illegal to upgrade a non-existent Function.");
  
    // Quickly eliminate it, if it's not a candidate.
    StringRef Name = F->getName();
    if (Name.size() <= 8 || !Name.startswith("llvm."))
      return false;
    Name = Name.substr(5); // Strip off "llvm."
  
    switch (Name[0]) {
    default: break;
    case 'a': {
      if (Name.startswith("arm.rbit") || Name.startswith("aarch64.rbit")) {
        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
                                          F->arg_begin()->getType());
        return true;
      }
      if (Name.startswith("arm.neon.vclz")) {
        Type* args[2] = {
          F->arg_begin()->getType(),
          Type::getInt1Ty(F->getContext())
        };
        // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
        // the end of the name. Change name from llvm.arm.neon.vclz.* to
        //  llvm.ctlz.*
        FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
        NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),
                                 "llvm.ctlz." + Name.substr(14), F->getParent());
        return true;
      }
      if (Name.startswith("arm.neon.vcnt")) {
        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
                                          F->arg_begin()->getType());
        return true;
      }
      static const Regex vldRegex("^arm\\.neon\\.vld([1234]|[234]lane)\\.v[a-z0-9]*$");
      if (vldRegex.match(Name)) {
        auto fArgs = F->getFunctionType()->params();
        SmallVector<Type *, 4> Tys(fArgs.begin(), fArgs.end());
        // Can't use Intrinsic::getDeclaration here as the return types might
        // then only be structurally equal.
        FunctionType* fType = FunctionType::get(F->getReturnType(), Tys, false);
        NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),
                                 "llvm." + Name + ".p0i8", F->getParent());
        return true;
      }
      static const Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");
      if (vstRegex.match(Name)) {
        static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,
                                                  Intrinsic::arm_neon_vst2,
                                                  Intrinsic::arm_neon_vst3,
                                                  Intrinsic::arm_neon_vst4};
  
        static const Intrinsic::ID StoreLaneInts[] = {
          Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
          Intrinsic::arm_neon_vst4lane
        };
  
        auto fArgs = F->getFunctionType()->params();
        Type *Tys[] = {fArgs[0], fArgs[1]};
        if (Name.find("lane") == StringRef::npos)
          NewFn = Intrinsic::getDeclaration(F->getParent(),
                                            StoreInts[fArgs.size() - 3], Tys);
        else
          NewFn = Intrinsic::getDeclaration(F->getParent(),
                                            StoreLaneInts[fArgs.size() - 5], Tys);
        return true;
      }
      if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") {
        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);
        return true;
      }
      if (Name.startswith("arm.neon.vqadds.")) {
        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::sadd_sat,
                                          F->arg_begin()->getType());
        return true;
      }
      if (Name.startswith("arm.neon.vqaddu.")) {
        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::uadd_sat,
                                          F->arg_begin()->getType());
        return true;
      }
      if (Name.startswith("arm.neon.vqsubs.")) {
        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ssub_sat,
                                          F->arg_begin()->getType());
        return true;
      }
      if (Name.startswith("arm.neon.vqsubu.")) {
        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::usub_sat,
                                          F->arg_begin()->getType());
        return true;
      }
      if (Name.startswith("aarch64.neon.addp")) {
        if (F->arg_size() != 2)
          break; // Invalid IR.
        VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
        if (Ty && Ty->getElementType()->isFloatingPointTy()) {
          NewFn = Intrinsic::getDeclaration(F->getParent(),
                                            Intrinsic::aarch64_neon_faddp, Ty);
          return true;
        }
      }
  
      // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and v16i8
      // respectively
      if ((Name.startswith("arm.neon.bfdot.") ||
           Name.startswith("aarch64.neon.bfdot.")) &&
          Name.endswith("i8")) {
        Intrinsic::ID IID =
            StringSwitch<Intrinsic::ID>(Name)
                .Cases("arm.neon.bfdot.v2f32.v8i8",
                       "arm.neon.bfdot.v4f32.v16i8",
                       Intrinsic::arm_neon_bfdot)
                .Cases("aarch64.neon.bfdot.v2f32.v8i8",
                       "aarch64.neon.bfdot.v4f32.v16i8",
                       Intrinsic::aarch64_neon_bfdot)
                .Default(Intrinsic::not_intrinsic);
        if (IID == Intrinsic::not_intrinsic)
          break;
  
        size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
        assert((OperandWidth == 64 || OperandWidth == 128) &&
               "Unexpected operand width");
        LLVMContext &Ctx = F->getParent()->getContext();
        std::array<Type *, 2> Tys {{
          F->getReturnType(),
          FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)
        }};
        NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
        return true;
      }
  
      // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic anymore
      // and accept v8bf16 instead of v16i8
      if ((Name.startswith("arm.neon.bfm") ||
           Name.startswith("aarch64.neon.bfm")) &&
          Name.endswith(".v4f32.v16i8")) {
        Intrinsic::ID IID =
            StringSwitch<Intrinsic::ID>(Name)
                .Case("arm.neon.bfmmla.v4f32.v16i8",
                      Intrinsic::arm_neon_bfmmla)
                .Case("arm.neon.bfmlalb.v4f32.v16i8",
                      Intrinsic::arm_neon_bfmlalb)
                .Case("arm.neon.bfmlalt.v4f32.v16i8",
                      Intrinsic::arm_neon_bfmlalt)
                .Case("aarch64.neon.bfmmla.v4f32.v16i8",
                      Intrinsic::aarch64_neon_bfmmla)
                .Case("aarch64.neon.bfmlalb.v4f32.v16i8",
                      Intrinsic::aarch64_neon_bfmlalb)
                .Case("aarch64.neon.bfmlalt.v4f32.v16i8",
                      Intrinsic::aarch64_neon_bfmlalt)
                .Default(Intrinsic::not_intrinsic);
        if (IID == Intrinsic::not_intrinsic)
          break;
  
        std::array<Type *, 0> Tys;
        NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
        return true;
      }
      break;
    }
  
    case 'c': {
      if (Name.startswith("ctlz.") && F->arg_size() == 1) {
        rename(F);
        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
                                          F->arg_begin()->getType());
        return true;
      }
      if (Name.startswith("cttz.") && F->arg_size() == 1) {
        rename(F);
        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
                                          F->arg_begin()->getType());
        return true;
      }
      break;
    }
    case 'd': {
      if (Name == "dbg.value" && F->arg_size() == 4) {
        rename(F);
        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
        return true;
      }
      break;
    }
    case 'e': {
      SmallVector<StringRef, 2> Groups;
      static const Regex R("^experimental.vector.reduce.([a-z]+)\\.[fi][0-9]+");
      if (R.match(Name, &Groups)) {
        Intrinsic::ID ID = Intrinsic::not_intrinsic;
        if (Groups[1] == "fadd")
          ID = Intrinsic::experimental_vector_reduce_v2_fadd;
        if (Groups[1] == "fmul")
          ID = Intrinsic::experimental_vector_reduce_v2_fmul;
  
        if (ID != Intrinsic::not_intrinsic) {
          rename(F);
          auto Args = F->getFunctionType()->params();
          Type *Tys[] = {F->getFunctionType()->getReturnType(), Args[1]};
          NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
          return true;
        }
      }
      break;
    }
    case 'i':
    case 'l': {
      bool IsLifetimeStart = Name.startswith("lifetime.start");
      if (IsLifetimeStart || Name.startswith("invariant.start")) {
        Intrinsic::ID ID = IsLifetimeStart ?
          Intrinsic::lifetime_start : Intrinsic::invariant_start;
        auto Args = F->getFunctionType()->params();
        Type* ObjectPtr[1] = {Args[1]};
        if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) {
          rename(F);
          NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
          return true;
        }
      }
  
      bool IsLifetimeEnd = Name.startswith("lifetime.end");
      if (IsLifetimeEnd || Name.startswith("invariant.end")) {
        Intrinsic::ID ID = IsLifetimeEnd ?
          Intrinsic::lifetime_end : Intrinsic::invariant_end;
  
        auto Args = F->getFunctionType()->params();
        Type* ObjectPtr[1] = {Args[IsLifetimeEnd ? 1 : 2]};
        if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) {
          rename(F);
          NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
          return true;
        }
      }
      if (Name.startswith("invariant.group.barrier")) {
        // Rename invariant.group.barrier to launder.invariant.group
        auto Args = F->getFunctionType()->params();
        Type* ObjectPtr[1] = {Args[0]};
        rename(F);
        NewFn = Intrinsic::getDeclaration(F->getParent(),
            Intrinsic::launder_invariant_group, ObjectPtr);
        return true;
  
      }
  
      break;
    }
    case 'm': {
      if (Name.startswith("masked.load.")) {
        Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() };
        if (F->getName() != Intrinsic::getName(Intrinsic::masked_load, Tys)) {
          rename(F);
          NewFn = Intrinsic::getDeclaration(F->getParent(),
                                            Intrinsic::masked_load,
                                            Tys);
          return true;
        }
      }
      if (Name.startswith("masked.store.")) {
        auto Args = F->getFunctionType()->params();
        Type *Tys[] = { Args[0], Args[1] };
        if (F->getName() != Intrinsic::getName(Intrinsic::masked_store, Tys)) {
          rename(F);
          NewFn = Intrinsic::getDeclaration(F->getParent(),
                                            Intrinsic::masked_store,
                                            Tys);
          return true;
        }
      }
      // Renaming gather/scatter intrinsics with no address space overloading
      // to the new overload which includes an address space
      if (Name.startswith("masked.gather.")) {
        Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
        if (F->getName() != Intrinsic::getName(Intrinsic::masked_gather, Tys)) {
          rename(F);
          NewFn = Intrinsic::getDeclaration(F->getParent(),
                                            Intrinsic::masked_gather, Tys);
          return true;
        }
      }
      if (Name.startswith("masked.scatter.")) {
        auto Args = F->getFunctionType()->params();
        Type *Tys[] = {Args[0], Args[1]};
        if (F->getName() != Intrinsic::getName(Intrinsic::masked_scatter, Tys)) {
          rename(F);
          NewFn = Intrinsic::getDeclaration(F->getParent(),
                                            Intrinsic::masked_scatter, Tys);
          return true;
        }
      }
      // Updating the memory intrinsics (memcpy/memmove/memset) that have an
      // alignment parameter to embedding the alignment as an attribute of
      // the pointer args.
      if (Name.startswith("memcpy.") && F->arg_size() == 5) {
        rename(F);
        // Get the types of dest, src, and len
        ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);
        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memcpy,
                                          ParamTypes);
        return true;
      }
      if (Name.startswith("memmove.") && F->arg_size() == 5) {
        rename(F);
        // Get the types of dest, src, and len
        ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);
        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memmove,
                                          ParamTypes);
        return true;
      }
      if (Name.startswith("memset.") && F->arg_size() == 5) {
        rename(F);
        // Get the types of dest, and len
        const auto *FT = F->getFunctionType();
        Type *ParamTypes[2] = {
            FT->getParamType(0), // Dest
            FT->getParamType(2)  // len
        };
        NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset,
                                          ParamTypes);
        return true;
      }
      break;
    }
    case 'n': {
      if (Name.startswith("nvvm.")) {
        Name = Name.substr(5);
  
        // The following nvvm intrinsics correspond exactly to an LLVM intrinsic.
        Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Name)
                                .Cases("brev32", "brev64", Intrinsic::bitreverse)
                                .Case("clz.i", Intrinsic::ctlz)
                                .Case("popc.i", Intrinsic::ctpop)
                                .Default(Intrinsic::not_intrinsic);
        if (IID != Intrinsic::not_intrinsic && F->arg_size() == 1) {
          NewFn = Intrinsic::getDeclaration(F->getParent(), IID,
                                            {F->getReturnType()});
          return true;
        }
  
        // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
        // not to an intrinsic alone.  We expand them in UpgradeIntrinsicCall.
        //
        // TODO: We could add lohi.i2d.
        bool Expand = StringSwitch<bool>(Name)
                          .Cases("abs.i", "abs.ll", true)
                          .Cases("clz.ll", "popc.ll", "h2f", true)
                          .Cases("max.i", "max.ll", "max.ui", "max.ull", true)
                          .Cases("min.i", "min.ll", "min.ui", "min.ull", true)
                          .StartsWith("atomic.load.add.f32.p", true)
                          .StartsWith("atomic.load.add.f64.p", true)
                          .Default(false);
        if (Expand) {
          NewFn = nullptr;
          return true;
        }
      }
      break;
    }
    case 'o':
      // We only need to change the name to match the mangling including the
      // address space.
      if (Name.startswith("objectsize.")) {
        Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
        if (F->arg_size() == 2 || F->arg_size() == 3 ||
            F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
          rename(F);
          NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize,
                                            Tys);
          return true;
        }
      }
      break;
  
    case 'p':
      if (Name == "prefetch") {
        // Handle address space overloading.
        Type *Tys[] = {F->arg_begin()->getType()};
        if (F->getName() != Intrinsic::getName(Intrinsic::prefetch, Tys)) {
          rename(F);
          NewFn =
              Intrinsic::getDeclaration(F->getParent(), Intrinsic::prefetch, Tys);
          return true;
        }
      }
      break;
  
    case 's':
      if (Name == "stackprotectorcheck") {
        NewFn = nullptr;
        return true;
      }
      break;
  
    case 'x':
      if (UpgradeX86IntrinsicFunction(F, Name, NewFn))
        return true;
    }
    // Remangle our intrinsic since we upgrade the mangling
    auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
    if (Result != None) {
      NewFn = Result.getValue();
      return true;
    }
  
    //  This may not belong here. This function is effectively being overloaded
    //  to both detect an intrinsic which needs upgrading, and to provide the
    //  upgraded form of the intrinsic. We should perhaps have two separate
    //  functions for this.
    return false;
  }
  
  bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
    NewFn = nullptr;
    bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
    assert(F != NewFn && "Intrinsic function upgraded to the same function");
  
    // Upgrade intrinsic attributes.  This does not change the function.
    if (NewFn)
      F = NewFn;
    if (Intrinsic::ID id = F->getIntrinsicID())
      F->setAttributes(Intrinsic::getAttributes(F->getContext(), id));
    return Upgraded;
  }
  
  GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
    if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
                            GV->getName() == "llvm.global_dtors")) ||
        !GV->hasInitializer())
      return nullptr;
    ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
    if (!ATy)
      return nullptr;
    StructType *STy = dyn_cast<StructType>(ATy->getElementType());
    if (!STy || STy->getNumElements() != 2)
      return nullptr;
  
    LLVMContext &C = GV->getContext();
    IRBuilder<> IRB(C);
    auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
                                 IRB.getInt8PtrTy());
    Constant *Init = GV->getInitializer();
    unsigned N = Init->getNumOperands();
    std::vector<Constant *> NewCtors(N);
    for (unsigned i = 0; i != N; ++i) {
      auto Ctor = cast<Constant>(Init->getOperand(i));
      NewCtors[i] = ConstantStruct::get(
          EltTy, Ctor->getAggregateElement(0u), Ctor->getAggregateElement(1),
          Constant::getNullValue(IRB.getInt8PtrTy()));
    }
    Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
  
    return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
                              NewInit, GV->getName());
  }
  
  // Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
  // to byte shuffles.
  static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder,
                                           Value *Op, unsigned Shift) {
    auto *ResultTy = cast<FixedVectorType>(Op->getType());
    unsigned NumElts = ResultTy->getNumElements() * 8;
  
    // Bitcast from a 64-bit element type to a byte element type.
    Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
    Op = Builder.CreateBitCast(Op, VecTy, "cast");
  
    // We'll be shuffling in zeroes.
    Value *Res = Constant::getNullValue(VecTy);
  
    // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
    // we'll just return the zero vector.
    if (Shift < 16) {
      int Idxs[64];
      // 256/512-bit version is split into 2/4 16-byte lanes.
      for (unsigned l = 0; l != NumElts; l += 16)
        for (unsigned i = 0; i != 16; ++i) {
          unsigned Idx = NumElts + i - Shift;
          if (Idx < NumElts)
            Idx -= NumElts - 16; // end of lane, switch operand.
          Idxs[l + i] = Idx + l;
        }
  
      Res = Builder.CreateShuffleVector(Res, Op, makeArrayRef(Idxs, NumElts));
    }
  
    // Bitcast back to a 64-bit element type.
    return Builder.CreateBitCast(Res, ResultTy, "cast");
  }
  
  // Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
  // to byte shuffles.
  static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
                                           unsigned Shift) {
    auto *ResultTy = cast<FixedVectorType>(Op->getType());
    unsigned NumElts = ResultTy->getNumElements() * 8;
  
    // Bitcast from a 64-bit element type to a byte element type.
    Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
    Op = Builder.CreateBitCast(Op, VecTy, "cast");
  
    // We'll be shuffling in zeroes.
    Value *Res = Constant::getNullValue(VecTy);
  
    // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
    // we'll just return the zero vector.
    if (Shift < 16) {
      int Idxs[64];
      // 256/512-bit version is split into 2/4 16-byte lanes.
      for (unsigned l = 0; l != NumElts; l += 16)
        for (unsigned i = 0; i != 16; ++i) {
          unsigned Idx = i + Shift;
          if (Idx >= 16)
            Idx += NumElts - 16; // end of lane, switch operand.
          Idxs[l + i] = Idx + l;
        }
  
      Res = Builder.CreateShuffleVector(Op, Res, makeArrayRef(Idxs, NumElts));
    }
  
    // Bitcast back to a 64-bit element type.
    return Builder.CreateBitCast(Res, ResultTy, "cast");
  }
  
  static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
                              unsigned NumElts) {
    assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
    llvm::VectorType *MaskTy = FixedVectorType::get(
        Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
    Mask = Builder.CreateBitCast(Mask, MaskTy);
  
    // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
    // i8 and we need to extract down to the right number of elements.
    if (NumElts <= 4) {
      int Indices[4];
      for (unsigned i = 0; i != NumElts; ++i)
        Indices[i] = i;
      Mask = Builder.CreateShuffleVector(
          Mask, Mask, makeArrayRef(Indices, NumElts), "extract");
    }
  
    return Mask;
  }
  
  static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask,
                              Value *Op0, Value *Op1) {
    // If the mask is all ones just emit the first operation.
    if (const auto *C = dyn_cast<Constant>(Mask))
      if (C->isAllOnesValue())
        return Op0;
  
    Mask = getX86MaskVec(Builder, Mask,
                         cast<FixedVectorType>(Op0->getType())->getNumElements());
    return Builder.CreateSelect(Mask, Op0, Op1);
  }
  
  static Value *EmitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask,
                                    Value *Op0, Value *Op1) {
    // If the mask is all ones just emit the first operation.
    if (const auto *C = dyn_cast<Constant>(Mask))
      if (C->isAllOnesValue())
        return Op0;
  
    auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
                                        Mask->getType()->getIntegerBitWidth());
    Mask = Builder.CreateBitCast(Mask, MaskTy);
    Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
    return Builder.CreateSelect(Mask, Op0, Op1);
  }
  
  // Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
  // PALIGNR handles large immediates by shifting while VALIGN masks the immediate
  // so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
  static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
                                          Value *Op1, Value *Shift,
                                          Value *Passthru, Value *Mask,
                                          bool IsVALIGN) {
    unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
  
    unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
    assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
    assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
    assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
  
    // Mask the immediate for VALIGN.
    if (IsVALIGN)
      ShiftVal &= (NumElts - 1);
  
    // If palignr is shifting the pair of vectors more than the size of two
    // lanes, emit zero.
    if (ShiftVal >= 32)
      return llvm::Constant::getNullValue(Op0->getType());
  
    // If palignr is shifting the pair of input vectors more than one lane,
    // but less than two lanes, convert to shifting in zeroes.
    if (ShiftVal > 16) {
      ShiftVal -= 16;
      Op1 = Op0;
      Op0 = llvm::Constant::getNullValue(Op0->getType());
    }
  
    int Indices[64];
    // 256-bit palignr operates on 128-bit lanes so we need to handle that
    for (unsigned l = 0; l < NumElts; l += 16) {
      for (unsigned i = 0; i != 16; ++i) {
        unsigned Idx = ShiftVal + i;
        if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
          Idx += NumElts - 16; // End of lane, switch operand.
        Indices[l + i] = Idx + l;
      }
    }
  
    Value *Align = Builder.CreateShuffleVector(Op1, Op0,
                                               makeArrayRef(Indices, NumElts),
                                               "palignr");
  
    return EmitX86Select(Builder, Mask, Align, Passthru);
  }
  
  static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallInst &CI,
                                            bool ZeroMask, bool IndexForm) {
    Type *Ty = CI.getType();
    unsigned VecWidth = Ty->getPrimitiveSizeInBits();
    unsigned EltWidth = Ty->getScalarSizeInBits();
    bool IsFloat = Ty->isFPOrFPVectorTy();
    Intrinsic::ID IID;
    if (VecWidth == 128 && EltWidth == 32 && IsFloat)
      IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
    else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
      IID = Intrinsic::x86_avx512_vpermi2var_d_128;
    else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
      IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
    else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
      IID = Intrinsic::x86_avx512_vpermi2var_q_128;
    else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
      IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
    else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
      IID = Intrinsic::x86_avx512_vpermi2var_d_256;
    else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
      IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
    else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
      IID = Intrinsic::x86_avx512_vpermi2var_q_256;
    else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
      IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
    else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
      IID = Intrinsic::x86_avx512_vpermi2var_d_512;
    else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
      IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
    else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
      IID = Intrinsic::x86_avx512_vpermi2var_q_512;
    else if (VecWidth == 128 && EltWidth == 16)
      IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
    else if (VecWidth == 256 && EltWidth == 16)
      IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
    else if (VecWidth == 512 && EltWidth == 16)
      IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
    else if (VecWidth == 128 && EltWidth == 8)
      IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
    else if (VecWidth == 256 && EltWidth == 8)
      IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
    else if (VecWidth == 512 && EltWidth == 8)
      IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
    else
      llvm_unreachable("Unexpected intrinsic");
  
    Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
                      CI.getArgOperand(2) };
  
    // If this isn't index form we need to swap operand 0 and 1.
    if (!IndexForm)
      std::swap(Args[0], Args[1]);
  
    Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
                                  Args);
    Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
                               : Builder.CreateBitCast(CI.getArgOperand(1),
                                                       Ty);
    return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
  }
  
  static Value *UpgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallInst &CI,
                                           Intrinsic::ID IID) {
    Type *Ty = CI.getType();
    Value *Op0 = CI.getOperand(0);
    Value *Op1 = CI.getOperand(1);
    Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
    Value *Res = Builder.CreateCall(Intrin, {Op0, Op1});
  
    if (CI.getNumArgOperands() == 4) { // For masked intrinsics.
      Value *VecSrc = CI.getOperand(2);
      Value *Mask = CI.getOperand(3);
      Res = EmitX86Select(Builder, Mask, Res, VecSrc);
    }
    return Res;
  }
  
  static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallInst &CI,
                                 bool IsRotateRight) {
    Type *Ty = CI.getType();
    Value *Src = CI.getArgOperand(0);
    Value *Amt = CI.getArgOperand(1);
  
    // Amount may be scalar immediate, in which case create a splat vector.
    // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
    // we only care about the lowest log2 bits anyway.
    if (Amt->getType() != Ty) {
      unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
      Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
      Amt = Builder.CreateVectorSplat(NumElts, Amt);
    }
  
    Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
    Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
    Value *Res = Builder.CreateCall(Intrin, {Src, Src, Amt});
  
    if (CI.getNumArgOperands() == 4) { // For masked intrinsics.
      Value *VecSrc = CI.getOperand(2);
      Value *Mask = CI.getOperand(3);
      Res = EmitX86Select(Builder, Mask, Res, VecSrc);
    }
    return Res;
  }
  
  static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallInst &CI, unsigned Imm,
                                bool IsSigned) {
    Type *Ty = CI.getType();
    Value *LHS = CI.getArgOperand(0);
    Value *RHS = CI.getArgOperand(1);
  
    CmpInst::Predicate Pred;
    switch (Imm) {
    case 0x0:
      Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
      break;
    case 0x1:
      Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
      break;
    case 0x2:
      Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
      break;
    case 0x3:
      Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
      break;
    case 0x4:
      Pred = ICmpInst::ICMP_EQ;
      break;
    case 0x5:
      Pred = ICmpInst::ICMP_NE;
      break;
    case 0x6:
      return Constant::getNullValue(Ty); // FALSE
    case 0x7:
      return Constant::getAllOnesValue(Ty); // TRUE
    default:
      llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
    }
  
    Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
    Value *Ext = Builder.CreateSExt(Cmp, Ty);
    return Ext;
  }
  
  static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallInst &CI,
                                      bool IsShiftRight, bool ZeroMask) {
    Type *Ty = CI.getType();
    Value *Op0 = CI.getArgOperand(0);
    Value *Op1 = CI.getArgOperand(1);
    Value *Amt = CI.getArgOperand(2);
  
    if (IsShiftRight)
      std::swap(Op0, Op1);
  
    // Amount may be scalar immediate, in which case create a splat vector.
    // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
    // we only care about the lowest log2 bits anyway.
    if (Amt->getType() != Ty) {
      unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
      Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
      Amt = Builder.CreateVectorSplat(NumElts, Amt);
    }
  
    Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
    Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
    Value *Res = Builder.CreateCall(Intrin, {Op0, Op1, Amt});
  
    unsigned NumArgs = CI.getNumArgOperands();
    if (NumArgs >= 4) { // For masked intrinsics.
      Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
                      ZeroMask     ? ConstantAggregateZero::get(CI.getType()) :
                                     CI.getArgOperand(0);
      Value *Mask = CI.getOperand(NumArgs - 1);
      Res = EmitX86Select(Builder, Mask, Res, VecSrc);
    }
    return Res;
  }
  
  static Value *UpgradeMaskedStore(IRBuilder<> &Builder,
                                   Value *Ptr, Value *Data, Value *Mask,
                                   bool Aligned) {
    // Cast the pointer to the right type.
    Ptr = Builder.CreateBitCast(Ptr,
                                llvm::PointerType::getUnqual(Data->getType()));
    const Align Alignment =
        Aligned
            ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedSize() / 8)
            : Align(1);
  
    // If the mask is all ones just emit a regular store.
    if (const auto *C = dyn_cast<Constant>(Mask))
      if (C->isAllOnesValue())
        return Builder.CreateAlignedStore(Data, Ptr, Alignment);
  
    // Convert the mask from an integer type to a vector of i1.
    unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
    Mask = getX86MaskVec(Builder, Mask, NumElts);
    return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
  }
  
  static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,
                                  Value *Ptr, Value *Passthru, Value *Mask,
                                  bool Aligned) {
    Type *ValTy = Passthru->getType();
    // Cast the pointer to the right type.
    Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ValTy));
    const Align Alignment =
        Aligned
            ? Align(Passthru->getType()->getPrimitiveSizeInBits().getFixedSize() /
                    8)
            : Align(1);
  
    // If the mask is all ones just emit a regular store.
    if (const auto *C = dyn_cast<Constant>(Mask))
      if (C->isAllOnesValue())
        return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
  
    // Convert the mask from an integer type to a vector of i1.
    unsigned NumElts =
        cast<FixedVectorType>(Passthru->getType())->getNumElements();
    Mask = getX86MaskVec(Builder, Mask, NumElts);
    return Builder.CreateMaskedLoad(Ptr, Alignment, Mask, Passthru);
  }
  
  static Value *upgradeAbs(IRBuilder<> &Builder, CallInst &CI) {
    Type *Ty = CI.getType();
    Value *Op0 = CI.getArgOperand(0);
    Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty);
    Value *Res = Builder.CreateCall(F, {Op0, Builder.getInt1(false)});
    if (CI.getNumArgOperands() == 3)
      Res = EmitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
    return Res;
  }
  
  static Value *upgradePMULDQ(IRBuilder<> &Builder, CallInst &CI, bool IsSigned) {
    Type *Ty = CI.getType();
  
    // Arguments have a vXi32 type so cast to vXi64.
    Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
    Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
  
    if (IsSigned) {
      // Shift left then arithmetic shift right.
      Constant *ShiftAmt = ConstantInt::get(Ty, 32);
      LHS = Builder.CreateShl(LHS, ShiftAmt);
      LHS = Builder.CreateAShr(LHS, ShiftAmt);
      RHS = Builder.CreateShl(RHS, ShiftAmt);
      RHS = Builder.CreateAShr(RHS, ShiftAmt);
    } else {
      // Clear the upper bits.
      Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
      LHS = Builder.CreateAnd(LHS, Mask);
      RHS = Builder.CreateAnd(RHS, Mask);
    }
  
    Value *Res = Builder.CreateMul(LHS, RHS);
  
    if (CI.getNumArgOperands() == 4)
      Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
  
    return Res;
  }
  
  // Applying mask on vector of i1's and make sure result is at least 8 bits wide.
  static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
                                       Value *Mask) {
    unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
    if (Mask) {
      const auto *C = dyn_cast<Constant>(Mask);
      if (!C || !C->isAllOnesValue())
        Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
    }
  
    if (NumElts < 8) {
      int Indices[8];
      for (unsigned i = 0; i != NumElts; ++i)
        Indices[i] = i;
      for (unsigned i = NumElts; i != 8; ++i)
        Indices[i] = NumElts + i % NumElts;
      Vec = Builder.CreateShuffleVector(Vec,
                                        Constant::getNullValue(Vec->getType()),
                                        Indices);
    }
    return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
  }
  
  static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI,
                                     unsigned CC, bool Signed) {
    Value *Op0 = CI.getArgOperand(0);
    unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
  
    Value *Cmp;
    if (CC == 3) {
      Cmp = Constant::getNullValue(
          FixedVectorType::get(Builder.getInt1Ty(), NumElts));
    } else if (CC == 7) {
      Cmp = Constant::getAllOnesValue(
          FixedVectorType::get(Builder.getInt1Ty(), NumElts));
    } else {
      ICmpInst::Predicate Pred;
      switch (CC) {
      default: llvm_unreachable("Unknown condition code");
      case 0: Pred = ICmpInst::ICMP_EQ;  break;
      case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
      case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
      case 4: Pred = ICmpInst::ICMP_NE;  break;
      case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
      case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
      }
      Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
    }
  
    Value *Mask = CI.getArgOperand(CI.getNumArgOperands() - 1);
  
    return ApplyX86MaskOn1BitsVec(Builder, Cmp, Mask);
  }
  
  // Replace a masked intrinsic with an older unmasked intrinsic.
  static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI,
                                      Intrinsic::ID IID) {
    Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID);
    Value *Rep = Builder.CreateCall(Intrin,
                                   { CI.getArgOperand(0), CI.getArgOperand(1) });
    return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
  }
  
  static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) {
    Value* A = CI.getArgOperand(0);
    Value* B = CI.getArgOperand(1);
    Value* Src = CI.getArgOperand(2);
    Value* Mask = CI.getArgOperand(3);
  
    Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
    Value* Cmp = Builder.CreateIsNotNull(AndNode);
    Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
    Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
    Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
    return Builder.CreateInsertElement(A, Select, (uint64_t)0);
  }
  
  
  static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallInst &CI) {
    Value* Op = CI.getArgOperand(0);
    Type* ReturnOp = CI.getType();
    unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
    Value *Mask = getX86MaskVec(Builder, Op, NumElts);
    return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
  }
  
  // Replace intrinsic with unmasked version and a select.
  static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
                                        CallInst &CI, Value *&Rep) {
    Name = Name.substr(12); // Remove avx512.mask.
  
    unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
    unsigned EltWidth = CI.getType()->getScalarSizeInBits();
    Intrinsic::ID IID;
    if (Name.startswith("max.p")) {
      if (VecWidth == 128 && EltWidth == 32)
        IID = Intrinsic::x86_sse_max_ps;
      else if (VecWidth == 128 && EltWidth == 64)
        IID = Intrinsic::x86_sse2_max_pd;
      else if (VecWidth == 256 && EltWidth == 32)
        IID = Intrinsic::x86_avx_max_ps_256;
      else if (VecWidth == 256 && EltWidth == 64)
        IID = Intrinsic::x86_avx_max_pd_256;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("min.p")) {
      if (VecWidth == 128 && EltWidth == 32)
        IID = Intrinsic::x86_sse_min_ps;
      else if (VecWidth == 128 && EltWidth == 64)
        IID = Intrinsic::x86_sse2_min_pd;
      else if (VecWidth == 256 && EltWidth == 32)
        IID = Intrinsic::x86_avx_min_ps_256;
      else if (VecWidth == 256 && EltWidth == 64)
        IID = Intrinsic::x86_avx_min_pd_256;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("pshuf.b.")) {
      if (VecWidth == 128)
        IID = Intrinsic::x86_ssse3_pshuf_b_128;
      else if (VecWidth == 256)
        IID = Intrinsic::x86_avx2_pshuf_b;
      else if (VecWidth == 512)
        IID = Intrinsic::x86_avx512_pshuf_b_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("pmul.hr.sw.")) {
      if (VecWidth == 128)
        IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
      else if (VecWidth == 256)
        IID = Intrinsic::x86_avx2_pmul_hr_sw;
      else if (VecWidth == 512)
        IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("pmulh.w.")) {
      if (VecWidth == 128)
        IID = Intrinsic::x86_sse2_pmulh_w;
      else if (VecWidth == 256)
        IID = Intrinsic::x86_avx2_pmulh_w;
      else if (VecWidth == 512)
        IID = Intrinsic::x86_avx512_pmulh_w_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("pmulhu.w.")) {
      if (VecWidth == 128)
        IID = Intrinsic::x86_sse2_pmulhu_w;
      else if (VecWidth == 256)
        IID = Intrinsic::x86_avx2_pmulhu_w;
      else if (VecWidth == 512)
        IID = Intrinsic::x86_avx512_pmulhu_w_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("pmaddw.d.")) {
      if (VecWidth == 128)
        IID = Intrinsic::x86_sse2_pmadd_wd;
      else if (VecWidth == 256)
        IID = Intrinsic::x86_avx2_pmadd_wd;
      else if (VecWidth == 512)
        IID = Intrinsic::x86_avx512_pmaddw_d_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("pmaddubs.w.")) {
      if (VecWidth == 128)
        IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
      else if (VecWidth == 256)
        IID = Intrinsic::x86_avx2_pmadd_ub_sw;
      else if (VecWidth == 512)
        IID = Intrinsic::x86_avx512_pmaddubs_w_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("packsswb.")) {
      if (VecWidth == 128)
        IID = Intrinsic::x86_sse2_packsswb_128;
      else if (VecWidth == 256)
        IID = Intrinsic::x86_avx2_packsswb;
      else if (VecWidth == 512)
        IID = Intrinsic::x86_avx512_packsswb_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("packssdw.")) {
      if (VecWidth == 128)
        IID = Intrinsic::x86_sse2_packssdw_128;
      else if (VecWidth == 256)
        IID = Intrinsic::x86_avx2_packssdw;
      else if (VecWidth == 512)
        IID = Intrinsic::x86_avx512_packssdw_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("packuswb.")) {
      if (VecWidth == 128)
        IID = Intrinsic::x86_sse2_packuswb_128;
      else if (VecWidth == 256)
        IID = Intrinsic::x86_avx2_packuswb;
      else if (VecWidth == 512)
        IID = Intrinsic::x86_avx512_packuswb_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("packusdw.")) {
      if (VecWidth == 128)
        IID = Intrinsic::x86_sse41_packusdw;
      else if (VecWidth == 256)
        IID = Intrinsic::x86_avx2_packusdw;
      else if (VecWidth == 512)
        IID = Intrinsic::x86_avx512_packusdw_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("vpermilvar.")) {
      if (VecWidth == 128 && EltWidth == 32)
        IID = Intrinsic::x86_avx_vpermilvar_ps;
      else if (VecWidth == 128 && EltWidth == 64)
        IID = Intrinsic::x86_avx_vpermilvar_pd;
      else if (VecWidth == 256 && EltWidth == 32)
        IID = Intrinsic::x86_avx_vpermilvar_ps_256;
      else if (VecWidth == 256 && EltWidth == 64)
        IID = Intrinsic::x86_avx_vpermilvar_pd_256;
      else if (VecWidth == 512 && EltWidth == 32)
        IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
      else if (VecWidth == 512 && EltWidth == 64)
        IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name == "cvtpd2dq.256") {
      IID = Intrinsic::x86_avx_cvt_pd2dq_256;
    } else if (Name == "cvtpd2ps.256") {
      IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
    } else if (Name == "cvttpd2dq.256") {
      IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
    } else if (Name == "cvttps2dq.128") {
      IID = Intrinsic::x86_sse2_cvttps2dq;
    } else if (Name == "cvttps2dq.256") {
      IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
    } else if (Name.startswith("permvar.")) {
      bool IsFloat = CI.getType()->isFPOrFPVectorTy();
      if (VecWidth == 256 && EltWidth == 32 && IsFloat)
        IID = Intrinsic::x86_avx2_permps;
      else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
        IID = Intrinsic::x86_avx2_permd;
      else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
        IID = Intrinsic::x86_avx512_permvar_df_256;
      else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
        IID = Intrinsic::x86_avx512_permvar_di_256;
      else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
        IID = Intrinsic::x86_avx512_permvar_sf_512;
      else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
        IID = Intrinsic::x86_avx512_permvar_si_512;
      else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
        IID = Intrinsic::x86_avx512_permvar_df_512;
      else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
        IID = Intrinsic::x86_avx512_permvar_di_512;
      else if (VecWidth == 128 && EltWidth == 16)
        IID = Intrinsic::x86_avx512_permvar_hi_128;
      else if (VecWidth == 256 && EltWidth == 16)
        IID = Intrinsic::x86_avx512_permvar_hi_256;
      else if (VecWidth == 512 && EltWidth == 16)
        IID = Intrinsic::x86_avx512_permvar_hi_512;
      else if (VecWidth == 128 && EltWidth == 8)
        IID = Intrinsic::x86_avx512_permvar_qi_128;
      else if (VecWidth == 256 && EltWidth == 8)
        IID = Intrinsic::x86_avx512_permvar_qi_256;
      else if (VecWidth == 512 && EltWidth == 8)
        IID = Intrinsic::x86_avx512_permvar_qi_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("dbpsadbw.")) {
      if (VecWidth == 128)
        IID = Intrinsic::x86_avx512_dbpsadbw_128;
      else if (VecWidth == 256)
        IID = Intrinsic::x86_avx512_dbpsadbw_256;
      else if (VecWidth == 512)
        IID = Intrinsic::x86_avx512_dbpsadbw_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("pmultishift.qb.")) {
      if (VecWidth == 128)
        IID = Intrinsic::x86_avx512_pmultishift_qb_128;
      else if (VecWidth == 256)
        IID = Intrinsic::x86_avx512_pmultishift_qb_256;
      else if (VecWidth == 512)
        IID = Intrinsic::x86_avx512_pmultishift_qb_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("conflict.")) {
      if (Name[9] == 'd' && VecWidth == 128)
        IID = Intrinsic::x86_avx512_conflict_d_128;
      else if (Name[9] == 'd' && VecWidth == 256)
        IID = Intrinsic::x86_avx512_conflict_d_256;
      else if (Name[9] == 'd' && VecWidth == 512)
        IID = Intrinsic::x86_avx512_conflict_d_512;
      else if (Name[9] == 'q' && VecWidth == 128)
        IID = Intrinsic::x86_avx512_conflict_q_128;
      else if (Name[9] == 'q' && VecWidth == 256)
        IID = Intrinsic::x86_avx512_conflict_q_256;
      else if (Name[9] == 'q' && VecWidth == 512)
        IID = Intrinsic::x86_avx512_conflict_q_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else if (Name.startswith("pavg.")) {
      if (Name[5] == 'b' && VecWidth == 128)
        IID = Intrinsic::x86_sse2_pavg_b;
      else if (Name[5] == 'b' && VecWidth == 256)
        IID = Intrinsic::x86_avx2_pavg_b;
      else if (Name[5] == 'b' && VecWidth == 512)
        IID = Intrinsic::x86_avx512_pavg_b_512;
      else if (Name[5] == 'w' && VecWidth == 128)
        IID = Intrinsic::x86_sse2_pavg_w;
      else if (Name[5] == 'w' && VecWidth == 256)
        IID = Intrinsic::x86_avx2_pavg_w;
      else if (Name[5] == 'w' && VecWidth == 512)
        IID = Intrinsic::x86_avx512_pavg_w_512;
      else
        llvm_unreachable("Unexpected intrinsic");
    } else
      return false;
  
    SmallVector<Value *, 4> Args(CI.arg_operands().begin(),
                                 CI.arg_operands().end());
    Args.pop_back();
    Args.pop_back();
    Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
                             Args);
    unsigned NumArgs = CI.getNumArgOperands();
    Rep = EmitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
                        CI.getArgOperand(NumArgs - 2));
    return true;
  }
  
  /// Upgrade comment in call to inline asm that represents an objc retain release
  /// marker.
  void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
    size_t Pos;
    if (AsmStr->find("mov\tfp") == 0 &&
        AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
        (Pos = AsmStr->find("# marker")) != std::string::npos) {
      AsmStr->replace(Pos, 1, ";");
    }
    return;
  }
  
  /// Upgrade a call to an old intrinsic. All argument and return casting must be
  /// provided to seamlessly integrate with existing context.
  void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
    Function *F = CI->getCalledFunction();
    LLVMContext &C = CI->getContext();
    IRBuilder<> Builder(C);
    Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
  
    assert(F && "Intrinsic call is not direct?");
  
    if (!NewFn) {
      // Get the Function's name.
      StringRef Name = F->getName();
  
      assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'");
      Name = Name.substr(5);
  
      bool IsX86 = Name.startswith("x86.");
      if (IsX86)
        Name = Name.substr(4);
      bool IsNVVM = Name.startswith("nvvm.");
      if (IsNVVM)
        Name = Name.substr(5);
  
      if (IsX86 && Name.startswith("sse4a.movnt.")) {
        Module *M = F->getParent();
        SmallVector<Metadata *, 1> Elts;
        Elts.push_back(
            ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
        MDNode *Node = MDNode::get(C, Elts);
  
        Value *Arg0 = CI->getArgOperand(0);
        Value *Arg1 = CI->getArgOperand(1);
  
        // Nontemporal (unaligned) store of the 0'th element of the float/double
        // vector.
        Type *SrcEltTy = cast<VectorType>(Arg1->getType())->getElementType();
        PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy);
        Value *Addr = Builder.CreateBitCast(Arg0, EltPtrTy, "cast");
        Value *Extract =
            Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
  
        StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, Align(1));
        SI->setMetadata(M->getMDKindID("nontemporal"), Node);
  
        // Remove intrinsic.
        CI->eraseFromParent();
        return;
      }
  
      if (IsX86 && (Name.startswith("avx.movnt.") ||
                    Name.startswith("avx512.storent."))) {
        Module *M = F->getParent();
        SmallVector<Metadata *, 1> Elts;
        Elts.push_back(
            ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
        MDNode *Node = MDNode::get(C, Elts);
  
        Value *Arg0 = CI->getArgOperand(0);
        Value *Arg1 = CI->getArgOperand(1);
  
        // Convert the type of the pointer to a pointer to the stored type.
        Value *BC = Builder.CreateBitCast(Arg0,
                                          PointerType::getUnqual(Arg1->getType()),
                                          "cast");
        StoreInst *SI = Builder.CreateAlignedStore(
            Arg1, BC,
            Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedSize() / 8));
        SI->setMetadata(M->getMDKindID("nontemporal"), Node);
  
        // Remove intrinsic.
        CI->eraseFromParent();
        return;
      }
  
      if (IsX86 && Name == "sse2.storel.dq") {
        Value *Arg0 = CI->getArgOperand(0);
        Value *Arg1 = CI->getArgOperand(1);
  
        auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
        Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
        Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
        Value *BC = Builder.CreateBitCast(Arg0,
                                          PointerType::getUnqual(Elt->getType()),
                                          "cast");
        Builder.CreateAlignedStore(Elt, BC, Align(1));
  
        // Remove intrinsic.
        CI->eraseFromParent();
        return;
      }
  
      if (IsX86 && (Name.startswith("sse.storeu.") ||
                    Name.startswith("sse2.storeu.") ||
                    Name.startswith("avx.storeu."))) {
        Value *Arg0 = CI->getArgOperand(0);
        Value *Arg1 = CI->getArgOperand(1);
  
        Arg0 = Builder.CreateBitCast(Arg0,
                                     PointerType::getUnqual(Arg1->getType()),
                                     "cast");
        Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
  
        // Remove intrinsic.
        CI->eraseFromParent();
        return;
      }
  
      if (IsX86 && Name == "avx512.mask.store.ss") {
        Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
        UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
                           Mask, false);
  
        // Remove intrinsic.
        CI->eraseFromParent();
        return;
      }
  
      if (IsX86 && (Name.startswith("avx512.mask.store"))) {
        // "avx512.mask.storeu." or "avx512.mask.store."
        bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
        UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
                           CI->getArgOperand(2), Aligned);
  
        // Remove intrinsic.
        CI->eraseFromParent();
        return;
      }
  
      Value *Rep;
      // Upgrade packed integer vector compare intrinsics to compare instructions.
      if (IsX86 && (Name.startswith("sse2.pcmp") ||
                    Name.startswith("avx2.pcmp"))) {
        // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
        bool CmpEq = Name[9] == 'e';
        Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
                                 CI->getArgOperand(0), CI->getArgOperand(1));
        Rep = Builder.CreateSExt(Rep, CI->getType(), "");
      } else if (IsX86 && (Name.startswith("avx512.broadcastm"))) {
        Type *ExtTy = Type::getInt32Ty(C);
        if (CI->getOperand(0)->getType()->isIntegerTy(8))
          ExtTy = Type::getInt64Ty(C);
        unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
                           ExtTy->getPrimitiveSizeInBits();
        Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
        Rep = Builder.CreateVectorSplat(NumElts, Rep);
      } else if (IsX86 && (Name == "sse.sqrt.ss" ||
                           Name == "sse2.sqrt.sd")) {
        Value *Vec = CI->getArgOperand(0);
        Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
        Function *Intr = Intrinsic::getDeclaration(F->getParent(),
                                                   Intrinsic::sqrt, Elt0->getType());
        Elt0 = Builder.CreateCall(Intr, Elt0);
        Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
      } else if (IsX86 && (Name.startswith("avx.sqrt.p") ||
                           Name.startswith("sse2.sqrt.p") ||
                           Name.startswith("sse.sqrt.p"))) {
        Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
                                                           Intrinsic::sqrt,
                                                           CI->getType()),
                                 {CI->getArgOperand(0)});
      } else if (IsX86 && (Name.startswith("avx512.mask.sqrt.p"))) {
        if (CI->getNumArgOperands() == 4 &&
            (!isa<ConstantInt>(CI->getArgOperand(3)) ||
             cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
          Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
                                              : Intrinsic::x86_avx512_sqrt_pd_512;
  
          Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) };
          Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
                                                             IID), Args);
        } else {
          Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
                                                             Intrinsic::sqrt,
                                                             CI->getType()),
                                   {CI->getArgOperand(0)});
        }
        Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                            CI->getArgOperand(1));
      } else if (IsX86 && (Name.startswith("avx512.ptestm") ||
                           Name.startswith("avx512.ptestnm"))) {
        Value *Op0 = CI->getArgOperand(0);
        Value *Op1 = CI->getArgOperand(1);
        Value *Mask = CI->getArgOperand(2);
        Rep = Builder.CreateAnd(Op0, Op1);
        llvm::Type *Ty = Op0->getType();
        Value *Zero = llvm::Constant::getNullValue(Ty);
        ICmpInst::Predicate Pred =
          Name.startswith("avx512.ptestm") ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
        Rep = Builder.CreateICmp(Pred, Rep, Zero);
        Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask);
      } else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))){
        unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
                               ->getNumElements();
        Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
        Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                            CI->getArgOperand(1));
      } else if (IsX86 && (Name.startswith("avx512.kunpck"))) {
        unsigned NumElts = CI->getType()->getScalarSizeInBits();
        Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
        Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
        int Indices[64];
        for (unsigned i = 0; i != NumElts; ++i)
          Indices[i] = i;
  
        // First extract half of each vector. This gives better codegen than
        // doing it in a single shuffle.
        LHS = Builder.CreateShuffleVector(LHS, LHS,
                                          makeArrayRef(Indices, NumElts / 2));
        RHS = Builder.CreateShuffleVector(RHS, RHS,
                                          makeArrayRef(Indices, NumElts / 2));
        // Concat the vectors.
        // NOTE: Operands have to be swapped to match intrinsic definition.
        Rep = Builder.CreateShuffleVector(RHS, LHS,
                                          makeArrayRef(Indices, NumElts));
        Rep = Builder.CreateBitCast(Rep, CI->getType());
      } else if (IsX86 && Name == "avx512.kand.w") {
        Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
        Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
        Rep = Builder.CreateAnd(LHS, RHS);
        Rep = Builder.CreateBitCast(Rep, CI->getType());
      } else if (IsX86 && Name == "avx512.kandn.w") {
        Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
        Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
        LHS = Builder.CreateNot(LHS);
        Rep = Builder.CreateAnd(LHS, RHS);
        Rep = Builder.CreateBitCast(Rep, CI->getType());
      } else if (IsX86 && Name == "avx512.kor.w") {
        Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
        Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
        Rep = Builder.CreateOr(LHS, RHS);
        Rep = Builder.CreateBitCast(Rep, CI->getType());
      } else if (IsX86 && Name == "avx512.kxor.w") {
        Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
        Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
        Rep = Builder.CreateXor(LHS, RHS);
        Rep = Builder.CreateBitCast(Rep, CI->getType());
      } else if (IsX86 && Name == "avx512.kxnor.w") {
        Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
        Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
        LHS = Builder.CreateNot(LHS);
        Rep = Builder.CreateXor(LHS, RHS);
        Rep = Builder.CreateBitCast(Rep, CI->getType());
      } else if (IsX86 && Name == "avx512.knot.w") {
        Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
        Rep = Builder.CreateNot(Rep);
        Rep = Builder.CreateBitCast(Rep, CI->getType());
      } else if (IsX86 &&
                 (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w")) {
        Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
        Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
        Rep = Builder.CreateOr(LHS, RHS);
        Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
        Value *C;
        if (Name[14] == 'c')
          C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
        else
          C = ConstantInt::getNullValue(Builder.getInt16Ty());
        Rep = Builder.CreateICmpEQ(Rep, C);
        Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
      } else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
                           Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
                           Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
                           Name == "sse.div.ss" || Name == "sse2.div.sd")) {
        Type *I32Ty = Type::getInt32Ty(C);
        Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
                                                   ConstantInt::get(I32Ty, 0));
        Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
                                                   ConstantInt::get(I32Ty, 0));
        Value *EltOp;
        if (Name.contains(".add."))
          EltOp = Builder.CreateFAdd(Elt0, Elt1);
        else if (Name.contains(".sub."))
          EltOp = Builder.CreateFSub(Elt0, Elt1);
        else if (Name.contains(".mul."))
          EltOp = Builder.CreateFMul(Elt0, Elt1);
        else
          EltOp = Builder.CreateFDiv(Elt0, Elt1);
        Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
                                          ConstantInt::get(I32Ty, 0));
      } else if (IsX86 && Name.startswith("avx512.mask.pcmp")) {
        // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
        bool CmpEq = Name[16] == 'e';
        Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
      } else if (IsX86 && Name.startswith("avx512.mask.vpshufbitqmb.")) {
        Type *OpTy = CI->getArgOperand(0)->getType();
        unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
        Intrinsic::ID IID;
        switch (VecWidth) {
        default: llvm_unreachable("Unexpected intrinsic");
        case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128; break;
        case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256; break;
        case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512; break;
        }
  
        Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
                                 { CI->getOperand(0), CI->getArgOperand(1) });
        Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
      } else if (IsX86 && Name.startswith("avx512.mask.fpclass.p")) {
        Type *OpTy = CI->getArgOperand(0)->getType();
        unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
        unsigned EltWidth = OpTy->getScalarSizeInBits();
        Intrinsic::ID IID;
        if (VecWidth == 128 && EltWidth == 32)
          IID = Intrinsic::x86_avx512_fpclass_ps_128;
        else if (VecWidth == 256 && EltWidth == 32)
          IID = Intrinsic::x86_avx512_fpclass_ps_256;
        else if (VecWidth == 512 && EltWidth == 32)
          IID = Intrinsic::x86_avx512_fpclass_ps_512;
        else if (VecWidth == 128 && EltWidth == 64)
          IID = Intrinsic::x86_avx512_fpclass_pd_128;
        else if (VecWidth == 256 && EltWidth == 64)
          IID = Intrinsic::x86_avx512_fpclass_pd_256;
        else if (VecWidth == 512 && EltWidth == 64)
          IID = Intrinsic::x86_avx512_fpclass_pd_512;
        else
          llvm_unreachable("Unexpected intrinsic");
  
        Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
                                 { CI->getOperand(0), CI->getArgOperand(1) });
        Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
      } else if (IsX86 && Name.startswith("avx512.cmp.p")) {
        SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
                                     CI->arg_operands().end());
        Type *OpTy = Args[0]->getType();
        unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
        unsigned EltWidth = OpTy->getScalarSizeInBits();
        Intrinsic::ID IID;
        if (VecWidth == 128 && EltWidth == 32)
          IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
        else if (VecWidth == 256 && EltWidth == 32)
          IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
        else if (VecWidth == 512 && EltWidth == 32)
          IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
        else if (VecWidth == 128 && EltWidth == 64)
          IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
        else if (VecWidth == 256 && EltWidth == 64)
          IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
        else if (VecWidth == 512 && EltWidth == 64)
          IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
        else
          llvm_unreachable("Unexpected intrinsic");
  
        Value *Mask = Constant::getAllOnesValue(CI->getType());
        if (VecWidth == 512)
          std::swap(Mask, Args.back());
        Args.push_back(Mask);
  
        Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
                                 Args);
      } else if (IsX86 && Name.startswith("avx512.mask.cmp.")) {
        // Integer compare intrinsics.
        unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
        Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
      } else if (IsX86 && Name.startswith("avx512.mask.ucmp.")) {
        unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
        Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
      } else if (IsX86 && (Name.startswith("avx512.cvtb2mask.") ||
                           Name.startswith("avx512.cvtw2mask.") ||
                           Name.startswith("avx512.cvtd2mask.") ||
                           Name.startswith("avx512.cvtq2mask."))) {
        Value *Op = CI->getArgOperand(0);
        Value *Zero = llvm::Constant::getNullValue(Op->getType());
        Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
        Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, nullptr);
      } else if(IsX86 && (Name == "ssse3.pabs.b.128" ||
                          Name == "ssse3.pabs.w.128" ||
                          Name == "ssse3.pabs.d.128" ||
                          Name.startswith("avx2.pabs") ||
                          Name.startswith("avx512.mask.pabs"))) {
        Rep = upgradeAbs(Builder, *CI);
      } else if (IsX86 && (Name == "sse41.pmaxsb" ||
                           Name == "sse2.pmaxs.w" ||
                           Name == "sse41.pmaxsd" ||
                           Name.startswith("avx2.pmaxs") ||
                           Name.startswith("avx512.mask.pmaxs"))) {
        Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
      } else if (IsX86 && (Name == "sse2.pmaxu.b" ||
                           Name == "sse41.pmaxuw" ||
                           Name == "sse41.pmaxud" ||
                           Name.startswith("avx2.pmaxu") ||
                           Name.startswith("avx512.mask.pmaxu"))) {
        Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
      } else if (IsX86 && (Name == "sse41.pminsb" ||
                           Name == "sse2.pmins.w" ||
                           Name == "sse41.pminsd" ||
                           Name.startswith("avx2.pmins") ||
                           Name.startswith("avx512.mask.pmins"))) {
        Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
      } else if (IsX86 && (Name == "sse2.pminu.b" ||
                           Name == "sse41.pminuw" ||
                           Name == "sse41.pminud" ||
                           Name.startswith("avx2.pminu") ||
                           Name.startswith("avx512.mask.pminu"))) {
        Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
      } else if (IsX86 && (Name == "sse2.pmulu.dq" ||
                           Name == "avx2.pmulu.dq" ||
                           Name == "avx512.pmulu.dq.512" ||
                           Name.startswith("avx512.mask.pmulu.dq."))) {
        Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false);
      } else if (IsX86 && (Name == "sse41.pmuldq" ||
                           Name == "avx2.pmul.dq" ||
                           Name == "avx512.pmul.dq.512" ||
                           Name.startswith("avx512.mask.pmul.dq."))) {
        Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true);
      } else if (IsX86 && (Name == "sse.cvtsi2ss" ||
                           Name == "sse2.cvtsi2sd" ||
                           Name == "sse.cvtsi642ss" ||
                           Name == "sse2.cvtsi642sd")) {
        Rep = Builder.CreateSIToFP(
            CI->getArgOperand(1),
            cast<VectorType>(CI->getType())->getElementType());
        Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
      } else if (IsX86 && Name == "avx512.cvtusi2sd") {
        Rep = Builder.CreateUIToFP(
            CI->getArgOperand(1),
            cast<VectorType>(CI->getType())->getElementType());
        Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
      } else if (IsX86 && Name == "sse2.cvtss2sd") {
        Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
        Rep = Builder.CreateFPExt(
            Rep, cast<VectorType>(CI->getType())->getElementType());
        Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
      } else if (IsX86 && (Name == "sse2.cvtdq2pd" ||
                           Name == "sse2.cvtdq2ps" ||
                           Name == "avx.cvtdq2.pd.256" ||
                           Name == "avx.cvtdq2.ps.256" ||
                           Name.startswith("avx512.mask.cvtdq2pd.") ||
                           Name.startswith("avx512.mask.cvtudq2pd.") ||
                           Name.startswith("avx512.mask.cvtdq2ps.") ||
                           Name.startswith("avx512.mask.cvtudq2ps.") ||
                           Name.startswith("avx512.mask.cvtqq2pd.") ||
                           Name.startswith("avx512.mask.cvtuqq2pd.") ||
                           Name == "avx512.mask.cvtqq2ps.256" ||
                           Name == "avx512.mask.cvtqq2ps.512" ||
                           Name == "avx512.mask.cvtuqq2ps.256" ||
                           Name == "avx512.mask.cvtuqq2ps.512" ||
                           Name == "sse2.cvtps2pd" ||
                           Name == "avx.cvt.ps2.pd.256" ||
                           Name == "avx512.mask.cvtps2pd.128" ||
                           Name == "avx512.mask.cvtps2pd.256")) {
        auto *DstTy = cast<FixedVectorType>(CI->getType());
        Rep = CI->getArgOperand(0);
        auto *SrcTy = cast<FixedVectorType>(Rep->getType());
  
        unsigned NumDstElts = DstTy->getNumElements();
        if (NumDstElts < SrcTy->getNumElements()) {
          assert(NumDstElts == 2 && "Unexpected vector size");
          Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
        }
  
        bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
        bool IsUnsigned = (StringRef::npos != Name.find("cvtu"));
        if (IsPS2PD)
          Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
        else if (CI->getNumArgOperands() == 4 &&
                 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
                  cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
          Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
                                         : Intrinsic::x86_avx512_sitofp_round;
          Function *F = Intrinsic::getDeclaration(CI->getModule(), IID,
                                                  { DstTy, SrcTy });
          Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) });
        } else {
          Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
                           : Builder.CreateSIToFP(Rep, DstTy, "cvt");
        }
  
        if (CI->getNumArgOperands() >= 3)
          Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                              CI->getArgOperand(1));
      } else if (IsX86 && (Name.startswith("avx512.mask.vcvtph2ps.") ||
                           Name.startswith("vcvtph2ps."))) {
        auto *DstTy = cast<FixedVectorType>(CI->getType());
        Rep = CI->getArgOperand(0);
        auto *SrcTy = cast<FixedVectorType>(Rep->getType());
        unsigned NumDstElts = DstTy->getNumElements();
        if (NumDstElts != SrcTy->getNumElements()) {
          assert(NumDstElts == 4 && "Unexpected vector size");
          Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
        }
        Rep = Builder.CreateBitCast(
            Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
        Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
        if (CI->getNumArgOperands() >= 3)
          Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                              CI->getArgOperand(1));
      } else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) {
        Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
                                CI->getArgOperand(1), CI->getArgOperand(2),
                                /*Aligned*/false);
      } else if (IsX86 && (Name.startswith("avx512.mask.load."))) {
        Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
                                CI->getArgOperand(1),CI->getArgOperand(2),
                                /*Aligned*/true);
      } else if (IsX86 && Name.startswith("avx512.mask.expand.load.")) {
        auto *ResultTy = cast<FixedVectorType>(CI->getType());
        Type *PtrTy = ResultTy->getElementType();
  
        // Cast the pointer to element type.
        Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
                                           llvm::PointerType::getUnqual(PtrTy));
  
        Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
                                       ResultTy->getNumElements());
  
        Function *ELd = Intrinsic::getDeclaration(F->getParent(),
                                                  Intrinsic::masked_expandload,
                                                  ResultTy);
        Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) });
      } else if (IsX86 && Name.startswith("avx512.mask.compress.store.")) {
        auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
        Type *PtrTy = ResultTy->getElementType();
  
        // Cast the pointer to element type.
        Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
                                           llvm::PointerType::getUnqual(PtrTy));
  
        Value *MaskVec =
            getX86MaskVec(Builder, CI->getArgOperand(2),
                          cast<FixedVectorType>(ResultTy)->getNumElements());
  
        Function *CSt = Intrinsic::getDeclaration(F->getParent(),
                                                  Intrinsic::masked_compressstore,
                                                  ResultTy);
        Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec });
      } else if (IsX86 && (Name.startswith("avx512.mask.compress.") ||
                           Name.startswith("avx512.mask.expand."))) {
        auto *ResultTy = cast<FixedVectorType>(CI->getType());
  
        Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
                                       ResultTy->getNumElements());
  
        bool IsCompress = Name[12] == 'c';
        Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
                                       : Intrinsic::x86_avx512_mask_expand;
        Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy);
        Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1),
                                         MaskVec });
      } else if (IsX86 && Name.startswith("xop.vpcom")) {
        bool IsSigned;
        if (Name.endswith("ub") || Name.endswith("uw") || Name.endswith("ud") ||
            Name.endswith("uq"))
          IsSigned = false;
        else if (Name.endswith("b") || Name.endswith("w") || Name.endswith("d") ||
                 Name.endswith("q"))
          IsSigned = true;
        else
          llvm_unreachable("Unknown suffix");
  
        unsigned Imm;
        if (CI->getNumArgOperands() == 3) {
          Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
        } else {
          Name = Name.substr(9); // strip off "xop.vpcom"
          if (Name.startswith("lt"))
            Imm = 0;
          else if (Name.startswith("le"))
            Imm = 1;
          else if (Name.startswith("gt"))
            Imm = 2;
          else if (Name.startswith("ge"))
            Imm = 3;
          else if (Name.startswith("eq"))
            Imm = 4;
          else if (Name.startswith("ne"))
            Imm = 5;
          else if (Name.startswith("false"))
            Imm = 6;
          else if (Name.startswith("true"))
            Imm = 7;
          else
            llvm_unreachable("Unknown condition");
        }
  
        Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
      } else if (IsX86 && Name.startswith("xop.vpcmov")) {
        Value *Sel = CI->getArgOperand(2);
        Value *NotSel = Builder.CreateNot(Sel);
        Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
        Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
        Rep = Builder.CreateOr(Sel0, Sel1);
      } else if (IsX86 && (Name.startswith("xop.vprot") ||
                           Name.startswith("avx512.prol") ||
                           Name.startswith("avx512.mask.prol"))) {
        Rep = upgradeX86Rotate(Builder, *CI, false);
      } else if (IsX86 && (Name.startswith("avx512.pror") ||
                           Name.startswith("avx512.mask.pror"))) {
        Rep = upgradeX86Rotate(Builder, *CI, true);
      } else if (IsX86 && (Name.startswith("avx512.vpshld.") ||
                           Name.startswith("avx512.mask.vpshld") ||
                           Name.startswith("avx512.maskz.vpshld"))) {
        bool ZeroMask = Name[11] == 'z';
        Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
      } else if (IsX86 && (Name.startswith("avx512.vpshrd.") ||
                           Name.startswith("avx512.mask.vpshrd") ||
                           Name.startswith("avx512.maskz.vpshrd"))) {
        bool ZeroMask = Name[11] == 'z';
        Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
      } else if (IsX86 && Name == "sse42.crc32.64.8") {
        Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
                                                 Intrinsic::x86_sse42_crc32_32_8);
        Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
        Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});
        Rep = Builder.CreateZExt(Rep, CI->getType(), "");
      } else if (IsX86 && (Name.startswith("avx.vbroadcast.s") ||
                           Name.startswith("avx512.vbroadcast.s"))) {
        // Replace broadcasts with a series of insertelements.
        auto *VecTy = cast<FixedVectorType>(CI->getType());
        Type *EltTy = VecTy->getElementType();
        unsigned EltNum = VecTy->getNumElements();
        Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
                                            EltTy->getPointerTo());
        Value *Load = Builder.CreateLoad(EltTy, Cast);
        Type *I32Ty = Type::getInt32Ty(C);
        Rep = UndefValue::get(VecTy);
        for (unsigned I = 0; I < EltNum; ++I)
          Rep = Builder.CreateInsertElement(Rep, Load,
                                            ConstantInt::get(I32Ty, I));
      } else if (IsX86 && (Name.startswith("sse41.pmovsx") ||
                           Name.startswith("sse41.pmovzx") ||
                           Name.startswith("avx2.pmovsx") ||
                           Name.startswith("avx2.pmovzx") ||
                           Name.startswith("avx512.mask.pmovsx") ||
                           Name.startswith("avx512.mask.pmovzx"))) {
        auto *SrcTy = cast<FixedVectorType>(CI->getArgOperand(0)->getType());
        auto *DstTy = cast<FixedVectorType>(CI->getType());
        unsigned NumDstElts = DstTy->getNumElements();
  
        // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
        SmallVector<int, 8> ShuffleMask(NumDstElts);
        for (unsigned i = 0; i != NumDstElts; ++i)
          ShuffleMask[i] = i;
  
        Value *SV = Builder.CreateShuffleVector(
            CI->getArgOperand(0), UndefValue::get(SrcTy), ShuffleMask);
  
        bool DoSext = (StringRef::npos != Name.find("pmovsx"));
        Rep = DoSext ? Builder.CreateSExt(SV, DstTy)
                     : Builder.CreateZExt(SV, DstTy);
        // If there are 3 arguments, it's a masked intrinsic so we need a select.
        if (CI->getNumArgOperands() == 3)
          Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                              CI->getArgOperand(1));
      } else if (Name == "avx512.mask.pmov.qd.256" ||
                 Name == "avx512.mask.pmov.qd.512" ||
                 Name == "avx512.mask.pmov.wb.256" ||
                 Name == "avx512.mask.pmov.wb.512") {
        Type *Ty = CI->getArgOperand(1)->getType();
        Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
        Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                            CI->getArgOperand(1));
      } else if (IsX86 && (Name.startswith("avx.vbroadcastf128") ||
                           Name == "avx2.vbroadcasti128")) {
        // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
        Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
        unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
        auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
        Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),
                                              PointerType::getUnqual(VT));
        Value *Load = Builder.CreateAlignedLoad(VT, Op, Align(1));
        if (NumSrcElts == 2)
          Rep = Builder.CreateShuffleVector(
              Load, UndefValue::get(Load->getType()), ArrayRef<int>{0, 1, 0, 1});
        else
          Rep =
              Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
                                          ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
      } else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") ||
                           Name.startswith("avx512.mask.shuf.f"))) {
        unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
        Type *VT = CI->getType();
        unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
        unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
        unsigned ControlBitsMask = NumLanes - 1;
        unsigned NumControlBits = NumLanes / 2;
        SmallVector<int, 8> ShuffleMask(0);
  
        for (unsigned l = 0; l != NumLanes; ++l) {
          unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
          // We actually need the other source.
          if (l >= NumLanes / 2)
            LaneMask += NumLanes;
          for (unsigned i = 0; i != NumElementsInLane; ++i)
            ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
        }
        Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
                                          CI->getArgOperand(1), ShuffleMask);
        Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
                            CI->getArgOperand(3));
      }else if (IsX86 && (Name.startswith("avx512.mask.broadcastf") ||
                           Name.startswith("avx512.mask.broadcasti"))) {
        unsigned NumSrcElts =
            cast<FixedVectorType>(CI->getArgOperand(0)->getType())
                ->getNumElements();
        unsigned NumDstElts =
            cast<FixedVectorType>(CI->getType())->getNumElements();
  
        SmallVector<int, 8> ShuffleMask(NumDstElts);
        for (unsigned i = 0; i != NumDstElts; ++i)
          ShuffleMask[i] = i % NumSrcElts;
  
        Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
                                          CI->getArgOperand(0),
                                          ShuffleMask);
        Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                            CI->getArgOperand(1));
      } else if (IsX86 && (Name.startswith("avx2.pbroadcast") ||
                           Name.startswith("avx2.vbroadcast") ||
                           Name.startswith("avx512.pbroadcast") ||
                           Name.startswith("avx512.mask.broadcast.s"))) {
        // Replace vp?broadcasts with a vector shuffle.
        Value *Op = CI->getArgOperand(0);
        ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
        Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
        Rep = Builder.CreateShuffleVector(Op, UndefValue::get(Op->getType()),
                                          Constant::getNullValue(MaskTy));
  
        if (CI->getNumArgOperands() == 3)
          Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                              CI->getArgOperand(1));
      } else if (IsX86 && (Name.startswith("sse2.padds.") ||
                           Name.startswith("avx2.padds.") ||
                           Name.startswith("avx512.padds.") ||
                           Name.startswith("avx512.mask.padds."))) {
        Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
      } else if (IsX86 && (Name.startswith("sse2.psubs.") ||
                           Name.startswith("avx2.psubs.") ||
                           Name.startswith("avx512.psubs.") ||
                           Name.startswith("avx512.mask.psubs."))) {
        Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
      } else if (IsX86 && (Name.startswith("sse2.paddus.") ||
                           Name.startswith("avx2.paddus.") ||
                           Name.startswith("avx512.mask.paddus."))) {
        Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
      } else if (IsX86 && (Name.startswith("sse2.psubus.") ||
                           Name.startswith("avx2.psubus.") ||
                           Name.startswith("avx512.mask.psubus."))) {
        Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
      } else if (IsX86 && Name.startswith("avx512.mask.palignr.")) {
        Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
                                        CI->getArgOperand(1),
                                        CI->getArgOperand(2),
                                        CI->getArgOperand(3),
                                        CI->getArgOperand(4),
                                        false);
      } else if (IsX86 && Name.startswith("avx512.mask.valign.")) {
        Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
                                        CI->getArgOperand(1),
                                        CI->getArgOperand(2),
                                        CI->getArgOperand(3),
                                        CI->getArgOperand(4),
                                        true);
      } else if (IsX86 && (Name == "sse2.psll.dq" ||
                           Name == "avx2.psll.dq")) {
        // 128/256-bit shift left specified in bits.
        unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
        Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
                                         Shift / 8); // Shift is in bits.
      } else if (IsX86 && (Name == "sse2.psrl.dq" ||
                           Name == "avx2.psrl.dq")) {
        // 128/256-bit shift right specified in bits.
        unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
        Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
                                         Shift / 8); // Shift is in bits.
      } else if (IsX86 && (Name == "sse2.psll.dq.bs" ||
                           Name == "avx2.psll.dq.bs" ||
                           Name == "avx512.psll.dq.512")) {
        // 128/256/512-bit shift left specified in bytes.
        unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
        Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
      } else if (IsX86 && (Name == "sse2.psrl.dq.bs" ||
                           Name == "avx2.psrl.dq.bs" ||
                           Name == "avx512.psrl.dq.512")) {
        // 128/256/512-bit shift right specified in bytes.
        unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
        Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
      } else if (IsX86 && (Name == "sse41.pblendw" ||
                           Name.startswith("sse41.blendp") ||
                           Name.startswith("avx.blend.p") ||
                           Name == "avx2.pblendw" ||
                           Name.startswith("avx2.pblendd."))) {
        Value *Op0 = CI->getArgOperand(0);
        Value *Op1 = CI->getArgOperand(1);
        unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();
        auto *VecTy = cast<FixedVectorType>(CI->getType());
        unsigned NumElts = VecTy->getNumElements();
  
        SmallVector<int, 16> Idxs(NumElts);
        for (unsigned i = 0; i != NumElts; ++i)
          Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i;
  
        Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
      } else if (IsX86 && (Name.startswith("avx.vinsertf128.") ||
                           Name == "avx2.vinserti128" ||
                           Name.startswith("avx512.mask.insert"))) {
        Value *Op0 = CI->getArgOperand(0);
        Value *Op1 = CI->getArgOperand(1);
        unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
        unsigned DstNumElts =
            cast<FixedVectorType>(CI->getType())->getNumElements();
        unsigned SrcNumElts =
            cast<FixedVectorType>(Op1->getType())->getNumElements();
        unsigned Scale = DstNumElts / SrcNumElts;
  
        // Mask off the high bits of the immediate value; hardware ignores those.
        Imm = Imm % Scale;
  
        // Extend the second operand into a vector the size of the destination.
        Value *UndefV = UndefValue::get(Op1->getType());
        SmallVector<int, 8> Idxs(DstNumElts);
        for (unsigned i = 0; i != SrcNumElts; ++i)
          Idxs[i] = i;
        for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
          Idxs[i] = SrcNumElts;
        Rep = Builder.CreateShuffleVector(Op1, UndefV, Idxs);
  
        // Insert the second operand into the first operand.
  
        // Note that there is no guarantee that instruction lowering will actually
        // produce a vinsertf128 instruction for the created shuffles. In
        // particular, the 0 immediate case involves no lane changes, so it can
        // be handled as a blend.
  
        // Example of shuffle mask for 32-bit elements:
        // Imm = 1  <i32 0, i32 1, i32 2,  i32 3,  i32 8, i32 9, i32 10, i32 11>
        // Imm = 0  <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6,  i32 7 >
  
        // First fill with identify mask.
        for (unsigned i = 0; i != DstNumElts; ++i)
          Idxs[i] = i;
        // Then replace the elements where we need to insert.
        for (unsigned i = 0; i != SrcNumElts; ++i)
          Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
        Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
  
        // If the intrinsic has a mask operand, handle that.
        if (CI->getNumArgOperands() == 5)
          Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
                              CI->getArgOperand(3));
      } else if (IsX86 && (Name.startswith("avx.vextractf128.") ||
                           Name == "avx2.vextracti128" ||
                           Name.startswith("avx512.mask.vextract"))) {
        Value *Op0 = CI->getArgOperand(0);
        unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
        unsigned DstNumElts =
            cast<FixedVectorType>(CI->getType())->getNumElements();
        unsigned SrcNumElts =
            cast<FixedVectorType>(Op0->getType())->getNumElements();
        unsigned Scale = SrcNumElts / DstNumElts;
  
        // Mask off the high bits of the immediate value; hardware ignores those.
        Imm = Imm % Scale;
  
        // Get indexes for the subvector of the input vector.
        SmallVector<int, 8> Idxs(DstNumElts);
        for (unsigned i = 0; i != DstNumElts; ++i) {
          Idxs[i] = i + (Imm * DstNumElts);
        }
        Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
  
        // If the intrinsic has a mask operand, handle that.
        if (CI->getNumArgOperands() == 4)
          Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                              CI->getArgOperand(2));
      } else if (!IsX86 && Name == "stackprotectorcheck") {
        Rep = nullptr;
      } else if (IsX86 && (Name.startswith("avx512.mask.perm.df.") ||
                           Name.startswith("avx512.mask.perm.di."))) {
        Value *Op0 = CI->getArgOperand(0);
        unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
        auto *VecTy = cast<FixedVectorType>(CI->getType());
        unsigned NumElts = VecTy->getNumElements();
  
        SmallVector<int, 8> Idxs(NumElts);
        for (unsigned i = 0; i != NumElts; ++i)
          Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
  
        Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
  
        if (CI->getNumArgOperands() == 4)
          Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                              CI->getArgOperand(2));
      } else if (IsX86 && (Name.startswith("avx.vperm2f128.") ||
                           Name == "avx2.vperm2i128")) {
        // The immediate permute control byte looks like this:
        //    [1:0] - select 128 bits from sources for low half of destination
        //    [2]   - ignore
        //    [3]   - zero low half of destination
        //    [5:4] - select 128 bits from sources for high half of destination
        //    [6]   - ignore
        //    [7]   - zero high half of destination
  
        uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
  
        unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
        unsigned HalfSize = NumElts / 2;
        SmallVector<int, 8> ShuffleMask(NumElts);
  
        // Determine which operand(s) are actually in use for this instruction.
        Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
        Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
  
        // If needed, replace operands based on zero mask.
        V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
        V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
  
        // Permute low half of result.
        unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
        for (unsigned i = 0; i < HalfSize; ++i)
          ShuffleMask[i] = StartIndex + i;
  
        // Permute high half of result.
        StartIndex = (Imm & 0x10) ? HalfSize : 0;
        for (unsigned i = 0; i < HalfSize; ++i)
          ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
  
        Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
  
      } else if (IsX86 && (Name.startswith("avx.vpermil.") ||
                           Name == "sse2.pshuf.d" ||
                           Name.startswith("avx512.mask.vpermil.p") ||
                           Name.startswith("avx512.mask.pshuf.d."))) {
        Value *Op0 = CI->getArgOperand(0);
        unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
        auto *VecTy = cast<FixedVectorType>(CI->getType());
        unsigned NumElts = VecTy->getNumElements();
        // Calculate the size of each index in the immediate.
        unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
        unsigned IdxMask = ((1 << IdxSize) - 1);
  
        SmallVector<int, 8> Idxs(NumElts);
        // Lookup the bits for this element, wrapping around the immediate every
        // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
        // to offset by the first index of each group.
        for (unsigned i = 0; i != NumElts; ++i)
          Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
  
        Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
  
        if (CI->getNumArgOperands() == 4)
          Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                              CI->getArgOperand(2));
      } else if (IsX86 && (Name == "sse2.pshufl.w" ||
                           Name.startswith("avx512.mask.pshufl.w."))) {
        Value *Op0 = CI->getArgOperand(0);
        unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
        unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
  
        SmallVector<int, 16> Idxs(NumElts);
        for (unsigned l = 0; l != NumElts; l += 8) {
          for (unsigned i = 0; i != 4; ++i)
            Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
          for (unsigned i = 4; i != 8; ++i)
            Idxs[i + l] = i + l;
        }
  
        Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
  
        if (CI->getNumArgOperands() == 4)
          Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                              CI->getArgOperand(2));
      } else if (IsX86 && (Name == "sse2.pshufh.w" ||
                           Name.startswith("avx512.mask.pshufh.w."))) {
        Value *Op0 = CI->getArgOperand(0);
        unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
        unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
  
        SmallVector<int, 16> Idxs(NumElts);
        for (unsigned l = 0; l != NumElts; l += 8) {
          for (unsigned i = 0; i != 4; ++i)
            Idxs[i + l] = i + l;
          for (unsigned i = 0; i != 4; ++i)
            Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
        }
  
        Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
  
        if (CI->getNumArgOperands() == 4)
          Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                              CI->getArgOperand(2));
      } else if (IsX86 && Name.startswith("avx512.mask.shuf.p")) {
        Value *Op0 = CI->getArgOperand(0);
        Value *Op1 = CI->getArgOperand(1);
        unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
        unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
  
        unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
        unsigned HalfLaneElts = NumLaneElts / 2;
  
        SmallVector<int, 16> Idxs(NumElts);
        for (unsigned i = 0; i != NumElts; ++i) {
          // Base index is the starting element of the lane.
          Idxs[i] = i - (i % NumLaneElts);
          // If we are half way through the lane switch to the other source.
          if ((i % NumLaneElts) >= HalfLaneElts)
            Idxs[i] += NumElts;
          // Now select the specific element. By adding HalfLaneElts bits from
          // the immediate. Wrapping around the immediate every 8-bits.
          Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
        }
  
        Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
  
        Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
                            CI->getArgOperand(3));
      } else if (IsX86 && (Name.startswith("avx512.mask.movddup") ||
                           Name.startswith("avx512.mask.movshdup") ||
                           Name.startswith("avx512.mask.movsldup"))) {
        Value *Op0 = CI->getArgOperand(0);
        unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
        unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
  
        unsigned Offset = 0;
        if (Name.startswith("avx512.mask.movshdup."))
          Offset = 1;
  
        SmallVector<int, 16> Idxs(NumElts);
        for (unsigned l = 0; l != NumElts; l += NumLaneElts)
          for (unsigned i = 0; i != NumLaneElts; i += 2) {
            Idxs[i + l + 0] = i + l + Offset;
            Idxs[i + l + 1] = i + l + Offset;
          }
  
        Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
  
        Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                            CI->getArgOperand(1));
      } else if (IsX86 && (Name.startswith("avx512.mask.punpckl") ||
                           Name.startswith("avx512.mask.unpckl."))) {
        Value *Op0 = CI->getArgOperand(0);
        Value *Op1 = CI->getArgOperand(1);
        int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
        int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
  
        SmallVector<int, 64> Idxs(NumElts);
        for (int l = 0; l != NumElts; l += NumLaneElts)
          for (int i = 0; i != NumLaneElts; ++i)
            Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
  
        Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
  
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                            CI->getArgOperand(2));
      } else if (IsX86 && (Name.startswith("avx512.mask.punpckh") ||
                           Name.startswith("avx512.mask.unpckh."))) {
        Value *Op0 = CI->getArgOperand(0);
        Value *Op1 = CI->getArgOperand(1);
        int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
        int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
  
        SmallVector<int, 64> Idxs(NumElts);
        for (int l = 0; l != NumElts; l += NumLaneElts)
          for (int i = 0; i != NumLaneElts; ++i)
            Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
  
        Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
  
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                            CI->getArgOperand(2));
      } else if (IsX86 && (Name.startswith("avx512.mask.and.") ||
                           Name.startswith("avx512.mask.pand."))) {
        VectorType *FTy = cast<VectorType>(CI->getType());
        VectorType *ITy = VectorType::getInteger(FTy);
        Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
                                Builder.CreateBitCast(CI->getArgOperand(1), ITy));
        Rep = Builder.CreateBitCast(Rep, FTy);
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                            CI->getArgOperand(2));
      } else if (IsX86 && (Name.startswith("avx512.mask.andn.") ||
                           Name.startswith("avx512.mask.pandn."))) {
        VectorType *FTy = cast<VectorType>(CI->getType());
        VectorType *ITy = VectorType::getInteger(FTy);
        Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
        Rep = Builder.CreateAnd(Rep,
                                Builder.CreateBitCast(CI->getArgOperand(1), ITy));
        Rep = Builder.CreateBitCast(Rep, FTy);
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                            CI->getArgOperand(2));
      } else if (IsX86 && (Name.startswith("avx512.mask.or.") ||
                           Name.startswith("avx512.mask.por."))) {
        VectorType *FTy = cast<VectorType>(CI->getType());
        VectorType *ITy = VectorType::getInteger(FTy);
        Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
                               Builder.CreateBitCast(CI->getArgOperand(1), ITy));
        Rep = Builder.CreateBitCast(Rep, FTy);
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                            CI->getArgOperand(2));
      } else if (IsX86 && (Name.startswith("avx512.mask.xor.") ||
                           Name.startswith("avx512.mask.pxor."))) {
        VectorType *FTy = cast<VectorType>(CI->getType());
        VectorType *ITy = VectorType::getInteger(FTy);
        Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
                                Builder.CreateBitCast(CI->getArgOperand(1), ITy));
        Rep = Builder.CreateBitCast(Rep, FTy);
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                            CI->getArgOperand(2));
      } else if (IsX86 && Name.startswith("avx512.mask.padd.")) {
        Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                            CI->getArgOperand(2));
      } else if (IsX86 && Name.startswith("avx512.mask.psub.")) {
        Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                            CI->getArgOperand(2));
      } else if (IsX86 && Name.startswith("avx512.mask.pmull.")) {
        Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                            CI->getArgOperand(2));
      } else if (IsX86 && Name.startswith("avx512.mask.add.p")) {
        if (Name.endswith(".512")) {
          Intrinsic::ID IID;
          if (Name[17] == 's')
            IID = Intrinsic::x86_avx512_add_ps_512;
          else
            IID = Intrinsic::x86_avx512_add_pd_512;
  
          Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
                                   { CI->getArgOperand(0), CI->getArgOperand(1),
                                     CI->getArgOperand(4) });
        } else {
          Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
        }
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                            CI->getArgOperand(2));
      } else if (IsX86 && Name.startswith("avx512.mask.div.p")) {
        if (Name.endswith(".512")) {
          Intrinsic::ID IID;
          if (Name[17] == 's')
            IID = Intrinsic::x86_avx512_div_ps_512;
          else
            IID = Intrinsic::x86_avx512_div_pd_512;
  
          Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
                                   { CI->getArgOperand(0), CI->getArgOperand(1),
                                     CI->getArgOperand(4) });
        } else {
          Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
        }
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                            CI->getArgOperand(2));
      } else if (IsX86 && Name.startswith("avx512.mask.mul.p")) {
        if (Name.endswith(".512")) {
          Intrinsic::ID IID;
          if (Name[17] == 's')
            IID = Intrinsic::x86_avx512_mul_ps_512;
          else
            IID = Intrinsic::x86_avx512_mul_pd_512;
  
          Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
                                   { CI->getArgOperand(0), CI->getArgOperand(1),
                                     CI->getArgOperand(4) });
        } else {
          Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
        }
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                            CI->getArgOperand(2));
      } else if (IsX86 && Name.startswith("avx512.mask.sub.p")) {
        if (Name.endswith(".512")) {
          Intrinsic::ID IID;
          if (Name[17] == 's')
            IID = Intrinsic::x86_avx512_sub_ps_512;
          else
            IID = Intrinsic::x86_avx512_sub_pd_512;
  
          Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
                                   { CI->getArgOperand(0), CI->getArgOperand(1),
                                     CI->getArgOperand(4) });
        } else {
          Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
        }
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                            CI->getArgOperand(2));
      } else if (IsX86 && (Name.startswith("avx512.mask.max.p") ||
                           Name.startswith("avx512.mask.min.p")) &&
                 Name.drop_front(18) == ".512") {
        bool IsDouble = Name[17] == 'd';
        bool IsMin = Name[13] == 'i';
        static const Intrinsic::ID MinMaxTbl[2][2] = {
          { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 },
          { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 }
        };
        Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
  
        Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
                                 { CI->getArgOperand(0), CI->getArgOperand(1),
                                   CI->getArgOperand(4) });
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                            CI->getArgOperand(2));
      } else if (IsX86 && Name.startswith("avx512.mask.lzcnt.")) {
        Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
                                                           Intrinsic::ctlz,
                                                           CI->getType()),
                                 { CI->getArgOperand(0), Builder.getInt1(false) });
        Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                            CI->getArgOperand(1));
      } else if (IsX86 && Name.startswith("avx512.mask.psll")) {
        bool IsImmediate = Name[16] == 'i' ||
                           (Name.size() > 18 && Name[18] == 'i');
        bool IsVariable = Name[16] == 'v';
        char Size = Name[16] == '.' ? Name[17] :
                    Name[17] == '.' ? Name[18] :
                    Name[18] == '.' ? Name[19] :
                                      Name[20];
  
        Intrinsic::ID IID;
        if (IsVariable && Name[17] != '.') {
          if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
            IID = Intrinsic::x86_avx2_psllv_q;
          else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
            IID = Intrinsic::x86_avx2_psllv_q_256;
          else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
            IID = Intrinsic::x86_avx2_psllv_d;
          else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
            IID = Intrinsic::x86_avx2_psllv_d_256;
          else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
            IID = Intrinsic::x86_avx512_psllv_w_128;
          else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
            IID = Intrinsic::x86_avx512_psllv_w_256;
          else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
            IID = Intrinsic::x86_avx512_psllv_w_512;
          else
            llvm_unreachable("Unexpected size");
        } else if (Name.endswith(".128")) {
          if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
            IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
                              : Intrinsic::x86_sse2_psll_d;
          else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
            IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
                              : Intrinsic::x86_sse2_psll_q;
          else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
            IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
                              : Intrinsic::x86_sse2_psll_w;
          else
            llvm_unreachable("Unexpected size");
        } else if (Name.endswith(".256")) {
          if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
            IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
                              : Intrinsic::x86_avx2_psll_d;
          else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
            IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
                              : Intrinsic::x86_avx2_psll_q;
          else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
            IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
                              : Intrinsic::x86_avx2_psll_w;
          else
            llvm_unreachable("Unexpected size");
        } else {
          if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
            IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :
                  IsVariable  ? Intrinsic::x86_avx512_psllv_d_512 :
                                Intrinsic::x86_avx512_psll_d_512;
          else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
            IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :
                  IsVariable  ? Intrinsic::x86_avx512_psllv_q_512 :
                                Intrinsic::x86_avx512_psll_q_512;
          else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
            IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
                              : Intrinsic::x86_avx512_psll_w_512;
          else
            llvm_unreachable("Unexpected size");
        }
  
        Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
      } else if (IsX86 && Name.startswith("avx512.mask.psrl")) {
        bool IsImmediate = Name[16] == 'i' ||
                           (Name.size() > 18 && Name[18] == 'i');
        bool IsVariable = Name[16] == 'v';
        char Size = Name[16] == '.' ? Name[17] :
                    Name[17] == '.' ? Name[18] :
                    Name[18] == '.' ? Name[19] :
                                      Name[20];
  
        Intrinsic::ID IID;
        if (IsVariable && Name[17] != '.') {
          if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
            IID = Intrinsic::x86_avx2_psrlv_q;
          else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
            IID = Intrinsic::x86_avx2_psrlv_q_256;
          else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
            IID = Intrinsic::x86_avx2_psrlv_d;
          else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
            IID = Intrinsic::x86_avx2_psrlv_d_256;
          else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
            IID = Intrinsic::x86_avx512_psrlv_w_128;
          else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
            IID = Intrinsic::x86_avx512_psrlv_w_256;
          else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
            IID = Intrinsic::x86_avx512_psrlv_w_512;
          else
            llvm_unreachable("Unexpected size");
        } else if (Name.endswith(".128")) {
          if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
            IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
                              : Intrinsic::x86_sse2_psrl_d;
          else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
            IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
                              : Intrinsic::x86_sse2_psrl_q;
          else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
            IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
                              : Intrinsic::x86_sse2_psrl_w;
          else
            llvm_unreachable("Unexpected size");
        } else if (Name.endswith(".256")) {
          if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
            IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
                              : Intrinsic::x86_avx2_psrl_d;
          else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
            IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
                              : Intrinsic::x86_avx2_psrl_q;
          else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
            IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
                              : Intrinsic::x86_avx2_psrl_w;
          else
            llvm_unreachable("Unexpected size");
        } else {
          if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
            IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :
                  IsVariable  ? Intrinsic::x86_avx512_psrlv_d_512 :
                                Intrinsic::x86_avx512_psrl_d_512;
          else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
            IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :
                  IsVariable  ? Intrinsic::x86_avx512_psrlv_q_512 :
                                Intrinsic::x86_avx512_psrl_q_512;
          else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
            IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
                              : Intrinsic::x86_avx512_psrl_w_512;
          else
            llvm_unreachable("Unexpected size");
        }
  
        Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
      } else if (IsX86 && Name.startswith("avx512.mask.psra")) {
        bool IsImmediate = Name[16] == 'i' ||
                           (Name.size() > 18 && Name[18] == 'i');
        bool IsVariable = Name[16] == 'v';
        char Size = Name[16] == '.' ? Name[17] :
                    Name[17] == '.' ? Name[18] :
                    Name[18] == '.' ? Name[19] :
                                      Name[20];
  
        Intrinsic::ID IID;
        if (IsVariable && Name[17] != '.') {
          if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
            IID = Intrinsic::x86_avx2_psrav_d;
          else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
            IID = Intrinsic::x86_avx2_psrav_d_256;
          else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
            IID = Intrinsic::x86_avx512_psrav_w_128;
          else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
            IID = Intrinsic::x86_avx512_psrav_w_256;
          else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
            IID = Intrinsic::x86_avx512_psrav_w_512;
          else
            llvm_unreachable("Unexpected size");
        } else if (Name.endswith(".128")) {
          if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
            IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
                              : Intrinsic::x86_sse2_psra_d;
          else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
            IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :
                  IsVariable  ? Intrinsic::x86_avx512_psrav_q_128 :
                                Intrinsic::x86_avx512_psra_q_128;
          else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
            IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
                              : Intrinsic::x86_sse2_psra_w;
          else
            llvm_unreachable("Unexpected size");
        } else if (Name.endswith(".256")) {
          if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
            IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
                              : Intrinsic::x86_avx2_psra_d;
          else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
            IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :
                  IsVariable  ? Intrinsic::x86_avx512_psrav_q_256 :
                                Intrinsic::x86_avx512_psra_q_256;
          else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
            IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
                              : Intrinsic::x86_avx2_psra_w;
          else
            llvm_unreachable("Unexpected size");
        } else {
          if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
            IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :
                  IsVariable  ? Intrinsic::x86_avx512_psrav_d_512 :
                                Intrinsic::x86_avx512_psra_d_512;
          else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
            IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :
                  IsVariable  ? Intrinsic::x86_avx512_psrav_q_512 :
                                Intrinsic::x86_avx512_psra_q_512;
          else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
            IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
                              : Intrinsic::x86_avx512_psra_w_512;
          else
            llvm_unreachable("Unexpected size");
        }
  
        Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
      } else if (IsX86 && Name.startswith("avx512.mask.move.s")) {
        Rep = upgradeMaskedMove(Builder, *CI);
      } else if (IsX86 && Name.startswith("avx512.cvtmask2")) {
        Rep = UpgradeMaskToInt(Builder, *CI);
      } else if (IsX86 && Name.endswith(".movntdqa")) {
        Module *M = F->getParent();
        MDNode *Node = MDNode::get(
            C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
  
        Value *Ptr = CI->getArgOperand(0);
  
        // Convert the type of the pointer to a pointer to the stored type.
        Value *BC = Builder.CreateBitCast(
            Ptr, PointerType::getUnqual(CI->getType()), "cast");
        LoadInst *LI = Builder.CreateAlignedLoad(
            CI->getType(), BC,
            Align(CI->getType()->getPrimitiveSizeInBits().getFixedSize() / 8));
        LI->setMetadata(M->getMDKindID("nontemporal"), Node);
        Rep = LI;
      } else if (IsX86 && (Name.startswith("fma.vfmadd.") ||
                           Name.startswith("fma.vfmsub.") ||
                           Name.startswith("fma.vfnmadd.") ||
                           Name.startswith("fma.vfnmsub."))) {
        bool NegMul = Name[6] == 'n';
        bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
        bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
  
        Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
                         CI->getArgOperand(2) };
  
        if (IsScalar) {
          Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
          Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
          Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
        }
  
        if (NegMul && !IsScalar)
          Ops[0] = Builder.CreateFNeg(Ops[0]);
        if (NegMul && IsScalar)
          Ops[1] = Builder.CreateFNeg(Ops[1]);
        if (NegAcc)
          Ops[2] = Builder.CreateFNeg(Ops[2]);
  
        Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
                                                           Intrinsic::fma,
                                                           Ops[0]->getType()),
                                 Ops);
  
        if (IsScalar)
          Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep,
                                            (uint64_t)0);
      } else if (IsX86 && Name.startswith("fma4.vfmadd.s")) {
        Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
                         CI->getArgOperand(2) };
  
        Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
        Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
        Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
  
        Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
                                                           Intrinsic::fma,
                                                           Ops[0]->getType()),
                                 Ops);
  
        Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
                                          Rep, (uint64_t)0);
      } else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.s") ||
                           Name.startswith("avx512.maskz.vfmadd.s") ||
                           Name.startswith("avx512.mask3.vfmadd.s") ||
                           Name.startswith("avx512.mask3.vfmsub.s") ||
                           Name.startswith("avx512.mask3.vfnmsub.s"))) {
        bool IsMask3 = Name[11] == '3';
        bool IsMaskZ = Name[11] == 'z';
        // Drop the "avx512.mask." to make it easier.
        Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
        bool NegMul = Name[2] == 'n';
        bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
  
        Value *A = CI->getArgOperand(0);
        Value *B = CI->getArgOperand(1);
        Value *C = CI->getArgOperand(2);
  
        if (NegMul && (IsMask3 || IsMaskZ))
          A = Builder.CreateFNeg(A);
        if (NegMul && !(IsMask3 || IsMaskZ))
          B = Builder.CreateFNeg(B);
        if (NegAcc)
          C = Builder.CreateFNeg(C);
  
        A = Builder.CreateExtractElement(A, (uint64_t)0);
        B = Builder.CreateExtractElement(B, (uint64_t)0);
        C = Builder.CreateExtractElement(C, (uint64_t)0);
  
        if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
            cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
          Value *Ops[] = { A, B, C, CI->getArgOperand(4) };
  
          Intrinsic::ID IID;
          if (Name.back() == 'd')
            IID = Intrinsic::x86_avx512_vfmadd_f64;
          else
            IID = Intrinsic::x86_avx512_vfmadd_f32;
          Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID);
          Rep = Builder.CreateCall(FMA, Ops);
        } else {
          Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
                                                    Intrinsic::fma,
                                                    A->getType());
          Rep = Builder.CreateCall(FMA, { A, B, C });
        }
  
        Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType()) :
                          IsMask3 ? C : A;
  
        // For Mask3 with NegAcc, we need to create a new extractelement that
        // avoids the negation above.
        if (NegAcc && IsMask3)
          PassThru = Builder.CreateExtractElement(CI->getArgOperand(2),
                                                  (uint64_t)0);
  
        Rep = EmitX86ScalarSelect(Builder, CI->getArgOperand(3),
                                  Rep, PassThru);
        Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0),
                                          Rep, (uint64_t)0);
      } else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.p") ||
                           Name.startswith("avx512.mask.vfnmadd.p") ||
                           Name.startswith("avx512.mask.vfnmsub.p") ||
                           Name.startswith("avx512.mask3.vfmadd.p") ||
                           Name.startswith("avx512.mask3.vfmsub.p") ||
                           Name.startswith("avx512.mask3.vfnmsub.p") ||
                           Name.startswith("avx512.maskz.vfmadd.p"))) {
        bool IsMask3 = Name[11] == '3';
        bool IsMaskZ = Name[11] == 'z';
        // Drop the "avx512.mask." to make it easier.
        Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
        bool NegMul = Name[2] == 'n';
        bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
  
        Value *A = CI->getArgOperand(0);
        Value *B = CI->getArgOperand(1);
        Value *C = CI->getArgOperand(2);
  
        if (NegMul && (IsMask3 || IsMaskZ))
          A = Builder.CreateFNeg(A);
        if (NegMul && !(IsMask3 || IsMaskZ))
          B = Builder.CreateFNeg(B);
        if (NegAcc)
          C = Builder.CreateFNeg(C);
  
        if (CI->getNumArgOperands() == 5 &&
            (!isa<ConstantInt>(CI->getArgOperand(4)) ||
             cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
          Intrinsic::ID IID;
          // Check the character before ".512" in string.
          if (Name[Name.size()-5] == 's')
            IID = Intrinsic::x86_avx512_vfmadd_ps_512;
          else
            IID = Intrinsic::x86_avx512_vfmadd_pd_512;
  
          Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
                                   { A, B, C, CI->getArgOperand(4) });
        } else {
          Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
                                                    Intrinsic::fma,
                                                    A->getType());
          Rep = Builder.CreateCall(FMA, { A, B, C });
        }
  
        Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
                          IsMask3 ? CI->getArgOperand(2) :
                                    CI->getArgOperand(0);
  
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
      } else if (IsX86 &&  Name.startswith("fma.vfmsubadd.p")) {
        unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
        unsigned EltWidth = CI->getType()->getScalarSizeInBits();
        Intrinsic::ID IID;
        if (VecWidth == 128 && EltWidth == 32)
          IID = Intrinsic::x86_fma_vfmaddsub_ps;
        else if (VecWidth == 256 && EltWidth == 32)
          IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
        else if (VecWidth == 128 && EltWidth == 64)
          IID = Intrinsic::x86_fma_vfmaddsub_pd;
        else if (VecWidth == 256 && EltWidth == 64)
          IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
        else
          llvm_unreachable("Unexpected intrinsic");
  
        Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
                         CI->getArgOperand(2) };
        Ops[2] = Builder.CreateFNeg(Ops[2]);
        Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
                                 Ops);
      } else if (IsX86 && (Name.startswith("avx512.mask.vfmaddsub.p") ||
                           Name.startswith("avx512.mask3.vfmaddsub.p") ||
                           Name.startswith("avx512.maskz.vfmaddsub.p") ||
                           Name.startswith("avx512.mask3.vfmsubadd.p"))) {
        bool IsMask3 = Name[11] == '3';
        bool IsMaskZ = Name[11] == 'z';
        // Drop the "avx512.mask." to make it easier.
        Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
        bool IsSubAdd = Name[3] == 's';
        if (CI->getNumArgOperands() == 5) {
          Intrinsic::ID IID;
          // Check the character before ".512" in string.
          if (Name[Name.size()-5] == 's')
            IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
          else
            IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
  
          Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
                           CI->getArgOperand(2), CI->getArgOperand(4) };
          if (IsSubAdd)
            Ops[2] = Builder.CreateFNeg(Ops[2]);
  
          Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
                                   Ops);
        } else {
          int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
  
          Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
                           CI->getArgOperand(2) };
  
          Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
                                                    Ops[0]->getType());
          Value *Odd = Builder.CreateCall(FMA, Ops);
          Ops[2] = Builder.CreateFNeg(Ops[2]);
          Value *Even = Builder.CreateCall(FMA, Ops);
  
          if (IsSubAdd)
            std::swap(Even, Odd);
  
          SmallVector<int, 32> Idxs(NumElts);
          for (int i = 0; i != NumElts; ++i)
            Idxs[i] = i + (i % 2) * NumElts;
  
          Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
        }
  
        Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
                          IsMask3 ? CI->getArgOperand(2) :
                                    CI->getArgOperand(0);
  
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
      } else if (IsX86 && (Name.startswith("avx512.mask.pternlog.") ||
                           Name.startswith("avx512.maskz.pternlog."))) {
        bool ZeroMask = Name[11] == 'z';
        unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
        unsigned EltWidth = CI->getType()->getScalarSizeInBits();
        Intrinsic::ID IID;
        if (VecWidth == 128 && EltWidth == 32)
          IID = Intrinsic::x86_avx512_pternlog_d_128;
        else if (VecWidth == 256 && EltWidth == 32)
          IID = Intrinsic::x86_avx512_pternlog_d_256;
        else if (VecWidth == 512 && EltWidth == 32)
          IID = Intrinsic::x86_avx512_pternlog_d_512;
        else if (VecWidth == 128 && EltWidth == 64)
          IID = Intrinsic::x86_avx512_pternlog_q_128;
        else if (VecWidth == 256 && EltWidth == 64)
          IID = Intrinsic::x86_avx512_pternlog_q_256;
        else if (VecWidth == 512 && EltWidth == 64)
          IID = Intrinsic::x86_avx512_pternlog_q_512;
        else
          llvm_unreachable("Unexpected intrinsic");
  
        Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
                          CI->getArgOperand(2), CI->getArgOperand(3) };
        Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
                                 Args);
        Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
                                   : CI->getArgOperand(0);
        Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
      } else if (IsX86 && (Name.startswith("avx512.mask.vpmadd52") ||
                           Name.startswith("avx512.maskz.vpmadd52"))) {
        bool ZeroMask = Name[11] == 'z';
        bool High = Name[20] == 'h' || Name[21] == 'h';
        unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
        Intrinsic::ID IID;
        if (VecWidth == 128 && !High)
          IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
        else if (VecWidth == 256 && !High)
          IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
        else if (VecWidth == 512 && !High)
          IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
        else if (VecWidth == 128 && High)
          IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
        else if (VecWidth == 256 && High)
          IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
        else if (VecWidth == 512 && High)
          IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
        else
          llvm_unreachable("Unexpected intrinsic");
  
        Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
                          CI->getArgOperand(2) };
        Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
                                 Args);
        Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
                                   : CI->getArgOperand(0);
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
      } else if (IsX86 && (Name.startswith("avx512.mask.vpermi2var.") ||
                           Name.startswith("avx512.mask.vpermt2var.") ||
                           Name.startswith("avx512.maskz.vpermt2var."))) {
        bool ZeroMask = Name[11] == 'z';
        bool IndexForm = Name[17] == 'i';
        Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
      } else if (IsX86 && (Name.startswith("avx512.mask.vpdpbusd.") ||
                           Name.startswith("avx512.maskz.vpdpbusd.") ||
                           Name.startswith("avx512.mask.vpdpbusds.") ||
                           Name.startswith("avx512.maskz.vpdpbusds."))) {
        bool ZeroMask = Name[11] == 'z';
        bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
        unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
        Intrinsic::ID IID;
        if (VecWidth == 128 && !IsSaturating)
          IID = Intrinsic::x86_avx512_vpdpbusd_128;
        else if (VecWidth == 256 && !IsSaturating)
          IID = Intrinsic::x86_avx512_vpdpbusd_256;
        else if (VecWidth == 512 && !IsSaturating)
          IID = Intrinsic::x86_avx512_vpdpbusd_512;
        else if (VecWidth == 128 && IsSaturating)
          IID = Intrinsic::x86_avx512_vpdpbusds_128;
        else if (VecWidth == 256 && IsSaturating)
          IID = Intrinsic::x86_avx512_vpdpbusds_256;
        else if (VecWidth == 512 && IsSaturating)
          IID = Intrinsic::x86_avx512_vpdpbusds_512;
        else
          llvm_unreachable("Unexpected intrinsic");
  
        Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
                          CI->getArgOperand(2)  };
        Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
                                 Args);
        Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
                                   : CI->getArgOperand(0);
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
      } else if (IsX86 && (Name.startswith("avx512.mask.vpdpwssd.") ||
                           Name.startswith("avx512.maskz.vpdpwssd.") ||
                           Name.startswith("avx512.mask.vpdpwssds.") ||
                           Name.startswith("avx512.maskz.vpdpwssds."))) {
        bool ZeroMask = Name[11] == 'z';
        bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
        unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
        Intrinsic::ID IID;
        if (VecWidth == 128 && !IsSaturating)
          IID = Intrinsic::x86_avx512_vpdpwssd_128;
        else if (VecWidth == 256 && !IsSaturating)
          IID = Intrinsic::x86_avx512_vpdpwssd_256;
        else if (VecWidth == 512 && !IsSaturating)
          IID = Intrinsic::x86_avx512_vpdpwssd_512;
        else if (VecWidth == 128 && IsSaturating)
          IID = Intrinsic::x86_avx512_vpdpwssds_128;
        else if (VecWidth == 256 && IsSaturating)
          IID = Intrinsic::x86_avx512_vpdpwssds_256;
        else if (VecWidth == 512 && IsSaturating)
          IID = Intrinsic::x86_avx512_vpdpwssds_512;
        else
          llvm_unreachable("Unexpected intrinsic");
  
        Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
                          CI->getArgOperand(2)  };
        Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
                                 Args);
        Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
                                   : CI->getArgOperand(0);
        Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
      } else if (IsX86 && (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
                           Name == "addcarry.u32" || Name == "addcarry.u64" ||
                           Name == "subborrow.u32" || Name == "subborrow.u64")) {
        Intrinsic::ID IID;
        if (Name[0] == 'a' && Name.back() == '2')
          IID = Intrinsic::x86_addcarry_32;
        else if (Name[0] == 'a' && Name.back() == '4')
          IID = Intrinsic::x86_addcarry_64;
        else if (Name[0] == 's' && Name.back() == '2')
          IID = Intrinsic::x86_subborrow_32;
        else if (Name[0] == 's' && Name.back() == '4')
          IID = Intrinsic::x86_subborrow_64;
        else
          llvm_unreachable("Unexpected intrinsic");
  
        // Make a call with 3 operands.
        Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
                          CI->getArgOperand(2)};
        Value *NewCall = Builder.CreateCall(
                                  Intrinsic::getDeclaration(CI->getModule(), IID),
                                  Args);
  
        // Extract the second result and store it.
        Value *Data = Builder.CreateExtractValue(NewCall, 1);
        // Cast the pointer to the right type.
        Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3),
                                   llvm::PointerType::getUnqual(Data->getType()));
        Builder.CreateAlignedStore(Data, Ptr, Align(1));
        // Replace the original call result with the first result of the new call.
        Value *CF = Builder.CreateExtractValue(NewCall, 0);
  
        CI->replaceAllUsesWith(CF);
        Rep = nullptr;
      } else if (IsX86 && Name.startswith("avx512.mask.") &&
                 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
        // Rep will be updated by the call in the condition.
      } else if (IsNVVM && (Name == "abs.i" || Name == "abs.ll")) {
        Value *Arg = CI->getArgOperand(0);
        Value *Neg = Builder.CreateNeg(Arg, "neg");
        Value *Cmp = Builder.CreateICmpSGE(
            Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond");
        Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");
      } else if (IsNVVM && (Name.startswith("atomic.load.add.f32.p") ||
                            Name.startswith("atomic.load.add.f64.p"))) {
        Value *Ptr = CI->getArgOperand(0);
        Value *Val = CI->getArgOperand(1);
        Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val,
                                      AtomicOrdering::SequentiallyConsistent);
      } else if (IsNVVM && (Name == "max.i" || Name == "max.ll" ||
                            Name == "max.ui" || Name == "max.ull")) {
        Value *Arg0 = CI->getArgOperand(0);
        Value *Arg1 = CI->getArgOperand(1);
        Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")
                         ? Builder.CreateICmpUGE(Arg0, Arg1, "max.cond")
                         : Builder.CreateICmpSGE(Arg0, Arg1, "max.cond");
        Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "max");
      } else if (IsNVVM && (Name == "min.i" || Name == "min.ll" ||
                            Name == "min.ui" || Name == "min.ull")) {
        Value *Arg0 = CI->getArgOperand(0);
        Value *Arg1 = CI->getArgOperand(1);
        Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")
                         ? Builder.CreateICmpULE(Arg0, Arg1, "min.cond")
                         : Builder.CreateICmpSLE(Arg0, Arg1, "min.cond");
        Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "min");
      } else if (IsNVVM && Name == "clz.ll") {
        // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 and returns an i64.
        Value *Arg = CI->getArgOperand(0);
        Value *Ctlz = Builder.CreateCall(
            Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
                                      {Arg->getType()}),
            {Arg, Builder.getFalse()}, "ctlz");
        Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
      } else if (IsNVVM && Name == "popc.ll") {
        // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 and returns an
        // i64.
        Value *Arg = CI->getArgOperand(0);
        Value *Popc = Builder.CreateCall(
            Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
                                      {Arg->getType()}),
            Arg, "ctpop");
        Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
      } else if (IsNVVM && Name == "h2f") {
        Rep = Builder.CreateCall(Intrinsic::getDeclaration(
                                     F->getParent(), Intrinsic::convert_from_fp16,
                                     {Builder.getFloatTy()}),
                                 CI->getArgOperand(0), "h2f");
      } else {
        llvm_unreachable("Unknown function for CallInst upgrade.");
      }
  
      if (Rep)
        CI->replaceAllUsesWith(Rep);
      CI->eraseFromParent();
      return;
    }
  
    const auto &DefaultCase = [&NewFn, &CI]() -> void {
      // Handle generic mangling change, but nothing else
      assert(
          (CI->getCalledFunction()->getName() != NewFn->getName()) &&
          "Unknown function for CallInst upgrade and isn't just a name change");
      CI->setCalledFunction(NewFn);
    };
    CallInst *NewCall = nullptr;
    switch (NewFn->getIntrinsicID()) {
    default: {
      DefaultCase();
      return;
    }
    case Intrinsic::experimental_vector_reduce_v2_fmul: {
      SmallVector<Value *, 2> Args;
      if (CI->isFast())
        Args.push_back(ConstantFP::get(CI->getOperand(0)->getType(), 1.0));
      else
        Args.push_back(CI->getOperand(0));
      Args.push_back(CI->getOperand(1));
      NewCall = Builder.CreateCall(NewFn, Args);
      cast<Instruction>(NewCall)->copyFastMathFlags(CI);
      break;
    }
    case Intrinsic::experimental_vector_reduce_v2_fadd: {
      SmallVector<Value *, 2> Args;
      if (CI->isFast())
        Args.push_back(Constant::getNullValue(CI->getOperand(0)->getType()));
      else
        Args.push_back(CI->getOperand(0));
      Args.push_back(CI->getOperand(1));
      NewCall = Builder.CreateCall(NewFn, Args);
      cast<Instruction>(NewCall)->copyFastMathFlags(CI);
      break;
    }
    case Intrinsic::arm_neon_vld1:
    case Intrinsic::arm_neon_vld2:
    case Intrinsic::arm_neon_vld3:
    case Intrinsic::arm_neon_vld4:
    case Intrinsic::arm_neon_vld2lane:
    case Intrinsic::arm_neon_vld3lane:
    case Intrinsic::arm_neon_vld4lane:
    case Intrinsic::arm_neon_vst1:
    case Intrinsic::arm_neon_vst2:
    case Intrinsic::arm_neon_vst3:
    case Intrinsic::arm_neon_vst4:
    case Intrinsic::arm_neon_vst2lane:
    case Intrinsic::arm_neon_vst3lane:
    case Intrinsic::arm_neon_vst4lane: {
      SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
                                   CI->arg_operands().end());
      NewCall = Builder.CreateCall(NewFn, Args);
      break;
    }
  
    case Intrinsic::arm_neon_bfdot:
    case Intrinsic::arm_neon_bfmmla:
    case Intrinsic::arm_neon_bfmlalb:
    case Intrinsic::arm_neon_bfmlalt:
    case Intrinsic::aarch64_neon_bfdot:
    case Intrinsic::aarch64_neon_bfmmla:
    case Intrinsic::aarch64_neon_bfmlalb:
    case Intrinsic::aarch64_neon_bfmlalt: {
      SmallVector<Value *, 3> Args;
      assert(CI->getNumArgOperands() == 3 &&
             "Mismatch between function args and call args");
      size_t OperandWidth =
          CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
      assert((OperandWidth == 64 || OperandWidth == 128) &&
             "Unexpected operand width");
      Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
      auto Iter = CI->arg_operands().begin();
      Args.push_back(*Iter++);
      Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
      Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
      NewCall = Builder.CreateCall(NewFn, Args);
      break;
    }
  
    case Intrinsic::bitreverse:
      NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
      break;
  
    case Intrinsic::ctlz:
    case Intrinsic::cttz:
      assert(CI->getNumArgOperands() == 1 &&
             "Mismatch between function args and call args");
      NewCall =
          Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
      break;
  
    case Intrinsic::objectsize: {
      Value *NullIsUnknownSize = CI->getNumArgOperands() == 2
                                     ? Builder.getFalse()
                                     : CI->getArgOperand(2);
      Value *Dynamic =
          CI->getNumArgOperands() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
      NewCall = Builder.CreateCall(
          NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
      break;
    }
  
    case Intrinsic::ctpop:
      NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
      break;
  
    case Intrinsic::convert_from_fp16:
      NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
      break;
  
    case Intrinsic::dbg_value:
      // Upgrade from the old version that had an extra offset argument.
      assert(CI->getNumArgOperands() == 4);
      // Drop nonzero offsets instead of attempting to upgrade them.
      if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
        if (Offset->isZeroValue()) {
          NewCall = Builder.CreateCall(
              NewFn,
              {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
          break;
        }
      CI->eraseFromParent();
      return;
  
    case Intrinsic::x86_xop_vfrcz_ss:
    case Intrinsic::x86_xop_vfrcz_sd:
      NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
      break;
  
    case Intrinsic::x86_xop_vpermil2pd:
    case Intrinsic::x86_xop_vpermil2ps:
    case Intrinsic::x86_xop_vpermil2pd_256:
    case Intrinsic::x86_xop_vpermil2ps_256: {
      SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
                                   CI->arg_operands().end());
      VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
      VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
      Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
      NewCall = Builder.CreateCall(NewFn, Args);
      break;
    }
  
    case Intrinsic::x86_sse41_ptestc:
    case Intrinsic::x86_sse41_ptestz:
    case Intrinsic::x86_sse41_ptestnzc: {
      // The arguments for these intrinsics used to be v4f32, and changed
      // to v2i64. This is purely a nop, since those are bitwise intrinsics.
      // So, the only thing required is a bitcast for both arguments.
      // First, check the arguments have the old type.
      Value *Arg0 = CI->getArgOperand(0);
      if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
        return;
  
      // Old intrinsic, add bitcasts
      Value *Arg1 = CI->getArgOperand(1);
  
      auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
  
      Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
      Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
  
      NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
      break;
    }
  
    case Intrinsic::x86_rdtscp: {
      // This used to take 1 arguments. If we have no arguments, it is already
      // upgraded.
      if (CI->getNumOperands() == 0)
        return;
  
      NewCall = Builder.CreateCall(NewFn);
      // Extract the second result and store it.
      Value *Data = Builder.CreateExtractValue(NewCall, 1);
      // Cast the pointer to the right type.
      Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0),
                                   llvm::PointerType::getUnqual(Data->getType()));
      Builder.CreateAlignedStore(Data, Ptr, Align(1));
      // Replace the original call result with the first result of the new call.
      Value *TSC = Builder.CreateExtractValue(NewCall, 0);
  
      NewCall->takeName(CI);
      CI->replaceAllUsesWith(TSC);
      CI->eraseFromParent();
      return;
    }
  
    case Intrinsic::x86_sse41_insertps:
    case Intrinsic::x86_sse41_dppd:
    case Intrinsic::x86_sse41_dpps:
    case Intrinsic::x86_sse41_mpsadbw:
    case Intrinsic::x86_avx_dp_ps_256:
    case Intrinsic::x86_avx2_mpsadbw: {
      // Need to truncate the last argument from i32 to i8 -- this argument models
      // an inherently 8-bit immediate operand to these x86 instructions.
      SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
                                   CI->arg_operands().end());
  
      // Replace the last argument with a trunc.
      Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
      NewCall = Builder.CreateCall(NewFn, Args);
      break;
    }
  
    case Intrinsic::x86_avx512_mask_cmp_pd_128:
    case Intrinsic::x86_avx512_mask_cmp_pd_256:
    case Intrinsic::x86_avx512_mask_cmp_pd_512:
    case Intrinsic::x86_avx512_mask_cmp_ps_128:
    case Intrinsic::x86_avx512_mask_cmp_ps_256:
    case Intrinsic::x86_avx512_mask_cmp_ps_512: {
      SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
                                   CI->arg_operands().end());
      unsigned NumElts =
          cast<FixedVectorType>(Args[0]->getType())->getNumElements();
      Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
  
      NewCall = Builder.CreateCall(NewFn, Args);
      Value *Res = ApplyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
  
      NewCall->takeName(CI);
      CI->replaceAllUsesWith(Res);
      CI->eraseFromParent();
      return;
    }
  
    case Intrinsic::thread_pointer: {
      NewCall = Builder.CreateCall(NewFn, {});
      break;
    }
  
    case Intrinsic::invariant_start:
    case Intrinsic::invariant_end:
    case Intrinsic::masked_load:
    case Intrinsic::masked_store:
    case Intrinsic::masked_gather:
    case Intrinsic::masked_scatter: {
      SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
                                   CI->arg_operands().end());
      NewCall = Builder.CreateCall(NewFn, Args);
      break;
    }
  
    case Intrinsic::memcpy:
    case Intrinsic::memmove:
    case Intrinsic::memset: {
      // We have to make sure that the call signature is what we're expecting.
      // We only want to change the old signatures by removing the alignment arg:
      //  @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
      //    -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
      //  @llvm.memset...(i8*, i8, i[32|64], i32, i1)
      //    -> @llvm.memset...(i8*, i8, i[32|64], i1)
      // Note: i8*'s in the above can be any pointer type
      if (CI->getNumArgOperands() != 5) {
        DefaultCase();
        return;
      }
      // Remove alignment argument (3), and add alignment attributes to the
      // dest/src pointers.
      Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
                        CI->getArgOperand(2), CI->getArgOperand(4)};
      NewCall = Builder.CreateCall(NewFn, Args);
      auto *MemCI = cast<MemIntrinsic>(NewCall);
      // All mem intrinsics support dest alignment.
      const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
      MemCI->setDestAlignment(Align->getMaybeAlignValue());
      // Memcpy/Memmove also support source alignment.
      if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
        MTI->setSourceAlignment(Align->getMaybeAlignValue());
      break;
    }
    }
    assert(NewCall && "Should have either set this variable or returned through "
                      "the default case");
    NewCall->takeName(CI);
    CI->replaceAllUsesWith(NewCall);
    CI->eraseFromParent();
  }
  
  void llvm::UpgradeCallsToIntrinsic(Function *F) {
    assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
  
    // Check if this function should be upgraded and get the replacement function
    // if there is one.
    Function *NewFn;
    if (UpgradeIntrinsicFunction(F, NewFn)) {
      // Replace all users of the old function with the new function or new
      // instructions. This is not a range loop because the call is deleted.
      for (auto UI = F->user_begin(), UE = F->user_end(); UI != UE; )
        if (CallInst *CI = dyn_cast<CallInst>(*UI++))
          UpgradeIntrinsicCall(CI, NewFn);
  
      // Remove old function, no longer used, from the module.
      F->eraseFromParent();
    }
  }
  
  MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
    // Check if the tag uses struct-path aware TBAA format.
    if (isa<MDNode>(MD.getOperand(0)) && MD.getNumOperands() >= 3)
      return &MD;
  
    auto &Context = MD.getContext();
    if (MD.getNumOperands() == 3) {
      Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
      MDNode *ScalarType = MDNode::get(Context, Elts);
      // Create a MDNode <ScalarType, ScalarType, offset 0, const>
      Metadata *Elts2[] = {ScalarType, ScalarType,
                           ConstantAsMetadata::get(
                               Constant::getNullValue(Type::getInt64Ty(Context))),
                           MD.getOperand(2)};
      return MDNode::get(Context, Elts2);
    }
    // Create a MDNode <MD, MD, offset 0>
    Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
                                      Type::getInt64Ty(Context)))};
    return MDNode::get(Context, Elts);
  }
  
  Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
                                        Instruction *&Temp) {
    if (Opc != Instruction::BitCast)
      return nullptr;
  
    Temp = nullptr;
    Type *SrcTy = V->getType();
    if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
        SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
      LLVMContext &Context = V->getContext();
  
      // We have no information about target data layout, so we assume that
      // the maximum pointer size is 64bit.
      Type *MidTy = Type::getInt64Ty(Context);
      Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
  
      return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
    }
  
    return nullptr;
  }
  
  Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
    if (Opc != Instruction::BitCast)
      return nullptr;
  
    Type *SrcTy = C->getType();
    if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
        SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
      LLVMContext &Context = C->getContext();
  
      // We have no information about target data layout, so we assume that
      // the maximum pointer size is 64bit.
      Type *MidTy = Type::getInt64Ty(Context);
  
      return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
                                       DestTy);
    }
  
    return nullptr;
  }
  
  /// Check the debug info version number, if it is out-dated, drop the debug
  /// info. Return true if module is modified.
  bool llvm::UpgradeDebugInfo(Module &M) {
    unsigned Version = getDebugMetadataVersionFromModule(M);
    if (Version == DEBUG_METADATA_VERSION) {
      bool BrokenDebugInfo = false;
      if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
        report_fatal_error("Broken module found, compilation aborted!");
      if (!BrokenDebugInfo)
        // Everything is ok.
        return false;
      else {
        // Diagnose malformed debug info.
        DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
        M.getContext().diagnose(Diag);
      }
    }
    bool Modified = StripDebugInfo(M);
    if (Modified && Version != DEBUG_METADATA_VERSION) {
      // Diagnose a version mismatch.
      DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
      M.getContext().diagnose(DiagVersion);
    }
    return Modified;
  }
  
  /// This checks for objc retain release marker which should be upgraded. It
  /// returns true if module is modified.
  static bool UpgradeRetainReleaseMarker(Module &M) {
    bool Changed = false;
    const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
    NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
    if (ModRetainReleaseMarker) {
      MDNode *Op = ModRetainReleaseMarker->getOperand(0);
      if (Op) {
        MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
        if (ID) {
          SmallVector<StringRef, 4> ValueComp;
          ID->getString().split(ValueComp, "#");
          if (ValueComp.size() == 2) {
            std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
            ID = MDString::get(M.getContext(), NewValue);
          }
          M.addModuleFlag(Module::Error, MarkerKey, ID);
          M.eraseNamedMetadata(ModRetainReleaseMarker);
          Changed = true;
        }
      }
    }
    return Changed;
  }
  
  void llvm::UpgradeARCRuntime(Module &M) {
    // This lambda converts normal function calls to ARC runtime functions to
    // intrinsic calls.
    auto UpgradeToIntrinsic = [&](const char *OldFunc,
                                  llvm::Intrinsic::ID IntrinsicFunc) {
      Function *Fn = M.getFunction(OldFunc);
  
      if (!Fn)
        return;
  
      Function *NewFn = llvm::Intrinsic::getDeclaration(&M, IntrinsicFunc);
  
      for (auto I = Fn->user_begin(), E = Fn->user_end(); I != E;) {
        CallInst *CI = dyn_cast<CallInst>(*I++);
        if (!CI || CI->getCalledFunction() != Fn)
          continue;
  
        IRBuilder<> Builder(CI->getParent(), CI->getIterator());
        FunctionType *NewFuncTy = NewFn->getFunctionType();
        SmallVector<Value *, 2> Args;
  
        // Don't upgrade the intrinsic if it's not valid to bitcast the return
        // value to the return type of the old function.
        if (NewFuncTy->getReturnType() != CI->getType() &&
            !CastInst::castIsValid(Instruction::BitCast, CI,
                                   NewFuncTy->getReturnType()))
          continue;
  
        bool InvalidCast = false;
  
        for (unsigned I = 0, E = CI->getNumArgOperands(); I != E; ++I) {
          Value *Arg = CI->getArgOperand(I);
  
          // Bitcast argument to the parameter type of the new function if it's
          // not a variadic argument.
          if (I < NewFuncTy->getNumParams()) {
            // Don't upgrade the intrinsic if it's not valid to bitcast the argument
            // to the parameter type of the new function.
            if (!CastInst::castIsValid(Instruction::BitCast, Arg,
                                       NewFuncTy->getParamType(I))) {
              InvalidCast = true;
              break;
            }
            Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
          }
          Args.push_back(Arg);
        }
  
        if (InvalidCast)
          continue;
  
        // Create a call instruction that calls the new function.
        CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
        NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
        NewCall->takeName(CI);
  
        // Bitcast the return value back to the type of the old call.
        Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
  
        if (!CI->use_empty())
          CI->replaceAllUsesWith(NewRetVal);
        CI->eraseFromParent();
      }
  
      if (Fn->use_empty())
        Fn->eraseFromParent();
    };
  
    // Unconditionally convert a call to "clang.arc.use" to a call to
    // "llvm.objc.clang.arc.use".
    UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
  
    // Upgrade the retain release marker. If there is no need to upgrade
    // the marker, that means either the module is already new enough to contain
    // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
    if (!UpgradeRetainReleaseMarker(M))
      return;
  
    std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
        {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
        {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
        {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
        {"objc_autoreleaseReturnValue",
         llvm::Intrinsic::objc_autoreleaseReturnValue},
        {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
        {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
        {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
        {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
        {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
        {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
        {"objc_release", llvm::Intrinsic::objc_release},
        {"objc_retain", llvm::Intrinsic::objc_retain},
        {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
        {"objc_retainAutoreleaseReturnValue",
         llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
        {"objc_retainAutoreleasedReturnValue",
         llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
        {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
        {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
        {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
        {"objc_unsafeClaimAutoreleasedReturnValue",
         llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
        {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
        {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
        {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
        {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
        {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
        {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
        {"objc_arc_annotation_topdown_bbstart",
         llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
        {"objc_arc_annotation_topdown_bbend",
         llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
        {"objc_arc_annotation_bottomup_bbstart",
         llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
        {"objc_arc_annotation_bottomup_bbend",
         llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
  
    for (auto &I : RuntimeFuncs)
      UpgradeToIntrinsic(I.first, I.second);
  }
  
  bool llvm::UpgradeModuleFlags(Module &M) {
    NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
    if (!ModFlags)
      return false;
  
    bool HasObjCFlag = false, HasClassProperties = false, Changed = false;
    bool HasSwiftVersionFlag = false;
    uint8_t SwiftMajorVersion, SwiftMinorVersion;
    uint32_t SwiftABIVersion;
    auto Int8Ty = Type::getInt8Ty(M.getContext());
    auto Int32Ty = Type::getInt32Ty(M.getContext());
  
    for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
      MDNode *Op = ModFlags->getOperand(I);
      if (Op->getNumOperands() != 3)
        continue;
      MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
      if (!ID)
        continue;
      if (ID->getString() == "Objective-C Image Info Version")
        HasObjCFlag = true;
      if (ID->getString() == "Objective-C Class Properties")
        HasClassProperties = true;
      // Upgrade PIC/PIE Module Flags. The module flag behavior for these two
      // field was Error and now they are Max.
      if (ID->getString() == "PIC Level" || ID->getString() == "PIE Level") {
        if (auto *Behavior =
                mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
          if (Behavior->getLimitedValue() == Module::Error) {
            Type *Int32Ty = Type::getInt32Ty(M.getContext());
            Metadata *Ops[3] = {
                ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Max)),
                MDString::get(M.getContext(), ID->getString()),
                Op->getOperand(2)};
            ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
            Changed = true;
          }
        }
      }
      // Upgrade Objective-C Image Info Section. Removed the whitespce in the
      // section name so that llvm-lto will not complain about mismatching
      // module flags that is functionally the same.
      if (ID->getString() == "Objective-C Image Info Section") {
        if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
          SmallVector<StringRef, 4> ValueComp;
          Value->getString().split(ValueComp, " ");
          if (ValueComp.size() != 1) {
            std::string NewValue;
            for (auto &S : ValueComp)
              NewValue += S.str();
            Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
                                MDString::get(M.getContext(), NewValue)};
            ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
            Changed = true;
          }
        }
      }
  
      // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
      // If the higher bits are set, it adds new module flag for swift info.
      if (ID->getString() == "Objective-C Garbage Collection") {
        auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
        if (Md) {
          assert(Md->getValue() && "Expected non-empty metadata");
          auto Type = Md->getValue()->getType();
          if (Type == Int8Ty)
            continue;
          unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
          if ((Val & 0xff) != Val) {
            HasSwiftVersionFlag = true;
            SwiftABIVersion = (Val & 0xff00) >> 8;
            SwiftMajorVersion = (Val & 0xff000000) >> 24;
            SwiftMinorVersion = (Val & 0xff0000) >> 16;
          }
          Metadata *Ops[3] = {
            ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
            Op->getOperand(1),
            ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
          ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
          Changed = true;
        }
      }
    }
  
    // "Objective-C Class Properties" is recently added for Objective-C. We
    // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
    // flag of value 0, so we can correclty downgrade this flag when trying to
    // link an ObjC bitcode without this module flag with an ObjC bitcode with
    // this module flag.
    if (HasObjCFlag && !HasClassProperties) {
      M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
                      (uint32_t)0);
      Changed = true;
    }
  
    if (HasSwiftVersionFlag) {
      M.addModuleFlag(Module::Error, "Swift ABI Version",
                      SwiftABIVersion);
      M.addModuleFlag(Module::Error, "Swift Major Version",
                      ConstantInt::get(Int8Ty, SwiftMajorVersion));
      M.addModuleFlag(Module::Error, "Swift Minor Version",
                      ConstantInt::get(Int8Ty, SwiftMinorVersion));
      Changed = true;
    }
  
    return Changed;
  }
  
  void llvm::UpgradeSectionAttributes(Module &M) {
    auto TrimSpaces = [](StringRef Section) -> std::string {
      SmallVector<StringRef, 5> Components;
      Section.split(Components, ',');
  
      SmallString<32> Buffer;
      raw_svector_ostream OS(Buffer);
  
      for (auto Component : Components)
        OS << ',' << Component.trim();
  
      return std::string(OS.str().substr(1));
    };
  
    for (auto &GV : M.globals()) {
      if (!GV.hasSection())
        continue;
  
      StringRef Section = GV.getSection();
  
      if (!Section.startswith("__DATA, __objc_catlist"))
        continue;
  
      // __DATA, __objc_catlist, regular, no_dead_strip
      // __DATA,__objc_catlist,regular,no_dead_strip
      GV.setSection(TrimSpaces(Section));
    }
  }
  
  namespace {
  // Prior to LLVM 10.0, the strictfp attribute could be used on individual
  // callsites within a function that did not also have the strictfp attribute.
  // Since 10.0, if strict FP semantics are needed within a function, the
  // function must have the strictfp attribute and all calls within the function
  // must also have the strictfp attribute. This latter restriction is
  // necessary to prevent unwanted libcall simplification when a function is
  // being cloned (such as for inlining).
  //
  // The "dangling" strictfp attribute usage was only used to prevent constant
  // folding and other libcall simplification. The nobuiltin attribute on the
  // callsite has the same effect.
  struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
    StrictFPUpgradeVisitor() {}
  
    void visitCallBase(CallBase &Call) {
      if (!Call.isStrictFP())
        return;
      if (isa<ConstrainedFPIntrinsic>(&Call))
        return;
      // If we get here, the caller doesn't have the strictfp attribute
      // but this callsite does. Replace the strictfp attribute with nobuiltin.
      Call.removeAttribute(AttributeList::FunctionIndex, Attribute::StrictFP);
      Call.addAttribute(AttributeList::FunctionIndex, Attribute::NoBuiltin);
    }
  };
  } // namespace
  
  void llvm::UpgradeFunctionAttributes(Function &F) {
    // If a function definition doesn't have the strictfp attribute,
    // convert any callsite strictfp attributes to nobuiltin.
    if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
      StrictFPUpgradeVisitor SFPV;
      SFPV.visit(F);
    }
  }
  
  static bool isOldLoopArgument(Metadata *MD) {
    auto *T = dyn_cast_or_null<MDTuple>(MD);
    if (!T)
      return false;
    if (T->getNumOperands() < 1)
      return false;
    auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
    if (!S)
      return false;
    return S->getString().startswith("llvm.vectorizer.");
  }
  
  static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
    StringRef OldPrefix = "llvm.vectorizer.";
    assert(OldTag.startswith(OldPrefix) && "Expected old prefix");
  
    if (OldTag == "llvm.vectorizer.unroll")
      return MDString::get(C, "llvm.loop.interleave.count");
  
    return MDString::get(
        C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
               .str());
  }
  
  static Metadata *upgradeLoopArgument(Metadata *MD) {
    auto *T = dyn_cast_or_null<MDTuple>(MD);
    if (!T)
      return MD;
    if (T->getNumOperands() < 1)
      return MD;
    auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
    if (!OldTag)
      return MD;
    if (!OldTag->getString().startswith("llvm.vectorizer."))
      return MD;
  
    // This has an old tag.  Upgrade it.
    SmallVector<Metadata *, 8> Ops;
    Ops.reserve(T->getNumOperands());
    Ops.push_back(upgradeLoopTag(T->getContext(), OldTag->getString()));
    for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
      Ops.push_back(T->getOperand(I));
  
    return MDTuple::get(T->getContext(), Ops);
  }
  
  MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
    auto *T = dyn_cast<MDTuple>(&N);
    if (!T)
      return &N;
  
    if (none_of(T->operands(), isOldLoopArgument))
      return &N;
  
    SmallVector<Metadata *, 8> Ops;
    Ops.reserve(T->getNumOperands());
    for (Metadata *MD : T->operands())
      Ops.push_back(upgradeLoopArgument(MD));
  
    return MDTuple::get(T->getContext(), Ops);
  }
  
  std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
    StringRef AddrSpaces = "-p270:32:32-p271:32:32-p272:64:64";
  
    // If X86, and the datalayout matches the expected format, add pointer size
    // address spaces to the datalayout.
    if (!Triple(TT).isX86() || DL.contains(AddrSpaces))
      return std::string(DL);
  
    SmallVector<StringRef, 4> Groups;
    Regex R("(e-m:[a-z](-p:32:32)?)(-[if]64:.*$)");
    if (!R.match(DL, &Groups))
      return std::string(DL);
  
    return (Groups[1] + AddrSpaces + Groups[3]).str();
  }
  
  void llvm::UpgradeAttributes(AttrBuilder &B) {
    StringRef FramePointer;
    if (B.contains("no-frame-pointer-elim")) {
      // The value can be "true" or "false".
      for (const auto &I : B.td_attrs())
        if (I.first == "no-frame-pointer-elim")
          FramePointer = I.second == "true" ? "all" : "none";
      B.removeAttribute("no-frame-pointer-elim");
    }
    if (B.contains("no-frame-pointer-elim-non-leaf")) {
      // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
      if (FramePointer != "all")
        FramePointer = "non-leaf";
      B.removeAttribute("no-frame-pointer-elim-non-leaf");
    }
    if (!FramePointer.empty())
      B.addAttribute("frame-pointer", FramePointer);
  
    if (B.contains("null-pointer-is-valid")) {
      // The value can be "true" or "false".
      bool NullPointerIsValid = false;
      for (const auto &I : B.td_attrs())
        if (I.first == "null-pointer-is-valid")
          NullPointerIsValid = I.second == "true";
      B.removeAttribute("null-pointer-is-valid");
      if (NullPointerIsValid)
        B.addAttribute(Attribute::NullPointerIsValid);
    }
  }
  