text: //===--- CGDecl.cpp - Emit LLVM Code for declarations ---------------------===//
  //
  // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
  // See https://llvm.org/LICENSE.txt for license information.
  // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  //
  //===----------------------------------------------------------------------===//
  //
  // This contains code to emit Decl nodes as LLVM code.
  //
  //===----------------------------------------------------------------------===//
  
  #include "CGBlocks.h"
  #include "CGCXXABI.h"
  #include "CGCleanup.h"
  #include "CGDebugInfo.h"
  #include "CGOpenCLRuntime.h"
  #include "CGOpenMPRuntime.h"
  #include "CGSYCLRuntime.h"
  #include "CodeGenFunction.h"
  #include "CodeGenModule.h"
  #include "ConstantEmitter.h"
  #include "PatternInit.h"
  #include "TargetInfo.h"
  #include "clang/AST/ASTContext.h"
  #include "clang/AST/Attr.h"
  #include "clang/AST/CharUnits.h"
  #include "clang/AST/Decl.h"
  #include "clang/AST/DeclObjC.h"
  #include "clang/AST/DeclOpenMP.h"
  #include "clang/Basic/CodeGenOptions.h"
  #include "clang/Basic/SourceManager.h"
  #include "clang/Basic/TargetInfo.h"
  #include "clang/CodeGen/CGFunctionInfo.h"
  #include "clang/Sema/Sema.h"
  #include "llvm/Analysis/ValueTracking.h"
  #include "llvm/IR/DataLayout.h"
  #include "llvm/IR/GlobalVariable.h"
  #include "llvm/IR/Intrinsics.h"
  #include "llvm/IR/Type.h"
  
  using namespace clang;
  using namespace CodeGen;
  
  static_assert(clang::Sema::MaximumAlignment <= llvm::Value::MaximumAlignment,
                "Clang max alignment greater than what LLVM supports?");
  
  void CodeGenFunction::EmitDecl(const Decl &D) {
    switch (D.getKind()) {
    case Decl::BuiltinTemplate:
    case Decl::TranslationUnit:
    case Decl::ExternCContext:
    case Decl::Namespace:
    case Decl::UnresolvedUsingTypename:
    case Decl::ClassTemplateSpecialization:
    case Decl::ClassTemplatePartialSpecialization:
    case Decl::VarTemplateSpecialization:
    case Decl::VarTemplatePartialSpecialization:
    case Decl::TemplateTypeParm:
    case Decl::UnresolvedUsingValue:
    case Decl::NonTypeTemplateParm:
    case Decl::CXXDeductionGuide:
    case Decl::CXXMethod:
    case Decl::CXXConstructor:
    case Decl::CXXDestructor:
    case Decl::CXXConversion:
    case Decl::Field:
    case Decl::MSProperty:
    case Decl::IndirectField:
    case Decl::ObjCIvar:
    case Decl::ObjCAtDefsField:
    case Decl::ParmVar:
    case Decl::ImplicitParam:
    case Decl::ClassTemplate:
    case Decl::VarTemplate:
    case Decl::FunctionTemplate:
    case Decl::TypeAliasTemplate:
    case Decl::TemplateTemplateParm:
    case Decl::ObjCMethod:
    case Decl::ObjCCategory:
    case Decl::ObjCProtocol:
    case Decl::ObjCInterface:
    case Decl::ObjCCategoryImpl:
    case Decl::ObjCImplementation:
    case Decl::ObjCProperty:
    case Decl::ObjCCompatibleAlias:
    case Decl::PragmaComment:
    case Decl::PragmaDetectMismatch:
    case Decl::AccessSpec:
    case Decl::LinkageSpec:
    case Decl::Export:
    case Decl::ObjCPropertyImpl:
    case Decl::FileScopeAsm:
    case Decl::Friend:
    case Decl::FriendTemplate:
    case Decl::Block:
    case Decl::Captured:
    case Decl::ClassScopeFunctionSpecialization:
    case Decl::UsingShadow:
    case Decl::ConstructorUsingShadow:
    case Decl::ObjCTypeParam:
    case Decl::Binding:
      llvm_unreachable("Declaration should not be in declstmts!");
    case Decl::Function:  // void X();
    case Decl::Record:    // struct/union/class X;
    case Decl::Enum:      // enum X;
    case Decl::EnumConstant: // enum ? { X = ? }
    case Decl::CXXRecord: // struct/union/class X; [C++]
    case Decl::StaticAssert: // static_assert(X, ""); [C++0x]
    case Decl::Label:        // __label__ x;
    case Decl::Import:
    case Decl::MSGuid:    // __declspec(uuid("..."))
    case Decl::OMPThreadPrivate:
    case Decl::OMPAllocate:
    case Decl::OMPCapturedExpr:
    case Decl::OMPRequires:
    case Decl::Empty:
    case Decl::Concept:
    case Decl::LifetimeExtendedTemporary:
    case Decl::RequiresExprBody:
      // None of these decls require codegen support.
      return;
  
    case Decl::NamespaceAlias:
      if (CGDebugInfo *DI = getDebugInfo())
          DI->EmitNamespaceAlias(cast<NamespaceAliasDecl>(D));
      return;
    case Decl::Using:          // using X; [C++]
      if (CGDebugInfo *DI = getDebugInfo())
          DI->EmitUsingDecl(cast<UsingDecl>(D));
      return;
    case Decl::UsingPack:
      for (auto *Using : cast<UsingPackDecl>(D).expansions())
        EmitDecl(*Using);
      return;
    case Decl::UsingDirective: // using namespace X; [C++]
      if (CGDebugInfo *DI = getDebugInfo())
        DI->EmitUsingDirective(cast<UsingDirectiveDecl>(D));
      return;
    case Decl::Var:
    case Decl::Decomposition: {
      const VarDecl &VD = cast<VarDecl>(D);
      assert(VD.isLocalVarDecl() &&
             "Should not see file-scope variables inside a function!");
      EmitVarDecl(VD);
      if (auto *DD = dyn_cast<DecompositionDecl>(&VD))
        for (auto *B : DD->bindings())
          if (auto *HD = B->getHoldingVar())
            EmitVarDecl(*HD);
      return;
    }
  
    case Decl::OMPDeclareReduction:
      return CGM.EmitOMPDeclareReduction(cast<OMPDeclareReductionDecl>(&D), this);
  
    case Decl::OMPDeclareMapper:
      return CGM.EmitOMPDeclareMapper(cast<OMPDeclareMapperDecl>(&D), this);
  
    case Decl::Typedef:      // typedef int X;
    case Decl::TypeAlias: {  // using X = int; [C++0x]
      const TypedefNameDecl &TD = cast<TypedefNameDecl>(D);
      QualType Ty = TD.getUnderlyingType();
  
      if (Ty->isVariablyModifiedType())
        EmitVariablyModifiedType(Ty);
  
      return;
    }
    }
  }
  
  /// EmitVarDecl - This method handles emission of any variable declaration
  /// inside a function, including static vars etc.
  void CodeGenFunction::EmitVarDecl(const VarDecl &D) {
    if (D.hasExternalStorage())
      // Don't emit it now, allow it to be emitted lazily on its first use.
      return;
  
    // Some function-scope variable does not have static storage but still
    // needs to be emitted like a static variable, e.g. a function-scope
    // variable in constant address space in OpenCL.
    if (D.getStorageDuration() != SD_Automatic) {
      // Static sampler variables translated to function calls.
      if (D.getType()->isSamplerT())
        return;
  
      llvm::GlobalValue::LinkageTypes Linkage =
          CGM.getLLVMLinkageVarDefinition(&D, /*IsConstant=*/false);
  
      // FIXME: We need to force the emission/use of a guard variable for
      // some variables even if we can constant-evaluate them because
      // we can't guarantee every translation unit will constant-evaluate them.
  
      return EmitStaticVarDecl(D, Linkage);
    }
  
    if (D.getType().getAddressSpace() == LangAS::opencl_local)
      return CGM.getOpenCLRuntime().EmitWorkGroupLocalVarDecl(*this, D);
  
    if (D.getAttr<SYCLScopeAttr>() && D.getAttr<SYCLScopeAttr>()->isWorkGroup())
      return CGM.getSYCLRuntime().emitWorkGroupLocalVarDecl(*this, D);
  
    assert(D.hasLocalStorage());
    return EmitAutoVarDecl(D);
  }
  
  static std::string getStaticDeclName(CodeGenModule &CGM, const VarDecl &D) {
    if (CGM.getLangOpts().CPlusPlus)
      return CGM.getMangledName(&D).str();
  
    // If this isn't C++, we don't need a mangled name, just a pretty one.
    assert(!D.isExternallyVisible() && "name shouldn't matter");
    std::string ContextName;
    const DeclContext *DC = D.getDeclContext();
    if (auto *CD = dyn_cast<CapturedDecl>(DC))
      DC = cast<DeclContext>(CD->getNonClosureContext());
    if (const auto *FD = dyn_cast<FunctionDecl>(DC))
      ContextName = std::string(CGM.getMangledName(FD));
    else if (const auto *BD = dyn_cast<BlockDecl>(DC))
      ContextName = std::string(CGM.getBlockMangledName(GlobalDecl(), BD));
    else if (const auto *OMD = dyn_cast<ObjCMethodDecl>(DC))
      ContextName = OMD->getSelector().getAsString();
    else
      llvm_unreachable("Unknown context for static var decl");
  
    ContextName += "." + D.getNameAsString();
    return ContextName;
  }
  
  llvm::Constant *CodeGenModule::getOrCreateStaticVarDecl(
      const VarDecl &D, llvm::GlobalValue::LinkageTypes Linkage) {
    // In general, we don't always emit static var decls once before we reference
    // them. It is possible to reference them before emitting the function that
    // contains them, and it is possible to emit the containing function multiple
    // times.
    if (llvm::Constant *ExistingGV = StaticLocalDeclMap[&D])
      return ExistingGV;
  
    QualType Ty = D.getType();
    assert(Ty->isConstantSizeType() && "VLAs can't be static");
  
    // Use the label if the variable is renamed with the asm-label extension.
    std::string Name;
    if (D.hasAttr<AsmLabelAttr>())
      Name = std::string(getMangledName(&D));
    else
      Name = getStaticDeclName(*this, D);
  
    llvm::Type *LTy = getTypes().ConvertTypeForMem(Ty);
    LangAS AS = GetGlobalVarAddressSpace(&D);
    unsigned TargetAS = getContext().getTargetAddressSpace(AS);
  
    // OpenCL variables in local address space and CUDA shared
    // variables cannot have an initializer.
    llvm::Constant *Init = nullptr;
    if (Ty.getAddressSpace() == LangAS::opencl_local ||
        D.hasAttr<CUDASharedAttr>() || D.hasAttr<LoaderUninitializedAttr>())
      Init = llvm::UndefValue::get(LTy);
    else
      Init = EmitNullConstant(Ty);
  
    llvm::GlobalVariable *GV = new llvm::GlobalVariable(
        getModule(), LTy, Ty.isConstant(getContext()), Linkage, Init, Name,
        nullptr, llvm::GlobalVariable::NotThreadLocal, TargetAS);
    GV->setAlignment(getContext().getDeclAlign(&D).getAsAlign());
  
    if (supportsCOMDAT() && GV->isWeakForLinker())
      GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
  
    if (D.getTLSKind())
      setTLSMode(GV, D);
  
    setGVProperties(GV, &D);
  
    // Make sure the result is of the correct type.
    LangAS ExpectedAS = Ty.getAddressSpace();
    llvm::Constant *Addr = GV;
    if (AS != ExpectedAS) {
      Addr = getTargetCodeGenInfo().performAddrSpaceCast(
          *this, GV, AS, ExpectedAS,
          LTy->getPointerTo(getContext().getTargetAddressSpace(ExpectedAS)));
    }
  
    setStaticLocalDeclAddress(&D, Addr);
  
    // Ensure that the static local gets initialized by making sure the parent
    // function gets emitted eventually.
    const Decl *DC = cast<Decl>(D.getDeclContext());
  
    // We can't name blocks or captured statements directly, so try to emit their
    // parents.
    if (isa<BlockDecl>(DC) || isa<CapturedDecl>(DC)) {
      DC = DC->getNonClosureContext();
      // FIXME: Ensure that global blocks get emitted.
      if (!DC)
        return Addr;
    }
  
    GlobalDecl GD;
    if (const auto *CD = dyn_cast<CXXConstructorDecl>(DC))
      GD = GlobalDecl(CD, Ctor_Base);
    else if (const auto *DD = dyn_cast<CXXDestructorDecl>(DC))
      GD = GlobalDecl(DD, Dtor_Base);
    else if (const auto *FD = dyn_cast<FunctionDecl>(DC))
      GD = GlobalDecl(FD);
    else {
      // Don't do anything for Obj-C method decls or global closures. We should
      // never defer them.
      assert(isa<ObjCMethodDecl>(DC) && "unexpected parent code decl");
    }
    if (GD.getDecl()) {
      // Disable emission of the parent function for the OpenMP device codegen.
      CGOpenMPRuntime::DisableAutoDeclareTargetRAII NoDeclTarget(*this);
      (void)GetAddrOfGlobal(GD);
    }
  
    return Addr;
  }
  
  /// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
  /// global variable that has already been created for it.  If the initializer
  /// has a different type than GV does, this may free GV and return a different
  /// one.  Otherwise it just returns GV.
  llvm::GlobalVariable *
  CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D,
                                                 llvm::GlobalVariable *GV) {
    if (getLangOpts().SYCLIsDevice) {
      auto *Scope = D.getAttr<SYCLScopeAttr>();
      if (Scope && Scope->isWorkGroup()) {
        // In SYCL device code globals which represent WG shared local variables
        // 1) (TODO) must have initializer emitted even if it is constant, because
        //    LLVM->SPIRV translation does not generate optional initializer for
        //    WG shared local variables.
        // 2) must not use guarded init because
        //   a) guarded init uses exceptions not supported in SYCL device code
        //   b) initialization is already safe because it happens in WG scope -
        //      once per WG - by specification, and this will be/ enforced in
        //       SYCL-specific Clang lowering (LowerWGScope.cpp) after CG.
        EmitCXXGlobalVarDeclInit(D, GV, /*PerformInit*/ true);
        // Remove const-ness, otherwise the initializer (a store into the
        // variable) won't have effect in SPIRV due to "Constant" decoration.
        GV->setConstant(false);
        return GV;
      }
    }
    ConstantEmitter emitter(*this);
    llvm::Constant *Init = emitter.tryEmitForInitializer(D);
  
    // If constant emission failed, then this should be a C++ static
    // initializer.
    if (!Init) {
      if (!getLangOpts().CPlusPlus)
        CGM.ErrorUnsupported(D.getInit(), "constant l-value expression");
      else if (HaveInsertPoint()) {
        // Since we have a static initializer, this global variable can't
        // be constant.
        GV->setConstant(false);
  
        EmitCXXGuardedInit(D, GV, /*PerformInit*/true);
      }
      return GV;
    }
  
    // The initializer may differ in type from the global. Rewrite
    // the global to match the initializer.  (We have to do this
    // because some types, like unions, can't be completely represented
    // in the LLVM type system.)
    if (GV->getValueType() != Init->getType()) {
      llvm::GlobalVariable *OldGV = GV;
  
      GV = new llvm::GlobalVariable(CGM.getModule(), Init->getType(),
                                    OldGV->isConstant(),
                                    OldGV->getLinkage(), Init, "",
                                    /*InsertBefore*/ OldGV,
                                    OldGV->getThreadLocalMode(),
                                    OldGV->getType()->getPointerAddressSpace());
      GV->setVisibility(OldGV->getVisibility());
      GV->setDSOLocal(OldGV->isDSOLocal());
      GV->setComdat(OldGV->getComdat());
  
      // Steal the name of the old global
      GV->takeName(OldGV);
  
      // Replace all uses of the old global with the new global
      llvm::Constant *NewPtrForOldDecl =
      llvm::ConstantExpr::getBitCast(GV, OldGV->getType());
      OldGV->replaceAllUsesWith(NewPtrForOldDecl);
  
      // Erase the old global, since it is no longer used.
      OldGV->eraseFromParent();
    }
  
    GV->setConstant(CGM.isTypeConstant(D.getType(), true));
    GV->setInitializer(Init);
  
    emitter.finalize(GV);
  
    if (D.needsDestruction(getContext()) == QualType::DK_cxx_destructor &&
        HaveInsertPoint()) {
      // We have a constant initializer, but a nontrivial destructor. We still
      // need to perform a guarded "initialization" in order to register the
      // destructor.
      EmitCXXGuardedInit(D, GV, /*PerformInit*/false);
    }
  
    return GV;
  }
  
  void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D,
                                        llvm::GlobalValue::LinkageTypes Linkage) {
    // Check to see if we already have a global variable for this
    // declaration.  This can happen when double-emitting function
    // bodies, e.g. with complete and base constructors.
    llvm::Constant *addr = CGM.getOrCreateStaticVarDecl(D, Linkage);
    CharUnits alignment = getContext().getDeclAlign(&D);
  
    // Store into LocalDeclMap before generating initializer to handle
    // circular references.
    setAddrOfLocalVar(&D, Address(addr, alignment));
  
    // We can't have a VLA here, but we can have a pointer to a VLA,
    // even though that doesn't really make any sense.
    // Make sure to evaluate VLA bounds now so that we have them for later.
    if (D.getType()->isVariablyModifiedType())
      EmitVariablyModifiedType(D.getType());
  
    // Save the type in case adding the initializer forces a type change.
    llvm::Type *expectedType = addr->getType();
  
    llvm::GlobalVariable *var =
      cast<llvm::GlobalVariable>(addr->stripPointerCasts());
  
    // CUDA's local and local static __shared__ variables should not
    // have any non-empty initializers. This is ensured by Sema.
    // Whatever initializer such variable may have when it gets here is
    // a no-op and should not be emitted.
    bool isCudaSharedVar = getLangOpts().CUDA && getLangOpts().CUDAIsDevice &&
                           D.hasAttr<CUDASharedAttr>();
    // If this value has an initializer, emit it.
    if (D.getInit() && !isCudaSharedVar)
      var = AddInitializerToStaticVarDecl(D, var);
  
    var->setAlignment(alignment.getAsAlign());
  
    if (D.hasAttr<AnnotateAttr>())
      CGM.AddGlobalAnnotations(&D, var);
  
    // Emit Intel FPGA attribute annotation for a local static variable.
    if (getLangOpts().SYCLIsDevice)
      CGM.addGlobalIntelFPGAAnnotation(&D, var);
  
    if (auto *SA = D.getAttr<PragmaClangBSSSectionAttr>())
      var->addAttribute("bss-section", SA->getName());
    if (auto *SA = D.getAttr<PragmaClangDataSectionAttr>())
      var->addAttribute("data-section", SA->getName());
    if (auto *SA = D.getAttr<PragmaClangRodataSectionAttr>())
      var->addAttribute("rodata-section", SA->getName());
    if (auto *SA = D.getAttr<PragmaClangRelroSectionAttr>())
      var->addAttribute("relro-section", SA->getName());
  
    if (const SectionAttr *SA = D.getAttr<SectionAttr>())
      var->setSection(SA->getName());
  
    if (D.hasAttr<UsedAttr>())
      CGM.addUsedGlobal(var);
  
    // We may have to cast the constant because of the initializer
    // mismatch above.
    //
    // FIXME: It is really dangerous to store this in the map; if anyone
    // RAUW's the GV uses of this constant will be invalid.
    llvm::Constant *castedAddr =
      llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(var, expectedType);
    if (var != castedAddr)
      LocalDeclMap.find(&D)->second = Address(castedAddr, alignment);
    CGM.setStaticLocalDeclAddress(&D, castedAddr);
  
    CGM.getSanitizerMetadata()->reportGlobalToASan(var, D);
  
    // Emit global variable debug descriptor for static vars.
    CGDebugInfo *DI = getDebugInfo();
    if (DI && CGM.getCodeGenOpts().hasReducedDebugInfo()) {
      DI->setLocation(D.getLocation());
      DI->EmitGlobalVariable(var, &D);
    }
  }
  
  namespace {
    struct DestroyObject final : EHScopeStack::Cleanup {
      DestroyObject(Address addr, QualType type,
                    CodeGenFunction::Destroyer *destroyer,
                    bool useEHCleanupForArray)
        : addr(addr), type(type), destroyer(destroyer),
          useEHCleanupForArray(useEHCleanupForArray) {}
  
      Address addr;
      QualType type;
      CodeGenFunction::Destroyer *destroyer;
      bool useEHCleanupForArray;
  
      void Emit(CodeGenFunction &CGF, Flags flags) override {
        // Don't use an EH cleanup recursively from an EH cleanup.
        bool useEHCleanupForArray =
          flags.isForNormalCleanup() && this->useEHCleanupForArray;
  
        CGF.emitDestroy(addr, type, destroyer, useEHCleanupForArray);
      }
    };
  
    template <class Derived>
    struct DestroyNRVOVariable : EHScopeStack::Cleanup {
      DestroyNRVOVariable(Address addr, QualType type, llvm::Value *NRVOFlag)
          : NRVOFlag(NRVOFlag), Loc(addr), Ty(type) {}
  
      llvm::Value *NRVOFlag;
      Address Loc;
      QualType Ty;
  
      void Emit(CodeGenFunction &CGF, Flags flags) override {
        // Along the exceptions path we always execute the dtor.
        bool NRVO = flags.isForNormalCleanup() && NRVOFlag;
  
        llvm::BasicBlock *SkipDtorBB = nullptr;
        if (NRVO) {
          // If we exited via NRVO, we skip the destructor call.
          llvm::BasicBlock *RunDtorBB = CGF.createBasicBlock("nrvo.unused");
          SkipDtorBB = CGF.createBasicBlock("nrvo.skipdtor");
          llvm::Value *DidNRVO =
            CGF.Builder.CreateFlagLoad(NRVOFlag, "nrvo.val");
          CGF.Builder.CreateCondBr(DidNRVO, SkipDtorBB, RunDtorBB);
          CGF.EmitBlock(RunDtorBB);
        }
  
        static_cast<Derived *>(this)->emitDestructorCall(CGF);
  
        if (NRVO) CGF.EmitBlock(SkipDtorBB);
      }
  
      virtual ~DestroyNRVOVariable() = default;
    };
  
    struct DestroyNRVOVariableCXX final
        : DestroyNRVOVariable<DestroyNRVOVariableCXX> {
      DestroyNRVOVariableCXX(Address addr, QualType type,
                             const CXXDestructorDecl *Dtor, llvm::Value *NRVOFlag)
          : DestroyNRVOVariable<DestroyNRVOVariableCXX>(addr, type, NRVOFlag),
            Dtor(Dtor) {}
  
      const CXXDestructorDecl *Dtor;
  
      void emitDestructorCall(CodeGenFunction &CGF) {
        CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
                                  /*ForVirtualBase=*/false,
                                  /*Delegating=*/false, Loc, Ty);
      }
    };
  
    struct DestroyNRVOVariableC final
        : DestroyNRVOVariable<DestroyNRVOVariableC> {
      DestroyNRVOVariableC(Address addr, llvm::Value *NRVOFlag, QualType Ty)
          : DestroyNRVOVariable<DestroyNRVOVariableC>(addr, Ty, NRVOFlag) {}
  
      void emitDestructorCall(CodeGenFunction &CGF) {
        CGF.destroyNonTrivialCStruct(CGF, Loc, Ty);
      }
    };
  
    struct CallStackRestore final : EHScopeStack::Cleanup {
      Address Stack;
      CallStackRestore(Address Stack) : Stack(Stack) {}
      void Emit(CodeGenFunction &CGF, Flags flags) override {
        llvm::Value *V = CGF.Builder.CreateLoad(Stack);
        llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
        CGF.Builder.CreateCall(F, V);
      }
    };
  
    struct ExtendGCLifetime final : EHScopeStack::Cleanup {
      const VarDecl &Var;
      ExtendGCLifetime(const VarDecl *var) : Var(*var) {}
  
      void Emit(CodeGenFunction &CGF, Flags flags) override {
        // Compute the address of the local variable, in case it's a
        // byref or something.
        DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(&Var), false,
                        Var.getType(), VK_LValue, SourceLocation());
        llvm::Value *value = CGF.EmitLoadOfScalar(CGF.EmitDeclRefLValue(&DRE),
                                                  SourceLocation());
        CGF.EmitExtendGCLifetime(value);
      }
    };
  
    struct CallCleanupFunction final : EHScopeStack::Cleanup {
      llvm::Constant *CleanupFn;
      const CGFunctionInfo &FnInfo;
      const VarDecl &Var;
  
      CallCleanupFunction(llvm::Constant *CleanupFn, const CGFunctionInfo *Info,
                          const VarDecl *Var)
        : CleanupFn(CleanupFn), FnInfo(*Info), Var(*Var) {}
  
      void Emit(CodeGenFunction &CGF, Flags flags) override {
        DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(&Var), false,
                        Var.getType(), VK_LValue, SourceLocation());
        // Compute the address of the local variable, in case it's a byref
        // or something.
        llvm::Value *Addr = CGF.EmitDeclRefLValue(&DRE).getPointer(CGF);
  
        // In some cases, the type of the function argument will be different from
        // the type of the pointer. An example of this is
        // void f(void* arg);
        // __attribute__((cleanup(f))) void *g;
        //
        // To fix this we insert a bitcast here.
        QualType ArgTy = FnInfo.arg_begin()->type;
        llvm::Value *Arg =
          CGF.Builder.CreateBitCast(Addr, CGF.ConvertType(ArgTy));
  
        CallArgList Args;
        Args.add(RValue::get(Arg),
                 CGF.getContext().getPointerType(Var.getType()));
        auto Callee = CGCallee::forDirect(CleanupFn);
        CGF.EmitCall(FnInfo, Callee, ReturnValueSlot(), Args);
      }
    };
  } // end anonymous namespace
  
  /// EmitAutoVarWithLifetime - Does the setup required for an automatic
  /// variable with lifetime.
  static void EmitAutoVarWithLifetime(CodeGenFunction &CGF, const VarDecl &var,
                                      Address addr,
                                      Qualifiers::ObjCLifetime lifetime) {
    switch (lifetime) {
    case Qualifiers::OCL_None:
      llvm_unreachable("present but none");
  
    case Qualifiers::OCL_ExplicitNone:
      // nothing to do
      break;
  
    case Qualifiers::OCL_Strong: {
      CodeGenFunction::Destroyer *destroyer =
        (var.hasAttr<ObjCPreciseLifetimeAttr>()
         ? CodeGenFunction::destroyARCStrongPrecise
         : CodeGenFunction::destroyARCStrongImprecise);
  
      CleanupKind cleanupKind = CGF.getARCCleanupKind();
      CGF.pushDestroy(cleanupKind, addr, var.getType(), destroyer,
                      cleanupKind & EHCleanup);
      break;
    }
    case Qualifiers::OCL_Autoreleasing:
      // nothing to do
      break;
  
    case Qualifiers::OCL_Weak:
      // __weak objects always get EH cleanups; otherwise, exceptions
      // could cause really nasty crashes instead of mere leaks.
      CGF.pushDestroy(NormalAndEHCleanup, addr, var.getType(),
                      CodeGenFunction::destroyARCWeak,
                      /*useEHCleanup*/ true);
      break;
    }
  }
  
  static bool isAccessedBy(const VarDecl &var, const Stmt *s) {
    if (const Expr *e = dyn_cast<Expr>(s)) {
      // Skip the most common kinds of expressions that make
      // hierarchy-walking expensive.
      s = e = e->IgnoreParenCasts();
  
      if (const DeclRefExpr *ref = dyn_cast<DeclRefExpr>(e))
        return (ref->getDecl() == &var);
      if (const BlockExpr *be = dyn_cast<BlockExpr>(e)) {
        const BlockDecl *block = be->getBlockDecl();
        for (const auto &I : block->captures()) {
          if (I.getVariable() == &var)
            return true;
        }
      }
    }
  
    for (const Stmt *SubStmt : s->children())
      // SubStmt might be null; as in missing decl or conditional of an if-stmt.
      if (SubStmt && isAccessedBy(var, SubStmt))
        return true;
  
    return false;
  }
  
  static bool isAccessedBy(const ValueDecl *decl, const Expr *e) {
    if (!decl) return false;
    if (!isa<VarDecl>(decl)) return false;
    const VarDecl *var = cast<VarDecl>(decl);
    return isAccessedBy(*var, e);
  }
  
  static bool tryEmitARCCopyWeakInit(CodeGenFunction &CGF,
                                     const LValue &destLV, const Expr *init) {
    bool needsCast = false;
  
    while (auto castExpr = dyn_cast<CastExpr>(init->IgnoreParens())) {
      switch (castExpr->getCastKind()) {
      // Look through casts that don't require representation changes.
      case CK_NoOp:
      case CK_BitCast:
      case CK_BlockPointerToObjCPointerCast:
        needsCast = true;
        break;
  
      // If we find an l-value to r-value cast from a __weak variable,
      // emit this operation as a copy or move.
      case CK_LValueToRValue: {
        const Expr *srcExpr = castExpr->getSubExpr();
        if (srcExpr->getType().getObjCLifetime() != Qualifiers::OCL_Weak)
          return false;
  
        // Emit the source l-value.
        LValue srcLV = CGF.EmitLValue(srcExpr);
  
        // Handle a formal type change to avoid asserting.
        auto srcAddr = srcLV.getAddress(CGF);
        if (needsCast) {
          srcAddr = CGF.Builder.CreateElementBitCast(
              srcAddr, destLV.getAddress(CGF).getElementType());
        }
  
        // If it was an l-value, use objc_copyWeak.
        if (srcExpr->getValueKind() == VK_LValue) {
          CGF.EmitARCCopyWeak(destLV.getAddress(CGF), srcAddr);
        } else {
          assert(srcExpr->getValueKind() == VK_XValue);
          CGF.EmitARCMoveWeak(destLV.getAddress(CGF), srcAddr);
        }
        return true;
      }
  
      // Stop at anything else.
      default:
        return false;
      }
  
      init = castExpr->getSubExpr();
    }
    return false;
  }
  
  static void drillIntoBlockVariable(CodeGenFunction &CGF,
                                     LValue &lvalue,
                                     const VarDecl *var) {
    lvalue.setAddress(CGF.emitBlockByrefAddress(lvalue.getAddress(CGF), var));
  }
  
  void CodeGenFunction::EmitNullabilityCheck(LValue LHS, llvm::Value *RHS,
                                             SourceLocation Loc) {
    if (!SanOpts.has(SanitizerKind::NullabilityAssign))
      return;
  
    auto Nullability = LHS.getType()->getNullability(getContext());
    if (!Nullability || *Nullability != NullabilityKind::NonNull)
      return;
  
    // Check if the right hand side of the assignment is nonnull, if the left
    // hand side must be nonnull.
    SanitizerScope SanScope(this);
    llvm::Value *IsNotNull = Builder.CreateIsNotNull(RHS);
    llvm::Constant *StaticData[] = {
        EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(LHS.getType()),
        llvm::ConstantInt::get(Int8Ty, 0), // The LogAlignment info is unused.
        llvm::ConstantInt::get(Int8Ty, TCK_NonnullAssign)};
    EmitCheck({{IsNotNull, SanitizerKind::NullabilityAssign}},
              SanitizerHandler::TypeMismatch, StaticData, RHS);
  }
  
  void CodeGenFunction::EmitScalarInit(const Expr *init, const ValueDecl *D,
                                       LValue lvalue, bool capturedByInit) {
    Qualifiers::ObjCLifetime lifetime = lvalue.getObjCLifetime();
    if (!lifetime) {
      llvm::Value *value = EmitScalarExpr(init);
      if (capturedByInit)
        drillIntoBlockVariable(*this, lvalue, cast<VarDecl>(D));
      EmitNullabilityCheck(lvalue, value, init->getExprLoc());
      EmitStoreThroughLValue(RValue::get(value), lvalue, true);
      return;
    }
  
    if (const CXXDefaultInitExpr *DIE = dyn_cast<CXXDefaultInitExpr>(init))
      init = DIE->getExpr();
  
    // If we're emitting a value with lifetime, we have to do the
    // initialization *before* we leave the cleanup scopes.
    if (const ExprWithCleanups *EWC = dyn_cast<ExprWithCleanups>(init))
      init = EWC->getSubExpr();
    CodeGenFunction::RunCleanupsScope Scope(*this);
  
    // We have to maintain the illusion that the variable is
    // zero-initialized.  If the variable might be accessed in its
    // initializer, zero-initialize before running the initializer, then
    // actually perform the initialization with an assign.
    bool accessedByInit = false;
    if (lifetime != Qualifiers::OCL_ExplicitNone)
      accessedByInit = (capturedByInit || isAccessedBy(D, init));
    if (accessedByInit) {
      LValue tempLV = lvalue;
      // Drill down to the __block object if necessary.
      if (capturedByInit) {
        // We can use a simple GEP for this because it can't have been
        // moved yet.
        tempLV.setAddress(emitBlockByrefAddress(tempLV.getAddress(*this),
                                                cast<VarDecl>(D),
                                                /*follow*/ false));
      }
  
      auto ty =
          cast<llvm::PointerType>(tempLV.getAddress(*this).getElementType());
      llvm::Value *zero = CGM.getNullPointer(ty, tempLV.getType());
  
      // If __weak, we want to use a barrier under certain conditions.
      if (lifetime == Qualifiers::OCL_Weak)
        EmitARCInitWeak(tempLV.getAddress(*this), zero);
  
      // Otherwise just do a simple store.
      else
        EmitStoreOfScalar(zero, tempLV, /* isInitialization */ true);
    }
  
    // Emit the initializer.
    llvm::Value *value = nullptr;
  
    switch (lifetime) {
    case Qualifiers::OCL_None:
      llvm_unreachable("present but none");
  
    case Qualifiers::OCL_Strong: {
      if (!D || !isa<VarDecl>(D) || !cast<VarDecl>(D)->isARCPseudoStrong()) {
        value = EmitARCRetainScalarExpr(init);
        break;
      }
      // If D is pseudo-strong, treat it like __unsafe_unretained here. This means
      // that we omit the retain, and causes non-autoreleased return values to be
      // immediately released.
      LLVM_FALLTHROUGH;
    }
  
    case Qualifiers::OCL_ExplicitNone:
      value = EmitARCUnsafeUnretainedScalarExpr(init);
      break;
  
    case Qualifiers::OCL_Weak: {
      // If it's not accessed by the initializer, try to emit the
      // initialization with a copy or move.
      if (!accessedByInit && tryEmitARCCopyWeakInit(*this, lvalue, init)) {
        return;
      }
  
      // No way to optimize a producing initializer into this.  It's not
      // worth optimizing for, because the value will immediately
      // disappear in the common case.
      value = EmitScalarExpr(init);
  
      if (capturedByInit) drillIntoBlockVariable(*this, lvalue, cast<VarDecl>(D));
      if (accessedByInit)
        EmitARCStoreWeak(lvalue.getAddress(*this), value, /*ignored*/ true);
      else
        EmitARCInitWeak(lvalue.getAddress(*this), value);
      return;
    }
  
    case Qualifiers::OCL_Autoreleasing:
      value = EmitARCRetainAutoreleaseScalarExpr(init);
      break;
    }
  
    if (capturedByInit) drillIntoBlockVariable(*this, lvalue, cast<VarDecl>(D));
  
    EmitNullabilityCheck(lvalue, value, init->getExprLoc());
  
    // If the variable might have been accessed by its initializer, we
    // might have to initialize with a barrier.  We have to do this for
    // both __weak and __strong, but __weak got filtered out above.
    if (accessedByInit && lifetime == Qualifiers::OCL_Strong) {
      llvm::Value *oldValue = EmitLoadOfScalar(lvalue, init->getExprLoc());
      EmitStoreOfScalar(value, lvalue, /* isInitialization */ true);
      EmitARCRelease(oldValue, ARCImpreciseLifetime);
      return;
    }
  
    EmitStoreOfScalar(value, lvalue, /* isInitialization */ true);
  }
  
  /// Decide whether we can emit the non-zero parts of the specified initializer
  /// with equal or fewer than NumStores scalar stores.
  static bool canEmitInitWithFewStoresAfterBZero(llvm::Constant *Init,
                                                 unsigned &NumStores) {
    // Zero and Undef never requires any extra stores.
    if (isa<llvm::ConstantAggregateZero>(Init) ||
        isa<llvm::ConstantPointerNull>(Init) ||
        isa<llvm::UndefValue>(Init))
      return true;
    if (isa<llvm::ConstantInt>(Init) || isa<llvm::ConstantFP>(Init) ||
        isa<llvm::ConstantVector>(Init) || isa<llvm::BlockAddress>(Init) ||
        isa<llvm::ConstantExpr>(Init))
      return Init->isNullValue() || NumStores--;
  
    // See if we can emit each element.
    if (isa<llvm::ConstantArray>(Init) || isa<llvm::ConstantStruct>(Init)) {
      for (unsigned i = 0, e = Init->getNumOperands(); i != e; ++i) {
        llvm::Constant *Elt = cast<llvm::Constant>(Init->getOperand(i));
        if (!canEmitInitWithFewStoresAfterBZero(Elt, NumStores))
          return false;
      }
      return true;
    }
  
    if (llvm::ConstantDataSequential *CDS =
          dyn_cast<llvm::ConstantDataSequential>(Init)) {
      for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
        llvm::Constant *Elt = CDS->getElementAsConstant(i);
        if (!canEmitInitWithFewStoresAfterBZero(Elt, NumStores))
          return false;
      }
      return true;
    }
  
    // Anything else is hard and scary.
    return false;
  }
  
  /// For inits that canEmitInitWithFewStoresAfterBZero returned true for, emit
  /// the scalar stores that would be required.
  static void emitStoresForInitAfterBZero(CodeGenModule &CGM,
                                          llvm::Constant *Init, Address Loc,
                                          bool isVolatile, CGBuilderTy &Builder) {
    assert(!Init->isNullValue() && !isa<llvm::UndefValue>(Init) &&
           "called emitStoresForInitAfterBZero for zero or undef value.");
  
    if (isa<llvm::ConstantInt>(Init) || isa<llvm::ConstantFP>(Init) ||
        isa<llvm::ConstantVector>(Init) || isa<llvm::BlockAddress>(Init) ||
        isa<llvm::ConstantExpr>(Init)) {
      Builder.CreateStore(Init, Loc, isVolatile);
      return;
    }
  
    if (llvm::ConstantDataSequential *CDS =
            dyn_cast<llvm::ConstantDataSequential>(Init)) {
      for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
        llvm::Constant *Elt = CDS->getElementAsConstant(i);
  
        // If necessary, get a pointer to the element and emit it.
        if (!Elt->isNullValue() && !isa<llvm::UndefValue>(Elt))
          emitStoresForInitAfterBZero(
              CGM, Elt, Builder.CreateConstInBoundsGEP2_32(Loc, 0, i), isVolatile,
              Builder);
      }
      return;
    }
  
    assert((isa<llvm::ConstantStruct>(Init) || isa<llvm::ConstantArray>(Init)) &&
           "Unknown value type!");
  
    for (unsigned i = 0, e = Init->getNumOperands(); i != e; ++i) {
      llvm::Constant *Elt = cast<llvm::Constant>(Init->getOperand(i));
  
      // If necessary, get a pointer to the element and emit it.
      if (!Elt->isNullValue() && !isa<llvm::UndefValue>(Elt))
        emitStoresForInitAfterBZero(CGM, Elt,
                                    Builder.CreateConstInBoundsGEP2_32(Loc, 0, i),
                                    isVolatile, Builder);
    }
  }
  
  /// Decide whether we should use bzero plus some stores to initialize a local
  /// variable instead of using a memcpy from a constant global.  It is beneficial
  /// to use bzero if the global is all zeros, or mostly zeros and large.
  static bool shouldUseBZeroPlusStoresToInitialize(llvm::Constant *Init,
                                                   uint64_t GlobalSize) {
    // If a global is all zeros, always use a bzero.
    if (isa<llvm::ConstantAggregateZero>(Init)) return true;
  
    // If a non-zero global is <= 32 bytes, always use a memcpy.  If it is large,
    // do it if it will require 6 or fewer scalar stores.
    // TODO: Should budget depends on the size?  Avoiding a large global warrants
    // plopping in more stores.
    unsigned StoreBudget = 6;
    uint64_t SizeLimit = 32;
  
    return GlobalSize > SizeLimit &&
           canEmitInitWithFewStoresAfterBZero(Init, StoreBudget);
  }
  
  /// Decide whether we should use memset to initialize a local variable instead
  /// of using a memcpy from a constant global. Assumes we've already decided to
  /// not user bzero.
  /// FIXME We could be more clever, as we are for bzero above, and generate
  ///       memset followed by stores. It's unclear that's worth the effort.
  static llvm::Value *shouldUseMemSetToInitialize(llvm::Constant *Init,
                                                  uint64_t GlobalSize,
                                                  const llvm::DataLayout &DL) {
    uint64_t SizeLimit = 32;
    if (GlobalSize <= SizeLimit)
      return nullptr;
    return llvm::isBytewiseValue(Init, DL);
  }
  
  /// Decide whether we want to split a constant structure or array store into a
  /// sequence of its fields' stores. This may cost us code size and compilation
  /// speed, but plays better with store optimizations.
  static bool shouldSplitConstantStore(CodeGenModule &CGM,
                                       uint64_t GlobalByteSize) {
    // Don't break things that occupy more than one cacheline.
    uint64_t ByteSizeLimit = 64;
    if (CGM.getCodeGenOpts().OptimizationLevel == 0)
      return false;
    if (GlobalByteSize <= ByteSizeLimit)
      return true;
    return false;
  }
  
  enum class IsPattern { No, Yes };
  
  /// Generate a constant filled with either a pattern or zeroes.
  static llvm::Constant *patternOrZeroFor(CodeGenModule &CGM, IsPattern isPattern,
                                          llvm::Type *Ty) {
    if (isPattern == IsPattern::Yes)
      return initializationPatternFor(CGM, Ty);
    else
      return llvm::Constant::getNullValue(Ty);
  }
  
  static llvm::Constant *constWithPadding(CodeGenModule &CGM, IsPattern isPattern,
                                          llvm::Constant *constant);
  
  /// Helper function for constWithPadding() to deal with padding in structures.
  static llvm::Constant *constStructWithPadding(CodeGenModule &CGM,
                                                IsPattern isPattern,
                                                llvm::StructType *STy,
                                                llvm::Constant *constant) {
    const llvm::DataLayout &DL = CGM.getDataLayout();
    const llvm::StructLayout *Layout = DL.getStructLayout(STy);
    llvm::Type *Int8Ty = llvm::IntegerType::getInt8Ty(CGM.getLLVMContext());
    unsigned SizeSoFar = 0;
    SmallVector<llvm::Constant *, 8> Values;
    bool NestedIntact = true;
    for (unsigned i = 0, e = STy->getNumElements(); i != e; i++) {
      unsigned CurOff = Layout->getElementOffset(i);
      if (SizeSoFar < CurOff) {
        assert(!STy->isPacked());
        auto *PadTy = llvm::ArrayType::get(Int8Ty, CurOff - SizeSoFar);
        Values.push_back(patternOrZeroFor(CGM, isPattern, PadTy));
      }
      llvm::Constant *CurOp;
      if (constant->isZeroValue())
        CurOp = llvm::Constant::getNullValue(STy->getElementType(i));
      else
        CurOp = cast<llvm::Constant>(constant->getAggregateElement(i));
      auto *NewOp = constWithPadding(CGM, isPattern, CurOp);
      if (CurOp != NewOp)
        NestedIntact = false;
      Values.push_back(NewOp);
      SizeSoFar = CurOff + DL.getTypeAllocSize(CurOp->getType());
    }
    unsigned TotalSize = Layout->getSizeInBytes();
    if (SizeSoFar < TotalSize) {
      auto *PadTy = llvm::ArrayType::get(Int8Ty, TotalSize - SizeSoFar);
      Values.push_back(patternOrZeroFor(CGM, isPattern, PadTy));
    }
    if (NestedIntact && Values.size() == STy->getNumElements())
      return constant;
    return llvm::ConstantStruct::getAnon(Values, STy->isPacked());
  }
  
  /// Replace all padding bytes in a given constant with either a pattern byte or
  /// 0x00.
  static llvm::Constant *constWithPadding(CodeGenModule &CGM, IsPattern isPattern,
                                          llvm::Constant *constant) {
    llvm::Type *OrigTy = constant->getType();
    if (const auto STy = dyn_cast<llvm::StructType>(OrigTy))
      return constStructWithPadding(CGM, isPattern, STy, constant);
    if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(OrigTy)) {
      llvm::SmallVector<llvm::Constant *, 8> Values;
      uint64_t Size = ArrayTy->getNumElements();
      if (!Size)
        return constant;
      llvm::Type *ElemTy = ArrayTy->getElementType();
      bool ZeroInitializer = constant->isNullValue();
      llvm::Constant *OpValue, *PaddedOp;
      if (ZeroInitializer) {
        OpValue = llvm::Constant::getNullValue(ElemTy);
        PaddedOp = constWithPadding(CGM, isPattern, OpValue);
      }
      for (unsigned Op = 0; Op != Size; ++Op) {
        if (!ZeroInitializer) {
          OpValue = constant->getAggregateElement(Op);
          PaddedOp = constWithPadding(CGM, isPattern, OpValue);
        }
        Values.push_back(PaddedOp);
      }
      auto *NewElemTy = Values[0]->getType();
      if (NewElemTy == ElemTy)
        return constant;
      auto *NewArrayTy = llvm::ArrayType::get(NewElemTy, Size);
      return llvm::ConstantArray::get(NewArrayTy, Values);
    }
    // FIXME: Add handling for tail padding in vectors. Vectors don't
    // have padding between or inside elements, but the total amount of
    // data can be less than the allocated size.
    return constant;
  }
  
  Address CodeGenModule::createUnnamedGlobalFrom(const VarDecl &D,
                                                 llvm::Constant *Constant,
                                                 CharUnits Align) {
    auto FunctionName = [&](const DeclContext *DC) -> std::string {
      if (const auto *FD = dyn_cast<FunctionDecl>(DC)) {
        if (const auto *CC = dyn_cast<CXXConstructorDecl>(FD))
          return CC->getNameAsString();
        if (const auto *CD = dyn_cast<CXXDestructorDecl>(FD))
          return CD->getNameAsString();
        return std::string(getMangledName(FD));
      } else if (const auto *OM = dyn_cast<ObjCMethodDecl>(DC)) {
        return OM->getNameAsString();
      } else if (isa<BlockDecl>(DC)) {
        return "<block>";
      } else if (isa<CapturedDecl>(DC)) {
        return "<captured>";
      } else {
        llvm_unreachable("expected a function or method");
      }
    };
  
    // Form a simple per-variable cache of these values in case we find we
    // want to reuse them.
    llvm::GlobalVariable *&CacheEntry = InitializerConstants[&D];
    if (!CacheEntry || CacheEntry->getInitializer() != Constant) {
      auto *Ty = Constant->getType();
      bool isConstant = true;
      llvm::GlobalVariable *InsertBefore = nullptr;
      unsigned AS =
          getContext().getTargetAddressSpace(getStringLiteralAddressSpace());
      std::string Name;
      if (D.hasGlobalStorage())
        Name = getMangledName(&D).str() + ".const";
      else if (const DeclContext *DC = D.getParentFunctionOrMethod())
        Name = ("__const." + FunctionName(DC) + "." + D.getName()).str();
      else
        llvm_unreachable("local variable has no parent function or method");
      llvm::GlobalVariable *GV = new llvm::GlobalVariable(
          getModule(), Ty, isConstant, llvm::GlobalValue::PrivateLinkage,
          Constant, Name, InsertBefore, llvm::GlobalValue::NotThreadLocal, AS);
      GV->setAlignment(Align.getAsAlign());
      GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
      CacheEntry = GV;
    } else if (CacheEntry->getAlignment() < Align.getQuantity()) {
      CacheEntry->setAlignment(Align.getAsAlign());
    }
  
    return Address(CacheEntry, Align);
  }
  
  static Address createUnnamedGlobalForMemcpyFrom(CodeGenModule &CGM,
                                                  const VarDecl &D,
                                                  CGBuilderTy &Builder,
                                                  llvm::Constant *Constant,
                                                  CharUnits Align) {
    Address SrcPtr = CGM.createUnnamedGlobalFrom(D, Constant, Align);
    llvm::Type *BP = llvm::PointerType::getInt8PtrTy(CGM.getLLVMContext(),
                                                     SrcPtr.getAddressSpace());
    if (SrcPtr.getType() != BP)
      SrcPtr = Builder.CreateBitCast(SrcPtr, BP);
    return SrcPtr;
  }
  
  static void emitStoresForConstant(CodeGenModule &CGM, const VarDecl &D,
                                    Address Loc, bool isVolatile,
                                    CGBuilderTy &Builder,
                                    llvm::Constant *constant) {
    auto *Ty = constant->getType();
    uint64_t ConstantSize = CGM.getDataLayout().getTypeAllocSize(Ty);
    if (!ConstantSize)
      return;
  
    bool canDoSingleStore = Ty->isIntOrIntVectorTy() ||
                            Ty->isPtrOrPtrVectorTy() || Ty->isFPOrFPVectorTy();
    if (canDoSingleStore) {
      Builder.CreateStore(constant, Loc, isVolatile);
      return;
    }
  
    auto *SizeVal = llvm::ConstantInt::get(CGM.IntPtrTy, ConstantSize);
  
    // If the initializer is all or mostly the same, codegen with bzero / memset
    // then do a few stores afterward.
    if (shouldUseBZeroPlusStoresToInitialize(constant, ConstantSize)) {
      Builder.CreateMemSet(Loc, llvm::ConstantInt::get(CGM.Int8Ty, 0), SizeVal,
                           isVolatile);
  
      bool valueAlreadyCorrect =
          constant->isNullValue() || isa<llvm::UndefValue>(constant);
      if (!valueAlreadyCorrect) {
        Loc = Builder.CreateBitCast(Loc, Ty->getPointerTo(Loc.getAddressSpace()));
        emitStoresForInitAfterBZero(CGM, constant, Loc, isVolatile, Builder);
      }
      return;
    }
  
    // If the initializer is a repeated byte pattern, use memset.
    llvm::Value *Pattern =
        shouldUseMemSetToInitialize(constant, ConstantSize, CGM.getDataLayout());
    if (Pattern) {
      uint64_t Value = 0x00;
      if (!isa<llvm::UndefValue>(Pattern)) {
        const llvm::APInt &AP = cast<llvm::ConstantInt>(Pattern)->getValue();
        assert(AP.getBitWidth() <= 8);
        Value = AP.getLimitedValue();
      }
      Builder.CreateMemSet(Loc, llvm::ConstantInt::get(CGM.Int8Ty, Value), SizeVal,
                           isVolatile);
      return;
    }
  
    // If the initializer is small, use a handful of stores.
    if (shouldSplitConstantStore(CGM, ConstantSize)) {
      if (auto *STy = dyn_cast<llvm::StructType>(Ty)) {
        // FIXME: handle the case when STy != Loc.getElementType().
        if (STy == Loc.getElementType()) {
          for (unsigned i = 0; i != constant->getNumOperands(); i++) {
            Address EltPtr = Builder.CreateStructGEP(Loc, i);
            emitStoresForConstant(
                CGM, D, EltPtr, isVolatile, Builder,
                cast<llvm::Constant>(Builder.CreateExtractValue(constant, i)));
          }
          return;
        }
      } else if (auto *ATy = dyn_cast<llvm::ArrayType>(Ty)) {
        // FIXME: handle the case when ATy != Loc.getElementType().
        if (ATy == Loc.getElementType()) {
          for (unsigned i = 0; i != ATy->getNumElements(); i++) {
            Address EltPtr = Builder.CreateConstArrayGEP(Loc, i);
            emitStoresForConstant(
                CGM, D, EltPtr, isVolatile, Builder,
                cast<llvm::Constant>(Builder.CreateExtractValue(constant, i)));
          }
          return;
        }
      }
    }
  
    // Copy from a global.
    Builder.CreateMemCpy(Loc,
                         createUnnamedGlobalForMemcpyFrom(
                             CGM, D, Builder, constant, Loc.getAlignment()),
                         SizeVal, isVolatile);
  }
  
  static void emitStoresForZeroInit(CodeGenModule &CGM, const VarDecl &D,
                                    Address Loc, bool isVolatile,
                                    CGBuilderTy &Builder) {
    llvm::Type *ElTy = Loc.getElementType();
    llvm::Constant *constant =
        constWithPadding(CGM, IsPattern::No, llvm::Constant::getNullValue(ElTy));
    emitStoresForConstant(CGM, D, Loc, isVolatile, Builder, constant);
  }
  
  static void emitStoresForPatternInit(CodeGenModule &CGM, const VarDecl &D,
                                       Address Loc, bool isVolatile,
                                       CGBuilderTy &Builder) {
    llvm::Type *ElTy = Loc.getElementType();
    llvm::Constant *constant = constWithPadding(
        CGM, IsPattern::Yes, initializationPatternFor(CGM, ElTy));
    assert(!isa<llvm::UndefValue>(constant));
    emitStoresForConstant(CGM, D, Loc, isVolatile, Builder, constant);
  }
  
  static bool containsUndef(llvm::Constant *constant) {
    auto *Ty = constant->getType();
    if (isa<llvm::UndefValue>(constant))
      return true;
    if (Ty->isStructTy() || Ty->isArrayTy() || Ty->isVectorTy())
      for (llvm::Use &Op : constant->operands())
        if (containsUndef(cast<llvm::Constant>(Op)))
          return true;
    return false;
  }
  
  static llvm::Constant *replaceUndef(CodeGenModule &CGM, IsPattern isPattern,
                                      llvm::Constant *constant) {
    auto *Ty = constant->getType();
    if (isa<llvm::UndefValue>(constant))
      return patternOrZeroFor(CGM, isPattern, Ty);
    if (!(Ty->isStructTy() || Ty->isArrayTy() || Ty->isVectorTy()))
      return constant;
    if (!containsUndef(constant))
      return constant;
    llvm::SmallVector<llvm::Constant *, 8> Values(constant->getNumOperands());
    for (unsigned Op = 0, NumOp = constant->getNumOperands(); Op != NumOp; ++Op) {
      auto *OpValue = cast<llvm::Constant>(constant->getOperand(Op));
      Values[Op] = replaceUndef(CGM, isPattern, OpValue);
    }
    if (Ty->isStructTy())
      return llvm::ConstantStruct::get(cast<llvm::StructType>(Ty), Values);
    if (Ty->isArrayTy())
      return llvm::ConstantArray::get(cast<llvm::ArrayType>(Ty), Values);
    assert(Ty->isVectorTy());
    return llvm::ConstantVector::get(Values);
  }
  
  /// EmitAutoVarDecl - Emit code and set up an entry in LocalDeclMap for a
  /// variable declaration with auto, register, or no storage class specifier.
  /// These turn into simple stack objects, or GlobalValues depending on target.
  void CodeGenFunction::EmitAutoVarDecl(const VarDecl &D) {
    AutoVarEmission emission = EmitAutoVarAlloca(D);
    EmitAutoVarInit(emission);
    EmitAutoVarCleanups(emission);
    // No alloca in case of NRVO (named return value optimization) variable.
    if (CGM.getLangOpts().SYCLIsDevice && !D.isNRVOVariable())
      CGM.getSYCLRuntime().actOnAutoVarEmit(
          *this, D, emission.getOriginalAllocatedAddress().getPointer());
  }
  
  /// Emit a lifetime.begin marker if some criteria are satisfied.
  /// \return a pointer to the temporary size Value if a marker was emitted, null
  /// otherwise
  llvm::Value *CodeGenFunction::EmitLifetimeStart(uint64_t Size,
                                                  llvm::Value *Addr) {
    if (!ShouldEmitLifetimeMarkers)
      return nullptr;
  
    assert(Addr->getType()->getPointerAddressSpace() ==
               CGM.getDataLayout().getAllocaAddrSpace() &&
           "Pointer should be in alloca address space");
    llvm::Value *SizeV = llvm::ConstantInt::get(Int64Ty, Size);
    Addr = Builder.CreateBitCast(Addr, AllocaInt8PtrTy);
    llvm::CallInst *C =
        Builder.CreateCall(CGM.getLLVMLifetimeStartFn(), {SizeV, Addr});
    C->setDoesNotThrow();
    return SizeV;
  }
  
  void CodeGenFunction::EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr) {
    assert(Addr->getType()->getPointerAddressSpace() ==
               CGM.getDataLayout().getAllocaAddrSpace() &&
           "Pointer should be in alloca address space");
    Addr = Builder.CreateBitCast(Addr, AllocaInt8PtrTy);
    llvm::CallInst *C =
        Builder.CreateCall(CGM.getLLVMLifetimeEndFn(), {Size, Addr});
    C->setDoesNotThrow();
  }
  
  void CodeGenFunction::EmitAndRegisterVariableArrayDimensions(
      CGDebugInfo *DI, const VarDecl &D, bool EmitDebugInfo) {
    // For each dimension stores its QualType and corresponding
    // size-expression Value.
    SmallVector<CodeGenFunction::VlaSizePair, 4> Dimensions;
    SmallVector<IdentifierInfo *, 4> VLAExprNames;
  
    // Break down the array into individual dimensions.
    QualType Type1D = D.getType();
    while (getContext().getAsVariableArrayType(Type1D)) {
      auto VlaSize = getVLAElements1D(Type1D);
      if (auto *C = dyn_cast<llvm::ConstantInt>(VlaSize.NumElts))
        Dimensions.emplace_back(C, Type1D.getUnqualifiedType());
      else {
        // Generate a locally unique name for the size expression.
        Twine Name = Twine("__vla_expr") + Twine(VLAExprCounter++);
        SmallString<12> Buffer;
        StringRef NameRef = Name.toStringRef(Buffer);
        auto &Ident = getContext().Idents.getOwn(NameRef);
        VLAExprNames.push_back(&Ident);
        auto SizeExprAddr =
            CreateDefaultAlignTempAlloca(VlaSize.NumElts->getType(), NameRef);
        Builder.CreateStore(VlaSize.NumElts, SizeExprAddr);
        Dimensions.emplace_back(SizeExprAddr.getPointer(),
                                Type1D.getUnqualifiedType());
      }
      Type1D = VlaSize.Type;
    }
  
    if (!EmitDebugInfo)
      return;
  
    // Register each dimension's size-expression with a DILocalVariable,
    // so that it can be used by CGDebugInfo when instantiating a DISubrange
    // to describe this array.
    unsigned NameIdx = 0;
    for (auto &VlaSize : Dimensions) {
      llvm::Metadata *MD;
      if (auto *C = dyn_cast<llvm::ConstantInt>(VlaSize.NumElts))
        MD = llvm::ConstantAsMetadata::get(C);
      else {
        // Create an artificial VarDecl to generate debug info for.
        IdentifierInfo *NameIdent = VLAExprNames[NameIdx++];
        auto VlaExprTy = VlaSize.NumElts->getType()->getPointerElementType();
        auto QT = getContext().getIntTypeForBitwidth(
            VlaExprTy->getScalarSizeInBits(), false);
        auto *ArtificialDecl = VarDecl::Create(
            getContext(), const_cast<DeclContext *>(D.getDeclContext()),
            D.getLocation(), D.getLocation(), NameIdent, QT,
            getContext().CreateTypeSourceInfo(QT), SC_Auto);
        ArtificialDecl->setImplicit();
  
        MD = DI->EmitDeclareOfAutoVariable(ArtificialDecl, VlaSize.NumElts,
                                           Builder);
      }
      assert(MD && "No Size expression debug node created");
      DI->registerVLASizeExpression(VlaSize.Type, MD);
    }
  }
  
  /// EmitAutoVarAlloca - Emit the alloca and debug information for a
  /// local variable.  Does not emit initialization or destruction.
  CodeGenFunction::AutoVarEmission
  CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
    QualType Ty = D.getType();
    assert(
        Ty.getAddressSpace() == LangAS::Default ||
        (Ty.getAddressSpace() == LangAS::opencl_private && getLangOpts().OpenCL));
  
    AutoVarEmission emission(D);
  
    bool isEscapingByRef = D.isEscapingByref();
    emission.IsEscapingByRef = isEscapingByRef;
  
    CharUnits alignment = getContext().getDeclAlign(&D);
  
    // If the type is variably-modified, emit all the VLA sizes for it.
    if (Ty->isVariablyModifiedType())
      EmitVariablyModifiedType(Ty);
  
    auto *DI = getDebugInfo();
    bool EmitDebugInfo = DI && CGM.getCodeGenOpts().hasReducedDebugInfo();
  
    Address address = Address::invalid();
    Address AllocaAddr = Address::invalid();
    Address OpenMPLocalAddr = Address::invalid();
    if (CGM.getLangOpts().OpenMPIRBuilder)
      OpenMPLocalAddr = OMPBuilderCBHelpers::getAddressOfLocalVariable(*this, &D);
    else
      OpenMPLocalAddr =
          getLangOpts().OpenMP
              ? CGM.getOpenMPRuntime().getAddressOfLocalVariable(*this, &D)
              : Address::invalid();
  
    bool NRVO = getLangOpts().ElideConstructors && D.isNRVOVariable();
  
    if (getLangOpts().OpenMP && OpenMPLocalAddr.isValid()) {
      address = OpenMPLocalAddr;
    } else if (Ty->isConstantSizeType()) {
      // If this value is an array or struct with a statically determinable
      // constant initializer, there are optimizations we can do.
      //
      // TODO: We should constant-evaluate the initializer of any variable,
      // as long as it is initialized by a constant expression. Currently,
      // isConstantInitializer produces wrong answers for structs with
      // reference or bitfield members, and a few other cases, and checking
      // for POD-ness protects us from some of these.
      if (D.getInit() && (Ty->isArrayType() || Ty->isRecordType()) &&
          (D.isConstexpr() ||
           ((Ty.isPODType(getContext()) ||
             getContext().getBaseElementType(Ty)->isObjCObjectPointerType()) &&
            D.getInit()->isConstantInitializer(getContext(), false)))) {
  
        // If the variable's a const type, and it's neither an NRVO
        // candidate nor a __block variable and has no mutable members,
        // emit it as a global instead.
        // Exception is if a variable is located in non-constant address space
        // in OpenCL.
        if ((!getLangOpts().OpenCL ||
             Ty.getAddressSpace() == LangAS::opencl_constant) &&
            (CGM.getCodeGenOpts().MergeAllConstants && !NRVO &&
             !isEscapingByRef && CGM.isTypeConstant(Ty, true))) {
          EmitStaticVarDecl(D, llvm::GlobalValue::InternalLinkage);
  
          // Signal this condition to later callbacks.
          emission.Addr = Address::invalid();
          assert(emission.wasEmittedAsGlobal());
          return emission;
        }
  
        // Otherwise, tell the initialization code that we're in this case.
        emission.IsConstantAggregate = true;
      }
  
      // A normal fixed sized variable becomes an alloca in the entry block,
      // unless:
      // - it's an NRVO variable.
      // - we are compiling OpenMP and it's an OpenMP local variable.
      if (NRVO) {
        // The named return value optimization: allocate this variable in the
        // return slot, so that we can elide the copy when returning this
        // variable (C++0x [class.copy]p34).
        address = ReturnValue;
  
        if (const RecordType *RecordTy = Ty->getAs<RecordType>()) {
          const auto *RD = RecordTy->getDecl();
          const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD);
          if ((CXXRD && !CXXRD->hasTrivialDestructor()) ||
              RD->isNonTrivialToPrimitiveDestroy()) {
            // Create a flag that is used to indicate when the NRVO was applied
            // to this variable. Set it to zero to indicate that NRVO was not
            // applied.
            llvm::Value *Zero = Builder.getFalse();
            Address NRVOFlag =
              CreateTempAlloca(Zero->getType(), CharUnits::One(), "nrvo");
            EnsureInsertPoint();
            Builder.CreateStore(Zero, NRVOFlag);
  
            // Record the NRVO flag for this variable.
            NRVOFlags[&D] = NRVOFlag.getPointer();
            emission.NRVOFlag = NRVOFlag.getPointer();
          }
        }
      } else {
        CharUnits allocaAlignment;
        llvm::Type *allocaTy;
        if (isEscapingByRef) {
          auto &byrefInfo = getBlockByrefInfo(&D);
          allocaTy = byrefInfo.Type;
          allocaAlignment = byrefInfo.ByrefAlignment;
        } else {
          allocaTy = ConvertTypeForMem(Ty);
          allocaAlignment = alignment;
        }
  
        // Create the alloca.  Note that we set the name separately from
        // building the instruction so that it's there even in no-asserts
        // builds.
        address = CreateTempAlloca(allocaTy, allocaAlignment, D.getName(),
                                   /*ArraySize=*/nullptr, &AllocaAddr);
  
        // Don't emit lifetime markers for MSVC catch parameters. The lifetime of
        // the catch parameter starts in the catchpad instruction, and we can't
        // insert code in those basic blocks.
        bool IsMSCatchParam =
            D.isExceptionVariable() && getTarget().getCXXABI().isMicrosoft();
  
        // Emit a lifetime intrinsic if meaningful. There's no point in doing this
        // if we don't have a valid insertion point (?).
        if (HaveInsertPoint() && !IsMSCatchParam) {
          // If there's a jump into the lifetime of this variable, its lifetime
          // gets broken up into several regions in IR, which requires more work
          // to handle correctly. For now, just omit the intrinsics; this is a
          // rare case, and it's better to just be conservatively correct.
          // PR28267.
          //
          // We have to do this in all language modes if there's a jump past the
          // declaration. We also have to do it in C if there's a jump to an
          // earlier point in the current block because non-VLA lifetimes begin as
          // soon as the containing block is entered, not when its variables
          // actually come into scope; suppressing the lifetime annotations
          // completely in this case is unnecessarily pessimistic, but again, this
          // is rare.
          if (!Bypasses.IsBypassed(&D) &&
              !(!getLangOpts().CPlusPlus && hasLabelBeenSeenInCurrentScope())) {
            llvm::TypeSize size =
                CGM.getDataLayout().getTypeAllocSize(allocaTy);
            emission.SizeForLifetimeMarkers =
                size.isScalable() ? EmitLifetimeStart(-1, AllocaAddr.getPointer())
                                  : EmitLifetimeStart(size.getFixedSize(),
                                                      AllocaAddr.getPointer());
          }
        } else {
          assert(!emission.useLifetimeMarkers());
        }
      }
    } else {
      EnsureInsertPoint();
  
      if (!DidCallStackSave) {
        // Save the stack.
        Address Stack =
          CreateTempAlloca(Int8PtrTy, getPointerAlign(), "saved_stack");
  
        llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::stacksave);
        llvm::Value *V = Builder.CreateCall(F);
        Builder.CreateStore(V, Stack);
  
        DidCallStackSave = true;
  
        // Push a cleanup block and restore the stack there.
        // FIXME: in general circumstances, this should be an EH cleanup.
        pushStackRestore(NormalCleanup, Stack);
      }
  
      auto VlaSize = getVLASize(Ty);
      llvm::Type *llvmTy = ConvertTypeForMem(VlaSize.Type);
  
      // Allocate memory for the array.
      address = CreateTempAlloca(llvmTy, alignment, "vla", VlaSize.NumElts,
                                 &AllocaAddr);
  
      // If we have debug info enabled, properly describe the VLA dimensions for
      // this type by registering the vla size expression for each of the
      // dimensions.
      EmitAndRegisterVariableArrayDimensions(DI, D, EmitDebugInfo);
    }
  
    setAddrOfLocalVar(&D, address);
    emission.Addr = address;
    emission.AllocaAddr = AllocaAddr;
  
    // Emit debug info for local var declaration.
    if (EmitDebugInfo && HaveInsertPoint()) {
      Address DebugAddr = address;
      bool UsePointerValue = NRVO && ReturnValuePointer.isValid();
      DI->setLocation(D.getLocation());
  
      // If NRVO, use a pointer to the return address.
      if (UsePointerValue)
        DebugAddr = ReturnValuePointer;
  
      (void)DI->EmitDeclareOfAutoVariable(&D, DebugAddr.getPointer(), Builder,
                                          UsePointerValue);
    }
  
    // Emit Intel FPGA attribute annotation for a local variable.
    if (getLangOpts().SYCLIsDevice) {
      SmallString<256> AnnotStr;
      CGM.generateIntelFPGAAnnotation(&D, AnnotStr);
      if (!AnnotStr.empty()) {
        llvm::Value *V = address.getPointer();
        EmitAnnotationCall(CGM.getIntrinsic(llvm::Intrinsic::var_annotation),
                           Builder.CreateBitCast(V, CGM.Int8PtrTy, V->getName()),
                           AnnotStr, D.getLocation());
      }
    }
  
    if (D.hasAttr<AnnotateAttr>() && HaveInsertPoint())
      EmitVarAnnotations(&D, address.getPointer());
  
    // Make sure we call @llvm.lifetime.end.
    if (emission.useLifetimeMarkers())
      EHStack.pushCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker,
                                           emission.getOriginalAllocatedAddress(),
                                           emission.getSizeForLifetimeMarkers());
  
    return emission;
  }
  
  static bool isCapturedBy(const VarDecl &, const Expr *);
  
  /// Determines whether the given __block variable is potentially
  /// captured by the given statement.
  static bool isCapturedBy(const VarDecl &Var, const Stmt *S) {
    if (const Expr *E = dyn_cast<Expr>(S))
      return isCapturedBy(Var, E);
    for (const Stmt *SubStmt : S->children())
      if (isCapturedBy(Var, SubStmt))
        return true;
    return false;
  }
  
  /// Determines whether the given __block variable is potentially
  /// captured by the given expression.
  static bool isCapturedBy(const VarDecl &Var, const Expr *E) {
    // Skip the most common kinds of expressions that make
    // hierarchy-walking expensive.
    E = E->IgnoreParenCasts();
  
    if (const BlockExpr *BE = dyn_cast<BlockExpr>(E)) {
      const BlockDecl *Block = BE->getBlockDecl();
      for (const auto &I : Block->captures()) {
        if (I.getVariable() == &Var)
          return true;
      }
  
      // No need to walk into the subexpressions.
      return false;
    }
  
    if (const StmtExpr *SE = dyn_cast<StmtExpr>(E)) {
      const CompoundStmt *CS = SE->getSubStmt();
      for (const auto *BI : CS->body())
        if (const auto *BIE = dyn_cast<Expr>(BI)) {
          if (isCapturedBy(Var, BIE))
            return true;
        }
        else if (const auto *DS = dyn_cast<DeclStmt>(BI)) {
            // special case declarations
            for (const auto *I : DS->decls()) {
                if (const auto *VD = dyn_cast<VarDecl>((I))) {
                  const Expr *Init = VD->getInit();
                  if (Init && isCapturedBy(Var, Init))
                    return true;
                }
            }
        }
        else
          // FIXME. Make safe assumption assuming arbitrary statements cause capturing.
          // Later, provide code to poke into statements for capture analysis.
          return true;
      return false;
    }
  
    for (const Stmt *SubStmt : E->children())
      if (isCapturedBy(Var, SubStmt))
        return true;
  
    return false;
  }
  
  /// Determine whether the given initializer is trivial in the sense
  /// that it requires no code to be generated.
  bool CodeGenFunction::isTrivialInitializer(const Expr *Init) {
    if (!Init)
      return true;
  
    if (const CXXConstructExpr *Construct = dyn_cast<CXXConstructExpr>(Init))
      if (CXXConstructorDecl *Constructor = Construct->getConstructor())
        if (Constructor->isTrivial() &&
            Constructor->isDefaultConstructor() &&
            !Construct->requiresZeroInitialization())
          return true;
  
    return false;
  }
  
  void CodeGenFunction::emitZeroOrPatternForAutoVarInit(QualType type,
                                                        const VarDecl &D,
                                                        Address Loc) {
    auto trivialAutoVarInit = getContext().getLangOpts().getTrivialAutoVarInit();
    CharUnits Size = getContext().getTypeSizeInChars(type);
    bool isVolatile = type.isVolatileQualified();
    if (!Size.isZero()) {
      switch (trivialAutoVarInit) {
      case LangOptions::TrivialAutoVarInitKind::Uninitialized:
        llvm_unreachable("Uninitialized handled by caller");
      case LangOptions::TrivialAutoVarInitKind::Zero:
        if (CGM.stopAutoInit())
          return;
        emitStoresForZeroInit(CGM, D, Loc, isVolatile, Builder);
        break;
      case LangOptions::TrivialAutoVarInitKind::Pattern:
        if (CGM.stopAutoInit())
          return;
        emitStoresForPatternInit(CGM, D, Loc, isVolatile, Builder);
        break;
      }
      return;
    }
  
    // VLAs look zero-sized to getTypeInfo. We can't emit constant stores to
    // them, so emit a memcpy with the VLA size to initialize each element.
    // Technically zero-sized or negative-sized VLAs are undefined, and UBSan
    // will catch that code, but there exists code which generates zero-sized
    // VLAs. Be nice and initialize whatever they requested.
    const auto *VlaType = getContext().getAsVariableArrayType(type);
    if (!VlaType)
      return;
    auto VlaSize = getVLASize(VlaType);
    auto SizeVal = VlaSize.NumElts;
    CharUnits EltSize = getContext().getTypeSizeInChars(VlaSize.Type);
    switch (trivialAutoVarInit) {
    case LangOptions::TrivialAutoVarInitKind::Uninitialized:
      llvm_unreachable("Uninitialized handled by caller");
  
    case LangOptions::TrivialAutoVarInitKind::Zero:
      if (CGM.stopAutoInit())
        return;
      if (!EltSize.isOne())
        SizeVal = Builder.CreateNUWMul(SizeVal, CGM.getSize(EltSize));
      Builder.CreateMemSet(Loc, llvm::ConstantInt::get(Int8Ty, 0), SizeVal,
                           isVolatile);
      break;
  
    case LangOptions::TrivialAutoVarInitKind::Pattern: {
      if (CGM.stopAutoInit())
        return;
      llvm::Type *ElTy = Loc.getElementType();
      llvm::Constant *Constant = constWithPadding(
          CGM, IsPattern::Yes, initializationPatternFor(CGM, ElTy));
      CharUnits ConstantAlign = getContext().getTypeAlignInChars(VlaSize.Type);
      llvm::BasicBlock *SetupBB = createBasicBlock("vla-setup.loop");
      llvm::BasicBlock *LoopBB = createBasicBlock("vla-init.loop");
      llvm::BasicBlock *ContBB = createBasicBlock("vla-init.cont");
      llvm::Value *IsZeroSizedVLA = Builder.CreateICmpEQ(
          SizeVal, llvm::ConstantInt::get(SizeVal->getType(), 0),
          "vla.iszerosized");
      Builder.CreateCondBr(IsZeroSizedVLA, ContBB, SetupBB);
      EmitBlock(SetupBB);
      if (!EltSize.isOne())
        SizeVal = Builder.CreateNUWMul(SizeVal, CGM.getSize(EltSize));
      llvm::Value *BaseSizeInChars =
          llvm::ConstantInt::get(IntPtrTy, EltSize.getQuantity());
      Address Begin = Builder.CreateElementBitCast(Loc, Int8Ty, "vla.begin");
      llvm::Value *End =
          Builder.CreateInBoundsGEP(Begin.getPointer(), SizeVal, "vla.end");
      llvm::BasicBlock *OriginBB = Builder.GetInsertBlock();
      EmitBlock(LoopBB);
      llvm::PHINode *Cur = Builder.CreatePHI(Begin.getType(), 2, "vla.cur");
      Cur->addIncoming(Begin.getPointer(), OriginBB);
      CharUnits CurAlign = Loc.getAlignment().alignmentOfArrayElement(EltSize);
      Builder.CreateMemCpy(Address(Cur, CurAlign),
                           createUnnamedGlobalForMemcpyFrom(
                               CGM, D, Builder, Constant, ConstantAlign),
                           BaseSizeInChars, isVolatile);
      llvm::Value *Next =
          Builder.CreateInBoundsGEP(Int8Ty, Cur, BaseSizeInChars, "vla.next");
      llvm::Value *Done = Builder.CreateICmpEQ(Next, End, "vla-init.isdone");
      Builder.CreateCondBr(Done, ContBB, LoopBB);
      Cur->addIncoming(Next, LoopBB);
      EmitBlock(ContBB);
    } break;
    }
  }
  
  void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) {
    assert(emission.Variable && "emission was not valid!");
  
    // If this was emitted as a global constant, we're done.
    if (emission.wasEmittedAsGlobal()) return;
  
    const VarDecl &D = *emission.Variable;
    auto DL = ApplyDebugLocation::CreateDefaultArtificial(*this, D.getLocation());
    QualType type = D.getType();
  
    // If this local has an initializer, emit it now.
    const Expr *Init = D.getInit();
  
    // If we are at an unreachable point, we don't need to emit the initializer
    // unless it contains a label.
    if (!HaveInsertPoint()) {
      if (!Init || !ContainsLabel(Init)) return;
      EnsureInsertPoint();
    }
  
    // Initialize the structure of a __block variable.
    if (emission.IsEscapingByRef)
      emitByrefStructureInit(emission);
  
    // Initialize the variable here if it doesn't have a initializer and it is a
    // C struct that is non-trivial to initialize or an array containing such a
    // struct.
    if (!Init &&
        type.isNonTrivialToPrimitiveDefaultInitialize() ==
            QualType::PDIK_Struct) {
      LValue Dst = MakeAddrLValue(emission.getAllocatedAddress(), type);
      if (emission.IsEscapingByRef)
        drillIntoBlockVariable(*this, Dst, &D);
      defaultInitNonTrivialCStructVar(Dst);
      return;
    }
  
    // Check whether this is a byref variable that's potentially
    // captured and moved by its own initializer.  If so, we'll need to
    // emit the initializer first, then copy into the variable.
    bool capturedByInit =
        Init && emission.IsEscapingByRef && isCapturedBy(D, Init);
  
    bool locIsByrefHeader = !capturedByInit;
    const Address Loc =
        locIsByrefHeader ? emission.getObjectAddress(*this) : emission.Addr;
  
    // Note: constexpr already initializes everything correctly.
    LangOptions::TrivialAutoVarInitKind trivialAutoVarInit =
        (D.isConstexpr()
             ? LangOptions::TrivialAutoVarInitKind::Uninitialized
             : (D.getAttr<UninitializedAttr>()
                    ? LangOptions::TrivialAutoVarInitKind::Uninitialized
                    : getContext().getLangOpts().getTrivialAutoVarInit()));
  
    auto initializeWhatIsTechnicallyUninitialized = [&](Address Loc) {
      if (trivialAutoVarInit ==
          LangOptions::TrivialAutoVarInitKind::Uninitialized)
        return;
  
      // Only initialize a __block's storage: we always initialize the header.
      if (emission.IsEscapingByRef && !locIsByrefHeader)
        Loc = emitBlockByrefAddress(Loc, &D, /*follow=*/false);
  
      return emitZeroOrPatternForAutoVarInit(type, D, Loc);
    };
  
    if (isTrivialInitializer(Init))
      return initializeWhatIsTechnicallyUninitialized(Loc);
  
    llvm::Constant *constant = nullptr;
    if (emission.IsConstantAggregate ||
        D.mightBeUsableInConstantExpressions(getContext())) {
      assert(!capturedByInit && "constant init contains a capturing block?");
      constant = ConstantEmitter(*this).tryEmitAbstractForInitializer(D);
      if (constant && !constant->isZeroValue() &&
          (trivialAutoVarInit !=
           LangOptions::TrivialAutoVarInitKind::Uninitialized)) {
        IsPattern isPattern =
            (trivialAutoVarInit == LangOptions::TrivialAutoVarInitKind::Pattern)
                ? IsPattern::Yes
                : IsPattern::No;
        // C guarantees that brace-init with fewer initializers than members in
        // the aggregate will initialize the rest of the aggregate as-if it were
        // static initialization. In turn static initialization guarantees that
        // padding is initialized to zero bits. We could instead pattern-init if D
        // has any ImplicitValueInitExpr, but that seems to be unintuitive
        // behavior.
        constant = constWithPadding(CGM, IsPattern::No,
                                    replaceUndef(CGM, isPattern, constant));
      }
    }
  
    if (!constant) {
      initializeWhatIsTechnicallyUninitialized(Loc);
      LValue lv = MakeAddrLValue(Loc, type);
      lv.setNonGC(true);
      return EmitExprAsInit(Init, &D, lv, capturedByInit);
    }
  
    if (!emission.IsConstantAggregate) {
      // For simple scalar/complex initialization, store the value directly.
      LValue lv = MakeAddrLValue(Loc, type);
      lv.setNonGC(true);
      return EmitStoreThroughLValue(RValue::get(constant), lv, true);
    }
  
    llvm::Type *BP = CGM.Int8Ty->getPointerTo(Loc.getAddressSpace());
    emitStoresForConstant(
        CGM, D, (Loc.getType() == BP) ? Loc : Builder.CreateBitCast(Loc, BP),
        type.isVolatileQualified(), Builder, constant);
  }
  
  /// Emit an expression as an initializer for an object (variable, field, etc.)
  /// at the given location.  The expression is not necessarily the normal
  /// initializer for the object, and the address is not necessarily
  /// its normal location.
  ///
  /// \param init the initializing expression
  /// \param D the object to act as if we're initializing
  /// \param lvalue the lvalue to initialize
  /// \param capturedByInit true if \p D is a __block variable
  ///   whose address is potentially changed by the initializer
  void CodeGenFunction::EmitExprAsInit(const Expr *init, const ValueDecl *D,
                                       LValue lvalue, bool capturedByInit) {
    QualType type = D->getType();
  
    if (type->isReferenceType()) {
      RValue rvalue = EmitReferenceBindingToExpr(init);
      if (capturedByInit)
        drillIntoBlockVariable(*this, lvalue, cast<VarDecl>(D));
      EmitStoreThroughLValue(rvalue, lvalue, true);
      return;
    }
    switch (getEvaluationKind(type)) {
    case TEK_Scalar:
      EmitScalarInit(init, D, lvalue, capturedByInit);
      return;
    case TEK_Complex: {
      ComplexPairTy complex = EmitComplexExpr(init);
      if (capturedByInit)
        drillIntoBlockVariable(*this, lvalue, cast<VarDecl>(D));
      EmitStoreOfComplex(complex, lvalue, /*init*/ true);
      return;
    }
    case TEK_Aggregate:
      if (type->isAtomicType()) {
        EmitAtomicInit(const_cast<Expr*>(init), lvalue);
      } else {
        AggValueSlot::Overlap_t Overlap = AggValueSlot::MayOverlap;
        if (isa<VarDecl>(D))
          Overlap = AggValueSlot::DoesNotOverlap;
        else if (auto *FD = dyn_cast<FieldDecl>(D))
          Overlap = getOverlapForFieldInit(FD);
        // TODO: how can we delay here if D is captured by its initializer?
        EmitAggExpr(init, AggValueSlot::forLValue(
                              lvalue, *this, AggValueSlot::IsDestructed,
                              AggValueSlot::DoesNotNeedGCBarriers,
                              AggValueSlot::IsNotAliased, Overlap));
      }
      return;
    }
    llvm_unreachable("bad evaluation kind");
  }
  
  /// Enter a destroy cleanup for the given local variable.
  void CodeGenFunction::emitAutoVarTypeCleanup(
                              const CodeGenFunction::AutoVarEmission &emission,
                              QualType::DestructionKind dtorKind) {
    assert(dtorKind != QualType::DK_none);
  
    // Note that for __block variables, we want to destroy the
    // original stack object, not the possibly forwarded object.
    Address addr = emission.getObjectAddress(*this);
  
    const VarDecl *var = emission.Variable;
    QualType type = var->getType();
  
    CleanupKind cleanupKind = NormalAndEHCleanup;
    CodeGenFunction::Destroyer *destroyer = nullptr;
  
    switch (dtorKind) {
    case QualType::DK_none:
      llvm_unreachable("no cleanup for trivially-destructible variable");
  
    case QualType::DK_cxx_destructor:
      // If there's an NRVO flag on the emission, we need a different
      // cleanup.
      if (emission.NRVOFlag) {
        assert(!type->isArrayType());
        CXXDestructorDecl *dtor = type->getAsCXXRecordDecl()->getDestructor();
        EHStack.pushCleanup<DestroyNRVOVariableCXX>(cleanupKind, addr, type, dtor,
                                                    emission.NRVOFlag);
        return;
      }
      break;
  
    case QualType::DK_objc_strong_lifetime:
      // Suppress cleanups for pseudo-strong variables.
      if (var->isARCPseudoStrong()) return;
  
      // Otherwise, consider whether to use an EH cleanup or not.
      cleanupKind = getARCCleanupKind();
  
      // Use the imprecise destroyer by default.
      if (!var->hasAttr<ObjCPreciseLifetimeAttr>())
        destroyer = CodeGenFunction::destroyARCStrongImprecise;
      break;
  
    case QualType::DK_objc_weak_lifetime:
      break;
  
    case QualType::DK_nontrivial_c_struct:
      destroyer = CodeGenFunction::destroyNonTrivialCStruct;
      if (emission.NRVOFlag) {
        assert(!type->isArrayType());
        EHStack.pushCleanup<DestroyNRVOVariableC>(cleanupKind, addr,
                                                  emission.NRVOFlag, type);
        return;
      }
      break;
    }
  
    // If we haven't chosen a more specific destroyer, use the default.
    if (!destroyer) destroyer = getDestroyer(dtorKind);
  
    // Use an EH cleanup in array destructors iff the destructor itself
    // is being pushed as an EH cleanup.
    bool useEHCleanup = (cleanupKind & EHCleanup);
    EHStack.pushCleanup<DestroyObject>(cleanupKind, addr, type, destroyer,
                                       useEHCleanup);
  }
  
  void CodeGenFunction::EmitAutoVarCleanups(const AutoVarEmission &emission) {
    assert(emission.Variable && "emission was not valid!");
  
    // If this was emitted as a global constant, we're done.
    if (emission.wasEmittedAsGlobal()) return;
  
    // If we don't have an insertion point, we're done.  Sema prevents
    // us from jumping into any of these scopes anyway.
    if (!HaveInsertPoint()) return;
  
    const VarDecl &D = *emission.Variable;
  
    // Check the type for a cleanup.
    if (QualType::DestructionKind dtorKind = D.needsDestruction(getContext()))
      emitAutoVarTypeCleanup(emission, dtorKind);
  
    // In GC mode, honor objc_precise_lifetime.
    if (getLangOpts().getGC() != LangOptions::NonGC &&
        D.hasAttr<ObjCPreciseLifetimeAttr>()) {
      EHStack.pushCleanup<ExtendGCLifetime>(NormalCleanup, &D);
    }
  
    // Handle the cleanup attribute.
    if (const CleanupAttr *CA = D.getAttr<CleanupAttr>()) {
      const FunctionDecl *FD = CA->getFunctionDecl();
  
      llvm::Constant *F = CGM.GetAddrOfFunction(FD);
      assert(F && "Could not find function!");
  
      const CGFunctionInfo &Info = CGM.getTypes().arrangeFunctionDeclaration(FD);
      EHStack.pushCleanup<CallCleanupFunction>(NormalAndEHCleanup, F, &Info, &D);
    }
  
    // If this is a block variable, call _Block_object_destroy
    // (on the unforwarded address). Don't enter this cleanup if we're in pure-GC
    // mode.
    if (emission.IsEscapingByRef &&
        CGM.getLangOpts().getGC() != LangOptions::GCOnly) {
      BlockFieldFlags Flags = BLOCK_FIELD_IS_BYREF;
      if (emission.Variable->getType().isObjCGCWeak())
        Flags |= BLOCK_FIELD_IS_WEAK;
      enterByrefCleanup(NormalAndEHCleanup, emission.Addr, Flags,
                        /*LoadBlockVarAddr*/ false,
                        cxxDestructorCanThrow(emission.Variable->getType()));
    }
  }
  
  CodeGenFunction::Destroyer *
  CodeGenFunction::getDestroyer(QualType::DestructionKind kind) {
    switch (kind) {
    case QualType::DK_none: llvm_unreachable("no destroyer for trivial dtor");
    case QualType::DK_cxx_destructor:
      return destroyCXXObject;
    case QualType::DK_objc_strong_lifetime:
      return destroyARCStrongPrecise;
    case QualType::DK_objc_weak_lifetime:
      return destroyARCWeak;
    case QualType::DK_nontrivial_c_struct:
      return destroyNonTrivialCStruct;
    }
    llvm_unreachable("Unknown DestructionKind");
  }
  
  /// pushEHDestroy - Push the standard destructor for the given type as
  /// an EH-only cleanup.
  void CodeGenFunction::pushEHDestroy(QualType::DestructionKind dtorKind,
                                      Address addr, QualType type) {
    assert(dtorKind && "cannot push destructor for trivial type");
    assert(needsEHCleanup(dtorKind));
  
    pushDestroy(EHCleanup, addr, type, getDestroyer(dtorKind), true);
  }
  
  /// pushDestroy - Push the standard destructor for the given type as
  /// at least a normal cleanup.
  void CodeGenFunction::pushDestroy(QualType::DestructionKind dtorKind,
                                    Address addr, QualType type) {
    assert(dtorKind && "cannot push destructor for trivial type");
  
    CleanupKind cleanupKind = getCleanupKind(dtorKind);
    pushDestroy(cleanupKind, addr, type, getDestroyer(dtorKind),
                cleanupKind & EHCleanup);
  }
  
  void CodeGenFunction::pushDestroy(CleanupKind cleanupKind, Address addr,
                                    QualType type, Destroyer *destroyer,
                                    bool useEHCleanupForArray) {
    pushFullExprCleanup<DestroyObject>(cleanupKind, addr, type,
                                       destroyer, useEHCleanupForArray);
  }
  
  void CodeGenFunction::pushStackRestore(CleanupKind Kind, Address SPMem) {
    EHStack.pushCleanup<CallStackRestore>(Kind, SPMem);
  }
  
  void CodeGenFunction::pushLifetimeExtendedDestroy(
      CleanupKind cleanupKind, Address addr, QualType type,
      Destroyer *destroyer, bool useEHCleanupForArray) {
    // Push an EH-only cleanup for the object now.
    // FIXME: When popping normal cleanups, we need to keep this EH cleanup
    // around in case a temporary's destructor throws an exception.
    if (cleanupKind & EHCleanup)
      EHStack.pushCleanup<DestroyObject>(
          static_cast<CleanupKind>(cleanupKind & ~NormalCleanup), addr, type,
          destroyer, useEHCleanupForArray);
  
    // Remember that we need to push a full cleanup for the object at the
    // end of the full-expression.
    pushCleanupAfterFullExpr<DestroyObject>(
        cleanupKind, addr, type, destroyer, useEHCleanupForArray);
  }
  
  /// emitDestroy - Immediately perform the destruction of the given
  /// object.
  ///
  /// \param addr - the address of the object; a type*
  /// \param type - the type of the object; if an array type, all
  ///   objects are destroyed in reverse order
  /// \param destroyer - the function to call to destroy individual
  ///   elements
  /// \param useEHCleanupForArray - whether an EH cleanup should be
  ///   used when destroying array elements, in case one of the
  ///   destructions throws an exception
  void CodeGenFunction::emitDestroy(Address addr, QualType type,
                                    Destroyer *destroyer,
                                    bool useEHCleanupForArray) {
    const ArrayType *arrayType = getContext().getAsArrayType(type);
    if (!arrayType)
      return destroyer(*this, addr, type);
  
    llvm::Value *length = emitArrayLength(arrayType, type, addr);
  
    CharUnits elementAlign =
      addr.getAlignment()
          .alignmentOfArrayElement(getContext().getTypeSizeInChars(type));
  
    // Normally we have to check whether the array is zero-length.
    bool checkZeroLength = true;
  
    // But if the array length is constant, we can suppress that.
    if (llvm::ConstantInt *constLength = dyn_cast<llvm::ConstantInt>(length)) {
      // ...and if it's constant zero, we can just skip the entire thing.
      if (constLength->isZero()) return;
      checkZeroLength = false;
    }
  
    llvm::Value *begin = addr.getPointer();
    llvm::Value *end = Builder.CreateInBoundsGEP(begin, length);
    emitArrayDestroy(begin, end, type, elementAlign, destroyer,
                     checkZeroLength, useEHCleanupForArray);
  }
  
  /// emitArrayDestroy - Destroys all the elements of the given array,
  /// beginning from last to first.  The array cannot be zero-length.
  ///
  /// \param begin - a type* denoting the first element of the array
  /// \param end - a type* denoting one past the end of the array
  /// \param elementType - the element type of the array
  /// \param destroyer - the function to call to destroy elements
  /// \param useEHCleanup - whether to push an EH cleanup to destroy
  ///   the remaining elements in case the destruction of a single
  ///   element throws
  void CodeGenFunction::emitArrayDestroy(llvm::Value *begin,
                                         llvm::Value *end,
                                         QualType elementType,
                                         CharUnits elementAlign,
                                         Destroyer *destroyer,
                                         bool checkZeroLength,
                                         bool useEHCleanup) {
    assert(!elementType->isArrayType());
  
    // The basic structure here is a do-while loop, because we don't
    // need to check for the zero-element case.
    llvm::BasicBlock *bodyBB = createBasicBlock("arraydestroy.body");
    llvm::BasicBlock *doneBB = createBasicBlock("arraydestroy.done");
  
    if (checkZeroLength) {
      llvm::Value *isEmpty = Builder.CreateICmpEQ(begin, end,
                                                  "arraydestroy.isempty");
      Builder.CreateCondBr(isEmpty, doneBB, bodyBB);
    }
  
    // Enter the loop body, making that address the current address.
    llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
    EmitBlock(bodyBB);
    llvm::PHINode *elementPast =
      Builder.CreatePHI(begin->getType(), 2, "arraydestroy.elementPast");
    elementPast->addIncoming(end, entryBB);
  
    // Shift the address back by one element.
    llvm::Value *negativeOne = llvm::ConstantInt::get(SizeTy, -1, true);
    llvm::Value *element = Builder.CreateInBoundsGEP(elementPast, negativeOne,
                                                     "arraydestroy.element");
  
    if (useEHCleanup)
      pushRegularPartialArrayCleanup(begin, element, elementType, elementAlign,
                                     destroyer);
  
    // Perform the actual destruction there.
    destroyer(*this, Address(element, elementAlign), elementType);
  
    if (useEHCleanup)
      PopCleanupBlock();
  
    // Check whether we've reached the end.
    llvm::Value *done = Builder.CreateICmpEQ(element, begin, "arraydestroy.done");
    Builder.CreateCondBr(done, doneBB, bodyBB);
    elementPast->addIncoming(element, Builder.GetInsertBlock());
  
    // Done.
    EmitBlock(doneBB);
  }
  
  /// Perform partial array destruction as if in an EH cleanup.  Unlike
  /// emitArrayDestroy, the element type here may still be an array type.
  static void emitPartialArrayDestroy(CodeGenFunction &CGF,
                                      llvm::Value *begin, llvm::Value *end,
                                      QualType type, CharUnits elementAlign,
                                      CodeGenFunction::Destroyer *destroyer) {
    // If the element type is itself an array, drill down.
    unsigned arrayDepth = 0;
    while (const ArrayType *arrayType = CGF.getContext().getAsArrayType(type)) {
      // VLAs don't require a GEP index to walk into.
      if (!isa<VariableArrayType>(arrayType))
        arrayDepth++;
      type = arrayType->getElementType();
    }
  
    if (arrayDepth) {
      llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
  
      SmallVector<llvm::Value*,4> gepIndices(arrayDepth+1, zero);
      begin = CGF.Builder.CreateInBoundsGEP(begin, gepIndices, "pad.arraybegin");
      end = CGF.Builder.CreateInBoundsGEP(end, gepIndices, "pad.arrayend");
    }
  
    // Destroy the array.  We don't ever need an EH cleanup because we
    // assume that we're in an EH cleanup ourselves, so a throwing
    // destructor causes an immediate terminate.
    CGF.emitArrayDestroy(begin, end, type, elementAlign, destroyer,
                         /*checkZeroLength*/ true, /*useEHCleanup*/ false);
  }
  
  namespace {
    /// RegularPartialArrayDestroy - a cleanup which performs a partial
    /// array destroy where the end pointer is regularly determined and
    /// does not need to be loaded from a local.
    class RegularPartialArrayDestroy final : public EHScopeStack::Cleanup {
      llvm::Value *ArrayBegin;
      llvm::Value *ArrayEnd;
      QualType ElementType;
      CodeGenFunction::Destroyer *Destroyer;
      CharUnits ElementAlign;
    public:
      RegularPartialArrayDestroy(llvm::Value *arrayBegin, llvm::Value *arrayEnd,
                                 QualType elementType, CharUnits elementAlign,
                                 CodeGenFunction::Destroyer *destroyer)
        : ArrayBegin(arrayBegin), ArrayEnd(arrayEnd),
          ElementType(elementType), Destroyer(destroyer),
          ElementAlign(elementAlign) {}
  
      void Emit(CodeGenFunction &CGF, Flags flags) override {
        emitPartialArrayDestroy(CGF, ArrayBegin, ArrayEnd,
                                ElementType, ElementAlign, Destroyer);
      }
    };
  
    /// IrregularPartialArrayDestroy - a cleanup which performs a
    /// partial array destroy where the end pointer is irregularly
    /// determined and must be loaded from a local.
    class IrregularPartialArrayDestroy final : public EHScopeStack::Cleanup {
      llvm::Value *ArrayBegin;
      Address ArrayEndPointer;
      QualType ElementType;
      CodeGenFunction::Destroyer *Destroyer;
      CharUnits ElementAlign;
    public:
      IrregularPartialArrayDestroy(llvm::Value *arrayBegin,
                                   Address arrayEndPointer,
                                   QualType elementType,
                                   CharUnits elementAlign,
                                   CodeGenFunction::Destroyer *destroyer)
        : ArrayBegin(arrayBegin), ArrayEndPointer(arrayEndPointer),
          ElementType(elementType), Destroyer(destroyer),
          ElementAlign(elementAlign) {}
  
      void Emit(CodeGenFunction &CGF, Flags flags) override {
        llvm::Value *arrayEnd = CGF.Builder.CreateLoad(ArrayEndPointer);
        emitPartialArrayDestroy(CGF, ArrayBegin, arrayEnd,
                                ElementType, ElementAlign, Destroyer);
      }
    };
  } // end anonymous namespace
  
  /// pushIrregularPartialArrayCleanup - Push an EH cleanup to destroy
  /// already-constructed elements of the given array.  The cleanup
  /// may be popped with DeactivateCleanupBlock or PopCleanupBlock.
  ///
  /// \param elementType - the immediate element type of the array;
  ///   possibly still an array type
  void CodeGenFunction::pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
                                                         Address arrayEndPointer,
                                                         QualType elementType,
                                                         CharUnits elementAlign,
                                                         Destroyer *destroyer) {
    pushFullExprCleanup<IrregularPartialArrayDestroy>(EHCleanup,
                                                      arrayBegin, arrayEndPointer,
                                                      elementType, elementAlign,
                                                      destroyer);
  }
  
  /// pushRegularPartialArrayCleanup - Push an EH cleanup to destroy
  /// already-constructed elements of the given array.  The cleanup
  /// may be popped with DeactivateCleanupBlock or PopCleanupBlock.
  ///
  /// \param elementType - the immediate element type of the array;
  ///   possibly still an array type
  void CodeGenFunction::pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,
                                                       llvm::Value *arrayEnd,
                                                       QualType elementType,
                                                       CharUnits elementAlign,
                                                       Destroyer *destroyer) {
    pushFullExprCleanup<RegularPartialArrayDestroy>(EHCleanup,
                                                    arrayBegin, arrayEnd,
                                                    elementType, elementAlign,
                                                    destroyer);
  }
  
  /// Lazily declare the @llvm.lifetime.start intrinsic.
  llvm::Function *CodeGenModule::getLLVMLifetimeStartFn() {
    if (LifetimeStartFn)
      return LifetimeStartFn;
    LifetimeStartFn = llvm::Intrinsic::getDeclaration(&getModule(),
      llvm::Intrinsic::lifetime_start, AllocaInt8PtrTy);
    return LifetimeStartFn;
  }
  
  /// Lazily declare the @llvm.lifetime.end intrinsic.
  llvm::Function *CodeGenModule::getLLVMLifetimeEndFn() {
    if (LifetimeEndFn)
      return LifetimeEndFn;
    LifetimeEndFn = llvm::Intrinsic::getDeclaration(&getModule(),
      llvm::Intrinsic::lifetime_end, AllocaInt8PtrTy);
    return LifetimeEndFn;
  }
  
  namespace {
    /// A cleanup to perform a release of an object at the end of a
    /// function.  This is used to balance out the incoming +1 of a
    /// ns_consumed argument when we can't reasonably do that just by
    /// not doing the initial retain for a __block argument.
    struct ConsumeARCParameter final : EHScopeStack::Cleanup {
      ConsumeARCParameter(llvm::Value *param,
                          ARCPreciseLifetime_t precise)
        : Param(param), Precise(precise) {}
  
      llvm::Value *Param;
      ARCPreciseLifetime_t Precise;
  
      void Emit(CodeGenFunction &CGF, Flags flags) override {
        CGF.EmitARCRelease(Param, Precise);
      }
    };
  } // end anonymous namespace
  
  /// Emit an alloca (or GlobalValue depending on target)
  /// for the specified parameter and set up LocalDeclMap.
  void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
                                     unsigned ArgNo) {
    // FIXME: Why isn't ImplicitParamDecl a ParmVarDecl?
    assert((isa<ParmVarDecl>(D) || isa<ImplicitParamDecl>(D)) &&
           "Invalid argument to EmitParmDecl");
  
    Arg.getAnyValue()->setName(D.getName());
  
    QualType Ty = D.getType();
  
    // Use better IR generation for certain implicit parameters.
    if (auto IPD = dyn_cast<ImplicitParamDecl>(&D)) {
      // The only implicit argument a block has is its literal.
      // This may be passed as an inalloca'ed value on Windows x86.
      if (BlockInfo) {
        llvm::Value *V = Arg.isIndirect()
                             ? Builder.CreateLoad(Arg.getIndirectAddress())
                             : Arg.getDirectValue();
        setBlockContextParameter(IPD, ArgNo, V);
        return;
      }
    }
  
    Address DeclPtr = Address::invalid();
    bool DoStore = false;
    bool IsScalar = hasScalarEvaluationKind(Ty);
    // If we already have a pointer to the argument, reuse the input pointer.
    if (Arg.isIndirect()) {
      DeclPtr = Arg.getIndirectAddress();
      // If we have a prettier pointer type at this point, bitcast to that.
      unsigned AS = DeclPtr.getType()->getAddressSpace();
      llvm::Type *IRTy = ConvertTypeForMem(Ty)->getPointerTo(AS);
      if (DeclPtr.getType() != IRTy)
        DeclPtr = Builder.CreateBitCast(DeclPtr, IRTy, D.getName());
      // Indirect argument is in alloca address space, which may be different
      // from the default address space.
      auto AllocaAS = CGM.getASTAllocaAddressSpace();
      auto *V = DeclPtr.getPointer();
      auto SrcLangAS = getLangOpts().OpenCL ? LangAS::opencl_private : AllocaAS;
      auto DestLangAS =
          getLangOpts().OpenCL ? LangAS::opencl_private : LangAS::Default;
      if (SrcLangAS != DestLangAS) {
        assert(getContext().getTargetAddressSpace(SrcLangAS) ==
               CGM.getDataLayout().getAllocaAddrSpace());
        auto DestAS = getContext().getTargetAddressSpace(DestLangAS);
        auto *T = V->getType()->getPointerElementType()->getPointerTo(DestAS);
        DeclPtr = Address(getTargetHooks().performAddrSpaceCast(
                              *this, V, SrcLangAS, DestLangAS, T, true),
                          DeclPtr.getAlignment());
      }
  
      // Push a destructor cleanup for this parameter if the ABI requires it.
      // Don't push a cleanup in a thunk for a method that will also emit a
      // cleanup.
      if (hasAggregateEvaluationKind(Ty) && !CurFuncIsThunk &&
          Ty->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) {
        if (QualType::DestructionKind DtorKind =
                D.needsDestruction(getContext())) {
          assert((DtorKind == QualType::DK_cxx_destructor ||
                  DtorKind == QualType::DK_nontrivial_c_struct) &&
                 "unexpected destructor type");
          pushDestroy(DtorKind, DeclPtr, Ty);
          CalleeDestructedParamCleanups[cast<ParmVarDecl>(&D)] =
              EHStack.stable_begin();
        }
      }
    } else {
      // Check if the parameter address is controlled by OpenMP runtime.
      Address OpenMPLocalAddr =
          getLangOpts().OpenMP
              ? CGM.getOpenMPRuntime().getAddressOfLocalVariable(*this, &D)
              : Address::invalid();
      if (getLangOpts().OpenMP && OpenMPLocalAddr.isValid()) {
        DeclPtr = OpenMPLocalAddr;
      } else {
        // Otherwise, create a temporary to hold the value.
        DeclPtr = CreateMemTemp(Ty, getContext().getDeclAlign(&D),
                                D.getName() + ".addr");
      }
      DoStore = true;
    }
  
    llvm::Value *ArgVal = (DoStore ? Arg.getDirectValue() : nullptr);
  
    LValue lv = MakeAddrLValue(DeclPtr, Ty);
    if (IsScalar) {
      Qualifiers qs = Ty.getQualifiers();
      if (Qualifiers::ObjCLifetime lt = qs.getObjCLifetime()) {
        // We honor __attribute__((ns_consumed)) for types with lifetime.
        // For __strong, it's handled by just skipping the initial retain;
        // otherwise we have to balance out the initial +1 with an extra
        // cleanup to do the release at the end of the function.
        bool isConsumed = D.hasAttr<NSConsumedAttr>();
  
        // If a parameter is pseudo-strong then we can omit the implicit retain.
        if (D.isARCPseudoStrong()) {
          assert(lt == Qualifiers::OCL_Strong &&
                 "pseudo-strong variable isn't strong?");
          assert(qs.hasConst() && "pseudo-strong variable should be const!");
          lt = Qualifiers::OCL_ExplicitNone;
        }
  
        // Load objects passed indirectly.
        if (Arg.isIndirect() && !ArgVal)
          ArgVal = Builder.CreateLoad(DeclPtr);
  
        if (lt == Qualifiers::OCL_Strong) {
          if (!isConsumed) {
            if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
              // use objc_storeStrong(&dest, value) for retaining the
              // object. But first, store a null into 'dest' because
              // objc_storeStrong attempts to release its old value.
              llvm::Value *Null = CGM.EmitNullConstant(D.getType());
              EmitStoreOfScalar(Null, lv, /* isInitialization */ true);
              EmitARCStoreStrongCall(lv.getAddress(*this), ArgVal, true);
              DoStore = false;
            }
            else
            // Don't use objc_retainBlock for block pointers, because we
            // don't want to Block_copy something just because we got it
            // as a parameter.
              ArgVal = EmitARCRetainNonBlock(ArgVal);
          }
        } else {
          // Push the cleanup for a consumed parameter.
          if (isConsumed) {
            ARCPreciseLifetime_t precise = (D.hasAttr<ObjCPreciseLifetimeAttr>()
                                  ? ARCPreciseLifetime : ARCImpreciseLifetime);
            EHStack.pushCleanup<ConsumeARCParameter>(getARCCleanupKind(), ArgVal,
                                                     precise);
          }
  
          if (lt == Qualifiers::OCL_Weak) {
            EmitARCInitWeak(DeclPtr, ArgVal);
            DoStore = false; // The weak init is a store, no need to do two.
          }
        }
  
        // Enter the cleanup scope.
        EmitAutoVarWithLifetime(*this, D, DeclPtr, lt);
      }
    }
  
    // Store the initial value into the alloca.
    if (DoStore)
      EmitStoreOfScalar(ArgVal, lv, /* isInitialization */ true);
  
    setAddrOfLocalVar(&D, DeclPtr);
  
    // Emit debug info for param declarations in non-thunk functions.
    if (CGDebugInfo *DI = getDebugInfo()) {
      if (CGM.getCodeGenOpts().hasReducedDebugInfo() && !CurFuncIsThunk) {
        DI->EmitDeclareOfArgVariable(&D, DeclPtr.getPointer(), ArgNo, Builder);
      }
    }
  
    if (D.hasAttr<AnnotateAttr>())
      EmitVarAnnotations(&D, DeclPtr.getPointer());
  
    // We can only check return value nullability if all arguments to the
    // function satisfy their nullability preconditions. This makes it necessary
    // to emit null checks for args in the function body itself.
    if (requiresReturnValueNullabilityCheck()) {
      auto Nullability = Ty->getNullability(getContext());
      if (Nullability && *Nullability == NullabilityKind::NonNull) {
        SanitizerScope SanScope(this);
        RetValNullabilityPrecondition =
            Builder.CreateAnd(RetValNullabilityPrecondition,
                              Builder.CreateIsNotNull(Arg.getAnyValue()));
      }
    }
  }
  
  void CodeGenModule::EmitOMPDeclareReduction(const OMPDeclareReductionDecl *D,
                                              CodeGenFunction *CGF) {
    if (!LangOpts.OpenMP || (!LangOpts.EmitAllDecls && !D->isUsed()))
      return;
    getOpenMPRuntime().emitUserDefinedReduction(CGF, D);
  }
  
  void CodeGenModule::EmitOMPDeclareMapper(const OMPDeclareMapperDecl *D,
                                           CodeGenFunction *CGF) {
    if (!LangOpts.OpenMP || LangOpts.OpenMPSimd ||
        (!LangOpts.EmitAllDecls && !D->isUsed()))
      return;
    getOpenMPRuntime().emitUserDefinedMapper(D, CGF);
  }
  
  void CodeGenModule::EmitOMPRequiresDecl(const OMPRequiresDecl *D) {
    getOpenMPRuntime().processRequiresDirective(D);
  }
  