<s> if ( CI -> isCallee ( & U ) ) Printfs . push_back ( CI ) ; } } if ( Printfs . empty ( ) ) return false ; if ( auto HostcallFunction = M . getFunction ( <STR_LIT> ) ) { for ( auto & U : HostcallFunction -> uses ( ) ) { if ( auto * CI = dyn_cast < CallInst > ( U . getUser ( ) ) ) { M . getContext ( ) . emitError ( CI , <STR_LIT> ) ; } } } TD = & M . getDataLayout ( ) ; auto DTWP = getAnalysisIfAvailable < DominatorTreeWrapperPass > ( ) ; DT = DTWP ? & DTWP -> getDomTree ( ) : nullptr ; auto GetTLI = [ this ] ( Function & F ) -> TargetLibraryInfo & {

return this -> getAnalysis < TargetLibraryInfoWrapperPass > ( ) . getTLI ( F ) ;


<s> if ( Specifier != 's' ) return false ; const PointerType * PT = dyn_cast < PointerType > ( OpType ) ; if ( ! PT || PT -> getAddressSpace ( ) != <ISA_LIT>AS :: CONSTANT_ADDRESS ) return false ;

Type * ElemType = PT -> getContainedType ( <NUM_LIT> ) ;


<s> bool <ISA_LIT>PrintfRuntimeBinding :: runOnModule ( Module & M ) { Triple TT ( M . getTargetTriple ( ) ) ; if ( TT . getArch ( ) == Triple :: r600 ) return false ; visit ( M ) ; if ( Printfs . empty ( ) ) return false ; TD = & M . getDataLayout ( ) ; auto DTWP = getAnalysisIfAvailable < DominatorTreeWrapperPass > ( ) ; DT = DTWP ? & DTWP -> getDomTree ( ) : nullptr ; TLI = & getAnalysis < TargetLibraryInfoWrapperPass > ( ) . getTLI ( ) ;


<s> Triple TT ( M . getTargetTriple ( ) ) ; if ( TT . getArch ( ) == Triple :: r600 ) return false ; visit ( M ) ; if ( Printfs . empty ( ) ) return false ; TD = & M . getDataLayout ( ) ; auto DTWP = getAnalysisIfAvailable < DominatorTreeWrapperPass > ( ) ; DT = DTWP ? & DTWP -> getDomTree ( ) : nullptr ; TLI = & getAnalysis < TargetLibraryInfoWrapperPass > ( ) . getTLI ( ) ; return lowerPrintfForGpu ( M ) ;


<s> Value * simplify ( Instruction * I ) {

return SimplifyInstruction ( I , { * TD , TLI , DT } ) ;


<s> Function * F = CS . getCalledFunction ( ) ; if ( F && F -> hasName ( ) && F -> getName ( ) == <STR_LIT> ) Printfs . push_back ( CS . getInstruction ( ) ) ;


<s> if ( F && F -> hasName ( ) && F -> getName ( ) == <STR_LIT> ) Printfs . push_back ( CS . getInstruction ( ) ) ;


<s> auto DTWP = getAnalysisIfAvailable < DominatorTreeWrapperPass > ( ) ; DT = DTWP ? & DTWP -> getDomTree ( ) : nullptr ; auto GetTLI = [ this ] ( Function & F ) -> TargetLibraryInfo & { return this -> getAnalysis < TargetLibraryInfoWrapperPass > ( ) . getTLI ( F ) ; } ; return lowerPrintfForGpu ( M , GetTLI ) ;


<s> auto PrintfFunction = M . getFunction ( <STR_LIT> ) ; if ( ! PrintfFunction ) return false ; for ( auto & U : PrintfFunction -> uses ( ) ) { if ( auto * CI = dyn_cast < CallInst > ( U . getUser ( ) ) ) { if ( CI -> isCallee ( & U ) ) Printfs . push_back ( CI ) ; } }

if ( Printfs . empty ( ) ) return false ;


<s> <ISA_LIT>PrintfRuntimeBinding :: <ISA_LIT>PrintfRuntimeBinding ( ) : ModulePass ( ID ) {


<s> <ISA_LIT>PrintfRuntimeBindingImpl ( function_ref < const DominatorTree & ( Function & ) > GetDT , function_ref < const TargetLibraryInfo & ( Function & ) > GetTLI ) : GetDT ( GetDT ) , GetTLI ( GetTLI ) {


<s> ModulePass * create<ISA_LIT>PrintfRuntimeBinding ( ) {

return new <ISA_LIT>PrintfRuntimeBinding ( ) ;


<s> ModulePass * create<ISA_LIT>PrintfRuntimeBinding ( ) { return new <ISA_LIT>PrintfRuntimeBinding ( ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < TargetLibraryInfoWrapperPass > ( ) ;

AU . addRequired < DominatorTreeWrapperPass > ( ) ;


<s> AU . addRequired < TargetLibraryInfoWrapperPass > ( ) ;


<s> void <ISA_LIT>PrintfRuntimeBindingImpl :: getConversionSpecifiers ( SmallVectorImpl < char > & OpConvSpecifiers , StringRef Fmt , size_t NumOps ) const { static const char ConvSpecifiers [ ] = <STR_LIT> ; size_t CurFmtSpecifierIdx = <NUM_LIT> ; size_t PrevFmtSpecifierIdx = <NUM_LIT> ; while ( ( CurFmtSpecifierIdx = Fmt . find_first_of ( ConvSpecifiers , CurFmtSpecifierIdx ) ) != StringRef :: npos ) {


<s> ArgDump = true ; while ( pTag && CurFmt [ -- pTag ] == '%' ) { ArgDump = ! ArgDump ; } } if ( ArgDump ) OpConvSpecifiers . push_back ( Fmt [ CurFmtSpecifierIdx ] ) ; PrevFmtSpecifierIdx = ++ CurFmtSpecifierIdx ; }


<s> } ; auto GetTLI = [ & FAM ] ( Function & F ) -> TargetLibraryInfo & { return FAM . getResult < TargetLibraryAnalysis > ( F ) ; } ; bool Changed = <ISA_LIT>PrintfRuntimeBindingImpl ( GetDT , GetTLI ) . run ( M ) ;

return Changed ? PreservedAnalyses :: none ( ) : PreservedAnalyses :: all ( ) ;


<s> return this -> getAnalysis < DominatorTreeWrapperPass > ( F ) . getDomTree ( ) ; } ;

auto GetTLI = [ this ] ( Function & F ) -> TargetLibraryInfo & {


<s> if ( Specifier != 's' ) return false ; const PointerType * PT = dyn_cast < PointerType > ( OpType ) ; if ( ! PT || PT -> getAddressSpace ( ) != <ISA_LIT>AS :: CONSTANT_ADDRESS ) return false ; Type * ElemType = PT -> getContainedType ( <NUM_LIT> ) ;

if ( ElemType -> getTypeID ( ) != Type :: IntegerTyID ) return false ;


<s> if ( Specifier != 's' ) return false ; const PointerType * PT = dyn_cast < PointerType > ( OpType ) ;

if ( ! PT || PT -> getAddressSpace ( ) != <ISA_LIT>AS :: CONSTANT_ADDRESS ) return false ;


<s> Value * simplify ( Instruction * I , const TargetLibraryInfo * TLI , const DominatorTree * DT ) {


<s> return SimplifyInstruction ( I , { * TD , TLI , DT } ) ;


<s> <ISA_LIT>PromoteAlloca ( const <ISA_LIT>Subtarget & st ) : FunctionPass ( ID ) , ST ( st ) , LocalMemAvailable ( <NUM_LIT> ) {


<s> static VectorType * arrayTypeToVecType ( Type * ArrayTy ) { return VectorType :: get ( ArrayTy -> getArrayElementType ( ) , ArrayTy -> getArrayNumElements ( ) ) ;


<s> static Value * calculateVectorIndex ( Value * Ptr , const std :: map < GetElementPtrInst * , Value * > & GEPIdx ) {

if ( isa < AllocaInst > ( Ptr ) ) return Constant :: getNullValue ( Type :: getInt32Ty ( Ptr -> getContext ( ) ) ) ;


<s> switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : case Instruction :: BitCast : case Instruction :: AddrSpaceCast : return true ; case Instruction :: Store : {

StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ;


<s> if ( CallInst * CI = dyn_cast < CallInst > ( User ) ) { if ( ! CI -> getCalledFunction ( ) ) return false ; WorkList . push_back ( User ) ; continue ; } Instruction * UseInst = dyn_cast < Instruction > ( User ) ;

if ( UseInst && UseInst -> getOpcode ( ) == Instruction :: PtrToInt ) return false ;


<s> FunctionPass * llvm :: create<ISA_LIT>PromoteAlloca ( const <ISA_LIT>Subtarget & ST ) { return new <ISA_LIT>PromoteAlloca ( ST ) ;


<s> if ( ! I0 || ! I0 -> isZero ( ) ) return NULL ;

return GEP -> getOperand ( <NUM_LIT> ) ;


<s> static Value * GEPToVectorIndex ( GetElementPtrInst * GEP ) {

if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) return NULL ;


<s> const char * getPassName ( ) const override {


<s> PointerType * GVTy = GV -> getType ( ) ; if ( GVTy -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( Value :: use_iterator U = GV -> use_begin ( ) , UE = GV -> use_end ( ) ; U != UE ; ++ U ) { Instruction * Use = dyn_cast < Instruction > ( * U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) LocalMemAvailable -= Mod -> getDataLayout ( ) . getTypeAllocSize ( GVTy -> getElementType ( ) ) ; }


<s> DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( std :: vector < Value * > :: iterator I = WorkList . begin ( ) , E = WorkList . end ( ) ; I != E ; ++ I ) { Instruction * Inst = cast < Instruction > ( * I ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { Value * Ptr = Inst -> getOperand ( <NUM_LIT> ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VectorTy -> getPointerTo ( <NUM_LIT> ) ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { Value * Ptr = Inst -> getOperand ( <NUM_LIT> ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VectorTy -> getPointerTo ( <NUM_LIT> ) ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * NewVecValue = Builder . CreateInsertElement ( VecValue , Inst -> getOperand ( <NUM_LIT> ) , Index ) ; Builder . CreateStore ( NewVecValue , BitCast ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: BitCast : case Instruction :: AddrSpaceCast : break ; default : Inst -> dump ( ) ; llvm_unreachable ( <STR_LIT> ) ; }


<s> Value * TIdZ = Builder . CreateCall ( ReadTIDIGZ , { } ) ; Value * Tmp0 = Builder . CreateMul ( TCntY , TCntZ ) ; Tmp0 = Builder . CreateMul ( Tmp0 , TIdX ) ; Value * Tmp1 = Builder . CreateMul ( TIdY , TCntZ ) ; Value * TID = Builder . CreateAdd ( Tmp0 , Tmp1 ) ; TID = Builder . CreateAdd ( TID , TIdZ ) ; std :: vector < Value * > Indices ; Indices . push_back ( Constant :: getNullValue ( Type :: getInt32Ty ( Mod -> getContext ( ) ) ) ) ; Indices . push_back ( TID ) ; Value * Offset = Builder . CreateGEP ( GVTy , GV , Indices ) ; I . mutateType ( Offset -> getType ( ) ) ; I . replaceAllUsesWith ( Offset ) ; I . eraseFromParent ( ) ; for ( std :: vector < Value * > :: iterator i = WorkList . begin ( ) , e = WorkList . end ( ) ; i != e ; ++ i ) { Value * V = * i ; CallInst * Call = dyn_cast < CallInst > ( V ) ; if ( ! Call ) { Type * EltTy = V -> getType ( ) -> getPointerElementType ( ) ; PointerType * NewTy = PointerType :: get ( EltTy , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; if ( isa < AddrSpaceCastInst > ( V ) ) continue ; V -> mutateType ( NewTy ) ; continue ; } IntrinsicInst * Intr = dyn_cast < IntrinsicInst > ( Call ) ; if ( ! Intr ) { std :: vector < Type * > ArgTypes ; for ( unsigned ArgIdx = <NUM_LIT> , ArgEnd = Call -> getNumArgOperands ( ) ; ArgIdx != ArgEnd ; ++ ArgIdx ) { ArgTypes . push_back ( Call -> getArgOperand ( ArgIdx ) -> getType ( ) ) ; } Function * F = Call -> getCalledFunction ( ) ; FunctionType * NewType = FunctionType :: get ( Call -> getType ( ) , ArgTypes , F -> isVarArg ( ) ) ; Constant * C = Mod -> getOrInsertFunction ( ( F -> getName ( ) + <STR_LIT> ) . str ( ) , NewType , F -> getAttributes ( ) ) ;

Function * NewF = cast < Function > ( C ) ;


<s> <ISA_LIT>PromoteAllocaImpl ( TargetMachine & TM ) : TM ( TM ) {


<s> if ( Val == OtherOp ) OtherOp = Inst -> getOperand ( OpIdx1 ) ; if ( isa < ConstantPointerNull > ( OtherOp ) ) return true ; Value * OtherObj = getUnderlyingObject ( OtherOp ) ; if ( ! isa < AllocaInst > ( OtherObj ) ) return false ; if ( OtherObj != BaseAlloca ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ;


<s> if ( Val == OtherOp ) OtherOp = Inst -> getOperand ( OpIdx1 ) ; if ( isa < ConstantPointerNull > ( OtherOp ) ) return true ;

Value * OtherObj = getUnderlyingObject ( OtherOp ) ;


<s> CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addRetAttr ( Attribute :: NoAlias ) ; DispatchPtr -> addRetAttr ( Attribute :: NonNull ) ; F . removeFnAttr ( <STR_LIT> ) ; DispatchPtr -> addDereferenceableRetAttr ( <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( I32Ty , GEPXY , Align ( <NUM_LIT> ) ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( I32Ty , GEPZU , Align ( <NUM_LIT> ) ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; ST . makeLIDRangeMetadata ( LoadZU ) ; Value * Y = Builder . CreateLShr ( LoadXY , <NUM_LIT> ) ;

return std :: make_pair ( Y , LoadZU ) ;


<s> AttrName = <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? ( <STR_LIT> ) <STR_LIT> : ( <STR_LIT> ) <STR_LIT> ; AttrName = <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } Function * WorkitemIdFn = <STR_LIT> ( Mod , IntrID ) ; CallInst * CI = Builder . CreateCall ( WorkitemIdFn ) ; ST . makeLIDRangeMetadata ( CI ) ; F -> removeFnAttr ( AttrName ) ; return CI ;


<s> Value * TCntY , * TCntZ ; std :: tie ( TCntY , TCntZ ) = getLocalSizeYZ ( Builder ) ; Value * TIdX = getWorkitemID ( Builder , <NUM_LIT> ) ; Value * TIdY = getWorkitemID ( Builder , <NUM_LIT> ) ; Value * TIdZ = getWorkitemID ( Builder , <NUM_LIT> ) ; Value * Tmp0 = Builder . CreateMul ( TCntY , TCntZ , <STR_LIT> , true , true ) ; Tmp0 = Builder . CreateMul ( Tmp0 , TIdX ) ; Value * Tmp1 = Builder . CreateMul ( TIdY , TCntZ , <STR_LIT> , true , true ) ; Value * TID = Builder . CreateAdd ( Tmp0 , Tmp1 ) ; TID = Builder . CreateAdd ( TID , TIdZ ) ; Value * Indices [ ] = { Constant :: getNullValue ( Type :: getInt32Ty ( Mod -> getContext ( ) ) ) , TID } ; Value * Offset = Builder . CreateInBoundsGEP ( GVTy , GV , Indices ) ; I . mutateType ( Offset -> getType ( ) ) ; I . replaceAllUsesWith ( Offset ) ; I . eraseFromParent ( ) ; SmallVector < IntrinsicInst * > DeferredIntrs ; for ( Value * V : WorkList ) { CallInst * Call = dyn_cast < CallInst > ( V ) ; if ( ! Call ) { if ( ICmpInst * CI = dyn_cast < ICmpInst > ( V ) ) { Value * Src0 = CI -> getOperand ( <NUM_LIT> ) ; PointerType * NewTy = PointerType :: getWithSamePointeeType ( cast < PointerType > ( Src0 -> getType ( ) ) , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; if ( isa < ConstantPointerNull > ( CI -> getOperand ( <NUM_LIT> ) ) ) CI -> setOperand ( <NUM_LIT> , ConstantPointerNull :: get ( NewTy ) ) ; if ( isa < ConstantPointerNull > ( CI -> getOperand ( <NUM_LIT> ) ) ) CI -> setOperand ( <NUM_LIT> , ConstantPointerNull :: get ( NewTy ) ) ; continue ; } if ( isa < AddrSpaceCastInst > ( V ) ) continue ; PointerType * NewTy = PointerType :: getWithSamePointeeType ( cast < PointerType > ( V -> getType ( ) ) , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; V -> mutateType ( NewTy ) ; if ( SelectInst * <ISA_LIT> = dyn_cast < SelectInst > ( V ) ) { if ( isa < ConstantPointerNull > ( <ISA_LIT> -> getOperand ( <NUM_LIT> ) ) ) <ISA_LIT> -> setOperand ( <NUM_LIT> , ConstantPointerNull :: get ( NewTy ) ) ;


<s> bool handlePromoteAllocaToVector ( AllocaInst & I , unsigned MaxVGPRs ) { if ( ! I . isStaticAlloca ( ) || I . isArrayAllocation ( ) ) return false ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << I << '\n' ) ;


<s> bool <ISA_LIT>PromoteAllocaImpl :: hasSufficientLocalMem ( const Function & F ) { FunctionType * FTy = F . getFunctionType ( ) ; const <ISA_LIT>Subtarget & ST = <ISA_LIT>Subtarget :: get ( TM , F ) ; for ( Type * ParamTy : FTy -> params ( ) ) { PointerType * PtrTy = dyn_cast < PointerType > ( ParamTy ) ; if ( PtrTy && PtrTy -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ) { LocalMemLimit = <NUM_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } } LocalMemLimit = ST . getLocalMemorySize ( ) ; if ( LocalMemLimit == <NUM_LIT> ) return false ; SmallVector < const Constant * , <NUM_LIT> > Stack ; SmallPtrSet < const Constant * , <NUM_LIT> > VisitedConstants ; SmallPtrSet < const GlobalVariable * , <NUM_LIT> > UsedLDS ; auto visitUsers = [ & ] ( const GlobalVariable * GV , const Constant * Val ) -> bool { for ( const User * U : Val -> users ( ) ) { if ( const Instruction * Use = dyn_cast < Instruction > ( U ) ) { if ( Use -> getParent ( ) -> getParent ( ) == & F ) return true ; } else { const Constant * C = cast < Constant > ( U ) ; if ( VisitedConstants . insert ( C ) . second ) Stack . push_back ( C ) ; } } return false ; } ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; if ( visitUsers ( & GV , & GV ) ) { UsedLDS . insert ( & GV ) ; Stack . clear ( ) ;


<s> const GCNSubtarget & ST = TM . getSubtarget < GCNSubtarget > ( F ) ; MaxVGPRs = ST . getMaxNumVGPRs ( ST . getWavesPerEU ( F ) . first ) ; if ( ! <STR_LIT> ( F . getCallingConv ( ) ) ) MaxVGPRs = std :: min ( MaxVGPRs , <NUM_LIT> ) ; } else { MaxVGPRs = <NUM_LIT> ; } bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ;

SmallVector < AllocaInst * , <NUM_LIT> > Allocas ;


<s> bool Changed = promoteAllocasToVector ( F , TM ) ; if ( Changed ) { PreservedAnalyses PA ; PA . preserveSet < CFGAnalyses > ( ) ; return PA ;


<s> PA . preserveSet < CFGAnalyses > ( ) ; return PA ; } return PreservedAnalyses :: all ( ) ;


<s> if ( skipFunction ( F ) ) return false ; if ( auto * TPC = getAnalysisIfAvailable < TargetPassConfig > ( ) ) {

return promoteAllocasToVector ( F , TPC -> getTM < TargetMachine > ( ) ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { if ( Inst -> getType ( ) == AllocaTy || Inst -> getType ( ) -> isVectorTy ( ) ) break ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; if ( ! Index ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; if ( Inst -> getType ( ) != VecEltTy ) ExtractElement = Builder . CreateBitOrPointerCast ( ExtractElement , Inst -> getType ( ) ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; if ( <ISA_LIT> -> getValueOperand ( ) -> getType ( ) == AllocaTy || <ISA_LIT> -> getValueOperand ( ) -> getType ( ) -> isVectorTy ( ) ) break ; Value * Ptr = <ISA_LIT> -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; if ( ! Index ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * Elt = <ISA_LIT> -> getValueOperand ( ) ; if ( Elt -> getType ( ) != VecEltTy ) Elt = Builder . CreateBitOrPointerCast ( Elt , VecEltTy ) ; Value * NewVecValue = Builder . CreateInsertElement ( VecValue , Elt , Index ) ; Builder . CreateStore ( NewVecValue , BitCast ) ; Inst -> eraseFromParent ( ) ; break ; } default : llvm_unreachable ( <STR_LIT> ) ; } }


<s> GetElementPtrInst * GEP = cast < GetElementPtrInst > ( Ptr ) ; auto I = GEPIdx . find ( GEP ) ;

return I == GEPIdx . end ( ) ? nullptr : I -> second ;


<s> auto I = GEPIdx . find ( GEP ) ; return I == GEPIdx . end ( ) ? nullptr : I -> second ;


<s> static bool canVectorizeInst ( Instruction * Inst , User * User ) { switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { LoadInst * LI = cast < LoadInst > ( Inst ) ; if ( isa < AllocaInst > ( User ) && LI -> getPointerOperandType ( ) == User -> getType ( ) && isa < VectorType > ( LI -> getType ( ) ) ) return true ; return isa < GetElementPtrInst > ( LI -> getPointerOperand ( ) ) && LI -> isSimple ( ) ; } case Instruction :: BitCast : return true ; case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; if ( isa < AllocaInst > ( User ) && <ISA_LIT> -> getPointerOperandType ( ) == User -> getType ( ) && isa < VectorType > ( <ISA_LIT> -> getValueOperand ( ) -> getType ( ) ) ) return true ; return ( <ISA_LIT> -> getPointerOperand ( ) == User ) && isa < GetElementPtrInst > ( User ) && <ISA_LIT> -> isSimple ( ) ; } default : return false ; }


<s> } case Instruction :: BitCast : return true ; case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; if ( isa < AllocaInst > ( User ) && <ISA_LIT> -> getPointerOperandType ( ) == User -> getType ( ) && isa < VectorType > ( <ISA_LIT> -> getValueOperand ( ) -> getType ( ) ) ) return true ; return ( <ISA_LIT> -> getPointerOperand ( ) == User ) && isa < GetElementPtrInst > ( User ) && <ISA_LIT> -> isSimple ( ) ; } default :


<s> } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ;

LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ;


<s> DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ;

MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ;


<s> case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } Function * WorkitemIdFn = <STR_LIT> ( Mod , IntrID ) ;

CallInst * CI = Builder . CreateCall ( WorkitemIdFn ) ;


<s> for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getType ( ) -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( const User * U : GV . users ( ) ) { const Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) { unsigned Align = GV . getAlignment ( ) ; if ( Align == <NUM_LIT> ) Align = DL . getABITypeAlignment ( GV . getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Align ) ; CurrentLocalMemUsage += AllocSize ; break ; } } } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage , F ) ; unsigned OccupancyHint = ST . getWavesPerEU ( F ) . second ; if ( OccupancyHint == <NUM_LIT> ) OccupancyHint = <NUM_LIT> ; OccupancyHint = std :: min ( OccupancyHint , ST . getMaxWavesPerEU ( ) ) ; MaxOccupancy = std :: min ( OccupancyHint , MaxOccupancy ) ;

unsigned MaxSizeWithWaveCount = ST . getMaxLocalMemSizeWithWaveCount ( MaxOccupancy , F ) ;


<s> bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ; for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ; I != E ; ) { AllocaInst * AI = dyn_cast < AllocaInst > ( I ) ; ++ I ; if ( AI ) Changed |= handleAlloca ( * AI , SufficientLDS ) ; }


<s> BasicBlock & EntryBB = * F . begin ( ) ; for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ; I != E ; ) { AllocaInst * AI = dyn_cast < AllocaInst > ( I ) ; ++ I ;

if ( AI ) Changed |= handleAlloca ( * AI , SufficientLDS ) ;


<s> case Instruction :: Load : { if ( Inst -> getType ( ) == AT ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; if ( <ISA_LIT> -> getValueOperand ( ) -> getType ( ) == AT ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = <ISA_LIT> -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * NewVecValue = Builder . CreateInsertElement ( VecValue , <ISA_LIT> -> getValueOperand ( ) , Index ) ; Builder . CreateStore ( NewVecValue , BitCast ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: BitCast : case Instruction :: AddrSpaceCast : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } return true ;


<s> return false ; } ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; if ( visitUsers ( & GV , & GV ) ) { UsedLDS . insert ( & GV ) ; Stack . clear ( ) ; continue ; } while ( ! Stack . empty ( ) ) { const Constant * C = Stack . pop_back_val ( ) ; if ( visitUsers ( & GV , C ) ) { UsedLDS . insert ( & GV ) ; Stack . clear ( ) ; break ; } } } const DataLayout & DL = Mod -> getDataLayout ( ) ; SmallVector < std :: pair < uint64_t , Align > , <NUM_LIT> > AllocatedSizes ; AllocatedSizes . reserve ( UsedLDS . size ( ) ) ; for ( const GlobalVariable * GV : UsedLDS ) { Align Alignment = DL . getValueOrABITypeAlignment ( GV -> getAlign ( ) , GV -> getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV -> getValueType ( ) ) ; if ( GV -> hasExternalLinkage ( ) && AllocSize == <NUM_LIT> ) { LocalMemLimit = <NUM_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> <STR_LIT> ) ; return false ; } AllocatedSizes . emplace_back ( AllocSize , Alignment ) ; } llvm :: sort ( AllocatedSizes , [ ] ( std :: pair < uint64_t , Align > LHS , std :: pair < uint64_t , Align > RHS ) { return LHS . second < RHS . second ; } ) ;


<s> <ISA_LIT>PromoteAlloca ( const TargetMachine * TM_ = nullptr ) : FunctionPass ( ID ) , TM ( TM_ ) {


<s> if ( ! isa < AllocaInst > ( OtherObj ) ) return false ; if ( OtherObj != BaseAlloca ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } return true ;


<s> Value * OtherObj = GetUnderlyingObject ( OtherOp , * DL ) ; if ( ! isa < AllocaInst > ( OtherObj ) ) return false ; if ( OtherObj != BaseAlloca ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> return new <ISA_LIT>PromoteAlloca ( TM ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>PromoteAlloca ( const TargetMachine * TM ) { return new <ISA_LIT>PromoteAlloca ( TM ) ;


<s> if ( ! TM ) return false ; Mod = & M ; DL = & Mod -> getDataLayout ( ) ;

const Triple & TT = TM -> getTargetTriple ( ) ;


<s> const Triple & TT = TM -> getTargetTriple ( ) ; IsAMDGCN = TT . getArch ( ) == Triple :: amdgcn ; IsAMDHSA = TT . getOS ( ) == Triple :: AMDHSA ; return false ;


<s> Function * LocalSizeYFn = <STR_LIT> ( Mod , <STR_LIT> ) ; Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , AS . CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; ST . makeLIDRangeMetadata ( LoadZU ) ;

Value * Y = Builder . CreateLShr ( LoadXY , <NUM_LIT> ) ;


<s> } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , AS . CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; ST . makeLIDRangeMetadata ( LoadZU ) ; Value * Y = Builder . CreateLShr ( LoadXY , <NUM_LIT> ) ; return std :: make_pair ( Y , LoadZU ) ;


<s> switch ( N ) { case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } Function * WorkitemIdFn = <STR_LIT> ( Mod , IntrID ) ; CallInst * CI = Builder . CreateCall ( WorkitemIdFn ) ;


<s> uint32_t AllocSize = WorkGroupSize * DL . getTypeAllocSize ( AllocaTy ) ; NewSize += AllocSize ; if ( NewSize > LocalMemLimit ) { DEBUG ( dbgs ( ) << <STR_LIT> << AllocSize << <STR_LIT> ) ; return ; } CurrentLocalMemUsage = NewSize ; std :: vector < Value * > WorkList ; if ( ! collectUsesWithPtrTypes ( & I , & I , WorkList ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } DEBUG ( dbgs ( ) << <STR_LIT> ) ; Function * F = I . getParent ( ) -> getParent ( ) ; Type * GVTy = ArrayType :: get ( I . getAllocatedType ( ) , WorkGroupSize ) ; GlobalVariable * GV = new GlobalVariable ( * Mod , GVTy , false , GlobalValue :: InternalLinkage , UndefValue :: get ( GVTy ) , Twine ( F -> getName ( ) ) + Twine ( '.' ) + I . getName ( ) , nullptr , GlobalVariable :: NotThreadLocal , AS . LOCAL_ADDRESS ) ; GV -> setUnnamedAddr ( GlobalValue :: UnnamedAddr :: Global ) ; GV -> setAlignment ( I . getAlignment ( ) ) ; Value * TCntY , * TCntZ ; std :: tie ( TCntY , TCntZ ) = getLocalSizeYZ ( Builder ) ; Value * TIdX = getWorkitemID ( Builder , <NUM_LIT> ) ; Value * TIdY = getWorkitemID ( Builder , <NUM_LIT> ) ; Value * TIdZ = getWorkitemID ( Builder , <NUM_LIT> ) ; Value * Tmp0 = Builder . CreateMul ( TCntY , TCntZ , <STR_LIT> , true , true ) ; Tmp0 = Builder . CreateMul ( Tmp0 , TIdX ) ; Value * Tmp1 = Builder . CreateMul ( TIdY , TCntZ , <STR_LIT> , true , true ) ; Value * TID = Builder . CreateAdd ( Tmp0 , Tmp1 ) ; TID = Builder . CreateAdd ( TID , TIdZ ) ; Value * Indices [ ] = { Constant :: getNullValue ( Type :: getInt32Ty ( Mod -> getContext ( ) ) ) , TID } ; Value * Offset = Builder . CreateInBoundsGEP ( GVTy , GV , Indices ) ; I . mutateType ( Offset -> getType ( ) ) ; I . replaceAllUsesWith ( Offset ) ; I . eraseFromParent ( ) ; for ( Value * V : WorkList ) { CallInst * Call = dyn_cast < CallInst > ( V ) ; if ( ! Call ) { if ( ICmpInst * CI = dyn_cast < ICmpInst > ( V ) ) { Value * Src0 = CI -> getOperand ( <NUM_LIT> ) ; Type * EltTy = Src0 -> getType ( ) -> getPointerElementType ( ) ; PointerType * NewTy = PointerType :: get ( EltTy , AS . LOCAL_ADDRESS ) ; if ( isa < ConstantPointerNull > ( CI -> getOperand ( <NUM_LIT> ) ) ) CI -> setOperand ( <NUM_LIT> , ConstantPointerNull :: get ( NewTy ) ) ;


<s> CurrentLocalMemUsage = <NUM_LIT> ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getType ( ) -> getAddressSpace ( ) != AS . LOCAL_ADDRESS ) continue ; for ( const User * U : GV . users ( ) ) { const Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) { unsigned Align = GV . getAlignment ( ) ; if ( Align == <NUM_LIT> ) Align = DL . getABITypeAlignment ( GV . getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Align ) ; CurrentLocalMemUsage += AllocSize ; break ; } } } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage , F ) ; unsigned OccupancyHint = ST . getWavesPerEU ( F ) . second ; if ( OccupancyHint == <NUM_LIT> ) OccupancyHint = <NUM_LIT> ; OccupancyHint = std :: min ( OccupancyHint , ST . getMaxWavesPerEU ( ) ) ; MaxOccupancy = std :: min ( OccupancyHint , MaxOccupancy ) ; unsigned MaxSizeWithWaveCount = ST . getMaxLocalMemSizeWithWaveCount ( MaxOccupancy , F ) ; if ( CurrentLocalMemUsage > MaxSizeWithWaveCount ) return false ;


<s> if ( Align == <NUM_LIT> ) Align = DL . getABITypeAlignment ( GV . getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Align ) ; CurrentLocalMemUsage += AllocSize ; break ; } } } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage , F ) ; unsigned OccupancyHint = ST . getWavesPerEU ( F ) . second ; if ( OccupancyHint == <NUM_LIT> ) OccupancyHint = <NUM_LIT> ; OccupancyHint = std :: min ( OccupancyHint , ST . getMaxWavesPerEU ( ) ) ; MaxOccupancy = std :: min ( OccupancyHint , MaxOccupancy ) ; unsigned MaxSizeWithWaveCount = ST . getMaxLocalMemSizeWithWaveCount ( MaxOccupancy , F ) ; if ( CurrentLocalMemUsage > MaxSizeWithWaveCount ) return false ; LocalMemLimit = MaxSizeWithWaveCount ; DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << CurrentLocalMemUsage << <STR_LIT> << <STR_LIT> << MaxSizeWithWaveCount << <STR_LIT> << MaxOccupancy << '\n' << <STR_LIT> << ( LocalMemLimit - CurrentLocalMemUsage ) << <STR_LIT> ) ; BasicBlock & EntryBB = * F . begin ( ) ; for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ; I != E ; ) { AllocaInst * AI = dyn_cast < AllocaInst > ( I ) ; ++ I ; if ( AI ) handleAlloca ( * AI ) ; }


<s> for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { Type * VecPtrTy = VectorTy -> getPointerTo ( AS . PRIVATE_ADDRESS ) ; Value * Ptr = Inst -> getOperand ( <NUM_LIT> ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { Type * VecPtrTy = VectorTy -> getPointerTo ( AS . PRIVATE_ADDRESS ) ; Value * Ptr = Inst -> getOperand ( <NUM_LIT> ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * NewVecValue = Builder . CreateInsertElement ( VecValue , Inst -> getOperand ( <NUM_LIT> ) , Index ) ; Builder . CreateStore ( NewVecValue , BitCast ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: BitCast : case Instruction :: AddrSpaceCast : break ; default : llvm_unreachable ( <STR_LIT> ) ; } }


<s> return isa < GetElementPtrInst > ( LI -> getPointerOperand ( ) ) && LI -> isSimple ( ) ; } case Instruction :: BitCast : return true ; case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; return ( <ISA_LIT> -> getPointerOperand ( ) == User ) && isa < GetElementPtrInst > ( User ) && <ISA_LIT> -> isSimple ( ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : return false ;


<s> const <ISA_LIT>Subtarget & ST = TM -> getSubtarget < <ISA_LIT>Subtarget > ( F ) ; if ( ! ST . isPromoteAllocaEnabled ( ) ) return false ; AS = <STR_LIT> ( * F . getParent ( ) ) ; bool SufficientLDS = hasSufficientLocalMem ( F ) ; bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ; for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ; I != E ; ) {

AllocaInst * AI = dyn_cast < AllocaInst > ( I ) ;


<s> if ( ! ST . isPromoteAllocaEnabled ( ) ) return false ; AS = <STR_LIT> ( * F . getParent ( ) ) ; bool SufficientLDS = hasSufficientLocalMem ( F ) ; bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ; for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ; I != E ; ) { AllocaInst * AI = dyn_cast < AllocaInst > ( I ) ; ++ I ; if ( AI ) Changed |= handleAlloca ( * AI , SufficientLDS ) ; } return Changed ;


<s> bool <ISA_LIT>PromoteAlloca :: doInitialization ( Module & M ) { if ( ! TM ) return false ; Mod = & M ; DL = & Mod -> getDataLayout ( ) ; MDBuilder MDB ( Mod -> getContext ( ) ) ; MaxWorkGroupSizeRange = MDB . createRange ( APInt ( <NUM_LIT> , <NUM_LIT> ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ; const Triple & TT = TM -> getTargetTriple ( ) ; IsAMDGCN = TT . getArch ( ) == Triple :: amdgcn ; IsAMDHSA = TT . getOS ( ) == Triple :: AMDHSA ;


<s> LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_range , MaxWorkGroupSizeRange ) ; Value * Y = Builder . CreateLShr ( LoadXY , <NUM_LIT> ) ; return std :: make_pair ( Y , LoadZU ) ;


<s> Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_range , MaxWorkGroupSizeRange ) ; Value * Y = Builder . CreateLShr ( LoadXY , <NUM_LIT> ) ; return std :: make_pair ( Y , LoadZU ) ;


<s> llvm_unreachable ( <STR_LIT> ) ; } Function * WorkitemIdFn = <STR_LIT> ( Mod , IntrID ) ; CallInst * CI = Builder . CreateCall ( WorkitemIdFn ) ; CI -> setMetadata ( LLVMContext :: MD_range , MaxWorkGroupSizeRange ) ; return CI ;


<s> if ( Use -> getParent ( ) -> getParent ( ) == & F ) { unsigned Align = GV . getAlignment ( ) ; if ( Align == <NUM_LIT> ) Align = DL . getABITypeAlignment ( GV . getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Align ) ; CurrentLocalMemUsage += AllocSize ; break ; } } } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage ) ; unsigned OccupancyHint = ST . getWavesPerEU ( F ) . second ; if ( OccupancyHint == <NUM_LIT> ) OccupancyHint = <NUM_LIT> ; OccupancyHint = std :: min ( OccupancyHint , ST . getMaxWavesPerEU ( ) ) ; MaxOccupancy = std :: min ( OccupancyHint , MaxOccupancy ) ; unsigned MaxSizeWithWaveCount = ST . getMaxLocalMemSizeWithWaveCount ( MaxOccupancy ) ; if ( CurrentLocalMemUsage > MaxSizeWithWaveCount ) return false ; LocalMemLimit = MaxSizeWithWaveCount ; DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << CurrentLocalMemUsage << <STR_LIT> << <STR_LIT> << MaxSizeWithWaveCount << <STR_LIT> << MaxOccupancy << '\n' << <STR_LIT> << ( LocalMemLimit - CurrentLocalMemUsage ) << <STR_LIT> ) ; BasicBlock & EntryBB = * F . begin ( ) ; for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ; I != E ; ) {


<s> GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( AllocaUser ) ; if ( ! GEP ) { if ( ! canVectorizeInst ( cast < Instruction > ( AllocaUser ) , Alloca ) ) return false ; WorkList . push_back ( AllocaUser ) ; continue ; } Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( User * GEPUser : AllocaUser -> users ( ) ) { if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) , AllocaUser ) ) return false ; WorkList . push_back ( GEPUser ) ; } } VectorType * VectorTy = arrayTypeToVecType ( AllocaTy ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ;


<s> if ( ! canVectorizeInst ( cast < Instruction > ( AllocaUser ) , Alloca ) ) return false ; WorkList . push_back ( AllocaUser ) ; continue ; } Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( User * GEPUser : AllocaUser -> users ( ) ) { if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) , AllocaUser ) ) return false ; WorkList . push_back ( GEPUser ) ; } } VectorType * VectorTy = arrayTypeToVecType ( AllocaTy ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = Inst -> getOperand ( <NUM_LIT> ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : {


<s> GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( AllocaUser ) ; if ( ! GEP ) { if ( ! canVectorizeInst ( Inst , UseUser , DL ) ) return false ; if ( Inst -> getOpcode ( ) == Instruction :: BitCast ) { Type * FromTy = Inst -> getOperand ( <NUM_LIT> ) -> getType ( ) -> getPointerElementType ( ) ; Type * ToTy = Inst -> getType ( ) -> getPointerElementType ( ) ; if ( FromTy -> isAggregateType ( ) || ToTy -> isAggregateType ( ) || DL . getTypeSizeInBits ( FromTy ) != DL . getTypeSizeInBits ( ToTy ) ) continue ; for ( User * CastUser : Inst -> users ( ) ) { if ( isAssumeLikeIntrinsic ( cast < Instruction > ( CastUser ) ) ) continue ; Users . push_back ( CastUser ) ; UseUsers . push_back ( Inst ) ; } continue ; } WorkList . push_back ( AllocaUser ) ; continue ; } Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; Users . append ( GEP -> user_begin ( ) , GEP -> user_end ( ) ) ; UseUsers . append ( GEP -> getNumUses ( ) , GEP ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { if ( Inst -> getType ( ) == AllocaTy || Inst -> getType ( ) -> isVectorTy ( ) ) break ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; if ( ! Index ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; if ( Inst -> getType ( ) != VecEltTy ) ExtractElement = Builder . CreateBitCast ( ExtractElement , Inst -> getType ( ) ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; if ( <ISA_LIT> -> getValueOperand ( ) -> getType ( ) == AllocaTy || <ISA_LIT> -> getValueOperand ( ) -> getType ( ) -> isVectorTy ( ) ) break ;

Value * Ptr = <ISA_LIT> -> getPointerOperand ( ) ;


<s> <ISA_LIT>PromoteAlloca ( ) : FunctionPass ( ID ) {


<s> <ISA_LIT>PromoteAllocaToVector ( ) : FunctionPass ( ID ) {


<s> return FixedVectorType :: get ( ArrayTy -> getElementType ( ) , ArrayTy -> getNumElements ( ) ) ;


<s> static Value * calculateVectorIndex ( Value * Ptr , const std :: map < GetElementPtrInst * , Value * > & GEPIdx ) {

GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( stripBitcasts ( Ptr ) ) ;


<s> static Value * calculateVectorIndex ( Value * Ptr , const std :: map < GetElementPtrInst * , Value * > & GEPIdx ) { GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( stripBitcasts ( Ptr ) ) ;


<s> static bool canVectorizeInst ( Instruction * Inst , User * User , const DataLayout & DL ) { switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { LoadInst * LI = cast < LoadInst > ( Inst ) ; if ( isa < AllocaInst > ( User ) && LI -> getPointerOperandType ( ) == User -> getType ( ) && isa < VectorType > ( LI -> getType ( ) ) ) return true ; Instruction * PtrInst = dyn_cast < Instruction > ( LI -> getPointerOperand ( ) ) ; if ( ! PtrInst ) return false ; return ( PtrInst -> getOpcode ( ) == Instruction :: GetElementPtr || PtrInst -> getOpcode ( ) == Instruction :: BitCast ) && LI -> isSimple ( ) ; } case Instruction :: BitCast : return true ; case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; if ( isa < AllocaInst > ( User ) && <ISA_LIT> -> getPointerOperandType ( ) == User -> getType ( ) && isa < VectorType > ( <ISA_LIT> -> getValueOperand ( ) -> getType ( ) ) ) return true ; Instruction * UserInst = dyn_cast < Instruction > ( User ) ; if ( ! UserInst ) return false ; return ( <ISA_LIT> -> getPointerOperand ( ) == User ) && ( UserInst -> getOpcode ( ) == Instruction :: GetElementPtr || UserInst -> getOpcode ( ) == Instruction :: BitCast ) && <ISA_LIT> -> isSimple ( ) ; } default : return false ;


<s> LoadInst * LI = cast < LoadInst > ( Inst ) ; if ( isa < AllocaInst > ( User ) && LI -> getPointerOperandType ( ) == User -> getType ( ) && isa < VectorType > ( LI -> getType ( ) ) ) return true ; Instruction * PtrInst = dyn_cast < Instruction > ( LI -> getPointerOperand ( ) ) ; if ( ! PtrInst ) return false ; return ( PtrInst -> getOpcode ( ) == Instruction :: GetElementPtr || PtrInst -> getOpcode ( ) == Instruction :: BitCast ) && LI -> isSimple ( ) ; } case Instruction :: BitCast : return true ; case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; if ( isa < AllocaInst > ( User ) && <ISA_LIT> -> getPointerOperandType ( ) == User -> getType ( ) && isa < VectorType > ( <ISA_LIT> -> getValueOperand ( ) -> getType ( ) ) ) return true ; Instruction * UserInst = dyn_cast < Instruction > ( User ) ; if ( ! UserInst ) return false ; return ( <ISA_LIT> -> getPointerOperand ( ) == User ) && ( UserInst -> getOpcode ( ) == Instruction :: GetElementPtr || UserInst -> getOpcode ( ) == Instruction :: BitCast ) && <ISA_LIT> -> isSimple ( ) ; }


<s> return new <ISA_LIT>PromoteAllocaToVector ( ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>PromoteAllocaToVector ( ) {

return new <ISA_LIT>PromoteAllocaToVector ( ) ;


<s> if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) return nullptr ;

ConstantInt * I0 = dyn_cast < ConstantInt > ( GEP -> getOperand ( <NUM_LIT> ) ) ;


FunctionPass :: getAnalysisUsage ( AU ) ;


<s> const <ISA_LIT>Subtarget & ST = <ISA_LIT>Subtarget :: get ( * TM , * Builder . GetInsertBlock ( ) -> getParent ( ) ) ; <STR_LIT> IntrID = <STR_LIT> ; switch ( N ) { case <NUM_LIT> : IntrID = IsAMDGCN ? ( <STR_LIT> ) <STR_LIT> : ( <STR_LIT> ) <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? ( <STR_LIT> ) <STR_LIT> : ( <STR_LIT> ) <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? ( <STR_LIT> ) <STR_LIT> : ( <STR_LIT> ) <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } Function * WorkitemIdFn = <STR_LIT> ( Mod , IntrID ) ; CallInst * CI = Builder . CreateCall ( WorkitemIdFn ) ; ST . makeLIDRangeMetadata ( CI ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << I << '\n' ) ; Module * Mod = I . getParent ( ) -> getParent ( ) -> getParent ( ) ; return tryPromoteAllocaToVector ( & I , Mod -> getDataLayout ( ) ) ;


<s> uint64_t AllocSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Align ) ; CurrentLocalMemUsage += AllocSize ; break ; } } } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage , F ) ; unsigned OccupancyHint = ST . getWavesPerEU ( F ) . second ; if ( OccupancyHint == <NUM_LIT> ) OccupancyHint = <NUM_LIT> ; OccupancyHint = std :: min ( OccupancyHint , ST . getMaxWavesPerEU ( ) ) ; MaxOccupancy = std :: min ( OccupancyHint , MaxOccupancy ) ; unsigned MaxSizeWithWaveCount = ST . getMaxLocalMemSizeWithWaveCount ( MaxOccupancy , F ) ; if ( CurrentLocalMemUsage > MaxSizeWithWaveCount ) return false ; LocalMemLimit = MaxSizeWithWaveCount ; LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << CurrentLocalMemUsage << <STR_LIT> << <STR_LIT> << MaxSizeWithWaveCount << <STR_LIT> << MaxOccupancy << '\n' << <STR_LIT> << ( LocalMemLimit - CurrentLocalMemUsage ) << <STR_LIT> ) ; return true ;


<s> IntrinsicInst * II = dyn_cast < IntrinsicInst > ( CI ) ; if ( ! II ) return false ; switch ( II -> getIntrinsicID ( ) ) {


<s> bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ; SmallVector < AllocaInst * , <NUM_LIT> > Allocas ;


<s> SmallVector < AllocaInst * , <NUM_LIT> > Allocas ; for ( Instruction & I : EntryBB ) { if ( AllocaInst * AI = dyn_cast < AllocaInst > ( & I ) ) Allocas . push_back ( AI ) ; }


<s> if ( ! ST . isPromoteAllocaEnabled ( ) ) return false ; unsigned MaxVGPRs ; if ( TM . getTargetTriple ( ) . getArch ( ) == Triple :: amdgcn ) { const GCNSubtarget & ST = TM . getSubtarget < GCNSubtarget > ( F ) ; MaxVGPRs = ST . getMaxNumVGPRs ( ST . getWavesPerEU ( F ) . first ) ; } else { MaxVGPRs = <NUM_LIT> ; } bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ; SmallVector < AllocaInst * , <NUM_LIT> > Allocas ; for ( Instruction & I : EntryBB ) { if ( AllocaInst * AI = dyn_cast < AllocaInst > ( & I ) ) Allocas . push_back ( AI ) ;


<s> DL = & Mod -> getDataLayout ( ) ; const Triple & TT = TM . getTargetTriple ( ) ; IsAMDGCN = TT . getArch ( ) == Triple :: amdgcn ; IsAMDHSA = TT . getOS ( ) == Triple :: AMDHSA ; const <ISA_LIT>Subtarget & ST = <ISA_LIT>Subtarget :: get ( TM , F ) ; if ( ! ST . isPromoteAllocaEnabled ( ) ) return false ; if ( IsAMDGCN ) { const GCNSubtarget & ST = TM . getSubtarget < GCNSubtarget > ( F ) ; MaxVGPRs = ST . getMaxNumVGPRs ( ST . getWavesPerEU ( F ) . first ) ; } else { MaxVGPRs = <NUM_LIT> ; }

bool SufficientLDS = hasSufficientLocalMem ( F ) ;


<s> if ( CAS -> isVolatile ( ) ) return false ; } if ( ICmpInst * ICmp = dyn_cast < ICmpInst > ( UseInst ) ) { if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , ICmp , <NUM_LIT> , <NUM_LIT> ) ) return false ; WorkList . push_back ( ICmp ) ; } if ( UseInst -> getOpcode ( ) == Instruction :: AddrSpaceCast ) { if ( PointerMayBeCaptured ( UseInst , true , true ) ) return false ; WorkList . push_back ( User ) ; continue ; } if ( ! User -> getType ( ) -> isPointerTy ( ) ) continue ; if ( GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( UseInst ) ) { if ( ! GEP -> isInBounds ( ) ) return false ; } if ( SelectInst * <ISA_LIT> = dyn_cast < SelectInst > ( UseInst ) ) { if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , <ISA_LIT> , <NUM_LIT> , <NUM_LIT> ) ) return false ; } if ( PHINode * Phi = dyn_cast < PHINode > ( UseInst ) ) { switch ( Phi -> getNumIncomingValues ( ) ) { case <NUM_LIT> : break ; case <NUM_LIT> : if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , Phi , <NUM_LIT> , <NUM_LIT> ) ) return false ; break ; default : return false ; }


<s> DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( I32Ty , GEPXY , Align ( <NUM_LIT> ) ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( I32Ty , GEPZU , Align ( <NUM_LIT> ) ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ;

LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ;


<s> IntrID = IsAMDGCN ? ( <STR_LIT> ) <STR_LIT> : ( <STR_LIT> ) <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? ( <STR_LIT> ) <STR_LIT> : ( <STR_LIT> ) <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? ( <STR_LIT> ) <STR_LIT> : ( <STR_LIT> ) <STR_LIT> ; break ; default :


<s> if ( ! TM || skipFunction ( F ) ) return false ; const <ISA_LIT>Subtarget & ST = TM -> getSubtarget < <ISA_LIT>Subtarget > ( F ) ; if ( ! ST . isPromoteAllocaEnabled ( ) ) return false ; FunctionType * FTy = F . getFunctionType ( ) ; for ( Type * ParamTy : FTy -> params ( ) ) { PointerType * PtrTy = dyn_cast < PointerType > ( ParamTy ) ; if ( PtrTy && PtrTy -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ) { LocalMemLimit = <NUM_LIT> ; DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } } LocalMemLimit = ST . getLocalMemorySize ( ) ; if ( LocalMemLimit == <NUM_LIT> ) return false ; const DataLayout & DL = Mod -> getDataLayout ( ) ; CurrentLocalMemUsage = <NUM_LIT> ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getType ( ) -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( const User * U : GV . users ( ) ) { const Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) { unsigned Align = GV . getAlignment ( ) ; if ( Align == <NUM_LIT> ) Align = DL . getABITypeAlignment ( GV . getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Align ) ; CurrentLocalMemUsage += AllocSize ; break ; } } } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage , F ) ; unsigned OccupancyHint = ST . getWavesPerEU ( F ) . second ;

if ( OccupancyHint == <NUM_LIT> ) OccupancyHint = <NUM_LIT> ;


<s> bool <ISA_LIT>PromoteAlloca :: binaryOpIsDerivedFromSameAlloca ( Value * BaseAlloca , Value * Val , Instruction * Inst , int OpIdx0 , int OpIdx1 ) const { Value * OtherOp = Inst -> getOperand ( OpIdx0 ) ; if ( Val == OtherOp ) OtherOp = Inst -> getOperand ( OpIdx1 ) ; if ( isa < ConstantPointerNull > ( OtherOp ) ) return true ; Value * OtherObj = getUnderlyingObject ( OtherOp ) ;

if ( ! isa < AllocaInst > ( OtherObj ) ) return false ;


<s> bool <ISA_LIT>PromoteAllocaToVector :: handleAlloca ( AllocaInst & I ) { if ( ! I . isStaticAlloca ( ) || I . isArrayAllocation ( ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << I << '\n' ) ; Module * Mod = I . getParent ( ) -> getParent ( ) -> getParent ( ) ; return tryPromoteAllocaToVector ( & I , Mod -> getDataLayout ( ) , MaxVGPRs ) ;


<s> const DataLayout & DL = Mod -> getDataLayout ( ) ; SmallVector < std :: pair < uint64_t , Align > , <NUM_LIT> > AllocatedSizes ; AllocatedSizes . reserve ( UsedLDS . size ( ) ) ; for ( const GlobalVariable * GV : UsedLDS ) { Align Alignment = DL . getValueOrABITypeAlignment ( GV -> getAlign ( ) , GV -> getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV -> getValueType ( ) ) ; AllocatedSizes . emplace_back ( AllocSize , Alignment ) ; } llvm :: sort ( AllocatedSizes . begin ( ) , AllocatedSizes . end ( ) , [ ] ( std :: pair < uint64_t , Align > LHS , std :: pair < uint64_t , Align > RHS ) { return LHS . second < RHS . second ; } ) ; CurrentLocalMemUsage = <NUM_LIT> ; for ( auto Alloc : AllocatedSizes ) { CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Alloc . second ) ; CurrentLocalMemUsage += Alloc . first ; } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage , F ) ; unsigned OccupancyHint = ST . getWavesPerEU ( F ) . second ; if ( OccupancyHint == <NUM_LIT> ) OccupancyHint = <NUM_LIT> ; OccupancyHint = std :: min ( OccupancyHint , ST . getMaxWavesPerEU ( ) ) ; MaxOccupancy = std :: min ( OccupancyHint , MaxOccupancy ) ; unsigned MaxSizeWithWaveCount = ST . getMaxLocalMemSizeWithWaveCount ( MaxOccupancy , F ) ; if ( CurrentLocalMemUsage > MaxSizeWithWaveCount ) return false ; LocalMemLimit = MaxSizeWithWaveCount ;

LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << CurrentLocalMemUsage << <STR_LIT> << <STR_LIT> << MaxSizeWithWaveCount << <STR_LIT> << MaxOccupancy << '\n' << <STR_LIT> << ( LocalMemLimit - CurrentLocalMemUsage ) << <STR_LIT> ) ;


<s> MaxVGPRs = ST . getMaxNumVGPRs ( ST . getWavesPerEU ( F ) . first ) ; } else { MaxVGPRs = <NUM_LIT> ; } bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ; SmallVector < AllocaInst * , <NUM_LIT> > Allocas ; for ( Instruction & I : EntryBB ) { if ( AllocaInst * AI = dyn_cast < AllocaInst > ( & I ) ) Allocas . push_back ( AI ) ; } for ( AllocaInst * AI : Allocas ) {

if ( handleAlloca ( * AI ) ) Changed = true ;


<s> static Value * calculateVectorIndex ( Value * Ptr , const std :: map < GetElementPtrInst * , Value * > & GEPIdx ) { auto * GEP = dyn_cast < GetElementPtrInst > ( Ptr -> stripPointerCasts ( ) ) ; if ( ! GEP ) return ConstantInt :: getNullValue ( Type :: getInt32Ty ( Ptr -> getContext ( ) ) ) ;


<s> if ( ! GEP ) return ConstantInt :: getNullValue ( Type :: getInt32Ty ( Ptr -> getContext ( ) ) ) ; auto I = GEPIdx . find ( GEP ) ;

assert ( I != GEPIdx . end ( ) && <STR_LIT> ) ;


<s> static Value * GEPToVectorIndex ( GetElementPtrInst * GEP , AllocaInst * Alloca , Type * VecElemTy , const DataLayout & DL ) { unsigned BW = DL . getIndexTypeSizeInBits ( GEP -> getType ( ) ) ; MapVector < Value * , APInt > VarOffsets ; APInt ConstOffset ( BW , <NUM_LIT> ) ; if ( GEP -> getPointerOperand ( ) -> stripPointerCasts ( ) != Alloca || ! GEP -> collectOffset ( DL , BW , VarOffsets , ConstOffset ) ) return nullptr ; unsigned VecElemSize = DL . getTypeAllocSize ( VecElemTy ) ; if ( VarOffsets . size ( ) > <NUM_LIT> ) return nullptr ; if ( VarOffsets . size ( ) == <NUM_LIT> ) { const auto & VarOffset = VarOffsets . front ( ) ; if ( ! ConstOffset . isZero ( ) || VarOffset . second != VecElemSize ) return nullptr ; return VarOffset . first ; } APInt Quot ; uint64_t Rem ; APInt :: udivrem ( ConstOffset , VecElemSize , Quot , Rem ) ; if ( Rem != <NUM_LIT> ) return nullptr ;

return ConstantInt :: get ( GEP -> getContext ( ) , Quot ) ;


<s> if ( isa < BitCastInst > ( Inst ) ) { for ( Use & U : Inst -> uses ( ) ) Uses . push_back ( & U ) ; continue ; } if ( auto * GEP = dyn_cast < GetElementPtrInst > ( Inst ) ) { Value * Index = GEPToVectorIndex ( GEP , Alloca , VecEltTy , DL ) ; if ( ! Index ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( Use & U : Inst -> uses ( ) ) Uses . push_back ( & U ) ; continue ; } if ( isAssumeLikeIntrinsic ( Inst ) ) continue ; if ( isa < ICmpInst > ( Inst ) && all_of ( Inst -> users ( ) , [ ] ( User * U ) { return isAssumeLikeIntrinsic ( cast < Instruction > ( U ) ) ; } ) ) continue ; return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Instruction * Inst : WorkList ) { IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Type * VecPtrTy = VectorTy -> getPointerTo ( Alloca -> getAddressSpace ( ) ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; if ( Inst -> getType ( ) != VecEltTy ) ExtractElement = Builder . CreateBitOrPointerCast ( ExtractElement , Inst -> getType ( ) ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; Value * Ptr = <ISA_LIT> -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Type * VecPtrTy = VectorTy -> getPointerTo ( Alloca -> getAddressSpace ( ) ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * Elt = <ISA_LIT> -> getValueOperand ( ) ; if ( Elt -> getType ( ) != VecEltTy ) Elt = Builder . CreateBitOrPointerCast ( Elt , VecEltTy ) ; Value * NewVecValue = Builder . CreateInsertElement ( VecValue , Elt , Index ) ; Builder . CreateStore ( NewVecValue , BitCast ) ; Inst -> eraseFromParent ( ) ; break ;


<s> GetElementPtrInst * GEP = cast < GetElementPtrInst > ( stripBitcasts ( Ptr ) ) ; auto I = GEPIdx . find ( GEP ) ;

return I == GEPIdx . end ( ) ? nullptr : I -> second ;


<s> } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { if ( Inst -> getType ( ) == AllocaTy || Inst -> getType ( ) -> isVectorTy ( ) ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; if ( Inst -> getType ( ) != VecEltTy ) ExtractElement = Builder . CreateBitCast ( ExtractElement , Inst -> getType ( ) ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; if ( <ISA_LIT> -> getValueOperand ( ) -> getType ( ) == AllocaTy || <ISA_LIT> -> getValueOperand ( ) -> getType ( ) -> isVectorTy ( ) ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = <ISA_LIT> -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * Elt = <ISA_LIT> -> getValueOperand ( ) ; if ( Elt -> getType ( ) != VecEltTy ) Elt = Builder . CreateBitCast ( Elt , VecEltTy ) ; Value * NewVecValue = Builder . CreateInsertElement ( VecValue , Elt , Index ) ;

Builder . CreateStore ( NewVecValue , BitCast ) ;


<s> <ISA_LIT>PromoteAlloca ( const TargetMachine * TM_ = nullptr ) :

FunctionPass ( ID ) , TM ( TM_ ) , Mod ( nullptr ) , DL ( nullptr ) , MaxWorkGroupSizeRange ( nullptr ) , LocalMemLimit ( <NUM_LIT> ) , CurrentLocalMemUsage ( <NUM_LIT> ) , IsAMDGCN ( false ) , IsAMDHSA ( false ) {


<s> if ( std :: find ( WorkList . begin ( ) , WorkList . end ( ) , User ) != WorkList . end ( ) ) continue ; if ( CallInst * CI = dyn_cast < CallInst > ( User ) ) { if ( ! isCallPromotable ( CI ) ) return false ; WorkList . push_back ( User ) ; continue ; } Instruction * UseInst = cast < Instruction > ( User ) ; if ( UseInst -> getOpcode ( ) == Instruction :: PtrToInt ) return false ; if ( LoadInst * LI = dyn_cast < LoadInst > ( UseInst ) ) { if ( LI -> isVolatile ( ) ) return false ; continue ; } if ( StoreInst * <ISA_LIT> = dyn_cast < StoreInst > ( UseInst ) ) { if ( <ISA_LIT> -> isVolatile ( ) ) return false ; if ( <ISA_LIT> -> getPointerOperand ( ) != Val ) return false ; } else if ( AtomicRMWInst * RMW = dyn_cast < AtomicRMWInst > ( UseInst ) ) { if ( RMW -> isVolatile ( ) ) return false ; } else if ( AtomicCmpXchgInst * CAS = dyn_cast < AtomicCmpXchgInst > ( UseInst ) ) { if ( CAS -> isVolatile ( ) ) return false ; } if ( ICmpInst * ICmp = dyn_cast < ICmpInst > ( UseInst ) ) { if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , ICmp , <NUM_LIT> , <NUM_LIT> ) ) return false ; WorkList . push_back ( ICmp ) ; } if ( ! User -> getType ( ) -> isPointerTy ( ) ) continue ; if ( GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( UseInst ) ) { if ( ! GEP -> isInBounds ( ) ) return false ; } if ( SelectInst * <ISA_LIT> = dyn_cast < SelectInst > ( UseInst ) ) { if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , <ISA_LIT> , <NUM_LIT> , <NUM_LIT> ) ) return false ; } if ( PHINode * Phi = dyn_cast < PHINode > ( UseInst ) ) { switch ( Phi -> getNumIncomingValues ( ) ) { case <NUM_LIT> : break ; case <NUM_LIT> : if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , Phi , <NUM_LIT> , <NUM_LIT> ) ) return false ; break ; default : return false ; }


<s> Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; LocalSizeY -> setMetadata ( LLVMContext :: MD_range , MaxWorkGroupSizeRange ) ; LocalSizeZ -> setMetadata ( LLVMContext :: MD_range , MaxWorkGroupSizeRange ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeSet :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeSet :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeSet :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = llvm :: MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ;

LoadZU -> setMetadata ( LLVMContext :: MD_range , MaxWorkGroupSizeRange ) ;


<s> } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeSet :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeSet :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeSet :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ;

MDNode * MD = llvm :: MDNode :: get ( Mod -> getContext ( ) , None ) ;


<s> PointerType * PtrTy = dyn_cast < PointerType > ( ParamTy ) ; if ( PtrTy && PtrTy -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ) { LocalMemLimit = <NUM_LIT> ; DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } } LocalMemLimit = ST . getLocalMemorySize ( ) ; if ( LocalMemLimit == <NUM_LIT> ) return false ; const DataLayout & DL = Mod -> getDataLayout ( ) ; CurrentLocalMemUsage = <NUM_LIT> ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getType ( ) -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( const User * U : GV . users ( ) ) { const Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) { unsigned Align = GV . getAlignment ( ) ; if ( Align == <NUM_LIT> ) Align = DL . getABITypeAlignment ( GV . getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Align ) ; CurrentLocalMemUsage += AllocSize ; break ; } } } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage ) ; unsigned OccupancyHint = <STR_LIT> ( F , <STR_LIT> , <NUM_LIT> ) ; if ( OccupancyHint == <NUM_LIT> ) OccupancyHint = <NUM_LIT> ; OccupancyHint = std :: min ( OccupancyHint , ST . getMaxWavesPerCU ( ) ) ;

MaxOccupancy = std :: min ( OccupancyHint , MaxOccupancy ) ;


<s> std :: pair < Value * , Value * > <ISA_LIT>PromoteAllocaImpl :: getLocalSizeYZ ( IRBuilder < > & Builder ) { const Function & F = * Builder . GetInsertBlock ( ) -> getParent ( ) ; const <ISA_LIT>Subtarget & ST = <ISA_LIT>Subtarget :: get ( TM , F ) ; if ( ! IsAMDHSA ) { Function * LocalSizeYFn = <STR_LIT> ( Mod , <STR_LIT> ) ; Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addRetAttr ( Attribute :: NoAlias ) ; DispatchPtr -> addRetAttr ( Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableRetAttr ( <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( I32Ty , GEPXY , Align ( <NUM_LIT> ) ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( I32Ty , GEPZU , Align ( <NUM_LIT> ) ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ;

LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ;


<s> if ( ! IsAMDHSA ) { Function * LocalSizeYFn = <STR_LIT> ( Mod , <STR_LIT> ) ; Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addRetAttr ( Attribute :: NoAlias ) ; DispatchPtr -> addRetAttr ( Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableRetAttr ( <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( I32Ty , GEPXY , Align ( <NUM_LIT> ) ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( I32Ty , GEPZU , Align ( <NUM_LIT> ) ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ;


<s> if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) , AllocaUser ) ) return false ; WorkList . push_back ( GEPUser ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { if ( Inst -> getType ( ) == AllocaTy ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; if ( <ISA_LIT> -> getValueOperand ( ) -> getType ( ) == AllocaTy ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = <ISA_LIT> -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ;


<s> if ( ! canVectorizeInst ( cast < Instruction > ( AllocaUser ) , Alloca ) ) return false ; WorkList . push_back ( AllocaUser ) ; continue ; } Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( User * GEPUser : AllocaUser -> users ( ) ) { if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) , AllocaUser ) ) return false ; WorkList . push_back ( GEPUser ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { if ( Inst -> getType ( ) == AllocaTy ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; if ( <ISA_LIT> -> getValueOperand ( ) -> getType ( ) == AllocaTy ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = <ISA_LIT> -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * NewVecValue = Builder . CreateInsertElement ( VecValue , <ISA_LIT> -> getValueOperand ( ) , Index ) ; Builder . CreateStore ( NewVecValue , BitCast ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: BitCast : case Instruction :: AddrSpaceCast : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } return true ;


<s> std :: pair < Value * , Value * > <ISA_LIT>PromoteAllocaImpl :: getLocalSizeYZ ( IRBuilder < > & Builder ) { Function & F = * Builder . GetInsertBlock ( ) -> getParent ( ) ; const <ISA_LIT>Subtarget & ST = <ISA_LIT>Subtarget :: get ( TM , F ) ; if ( ! IsAMDHSA ) { Function * LocalSizeYFn = <STR_LIT> ( Mod , <STR_LIT> ) ; Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addRetAttr ( Attribute :: NoAlias ) ; DispatchPtr -> addRetAttr ( Attribute :: NonNull ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ;


<s> Function * LocalSizeYFn = <STR_LIT> ( Mod , <STR_LIT> ) ; Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addRetAttr ( Attribute :: NoAlias ) ; DispatchPtr -> addRetAttr ( Attribute :: NonNull ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; F . removeFnAttr ( <STR_LIT> ) ; DispatchPtr -> addDereferenceableRetAttr ( <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( I32Ty , GEPXY , Align ( <NUM_LIT> ) ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( I32Ty , GEPZU , Align ( <NUM_LIT> ) ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ;


<s> if ( <ISA_LIT> -> getPointerOperand ( ) != Val ) return false ; } else if ( AtomicRMWInst * RMW = dyn_cast_or_null < AtomicRMWInst > ( UseInst ) ) { if ( RMW -> isVolatile ( ) ) return false ; } else if ( AtomicCmpXchgInst * CAS = dyn_cast_or_null < AtomicCmpXchgInst > ( UseInst ) ) { if ( CAS -> isVolatile ( ) ) return false ; } if ( ICmpInst * ICmp = dyn_cast < ICmpInst > ( UseInst ) ) { if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , ICmp , <NUM_LIT> , <NUM_LIT> ) ) return false ; WorkList . push_back ( ICmp ) ; } if ( ! User -> getType ( ) -> isPointerTy ( ) ) continue ; if ( GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( UseInst ) ) { if ( ! GEP -> isInBounds ( ) ) return false ; } if ( SelectInst * <ISA_LIT> = dyn_cast < SelectInst > ( UseInst ) ) { if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , <ISA_LIT> , <NUM_LIT> , <NUM_LIT> ) ) return false ; } if ( PHINode * Phi = dyn_cast < PHINode > ( UseInst ) ) { switch ( Phi -> getNumIncomingValues ( ) ) { case <NUM_LIT> : break ; case <NUM_LIT> : if ( ! binaryOpIsDerivedFromSameAlloca ( BaseAlloca , Val , Phi , <NUM_LIT> , <NUM_LIT> ) ) return false ; break ; default :


<s> if ( isa < ConstantPointerNull > ( Phi -> getIncomingValue ( I ) ) ) Phi -> setIncomingValue ( I , ConstantPointerNull :: get ( NewTy ) ) ; } } continue ; } IntrinsicInst * Intr = dyn_cast < IntrinsicInst > ( Call ) ; if ( ! Intr ) { std :: vector < Type * > ArgTypes ; for ( unsigned ArgIdx = <NUM_LIT> , ArgEnd = Call -> getNumArgOperands ( ) ; ArgIdx != ArgEnd ; ++ ArgIdx ) { ArgTypes . push_back ( Call -> getArgOperand ( ArgIdx ) -> getType ( ) ) ; } Function * F = Call -> getCalledFunction ( ) ; FunctionType * NewType = FunctionType :: get ( Call -> getType ( ) , ArgTypes , F -> isVarArg ( ) ) ; Constant * C = Mod -> getOrInsertFunction ( ( F -> getName ( ) + <STR_LIT> ) . str ( ) , NewType , F -> getAttributes ( ) ) ; Function * NewF = cast < Function > ( C ) ; Call -> setCalledFunction ( NewF ) ; continue ; } Builder . SetInsertPoint ( Intr ) ; switch ( Intr -> getIntrinsicID ( ) ) { case <STR_LIT> : case <STR_LIT> : Intr -> eraseFromParent ( ) ; continue ; case <STR_LIT> : { MemCpyInst * MemCpy = cast < MemCpyInst > ( Intr ) ; Builder . CreateMemCpy ( MemCpy -> getRawDest ( ) , MemCpy -> getRawSource ( ) , MemCpy -> getLength ( ) , MemCpy -> getAlignment ( ) , MemCpy -> isVolatile ( ) ) ; Intr -> eraseFromParent ( ) ; continue ; } case <STR_LIT> : { MemMoveInst * MemMove = cast < MemMoveInst > ( Intr ) ; Builder . CreateMemMove ( MemMove -> getRawDest ( ) , MemMove -> getRawSource ( ) , MemMove -> getLength ( ) , MemMove -> getAlignment ( ) , MemMove -> isVolatile ( ) ) ; Intr -> eraseFromParent ( ) ; continue ; } case <STR_LIT> : { MemSetInst * MemSet = cast < MemSetInst > ( Intr ) ; Builder . CreateMemSet ( MemSet -> getRawDest ( ) , MemSet -> getValue ( ) , MemSet -> getLength ( ) , MemSet -> getAlignment ( ) , MemSet -> isVolatile ( ) ) ; Intr -> eraseFromParent ( ) ; continue ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Intr -> eraseFromParent ( ) ; continue ; case <STR_LIT> : {

Value * Src = Intr -> getOperand ( <NUM_LIT> ) ;


<s> for ( const User * U : GV . users ( ) ) { const Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) { unsigned Align = GV . getAlignment ( ) ; if ( Align == <NUM_LIT> ) Align = DL . getABITypeAlignment ( GV . getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Align ) ; CurrentLocalMemUsage += AllocSize ; break ; } } } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage ) ; unsigned OccupancyHint = <STR_LIT> ( F , <STR_LIT> , <NUM_LIT> ) ; if ( OccupancyHint == <NUM_LIT> ) OccupancyHint = <NUM_LIT> ; OccupancyHint = std :: min ( OccupancyHint , ST . getMaxWavesPerCU ( ) ) ; MaxOccupancy = std :: min ( OccupancyHint , MaxOccupancy ) ; unsigned MaxSizeWithWaveCount = ST . getMaxLocalMemSizeWithWaveCount ( MaxOccupancy ) ; if ( CurrentLocalMemUsage > MaxSizeWithWaveCount ) return false ; LocalMemLimit = MaxSizeWithWaveCount ; DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << CurrentLocalMemUsage << <STR_LIT> << <STR_LIT> << MaxSizeWithWaveCount << <STR_LIT> << MaxOccupancy << '\n' << <STR_LIT> << ( LocalMemLimit - CurrentLocalMemUsage ) << <STR_LIT> ) ; BasicBlock & EntryBB = * F . begin ( ) ; for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ; I != E ; ) { AllocaInst * AI = dyn_cast < AllocaInst > ( I ) ; ++ I ; if ( AI ) handleAlloca ( * AI ) ; } return true ;


<s> bool <ISA_LIT>PromoteAlloca :: runOnFunction ( Function & F ) { FunctionType * FTy = F . getFunctionType ( ) ; LocalMemAvailable = ST . getLocalMemorySize ( ) ; for ( unsigned i = <NUM_LIT> , e = FTy -> getNumParams ( ) ; i != e ; ++ i ) { Type * ParamTy = FTy -> getParamType ( i ) ; if ( ParamTy -> isPointerTy ( ) && ParamTy -> getPointerAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ) { LocalMemAvailable = <NUM_LIT> ; DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; break ; } } if ( LocalMemAvailable > <NUM_LIT> ) { for ( Module :: global_iterator I = Mod -> global_begin ( ) , E = Mod -> global_end ( ) ; I != E ; ++ I ) { GlobalVariable * GV = & * I ; if ( GV -> getType ( ) -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( Value :: use_iterator U = GV -> use_begin ( ) , UE = GV -> use_end ( ) ; U != UE ; ++ U ) { Instruction * Use = dyn_cast < Instruction > ( * U ) ;


<s> Value * TIdY = Builder . CreateCall ( ReadTIDIGY , { } ) ; Value * TIdZ = Builder . CreateCall ( ReadTIDIGZ , { } ) ; Value * Tmp0 = Builder . CreateMul ( TCntY , TCntZ ) ; Tmp0 = Builder . CreateMul ( Tmp0 , TIdX ) ; Value * Tmp1 = Builder . CreateMul ( TIdY , TCntZ ) ; Value * TID = Builder . CreateAdd ( Tmp0 , Tmp1 ) ; TID = Builder . CreateAdd ( TID , TIdZ ) ; std :: vector < Value * > Indices ; Indices . push_back ( Constant :: getNullValue ( Type :: getInt32Ty ( Mod -> getContext ( ) ) ) ) ; Indices . push_back ( TID ) ; Value * Offset = Builder . CreateGEP ( GVTy , GV , Indices ) ; I . mutateType ( Offset -> getType ( ) ) ; I . replaceAllUsesWith ( Offset ) ; I . eraseFromParent ( ) ; for ( std :: vector < Value * > :: iterator i = WorkList . begin ( ) , e = WorkList . end ( ) ; i != e ; ++ i ) { Value * V = * i ; CallInst * Call = dyn_cast < CallInst > ( V ) ; if ( ! Call ) { Type * EltTy = V -> getType ( ) -> getPointerElementType ( ) ; PointerType * NewTy = PointerType :: get ( EltTy , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; if ( isa < AddrSpaceCastInst > ( V ) ) continue ; V -> mutateType ( NewTy ) ; continue ; } IntrinsicInst * Intr = dyn_cast < IntrinsicInst > ( Call ) ; if ( ! Intr ) { std :: vector < Type * > ArgTypes ; for ( unsigned ArgIdx = <NUM_LIT> , ArgEnd = Call -> getNumArgOperands ( ) ; ArgIdx != ArgEnd ; ++ ArgIdx ) { ArgTypes . push_back ( Call -> getArgOperand ( ArgIdx ) -> getType ( ) ) ; } Function * F = Call -> getCalledFunction ( ) ; FunctionType * NewType = FunctionType :: get ( Call -> getType ( ) , ArgTypes , F -> isVarArg ( ) ) ; Constant * C = Mod -> getOrInsertFunction ( ( F -> getName ( ) + <STR_LIT> ) . str ( ) , NewType , F -> getAttributes ( ) ) ; Function * NewF = cast < Function > ( C ) ; Call -> setCalledFunction ( NewF ) ; continue ; } Builder . SetInsertPoint ( Intr ) ; switch ( Intr -> getIntrinsicID ( ) ) { case <STR_LIT> : case <STR_LIT> : Intr -> eraseFromParent ( ) ; continue ; case <STR_LIT> : {

MemCpyInst * MemCpy = cast < MemCpyInst > ( Intr ) ;


<s> return VectorType :: get ( ArrayTy -> getArrayElementType ( ) , ArrayTy -> getArrayNumElements ( ) ) ;


<s> } } if ( LocalMemAvailable > <NUM_LIT> ) { for ( Module :: global_iterator I = Mod -> global_begin ( ) , E = Mod -> global_end ( ) ; I != E ; ++ I ) { GlobalVariable * GV = I ; PointerType * GVTy = GV -> getType ( ) ; if ( GVTy -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ;

for ( Value :: use_iterator U = GV -> use_begin ( ) , UE = GV -> use_end ( ) ; U != UE ; ++ U ) {


<s> if ( GVTy -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( Value :: use_iterator U = GV -> use_begin ( ) , UE = GV -> use_end ( ) ; U != UE ; ++ U ) { Instruction * Use = dyn_cast < Instruction > ( * U ) ; if ( ! Use ) continue ;

if ( Use -> getParent ( ) -> getParent ( ) == & F ) LocalMemAvailable -= Mod -> getDataLayout ( ) . getTypeAllocSize ( GVTy -> getElementType ( ) ) ;


<s> for ( std :: vector < Value * > :: iterator i = WorkList . begin ( ) , e = WorkList . end ( ) ; i != e ; ++ i ) { Value * V = * i ; CallInst * Call = dyn_cast < CallInst > ( V ) ; if ( ! Call ) { Type * EltTy = V -> getType ( ) -> getPointerElementType ( ) ; PointerType * NewTy = PointerType :: get ( EltTy , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; if ( isa < AddrSpaceCastInst > ( V ) ) continue ; V -> mutateType ( NewTy ) ; continue ; } IntrinsicInst * Intr = dyn_cast < IntrinsicInst > ( Call ) ; if ( ! Intr ) { std :: vector < Type * > ArgTypes ; for ( unsigned ArgIdx = <NUM_LIT> , ArgEnd = Call -> getNumArgOperands ( ) ; ArgIdx != ArgEnd ; ++ ArgIdx ) { ArgTypes . push_back ( Call -> getArgOperand ( ArgIdx ) -> getType ( ) ) ; } Function * F = Call -> getCalledFunction ( ) ; FunctionType * NewType = FunctionType :: get ( Call -> getType ( ) , ArgTypes , F -> isVarArg ( ) ) ; Constant * C = Mod -> getOrInsertFunction ( ( F -> getName ( ) + <STR_LIT> ) . str ( ) , NewType , F -> getAttributes ( ) ) ; Function * NewF = cast < Function > ( C ) ; Call -> setCalledFunction ( NewF ) ; continue ; } Builder . SetInsertPoint ( Intr ) ; switch ( Intr -> getIntrinsicID ( ) ) { case <STR_LIT> : case <STR_LIT> : Intr -> eraseFromParent ( ) ; continue ; case <STR_LIT> : { MemCpyInst * MemCpy = cast < MemCpyInst > ( Intr ) ; Builder . CreateMemCpy ( MemCpy -> getRawDest ( ) , MemCpy -> getRawSource ( ) , MemCpy -> getLength ( ) , MemCpy -> getAlignment ( ) , MemCpy -> isVolatile ( ) ) ; Intr -> eraseFromParent ( ) ; continue ; } case <STR_LIT> : { MemSetInst * MemSet = cast < MemSetInst > ( Intr ) ; Builder . CreateMemSet ( MemSet -> getRawDest ( ) , MemSet -> getValue ( ) , MemSet -> getLength ( ) , MemSet -> getAlignment ( ) , MemSet -> isVolatile ( ) ) ; Intr -> eraseFromParent ( ) ; continue ; } default : Intr -> dump ( ) ; llvm_unreachable ( <STR_LIT> ) ; }


<s> } case Instruction :: BitCast : case Instruction :: AddrSpaceCast : return true ; case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; return ( <ISA_LIT> -> getPointerOperand ( ) == User ) && isa < GetElementPtrInst > ( User ) && ! <ISA_LIT> -> isVolatile ( ) ;


<s> return true ; case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; return ( <ISA_LIT> -> getPointerOperand ( ) == User ) && isa < GetElementPtrInst > ( User ) && ! <ISA_LIT> -> isVolatile ( ) ; } default : return false ; }


<s> LocalMemLimit = ST . getLocalMemorySize ( ) ; if ( LocalMemLimit == <NUM_LIT> ) return false ; const DataLayout & DL = Mod -> getDataLayout ( ) ; CurrentLocalMemUsage = <NUM_LIT> ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getType ( ) -> getAddressSpace ( ) != AS . LOCAL_ADDRESS ) continue ; for ( const User * U : GV . users ( ) ) { const Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) { unsigned Align = GV . getAlignment ( ) ; if ( Align == <NUM_LIT> ) Align = DL . getABITypeAlignment ( GV . getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Align ) ; CurrentLocalMemUsage += AllocSize ; break ; } }


<s> Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( User * GEPUser : AllocaUser -> users ( ) ) { if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) , AllocaUser ) ) return false ; WorkList . push_back ( GEPUser ) ; } } VectorType * VectorTy = arrayTypeToVecType ( AllocaTy ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { Type * VecPtrTy = VectorTy -> getPointerTo ( AS . PRIVATE_ADDRESS ) ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ;


<s> Function * LocalSizeYFn = <STR_LIT> ( Mod , <STR_LIT> ) ; Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ;

Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ;


<s> const <ISA_LIT>Subtarget & ST = <ISA_LIT>Subtarget :: get ( * TM , F ) ; if ( ! IsAMDHSA ) { Function * LocalSizeYFn = <STR_LIT> ( Mod , <STR_LIT> ) ; Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ;

Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ;


<s> GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( AllocaUser ) ; if ( ! GEP ) { if ( ! canVectorizeInst ( cast < Instruction > ( AllocaUser ) , Alloca ) ) return false ; WorkList . push_back ( AllocaUser ) ; continue ; } Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( User * GEPUser : AllocaUser -> users ( ) ) { if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) , AllocaUser ) ) return false ; WorkList . push_back ( GEPUser ) ; } } VectorType * VectorTy = dyn_cast < VectorType > ( AllocaTy ) ; if ( ! VectorTy ) VectorTy = arrayTypeToVecType ( cast < ArrayType > ( AllocaTy ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { if ( Inst -> getType ( ) == AT ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; }


<s> return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , AS . CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ;


<s> <STR_LIT> IntrID = <STR_LIT> :: not_intrinsic ; switch ( N ) { case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } Function * WorkitemIdFn = <STR_LIT> ( Mod , IntrID ) ;

CallInst * CI = Builder . CreateCall ( WorkitemIdFn ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } } LocalMemLimit = ST . getLocalMemorySize ( ) ; if ( LocalMemLimit == <NUM_LIT> ) return false ; const DataLayout & DL = Mod -> getDataLayout ( ) ; CurrentLocalMemUsage = <NUM_LIT> ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getType ( ) -> getAddressSpace ( ) != AS . LOCAL_ADDRESS ) continue ; for ( const User * U : GV . users ( ) ) { const Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) {

unsigned Align = GV . getAlignment ( ) ;


<s> IsAMDGCN = TT . getArch ( ) == Triple :: amdgcn ; IsAMDHSA = TT . getOS ( ) == Triple :: AMDHSA ; const <ISA_LIT>CommonSubtarget & ST = <ISA_LIT>CommonSubtarget :: get ( * TM , F ) ; if ( ! ST . isPromoteAllocaEnabled ( ) ) return false ; AS = <STR_LIT> ( * F . getParent ( ) ) ; bool SufficientLDS = hasSufficientLocalMem ( F ) ; bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ;

for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ; I != E ; ) {


<s> FunctionPass ( ID ) , TM ( TM_ ) , Mod ( nullptr ) , MaxWorkGroupSizeRange ( nullptr ) , LocalMemAvailable ( <NUM_LIT> ) , IsAMDGCN ( false ) , IsAMDHSA ( false ) {