<s> } auto RuntimeHandle = ( F . getName ( ) + <STR_LIT> ) . str ( ) ; auto * GV = new GlobalVariable ( M , Type :: getInt8Ty ( C ) -> getPointerTo ( <ISA_LIT>AS :: GLOBAL_ADDRESS ) , true , GlobalValue :: ExternalLinkage , nullptr , RuntimeHandle , nullptr , GlobalValue :: NotThreadLocal , <ISA_LIT>AS :: GLOBAL_ADDRESS , true ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * GV << '\n' ) ; for ( auto U : F . users ( ) ) { if ( ! isa < ConstantExpr > ( & * U ) ) continue ; auto * BitCast = cast < ConstantExpr > ( & * U ) ; auto * NewPtr = ConstantExpr :: getPointerCast ( GV , BitCast -> getType ( ) ) ; BitCast -> replaceAllUsesWith ( NewPtr ) ; F . addFnAttr ( <STR_LIT> , RuntimeHandle ) ; F . setLinkage ( GlobalValue :: ExternalLinkage ) ; for ( auto U : NewPtr -> users ( ) ) { if ( auto * I = dyn_cast < Instruction > ( & * U ) ) { auto * F = I -> getParent ( ) -> getParent ( ) ; Callers . insert ( F ) ; collectCallers ( F , Callers ) ; } } Changed =


<s> if ( F . hasFnAttribute ( <STR_LIT> ) ) { if ( ! F . hasOneUse ( ) || ! F . user_begin ( ) -> hasOneUse ( ) || ! isa < ConstantExpr > ( * F . user_begin ( ) ) || ! isa < ConstantExpr > ( * F . user_begin ( ) -> user_begin ( ) ) ) { continue ; } auto * BitCast = cast < ConstantExpr > ( * F . user_begin ( ) ) ; auto * AddrCast = cast < ConstantExpr > ( * BitCast -> user_begin ( ) ) ; auto RuntimeHandle = ( F . getName ( ) + <STR_LIT> ) . str ( ) ; auto * GV = new GlobalVariable ( M , Type :: getInt8Ty ( C ) -> getPointerTo ( <ISA_LIT>AS :: GLOBAL_ADDRESS ) , true , GlobalValue :: ExternalLinkage , nullptr , RuntimeHandle , nullptr , GlobalValue :: NotThreadLocal , <ISA_LIT>AS :: GLOBAL_ADDRESS , true ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * GV


<s> DEBUG ( dbgs ( ) << <STR_LIT> << * GV << '\n' ) ; auto * NewPtr = ConstantExpr :: getPointerCast ( GV , AddrCast -> getType ( ) ) ; AddrCast -> replaceAllUsesWith ( NewPtr ) ; F . addFnAttr ( <STR_LIT> , RuntimeHandle ) ; F . setLinkage ( GlobalValue :: ExternalLinkage ) ; for ( auto U : NewPtr -> users ( ) ) { if ( auto * I = dyn_cast < Instruction > ( & * U ) ) { auto * F = I -> getParent ( ) -> getParent ( ) ; Callers . insert ( F ) ; collectCallers ( F , Callers ) ; } } Changed = true ; } } for ( auto F : Callers ) { if ( F -> getCallingConv ( ) != CallingConv :: <ISA_LIT>_KERNEL


<s> bool <ISA_LIT>OpenCLEnqueuedBlockLowering :: runOnModule ( Module & M ) { DenseSet < Function * > Callers ; auto & C = M . getContext ( ) ; bool Changed = false ; for ( auto & F : M . functions ( ) ) { if ( F . hasFnAttribute ( <STR_LIT> ) ) { if ( ! F . hasName ( ) ) { SmallString < <NUM_LIT> > Name ; Mangler :: getNameWithPrefix ( Name , <STR_LIT> , M . getDataLayout ( ) ) ; F . setName ( Name ) ; } DEBUG ( dbgs ( ) << <STR_LIT> << F . getName ( ) << '\n' ) ; auto RuntimeHandle = ( F . getName ( ) + <STR_LIT> ) . str ( ) ; auto T = Type :: getInt8Ty ( C ) -> getPointerTo ( <ISA_LIT>AS :: GLOBAL_ADDRESS


<s> FunctionType * FT = F -> getFunctionType ( ) ; SmallVector < Type * , <NUM_LIT> > ArgTypes ; KernelArgMD NewArgMDs ; PushArgMD ( NewArgMDs , GetArgMD ( KernelMDNode , <NUM_LIT> ) ) ; for ( unsigned i = <NUM_LIT> ; i < FT -> getNumParams ( ) ; ++ i ) { ArgTypes . push_back ( FT -> getParamType ( i ) ) ; MDVector ArgMD = GetArgMD ( KernelMDNode , i + <NUM_LIT> ) ; PushArgMD ( NewArgMDs , ArgMD ) ; if ( ! IsImageType ( ArgTypeFromMD ( KernelMDNode , i ) ) ) continue ; ArgTypes . push_back ( ImageSizeType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageSizeArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; ArgTypes . push_back ( ImageFormatType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageFormatArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; Modified = true ; } if ( ! Modified ) { return std :: make_tuple ( nullptr , nullptr ) ; } auto NewFT = FunctionType :: get ( FT -> getReturnType ( ) , ArgTypes , false ) ; auto NewF = Function :: Create ( NewFT , F -> getLinkage ( ) , F -> getName ( ) ) ; ValueToValueMapTy VMap ; auto NewFArgIt = NewF -> arg_begin ( ) ; for ( auto & Arg : F -> args ( ) ) { auto ArgName = Arg . getName ( ) ; NewFArgIt -> setName ( ArgName ) ; VMap [ & Arg ] = & ( * NewFArgIt ++ ) ; if ( IsImageType ( ArgTypeFromMD ( KernelMDNode , Arg . getArgNo ( ) ) ) ) { ( NewFArgIt ++ ) -> setName ( Twine ( <STR_LIT> ) + ArgName ) ; ( NewFArgIt ++ ) -> setName ( Twine ( <STR_LIT> ) + ArgName ) ; } } SmallVector < ReturnInst * , <NUM_LIT> >


<s> auto F = mdconst :: dyn_extract < Function > ( Node -> getOperand ( <NUM_LIT> ) ) ; if ( ! F ) return nullptr ; size_t ExpectNumArgNodeOps = F -> arg_size ( ) + <NUM_LIT> ; for ( size_t i = <NUM_LIT> ; i < NumKernelArgMDNodes ; ++ i ) { MDNode * ArgNode = dyn_cast_or_null < MDNode > ( Node -> getOperand ( i + <NUM_LIT> ) ) ; if ( ArgNode -> getNumOperands ( ) != ExpectNumArgNodeOps ) return nullptr ; if ( ! ArgNode -> getOperand ( <NUM_LIT> ) ) return nullptr ; assert ( cast < MDString > ( ArgNode -> getOperand ( <NUM_LIT> ) ) -> getString ( ) == KernelArgMDNodeNames [


<s> MDNode * ArgAQNode = cast < MDNode > ( KernelMDNode -> getOperand ( <NUM_LIT> ) ) ; return cast < MDString > ( ArgAQNode -> getOperand ( ArgIdx + <NUM_LIT> )


<s> FunctionType * FT = F -> getFunctionType ( ) ; SmallVector < Type * , <NUM_LIT> > ArgTypes ; KernelArgMD NewArgMDs ; PushArgMD ( NewArgMDs , GetArgMD ( KernelMDNode , <NUM_LIT> ) ) ; for ( unsigned i = <NUM_LIT> ; i < FT -> getNumParams ( ) ; ++ i ) { ArgTypes . push_back ( FT -> getParamType ( i ) ) ; MDVector ArgMD = GetArgMD ( KernelMDNode , i + <NUM_LIT> ) ; PushArgMD ( NewArgMDs , ArgMD ) ; if ( ! IsImageType ( ArgTypeFromMD ( KernelMDNode , i ) ) ) continue ; ArgTypes . push_back ( ImageSizeType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageSizeArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; ArgTypes . push_back ( ImageFormatType ) ; ArgMD [ <NUM_LIT> ] = ArgMD [ <NUM_LIT> ] = MDString :: get ( * Context , ImageFormatArgMDType ) ; PushArgMD ( NewArgMDs , ArgMD ) ; Modified = true ; } if ( ! Modified ) { return std :: make_tuple ( nullptr , nullptr ) ; } auto NewFT = FunctionType :: get ( FT -> getReturnType ( ) , ArgTypes , false ) ; auto NewF = Function :: Create ( NewFT , F -> getLinkage ( ) , F -> getName ( ) ) ; ValueToValueMapTy VMap ; auto NewFArgIt = NewF -> arg_begin ( ) ; for ( auto & Arg : F -> args ( ) ) { auto ArgName = Arg . getName ( ) ; NewFArgIt -> setName ( ArgName ) ; VMap [ & Arg ] = & ( * NewFArgIt ++ ) ; if ( IsImageType ( ArgTypeFromMD ( KernelMDNode , Arg . getArgNo ( ) ) ) ) { ( NewFArgIt ++ ) -> setName ( Twine ( <STR_LIT> ) + ArgName ) ; ( NewFArgIt ++ ) -> setName ( Twine ( <STR_LIT> ) + ArgName ) ; } } SmallVector < ReturnInst * , <NUM_LIT> > Returns ; CloneFunctionInto ( NewF , F , VMap


<s> MDNode * ArgTypeNode = cast < MDNode > ( KernelMDNode -> getOperand ( <NUM_LIT> ) ) ; return cast < MDString > ( ArgTypeNode -> getOperand ( ArgIdx + <NUM_LIT> )


<s> static StringRef ArgTypeFromMD ( MDNode * KernelMDNode , unsigned ArgIdx ) { MDNode * ArgTypeNode = cast < MDNode > ( KernelMDNode -> getOperand ( <NUM_LIT> ) ) ; return cast < MDString > ( ArgTypeNode -> getOperand (

ArgIdx + <NUM_LIT> ) ) -> getString ( ) ;


<s> static MDVector GetArgMD ( MDNode * KernelMDNode , unsigned OpIdx ) { MDVector Res ; for ( unsigned i = <NUM_LIT> ; i < NumKernelArgMDNodes ; ++ i ) { MDNode * Node = cast < MDNode > (

KernelMDNode -> getOperand ( i + <NUM_LIT> ) ) ;


<s> for ( size_t i = <NUM_LIT> ; i < NumKernelArgMDNodes ; ++ i ) { MDNode * ArgNode = dyn_cast_or_null < MDNode > ( Node -> getOperand ( i + <NUM_LIT> ) ) ; if ( ArgNode -> getNumOperands ( ) != ExpectNumArgNodeOps ) return nullptr ; if ( ! ArgNode -> getOperand ( <NUM_LIT> ) ) return nullptr ; MDString * StringNode = dyn_cast < MDString > ( ArgNode -> getOperand ( <NUM_LIT> ) ) ; if ( ! StringNode || StringNode -> getString ( ) != KernelArgMDNodeNames [ i


<s> uint32_t ResourceID ; if ( AccessQual == <STR_LIT> ) { ResourceID = NumReadOnlyImageArgs ++ ; } else if ( AccessQual == <STR_LIT> ) { ResourceID = NumWriteOnlyImageArgs ++ ; } else { llvm_unreachable ( <STR_LIT> ) ; } Argument & SizeArg = * ( ++ ArgI ) ; Argument & FormatArg = * ( ++ ArgI ) ; Modified |= replaceImageUses ( Arg , ResourceID , SizeArg , FormatArg ) ; } else if ( IsSamplerType ( Type ) ) { uint32_t ResourceID = NumSamplerArgs ++ ; Modified |= replaceSamplerUses ( Arg , ResourceID ) ; } } for ( unsigned i = <NUM_LIT> ; i < InstsToErase . size ( ) ; ++ i ) { InstsToErase [ i ] -> eraseFromParent


<s> auto Inst = dyn_cast < CallInst > ( Use . getUser ( ) ) ; if ( ! Inst ) { continue ; } Function * F = Inst -> getCalledFunction ( ) ; if ( ! F ) continue ; Value * Replacement = nullptr ; StringRef Name = F -> getName ( ) ; if ( Name . startswith ( GetImageResourceIDFunc ) ) { Replacement = ConstantInt :: get ( Int32Type , ResourceID ) ; } else if ( Name . startswith ( GetImageSizeFunc ) ) { Replacement = & ImageSizeArg ; } else if ( Name . startswith ( GetImageFormatFunc ) ) { Replacement = & ImageFormatArg ; } else { continue ; } Inst -> replaceAllUsesWith ( Replacement )


<s> Context = & M . getContext ( ) ; Int32Type = Type :: getInt32Ty ( M . getContext ( ) ) ; ImageSizeType = ArrayType :: get ( Int32Type , <NUM_LIT> ) ; ImageFormatType = ArrayType :: get ( Int32Type , <NUM_LIT> ) ; return transformKernels (


<s> msgpack :: MapDocNode <ISA_LIT>PALMetadata :: getShaderFunction ( StringRef Name ) { auto Functions = getShaderFunctions (


<s> if ( ShaderFunctions . isEmpty ( ) ) ShaderFunctions


<s> msgpack :: MapDocNode <ISA_LIT>PALMetadata :: getShaderFunctions ( ) { if ( ShaderFunctions . isEmpty ( ) ) ShaderFunctions = refShaderFunctions ( ) ; return ShaderFunctions .


<s> case CallingConv :: <ISA_LIT>_PS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_VS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_GS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_ES : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_HS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_LS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_Gfx : llvm_unreachable ( <STR_LIT> ) ; default : return <STR_LIT>


<s> auto & N = MsgPackDoc . getRoot ( ) . getMap ( true ) [ MsgPackDoc . getNode ( <STR_LIT> ) ]

. getArray ( true ) [ <NUM_LIT> ] . getMap ( true ) [ MsgPackDoc . getNode ( <STR_LIT> ) ] ;


<s> void <ISA_LIT>PALMetadata :: readFromIR ( Module & M ) { auto NamedMD = M . getNamedMetadata ( <STR_LIT> ) ; if ( NamedMD && NamedMD -> getNumOperands ( ) ) { BlobType = ELF :: NT_<ISA_LIT>_METADATA ; auto MDN = dyn_cast < MDTuple > ( NamedMD -> getOperand ( <NUM_LIT> ) ) ; if ( MDN && MDN -> getNumOperands ( ) ) { if ( auto MDS = dyn_cast < MDString > ( MDN -> getOperand ( <NUM_LIT> ) ) ) setFromMsgPackBlob ( MDS -> getString ( ) ) ; } return ; } BlobType = ELF :: NT_AMD_<ISA_LIT>_PAL_METADATA ; NamedMD = M . getNamedMetadata ( <STR_LIT> ) ; if ( ! NamedMD || ! NamedMD -> getNumOperands ( ) ) return ; auto Tuple = dyn_cast < MDTuple > ( NamedMD -> getOperand (


<s> if ( NamedMD && NamedMD -> getNumOperands ( ) ) { BlobType = ELF :: NT_<ISA_LIT>_METADATA ; auto MDN = dyn_cast < MDTuple > ( NamedMD -> getOperand ( <NUM_LIT> ) ) ; if ( MDN && MDN -> getNumOperands ( ) ) { if ( auto MDS = dyn_cast < MDString > ( MDN -> getOperand ( <NUM_LIT> ) ) ) setFromMsgPackBlob ( MDS -> getString ( ) ) ; } return ; } BlobType = ELF :: NT_AMD_<ISA_LIT>_PAL_METADATA ; NamedMD = M . getNamedMetadata ( <STR_LIT> ) ; if ( ! NamedMD || ! NamedMD -> getNumOperands ( ) ) return ; auto Tuple = dyn_cast < MDTuple > ( NamedMD -> getOperand ( <NUM_LIT> ) ) ; if ( ! Tuple ) return ; for ( unsigned I = <NUM_LIT> , E = Tuple -> getNumOperands ( ) & - <NUM_LIT> ; I != E ; I += <NUM_LIT> ) { auto Key = mdconst :: dyn_extract < ConstantInt > ( Tuple -> getOperand (


<s> if ( Type == ELF :: NT_AMD_PAL_METADATA ) return setFromLegacyBlob


<s> auto Node = getShaderFunction ( MF . getFunction ( ) . getName ( ) ) ; Node [ <STR_LIT> ] = MsgPackDoc . getNode (


<s> void <ISA_LIT>PALMetadata :: setFunctionNumUsedSgprs ( const MachineFunction & MF


<s> void <ISA_LIT>PALMetadata :: setFunctionNumUsedVgprs ( const MachineFunction & MF ,


<s> auto Node = getShaderFunction ( MF . getFunction ( ) . getName ( ) ) ; Node [ <STR_LIT> ] = MsgPackDoc . getNode ( Val


<s> if ( Type == ELF :: NT_AMD_PAL_METADATA ) toLegacyBlob ( Blob ) ; else if ( Type ) toMsgPackBlob


<s> msgpack :: MapDocNode <ISA_LIT>PALMetadata :: getHwStage ( unsigned CC ) { if ( HwStages . isEmpty ( ) ) HwStages = MsgPackDoc . getRoot ( ) . getMap ( true ) [ <STR_LIT> ] . getArray ( true ) [ <NUM_LIT> ] . getMap ( true ) [ <STR_LIT>


<s> msgpack :: MapDocNode <ISA_LIT>PALMetadata :: getHwStage ( unsigned CC ) { if ( HwStages . isEmpty ( ) ) HwStages = MsgPackDoc . getRoot ( ) . getMap ( true ) [ <STR_LIT> ] . getArray ( true ) [ <NUM_LIT> ] . getMap ( true ) [ <STR_LIT> ] . getMap ( true ) ; return HwStages . getMap ( ) [ getStageName ( CC ) ] .


<s> auto N = It -> second ; if ( N . getKind ( ) != msgpack :: Type ::


<s> msgpack :: MapDocNode <ISA_LIT>PALMetadata :: getRegisters ( ) { if ( Registers . isEmpty ( ) ) Registers = refRegisters ( ) ; return Registers . getMap (


<s> case CallingConv :: <ISA_LIT>_ES : return PALMD :: R_2CCA_SPI_SHADER_PGM_RSRC1_ES ; case CallingConv :: <ISA_LIT>_GS : return PALMD :: R_2C8A_SPI_SHADER_PGM_RSRC1_GS ; case CallingConv :: <ISA_LIT>_VS : return PALMD ::


<s> switch ( CC ) { case CallingConv :: <ISA_LIT>_PS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_VS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_GS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_ES : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_HS : return <STR_LIT> ; case CallingConv :: <ISA_LIT>_LS : return <STR_LIT> ; default : return <STR_LIT>


<s> unsigned <ISA_LIT>PALMetadata :: getType ( ) const { return BlobType


<s> unsigned <ISA_LIT>PALMetadata :: getType


<s> bool <ISA_LIT>PALMetadata :: isLegacy ( )


<s> void <ISA_LIT>PALMetadata :: setEntryPoint ( unsigned CC , StringRef Name


<s> for ( unsigned I = <NUM_LIT> ; I != Blob . size ( ) / sizeof ( uint32_t ) / <NUM_LIT> ; ++ I

) setRegister ( Data [ I * <NUM_LIT> ] , Data [ I * <NUM_LIT> + <NUM_LIT> ] ) ;


<s> return MsgPackDoc . readFromBlob ( Blob ,


<s> bool <ISA_LIT>PALMetadata :: setFromString ( StringRef S ) { BlobType = ELF :: NT_<ISA_LIT>_METADATA ; if ( ! MsgPackDoc . fromYAML ( S ) ) return false ; auto & RegsObj = refRegisters ( ) ; auto OrigRegs = RegsObj ; RegsObj = MsgPackDoc . getMapNode ( ) ; Registers = RegsObj . getMap ( ) ; bool Ok = true ; for ( auto I : OrigRegs . getMap ( ) ) { auto Key = I . first ; if ( Key . getKind ( ) == msgpack :: Type :: String ) { StringRef S = Key . getString ( ) ; uint64_t Val ; if ( S . consumeInteger ( <NUM_LIT> ,


<s> void <ISA_LIT>PALMetadata :: setLegacy ( )


<s> void <ISA_LIT>PALMetadata :: setLegacy ( ) { BlobType = ELF


<s> unsigned NumUsedSgprsKey = getScratchSizeKey ( CC ) + PALMD :: Key :: VS_NUM_USED_SGPRS - PALMD :: Key :: VS_SCRATCH_<ISA_LIT>ZE ; setRegister ( NumUsedSgprsKey , Val ) ; return ; } getHwStage ( CC ) [ <STR_LIT> ] = MsgPackDoc .


<s> unsigned NumUsedVgprsKey = getScratchSizeKey ( CC ) + PALMD :: Key :: VS_NUM_USED_VGPRS - PALMD :: Key :: VS_SCRATCH_<ISA_LIT>ZE ; setRegister ( NumUsedVgprsKey , Val ) ; return ; } getHwStage ( CC ) [ <STR_LIT> ] = MsgPackDoc .


<s> auto & N = getRegisters ( ) [ MsgPackDoc . getNode ( Reg ) ] ; if ( N . getKind ( ) == msgpack :: Type :: UInt ) Val |= N . getUInt ( ) ; N = N . getDocument ( ) ->


<s> setRegister ( getRsrc1Reg ( CC ) + <NUM_LIT> ,


<s> setRegister ( PALMD :: R_A1B4_SPI_PS_INPUT_ADDR


<s> void <ISA_LIT>PALMetadata :: setSpiPsInputEna ( unsigned Val ) { setRegister ( PALMD :: R_A1B3_SPI_PS_INPUT_ENA , Val


<s> void <ISA_LIT>PALMetadata :: toBlob ( unsigned Type , std :: string &


<s> if ( Type == ELF :: NT_AMD_<ISA_LIT>_PAL_METADATA ) toLegacyBlob ( Blob ) ; else if ( Type ) toMsgPackBlob ( Blob


<s> raw_string_ostream OS ( Blob ) ; <STR_LIT> :: Writer EW ( OS , <STR_LIT> :: little ) ; for ( auto I : Registers . getMap ( ) ) { EW . write ( uint32_t ( I . first


<s>
raw_string_ostream
Stream
(
String
)
;
if
(
isLegacy
(
)
)
{
if
(
MsgPackDoc
.
getRoot
(
)
.
getKind
(
)
==
msgpack
::
Type
::
)
return
;
Stream
<<
'\t'
<<
<STR_LIT>
::
AssemblerDirective
<<
' '
;
auto
Regs
=
getRegisters
(
)
;
for
(
auto
I
=
Regs
.
begin
(
)
,
E
=
Regs
.
end
(
)
;
I
!=
E
;
++
I
)
{
if
(
I
!=
Regs
.
begin
(
)
)
Stream
<<
','
;
unsigned
Reg
=
I
->
first
.
getUInt
(
)
;
unsigned
Val
=
I
->
second
.
getUInt
(
)
;
Stream
<<
<STR_LIT>
<<
Twine
::
utohexstr
(
Reg
)
<<
<STR_LIT>
<<
Twine
::
utohexstr
(
Val
)
;
}
Stream
<<
'\n'
;
return
;
}
MsgPackDoc
.
setHexMode
(
)
;
auto
&
RegsObj
=
refRegisters
(


<s> static std :: pair < const Value * , const Type * > getMemoryInstrPtrAndType ( const Instruction * Inst ) { if ( auto LI = dyn_cast < LoadInst > ( Inst ) ) return { LI -> getPointerOperand ( ) , LI -> getType ( ) } ; if ( auto <ISA_LIT> = dyn_cast < StoreInst > ( Inst ) ) return { <ISA_LIT> -> getPointerOperand ( ) , <ISA_LIT> -> getValueOperand ( ) -> getType ( ) } ; if ( auto AI = dyn_cast < AtomicCmpXchgInst > ( Inst ) ) return { AI -> getPointerOperand ( ) , AI -> getCompareOperand ( ) -> getType ( ) } ; if ( auto AI = dyn_cast < AtomicRMWInst > ( Inst ) ) return { AI -> getPointerOperand ( ) , AI -> getValOperand ( ) -> getType ( ) } ; if ( auto MI = dyn_cast < AnyMemIntrinsic > ( Inst ) ) return { MI -> getRawDest ( ) , Type :: getInt8Ty ( MI -> getContext ( ) ) } ; return { nullptr


<s> if ( auto PT = dyn_cast < PointerType > ( V -> getType ( ) ) ) { unsigned As = PT ->


<s> if ( auto PT = dyn_cast < PointerType > ( V


<s> bool <ISA_LIT>PerfHint :: isIndirectAccess ( const Instruction * Inst ) const { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Inst << '\n' ) ; SmallSet < const Value * , <NUM_LIT> > WorkSet ; SmallSet < const Value * , <NUM_LIT> > Visited ; if ( const Value * MO = getMemoryInstrPtrAndType ( Inst ) . first ) { if ( isGlobalAddr ( MO ) ) WorkSet . insert ( MO ) ; } while ( ! WorkSet . empty ( ) ) { const Value * V = * WorkSet . begin ( ) ; WorkSet . erase ( * WorkSet . begin ( ) ) ; if ( ! Visited . insert ( V ) . second ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * V << '\n' ) ; if ( auto LD = dyn_cast < LoadInst > ( V ) ) { auto M = LD -> getPointerOperand ( ) ; if ( isGlobalAddr ( M ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return true ; } continue ; } if ( auto GEP = dyn_cast < GetElementPtrInst > ( V ) ) { auto P = GEP -> getPointerOperand ( ) ; WorkSet . insert ( P ) ; for ( unsigned I = <NUM_LIT> , E = GEP -> getNumIndices ( ) + <NUM_LIT> ; I != E ; ++ I ) WorkSet . insert ( GEP -> getOperand ( I ) ) ; continue ; } if ( auto U = dyn_cast < UnaryInstruction > ( V ) ) { WorkSet . insert ( U -> getOperand ( <NUM_LIT> ) ) ; continue ; } if ( auto BO = dyn_cast < BinaryOperator > ( V ) ) { WorkSet . insert ( BO -> getOperand ( <NUM_LIT> ) ) ; WorkSet . insert ( BO -> getOperand ( <NUM_LIT> ) ) ; continue ; } if ( auto S = dyn_cast < SelectInst > ( V ) ) { WorkSet . insert ( S -> getFalseValue ( ) ) ; WorkSet . insert ( S -> getTrueValue ( ) ) ; continue ; } if ( auto E = dyn_cast < ExtractElementInst > ( V ) ) { WorkSet . insert ( E -> getVectorOperand ( ) ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT>


<s> } if ( auto GEP = dyn_cast < GetElementPtrInst > ( V ) ) { auto P = GEP -> getPointerOperand ( ) ; WorkSet . insert ( P ) ; for ( unsigned I = <NUM_LIT> , E = GEP -> getNumIndices ( ) + <NUM_LIT> ; I != E ; ++ I ) WorkSet . insert ( GEP -> getOperand ( I ) ) ; continue ; } if ( auto U = dyn_cast < UnaryInstruction > ( V ) ) { WorkSet . insert ( U -> getOperand ( <NUM_LIT> ) ) ; continue ; } if ( auto BO = dyn_cast < BinaryOperator > ( V ) ) { WorkSet . insert ( BO -> getOperand ( <NUM_LIT> ) ) ; WorkSet . insert ( BO -> getOperand ( <NUM_LIT> ) ) ; continue ; } if ( auto S = dyn_cast < SelectInst > ( V ) ) { WorkSet . insert ( S -> getFalseValue ( ) ) ; WorkSet . insert ( S -> getTrueValue


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Inst << '\n' ) ; MemAccessInfo MAI = makeMemAccessInfo ( const_cast < Instruction * > ( Inst ) ) ; bool IsLargeStride = MAI . isLargeStride (


<s> bool <ISA_LIT>PerfHint :: isLocalAddr ( const Value * V ) const { if ( auto PT = dyn_cast < PointerType > ( V -> getType ( ) ) ) return PT -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ; return false


<s> bool <ISA_LIT>PerfHint :: isLocalAddr ( const Value * V ) const { if ( auto PT = dyn_cast < PointerType > ( V -> getType ( ) ) ) return PT -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ; return


<s> bool <ISA_LIT>PerfHint :: isMemBound ( const <ISA_LIT>PerfHintAnalysis :: FuncInfo & FI ) { return FI . MemInstCost * <NUM_LIT> / FI . InstCost >


<s> <ISA_LIT>PerfHint :: MemAccessInfo <ISA_LIT>PerfHint :: makeMemAccessInfo ( Instruction * Inst ) const { MemAccessInfo MAI ; const Value * MO = getMemoryInstrPtrAndType ( Inst )


<s> <ISA_LIT>PerfHint :: MemAccessInfo <ISA_LIT>PerfHint :: makeMemAccessInfo ( Instruction * Inst ) const { MemAccessInfo MAI ; const Value * MO = getMemoryInstrPtrAndType ( Inst ) . first ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MO << '\n' ) ; if ( isLocalAddr ( MO ) ) return MAI ; MAI . V = MO ; MAI . Base = GetPointerBaseWithConstantOffset ( MO , MAI .


<s> bool <ISA_LIT>PerfHint :: needLimitWave ( const <ISA_LIT>PerfHintAnalysis :: FuncInfo & FI ) { return ( ( FI . MemInstCost + FI . IAMInstCost * IAWeight + FI . LSMInstCost * LSWeight ) * <NUM_LIT> /


<s> auto FI = FIM . find ( F ) ; if ( FI == FIM . end ( )


<s> return Printable ( [ this ] ( raw_ostream & OS ) { OS << <STR_LIT> << * V << '\n' << <STR_LIT> << * Base << <STR_LIT> << Offset


<s> DL = & M . getDataLayout ( ) ; if ( F . hasFnAttribute ( <STR_LIT> ) && F . hasFnAttribute ( <STR_LIT> ) ) return false ; const <ISA_LIT>PerfHintAnalysis :: FuncInfo * Info = visit ( F ) ; LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << Info -> MemInstCost << '\n' << <STR_LIT> << Info -> IAMInstCost << '\n' << <STR_LIT> << Info -> LSMInstCost << '\n' << <STR_LIT> << Info -> InstCost << '\n' ) ; bool Changed = false ; if ( isMemBound ( * Info ) ) { LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> ) ; NumMemBound ++ ; F . addFnAttr ( <STR_LIT> , <STR_LIT> ) ; Changed = true ; } if ( <STR_LIT> ( F . getCallingConv ( ) ) && needLimitWave (


<s> if ( ! F || F -> isDeclaration ( ) ) continue ; const TargetSubtargetInfo * ST = TM . getSubtargetImpl ( * F ) ; <ISA_LIT>PerfHint Analyzer ( FIM , ST -> getTargetLowering


<s> if ( isIndirectAccess ( & I ) ) FI . IAMInstCost += Size ; if ( isLargeStride ( & I ) ) FI . LSMInstCost += Size ; FI . MemInstCost += Size ; FI . InstCost += Size ; continue ; } if ( auto * CB = dyn_cast < CallBase > ( & I ) ) { Function * Callee = CB -> getCalledFunction ( ) ; if ( ! Callee || Callee -> isDeclaration ( ) ) { ++ FI . InstCost ; continue ; } if ( & F == Callee ) continue ; auto Loc = FIM . find ( Callee ) ; if ( Loc == FIM . end ( ) ) continue ; FI . MemInstCost += Loc -> second . MemInstCost ; FI . InstCost += Loc -> second . InstCost ; FI . IAMInstCost += Loc -> second . IAMInstCost ; FI . LSMInstCost += Loc -> second . LSMInstCost ; } else if ( auto * GEP = dyn_cast < GetElementPtrInst > ( & I ) ) { TargetLoweringBase :: AddrMode AM ; auto * Ptr = GetPointerBaseWithConstantOffset ( GEP , AM . BaseOffs , * DL ) ; AM . BaseGV = dyn_cast_or_null < GlobalValue > ( const_cast < Value * >


<s> if ( ! Callee || Callee -> isDeclaration ( ) ) { ++ FI . InstCost ; continue ; } if ( & F == Callee ) continue ; auto Loc = FIM . find ( Callee ) ; if ( Loc == FIM . end ( ) ) continue ; FI . MemInstCost += Loc -> second . MemInstCost ; FI . InstCost += Loc -> second . InstCost ; FI . IAMInstCost += Loc -> second . IAMInstCost ; FI . LSMInstCost += Loc -> second . LSMInstCost ; } else if ( auto * GEP = dyn_cast < GetElementPtrInst > ( & I ) ) { TargetLoweringBase :: AddrMode AM ; auto * Ptr = GetPointerBaseWithConstantOffset ( GEP , AM . BaseOffs , * DL ) ; AM . BaseGV = dyn_cast_or_null < GlobalValue > ( const_cast < Value * > ( Ptr ) ) ; AM . HasBaseReg = ! AM . BaseGV ; if ( TLI -> isLegalAddressingMode ( * DL , AM , GEP -> getResultElementType ( ) , GEP -> getPointerAddressSpace ( ) ) ) continue ; ++ FI . InstCost ; }


<s> bool <ISA_LIT>PerfHint :: runOnFunction ( Function & F ) { const Module & M = * F . getParent ( ) ; DL = & M . getDataLayout ( ) ; if ( F . hasFnAttribute ( <STR_LIT> ) && F . hasFnAttribute ( <STR_LIT> ) ) return false ; const <ISA_LIT>PerfHintAnalysis :: FuncInfo * Info = visit ( F ) ; LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << Info -> MemInstCost << '\n' << <STR_LIT> << Info -> IAMInstCost << '\n' << <STR_LIT> << Info -> LSMInstCost << '\n' << <STR_LIT> << Info -> InstCost << '\n' ) ; if ( isMemBound ( * Info ) ) { LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> ) ; NumMemBound ++ ; F . addFnAttr ( <STR_LIT> , <STR_LIT> ) ; } if ( <STR_LIT> ( F . getCallingConv ( ) ) && needLimitWave ( * Info ) ) { LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> ) ; NumLimitWave


<s> NumMemBound ++ ; F . addFnAttr ( <STR_LIT> , <STR_LIT> ) ; } if ( <STR_LIT> ( F . getCallingConv ( ) ) && needLimitWave ( * Info ) ) { LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> ) ; NumLimitWave ++ ; F . addFnAttr ( <STR_LIT> , <STR_LIT> ) ; } return


<s> Function * Callee = CS . getCalledFunction ( ) ; if ( ! Callee || Callee -> isDeclaration ( ) ) { ++ FI . InstCount ; continue ; } if ( & F == Callee ) continue ; auto Loc = FIM . find ( Callee ) ; if ( Loc == FIM . end ( ) ) continue ; FI . MemInstCount += Loc -> second . MemInstCount ; FI . InstCount += Loc -> second . InstCount ; FI . IAMInstCount += Loc -> second . IAMInstCount ; FI . LSMInstCount += Loc -> second . LSMInstCount ; } else if ( auto * GEP = dyn_cast < GetElementPtrInst > ( & I ) ) { TargetLoweringBase :: AddrMode AM ; auto * Ptr = GetPointerBaseWithConstantOffset ( GEP , AM . BaseOffs , * DL ) ; AM . BaseGV = dyn_cast_or_null < GlobalValue > ( const_cast < Value * > ( Ptr ) ) ; AM . HasBaseReg = ! AM . BaseGV ; if ( TLI -> isLegalAddressingMode ( * DL , AM , GEP -> getResultElementType ( ) , GEP -> getPointerAddressSpace ( ) ) ) continue ; ++ FI . InstCount ; } else { ++ FI .


<s> assert ( Loc != FIM . end ( ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << Loc -> second . MemInstCount << '\n' << <STR_LIT> << Loc -> second . IAMInstCount << '\n' <<

<STR_LIT> << Loc -> second . LSMInstCount << '\n' << <STR_LIT> << Loc -> second . InstCount << '\n' ) ;


<s> LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << Loc -> second . MemInstCount << '\n' << <STR_LIT> << Loc -> second . IAMInstCount << '\n' << <STR_LIT> << Loc -> second . LSMInstCount << '\n' << <STR_LIT> << Loc -> second . InstCount << '\n' ) ; auto & FI = Loc -> second ; if ( isMemBound ( FI ) ) { LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> ) ; NumMemBound ++


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << F . getName ( ) << '\n' ) ; for ( auto & B : F ) { LastAccess = MemAccessInfo ( ) ; for ( auto & I : B ) { if ( getMemoryInstrPtr ( & I ) ) { if ( isIndirectAccess ( & I ) ) ++ FI . IAMInstCount ; if ( isLargeStride ( & I ) ) ++ FI . LSMInstCount ; ++ FI . MemInstCount ; ++ FI . InstCount ; continue ; } if ( auto * CB = dyn_cast < CallBase > ( & I ) ) { Function * Callee = CB -> getCalledFunction ( ) ; if ( ! Callee || Callee -> isDeclaration ( )


<s> if ( Loc == FIM . end ( ) ) continue ; FI . MemInstCount += Loc -> second . MemInstCount ; FI . InstCount += Loc -> second . InstCount ; FI . IAMInstCount += Loc -> second . IAMInstCount ; FI . LSMInstCount += Loc -> second . LSMInstCount ; } else if ( auto * GEP = dyn_cast < GetElementPtrInst > ( & I ) ) { TargetLoweringBase :: AddrMode AM ; auto * Ptr = GetPointerBaseWithConstantOffset ( GEP , AM . BaseOffs , * DL ) ; AM . BaseGV = dyn_cast_or_null < GlobalValue > ( const_cast < Value * > ( Ptr ) ) ; AM . HasBaseReg = ! AM . BaseGV ; if ( TLI -> isLegalAddressingMode ( * DL , AM , GEP -> getResultElementType ( ) , GEP -> getPointerAddressSpace ( ) ) ) continue ; ++ FI


<s> bool <ISA_LIT>PerfHint :: isGlobalAddr ( const Value * V ) const { if ( auto PT = dyn_cast < PointerType > ( V -> getType ( ) ) ) { unsigned As = PT -> getAddressSpace


<s> if ( auto PT = dyn_cast < PointerType > ( V -> getType ( )

) ) return PT -> getAddressSpace ( ) == AS . LOCAL_ADDRESS ;


<s> return FI . MemInstCount * <NUM_LIT> / FI


<s> const Value * MO = getMemoryInstrPtr ( Inst ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MO << '\n' ) ; if ( isLocalAddr ( MO


<s> bool <ISA_LIT>PerfHintAnalysis :: runOnFunction ( Function & F ) { auto * TPC = getAnalysisIfAvailable < TargetPassConfig > ( ) ; if ( ! TPC ) return false ; const TargetMachine & TM = TPC -> getTM < TargetMachine > ( ) ; const TargetSubtargetInfo * ST = TM . getSubtargetImpl


<s> FI . IAMInstCount += Loc -> second . IAMInstCount ; FI . LSMInstCount += Loc -> second . LSMInstCount ; } else if ( auto * GEP = dyn_cast < GetElementPtrInst > ( & I ) ) { TargetLoweringBase :: AddrMode AM ; auto * Ptr = GetPointerBaseWithConstantOffset ( GEP , AM . BaseOffs , * DL ) ; AM . BaseGV = dyn_cast_or_null < GlobalValue > ( const_cast < Value * > ( Ptr ) ) ; AM . HasBaseReg = ! AM . BaseGV ; if ( TLI -> isLegalAddressingMode ( * DL , AM , GEP -> getResultElementType ( ) , GEP -> getPointerAddressSpace ( ) ) ) continue ; ++ FI .


<s> ++ FI . MemInstCount ; ++ FI . InstCount ; continue ; } CallSite CS ( const_cast < Instruction * > ( & I ) ) ; if ( CS ) { Function * Callee = CS . getCalledFunction ( ) ; if ( ! Callee || Callee -> isDeclaration ( ) ) { ++ FI . InstCount ; continue ; } if ( & F == Callee ) continue ; visit ( * Callee ) ; auto Loc = FIM . find ( Callee ) ; assert ( Loc != FIM . end ( ) && <STR_LIT> ) ; FI . MemInstCount += Loc -> second . MemInstCount ; FI . InstCount += Loc -> second . InstCount ; FI . IAMInstCount += Loc -> second . IAMInstCount ; FI . LSMInstCount += Loc -> second . LSMInstCount ; } else if ( auto * GEP = dyn_cast < GetElementPtrInst > ( & I ) ) { TargetLoweringBase :: AddrMode AM ; auto * Ptr = GetPointerBaseWithConstantOffset ( GEP , AM . BaseOffs , * DL ) ; AM . BaseGV = dyn_cast_or_null < GlobalValue > ( const_cast < Value * > ( Ptr ) ) ; AM . HasBaseReg = ! AM . BaseGV ; if ( TLI -> isLegalAddressingMode ( * DL , AM , GEP -> getResultElementType ( ) , GEP -> getPointerAddressSpace ( ) ) ) continue ; ++ FI . InstCount ; }


<s> if ( Ty == S32 ) { B . buildInstr ( <STR_LIT> , { DstReg } , { SrcReg } , MI . getFlags ( ) ) ; } else { auto Cvt0 = B . buildInstr ( <STR_LIT> , { S32 } , { SrcReg } , MI . getFlags ( ) ) ; B . buildFPTrunc ( DstReg , Cvt0 , MI . getFlags ( ) ) ; } MI . eraseFromParent (


<s> MachineIRBuilder B ( MI ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT Ty = B . getMRI (


<s> if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_SHL : case TargetOpcode :: G_LSHR : case TargetOpcode :: G_ASHR


<s> CombinerHelper Helper ( Observer , B , KB , MDT , LInfo ) ; <ISA_LIT>GenPostLegalizerCombinerHelper Generated ( GeneratedRuleCfg ) ; if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode


<s> <ISA_LIT>PostLegalizerCombinerHelper ( MachineIRBuilder & B , CombinerHelper & Helper ) : B ( B ) , MF ( B . getMF ( )

) , MRI ( * B . getMRI ( ) ) , Helper ( Helper ) { }


<s> <ISA_LIT>PostLegalizerCombinerHelper ( MachineIRBuilder & B , CombinerHelper & Helper ) : B ( B ) , MF ( B . getMF ( ) ) , MRI (

* B . getMRI ( ) ) , Helper ( Helper ) { }


<s> unsigned NewOpc = <STR_LIT> + MatchInfo . ShiftOffset / <NUM_LIT> ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; Register CvtSrc = MatchInfo . CvtVal ; LLT SrcTy = MRI . getType ( MatchInfo . CvtVal ) ; if ( SrcTy != S32 ) { assert ( SrcTy . isScalar ( ) && SrcTy . getSizeInBits ( ) >= <NUM_LIT> ) ; CvtSrc = B . buildAnyExt ( S32 , CvtSrc ) . getReg ( <NUM_LIT> ) ; } assert ( MI . getOpcode ( ) != NewOpc ) ; B . buildInstr ( NewOpc , { MI . getOperand ( <NUM_LIT> ) } , {


<s> B . buildInstr ( Opc , { MI . getOperand ( <NUM_LIT> ) } , { X , Y } , MI . getFlags ( ) ) ; } ; switch ( Info . Pred ) { case CmpInst :: FCMP_ULT : case CmpInst :: FCMP_ULE : if ( Info . LHS == Info . True ) buildNewInst ( <STR_LIT> , Info . RHS , Info . LHS ) ; else buildNewInst ( <STR_LIT> , Info . LHS , Info . RHS ) ; break ; case CmpInst :: FCMP_OLE : case CmpInst ::


<s> <ISA_LIT>PostLegalizerCombinerHelper PostLegalizerHelper ( B , Helper ) ; <ISA_LIT>GenPostLegalizerCombinerHelper Generated ( GeneratedRuleCfg , Helper , PostLegalizerHelper ) ; if ( Generated . tryCombineAll ( Observer , MI , B ) ) return true ; switch ( MI .


<s> Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; mi_match ( SrcReg , MRI , m_GZExt ( m_Reg ( SrcReg ) ) ) ; Register Src0 ; int64_t ShiftAmt ; bool IsShr = mi_match ( SrcReg , MRI , m_GLShr ( m_Reg ( Src0 ) , m_ICst ( ShiftAmt ) ) ) ; if ( IsShr || mi_match ( SrcReg , MRI , m_GShl ( m_Reg ( Src0 ) , m_ICst ( ShiftAmt ) ) ) ) { const unsigned Offset = MI .


<s> Info . True = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Info . False = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! ( Info . LHS == Info . True && Info . RHS == Info . False ) && ! ( Info . LHS == Info . False && Info . RHS == Info . True ) ) return false ; switch ( Info . Pred ) { case CmpInst :: FCMP_FALSE : case CmpInst ::


<s> case TargetOpcode :: G_SHL : case TargetOpcode :: G_LSHR : case TargetOpcode :: G_ASHR : return Helper . tryCombineShiftToUnmerge ( MI , <NUM_LIT> ) ; } return


<s> <ISA_LIT>PostLegalizerCombinerHelper ( MachineIRBuilder & B , <ISA_LIT>CombinerHelper & Helper ) : B ( B ) , MF ( B . getMF ( ) ) , MRI ( * B . getMRI ( )


<s> <ISA_LIT>PostLegalizerCombinerHelper ( MachineIRBuilder & B , <ISA_LIT>CombinerHelper & Helper ) : B ( B ) , MF ( B . getMF ( ) ) , MRI ( * B . getMRI ( ) ) , Helper


<s> <ISA_LIT>PostLegalizerCombiner :: <ISA_LIT>PostLegalizerCombiner ( bool IsOptNone ) : MachineFunctionPass


<s> } case CmpInst :: FCMP_UGE : case CmpInst :: FCMP_UGT : { if ( Info . LHS == Info . True ) buildNewInst ( <STR_LIT> , Info . RHS , Info . LHS ) ; else buildNewInst ( <STR_LIT> , Info . LHS , Info . RHS ) ; break ; } case CmpInst :: FCMP_OGT : case CmpInst :: FCMP_OGE : { if ( Info . LHS == Info . True ) buildNewInst ( <STR_LIT> , Info . LHS , Info . RHS ) ; else buildNewInst ( <STR_LIT> , Info . RHS , Info . LHS ) ; break ; } default : llvm_unreachable ( <STR_LIT>


<s> LLT Ty = B . getMRI ( ) -> getType ( DstReg ) ; if ( Ty == S32 ) { B . buildInstr ( <STR_LIT> , { DstReg } , { MI . getOperand ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; } else { auto Cvt0 = B . buildInstr ( <STR_LIT> , { S32 } , { MI . getOperand ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; B . buildFPTrunc ( DstReg , Cvt0 , MI . getFlags ( ) ) ; } MI . eraseFromParent


<s> } else { auto Cvt0 = B . buildInstr ( <STR_LIT> , { S32 } , { MI . getOperand ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; B . buildFPTrunc ( DstReg , Cvt0 , MI .


<s> return new <ISA_LIT>PostLegalizerCombiner (


<s> FunctionPass * create<ISA_LIT>PostLegalizeCombiner ( bool IsOptNone ) { return new <ISA_LIT>PostLegalizerCombiner ( IsOptNone


<s> Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; mi_match ( SrcReg , MRI , m_GZExt ( m_Reg ( SrcReg ) ) ) ; Register Src0 ; int64_t ShiftAmt ; bool IsShr = mi_match ( SrcReg , MRI , m_GLShr ( m_Reg ( Src0 ) , m_ICst ( ShiftAmt ) ) ) ; if ( IsShr || mi_match ( SrcReg , MRI , m_GShl ( m_Reg ( Src0 ) , m_ICst ( ShiftAmt ) ) ) ) { const unsigned Offset = MI . getOpcode ( ) - <STR_LIT> ; unsigned ShiftOffset = <NUM_LIT> * Offset ; if ( IsShr ) ShiftOffset += ShiftAmt ; else ShiftOffset -=


<s> Info . True = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Info . False = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! ( Info . LHS == Info . True && Info . RHS == Info . False ) && ! ( Info . LHS == Info . False && Info . RHS == Info . True ) ) return false ; switch ( Info . Pred ) { case CmpInst :: FCMP_FALSE : case CmpInst :: FCMP_OEQ : case CmpInst :: FCMP_ONE : case CmpInst :: FCMP_ORD : case CmpInst :: FCMP_UNO : case CmpInst :: FCMP_UEQ : case CmpInst :: FCMP_UNE : case CmpInst :: FCMP_TRUE : return false ; default : return true


<s> static bool matchUCharToFloat ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineFunction & MF , CombinerHelper & Helper ) { Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT Ty = MRI . getType ( DstReg ) ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) || Ty == LLT :: scalar ( <NUM_LIT> ) ) { const APInt Mask = APInt ::

getHighBitsSet ( <NUM_LIT> , <NUM_LIT> ) ;


<s> if ( MF . getProperties ( ) . hasProperty ( MachineFunctionProperties :: Property :: FailedISel ) ) return false ; auto * TPC = & getAnalysis < TargetPassConfig > ( ) ; const Function & F = MF . getFunction ( ) ; bool EnableOpt = MF . getTarget ( ) . getOptLevel ( ) != CodeGenOpt :: None && ! skipFunction ( F ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>LegalizerInfo * LI = static_cast < const <ISA_LIT>LegalizerInfo * > ( ST . getLegalizerInfo ( ) ) ; GISelKnownBits * KB = & getAnalysis < GISelKnownBitsAnalysis > ( ) . get ( MF ) ; MachineDominatorTree * MDT = IsOptNone ? nullptr : & getAnalysis < MachineDominatorTree > ( ) ; <ISA_LIT>PostLegalizerCombinerInfo PCInfo ( EnableOpt , F . hasOptSize ( ) , F . hasMinSize ( ) , LI , KB , MDT ) ; Combiner C ( PCInfo , TPC ) ; return C . combineMachineInstrs (


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>LegalizerInfo * LI = static_cast < const <ISA_LIT>LegalizerInfo * > ( ST . getLegalizerInfo ( ) ) ; GISelKnownBits * KB = & getAnalysis < GISelKnownBitsAnalysis > ( ) . get ( MF ) ; MachineDominatorTree * MDT = IsOptNone ? nullptr : & getAnalysis < MachineDominatorTree > ( ) ; <ISA_LIT>PostLegalizerCombinerInfo PCInfo ( EnableOpt , F . hasOptSize ( ) , F . hasMinSize ( ) , LI , KB , MDT ) ; Combiner C ( PCInfo , TPC ) ; return C . combineMachineInstrs ( MF , nullptr


<s> if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_CONCAT_VECTORS : return Helper . tryCombineConcatVectors ( MI ) ; case TargetOpcode ::


<s> if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_CONCAT_VECTORS : return Helper . tryCombineConcatVectors ( MI ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return Helper . tryCombineShuffleVector ( MI ) ; } return


<s> <ISA_LIT>PreLegalizerCombinerHelper ( MachineIRBuilder & B , CombinerHelper & Helper ) : B ( B ) , MF ( B . getMF

( ) ) , MRI ( * B . getMRI ( ) ) , Helper ( Helper ) { }


<s> <ISA_LIT>PreLegalizerCombinerHelper ( MachineIRBuilder & B , CombinerHelper & Helper ) : B ( B ) , MF ( B . getMF ( )


<s> <ISA_LIT>PreLegalizerCombinerInfo ( bool EnableOpt , bool OptSize , bool MinSize , GISelKnownBits * KB , MachineDominatorTree * MDT ) : CombinerInfo ( true , false , nullptr , EnableOpt , OptSize , MinSize ) , KB (


<s> void <ISA_LIT>PreLegalizerCombinerHelper :: applyClampI64ToI16 ( MachineInstr & MI , const ClampI64ToI16MatchInfo & MatchInfo ) { Register Src = MatchInfo . Origin ; assert ( MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) . getType ( Src ) == LLT :: scalar ( <NUM_LIT> ) ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; B . setMBB ( * MI . getParent ( ) ) ; B . setInstrAndDebugLoc ( MI ) ; auto Unmerge = B . buildUnmerge ( S32 , Src ) ; assert ( MI . getOpcode ( ) != <STR_LIT> ) ; const LLT V2S16 = LLT :: fixed_vector ( <NUM_LIT> , <NUM_LIT> ) ; auto CvtPk = B . buildInstr ( <STR_LIT> , { V2S16 } , { Unmerge . getReg ( <NUM_LIT> ) , Unmerge . getReg ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; auto MinBoundary = std :: min ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MaxBoundary = std :: max ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MinBoundaryDst = B . buildConstant ( S32 , MinBoundary ) ; auto MaxBoundaryDst = B . buildConstant ( S32 , MaxBoundary ) ; auto Bitcast = B . buildBitcast ( { S32 } , CvtPk ) ; auto Med3 = B . buildInstr ( <STR_LIT> , { S32 } , { MinBoundaryDst . getReg ( <NUM_LIT> ) , Bitcast . getReg ( <NUM_LIT> ) , MaxBoundaryDst . getReg ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; B . buildTrunc ( MI . getOperand ( <NUM_LIT> ) . getReg (


<s> Register Src = MatchInfo . Origin ; assert ( MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) . getType ( Src ) == LLT :: scalar ( <NUM_LIT> ) ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; B . setMBB ( * MI . getParent ( ) ) ; B . setInstrAndDebugLoc ( MI ) ; auto Unmerge = B . buildUnmerge ( S32 , Src ) ; assert ( MI . getOpcode ( ) != <STR_LIT> ) ; const LLT V2S16 = LLT :: fixed_vector ( <NUM_LIT> , <NUM_LIT> ) ; auto CvtPk = B . buildInstr ( <STR_LIT> , { V2S16 } , { Unmerge . getReg ( <NUM_LIT> ) , Unmerge . getReg ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; auto MinBoundary = std :: min ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MaxBoundary = std :: max ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MinBoundaryDst = B . buildConstant ( S32 , MinBoundary ) ; auto MaxBoundaryDst = B . buildConstant ( S32 , MaxBoundary ) ; auto Bitcast = B . buildBitcast ( { S32 } , CvtPk ) ; auto Med3 = B . buildInstr ( <STR_LIT> , { S32 } , { MinBoundaryDst . getReg ( <NUM_LIT> ) , Bitcast . getReg ( <NUM_LIT> ) , MaxBoundaryDst . getReg ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; B . buildTrunc ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Med3 ) ; MI . eraseFromParent (


<s> <ISA_LIT>PreLegalizerCombinerHelper PreLegalizerHelper ( B , Helper ) ; <ISA_LIT>GenPreLegalizerCombinerHelper Generated ( GeneratedRuleCfg , Helper , PreLegalizerHelper ) ; if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_MEMCPY_INLINE : return Helper . tryEmitMemcpyInline ( MI ) ; case TargetOpcode :: G_CONCAT_VECTORS : return Helper . tryCombineConcatVectors ( MI ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return Helper . tryCombineShuffleVector ( MI ) ; } return false


<s> if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_MEMCPY_INLINE : return Helper . tryEmitMemcpyInline ( MI ) ; case TargetOpcode :: G_CONCAT_VECTORS : return Helper . tryCombineConcatVectors ( MI ) ; case TargetOpcode


<s> if ( ! IsOptNone ) { AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ; } AU . addRequired < GISelCSEAnalysisWrapperPass > ( ) ; AU . addPreserved < GISelCSEAnalysisWrapperPass > ( ) ; MachineFunctionPass :: getAnalysisUsage ( AU


<s> GISelKnownBits * KB = & getAnalysis < GISelKnownBitsAnalysis > ( ) . get ( MF ) ; MachineDominatorTree * MDT = IsOptNone ? nullptr : & getAnalysis < MachineDominatorTree > ( ) ; <ISA_LIT>PreLegalizerCombinerInfo PCInfo ( EnableOpt , F . hasOptSize ( ) , F . hasMinSize ( ) , KB , MDT ) ; GISelCSEAnalysisWrapper & Wrapper = getAnalysis < GISelCSEAnalysisWrapperPass >


<s> GISelKnownBits * KB = & getAnalysis < GISelKnownBitsAnalysis > ( ) . get ( MF ) ; MachineDominatorTree * MDT = IsOptNone ? nullptr : & getAnalysis < MachineDominatorTree > ( ) ; <ISA_LIT>PreLegalizerCombinerInfo PCInfo ( EnableOpt , F . hasOptSize ( ) , F . hasMinSize ( ) , KB , MDT ) ; GISelCSEAnalysisWrapper & Wrapper = getAnalysis < GISelCSEAnalysisWrapperPass > ( ) . getCSEWrapper ( ) ; auto * CSEInfo = & Wrapper . get ( TPC -> getCSEConfig ( ) ) ; Combiner C ( PCInfo , TPC )


<s> <ISA_LIT>PreLegalizerCombinerHelper ( MachineIRBuilder & B , <ISA_LIT>CombinerHelper & Helper ) : B ( B ) , MF ( B . getMF (

) ) , MRI ( * B . getMRI ( ) ) , Helper ( Helper ) { }


<s> <ISA_LIT>PreLegalizerCombinerHelper ( MachineIRBuilder & B , <ISA_LIT>CombinerHelper & Helper ) : B ( B ) , MF ( B . getMF ( ) ) , MRI ( * B . getMRI ( ) ) , Helper


<s> B . setInstrAndDebugLoc ( MI ) ; auto Unmerge = B . buildUnmerge ( S32 , Src ) ; assert ( MI . getOpcode ( ) != <STR_LIT> ) ; const LLT V2S16 = LLT :: fixed_vector ( <NUM_LIT> , <NUM_LIT> ) ; auto CvtPk = B . buildInstr ( <STR_LIT> , { V2S16 } , { Unmerge . getReg ( <NUM_LIT> ) , Unmerge . getReg ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; auto MinBoundary = std :: min ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MaxBoundary = std :: max ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MinBoundaryDst = B . buildConstant ( S32 , MinBoundary ) ; auto MaxBoundaryDst = B . buildConstant (


<s> void <ISA_LIT>PreLegalizerCombinerHelper :: applyClampI64ToI16 ( MachineInstr & MI , const ClampI64ToI16MatchInfo & MatchInfo ) { Register Src = MatchInfo . Origin ; assert ( MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) . getType ( Src ) == LLT :: scalar ( <NUM_LIT> ) ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; B . setInstrAndDebugLoc ( MI ) ; auto Unmerge = B . buildUnmerge ( S32 , Src ) ; assert ( MI . getOpcode ( ) != <STR_LIT> ) ; const LLT V2S16 = LLT :: fixed_vector ( <NUM_LIT> , <NUM_LIT> ) ; auto CvtPk = B . buildInstr ( <STR_LIT> , { V2S16 } , { Unmerge . getReg ( <NUM_LIT> ) , Unmerge . getReg ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; auto MinBoundary = std :: min ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MaxBoundary = std :: max ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MinBoundaryDst = B . buildConstant ( S32 , MinBoundary ) ; auto MaxBoundaryDst = B . buildConstant ( S32 , MaxBoundary


<s> auto Unmerge = B . buildUnmerge ( S32 , Src ) ; assert ( MI . getOpcode ( ) != <STR_LIT> ) ; const LLT V2S16 = LLT :: vector ( <NUM_LIT> , <NUM_LIT> ) ; auto CvtPk = B . buildInstr ( <STR_LIT> , { V2S16 } , { Unmerge . getReg ( <NUM_LIT> ) , Unmerge . getReg ( <NUM_LIT> ) } , MI . getFlags ( ) ) ; auto MinBoundary = std :: min ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MaxBoundary = std :: max ( MatchInfo . Cmp1 , MatchInfo . Cmp2 ) ; auto MinBoundaryDst = B . buildConstant ( S32 , MinBoundary ) ; auto MaxBoundaryDst = B . buildConstant (


<s> <ISA_LIT>PreLegalizerCombinerInfo ( bool EnableOpt , bool OptSize , bool MinSize , GISelKnownBits * KB , MachineDominatorTree * MDT ) : CombinerInfo ( true , false , nullptr , EnableOpt , OptSize , MinSize ) , KB


<s> CombinerHelper Helper ( Observer , B , KB , MDT ) ; if ( Generated . tryCombineAll ( Observer , MI , B , Helper ) ) return true ; switch ( MI . getOpcode ( ) ) { case TargetOpcode ::


<s> return new <ISA_LIT>PreLegalizerCombiner (


<s> AU . addRequired < GISelKnownBitsAnalysis > ( ) ; AU . addPreserved < GISelKnownBitsAnalysis > ( ) ; if ( ! IsOptNone ) { AU . addRequired < MachineDominatorTree


<s> auto * TPC = & getAnalysis < TargetPassConfig > ( ) ; const Function & F = MF . getFunction ( ) ; bool EnableOpt = MF . getTarget ( ) . getOptLevel ( ) != CodeGenOpt :: None && ! skipFunction ( F ) ; GISelKnownBits * KB = & getAnalysis < GISelKnownBitsAnalysis > ( ) . get ( MF ) ; MachineDominatorTree * MDT = IsOptNone ? nullptr : & getAnalysis < MachineDominatorTree > ( ) ; <ISA_LIT>PreLegalizerCombinerInfo PCInfo ( EnableOpt , F . hasOptSize ( ) ,


<s> size_t CurFmtSpecifierIdx = <NUM_LIT> ; size_t PrevFmtSpecifierIdx = <NUM_LIT> ; while ( ( CurFmtSpecifierIdx = Fmt . find_first_of ( ConvSpecifiers , CurFmtSpecifierIdx ) ) != StringRef :: npos ) { bool ArgDump = false ; StringRef CurFmt = Fmt . substr ( PrevFmtSpecifierIdx , CurFmtSpecifierIdx - PrevFmtSpecifierIdx ) ; size_t pTag = CurFmt . find_last_of ( <STR_LIT> ) ; if ( pTag != StringRef :: npos ) { ArgDump = true ; while ( pTag && CurFmt [ -- pTag ] == '%' ) { ArgDump = ! ArgDump ; } } if ( ArgDump ) OpConvSpecifiers . push_back ( Fmt [ CurFmtSpecifierIdx ] ) ; PrevFmtSpecifierIdx = ++


<s> if ( CI -> isCallee ( & U ) ) Printfs . push_back ( CI ) ; } } if ( Printfs . empty ( ) ) return false ; if ( auto HostcallFunction = M . getFunction ( <STR_LIT> ) ) { for ( auto & U : HostcallFunction -> uses ( ) ) { if ( auto * CI = dyn_cast < CallInst > ( U . getUser ( ) ) ) { M . getContext ( ) . emitError ( CI , <STR_LIT> ) ; } } } TD = & M . getDataLayout ( ) ; auto DTWP = getAnalysisIfAvailable < DominatorTreeWrapperPass > ( ) ; DT = DTWP ? & DTWP -> getDomTree ( ) : nullptr ; auto GetTLI = [ this ] ( Function & F ) -> TargetLibraryInfo & { return this -> getAnalysis < TargetLibraryInfoWrapperPass > ( ) . getTLI (


<s> if ( Specifier != 's' ) return false ; const PointerType * PT = dyn_cast < PointerType > ( OpType ) ; if ( ! PT || PT -> getAddressSpace ( ) != <ISA_LIT>AS :: CONSTANT_ADDRESS ) return false ; Type * ElemType = PT -> getContainedType (


<s> return SimplifyInstruction ( I , { * TD , TLI , DT }


<s> return SimplifyInstruction ( I , { * TD


<s> bool <ISA_LIT>PrintfRuntimeBinding :: runOnModule ( Module & M ) { Triple TT ( M . getTargetTriple ( ) ) ; if ( TT . getArch ( ) == Triple :: r600 ) return false ; visit ( M ) ; if ( Printfs . empty ( ) ) return false ; TD = & M . getDataLayout ( ) ; auto DTWP = getAnalysisIfAvailable < DominatorTreeWrapperPass > ( ) ; DT = DTWP ? & DTWP -> getDomTree ( ) : nullptr ; TLI = & getAnalysis < TargetLibraryInfoWrapperPass > ( ) . getTLI ( ) ; return lowerPrintfForGpu ( M


<s> Value * simplify ( Instruction * I ) { return SimplifyInstruction ( I , { * TD , TLI ,


<s> auto PrintfFunction = M . getFunction ( <STR_LIT> ) ; if ( ! PrintfFunction ) return false ; for ( auto & U : PrintfFunction -> uses ( ) ) { if ( auto * CI = dyn_cast < CallInst > ( U . getUser ( ) ) ) { if ( CI -> isCallee ( & U ) ) Printfs . push_back ( CI ) ; } } if ( Printfs . empty


<s> ModulePass * create<ISA_LIT>PrintfRuntimeBinding ( ) { return new <ISA_LIT>PrintfRuntimeBinding (


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < TargetLibraryInfoWrapperPass > ( ) ; AU . addRequired <


<s> AU . addRequired < TargetLibraryInfoWrapperPass > ( ) ; AU . addRequired < DominatorTreeWrapperPass > ( )


<s> void <ISA_LIT>PrintfRuntimeBindingImpl :: getConversionSpecifiers ( SmallVectorImpl < char > & OpConvSpecifiers , StringRef Fmt , size_t NumOps ) const { static const char ConvSpecifiers [ ] = <STR_LIT> ; size_t CurFmtSpecifierIdx = <NUM_LIT> ; size_t PrevFmtSpecifierIdx = <NUM_LIT> ; while ( ( CurFmtSpecifierIdx = Fmt . find_first_of ( ConvSpecifiers , CurFmtSpecifierIdx ) ) != StringRef :: npos ) { bool ArgDump = false


<s> } ; auto GetTLI = [ & FAM ] ( Function & F ) -> TargetLibraryInfo & { return FAM . getResult < TargetLibraryAnalysis > ( F ) ; } ; bool Changed = <ISA_LIT>PrintfRuntimeBindingImpl ( GetDT , GetTLI ) . run ( M ) ; return Changed ? PreservedAnalyses :: none ( )


<s> return this -> getAnalysis < DominatorTreeWrapperPass > ( F ) . getDomTree ( ) ; } ; auto GetTLI = [ this ] ( Function & F )


<s> if ( Specifier != 's' ) return false ; const PointerType * PT = dyn_cast < PointerType > ( OpType ) ; if ( ! PT || PT -> getAddressSpace ( ) != <ISA_LIT>AS :: CONSTANT_ADDRESS ) return false ; Type * ElemType = PT -> getContainedType ( <NUM_LIT> ) ; if ( ElemType -> getTypeID ( ) !=


<s> if ( Specifier != 's' ) return false ; const PointerType * PT = dyn_cast < PointerType > ( OpType ) ; if ( ! PT || PT -> getAddressSpace ( ) != <ISA_LIT>AS ::


<s> Value * simplify ( Instruction * I , const TargetLibraryInfo * TLI , const DominatorTree * DT ) { return SimplifyInstruction ( I , { * TD


<s> return VectorType :: get ( ArrayTy -> getArrayElementType (

) , ArrayTy -> getArrayNumElements ( ) ) ;


<s> static Value * calculateVectorIndex ( Value * Ptr , const std :: map < GetElementPtrInst * , Value * > & GEPIdx ) { if ( isa < AllocaInst > ( Ptr ) ) return Constant :: getNullValue ( Type :: getInt32Ty ( Ptr -> getContext


<s> switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : case Instruction :: BitCast : case Instruction :: AddrSpaceCast : return true ; case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst >


<s> if ( CallInst * CI = dyn_cast < CallInst > ( User ) ) { if ( ! CI -> getCalledFunction ( ) ) return false ; WorkList . push_back ( User ) ; continue ; } Instruction * UseInst = dyn_cast < Instruction > ( User ) ; if ( UseInst && UseInst -> getOpcode ( ) == Instruction :: PtrToInt )


<s> FunctionPass * llvm :: create<ISA_LIT>PromoteAlloca ( const


<s> if ( ! I0 || ! I0 -> isZero ( ) ) return NULL ; return GEP -> getOperand ( <NUM_LIT>


<s> static Value * GEPToVectorIndex ( GetElementPtrInst * GEP ) { if ( GEP -> getNumOperands ( ) !=


<s> const char * getPassName ( ) const override { return <STR_LIT>


<s> Value * TIdZ = Builder . CreateCall ( ReadTIDIGZ , { } ) ; Value * Tmp0 = Builder . CreateMul ( TCntY , TCntZ ) ; Tmp0 = Builder . CreateMul ( Tmp0 , TIdX ) ; Value * Tmp1 = Builder . CreateMul ( TIdY , TCntZ ) ; Value * TID = Builder . CreateAdd ( Tmp0 , Tmp1 ) ; TID = Builder . CreateAdd ( TID , TIdZ ) ; std :: vector < Value * > Indices ; Indices . push_back ( Constant :: getNullValue ( Type :: getInt32Ty ( Mod -> getContext ( ) ) ) ) ; Indices . push_back ( TID ) ; Value * Offset = Builder . CreateGEP ( GVTy , GV , Indices ) ; I . mutateType ( Offset -> getType ( ) ) ; I . replaceAllUsesWith ( Offset ) ; I . eraseFromParent ( ) ; for ( std :: vector < Value * > :: iterator i = WorkList . begin ( ) , e = WorkList . end ( ) ; i != e ; ++ i ) { Value * V = * i ; CallInst * Call = dyn_cast < CallInst > ( V ) ; if ( ! Call ) { Type * EltTy = V -> getType ( ) -> getPointerElementType ( ) ; PointerType * NewTy = PointerType :: get ( EltTy , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; if ( isa < AddrSpaceCastInst > ( V ) ) continue ; V -> mutateType ( NewTy ) ; continue ; } IntrinsicInst * Intr = dyn_cast < IntrinsicInst > ( Call ) ; if ( ! Intr ) { std :: vector < Type * > ArgTypes ; for ( unsigned ArgIdx = <NUM_LIT> , ArgEnd = Call -> getNumArgOperands ( ) ; ArgIdx != ArgEnd ; ++ ArgIdx ) { ArgTypes . push_back ( Call -> getArgOperand ( ArgIdx ) -> getType ( ) ) ; } Function * F = Call -> getCalledFunction ( ) ; FunctionType * NewType = FunctionType :: get ( Call -> getType ( ) , ArgTypes , F -> isVarArg ( ) ) ; Constant * C = Mod -> getOrInsertFunction ( ( F -> getName ( ) + <STR_LIT> ) . str ( ) , NewType , F -> getAttributes ( ) ) ; Function * NewF = cast <


<s> if ( Val == OtherOp ) OtherOp = Inst -> getOperand ( OpIdx1 ) ; if ( isa < ConstantPointerNull > ( OtherOp ) ) return true ; Value * OtherObj = getUnderlyingObject (


<s> CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addRetAttr ( Attribute :: NoAlias ) ; DispatchPtr -> addRetAttr ( Attribute :: NonNull ) ; F . removeFnAttr ( <STR_LIT> ) ; DispatchPtr -> addDereferenceableRetAttr ( <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( I32Ty , GEPXY , Align ( <NUM_LIT> ) ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( I32Ty , GEPZU , Align ( <NUM_LIT> ) ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; ST . makeLIDRangeMetadata ( LoadZU ) ; Value * Y = Builder . CreateLShr ( LoadXY , <NUM_LIT> ) ; return std :: make_pair ( Y , LoadZU


<s> Value * TCntY , * TCntZ ; std :: tie ( TCntY , TCntZ ) = getLocalSizeYZ ( Builder ) ; Value * TIdX = getWorkitemID ( Builder , <NUM_LIT> ) ; Value * TIdY = getWorkitemID ( Builder , <NUM_LIT> ) ; Value * TIdZ = getWorkitemID ( Builder , <NUM_LIT> ) ; Value * Tmp0 = Builder . CreateMul ( TCntY , TCntZ , <STR_LIT> , true , true ) ; Tmp0 = Builder . CreateMul ( Tmp0 , TIdX ) ; Value * Tmp1 = Builder . CreateMul ( TIdY , TCntZ , <STR_LIT> , true , true ) ; Value * TID = Builder . CreateAdd ( Tmp0 , Tmp1 ) ; TID = Builder . CreateAdd ( TID , TIdZ ) ; Value * Indices [ ] = { Constant :: getNullValue ( Type :: getInt32Ty ( Mod -> getContext ( ) ) ) , TID } ; Value * Offset = Builder . CreateInBoundsGEP ( GVTy , GV , Indices ) ; I . mutateType ( Offset -> getType ( ) ) ; I . replaceAllUsesWith ( Offset ) ; I . eraseFromParent ( ) ; SmallVector < IntrinsicInst * > DeferredIntrs ; for ( Value * V : WorkList ) { CallInst * Call = dyn_cast < CallInst > ( V ) ; if ( ! Call ) { if ( ICmpInst * CI = dyn_cast < ICmpInst > ( V ) ) { Value * Src0 = CI -> getOperand ( <NUM_LIT> ) ; PointerType * NewTy = PointerType :: getWithSamePointeeType ( cast < PointerType > ( Src0 -> getType ( ) ) , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; if ( isa < ConstantPointerNull > ( CI -> getOperand ( <NUM_LIT> ) ) ) CI -> setOperand ( <NUM_LIT> , ConstantPointerNull :: get ( NewTy ) ) ; if ( isa < ConstantPointerNull > ( CI -> getOperand ( <NUM_LIT> ) ) ) CI -> setOperand ( <NUM_LIT> , ConstantPointerNull :: get ( NewTy ) ) ; continue ; } if ( isa < AddrSpaceCastInst > ( V ) ) continue ; PointerType * NewTy = PointerType :: getWithSamePointeeType ( cast < PointerType > ( V -> getType ( ) ) , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; V -> mutateType ( NewTy ) ; if ( SelectInst * <ISA_LIT> = dyn_cast < SelectInst > ( V ) ) { if ( isa < ConstantPointerNull > ( <ISA_LIT> -> getOperand ( <NUM_LIT> ) ) ) <ISA_LIT> -> setOperand ( <NUM_LIT> , ConstantPointerNull :: get ( NewTy ) ) ; if ( isa < ConstantPointerNull > ( <ISA_LIT> -> getOperand ( <NUM_LIT> ) ) ) <ISA_LIT> ->


<s> bool handlePromoteAllocaToVector ( AllocaInst & I , unsigned MaxVGPRs ) { if ( ! I . isStaticAlloca ( ) || I . isArrayAllocation ( ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << I


<s> const GCNSubtarget & ST = TM . getSubtarget < GCNSubtarget > ( F ) ; MaxVGPRs = ST . getMaxNumVGPRs ( ST . getWavesPerEU ( F ) . first ) ; if ( ! <STR_LIT> ( F . getCallingConv ( ) ) ) MaxVGPRs = std :: min ( MaxVGPRs , <NUM_LIT> ) ; } else { MaxVGPRs = <NUM_LIT> ; } bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ; SmallVector < AllocaInst * , <NUM_LIT> >


<s> if ( skipFunction ( F ) ) return false ; if ( auto * TPC = getAnalysisIfAvailable < TargetPassConfig > ( ) ) { return promoteAllocasToVector ( F , TPC -> getTM < TargetMachine > (


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { if ( Inst -> getType ( ) == AllocaTy || Inst -> getType ( ) -> isVectorTy ( ) ) break ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; if ( ! Index ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; if ( Inst -> getType ( ) != VecEltTy ) ExtractElement = Builder . CreateBitOrPointerCast ( ExtractElement , Inst -> getType ( ) ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; if ( <ISA_LIT> -> getValueOperand ( ) -> getType ( ) == AllocaTy || <ISA_LIT> -> getValueOperand ( ) -> getType ( ) -> isVectorTy ( ) ) break ; Value * Ptr = <ISA_LIT> -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; if ( ! Index ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * Elt = <ISA_LIT> -> getValueOperand ( ) ; if ( Elt -> getType ( ) != VecEltTy ) Elt = Builder . CreateBitOrPointerCast ( Elt , VecEltTy ) ; Value * NewVecValue = Builder . CreateInsertElement ( VecValue , Elt , Index ) ; Builder . CreateStore ( NewVecValue , BitCast ) ; Inst -> eraseFromParent ( ) ; break ; } default : llvm_unreachable ( <STR_LIT> ) ; } } return true


<s> static VectorType * arrayTypeToVecType (


<s> GetElementPtrInst * GEP = cast < GetElementPtrInst > ( Ptr ) ; auto I = GEPIdx . find ( GEP ) ; return I == GEPIdx . end ( ) ? nullptr :


<s> } case Instruction :: BitCast : return true ; case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; if ( isa < AllocaInst > ( User ) && <ISA_LIT> -> getPointerOperandType ( ) == User -> getType ( ) && isa < VectorType > ( <ISA_LIT> -> getValueOperand ( ) -> getType ( ) ) ) return true ; return ( <ISA_LIT> -> getPointerOperand ( ) == User ) && isa < GetElementPtrInst > ( User ) && <ISA_LIT> -> isSimple ( ) ; } default : return false


<s> } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY


<s> DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None


<s> case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } Function * WorkitemIdFn = <STR_LIT> ( Mod , IntrID ) ; CallInst * CI = Builder .


<s> for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getType ( ) -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( const User * U : GV . users ( ) ) { const Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) { unsigned Align = GV . getAlignment ( ) ; if ( Align == <NUM_LIT> ) Align = DL . getABITypeAlignment ( GV . getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Align ) ; CurrentLocalMemUsage += AllocSize ; break ; } } } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage , F ) ; unsigned OccupancyHint = ST . getWavesPerEU ( F ) . second ; if ( OccupancyHint == <NUM_LIT> ) OccupancyHint = <NUM_LIT> ; OccupancyHint = std :: min ( OccupancyHint , ST . getMaxWavesPerEU ( ) ) ; MaxOccupancy = std :: min ( OccupancyHint , MaxOccupancy ) ; unsigned MaxSizeWithWaveCount = ST . getMaxLocalMemSizeWithWaveCount ( MaxOccupancy , F


<s> bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ; for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ; I != E ; ) { AllocaInst * AI = dyn_cast < AllocaInst > ( I ) ; ++ I ; if ( AI ) Changed |= handleAlloca ( * AI , SufficientLDS ) ; } return


<s> BasicBlock & EntryBB = * F . begin ( ) ; for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ; I != E ; ) { AllocaInst * AI = dyn_cast < AllocaInst > ( I ) ; ++ I ; if ( AI ) Changed |= handleAlloca ( * AI ,


<s> return false ; } ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; if ( visitUsers ( & GV , & GV ) ) { UsedLDS . insert ( & GV ) ; Stack . clear ( ) ; continue ; } while ( ! Stack . empty ( ) ) { const Constant * C = Stack . pop_back_val ( ) ; if ( visitUsers ( & GV , C ) ) { UsedLDS . insert ( & GV ) ; Stack . clear ( ) ; break ; } } } const DataLayout & DL = Mod -> getDataLayout ( ) ; SmallVector < std :: pair < uint64_t , Align > , <NUM_LIT> > AllocatedSizes ; AllocatedSizes . reserve ( UsedLDS . size ( ) ) ; for ( const GlobalVariable * GV : UsedLDS ) { Align Alignment = DL . getValueOrABITypeAlignment ( GV -> getAlign ( ) , GV -> getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV -> getValueType ( ) ) ; if ( GV -> hasExternalLinkage ( ) && AllocSize == <NUM_LIT> ) { LocalMemLimit = <NUM_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> <STR_LIT> ) ; return false ; } AllocatedSizes . emplace_back ( AllocSize , Alignment ) ; } llvm :: sort ( AllocatedSizes , [ ] ( std :: pair < uint64_t , Align > LHS , std :: pair < uint64_t , Align > RHS ) { return LHS . second < RHS . second ; } ) ; CurrentLocalMemUsage =


<s> Value * OtherObj = GetUnderlyingObject ( OtherOp , * DL ) ; if ( ! isa < AllocaInst > ( OtherObj ) ) return false ; if ( OtherObj != BaseAlloca ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false


<s> if ( ! TM ) return false ; Mod = & M ; DL = & Mod -> getDataLayout ( ) ; const Triple & TT = TM -> getTargetTriple


<s> Function * LocalSizeYFn = <STR_LIT> ( Mod , <STR_LIT> ) ; Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeList :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , AS . CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; ST . makeLIDRangeMetadata ( LoadZU ) ; Value * Y = Builder . CreateLShr


<s> switch ( N ) { case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? <STR_LIT> : <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } Function * WorkitemIdFn = <STR_LIT> ( Mod , IntrID ) ; CallInst * CI = Builder . CreateCall ( WorkitemIdFn ) ; ST . makeLIDRangeMetadata (


<s> CurrentLocalMemUsage = <NUM_LIT> ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getType ( ) -> getAddressSpace ( ) != AS . LOCAL_ADDRESS ) continue ; for ( const User * U : GV . users ( ) ) { const Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) { unsigned Align = GV . getAlignment ( ) ; if ( Align == <NUM_LIT> ) Align = DL . getABITypeAlignment ( GV . getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Align ) ; CurrentLocalMemUsage += AllocSize ; break ; } } } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage , F ) ; unsigned OccupancyHint = ST . getWavesPerEU ( F ) . second ; if ( OccupancyHint == <NUM_LIT> ) OccupancyHint = <NUM_LIT> ; OccupancyHint = std :: min ( OccupancyHint , ST . getMaxWavesPerEU ( ) ) ; MaxOccupancy = std :: min ( OccupancyHint , MaxOccupancy ) ; unsigned MaxSizeWithWaveCount = ST . getMaxLocalMemSizeWithWaveCount ( MaxOccupancy , F ) ; if ( CurrentLocalMemUsage > MaxSizeWithWaveCount ) return false ; LocalMemLimit = MaxSizeWithWaveCount


<s> if ( Align == <NUM_LIT> ) Align = DL . getABITypeAlignment ( GV . getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Align ) ; CurrentLocalMemUsage += AllocSize ; break ; } } } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage , F ) ; unsigned OccupancyHint = ST . getWavesPerEU ( F ) . second ; if ( OccupancyHint == <NUM_LIT> ) OccupancyHint = <NUM_LIT> ; OccupancyHint = std :: min ( OccupancyHint , ST . getMaxWavesPerEU ( ) ) ; MaxOccupancy = std :: min ( OccupancyHint , MaxOccupancy ) ; unsigned MaxSizeWithWaveCount = ST . getMaxLocalMemSizeWithWaveCount ( MaxOccupancy , F ) ; if ( CurrentLocalMemUsage > MaxSizeWithWaveCount ) return false ; LocalMemLimit = MaxSizeWithWaveCount ; DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << CurrentLocalMemUsage << <STR_LIT> << <STR_LIT> << MaxSizeWithWaveCount << <STR_LIT> << MaxOccupancy << '\n' << <STR_LIT> << ( LocalMemLimit - CurrentLocalMemUsage ) << <STR_LIT> ) ; BasicBlock & EntryBB = * F . begin ( ) ; for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ; I != E ; ) { AllocaInst * AI = dyn_cast < AllocaInst > ( I ) ; ++ I ; if ( AI ) handleAlloca ( * AI ) ; } return


<s> for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { Type * VecPtrTy = VectorTy -> getPointerTo ( AS . PRIVATE_ADDRESS ) ; Value * Ptr = Inst -> getOperand ( <NUM_LIT> ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { Type * VecPtrTy = VectorTy -> getPointerTo ( AS . PRIVATE_ADDRESS ) ; Value * Ptr = Inst -> getOperand ( <NUM_LIT> ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * NewVecValue = Builder . CreateInsertElement ( VecValue , Inst -> getOperand ( <NUM_LIT> ) , Index ) ; Builder . CreateStore ( NewVecValue , BitCast ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: BitCast : case Instruction :: AddrSpaceCast : break ; default : llvm_unreachable ( <STR_LIT> ) ; } } return


<s> const <ISA_LIT>Subtarget & ST = TM -> getSubtarget < <ISA_LIT>Subtarget > ( F ) ; if ( ! ST . isPromoteAllocaEnabled ( ) ) return false ; AS = <STR_LIT> ( * F . getParent ( ) ) ; bool SufficientLDS = hasSufficientLocalMem ( F ) ; bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ; for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ; I != E ; ) { AllocaInst * AI = dyn_cast < AllocaInst


<s> bool <ISA_LIT>PromoteAlloca :: doInitialization ( Module & M ) { if ( ! TM ) return false ; Mod = & M ; DL = & Mod -> getDataLayout ( ) ; MDBuilder MDB ( Mod -> getContext ( ) ) ; MaxWorkGroupSizeRange = MDB . createRange ( APInt ( <NUM_LIT> , <NUM_LIT> ) , APInt ( <NUM_LIT> , <NUM_LIT> ) ) ; const Triple & TT = TM -> getTargetTriple ( ) ; IsAMDGCN = TT . getArch ( ) == Triple :: amdgcn ; IsAMDHSA = TT . getOS ( ) == Triple :: AMDHSA ; return false


<s> if ( Use -> getParent ( ) -> getParent ( ) == & F ) { unsigned Align = GV . getAlignment ( ) ; if ( Align == <NUM_LIT> ) Align = DL . getABITypeAlignment ( GV . getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Align ) ; CurrentLocalMemUsage += AllocSize ; break ; } } } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage ) ; unsigned OccupancyHint = ST . getWavesPerEU ( F ) . second ; if ( OccupancyHint == <NUM_LIT> ) OccupancyHint = <NUM_LIT> ; OccupancyHint = std :: min ( OccupancyHint , ST . getMaxWavesPerEU ( ) ) ; MaxOccupancy = std :: min ( OccupancyHint , MaxOccupancy ) ; unsigned MaxSizeWithWaveCount = ST . getMaxLocalMemSizeWithWaveCount ( MaxOccupancy ) ; if ( CurrentLocalMemUsage > MaxSizeWithWaveCount ) return false ; LocalMemLimit = MaxSizeWithWaveCount ; DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << CurrentLocalMemUsage << <STR_LIT> << <STR_LIT> << MaxSizeWithWaveCount << <STR_LIT> << MaxOccupancy << '\n' << <STR_LIT> << ( LocalMemLimit - CurrentLocalMemUsage ) << <STR_LIT> ) ; BasicBlock & EntryBB = * F . begin ( ) ; for ( auto I = EntryBB . begin ( ) , E = EntryBB . end ( ) ; I != E ; ) { AllocaInst * AI = dyn_cast < AllocaInst > ( I


<s> GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( AllocaUser ) ; if ( ! GEP ) { if ( ! canVectorizeInst ( cast < Instruction > ( AllocaUser ) , Alloca ) ) return false ; WorkList . push_back ( AllocaUser ) ; continue ; } Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( User * GEPUser : AllocaUser -> users ( ) ) { if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) , AllocaUser ) ) return false ; WorkList . push_back ( GEPUser ) ; } } VectorType * VectorTy = arrayTypeToVecType ( AllocaTy ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode


<s> if ( ! canVectorizeInst ( cast < Instruction > ( AllocaUser ) , Alloca ) ) return false ; WorkList . push_back ( AllocaUser ) ; continue ; } Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; for ( User * GEPUser : AllocaUser -> users ( ) ) { if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) , AllocaUser ) ) return false ; WorkList . push_back ( GEPUser ) ; } } VectorType * VectorTy = arrayTypeToVecType ( AllocaTy ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = Inst -> getOperand ( <NUM_LIT> ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS


<s> GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( AllocaUser ) ; if ( ! GEP ) { if ( ! canVectorizeInst ( Inst , UseUser , DL ) ) return false ; if ( Inst -> getOpcode ( ) == Instruction :: BitCast ) { Type * FromTy = Inst -> getOperand ( <NUM_LIT> ) -> getType ( ) -> getPointerElementType ( ) ; Type * ToTy = Inst -> getType ( ) -> getPointerElementType ( ) ; if ( FromTy -> isAggregateType ( ) || ToTy -> isAggregateType ( ) || DL . getTypeSizeInBits ( FromTy ) != DL . getTypeSizeInBits ( ToTy ) ) continue ; for ( User * CastUser : Inst -> users ( ) ) { if ( isAssumeLikeIntrinsic ( cast < Instruction > ( CastUser ) ) ) continue ; Users . push_back ( CastUser ) ; UseUsers . push_back ( Inst ) ; } continue ; } WorkList . push_back ( AllocaUser ) ; continue ; } Value * Index = GEPToVectorIndex ( GEP ) ; if ( ! Index ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * GEP << '\n' ) ; return false ; } GEPVectorIdx [ GEP ] = Index ; Users . append ( GEP -> user_begin ( ) , GEP -> user_end ( ) ) ; UseUsers . append ( GEP -> getNumUses ( ) , GEP ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { if ( Inst -> getType ( ) == AllocaTy || Inst -> getType ( ) -> isVectorTy ( ) ) break ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; if ( ! Index ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; if ( Inst -> getType ( ) != VecEltTy ) ExtractElement = Builder . CreateBitCast ( ExtractElement , Inst -> getType ( ) ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; if ( <ISA_LIT> -> getValueOperand ( ) -> getType ( ) == AllocaTy || <ISA_LIT> -> getValueOperand ( ) -> getType ( ) -> isVectorTy ( ) ) break ; Value * Ptr = <ISA_LIT>


<s> return FixedVectorType :: get ( ArrayTy -> getElementType ( ) , ArrayTy ->


<s> static Value * calculateVectorIndex ( Value * Ptr , const std :: map < GetElementPtrInst * , Value * > & GEPIdx ) { GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( stripBitcasts


<s> static Value * calculateVectorIndex ( Value * Ptr , const std :: map < GetElementPtrInst * , Value * > & GEPIdx ) { GetElementPtrInst * GEP = dyn_cast < GetElementPtrInst > ( stripBitcasts ( Ptr ) ) ; if ( ! GEP


<s> FunctionPass * llvm :: create<ISA_LIT>PromoteAlloca ( )


<s> FunctionPass * llvm :: create<ISA_LIT>PromoteAllocaToVector ( ) { return new <ISA_LIT>PromoteAllocaToVector ( )


<s> Mod = & M ; DL = & Mod -> getDataLayout (


<s> if ( GEP -> getNumOperands ( ) != <NUM_LIT> ) return nullptr ; ConstantInt * I0 = dyn_cast < ConstantInt > ( GEP -> getOperand ( <NUM_LIT>


<s> AU . setPreservesCFG ( ) ; FunctionPass :: getAnalysisUsage (


<s> const <ISA_LIT>Subtarget & ST = <ISA_LIT>Subtarget :: get ( * TM , * Builder . GetInsertBlock ( ) -> getParent ( ) ) ; <STR_LIT> IntrID = <STR_LIT> ; switch ( N ) { case <NUM_LIT> : IntrID = IsAMDGCN ? ( <STR_LIT> ) <STR_LIT> : ( <STR_LIT> ) <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? ( <STR_LIT> ) <STR_LIT> : ( <STR_LIT> ) <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? ( <STR_LIT> ) <STR_LIT> : ( <STR_LIT> ) <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } Function * WorkitemIdFn = <STR_LIT> ( Mod , IntrID ) ; CallInst * CI = Builder . CreateCall ( WorkitemIdFn ) ; ST . makeLIDRangeMetadata ( CI ) ; return CI


<s> IntrinsicInst * II = dyn_cast < IntrinsicInst > ( CI ) ; if ( ! II ) return false ; switch ( II -> getIntrinsicID ( ) ) { case <STR_LIT>


<s> bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ; SmallVector < AllocaInst * , <NUM_LIT> > Allocas ; for ( Instruction & I


<s> SmallVector < AllocaInst * , <NUM_LIT> > Allocas ; for ( Instruction & I : EntryBB ) { if ( AllocaInst * AI = dyn_cast < AllocaInst > ( & I ) ) Allocas . push_back ( AI ) ; } for ( AllocaInst * AI : Allocas


<s> DL = & Mod -> getDataLayout ( ) ; const Triple & TT = TM . getTargetTriple ( ) ; IsAMDGCN = TT . getArch ( ) == Triple :: amdgcn ; IsAMDHSA = TT . getOS ( ) == Triple :: AMDHSA ; const <ISA_LIT>Subtarget & ST = <ISA_LIT>Subtarget :: get ( TM , F ) ; if ( ! ST . isPromoteAllocaEnabled ( ) ) return false ; if ( IsAMDGCN ) { const GCNSubtarget & ST = TM . getSubtarget < GCNSubtarget > ( F ) ; MaxVGPRs = ST . getMaxNumVGPRs ( ST . getWavesPerEU ( F ) . first ) ; } else { MaxVGPRs = <NUM_LIT> ; } bool SufficientLDS = hasSufficientLocalMem ( F


<s> DispatchPtr -> addDereferenceableAttr ( AttributeList :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( I32Ty , GEPXY , Align ( <NUM_LIT> ) ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( I32Ty , GEPZU , Align ( <NUM_LIT> ) ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load


<s> IntrID = IsAMDGCN ? ( <STR_LIT> ) <STR_LIT> : ( <STR_LIT> ) <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? ( <STR_LIT> ) <STR_LIT> : ( <STR_LIT> ) <STR_LIT> ; break ; case <NUM_LIT> : IntrID = IsAMDGCN ? ( <STR_LIT> ) <STR_LIT> : ( <STR_LIT> ) <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT>


<s> if ( ! TM || skipFunction ( F ) ) return false ; const <ISA_LIT>Subtarget & ST = TM -> getSubtarget < <ISA_LIT>Subtarget > ( F ) ; if ( ! ST . isPromoteAllocaEnabled ( ) ) return false ; FunctionType * FTy = F . getFunctionType ( ) ; for ( Type * ParamTy : FTy -> params ( ) ) { PointerType * PtrTy = dyn_cast < PointerType > ( ParamTy ) ; if ( PtrTy && PtrTy -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ) { LocalMemLimit = <NUM_LIT> ; DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } } LocalMemLimit = ST . getLocalMemorySize ( ) ; if ( LocalMemLimit == <NUM_LIT> ) return false ; const DataLayout & DL = Mod -> getDataLayout ( ) ; CurrentLocalMemUsage = <NUM_LIT> ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getType ( ) -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( const User * U : GV . users ( ) ) { const Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) { unsigned Align = GV . getAlignment ( ) ; if ( Align == <NUM_LIT> ) Align = DL . getABITypeAlignment ( GV . getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Align ) ; CurrentLocalMemUsage += AllocSize ; break ; } } } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage , F ) ; unsigned OccupancyHint = ST . getWavesPerEU ( F ) . second ; if ( OccupancyHint == <NUM_LIT> ) OccupancyHint


<s> bool <ISA_LIT>PromoteAlloca :: binaryOpIsDerivedFromSameAlloca ( Value * BaseAlloca , Value * Val , Instruction * Inst , int OpIdx0 , int OpIdx1 ) const { Value * OtherOp = Inst -> getOperand ( OpIdx0 ) ; if ( Val == OtherOp ) OtherOp = Inst -> getOperand ( OpIdx1 ) ; if ( isa < ConstantPointerNull > ( OtherOp ) ) return true ; Value * OtherObj = getUnderlyingObject ( OtherOp ) ; if ( ! isa < AllocaInst >


<s> const DataLayout & DL = Mod -> getDataLayout ( ) ; SmallVector < std :: pair < uint64_t , Align > , <NUM_LIT> > AllocatedSizes ; AllocatedSizes . reserve ( UsedLDS . size ( ) ) ; for ( const GlobalVariable * GV : UsedLDS ) { Align Alignment = DL . getValueOrABITypeAlignment ( GV -> getAlign ( ) , GV -> getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV -> getValueType ( ) ) ; AllocatedSizes . emplace_back ( AllocSize , Alignment ) ; } llvm :: sort ( AllocatedSizes . begin ( ) , AllocatedSizes . end ( ) , [ ] ( std :: pair < uint64_t , Align > LHS , std :: pair < uint64_t , Align > RHS ) { return LHS . second < RHS . second ; } ) ; CurrentLocalMemUsage = <NUM_LIT> ; for ( auto Alloc : AllocatedSizes ) { CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Alloc . second ) ; CurrentLocalMemUsage += Alloc . first ; } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage , F ) ; unsigned OccupancyHint = ST . getWavesPerEU ( F ) . second ; if ( OccupancyHint == <NUM_LIT> ) OccupancyHint = <NUM_LIT> ; OccupancyHint = std :: min ( OccupancyHint , ST . getMaxWavesPerEU ( ) ) ; MaxOccupancy = std :: min ( OccupancyHint , MaxOccupancy ) ; unsigned MaxSizeWithWaveCount = ST . getMaxLocalMemSizeWithWaveCount ( MaxOccupancy , F ) ; if ( CurrentLocalMemUsage > MaxSizeWithWaveCount ) return false ; LocalMemLimit = MaxSizeWithWaveCount ; LLVM_DEBUG ( dbgs ( ) << F . getName ( ) << <STR_LIT> << CurrentLocalMemUsage << <STR_LIT> << <STR_LIT>

<< MaxSizeWithWaveCount << <STR_LIT> << MaxOccupancy << '\n' << <STR_LIT> << ( LocalMemLimit - CurrentLocalMemUsage ) << <STR_LIT> ) ;


<s> MaxVGPRs = ST . getMaxNumVGPRs ( ST . getWavesPerEU ( F ) . first ) ; } else { MaxVGPRs = <NUM_LIT> ; } bool Changed = false ; BasicBlock & EntryBB = * F . begin ( ) ; SmallVector < AllocaInst * , <NUM_LIT> > Allocas ; for ( Instruction & I : EntryBB ) { if ( AllocaInst * AI = dyn_cast < AllocaInst > ( & I ) ) Allocas . push_back ( AI ) ; } for ( AllocaInst * AI : Allocas ) { if ( handleAlloca ( * AI ) )


<s> static Value * calculateVectorIndex ( Value * Ptr , const std :: map < GetElementPtrInst * , Value * > & GEPIdx ) { auto * GEP = dyn_cast < GetElementPtrInst > ( Ptr -> stripPointerCasts ( ) ) ; if ( ! GEP ) return ConstantInt :: getNullValue ( Type :: getInt32Ty ( Ptr -> getContext ( ) ) ) ; auto I = GEPIdx . find (


<s> if ( ! GEP ) return ConstantInt :: getNullValue ( Type :: getInt32Ty ( Ptr -> getContext ( ) ) ) ; auto I = GEPIdx . find ( GEP ) ; assert ( I != GEPIdx . end ( ) && <STR_LIT>


<s> static Value * GEPToVectorIndex ( GetElementPtrInst * GEP , AllocaInst * Alloca , Type * VecElemTy , const DataLayout & DL ) { unsigned BW = DL . getIndexTypeSizeInBits ( GEP -> getType ( ) ) ; MapVector < Value * , APInt > VarOffsets ; APInt ConstOffset ( BW , <NUM_LIT> ) ; if ( GEP -> getPointerOperand ( ) -> stripPointerCasts ( ) != Alloca || ! GEP -> collectOffset ( DL , BW , VarOffsets , ConstOffset ) ) return nullptr ; unsigned VecElemSize = DL . getTypeAllocSize ( VecElemTy ) ; if ( VarOffsets . size ( ) > <NUM_LIT> ) return nullptr ; if ( VarOffsets . size ( ) == <NUM_LIT> ) { const auto & VarOffset = VarOffsets . front ( ) ; if ( ! ConstOffset . isZero ( ) || VarOffset . second != VecElemSize ) return nullptr ; return VarOffset . first ; } APInt Quot ; uint64_t Rem ; APInt :: udivrem ( ConstOffset , VecElemSize , Quot , Rem ) ; if ( Rem != <NUM_LIT> ) return nullptr ; return ConstantInt :: get ( GEP -> getContext ( )


<s> GetElementPtrInst * GEP = cast < GetElementPtrInst > ( stripBitcasts ( Ptr ) ) ; auto I = GEPIdx . find ( GEP ) ; return I == GEPIdx . end ( ) ? nullptr : I


<s> } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { if ( Inst -> getType ( ) == AllocaTy || Inst -> getType ( ) -> isVectorTy ( ) ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; if ( Inst -> getType ( ) != VecEltTy ) ExtractElement = Builder . CreateBitCast ( ExtractElement , Inst -> getType ( ) ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; if ( <ISA_LIT> -> getValueOperand ( ) -> getType ( ) == AllocaTy || <ISA_LIT> -> getValueOperand ( ) -> getType ( ) -> isVectorTy ( ) ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = <ISA_LIT> -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * Elt = <ISA_LIT> -> getValueOperand ( ) ; if ( Elt -> getType ( ) != VecEltTy ) Elt = Builder . CreateBitCast ( Elt , VecEltTy ) ; Value * NewVecValue = Builder . CreateInsertElement ( VecValue , Elt , Index ) ; Builder . CreateStore ( NewVecValue , BitCast


<s> <ISA_LIT>PromoteAlloca ( const TargetMachine * TM_ = nullptr ) : FunctionPass ( ID ) , TM ( TM_ ) , Mod ( nullptr ) , DL ( nullptr ) , MaxWorkGroupSizeRange ( nullptr ) , LocalMemLimit ( <NUM_LIT> ) , CurrentLocalMemUsage ( <NUM_LIT> )

, IsAMDGCN ( false ) , IsAMDHSA ( false ) {


<s> Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; LocalSizeY -> setMetadata ( LLVMContext :: MD_range , MaxWorkGroupSizeRange ) ; LocalSizeZ -> setMetadata ( LLVMContext :: MD_range , MaxWorkGroupSizeRange ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeSet :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeSet :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeSet :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = llvm :: MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_range ,


<s> } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addAttribute ( AttributeSet :: ReturnIndex , Attribute :: NoAlias ) ; DispatchPtr -> addAttribute ( AttributeSet :: ReturnIndex , Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableAttr ( AttributeSet :: ReturnIndex , <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( GEPXY , <NUM_LIT> ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( GEPZU , <NUM_LIT> ) ; MDNode * MD = llvm :: MDNode :: get ( Mod -> getContext


<s> PointerType * PtrTy = dyn_cast < PointerType > ( ParamTy ) ; if ( PtrTy && PtrTy -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ) { LocalMemLimit = <NUM_LIT> ; DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } } LocalMemLimit = ST . getLocalMemorySize ( ) ; if ( LocalMemLimit == <NUM_LIT> ) return false ; const DataLayout & DL = Mod -> getDataLayout ( ) ; CurrentLocalMemUsage = <NUM_LIT> ; for ( GlobalVariable & GV : Mod -> globals ( ) ) { if ( GV . getType ( ) -> getAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) continue ; for ( const User * U : GV . users ( ) ) { const Instruction * Use = dyn_cast < Instruction > ( U ) ; if ( ! Use ) continue ; if ( Use -> getParent ( ) -> getParent ( ) == & F ) { unsigned Align = GV . getAlignment ( ) ; if ( Align == <NUM_LIT> ) Align = DL . getABITypeAlignment ( GV . getValueType ( ) ) ; uint64_t AllocSize = DL . getTypeAllocSize ( GV . getValueType ( ) ) ; CurrentLocalMemUsage = alignTo ( CurrentLocalMemUsage , Align ) ; CurrentLocalMemUsage += AllocSize ; break ; } } } unsigned MaxOccupancy = ST . getOccupancyWithLocalMemSize ( CurrentLocalMemUsage ) ; unsigned OccupancyHint = <STR_LIT> ( F , <STR_LIT> , <NUM_LIT> ) ; if ( OccupancyHint == <NUM_LIT> ) OccupancyHint = <NUM_LIT> ; OccupancyHint = std :: min ( OccupancyHint , ST . getMaxWavesPerCU ( ) ) ; MaxOccupancy = std :: min ( OccupancyHint , MaxOccupancy


<s> std :: pair < Value * , Value * > <ISA_LIT>PromoteAllocaImpl :: getLocalSizeYZ ( IRBuilder < > & Builder ) { const Function & F = * Builder . GetInsertBlock ( ) -> getParent ( ) ; const <ISA_LIT>Subtarget & ST = <ISA_LIT>Subtarget :: get ( TM , F ) ; if ( ! IsAMDHSA ) { Function * LocalSizeYFn = <STR_LIT> ( Mod , <STR_LIT> ) ; Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addRetAttr ( Attribute :: NoAlias ) ; DispatchPtr -> addRetAttr ( Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableRetAttr ( <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( I32Ty , GEPXY , Align ( <NUM_LIT> ) ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( I32Ty , GEPZU , Align ( <NUM_LIT> ) ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD ) ; LoadZU -> setMetadata ( LLVMContext :: MD_invariant_load


<s> if ( ! IsAMDHSA ) { Function * LocalSizeYFn = <STR_LIT> ( Mod , <STR_LIT> ) ; Function * LocalSizeZFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * LocalSizeY = Builder . CreateCall ( LocalSizeYFn , { } ) ; CallInst * LocalSizeZ = Builder . CreateCall ( LocalSizeZFn , { } ) ; ST . makeLIDRangeMetadata ( LocalSizeY ) ; ST . makeLIDRangeMetadata ( LocalSizeZ ) ; return std :: make_pair ( LocalSizeY , LocalSizeZ ) ; } assert ( IsAMDGCN ) ; Function * DispatchPtrFn = <STR_LIT> ( Mod , <STR_LIT> ) ; CallInst * DispatchPtr = Builder . CreateCall ( DispatchPtrFn , { } ) ; DispatchPtr -> addRetAttr ( Attribute :: NoAlias ) ; DispatchPtr -> addRetAttr ( Attribute :: NonNull ) ; DispatchPtr -> addDereferenceableRetAttr ( <NUM_LIT> ) ; Type * I32Ty = Type :: getInt32Ty ( Mod -> getContext ( ) ) ; Value * CastDispatchPtr = Builder . CreateBitCast ( DispatchPtr , PointerType :: get ( I32Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; Value * GEPXY = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadXY = Builder . CreateAlignedLoad ( I32Ty , GEPXY , Align ( <NUM_LIT> ) ) ; Value * GEPZU = Builder . CreateConstInBoundsGEP1_64 ( I32Ty , CastDispatchPtr , <NUM_LIT> ) ; LoadInst * LoadZU = Builder . CreateAlignedLoad ( I32Ty , GEPZU , Align ( <NUM_LIT> ) ) ; MDNode * MD = MDNode :: get ( Mod -> getContext ( ) , None ) ; LoadXY -> setMetadata ( LLVMContext :: MD_invariant_load , MD


<s> if ( ! canVectorizeInst ( cast < Instruction > ( GEPUser ) , AllocaUser ) ) return false ; WorkList . push_back ( GEPUser ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * AllocaTy << <STR_LIT> << * VectorTy << '\n' ) ; for ( Value * V : WorkList ) { Instruction * Inst = cast < Instruction > ( V ) ; IRBuilder < > Builder ( Inst ) ; switch ( Inst -> getOpcode ( ) ) { case Instruction :: Load : { if ( Inst -> getType ( ) == AllocaTy ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = cast < LoadInst > ( Inst ) -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad ( VectorTy , BitCast ) ; Value * ExtractElement = Builder . CreateExtractElement ( VecValue , Index ) ; Inst -> replaceAllUsesWith ( ExtractElement ) ; Inst -> eraseFromParent ( ) ; break ; } case Instruction :: Store : { StoreInst * <ISA_LIT> = cast < StoreInst > ( Inst ) ; if ( <ISA_LIT> -> getValueOperand ( ) -> getType ( ) == AllocaTy ) break ; Type * VecPtrTy = VectorTy -> getPointerTo ( <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; Value * Ptr = <ISA_LIT> -> getPointerOperand ( ) ; Value * Index = calculateVectorIndex ( Ptr , GEPVectorIdx ) ; Value * BitCast = Builder . CreateBitCast ( Alloca , VecPtrTy ) ; Value * VecValue = Builder . CreateLoad