<s> unsigned encodeVmcnt ( const IsaVersion & Version , unsigned Waitcnt , unsigned Vmcnt ) { Waitcnt = packBits ( Vmcnt , Waitcnt , getVmcntBitShiftLo ( Version . Major ) , getVmcntBitWidthLo


<s> unsigned getExpcntBitMask ( const IsaVersion & Version ) { return ( <NUM_LIT> << getExpcntBitWidth ( Version . Major


<s> return ( <NUM_LIT> << getExpcntBitWidth ( Version . Major


<s> unsigned getExpcntBitShift ( unsigned VersionMajor ) { return VersionMajor >= <NUM_LIT> ? <NUM_LIT> :


<s> unsigned getExpcntBitWidth ( unsigned VersionMajor ) { return


<s> const GcnBufferFormatInfo * getGcnBufferFormatInfo ( uint8_t Format , const MCSubtargetInfo & STI ) { return isGFX11Plus ( STI ) ? getGfx11PlusBufferFormatInfo ( Format ) : isGFX10 ( STI ) ? getGfx10BufferFormatInfo ( Format ) : getGfx9BufferFormatInfo


<s> return ( MsgId == ID_SYSMSG ) ? OpSysSymbolic [ OpId ] :


<s> assert ( msgRequiresOp ( MsgId , STI ) ) ; return ( MsgId == ID_SYSMSG ) ? OpSysSymbolic [ OpId ] :


<s> for ( int Id = UfmtGFX10 :: UFMT_FIRST ; Id <= UfmtGFX10 :: UFMT_LAST ; ++ Id ) { if ( Name == UfmtSymbolicGFX10 [ Id ] ) return Id ; } } return


<s> if ( isValidUnifiedFormat ( Id , STI ) ) return isGFX10 ( STI ) ? UfmtSymbolicGFX10 [ Id ] : UfmtSymbolicGFX11 [ Id ] ; return


<s> return ( <NUM_LIT> << ( getVmcntBitWidthLo ( Version . Major ) + getVmcntBitWidthHi ( Version . Major )


<s> unsigned getVmcntBitMask ( const IsaVersion & Version ) { return ( <NUM_LIT> << ( getVmcntBitWidthLo ( Version . Major ) +

getVmcntBitWidthHi ( Version . Major ) ) ) - <NUM_LIT> ;


<s> unsigned getVmcntBitShiftHi ( unsigned VersionMajor ) { return <NUM_LIT>


<s> unsigned getVmcntBitShiftLo ( unsigned VersionMajor ) { return VersionMajor >= <NUM_LIT> ?


<s> unsigned getVmcntBitWidthLo ( unsigned VersionMajor ) { return VersionMajor >= <NUM_LIT> ? <NUM_LIT>


<s> unsigned getVmcntBitWidthLo ( unsigned VersionMajor ) { return VersionMajor >= <NUM_LIT> ?


<s> unsigned VmcntLo = getBitMask ( getVmcntBitShiftLo ( Version . Major ) , getVmcntBitWidthLo ( Version . Major ) ) ; unsigned Expcnt = getBitMask ( getExpcntBitShift ( Version . Major ) , getExpcntBitWidth ( Version


<s> bool hasPackedD16 ( const MCSubtargetInfo & STI ) { return ! STI . getFeatureBits ( ) [ <STR_LIT> ] && ! isCI ( STI ) && ! is<ISA_LIT>


<s> bool hasPackedD16 ( const MCSubtargetInfo & STI ) { return ! STI . getFeatureBits ( ) [ <STR_LIT> ] && ! isCI ( STI ) && !


<s> bool isGFX10Plus ( const MCSubtargetInfo & STI


<s> bool isGFX8_GFX9_GFX10 ( const MCSubtargetInfo & STI ) { return isVI ( STI ) || isGFX9 ( STI ) || isGFX10 (


<s> bool isGFX8_GFX9_GFX10 ( const MCSubtargetInfo & STI ) { return isVI ( STI ) || isGFX9 (


<s> switch ( Id ) { case ET_NULL : return ! isGFX11Plus ( STI ) ; case ET_POS4 : case ET_PRIM : return isGFX10Plus ( STI ) ; case ET_DUAL_SRC_BLEND0 : case ET_DUAL_SRC_BLEND1 : return isGFX11Plus ( STI ) ; default : if ( Id >= ET_PARAM0 && Id <= ET_PARAM31 ) return ! isGFX11Plus (


<s> bool isValidMsgId ( int64_t MsgId , const


<s> switch ( MsgId ) { case ID_GS_PreGFX11 : return ( OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_ ) && OpId != OP_GS_NOP ; case ID_GS_DONE_PreGFX11 : return OP_GS_FIRST_ <= OpId &&


<s> bool isValidMsgStream ( int64_t MsgId , int64_t OpId , int64_t StreamId , const MCSubtargetInfo & STI , bool Strict ) { assert ( isValidMsgOp ( MsgId , OpId , STI , Strict ) ) ; if ( ! Strict ) return <NUM_LIT> <= StreamId && isUInt < STREAM_ID_WIDTH_ > ( StreamId ) ; if ( ! isGFX11Plus ( STI ) ) { switch ( MsgId ) { case ID_GS_PreGFX11 : return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_ ; case


<s> return isGFX10 ( STI ) ? Id <= UfmtGFX10 ::


<s> return MsgId == ID_SYSMSG || ( ! isGFX11Plus ( STI ) && ( MsgId ==

ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11 ) ) ;


<s> bool msgSupportsStream ( int64_t MsgId , int64_t OpId , const


<s> return ! isGFX11Plus ( STI ) && ( MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11 ) && OpId


<s> return ( ( Src << Shift ) & Mask ) |


<s> if ( isGFX10 ( STI ) ) return UFMT_DEFAULT ; return


<s> unsigned getDefaultFormatEncoding ( const MCSubtargetInfo & STI ) { if ( isGFX10 ( STI ) ) return UFMT_DEFAULT ; return DFMT_NFMT_DEFAULT


<s> return isGFX10 ( STI ) ? ( Val <= UFMT_MAX )


<s> return <STR_LIT> :: HOSTCALL_PTR_OFFSET ; default : llvm_unreachable ( <STR_LIT> ) ; return <NUM_LIT>


<s> int Idx = getOprIdx < const MCSubtargetInfo & > ( Id , Opr , OPR_<ISA_LIT>ZE , STI ) ; return ( Idx < <NUM_LIT> ) ? <STR_LIT>


<s> StringRef getHwreg ( unsigned Id , const MCSubtargetInfo & STI ) { int Idx = getOprIdx < const MCSubtargetInfo & > ( Id , Opr


<s> bool getMAIIsDGEMM ( unsigned Opc ) { const MAIInstInfo * Info = getMAIInstInfoHelper ( Opc ) ; return Info ? Info


<s> bool getMAIIsDGEMM ( unsigned Opc ) { const MAIInstInfo * Info = getMAIInstInfoHelper


<s> int Idx = getOprIdx < const MCSubtargetInfo & > ( Name , Msg , MSG_<ISA_LIT>ZE , STI ) ; return ( Idx < <NUM_LIT> ) ? Idx


<s> int Idx = getOprIdx < const MCSubtargetInfo & > ( MsgId , Msg , MSG_<ISA_LIT>ZE , STI ) ; return ( Idx < <NUM_LIT> ) ? <STR_LIT> : Msg [


<s> static int getOprIdx ( int Id , const CustomOperand < T > OpInfo [ ] , int OpInfoSize , T Context , bool QuickCheck = true ) { auto Test = [ = ] ( const CustomOperand < T > & Op ) { return Op . Encoding == Id && ! Op . Name . empty ( ) ; } ; if ( QuickCheck && isValidOpr < T > ( Id , OpInfo , OpInfoSize , Context ) && OpInfo [ Id ]


<s> if ( has90AInsts && ArgNumAGPR ) return alignTo ( ArgNumVGPR , <NUM_LIT>


<s> bool hasMAIInsts ( const MCSubtargetInfo & STI ) { return STI . getFeatureBits ( ) [ <STR_LIT>


<s> return isGFX10 ( STI ) && ! <STR_LIT> ( STI


<s> bool isGFX10Before1030 ( const MCSubtargetInfo


<s> bool isGFX8Plus ( const MCSubtargetInfo & STI ) { return isVI ( STI ) || isGFX9Plus ( STI


<s> return <STR_LIT> ( Func -> getCallingConv ( )


<s> bool isNotGFX10Plus ( const MCSubtargetInfo & STI ) { return is<ISA_LIT> ( STI ) || isCI ( STI ) || isVI ( STI ) || isGFX9


<s> if ( ! Strict ) return <NUM_LIT> <= OpId && isUInt < OP_WIDTH_ > ( OpId ) ; switch ( MsgId ) { case ID_GS : return ( OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_ ) && OpId != OP_GS_NOP ; case ID_GS_DONE : return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_ ; case ID_SYSMSG : return OP_SYS_FIRST_ <= OpId &&


<s> return <NUM_LIT> <= Idx && Idx < OpInfoSize && ! OpInfo [ Idx ] . Name . empty ( ) && (

! OpInfo [ Idx ] . Cond || OpInfo [ Idx ] . Cond ( Context ) ) ;


<s> static bool isValidOpr ( int Idx , const CustomOperand < T > OpInfo [ ] , int OpInfoSize


<s> * this = getDefaultForCallingConv ( F . getCallingConv ( ) ) ; StringRef IEEEAttr = F . getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) ; if ( ! IEEEAttr . empty ( ) ) IEEE = IEEEAttr == <STR_LIT> ; StringRef DX10ClampAttr = F . getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) ; if ( ! DX10ClampAttr . empty ( ) ) DX10Clamp = DX10ClampAttr == <STR_LIT> ; FP32Denormals = ST . hasFP32Denormals ( F


<s> <STR_LIT> ( const Function & F , const GCNSubtarget & ST ) { * this = getDefaultForCallingConv ( F . getCallingConv ( ) ) ; StringRef IEEEAttr = F . getFnAttribute ( <STR_LIT> )


<s> if ( DX10Clamp != CalleeMode . DX10Clamp ) return false ; if ( IEEE != CalleeMode . IEEE ) return false ; return oneWayCompatible ( FP64FP16Denormals , CalleeMode . FP64FP16Denormals ) && oneWayCompatible ( FP32Denormals , CalleeMode


<s> if ( IEEE != CalleeMode . IEEE ) return false ; return oneWayCompatible ( FP64FP16Denormals , CalleeMode . FP64FP16Denormals ) && oneWayCompatible ( FP32Denormals , CalleeMode


<s> <ISA_LIT>ModeRegisterDefaults Mode ; Mode . DX10Clamp = true ; Mode . IEEE = IsCompute ; Mode . FP32InputDenormals = false ; Mode . FP32OutputDenormals = false ; Mode . FP64FP16InputDenormals


<s> <ISA_LIT>ModeRegisterDefaults Mode ; Mode . DX10Clamp = true ; Mode . IEEE = IsCompute ; Mode . FP32InputDenormals = false ; Mode . FP32OutputDenormals = false


<s> } unsigned SubReg0 = TRI -> getSubReg ( Reg0 , <NUM_LIT> ) ; if ( SubReg0 == <NUM_LIT> ) { return TRI -> getSubRegIndex ( Reg1 , Reg0 ) > <NUM_LIT> ; } for ( unsigned Idx = <NUM_LIT> ; SubReg0 > <NUM_LIT> ; ++ Idx ) { if ( isRegIntersect ( Reg1 ,


( FeatureCuMode ) ? <NUM_LIT> : <NUM_LIT> ) ;


<s> Header . amd_kernel_code_version_minor = <NUM_LIT> ; Header . amd_machine_kind = <NUM_LIT> ; Header . amd_machine_version_major = Version . Major ; Header . amd_machine_version_minor = Version . Minor ; Header . amd_machine_version_stepping = Version . Stepping ; Header . kernel_code_entry_byte_offset = sizeof ( Header ) ; Header . wavefront_size = <NUM_LIT> ; Header . call_convention = - <NUM_LIT> ; Header . kernarg_segment_alignment =


<s> if ( Features . test ( FeatureISAVersion8_0_3 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_1_0 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureVolcanicIslands ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_0 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_2 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_4 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_6 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureGFX9 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureSouthernIslands ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; return { <NUM_LIT> , <NUM_LIT> ,


<s> if ( Features . test ( FeatureISAVersion6_0_0 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion6_0_1 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion7_0_0 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion7_0_1 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion7_0_2 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion7_0_3 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion7_0_4 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureSeaIslands ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_0_1 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_0_2 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_0_3 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_1_0 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureVolcanicIslands ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_0 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_2 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_4 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_6 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureGFX9 ) ) return { <NUM_LIT> ,


<s> return getMaxWavesPerEU ( ) * getEUsPerCU (


<s> return getMaxWavesPerEU ( ) * getEUsPerCU


<s> if ( WavesPerEU >= getMaxWavesPerEU ( ) ) return <NUM_LIT> ; unsigned MinNumSGPRs = getTotalNumSGPRs ( Features ) / ( WavesPerEU + <NUM_LIT> ) ; if ( Features . test ( FeatureTrapHandler ) ) MinNumSGPRs -= std :: min ( MinNumSGPRs , ( unsigned ) TRAP_NUM_SGPRS ) ; MinNumSGPRs = alignDown ( MinNumSGPRs , getSGPRAllocGranule ( Features )


<s> unsigned getMinNumVGPRs ( const FeatureBitset & Features , unsigned WavesPerEU ) { assert ( WavesPerEU != <NUM_LIT> ) ; if ( WavesPerEU >= getMaxWavesPerEU ( ) ) return <NUM_LIT> ; unsigned MinNumVGPRs = alignDown ( getTotalNumVGPRs ( Features ) / ( WavesPerEU + <NUM_LIT> ) , getVGPRAllocGranule ( Features


<s> return getNumExtraSGPRs ( Features , VCCUsed , FlatScrUsed , Features


<s> NumVGPRs = alignTo ( std :: max ( <NUM_LIT> , NumVGPRs ) , getVGPREncodingGranule (


<s> auto TargetTriple = STI -> getTargetTriple ( ) ; auto ISAVersion = IsaInfo :: getIsaVersion ( STI -> getFeatureBits ( ) ) ; Stream << TargetTriple . getArchName ( ) << '-' << TargetTriple . getVendorName ( ) << '-' << TargetTriple . getOSName ( ) << '-' << TargetTriple . getEnvironmentName ( ) << '-' << <STR_LIT>

<< ISAVersion . Major << ISAVersion . Minor << ISAVersion . Stepping ;


<s> void streamIsaVersion ( const MCSubtargetInfo * STI , raw_ostream & Stream ) { auto TargetTriple = STI -> getTargetTriple ( ) ; auto ISAVersion = IsaInfo :: getIsaVersion ( STI -> getFeatureBits (


<s> if ( Literal >= - <NUM_LIT> && Literal <= <NUM_LIT> ) return true ; uint16_t Val = static_cast < uint16_t >


<s> bool is<ISA_LIT>SrcInlinableOperand ( const MCInstrDesc & Desc , unsigned OpNo ) { unsigned OpType = Desc . OpInfo [ OpNo ] . OperandType ; return OpType >= <STR_LIT> &&


<s> bool is<ISA_LIT>SrcOperand ( const MCInstrDesc & Desc , unsigned OpNo ) { unsigned OpType = Desc . OpInfo [ OpNo ] . OperandType ; return OpType >= <STR_LIT> && OpType


<s> bool is<ISA_LIT>SrcOperand ( const MCInstrDesc & Desc , unsigned OpNo ) { unsigned OpType = Desc . OpInfo [ OpNo ] . OperandType ; return OpType >= <STR_LIT> &&


<s> bool isGFX9Plus ( const MCSubtargetInfo & STI


<s> bool isGFX9Plus ( const MCSubtargetInfo & STI ) { return isGFX9 ( STI ) ||


<s> else { AS . FLAT_ADDRESS = <NUM_LIT> ; AS . CONSTANT_ADDRESS = <NUM_LIT> ; AS . PRIVATE_ADDRESS = <NUM_LIT> ; AS . REGION_ADDRESS =


<s> Stream << TargetTriple . getArchName ( ) << '-' << TargetTriple . getVendorName ( ) << '-' << TargetTriple . getOSName ( ) << '-' << TargetTriple . getEnvironmentName ( ) << '-' << <STR_LIT> << Version . Major << Version . Minor << Version . Stepping ; if ( hasXNACK ( * STI )


<s> Stream << TargetTriple . getArchName ( ) << '-' << TargetTriple . getVendorName ( ) << '-' << TargetTriple . getOSName ( ) << '-' << TargetTriple . getEnvironmentName ( ) << '-' << <STR_LIT> << Version . Major << Version . Minor << Version . Stepping ; if ( hasXNACK ( * STI


<s> if ( ! isDwordAligned ( ByteOffset ) && ! hasSMEMByteOffset ( ST ) ) return None ; assert ( isCI ( ST ) ) ; int64_t EncodedOffset = convertSMRDOffsetUnits ( ST ,


<s> Optional < int64_t > getSMRDEncodedOffset ( const MCSubtargetInfo & ST , int64_t ByteOffset ) { if ( ! isDwordAligned ( ByteOffset ) && ! hasSMEMByteOffset


<s> return hasSMEMByteOffset ( ST ) ? isUInt < <NUM_LIT> > ( EncodedOffset ) : isUInt < <NUM_LIT> > (


<s> StringRef DX10ClampAttr = F . getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) ; if ( ! DX10ClampAttr . empty ( ) ) DX10Clamp = DX10ClampAttr == <STR_LIT> ; FP32InputDenormals = ST . hasFP32Denormals ( F ) ; FP32OutputDenormals =


<s> return packBits ( Vmcnt , Waitcnt , getVmcntBitShift (


<s> if ( Features . test ( FeatureISAVersion8_0_1 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_0_2 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_0_3 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_0_4 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_1_0 ) ) return { <NUM_LIT> , <NUM_LIT>


<s> if ( Features . test ( FeatureISAVersion8_0_3 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_0_4 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_1_0 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( ! Features . test ( FeatureGCN ) || Features . test (

FeatureSouthernIslands ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ;


<s> unsigned getVmcntBitMask ( const IsaInfo :: IsaVersion & Version ) { return ( <NUM_LIT> << getVmcntBitWidth ( ) ) -


<s> unsigned getVmcntBitMask ( const IsaInfo :: IsaVersion


<s> unsigned Lgkmcnt = getBitMask ( getLgkmcntBitShift ( ) , getLgkmcntBitWidth ( ) ) ; return Vmcnt | Expcnt


<s> unsigned getWaitcntBitMask ( const IsaInfo :: IsaVersion & Version ) { unsigned Vmcnt = getBitMask ( getVmcntBitShift ( ) , getVmcntBitWidth (


<s> AS . FLAT_ADDRESS = <NUM_LIT> ; AS . PRIVATE_ADDRESS = <NUM_LIT> ; AS . REGION_ADDRESS =


<s> return getIntegerAttribute ( F , <STR_LIT> , <NUM_LIT>


<s> unsigned getMaximumWorkGroupSize ( const Function &


<s> return unpackBits ( Waitcnt , getExpcntBitShift ( )


<s> return unpackBits ( Waitcnt , getLgkmcntBitShift (


<s> unsigned VmcntLo = unpackBits ( Waitcnt , getVmcntBitShiftLo ( ) , getVmcntBitWidthLo ( ) ) ; if ( Version . Major < <NUM_LIT> ) return VmcntLo ; unsigned VmcntHi = unpackBits ( Waitcnt , getVmcntBitShiftHi ( ) , getVmcntBitWidthHi ( ) ) ; VmcntHi <<= getVmcntBitWidthLo ( ) ; return VmcntLo | VmcntHi


<s> Waitcnt = packBits ( Vmcnt , Waitcnt , getVmcntBitShiftLo ( ) , getVmcntBitWidthLo ( ) ) ; if ( Version . Major < <NUM_LIT> ) return


<s> unsigned encodeWaitcnt ( const IsaVersion & Version , const Waitcnt & Decoded ) { return encodeWaitcnt ( Version , Decoded . VmCnt , Decoded . ExpCnt , Decoded


<s> unsigned encodeWaitcnt ( const IsaVersion & Version , const Waitcnt & Decoded ) { return encodeWaitcnt ( Version , Decoded . VmCnt , Decoded .


<s> if ( STI -> getFeatureBits ( ) . test ( FeatureSGPRInitBug ) ) return FIXED_NUM_SGPRS_FOR_INIT_BUG ; IsaVersion Version = getIsaVersion ( STI -> getCPU ( ) ) ; if ( Version . Major >= <NUM_LIT> ) return <NUM_LIT> ; return <NUM_LIT>


<s> unsigned getAddressableNumVGPRs ( const MCSubtargetInfo * STI ) { return getTotalNumVGPRs (


<s> unsigned getBitMask ( unsigned Shift , unsigned Width ) { return ( ( <NUM_LIT> << Width


<s> unsigned getEUsPerCU ( const MCSubtargetInfo * STI ) { return <NUM_LIT>


<s> return ( <NUM_LIT> << getExpcntBitWidth ( ) ) -


<s> unsigned getExpcntBitWidth ( ) { return <NUM_LIT>


<s> unsigned getInitialP<ISA_LIT>nputAddr ( const Function


<s> unsigned getInitialP<ISA_LIT>nputAddr ( const Function & F ) { return getIntegerAttribute ( F , <STR_LIT>


<s> if ( ! A . isStringAttribute ( ) ) return Default ; LLVMContext & Ctx = F . getContext ( ) ; std :: pair < int , int > Ints = Default ; std :: pair < StringRef , StringRef > Strs = A . getValueAsString ( ) . split ( ',' ) ; if ( Strs . first . trim ( ) . getAsInteger (


<s> unsigned getLgkmcntBitMask ( const IsaVersion & Version ) { return ( <NUM_LIT> << getLgkmcntBitWidth (


<s> unsigned getLgkmcntBitMask ( const IsaVersion & Version


<s> unsigned getLgkmcntBitWidth ( ) { return <NUM_LIT>


<s> unsigned getLgkmcntBitWidth ( ) { return


<s> if ( STI -> getFeatureBits ( ) . test ( FeatureLocalMemorySize32768 ) ) return <NUM_LIT> ; if ( STI -> getFeatureBits ( ) . test ( FeatureLocalMemorySize65536 ) ) return <NUM_LIT> ; return <NUM_LIT>


<s> const MIMGInfo * NewInfo = getMIMGOpcodeHelper ( OrigInfo -> BaseOpcode , OrigInfo -> MIMGEncoding , NewChannels , OrigInfo -> VAddrDwords ) ; return NewInfo ? NewInfo ->


<s> unsigned getMaxFlatWorkGroupSize ( const MCSubtargetInfo *


<s> unsigned getMaxNumVGPRs ( const MCSubtargetInfo * STI , unsigned WavesPerEU ) { assert ( WavesPerEU != <NUM_LIT> ) ; unsigned MaxNumVGPRs = alignDown ( getTotalNumVGPRs ( STI ) / WavesPerEU , getVGPRAllocGranule ( STI ) ) ; unsigned AddressableNumVGPRs = getAddressableNumVGPRs ( STI ) ; return std :: min (


<s> unsigned getMaxWavesPerCU ( const MCSubtargetInfo * STI ,


<s> if ( ! STI -> getFeatureBits ( ) . test ( FeatureGCN ) ) return <NUM_LIT> ; unsigned N = getWavesPerWorkGroup ( STI , FlatWorkGroupSize ) ; if ( N == <NUM_LIT> )


<s> return getMCOpcodeGen ( Opcode , static_cast < Subtarget >


<s> const MIMGInfo * Info = getMIMGOpcodeHelper ( BaseOpcode , MIMGEncoding , VDataDwords , VAddrDwords ) ; return Info ? Info -> Opcode :


<s> int getMIMGOpcode ( unsigned BaseOpcode , unsigned MIMGEncoding , unsigned VDataDwords , unsigned VAddrDwords ) { const MIMGInfo * Info = getMIMGOpcodeHelper ( BaseOpcode , MIMGEncoding , VDataDwords ,


<s> unsigned getMinNumSGPRs ( const MCSubtargetInfo * STI , unsigned WavesPerEU ) { assert ( WavesPerEU != <NUM_LIT> ) ; if ( WavesPerEU >= getMaxWavesPerEU ( ) ) return <NUM_LIT> ; unsigned MinNumSGPRs = getTotalNumSGPRs ( STI ) / ( WavesPerEU + <NUM_LIT> ) ; if ( STI -> getFeatureBits ( ) . test ( FeatureTrapHandler ) ) MinNumSGPRs -= std :: min ( MinNumSGPRs , (


<s> const MUBUFInfo * Info = getMUBUFInfoFromOpcode ( Opc ) ; return Info ? Info -> BaseOpcode :


<s> const MUBUFInfo * Info = getMUBUFOpcodeHelper ( Opc ) ; return Info ? Info ->


<s> const MUBUFInfo * Info = getMUBUFOpcodeHelper ( Opc


<s> return getNumExtraSGPRs ( STI , VCCUsed , FlatScrUsed , STI ->

getFeatureBits ( ) . test ( <STR_LIT> ) ) ;


<s> return getNumExtraSGPRs ( STI , VCCUsed , FlatScrUsed , STI -> getFeatureBits ( ) . test ( <STR_LIT>


<s> NumSGPRs = alignTo ( std :: max ( <NUM_LIT> , NumSGPRs ) , getSGPREncodingGranule ( STI


<s> unsigned getRegBitWidth ( const MCRegisterClass &


<s> unsigned getRegOperandSize ( const MCRegisterInfo * MRI , const MCInstrDesc & Desc , unsigned OpNo ) { assert ( OpNo < Desc .


<s> unsigned getSGPREncodingGranule ( const MCSubtargetInfo * STI ) { return


<s> if ( isGCN3Encoding ( ST ) ) return ByteOffset ; return ByteOffset >> <NUM_LIT>


<s> unsigned getVGPRAllocGranule ( const MCSubtargetInfo *


<s> unsigned getVmcntBitMask ( const IsaVersion & Version ) { unsigned VmcntLo = ( <NUM_LIT> << getVmcntBitWidthLo ( ) ) - <NUM_LIT> ; if ( Version . Major < <NUM_LIT> )


<s> unsigned getVmcntBitShiftLo ( ) { return <NUM_LIT>


<s> unsigned Expcnt = getBitMask ( getExpcntBitShift ( ) , getExpcntBitWidth ( ) ) ; unsigned Lgkmcnt = getBitMask ( getLgkmcntBitShift ( ) , getLgkmcntBitWidth ( ) ) ; unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt ; if ( Version . Major < <NUM_LIT> ) return Waitcnt ; unsigned VmcntHi = getBitMask ( getVmcntBitShiftHi ( ) , getVmcntBitWidthHi ( ) ) ; return Waitcnt |


<s> unsigned getWavefrontSize ( const MCSubtargetInfo * STI ) { if ( STI -> getFeatureBits ( ) . test ( FeatureWavefrontSize16 ) ) return <NUM_LIT> ; if ( STI -> getFeatureBits ( ) . test ( FeatureWavefrontSize32 ) ) return <NUM_LIT> ; return <NUM_LIT>


<s> return STI -> getTargetTriple ( ) . getOS ( ) == Triple :: AMDHSA && STI

-> getFeatureBits ( ) . test ( FeatureCodeObjectV3 ) ;


<s> return STI -> getTargetTriple ( ) . getOS ( ) == Triple :: AMDHSA &&

STI -> getFeatureBits ( ) . test ( FeatureCodeObjectV3 ) ;


<s> return STI . getFeatureBits ( ) [ <STR_LIT>


<s> bool hasMIMG_R128 ( const MCSubtargetInfo &


<s> bool hasSRAMECC ( const MCSubtargetInfo & STI ) { return STI . getFeatureBits ( ) [ <STR_LIT>


<s> Header . amd_machine_version_stepping = Version . Stepping ; Header . kernel_code_entry_byte_offset = sizeof ( Header ) ; Header . wavefront_size = <NUM_LIT> ; Header . call_convention = - <NUM_LIT> ; Header . kernarg_segment_alignment = <NUM_LIT> ; Header . group_segment_alignment = <NUM_LIT> ; Header . private_segment_alignment =


<s> bool isCompute ( CallingConv :: ID cc ) { return ! isShader ( cc ) || cc ==


<s> bool isGCN3Encoding ( const MCSubtargetInfo & STI ) { return STI . getFeatureBits (


<s> return ( Val == FloatToBits ( <NUM_LIT> ) ) || ( Val == FloatToBits ( <NUM_LIT> ) ) || ( Val == FloatToBits ( - <NUM_LIT> ) ) || ( Val == FloatToBits ( <NUM_LIT> ) ) || ( Val == FloatToBits ( - <NUM_LIT> ) ) || ( Val == FloatToBits ( <NUM_LIT> ) ) || ( Val == FloatToBits ( - <NUM_LIT> )

) || ( Val == FloatToBits ( <NUM_LIT> ) ) || ( Val == FloatToBits ( - <NUM_LIT> ) ) || ( Val == <NUM_LIT> && HasInv2Pi ) ;


<s> if ( Literal >= - <NUM_LIT> && Literal <= <NUM_LIT> ) return true ; uint64_t Val = static_cast < uint64_t > ( Literal ) ; return ( Val == DoubleToBits ( <NUM_LIT> ) ) || ( Val == DoubleToBits ( <NUM_LIT> ) ) || ( Val == DoubleToBits ( - <NUM_LIT> ) ) || ( Val == DoubleToBits ( <NUM_LIT> ) ) || ( Val == DoubleToBits ( - <NUM_LIT> ) ) || ( Val == DoubleToBits ( <NUM_LIT> ) ) || ( Val == DoubleToBits (

- <NUM_LIT> ) ) || ( Val == DoubleToBits ( <NUM_LIT> ) ) || ( Val == DoubleToBits ( - <NUM_LIT> ) ) || ( Val == <NUM_LIT> && HasInv2Pi ) ;


<s> return ( Val == DoubleToBits ( <NUM_LIT> ) ) || ( Val == DoubleToBits ( <NUM_LIT> ) ) || ( Val == DoubleToBits ( - <NUM_LIT> ) ) || ( Val == DoubleToBits ( <NUM_LIT> ) ) || ( Val == DoubleToBits ( - <NUM_LIT> ) ) || ( Val == DoubleToBits ( <NUM_LIT> ) ) ||

( Val == DoubleToBits ( - <NUM_LIT> ) ) || ( Val == DoubleToBits ( <NUM_LIT> ) ) || ( Val == DoubleToBits ( - <NUM_LIT> ) ) || ( Val == <NUM_LIT> && HasInv2Pi ) ;


<s> return isGCN3Encoding ( ST ) ? isUInt < <NUM_LIT> > ( EncodedOffset ) : isUInt < <NUM_LIT>


<s> for ( MCRegAliasIterator R ( Reg0 , TRI , true ) ; R . isValid ( ) ; ++ R ) { if ( * R == Reg1


<s> for ( MCRegAliasIterator R ( Reg0 , TRI , true ) ; R . isValid ( ) ; ++ R ) { if ( * R == Reg1 ) return true ; } return


<s> return SGPRClass . contains ( FirstSubReg != <NUM_LIT> ? FirstSubReg : Reg


<s> switch ( cc ) { case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_LS : case CallingConv


<s> bool isShader ( CallingConv :: ID cc ) { switch ( cc ) { case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_LS : case CallingConv ::


<s> bool is<ISA_LIT>SrcInlinableOperand ( const MCInstrDesc & Desc , unsigned OpNo ) { assert ( OpNo < Desc .


<s> unsigned OpType = Desc . OpInfo [ OpNo ] . OperandType ; return OpType >= <STR_LIT> &&


<s> auto Version = getIsaVersion ( STI -> getCPU ( ) ) ; Stream << TargetTriple . getArchName ( ) << '-' << TargetTriple . getVendorName ( ) << '-' << TargetTriple . getOSName ( ) << '-' << TargetTriple . getEnvironmentName ( ) << '-' << <STR_LIT> << Version . Major << Version . Minor << Version . Stepping ; if ( hasXNACK ( * STI )


<s> static unsigned getCallOpcode ( const MachineFunction & CallerF , bool IsIndirect , bool


<s> return false ; } if ( any_of ( CallerF . args ( ) , [ ] ( const Argument & A ) { return A . hasByValAttr ( ) || A . hasSwiftErrorAttr ( ) ; } ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( MF . getTarget ( ) . Options . GuaranteedTailCallOpt ) return canGuaranteeTCO ( CalleeCC ) && CalleeCC == CallerF . getCallingConv ( ) ; if ( ! doCallerAndCalleePassArgsTheSameWay ( Info , MF , InArgs ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( ! areCalleeOutgoingArgsTailCallable ( Info , MF ,


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineFunction & MF = MIRBuilder . getMF ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const Function & F = MF . getFunction ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; SmallVector < ArgInfo , <NUM_LIT> > OutArgs ; for ( auto & OrigArg : Info . OrigArgs ) splitToValueTypes ( OrigArg , OutArgs , DL , Info . CallConv ) ; SmallVector < ArgInfo , <NUM_LIT> > InArgs ; if ( Info . CanLowerReturn && ! Info . OrigRet . Ty -> isVoidTy ( ) ) splitToValueTypes ( Info . OrigRet , InArgs , DL , Info . CallConv ) ; bool CanTailCallOpt = isEligibleForTailCallOptimization ( MIRBuilder , Info , InArgs , OutArgs ) ; if ( Info . IsMustTailCall && ! CanTailCallOpt ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } Info . IsTailCall = CanTailCallOpt ; if ( CanTailCallOpt ) return lowerTailCall ( MIRBuilder , Info , OutArgs ) ; CCAssignFn * AssignFnFixed ; CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( Info . CallConv , TLI ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , false ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; MIB . addDef ( TRI -> getReturnAddressReg ( MF ) ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , Info . CallConv ) ; MIB . addRegMask (


<s> MFI -> setIfReturnsVoid ( ! Val ) ; assert ( ! Val == VRegs . empty ( ) && <STR_LIT> ) ; CallingConv :: ID CC = B . getMF ( ) . getFunction ( ) . getCallingConv ( ) ; const bool IsShader = <STR_LIT> ( CC ) ; const bool IsWaveEnd = ( IsShader && MFI -> returnsVoid ( ) ) || <STR_LIT> ( CC ) ; if ( IsWaveEnd ) { B . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) ; return true ; } auto const & ST = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned ReturnOpc = <NUM_LIT> ; if ( IsShader ) ReturnOpc = <STR_LIT> ; else if ( CC == CallingConv :: <ISA_LIT>_Gfx ) ReturnOpc = <STR_LIT> ; else ReturnOpc = <STR_LIT> ; auto Ret = B . buildInstrNoInsert ( ReturnOpc ) ; Register ReturnAddrVReg ; if ( ReturnOpc == <STR_LIT> ) { ReturnAddrVReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Ret . addUse ( ReturnAddrVReg ) ; } else if ( ReturnOpc == <STR_LIT> ) { ReturnAddrVReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Ret . addUse


<s> if ( ! IsSibCall ) { unsigned NumReusableBytes = FuncInfo -> getBytesInStackArgArea ( ) ; SmallVector < CCValAssign , <NUM_LIT> > OutLocs ; CCState OutInfo ( CalleeCC , false , MF , OutLocs , F . getContext ( ) ) ; OutgoingValueAssigner CalleeAssigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( CalleeAssigner , OutArgs , OutInfo ) ) return false ; NumBytes = alignTo ( OutInfo . getNextStackOffset ( ) , ST . getStackAlignment ( ) ) ; FPDiff = NumReusableBytes - NumBytes ; assert ( isAligned ( ST . getStackAlignment ( ) , FPDiff ) && <STR_LIT> ) ; } SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ; SmallVector < std :: pair < MCRegister , Register > , <NUM_LIT> > ImplicitArgRegs ; if ( Info . CallConv != CallingConv :: <ISA_LIT>_Gfx ) { if ( ! passSpecialInputs ( MIRBuilder , CCInfo , ImplicitArgRegs , Info ) ) return false ; } OutgoingValueAssigner Assigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( Assigner , OutArgs , CCInfo ) ) return false ; <ISA_LIT>OutgoingArgHandler Handler ( MIRBuilder , MRI , MIB , true , FPDiff ) ; if ( ! handleAssignments ( Handler , OutArgs , CCInfo , ArgLocs , MIRBuilder ) ) return false ; handleImplicitCallArguments ( MIRBuilder , MIB , ST , * FuncInfo , ImplicitArgRegs ) ; if ( ! IsSibCall ) { MIB -> getOperand ( <NUM_LIT> ) . setImm ( FPDiff ) ; CallSeqStart . addImm ( NumBytes ) . addImm ( <NUM_LIT> ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addImm (


<s> CCAssignFn * AssignFnFixed ; CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( CalleeCC , TLI ) ; MachineInstrBuilder CallSeqStart ; if ( ! IsSibCall ) CallSeqStart = MIRBuilder . buildInstr ( <STR_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , true ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; MIB . addImm ( <NUM_LIT> ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , CalleeCC ) ; MIB . addRegMask ( Mask ) ; int FPDiff = <NUM_LIT> ; unsigned NumBytes = <NUM_LIT> ; if ( ! IsSibCall ) { unsigned NumReusableBytes = FuncInfo -> getBytesInStackArgArea ( ) ; SmallVector < CCValAssign , <NUM_LIT> > OutLocs ; CCState OutInfo ( CalleeCC , false , MF , OutLocs , F . getContext ( ) ) ; OutgoingValueAssigner CalleeAssigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( CalleeAssigner , OutArgs , OutInfo ) ) return false ; NumBytes = alignTo ( OutInfo . getNextStackOffset ( ) , ST . getStackAlignment ( ) ) ; FPDiff = NumReusableBytes - NumBytes ; assert ( isAligned ( ST . getStackAlignment ( ) , FPDiff ) && <STR_LIT> ) ; } SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ; SmallVector < std :: pair < MCRegister , Register > , <NUM_LIT> > ImplicitArgRegs ; if ( Info . CallConv != CallingConv :: <ISA_LIT>_Gfx ) { if ( ! passSpecialInputs ( MIRBuilder , CCInfo , ImplicitArgRegs , Info ) ) return false ; } OutgoingValueAssigner Assigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( Assigner , OutArgs , CCInfo ) ) return false ; <ISA_LIT>OutgoingArgHandler Handler ( MIRBuilder , MRI , MIB , true , FPDiff ) ; if ( ! handleAssignments ( Handler , OutArgs , CCInfo , ArgLocs , MIRBuilder ) ) return false ; handleImplicitCallArguments ( MIRBuilder , MIB , ST , * FuncInfo , ImplicitArgRegs ) ; if ( ! IsSibCall ) { MIB -> getOperand ( <NUM_LIT> ) . setImm ( FPDiff ) ; CallSeqStart . addImm ( NumBytes )


<s> if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , ArgTy ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , ArgTy ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) return false ; auto WorkitemIDX = CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ; auto WorkitemIDY = CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ; auto WorkitemIDZ = CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ; const ArgDescriptor * IncomingArgX = std :: get < <NUM_LIT> > ( WorkitemIDX ) ; const ArgDescriptor * IncomingArgY = std :: get < <NUM_LIT> > ( WorkitemIDY ) ; const ArgDescriptor * IncomingArgZ = std :: get < <NUM_LIT> > ( WorkitemIDZ ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const bool NeedWorkItemIDX = ! Info . CB -> hasFnAttr ( <STR_LIT> ) ; const bool NeedWorkItemIDY = ! Info . CB -> hasFnAttr ( <STR_LIT> ) ; const bool NeedWorkItemIDZ = ! Info . CB -> hasFnAttr ( <STR_LIT> ) ; Register InputReg ; if ( IncomingArgX && ! IncomingArgX -> isMasked ( ) && CalleeArgInfo -> WorkItemIDX && NeedWorkItemIDX ) { InputReg = MRI . createGenericVirtualRegister ( S32 ) ; LI -> loadInputValue ( InputReg , MIRBuilder , IncomingArgX , std :: get < <NUM_LIT> > ( WorkitemIDX ) , std :: get < <NUM_LIT> > ( WorkitemIDX ) ) ; } if ( IncomingArgY && ! IncomingArgY -> isMasked ( ) && CalleeArgInfo -> WorkItemIDY && NeedWorkItemIDY ) { Register Y = MRI . createGenericVirtualRegister ( S32 ) ; LI -> loadInputValue ( Y , MIRBuilder , IncomingArgY , std :: get < <NUM_LIT> > ( WorkitemIDY ) , std :: get < <NUM_LIT> > ( WorkitemIDY ) ) ; Y = MIRBuilder . buildShl ( S32 , Y , MIRBuilder . buildConstant ( S32 , <NUM_LIT> ) ) . getReg ( <NUM_LIT> ) ; InputReg = InputReg ? MIRBuilder . buildOr ( S32 , InputReg , Y ) . getReg ( <NUM_LIT> ) : Y ; } if ( IncomingArgZ && ! IncomingArgZ -> isMasked ( ) && CalleeArgInfo -> WorkItemIDZ && NeedWorkItemIDZ ) { Register Z = MRI . createGenericVirtualRegister ( S32 ) ; LI -> loadInputValue ( Z , MIRBuilder , IncomingArgZ , std :: get < <NUM_LIT> > ( WorkitemIDZ ) , std :: get < <NUM_LIT> > ( WorkitemIDZ ) ) ; Z = MIRBuilder . buildShl ( S32 , Z , MIRBuilder . buildConstant ( S32 , <NUM_LIT> ) ) . getReg ( <NUM_LIT> ) ; InputReg = InputReg ? MIRBuilder . buildOr ( S32 , InputReg , Z ) . getReg ( <NUM_LIT> ) : Z ; } if ( ! InputReg && ( NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ ) ) { InputReg = MRI . createGenericVirtualRegister


<s> unsigned DispatchIDReg = Info -> addDispatchID ( * TRI ) ; CCInfo . AllocateReg ( DispatchIDReg ) ; } if ( Info -> hasFlatScratchInit ( ) ) { unsigned FlatScratchInitReg = Info -> addFlatScratchInit ( * TRI ) ; CCInfo . AllocateReg ( FlatScratchInitReg ) ; } unsigned NumArgs = F . arg_size ( ) ; Function :: const_arg_iterator CurOrigArg = F . arg_begin ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ CurOrigArg ) { EVT ValEVT = TLI . getValueType ( DL , CurOrigArg -> getType ( ) ) ; if ( ! ValEVT . isSimple ( ) ) return false ; MVT ValVT = ValEVT . getSimpleVT ( ) ; <STR_LIT> Flags ; ArgInfo OrigArg { VRegs [ i ] , CurOrigArg -> getType ( ) } ; setArgFlags ( OrigArg , i + <NUM_LIT> , DL , F ) ; Flags . setOrigAlign ( DL . getABITypeAlignment ( CurOrigArg -> getType ( ) ) ) ; CCAssignFn * AssignFn = CCAssignFnForCall ( F . getCallingConv ( ) , false ) ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( Res ) return false ; } Function :: const_arg_iterator Arg = F . arg_begin ( ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_VS ) { for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ Arg ) { CCValAssign & VA = ArgLocs [ i ] ; MRI . addLiveIn ( VA . getLocReg ( ) , VRegs [ i ] ) ; MIRBuilder . getMBB ( ) . addLiveIn ( VA . getLocReg ( )


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = * MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; unsigned KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; unsigned OffsetReg = MRI . createGenericVirtualRegister ( LLT :: scalar


<s> bool <ISA_LIT>CallLowering :: lowerReturn ( MachineIRBuilder & MIRBuilder , const


<s> } if ( Info -> hasFlatScratchInit ( ) ) { unsigned FlatScratchInitReg = Info -> addFlatScratchInit ( * TRI ) ; CCInfo . AllocateReg ( FlatScratchInitReg ) ; } unsigned NumArgs = F . arg_size ( ) ; Function :: const_arg_iterator CurOrigArg = F . arg_begin ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; unsigned P<ISA_LIT>nputNum = <NUM_LIT> ; BitVector Skipped ( NumArgs ) ; for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ CurOrigArg ) { EVT ValEVT = TLI . getValueType ( DL , CurOrigArg -> getType ( ) ) ; <STR_LIT> Flags ; ArgInfo OrigArg { VRegs [ i ] , CurOrigArg -> getType ( ) } ; setArgFlags ( OrigArg , i + <NUM_LIT> , DL , F ) ; Flags . setOrigAlign ( DL . getABITypeAlignment ( CurOrigArg -> getType ( ) ) ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && ! OrigArg . Flags . isInReg ( ) && ! OrigArg . Flags . isByVal ( ) && P<ISA_LIT>nputNum <= <NUM_LIT> ) { if ( CurOrigArg -> use_empty ( ) && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ) { Skipped . set ( i ) ; ++ P<ISA_LIT>nputNum ; continue ; } Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! CurOrigArg -> use_empty ( ) ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; ++ P<ISA_LIT>nputNum ; } CCAssignFn * AssignFn = CCAssignFnForCall ( F . getCallingConv ( ) , false ) ; if ( ValEVT . isVector ( ) ) { EVT ElemVT = ValEVT . getVectorElementType ( ) ; if ( ! ValEVT . isSimple ( ) ) return false ; MVT ValVT = ElemVT . getSimpleVT ( ) ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( ! Res ) return false ; } else { MVT ValVT = ValEVT . getSimpleVT ( ) ; if ( ! ValEVT . isSimple ( ) ) return false ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( Res ) return false


<s> const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; unsigned Align = DL . getABITypeAlignment ( ParamTy ) ; unsigned PtrReg = lowerParameterPtr ( MIRBuilder , ParamTy ,


<s> PointerType * PtrTy = PointerType :: get ( ParamTy , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; unsigned KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; unsigned OffsetReg = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildConstant ( OffsetReg , Offset ) ; MIRBuilder . buildGEP ( DstReg , KernArgSegmentVReg , OffsetReg ) ; return


<s> bool <ISA_LIT>CallLowering :: lowerReturn ( MachineIRBuilder & MIRBuilder , const Value * Val , unsigned VReg ) const { if ( Val ) return false ; MIRBuilder . buildInstr ( <STR_LIT>


<s> const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; unsigned PtrReg = lowerParameterPtr ( MIRBuilder , ParamTy ,


<s> void <ISA_LIT>CallLowering :: lowerParameter ( MachineIRBuilder & MIRBuilder , Type * ParamTy , uint64_t Offset , unsigned Align , unsigned DstReg ) const { MachineFunction & MF = MIRBuilder . getMF ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout


<s> const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; unsigned KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; unsigned OffsetReg = MRI . createGenericVirtualRegister ( LLT


<s> if ( Val ) return false ; MIRBuilder . buildInstr ( <STR_LIT> ) ; return true


<s> for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ CurOrigArg ) { EVT ValEVT = TLI . getValueType ( DL , CurOrigArg -> getType ( ) ) ; if ( ! ValEVT . isSimple ( ) ) return false ; MVT ValVT = ValEVT . getSimpleVT ( ) ; <STR_LIT> Flags ; ArgInfo OrigArg { VRegs [ i ] , CurOrigArg -> getType ( ) } ; setArgFlags ( OrigArg , i + <NUM_LIT> , DL , F ) ; Flags . setOrigAlign ( DL . getABITypeAlignment ( CurOrigArg -> getType ( ) ) ) ; CCAssignFn * AssignFn = CCAssignFnForCall ( F . getCallingConv ( ) , false ) ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( Res ) return false ; } Function :: const_arg_iterator Arg = F . arg_begin ( ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_VS ) { for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ Arg ) { CCValAssign & VA = ArgLocs [ i ] ; MRI . addLiveIn ( VA . getLocReg ( ) , VRegs [ i ] ) ; MIRBuilder . getMBB ( ) . addLiveIn ( VA . getLocReg ( ) ) ; MIRBuilder . buildCopy ( VRegs [ i ] , VA . getLocReg ( ) ) ; } return true ; } for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ Arg ) { CCValAssign & VA = ArgLocs [ i ] ; lowerParameter ( MIRBuilder , Arg -> getType ( ) , VA . getLocMemOffset ( ) + Subtarget -> getExplicitKernelArgOffset ( MF


<s> MIRBuilder . getMBB ( ) . addLiveIn ( InputPtrReg ) ; MIRBuilder . buildCopy ( VReg , InputPtrReg ) ; CCInfo . AllocateReg ( InputPtrReg ) ; } if ( Info -> hasDispatchID ( ) ) { unsigned DispatchIDReg = Info -> addDispatchID ( * TRI ) ; CCInfo . AllocateReg ( DispatchIDReg ) ; } if ( Info -> hasFlatScratchInit ( ) ) { unsigned FlatScratchInitReg = Info -> addFlatScratchInit ( * TRI ) ; CCInfo . AllocateReg ( FlatScratchInitReg ) ; } unsigned NumArgs = F . arg_size ( ) ; Function :: const_arg_iterator CurOrigArg = F . arg_begin ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ CurOrigArg ) { EVT ValEVT = TLI . getValueType ( DL , CurOrigArg -> getType ( ) ) ; if ( ! ValEVT . isSimple ( ) ) return false ; MVT ValVT = ValEVT . getSimpleVT ( ) ; <STR_LIT> Flags ; ArgInfo OrigArg { VRegs [ i ] , CurOrigArg -> getType ( ) } ; setArgFlags ( OrigArg , i + <NUM_LIT> , DL , F ) ; Flags . setOrigAlign ( DL . getABITypeAlignment ( CurOrigArg -> getType ( ) ) ) ; CCAssignFn * AssignFn = CCAssignFnForCall ( F . getCallingConv ( ) , false ) ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( Res ) return false ; } Function :: const_arg_iterator Arg = F . arg_begin ( ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_VS ) { for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ Arg ) { CCValAssign & VA = ArgLocs [ i ] ; MRI . addLiveIn ( VA . getLocReg ( ) , VRegs [ i ] ) ; MIRBuilder . getMBB ( ) . addLiveIn ( VA . getLocReg ( ) ) ; MIRBuilder . buildCopy ( VRegs [ i ] , VA . getLocReg ( ) ) ; } return true ; } for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i


<s> return AssignFn ( ValNo , ValVT , LocVT , LocInfo , Flags ,


<s> void assignValueToAddress ( Register ValVReg , Register Addr , uint64_t Size , MachinePointerInfo & MPO , CCValAssign & VA ) override { MachineFunction & MF = MIRBuilder . getMF (


<s> void assignValueToAddress ( Register ValVReg , Register Addr , uint64_t Size , MachinePointerInfo &


<s> void assignValueToReg ( Register ValVReg , Register PhysReg , CCValAssign & VA ) override { Register ExtReg ; if ( VA . getLocVT ( ) . getSizeInBits ( ) < <NUM_LIT> ) { ExtReg = MIRBuilder . buildAnyExt ( LLT :: scalar ( <NUM_LIT> ) , ValVReg ) . getReg


<s> } else ExtReg = extendRegister ( ValVReg , VA ) ; const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; if ( TRI -> isSGPRReg ( MRI , PhysReg ) ) { auto ToSGPR = MIRBuilder . buildIntrinsic ( <STR_LIT> , { MRI . getType ( ExtReg ) } , false ) . addReg ( ExtReg ) ; ExtReg = ToSGPR . getReg ( <NUM_LIT> ) ; } MIRBuilder . buildCopy ( PhysReg


<s> auto AddrReg = MIRBuilder . buildFrameIndex ( LLT :: pointer ( <ISA_LIT>AS :: PRIVATE_ADDRESS , <NUM_LIT> ) , FI ) ; StackUsed = std :: max ( StackUsed , Size + Offset


<s> bool isIncomingArgumentHandler ( ) const override { return true


<s> for ( auto & Arg : F . args ( ) ) { if ( DL . getTypeStoreSize ( Arg . getType ( ) ) == <NUM_LIT> ) continue ; const bool InReg = Arg . hasAttribute ( Attribute :: InReg ) ; if ( ! IsShader && InReg ) return false ; if ( Arg . hasAttribute ( Attribute :: SwiftSelf ) || Arg . hasAttribute ( Attribute :: SwiftError ) || Arg . hasAttribute ( Attribute :: Nest ) ) return false ; if ( CC == CallingConv :: <ISA_LIT>_PS && ! InReg && P<ISA_LIT>nputNum <= <NUM_LIT> ) { const bool ArgUsed = ! Arg . use_empty ( ) ; bool SkipArg = ! ArgUsed && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! SkipArg ) { Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ArgUsed ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; } ++ P<ISA_LIT>nputNum ; if ( SkipArg ) { for ( int I = <NUM_LIT> , E = VRegs [ Idx ] . size ( ) ; I != E ; ++ I ) B . buildUndef ( VRegs [ Idx ] [ I ] ) ; ++ Idx ; continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg . getType ( ) ) ; const unsigned OrigArgIdx = Idx + AttributeList :: FirstArgIndex ; setArgFlags ( OrigArg , OrigArgIdx , DL , F ) ; splitToValueTypes ( B , OrigArg , OrigArgIdx , SplitArgs , DL , CC , [ & ] ( ArrayRef < Register > Regs , Register DstReg , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { assert ( DstReg == VRegs [ Idx ] [ VTSplitIdx ] ) ; packSplitRegsToOrigType ( B , VRegs [ Idx ] [ VTSplitIdx ] , Regs , LLTy , PartLLT ) ; } ) ; ++ Idx ; } if ( CC == CallingConv :: <ISA_LIT>_PS ) { if ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> || ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> && Info -> isP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ) ) { CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT>


<s> const Align KernArgBaseAlign ( <NUM_LIT> ) ; const unsigned BaseOffset = Subtarget -> getExplicitKernelArgOffset ( F ) ; uint64_t ExplicitArgOffset = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { Type * ArgTy = Arg . getType ( ) ; unsigned AllocSize = DL . getTypeAllocSize ( ArgTy ) ; if ( AllocSize == <NUM_LIT> ) continue ; unsigned ABIAlign = DL . getABITypeAlignment ( ArgTy ) ; uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; ArrayRef < Register > OrigArgRegs = VRegs [ i ] ; Register ArgReg = OrigArgRegs . size ( ) == <NUM_LIT> ? OrigArgRegs [ <NUM_LIT> ] : MRI . createGenericVirtualRegister ( getLLTForType ( * ArgTy , DL ) ) ; Align Alignment = commonAlignment ( KernArgBaseAlign , ArgOffset ) ; ArgOffset = alignTo ( ArgOffset , DL . getABITypeAlignment ( ArgTy ) ) ; lowerParameter ( B , ArgTy , ArgOffset , Alignment , ArgReg ) ; if ( OrigArgRegs . size ( ) > <NUM_LIT> ) unpackRegs ( OrigArgRegs , ArgReg , ArgTy , B ) ; ++


<s> const Align KernArgBaseAlign ( <NUM_LIT> ) ; const unsigned BaseOffset = Subtarget -> getExplicitKernelArgOffset ( F ) ; uint64_t ExplicitArgOffset = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { Type * ArgTy = Arg . getType ( ) ; unsigned AllocSize = DL . getTypeAllocSize ( ArgTy ) ; if ( AllocSize == <NUM_LIT> ) continue ; unsigned ABIAlign = DL . getABITypeAlignment ( ArgTy ) ; uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; ArrayRef < Register > OrigArgRegs = VRegs [ i ] ; Register ArgReg = OrigArgRegs . size ( ) == <NUM_LIT> ? OrigArgRegs [ <NUM_LIT> ] : MRI . createGenericVirtualRegister ( getLLTForType ( * ArgTy , DL ) ) ; Align Alignment = commonAlignment ( KernArgBaseAlign , ArgOffset ) ; ArgOffset = alignTo ( ArgOffset , DL . getABITypeAlignment ( ArgTy ) ) ; lowerParameter ( B , ArgTy , ArgOffset , Alignment , ArgReg ) ; if ( OrigArgRegs . size ( ) > <NUM_LIT> ) unpackRegs ( OrigArgRegs , ArgReg , ArgTy , B ) ; ++ i ; } TLI . allocateSpecialEntryInputVGPRs ( CCInfo , MF , * TRI , * Info ) ; TLI . allocateSystemSGPRs ( CCInfo , MF , * Info , F . getCallingConv ( ) , false ) ; return


<s> const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; unsigned TypeSize = DL . getTypeStoreSize


<s> Register <ISA_LIT>CallLowering :: lowerParameterPtr ( MachineIRBuilder & B , Type * ParamTy , uint64_t Offset ) const { MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; Register KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; auto OffsetReg = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , Offset ) ; return B . buildPtrAdd ( PtrType , KernArgSegmentVReg , OffsetReg ) . getReg (


<s> auto & MF = B . getMF ( ) ; const auto & F = MF . getFunction ( ) ; const DataLayout & DL = MF . getDataLayout ( ) ; MachineRegisterInfo * MRI = B . getMRI ( ) ; CallingConv :: ID CC = F . getCallingConv ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; ArgInfo OrigRetInfo ( VRegs , Val -> getType ( ) ) ; setArgFlags ( OrigRetInfo , AttributeList :: ReturnIndex , DL , F ) ; SmallVector < ArgInfo , <NUM_LIT> > SplitRetInfos ; splitToValueTypes ( B , OrigRetInfo , AttributeList :: ReturnIndex , SplitRetInfos , DL , CC , [ & ] ( ArrayRef < Register > Regs , Register SrcReg , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { unpackRegsToOrigType ( B , Regs , SrcReg , SplitRetInfos [ VTSplitIdx ] , LLTy , PartLLT ) ; } ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForReturn ( CC , F . isVarArg ( ) ) ; OutgoingValueHandler RetHandler ( B , * MRI , Ret , AssignFn ) ; return handleAssignments ( B ,


<s> int NumDst = LCMTy . getSizeInBits ( ) / LLTy . getSizeInBits ( ) ; SmallVector < Register , <NUM_LIT> > PadDstRegs ( NumDst ) ; std :: copy ( DstRegs . begin ( ) , DstRegs . end ( ) , PadDstRegs . begin ( ) ) ; for ( int I = DstRegs . size ( ) ; I != NumDst ; ++ I ) PadDstRegs [ I ] = MRI . createGenericVirtualRegister ( LLTy ) ; return B . buildUnmerge ( PadDstRegs ,


<s> B . buildTrunc ( OrigRegs [ <NUM_LIT> ] , Widened ) ; } return ; } if ( LLTy . isVector ( ) && PartLLT . isVector ( ) ) { assert ( OrigRegs . size ( ) == <NUM_LIT> ) ; assert ( LLTy . getElementType ( ) == PartLLT . getElementType ( ) ) ; mergeVectorRegsToResultRegs ( B , OrigRegs , Regs ) ; return ; } assert ( LLTy . isVector ( ) && ! PartLLT . isVector ( ) ) ; LLT DstEltTy = LLTy . getElementType ( ) ; LLT RealDstEltTy = MRI . getType ( OrigRegs [ <NUM_LIT> ] ) . getElementType ( ) ; assert ( DstEltTy . getSizeInBits ( ) == RealDstEltTy . getSizeInBits ( ) ) ; if ( DstEltTy == PartLLT ) { if ( RealDstEltTy . isPointer ( ) ) { for ( Register Reg : Regs ) MRI . setType ( Reg , RealDstEltTy ) ; } B . buildBuildVector ( OrigRegs [ <NUM_LIT> ] , Regs ) ; } else if ( DstEltTy . getSizeInBits ( ) > PartLLT . getSizeInBits ( ) ) { SmallVector < Register , <NUM_LIT> > EltMerges ; int PartsPerElt = DstEltTy . getSizeInBits ( ) / PartLLT . getSizeInBits ( ) ; assert ( DstEltTy . getSizeInBits ( ) % PartLLT . getSizeInBits ( ) == <NUM_LIT> ) ; for ( int I = <NUM_LIT> , NumElts = LLTy . getNumElements ( ) ; I != NumElts ; ++ I ) { auto Merge = B . buildMerge ( RealDstEltTy , Regs . take_front ( PartsPerElt ) ) ; MRI . setType ( Merge . getReg ( <NUM_LIT> ) , RealDstEltTy ) ; EltMerges . push_back ( Merge . getReg ( <NUM_LIT> )


<s> static void packSplitRegsToOrigType ( MachineIRBuilder & B , ArrayRef < Register > OrigRegs , ArrayRef < Register > Regs , LLT LLTy , LLT PartLLT ) { MachineRegisterInfo & MRI = * B . getMRI ( ) ; if ( ! LLTy . isVector ( ) && ! PartLLT . isVector ( ) ) { assert ( OrigRegs . size ( ) == <NUM_LIT> ) ; LLT OrigTy = MRI . getType ( OrigRegs [ <NUM_LIT> ] ) ; unsigned SrcSize = PartLLT . getSizeInBits ( ) * Regs . size ( ) ; if ( SrcSize == OrigTy . getSizeInBits ( ) ) B . buildMerge ( OrigRegs [ <NUM_LIT> ] , Regs ) ; else { auto Widened = B . buildMerge ( LLT :: scalar ( SrcSize ) , Regs ) ; B . buildTrunc ( OrigRegs [ <NUM_LIT> ] , Widened ) ; } return ; } if ( LLTy . isVector ( ) && PartLLT . isVector ( ) ) { assert ( OrigRegs . size ( ) == <NUM_LIT> ) ; assert ( LLTy . getElementType ( ) == PartLLT . getElementType ( ) ) ; mergeVectorRegsToResultRegs ( B , OrigRegs , Regs ) ; return ; } assert ( LLTy . isVector ( ) && ! PartLLT . isVector ( ) ) ; LLT DstEltTy = LLTy . getElementType ( ) ; LLT RealDstEltTy = MRI . getType ( OrigRegs [ <NUM_LIT> ] ) . getElementType ( ) ; assert ( DstEltTy . getSizeInBits ( ) == RealDstEltTy . getSizeInBits ( ) ) ; if ( DstEltTy == PartLLT ) { if ( RealDstEltTy . isPointer ( ) ) { for ( Register Reg : Regs ) MRI . setType ( Reg , RealDstEltTy ) ; } B . buildBuildVector ( OrigRegs [ <NUM_LIT> ] , Regs ) ; } else if ( DstEltTy . getSizeInBits ( ) > PartLLT . getSizeInBits ( ) ) { SmallVector < Register , <NUM_LIT> > EltMerges ; int PartsPerElt = DstEltTy . getSizeInBits ( ) / PartLLT . getSizeInBits ( ) ; assert ( DstEltTy . getSizeInBits ( ) % PartLLT . getSizeInBits ( ) == <NUM_LIT> ) ; for ( int I = <NUM_LIT> , NumElts = LLTy . getNumElements ( ) ; I != NumElts ; ++ I ) { auto Merge = B . buildMerge ( RealDstEltTy , Regs . take_front


<s> const unsigned SrcSize = SrcTy . getSizeInBits ( ) ; const unsigned PartSize = PartTy . getSizeInBits ( ) ; if ( SrcTy . isVector ( ) && ! PartTy . isVector ( ) && PartSize > SrcTy . getElementType ( ) . getSizeInBits ( ) ) { auto UnmergeToEltTy = B . buildUnmerge ( SrcTy . getElementType ( ) , SrcReg ) ; for ( int i = <NUM_LIT> , e = DstRegs . size ( ) ; i != e ; ++ i ) B . buildAnyExt ( DstRegs [ i ] , UnmergeToEltTy . getReg ( i ) ) ; return ; } if ( SrcSize % PartSize == <NUM_LIT> ) { B . buildUnmerge ( DstRegs , SrcReg


<s> bool <ISA_LIT>CallLowering :: lowerFormalArguments ( MachineIRBuilder & MIRBuilder , const Function & F , ArrayRef < unsigned


<s> bool <ISA_LIT>CallLowering :: lowerReturn ( MachineIRBuilder & MIRBuilder , const Value * Val


<s> CallingConv :: ID CC = B . getMF ( ) . getFunction ( ) . getCallingConv ( ) ; const bool IsShader = <STR_LIT> ( CC ) ; const bool IsWaveEnd = ( IsShader && MFI -> returnsVoid ( ) ) || <STR_LIT> ( CC ) ; if ( IsWaveEnd ) { B . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) ; return true ; } unsigned ReturnOpc = IsShader ? <STR_LIT> : <STR_LIT> ; auto Ret = B . buildInstrNoInsert ( ReturnOpc ) ; if ( ! FLI . CanLowerReturn ) insertSRetStores ( B , Val -> getType ( ) , VRegs , FLI . DemoteRegister ) ; else if ( ! lowerReturnVal ( B , Val , VRegs , Ret ) ) return false ; B . insertInstr ( Ret ) ; return


<s> assert ( ! Val == VRegs . empty ( ) && <STR_LIT> ) ; CallingConv :: ID CC = B . getMF ( ) . getFunction ( ) . getCallingConv ( ) ; const bool IsShader = <STR_LIT> ( CC ) ; const bool IsWaveEnd = ( IsShader && MFI -> returnsVoid ( ) ) || <STR_LIT> ( CC ) ; if ( IsWaveEnd ) { B . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) ; return true ; } unsigned ReturnOpc = IsShader ? <STR_LIT> : <STR_LIT> ; auto Ret = B .


<s> case CCValAssign :: LocInfo :: SExt : case CCValAssign :: LocInfo :: ZExt : case CCValAssign :: LocInfo :: AExt : { auto Copy = MIRBuilder . buildCopy ( LLT { VA . getLocVT ( ) } , PhysReg ) ; MIRBuilder . buildTrunc ( ValVReg , Copy ) ; break ; } default : MIRBuilder . buildCopy (


<s> MPO = MachinePointerInfo :: getFixedStack ( MIRBuilder . getMF ( ) , FI ) ; Register AddrReg = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: PRIVATE_ADDRESS , <NUM_LIT> ) ) ; MIRBuilder . buildFrameIndex ( AddrReg , FI ) ; StackUsed = std :: max ( StackUsed , Size


<s> bool isArgumentHandler ( ) const override { return true


<s> const GCNSubtarget * Subtarget = & MF . getSubtarget < GCNSubtarget > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget -> getRegisterInfo ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( F . getCallingConv ( ) , F . isVarArg ( ) , MF , ArgLocs , F . getContext ( ) ) ; allocateHSAUserSGPRs ( CCInfo , MIRBuilder , MF , * TRI , * Info ) ; unsigned i = <NUM_LIT> ; const unsigned KernArgBaseAlign = <NUM_LIT> ; const unsigned BaseOffset = Subtarget -> getExplicitKernelArgOffset ( F ) ; uint64_t ExplicitArgOffset = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { Type * ArgTy = Arg . getType ( ) ; unsigned AllocSize = DL . getTypeAllocSize ( ArgTy ) ; if ( AllocSize == <NUM_LIT> ) continue ; unsigned ABIAlign = DL . getABITypeAlignment ( ArgTy ) ; uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; ArrayRef < Register > OrigArgRegs = VRegs [ i ] ; Register ArgReg = OrigArgRegs . size ( ) == <NUM_LIT> ? OrigArgRegs [ <NUM_LIT> ] : MRI . createGenericVirtualRegister ( getLLTForType ( * ArgTy , DL ) ) ; unsigned Align = MinAlign ( KernArgBaseAlign , ArgOffset ) ; ArgOffset = alignTo ( ArgOffset , DL . getABITypeAlignment ( ArgTy ) ) ; lowerParameter ( MIRBuilder , ArgTy , ArgOffset , Align , ArgReg ) ; if ( OrigArgRegs . size ( ) > <NUM_LIT> ) unpackRegs ( OrigArgRegs , ArgReg , ArgTy , MIRBuilder ) ; ++ i ; } TLI . allocateSpecialEntryInputVGPRs ( CCInfo , MF , * TRI , * Info ) ; TLI . allocateSystemSGPRs ( CCInfo , MF , * Info , F . getCallingConv ( )


<s> const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; Register PtrReg = lowerParameterPtr ( MIRBuilder , ParamTy


<s> const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; Register DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; Register KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register KernArgSegmentVReg = MRI .

getLiveInVirtReg ( KernArgSegmentPtr ) ;


<s> auto const & ST = MIRBuilder . getMF ( ) . getSubtarget < GCNSubtarget > ( ) ; unsigned ReturnOpc = IsShader ? <STR_LIT> : <STR_LIT> ; auto Ret = MIRBuilder . buildInstrNoInsert ( ReturnOpc ) ; Register ReturnAddrVReg ; if ( ReturnOpc == <STR_LIT> ) { ReturnAddrVReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Ret . addUse ( ReturnAddrVReg ) ; } if ( ! lowerReturnVal ( MIRBuilder , Val , VRegs , Ret ) ) return false ; if ( ReturnOpc == <STR_LIT> ) { const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; Register LiveInReturn = MF . addLiveIn ( TRI ->

getReturnAddressReg ( MF ) , & <STR_LIT> ) ;


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; ArgInfo OrigRetInfo ( VRegs , Val -> getType ( ) ) ; setArgFlags ( OrigRetInfo , AttributeList :: ReturnIndex , DL , F ) ; SmallVector < ArgInfo , <NUM_LIT> > SplitRetInfos ; splitToValueTypes ( OrigRetInfo , SplitRetInfos , DL , MRI , CC , [ & ] ( ArrayRef < Register > Regs , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { unpackRegsToOrigType ( MIRBuilder , Regs , VRegs [ VTSplitIdx ] ,


<s> splitToValueTypes ( OrigRetInfo , SplitRetInfos , DL , MRI , CC , [ & ] ( ArrayRef < Register > Regs , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { unpackRegsToOrigType ( MIRBuilder , Regs , VRegs [ VTSplitIdx ] , LLTy , PartLLT ) ; } ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForReturn ( CC , F . isVarArg ( ) ) ; OutgoingValueHandler RetHandler ( MIRBuilder , MF . getRegInfo ( ) , Ret , AssignFn ) ; return handleAssignments ( MIRBuilder ,


<s> const int NumRoundedParts = ( SrcSize + PartSize - <NUM_LIT> ) / PartSize ; LLT BigTy = getMultipleType ( PartTy , NumRoundedParts ) ; auto ImpDef = MIRBuilder . buildUndef ( BigTy ) ; Register BigReg = MRI . createGenericVirtualRegister ( BigTy ) ; MIRBuilder . buildInsert ( BigReg , ImpDef . getReg ( <NUM_LIT> ) , SrcReg , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; int64_t Offset = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> , e = DstRegs . size ( ) ; i != e ; ++

i , Offset += PartSize ) MIRBuilder . buildExtract ( DstRegs [ i ] , BigReg , Offset ) ;


<s> void assignValueToAddress ( const CallLowering :: ArgInfo & Arg , Register Addr , uint64_t MemSize , MachinePointerInfo & MPO , CCValAssign & VA ) override { Register ValVReg = VA . getLocInfo ( ) != CCValAssign :: LocInfo :: FPExt ? extendRegister ( Arg . Regs [ <NUM_LIT> ] , VA ) : Arg


<s> Register ExtReg = extendRegisterMin32 ( ValVReg , VA ) ; MIRBuilder . buildCopy ( PhysReg ,


<s> if ( VA . getLocVT ( ) . getSizeInBits ( ) < <NUM_LIT> ) { return MIRBuilder . buildAnyExt ( LLT :: scalar ( <NUM_LIT> ) , ValVReg ) . getReg (


<s> static unsigned getCallOpcode ( const MachineFunction & CallerF , bool IsIndirect , bool IsTailCall ) { return <STR_LIT>


<s> } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! SPReg ) SPReg = MIRBuilder . buildCopy ( PtrTy , MFI -> getStackPtrOffsetReg ( ) ) . getReg ( <NUM_LIT> ) ; auto OffsetReg = MIRBuilder . buildConstant ( S32 , Offset ) ; auto AddrReg = MIRBuilder . buildPtrAdd ( PtrTy , SPReg , OffsetReg ) ; MPO = MachinePointerInfo :: getStack (


<s> Register getStackAddress ( uint64_t Size , int64_t Offset , MachinePointerInfo & MPO ) override { MachineFunction & MF = MIRBuilder . getMF ( ) ; const LLT PtrTy = LLT :: pointer ( <ISA_LIT>AS :: PRIVATE_ADDRESS , <NUM_LIT> ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; if ( IsTailCall ) { llvm_unreachable ( <STR_LIT> ) ; } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! SPReg ) SPReg = MIRBuilder . buildCopy ( PtrTy , MFI -> getStackPtrOffsetReg ( ) ) . getReg ( <NUM_LIT> ) ; auto OffsetReg = MIRBuilder . buildConstant ( S32 , Offset ) ; auto AddrReg = MIRBuilder . buildPtrAdd ( PtrTy , SPReg , OffsetReg ) ; MPO = MachinePointerInfo :: getStack ( MF , Offset ) ; return AddrReg . getReg (


<s> MIRBuilder . buildCopy ( <STR_LIT> , ScratchRSrcReg ) ; MIB . addReg ( <STR_LIT> , RegState :: Implicit ) ; } for ( std :: pair < MCRegister , Register > ArgReg : ImplicitArgRegs ) { MIRBuilder . buildCopy ( ( Register ) ArgReg . first , ArgReg . second ) ; MIB . addReg ( ArgReg . first , RegState :: Implicit ) ; } unsigned NumBytes = CCInfo . getNextStackOffset ( ) ; if ( MIB -> getOperand ( <NUM_LIT> ) . isReg ( ) ) { MIB -> getOperand ( <NUM_LIT> ) . setReg ( constrainOperandRegClass ( MF , * TRI , MRI , * ST . getInstrInfo ( ) , * ST . getRegBankInfo ( ) , * MIB , MIB -> getDesc ( ) , MIB -> getOperand ( <NUM_LIT> ) , <NUM_LIT> ) ) ; } auto OrigInsertPt = MIRBuilder . getInsertPt ( ) ; MIRBuilder . insertInstr ( MIB ) ; MachineInstrBuilder CallSeqEnd = MIRBuilder . buildInstr ( <STR_LIT> ) ; SmallVector < ArgInfo , <NUM_LIT> > InArgs ; if ( ! Info . CanLowerReturn ) { insertSRetLoads ( MIRBuilder , Info . OrigRet . Ty , Info . OrigRet . Regs , Info . DemoteRegister , Info . DemoteStackIndex ) ; } else if ( ! Info . OrigRet . Ty -> isVoidTy ( ) ) { SmallVector < ArgInfo , <NUM_LIT> > PreSplitRetInfos ; splitToValueTypes ( MIRBuilder , Info . OrigRet , PreSplitRetInfos , DL , Info . CallConv ) ; processSplitArgs ( MIRBuilder , Info . OrigRet , PreSplitRetInfos , InArgs , DL , Info . CallConv , false , [ & ] ( ArrayRef < Register > Regs , Register DstReg , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { assert ( DstReg == Info . OrigRet . Regs [ VTSplitIdx ] ) ; packSplitRegsToOrigType ( MIRBuilder , Info . OrigRet . Regs [ VTSplitIdx ] , Regs , LLTy , PartLLT ) ; } ) ; } MIRBuilder . setInsertPt ( MIRBuilder . getMBB ( ) , CallSeqEnd ) ; if ( Info . CanLowerReturn && ! Info . OrigRet . Ty -> isVoidTy ( ) ) { CCAssignFn * RetAssignFn = TLI . CCAssignFnForReturn ( Info . CallConv , Info . IsVarArg ) ; CallReturnHandler Handler ( MIRBuilder , MRI , MIB , RetAssignFn ) ; if ( ! handleAssignments ( MIRBuilder , InArgs ,


<s> uint64_t ExplicitArgOffset = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { const bool IsByRef = Arg . hasByRefAttr ( ) ; Type * ArgTy = IsByRef ? Arg . getParamByRefType ( ) : Arg . getType ( ) ; unsigned AllocSize = DL . getTypeAllocSize ( ArgTy ) ; if ( AllocSize == <NUM_LIT> ) continue ; MaybeAlign ABIAlign = IsByRef ? Arg . getParamAlign ( ) : None ; if ( ! ABIAlign ) ABIAlign = DL . getABITypeAlign ( ArgTy ) ; uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; if ( Arg . use_empty ( ) ) { ++ i ; continue ; } Align Alignment = commonAlignment ( KernArgBaseAlign , ArgOffset ) ; if ( IsByRef ) { unsigned ByRefAS = cast < PointerType > ( Arg . getType ( ) ) -> getAddressSpace ( ) ; assert ( VRegs [ i ] . size ( ) == <NUM_LIT> && <STR_LIT> ) ; if ( ByRefAS == <ISA_LIT>AS :: CONSTANT_ADDRESS ) { lowerParameterPtr ( VRegs [ i ] [ <NUM_LIT> ] , B , ArgTy , ArgOffset ) ; } else { const LLT ConstPtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register PtrReg = MRI . createGenericVirtualRegister ( ConstPtrTy ) ; lowerParameterPtr ( PtrReg , B , ArgTy , ArgOffset ) ; B . buildAddrSpaceCast ( VRegs [ i ] [ <NUM_LIT> ] , PtrReg ) ; } } else { ArrayRef < Register > OrigArgRegs = VRegs [ i ] ; Register ArgReg = OrigArgRegs . size ( ) == <NUM_LIT> ? OrigArgRegs [ <NUM_LIT> ] : MRI . createGenericVirtualRegister ( getLLTForType


<s> Register PtrReg = B . getMRI ( ) -> createGenericVirtualRegister ( PtrTy ) ; lowerParameterPtr ( PtrReg , B , ParamTy , Offset ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad |

MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , TypeSize , Alignment ) ;


<s> unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; LLT PtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register PtrReg = B . getMRI ( ) -> createGenericVirtualRegister ( PtrTy ) ; lowerParameterPtr ( PtrReg , B , ParamTy , Offset ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant


<s> MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; Register KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; auto OffsetReg = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , Offset ) ; B . buildPtrAdd ( DstReg , KernArgSegmentVReg ,


<s> Register KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; auto OffsetReg = B . buildConstant ( LLT :: scalar (


<s> SmallVector < ArgInfo , <NUM_LIT> > PreSplitRetInfos ; SmallVector < ArgInfo , <NUM_LIT> > SplitRetInfos ; for ( unsigned i = <NUM_LIT> ; i < SplitEVTs . size ( ) ; ++ i ) { EVT VT = SplitEVTs [ i ] ; Register Reg = VRegs [ i ] ; ArgInfo RetInfo ( Reg , VT . getTypeForEVT ( Ctx ) ) ; setArgFlags ( RetInfo , AttributeList :: ReturnIndex , DL , F ) ; if ( VT . isScalarInteger ( ) ) { unsigned ExtendOp = TargetOpcode :: G_ANYEXT ; if ( RetInfo . Flags [ <NUM_LIT> ] . isSExt ( ) ) { assert ( RetInfo . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_SEXT ; } else if ( RetInfo . Flags [ <NUM_LIT> ] . isZExt ( ) ) { assert ( RetInfo . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_ZEXT ; } EVT ExtVT = TLI . getTypeForExtReturn ( Ctx , VT , extOpcodeToISDExtOpcode ( ExtendOp ) ) ; if ( ExtVT != VT ) { RetInfo . Ty = ExtVT . getTypeForEVT ( Ctx ) ; LLT ExtTy = getLLTForType ( * RetInfo . Ty , DL ) ; Reg = B . buildInstr ( ExtendOp , { ExtTy } , { Reg


<s> SmallVector < ArgInfo , <NUM_LIT> > SplitRetInfos ; for ( unsigned i = <NUM_LIT> ; i < SplitEVTs . size ( ) ; ++ i ) { EVT VT = SplitEVTs [ i ] ; Register Reg = VRegs [ i ] ; ArgInfo RetInfo ( Reg , VT . getTypeForEVT ( Ctx ) ) ; setArgFlags ( RetInfo , AttributeList :: ReturnIndex , DL , F ) ; if ( VT . isScalarInteger ( ) ) { unsigned ExtendOp = TargetOpcode :: G_ANYEXT ; if ( RetInfo . Flags [ <NUM_LIT> ] . isSExt ( ) ) { assert ( RetInfo . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_SEXT ; } else if ( RetInfo . Flags [ <NUM_LIT> ] . isZExt ( ) ) { assert ( RetInfo . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_ZEXT ; } EVT ExtVT = TLI . getTypeForExtReturn ( Ctx , VT , extOpcodeToISDExtOpcode ( ExtendOp ) ) ; if ( ExtVT != VT ) { RetInfo . Ty = ExtVT . getTypeForEVT ( Ctx ) ; LLT ExtTy = getLLTForType ( * RetInfo . Ty , DL ) ; Reg = B . buildInstr ( ExtendOp , { ExtTy } , { Reg } ) . getReg ( <NUM_LIT> ) ; } } if ( Reg != RetInfo . Regs [ <NUM_LIT> ] ) { RetInfo . Regs [ <NUM_LIT> ] = Reg ; setArgFlags ( RetInfo , AttributeList :: ReturnIndex , DL , F ) ; } splitToValueTypes ( B , RetInfo , PreSplitRetInfos , DL , CC ) ; processSplitArgs ( B , RetInfo , PreSplitRetInfos , SplitRetInfos , DL , CC , true , [ & ] ( ArrayRef < Register > Regs , Register SrcReg , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { unpackRegsToOrigType ( B , Regs , SrcReg , PreSplitRetInfos [ VTSplitIdx ] , LLTy , PartLLT ) ; } ) ; PreSplitRetInfos . clear ( ) ; } CCAssignFn * AssignFn = TLI . CCAssignFnForReturn ( CC , F . isVarArg ( ) ) ; <ISA_LIT>OutgoingValueHandler RetHandler ( B , * MRI , Ret ,


<s> LLVMContext & Ctx = OrigArg . Ty -> getContext ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; for ( int SplitIdx = <NUM_LIT> , e = SplitArg . size ( ) ; SplitIdx != e ; ++ SplitIdx ) { const ArgInfo & CurSplitArg = SplitArg [ SplitIdx ] ; Register Reg = OrigArg . Regs [ SplitIdx ] ; EVT VT = EVT :: getEVT ( CurSplitArg . Ty ) ; LLT LLTy = getLLTForType ( * CurSplitArg . Ty , DL ) ; unsigned NumParts = TLI . getNumRegistersForCallingConv ( Ctx , CallConv , VT ) ; MVT RegVT = TLI . getRegisterTypeForCallingConv ( Ctx , CallConv , VT ) ; if ( NumParts == <NUM_LIT> ) { SplitArgs . emplace_back ( Reg , CurSplitArg . Ty , OrigArg . Flags , OrigArg . IsFixed ) ; continue ; } SmallVector < Register , <NUM_LIT> > SplitRegs ; Type * PartTy = EVT ( RegVT ) . getTypeForEVT ( Ctx ) ; LLT PartLLT = getLLTForType ( * PartTy , DL ) ; MachineRegisterInfo & MRI = * B .


<s> LLT DstTy = MRI . getType ( DstRegs [ <NUM_LIT> ] ) ; LLT LCMTy = getLCMType ( SrcTy , PartTy ) ; const unsigned LCMSize = LCMTy . getSizeInBits ( ) ; const unsigned DstSize = DstTy . getSizeInBits ( ) ; const unsigned SrcSize = SrcTy . getSizeInBits ( ) ; Register UnmergeSrc = SrcReg ; if ( LCMSize != SrcSize ) { Register Undef = B . buildUndef ( SrcTy ) . getReg ( <NUM_LIT> ) ; SmallVector < Register , <NUM_LIT> > MergeParts ( <NUM_LIT> , SrcReg ) ; for ( unsigned Size = SrcSize ; Size != LCMSize ; Size += SrcSize ) MergeParts . push_back ( Undef ) ; UnmergeSrc = B . buildMerge ( LCMTy , MergeParts )


<s> MIB . addUse ( PhysReg , RegState :: Implicit ) ; Register ExtReg = extendRegisterMin32 ( * this , ValVReg , VA ) ; MIRBuilder . buildCopy ( PhysReg , ExtReg


<s> void <ISA_LIT>CallLowering :: handleImplicitCallArguments ( MachineIRBuilder & MIRBuilder , MachineInstrBuilder & CallInst , const GCNSubtarget & ST , const <ISA_LIT>MachineFunctionInfo & FuncInfo , ArrayRef < std :: pair < MCRegister , Register >> ImplicitArgRegs ) const { if ( ! ST . enableFlatScratch ( ) ) { auto ScratchRSrcReg = MIRBuilder . buildCopy ( LLT :: vector ( <NUM_LIT> , <NUM_LIT> ) , FuncInfo . getScratchRSrcReg ( ) ) ; MIRBuilder . buildCopy (


<s> MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , false ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; MIB . addDef ( TRI -> getReturnAddressReg ( MF ) ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , Info . CallConv ) ; MIB . addRegMask ( Mask ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ; SmallVector < std :: pair < MCRegister , Register > , <NUM_LIT> > ImplicitArgRegs ; if ( <ISA_LIT>TargetMachine :: EnableFixedFunctionABI && Info . CallConv != CallingConv :: <ISA_LIT>_Gfx ) { if ( ! passSpecialInputs ( MIRBuilder , CCInfo , ImplicitArgRegs , Info ) ) return false ; } SmallVector < Register , <NUM_LIT> > PhysRegs ; OutgoingValueAssigner Assigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( Assigner , OutArgs , CCInfo ) ) return false ; <ISA_LIT>OutgoingArgHandler Handler ( MIRBuilder , MRI , MIB , false ) ; if ( ! handleAssignments ( Handler , OutArgs , CCInfo , ArgLocs , MIRBuilder ) ) return false ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; handleImplicitCallArguments ( MIRBuilder , MIB , ST , * MFI , ImplicitArgRegs ) ; unsigned NumBytes = CCInfo . getNextStackOffset ( ) ; if ( MIB -> getOperand ( <NUM_LIT> ) . isReg ( ) ) { MIB -> getOperand ( <NUM_LIT> ) . setReg ( constrainOperandRegClass ( MF , * TRI , MRI , * ST . getInstrInfo ( ) , * ST . getRegBankInfo ( ) , * MIB , MIB -> getDesc ( ) , MIB -> getOperand ( <NUM_LIT> ) , <NUM_LIT> ) ) ; } MIRBuilder . insertInstr ( MIB ) ; if ( Info . CanLowerReturn && ! Info . OrigRet . Ty -> isVoidTy ( ) ) { CCAssignFn * RetAssignFn = TLI . CCAssignFnForReturn ( Info . CallConv , Info .


<s> CCState CCInfo ( F . getCallingConv ( ) , F . isVarArg ( ) , MF , ArgLocs , F . getContext ( ) ) ; allocateHSAUserSGPRs ( CCInfo , B , MF , * TRI , * Info ) ; unsigned i = <NUM_LIT> ; const Align KernArgBaseAlign ( <NUM_LIT> ) ; const unsigned BaseOffset = Subtarget -> getExplicitKernelArgOffset ( F ) ; uint64_t ExplicitArgOffset = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { const bool IsByRef = Arg . hasByRefAttr ( ) ; Type * ArgTy = IsByRef ? Arg . getParamByRefType ( ) : Arg . getType ( ) ; unsigned AllocSize = DL . getTypeAllocSize ( ArgTy ) ; if ( AllocSize == <NUM_LIT> ) continue ; MaybeAlign ABIAlign = IsByRef ? Arg . getParamAlign ( ) : None ; if ( ! ABIAlign ) ABIAlign = DL . getABITypeAlign ( ArgTy ) ; uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; if ( Arg . use_empty ( ) ) { ++ i ; continue ; } Align Alignment = commonAlignment ( KernArgBaseAlign , ArgOffset ) ; if ( IsByRef ) { unsigned ByRefAS = cast < PointerType > ( Arg . getType ( ) ) -> getAddressSpace ( ) ; assert ( VRegs [ i ] . size ( ) == <NUM_LIT> && <STR_LIT> ) ; if ( ByRefAS == <ISA_LIT>AS :: CONSTANT_ADDRESS ) { lowerParameterPtr ( VRegs [ i ] [ <NUM_LIT> ] , B , ArgTy , ArgOffset ) ; } else { const LLT ConstPtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register PtrReg = MRI . createGenericVirtualRegister ( ConstPtrTy ) ; lowerParameterPtr ( PtrReg , B , ArgTy , ArgOffset ) ; B . buildAddrSpaceCast ( VRegs [ i ] [ <NUM_LIT> ] , PtrReg ) ; } } else { ArrayRef < Register > OrigArgRegs = VRegs [


<s> assert ( VRegs . size ( ) == SplitEVTs . size ( ) && <STR_LIT> ) ; SmallVector < ArgInfo , <NUM_LIT> > SplitRetInfos ; for ( unsigned i = <NUM_LIT> ; i < SplitEVTs . size ( ) ; ++ i ) { EVT VT = SplitEVTs [ i ] ; Register Reg = VRegs [ i ] ; ArgInfo RetInfo ( Reg , VT . getTypeForEVT ( Ctx ) ) ; setArgFlags ( RetInfo , AttributeList :: ReturnIndex , DL , F ) ; if ( VT . isScalarInteger ( ) ) { unsigned ExtendOp = TargetOpcode :: G_ANYEXT ; if ( RetInfo . Flags [ <NUM_LIT> ] . isSExt ( ) ) { assert ( RetInfo . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_SEXT ; } else if ( RetInfo . Flags [ <NUM_LIT> ] . isZExt ( ) ) { assert ( RetInfo . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_ZEXT ; } EVT ExtVT = TLI . getTypeForExtReturn ( Ctx , VT , extOpcodeToISDExtOpcode ( ExtendOp ) ) ; if ( ExtVT != VT ) { RetInfo . Ty = ExtVT . getTypeForEVT ( Ctx ) ; LLT ExtTy = getLLTForType ( * RetInfo . Ty , DL ) ; Reg = B . buildInstr ( ExtendOp , { ExtTy } , { Reg } ) . getReg ( <NUM_LIT> ) ; } } if ( Reg != RetInfo . Regs [ <NUM_LIT> ] ) { RetInfo . Regs [ <NUM_LIT> ] = Reg ; setArgFlags ( RetInfo , AttributeList :: ReturnIndex , DL , F ) ; } splitToValueTypes ( RetInfo , SplitRetInfos , DL , CC ) ; } CCAssignFn * AssignFn = TLI . CCAssignFnForReturn ( CC , F . isVarArg ( ) ) ; OutgoingValueAssigner Assigner ( AssignFn ) ; <ISA_LIT>OutgoingValueHandler RetHandler ( B , *


<s> continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg . getType ( ) ) ; setArgFlags ( OrigArg , Idx + AttributeList :: FirstArgIndex , DL , F ) ; splitToValueTypes ( OrigArg , SplitArgs , DL , MRI , CC , [ & ] ( ArrayRef < Register > Regs , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { packSplitRegsToOrigType ( B , VRegs [ Idx ] [ VTSplitIdx ] , Regs , LLTy , PartLLT ) ; } ) ; ++ Idx ; } if ( CC == CallingConv :: <ISA_LIT>_PS ) { if ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> || ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> && Info -> isP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ) ) { CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ; Info -> markP<ISA_LIT>nputEnabled ( <NUM_LIT> ) ; } if ( Subtarget . isAmdPalOS ( ) ) { unsigned PsInputBits = Info -> getP<ISA_LIT>nputAddr ( ) & Info -> getP<ISA_LIT>nputEnable ( ) ; if ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> || ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> && ( PsInputBits >> <NUM_LIT> & <NUM_LIT> ) ) ) Info -> markP<ISA_LIT>nputEnabled ( countTrailingZeros ( Info -> getP<ISA_LIT>nputAddr ( ) , ZB_Undefined ) ) ; } } const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForCall ( CC , F . isVarArg ( ) ) ; if ( ! MBB . empty ( ) ) B . setInstr ( * MBB . begin ( ) ) ; FormalArgHandler Handler ( B , MRI , AssignFn ) ; if ( ! handleAssignments ( CCInfo , ArgLocs , B , SplitArgs , Handler ) ) return false ; if ( ! IsEntryFunc ) { TLI . allocateSpecialInputVGPRs ( CCInfo , MF , * TRI , * Info ) ; } if ( IsEntryFunc ) { TLI . allocateSystemSGPRs ( CCInfo , MF , * Info , CC , IsShader ) ; } else { CCInfo . AllocateReg ( Info -> getScratchRSrcReg ( ) ) ; CCInfo . AllocateReg ( Info -> getScratchWaveOffsetReg ( ) ) ; CCInfo . AllocateReg ( Info -> getFrameOffsetReg ( ) ) ; TLI . allocateSpecialInputSGPRs ( CCInfo , MF , * TRI , * Info ) ; } B . setMBB (


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget -> getRegisterInfo ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( F . getCallingConv ( ) , F . isVarArg ( ) , MF , ArgLocs , F . getContext ( ) ) ; allocateHSAUserSGPRs ( CCInfo , B , MF , * TRI , * Info ) ; unsigned i = <NUM_LIT> ; const unsigned KernArgBaseAlign = <NUM_LIT> ; const unsigned BaseOffset = Subtarget -> getExplicitKernelArgOffset ( F ) ; uint64_t ExplicitArgOffset = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { Type * ArgTy = Arg . getType ( ) ; unsigned AllocSize = DL . getTypeAllocSize ( ArgTy ) ; if ( AllocSize == <NUM_LIT> ) continue ; unsigned ABIAlign = DL . getABITypeAlignment ( ArgTy ) ; uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; ArrayRef < Register > OrigArgRegs = VRegs [ i ] ; Register ArgReg = OrigArgRegs . size ( ) == <NUM_LIT> ? OrigArgRegs [ <NUM_LIT> ] : MRI . createGenericVirtualRegister ( getLLTForType ( * ArgTy , DL ) ) ; unsigned Align = MinAlign (


<s> void <ISA_LIT>CallLowering :: lowerParameter ( MachineIRBuilder & B , Type * ParamTy , uint64_t Offset , unsigned Align , Register DstReg ) const { MachineFunction & MF = B . getMF ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; Register PtrReg = lowerParameterPtr ( B , ParamTy , Offset ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , TypeSize , Align ) ; B . buildLoad ( DstReg , PtrReg


<s> const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; Register PtrReg = lowerParameterPtr ( B , ParamTy , Offset ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , TypeSize , Align ) ; B . buildLoad ( DstReg , PtrReg


<s> MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy


<s> PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; Register DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; Register KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; Register OffsetReg = MRI . createGenericVirtualRegister ( LLT


<s> assert ( ! Val == VRegs . empty ( ) && <STR_LIT> ) ; CallingConv :: ID CC = B . getMF ( ) . getFunction ( ) . getCallingConv ( ) ; const bool IsShader = <STR_LIT> ( CC ) ; const bool IsWaveEnd = ( IsShader && MFI -> returnsVoid ( ) ) || <STR_LIT> ( CC ) ; if ( IsWaveEnd ) { B . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) ; return true ; } auto const & ST = B . getMF ( ) . getSubtarget < GCNSubtarget > ( ) ; unsigned ReturnOpc = IsShader ? <STR_LIT> : <STR_LIT> ; auto Ret = B . buildInstrNoInsert ( ReturnOpc ) ; Register ReturnAddrVReg ; if ( ReturnOpc == <STR_LIT> ) { ReturnAddrVReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Ret . addUse ( ReturnAddrVReg ) ; } if ( ! lowerReturnVal ( B , Val , VRegs , Ret ) ) return false ; if ( ReturnOpc == <STR_LIT> ) { const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; Register LiveInReturn = MF . addLiveIn ( TRI -> getReturnAddressReg ( MF ) , & <STR_LIT> ) ; B . buildCopy (


<s> CallingConv :: ID CC = F . getCallingConv ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; ArgInfo OrigRetInfo ( VRegs , Val -> getType ( ) ) ; setArgFlags ( OrigRetInfo , AttributeList :: ReturnIndex , DL , F ) ; SmallVector < ArgInfo , <NUM_LIT> > SplitRetInfos ; splitToValueTypes ( OrigRetInfo , SplitRetInfos , DL , MRI , CC , [ & ] ( ArrayRef < Register > Regs , LLT


<s> return ; } if ( LLTy . isVector ( ) && PartLLT . isVector ( ) ) { assert ( LLTy . getElementType ( ) == PartLLT . getElementType ( ) ) ; int DstElts = LLTy . getNumElements ( ) ; int PartElts = PartLLT . getNumElements ( ) ; if ( DstElts % PartElts == <NUM_LIT> ) B . buildConcatVectors ( OrigRegs [ <NUM_LIT> ] , Regs ) ; else { assert ( PartElts == <NUM_LIT> && DstElts % <NUM_LIT> != <NUM_LIT> ) ; int RoundedElts = PartElts * ( ( DstElts + PartElts - <NUM_LIT> ) / PartElts ) ; LLT RoundedDestTy = LLT :: vector ( RoundedElts , PartLLT . getElementType ( ) ) ; auto RoundedConcat = B . buildConcatVectors ( RoundedDestTy , Regs ) ; B . buildExtract ( OrigRegs [ <NUM_LIT> ] , RoundedConcat , <NUM_LIT> ) ; } return ; } assert ( LLTy . isVector ( ) && ! PartLLT . isVector ( ) ) ; LLT DstEltTy = LLTy . getElementType ( ) ; if ( DstEltTy == PartLLT ) { B . buildBuildVector ( OrigRegs [ <NUM_LIT> ]


<s> } if ( SrcSize % PartSize == <NUM_LIT> ) { B . buildUnmerge ( DstRegs , SrcReg ) ; return ; } const int NumRoundedParts = ( SrcSize + PartSize - <NUM_LIT> ) / PartSize ; LLT BigTy = getMultipleType ( PartTy , NumRoundedParts ) ; auto ImpDef = B . buildUndef ( BigTy ) ; Register BigReg = MRI . createGenericVirtualRegister ( BigTy ) ; B . buildInsert ( BigReg , ImpDef . getReg ( <NUM_LIT> ) , SrcReg , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; int64_t Offset = <NUM_LIT>


<s> const unsigned PartSize = PartTy . getSizeInBits ( ) ; if ( SrcTy . isVector ( ) && ! PartTy . isVector ( ) && PartSize > SrcTy . getElementType ( ) . getSizeInBits ( ) ) { auto UnmergeToEltTy = B . buildUnmerge ( SrcTy . getElementType ( ) , SrcReg ) ; for ( int i = <NUM_LIT> , e = DstRegs . size ( ) ; i != e ; ++ i ) B . buildAnyExt ( DstRegs [ i ] , UnmergeToEltTy . getReg ( i ) ) ; return ; } if ( SrcSize % PartSize == <NUM_LIT> ) { B . buildUnmerge ( DstRegs , SrcReg ) ; return ; } const int NumRoundedParts = ( SrcSize + PartSize - <NUM_LIT> ) / PartSize ; LLT BigTy = getMultipleType ( PartTy , NumRoundedParts ) ; auto ImpDef = B . buildUndef ( BigTy ) ; Register BigReg = MRI . createGenericVirtualRegister ( BigTy ) ; B . buildInsert ( BigReg , ImpDef . getReg ( <NUM_LIT> ) , SrcReg , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; int64_t Offset = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> , e = DstRegs . size ( ) ; i != e ; ++ i , Offset += PartSize ) B . buildExtract ( DstRegs [ i ]


<s> void assignValueToReg ( Register ValVReg , Register PhysReg , CCValAssign & VA ) override { MIB . addUse ( PhysReg


<s> void assignValueToReg ( Register ValVReg , Register PhysReg , CCValAssign & VA ) override { MIB . addUse ( PhysReg ) ; MIRBuilder . buildCopy ( PhysReg


<s> PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; Register PtrReg = lowerParameterPtr ( MIRBuilder , ParamTy , Offset ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MONonTemporal | MachineMemOperand


<s> PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; Register PtrReg = lowerParameterPtr ( MIRBuilder , ParamTy ,


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MFI -> setIfReturnsVoid ( ! Val ) ; if ( ! Val ) { MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) ; return true ; } Register VReg = VRegs [ <NUM_LIT> ] ; const Function & F = MF . getFunction ( ) ; auto & DL = F . getParent ( ) -> getDataLayout ( ) ; if ( ! <STR_LIT> ( F . getCallingConv ( ) ) ) return false ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; SmallVector < EVT , <NUM_LIT> > SplitVTs ; SmallVector < uint64_t , <NUM_LIT> > Offsets ; ArgInfo OrigArg { VReg , Val ->


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = * MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; unsigned KernArgSegmentPtr = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR ) ; unsigned KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; unsigned OffsetReg = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildConstant ( OffsetReg , Offset ) ; MIRBuilder . buildGEP ( DstReg ,


<s> const <ISA_LIT>RegisterInfo * TRI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = * MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; unsigned KernArgSegmentPtr = TRI -> getPreloadedValue ( MF

, <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR ) ;


<s> Register ValVReg = VA . getLocInfo ( ) != CCValAssign :: LocInfo :: FPExt ? extendRegister ( Arg . Regs [ <NUM_LIT> ] , VA ) : Arg . Regs [ <NUM_LIT> ] ; const LLT RegTy = MRI . getType ( ValVReg ) ; if ( RegTy . getSizeInBytes ( ) > Size ) Size = RegTy . getSizeInBytes ( ) ; assignValueToAddress ( ValVReg , Addr , Size , MPO


<s> for ( int I = <NUM_LIT> , E = VRegs [ Idx ] . size ( ) ; I != E ; ++ I ) B . buildUndef ( VRegs [ Idx ] [ I ] ) ; ++ Idx ; continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg . getType ( ) ) ; const unsigned OrigArgIdx = Idx + AttributeList :: FirstArgIndex ; setArgFlags ( OrigArg , OrigArgIdx , DL , F ) ; splitToValueTypes ( B , OrigArg , SplitArgs , DL , CC , false , [ & ] ( ArrayRef < Register > Regs , Register DstReg , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { assert ( DstReg == VRegs [ Idx ] [ VTSplitIdx ] ) ; packSplitRegsToOrigType ( B , VRegs [ Idx ] [ VTSplitIdx ] , Regs , LLTy , PartLLT ) ; } ) ; ++ Idx ; } if ( CC == CallingConv :: <ISA_LIT>_PS ) { if ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> || ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> && Info -> isP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ) ) { CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ; Info -> markP<ISA_LIT>nputEnabled ( <NUM_LIT> ) ; } if ( Subtarget . isAmdPalOS ( ) ) { unsigned PsInputBits = Info -> getP<ISA_LIT>nputAddr ( ) & Info -> getP<ISA_LIT>nputEnable ( ) ; if ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> || ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> && ( PsInputBits >> <NUM_LIT> & <NUM_LIT> ) ) ) Info -> markP<ISA_LIT>nputEnabled ( countTrailingZeros ( Info -> getP<ISA_LIT>nputAddr ( ) , ZB_Undefined ) ) ; } } const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForCall ( CC , F . isVarArg ( ) ) ; if ( ! MBB . empty ( ) ) B . setInstr ( * MBB . begin ( ) ) ; if ( ! IsEntryFunc ) { if ( <ISA_LIT>TargetMachine :: EnableFixedFunctionABI ) TLI . allocateSpecialInputVGPRsFixed ( CCInfo , MF , * TRI , * Info ) ; } FormalArgHandler Handler ( B , MRI , AssignFn ) ; if ( ! handleAssignments ( CCInfo , ArgLocs , B , SplitArgs , Handler ) ) return false ; if ( ! IsEntryFunc && ! <ISA_LIT>TargetMachine :: EnableFixedFunctionABI ) { TLI . allocateSpecialInputVGPRs ( CCInfo , MF , * TRI , * Info ) ; } if ( IsEntryFunc ) { TLI . allocateSystemSGPRs ( CCInfo , MF , * Info


<s> setArgFlags ( OrigRetInfo , AttributeList :: ReturnIndex , DL , F ) ; SmallVector < ArgInfo , <NUM_LIT> > SplitRetInfos ; splitToValueTypes ( B , OrigRetInfo , SplitRetInfos , DL , CC , true , [ & ] ( ArrayRef < Register > Regs , Register SrcReg , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { unpackRegsToOrigType ( B , Regs , SrcReg , SplitRetInfos [ VTSplitIdx ] , LLTy , PartLLT ) ; } ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForReturn ( CC , F . isVarArg ( ) ) ; <ISA_LIT>OutgoingValueHandler RetHandler ( B , * MRI ,


<s> assert ( OrigArg . Regs . size ( ) == SplitVTs . size ( ) ) ; int SplitIdx = <NUM_LIT> ; for ( EVT VT : SplitVTs ) { Register Reg = OrigArg . Regs [ SplitIdx ] ; Type * Ty = VT . getTypeForEVT ( Ctx ) ; LLT LLTy = getLLTForType ( * Ty , DL ) ; if ( IsOutgoing && VT . isScalarInteger ( ) ) { unsigned ExtendOp = TargetOpcode :: G_ANYEXT ; if ( OrigArg . Flags [ <NUM_LIT> ] . isSExt ( ) ) { assert ( OrigArg . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_SEXT ; } else if ( OrigArg . Flags [ <NUM_LIT> ] . isZExt ( ) ) { assert ( OrigArg . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_ZEXT ; } EVT ExtVT = TLI . getTypeForExtReturn ( Ctx , VT , extOpcodeToISDExtOpcode ( ExtendOp ) ) ; if ( ExtVT . getSizeInBits ( ) != VT . getSizeInBits ( ) ) { VT = ExtVT ; Ty = ExtVT . getTypeForEVT ( Ctx ) ; LLTy = getLLTForType ( * Ty , DL ) ; Reg = B . buildInstr ( ExtendOp , { LLTy } , { Reg } ) . getReg ( <NUM_LIT> ) ; } } unsigned NumParts = TLI . getNumRegistersForCallingConv ( Ctx , CallConv , VT ) ; MVT RegVT = TLI . getRegisterTypeForCallingConv ( Ctx , CallConv , VT ) ; if ( NumParts == <NUM_LIT> ) { SplitArgs . emplace_back ( Reg , Ty , OrigArg . Flags , OrigArg . IsFixed ) ; ++ SplitIdx ; continue ; } SmallVector < Register , <NUM_LIT> > SplitRegs ; Type * PartTy = EVT ( RegVT ) . getTypeForEVT ( Ctx ) ; LLT PartLLT = getLLTForType ( * PartTy , DL ) ; MachineRegisterInfo & MRI = * B . getMRI ( ) ; for ( unsigned i = <NUM_LIT> ; i < NumParts ; ++ i ) { Register PartReg = MRI . createGenericVirtualRegister ( PartLLT ) ; SplitRegs . push_back ( PartReg ) ; SplitArgs . emplace_back ( ArrayRef < Register > ( PartReg ) , PartTy ,


<s> if ( CC == CallingConv :: <ISA_LIT>_PS && ! InReg && P<ISA_LIT>nputNum <= <NUM_LIT> ) { const bool ArgUsed = ! Arg . use_empty ( ) ; bool SkipArg = ! ArgUsed && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! SkipArg ) { Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ArgUsed ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; } ++ P<ISA_LIT>nputNum ; if ( SkipArg ) { for ( Register R : VRegs [ Idx ] ) B . buildUndef ( R ) ; ++ Idx ; continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg , Idx ) ; const unsigned OrigArgIdx = Idx + AttributeList :: FirstArgIndex ; setArgFlags ( OrigArg , OrigArgIdx , DL , F ) ; splitToValueTypes ( OrigArg , SplitArgs , DL , CC ) ; ++ Idx ; } if ( CC == CallingConv :: <ISA_LIT>_PS ) { if ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> || ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> && Info -> isP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ) ) { CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ; Info -> markP<ISA_LIT>nputEnabled ( <NUM_LIT> ) ; } if ( Subtarget . isAmdPalOS ( ) ) { unsigned PsInputBits = Info -> getP<ISA_LIT>nputAddr ( ) & Info -> getP<ISA_LIT>nputEnable ( ) ; if ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> || ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> && ( PsInputBits >> <NUM_LIT> & <NUM_LIT> ) ) ) Info -> markP<ISA_LIT>nputEnabled ( countTrailingZeros ( Info -> getP<ISA_LIT>nputAddr ( ) , ZB_Undefined ) ) ; } } const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForCall ( CC , F . isVarArg ( ) ) ; if ( ! MBB . empty ( ) ) B . setInstr (