<s> assert ( OpNo < Desc . NumOperands ) ; unsigned OpType = Desc . OpInfo [ OpNo ] . OperandType ; switch ( OpType ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> auto TargetTriple = STI . getTargetTriple ( ) ; auto Version = getIsaVersion ( STI . getCPU ( ) ) ; StreamRep << TargetTriple . getArchName ( ) << '-' << TargetTriple . getVendorName ( ) << '-' << TargetTriple . getOSName ( ) << '-' << TargetTriple . getEnvironmentName ( ) << '-' ; std :: string Processor = <STR_LIT> ; if ( Version . Major >= <NUM_LIT> ) Processor = STI . getCPU ( ) . str ( ) ; else Processor = ( Twine ( <STR_LIT> ) + Twine ( Version . Major ) + Twine ( Version . Minor ) + Twine ( Version . Stepping ) ) . str ( ) ; std :: string Features = <STR_LIT> ; if ( Optional < uint8_t > HsaAbiVersion = getHsaAbiVersion ( & STI ) ) { switch ( * HsaAbiVersion ) { case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V2 : if ( Processor == <STR_LIT> ) { } else if ( Processor == <STR_LIT> ) { } else if ( Processor == <STR_LIT> ) { } else if ( Processor == <STR_LIT> ) { } else if ( Processor == <STR_LIT> ) { } else if ( Processor == <STR_LIT> ) { } else if ( Processor == <STR_LIT> ) { } else if ( Processor == <STR_LIT> ) { } else if ( Processor == <STR_LIT> ) { } else if ( Processor == <STR_LIT> ) { if ( ! isXnackOnOrAny ( ) ) report_fatal_error ( <STR_LIT> + Processor + <STR_LIT> ) ; } else if ( Processor == <STR_LIT> ) { } else if ( Processor == <STR_LIT> ) { } else if ( Processor == <STR_LIT> ) { } else if ( Processor == <STR_LIT> ) { if ( ! isXnackOnOrAny ( ) ) report_fatal_error ( <STR_LIT> + Processor + <STR_LIT> ) ; } else if ( Processor == <STR_LIT> ) { if ( isXnackOnOrAny ( ) ) Processor = <STR_LIT> ; } else if ( Processor == <STR_LIT> ) { if ( isXnackOnOrAny ( ) ) Processor = <STR_LIT> ; } else if ( Processor == <STR_LIT> ) { if ( isXnackOnOrAny ( ) ) Processor = <STR_LIT> ; } else if ( Processor == <STR_LIT> ) { if ( isXnackOnOrAny ( ) ) Processor = <STR_LIT> ; } else if ( Processor == <STR_LIT> ) { if ( isXnackOnOrAny ( ) ) report_fatal_error ( <STR_LIT> + Processor + <STR_LIT> ) ; } else { report_fatal_error ( <STR_LIT> + Processor ) ; } break ; case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V3 : if ( isXnackOnOrAny ( ) ) Features += <STR_LIT> ; if ( isSramEccOnOrAny ( ) ) Features += <STR_LIT> ; break ; case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V4 : if ( getSramEccSetting ( ) == TargetIDSetting :: Off ) Features += <STR_LIT> ; else if ( getSramEccSetting ( ) == TargetIDSetting :: On ) Features += <STR_LIT> ;

if ( getXnackSetting ( ) == TargetIDSetting :: Off ) Features += <STR_LIT> ;


<s> int64_t encodeHwreg ( int64_t Id , int64_t Offset , int64_t Width ) {

return ( Id << ID_SHIFT_ ) | ( Offset << OFFSET_SHIFT_ ) | ( ( Width - <NUM_LIT> ) << WIDTH_M1_SHIFT_ ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : return <NUM_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ;


<s> return decodeCustomOperand ( DepCtrInfo , DEP_CTR_<ISA_LIT>ZE , Code , Id , Name , Val , IsDefault , STI ) ;


<s> static int encodeCustomOperandVal ( const CustomOperandVal & Op , int64_t InputVal ) {

if ( InputVal < <NUM_LIT> || InputVal > Op . Max ) return OPR_VAL_INVALID ;


<s> if ( InputVal < <NUM_LIT> || InputVal > Op . Max ) return OPR_VAL_INVALID ;


<s> return encodeCustomOperand ( DepCtrInfo , DEP_CTR_<ISA_LIT>ZE , Name , Val , UsedOprMask , STI ) ;


<s> if ( Op . isSupported ( STI ) ) Enc |= Op . encode ( Op . Default ) ; } return Enc ;


if ( Default == - <NUM_LIT> ) Default = getDefaultCustomOperandEncoding ( DepCtrInfo , DEP_CTR_<ISA_LIT>ZE , STI ) ;


<s> case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <STR_LIT> :: MULTIGRID_SYNC_ARG_OFFSET ;


<s> for ( int Idx = <NUM_LIT> ; Idx < Size ; ++ Idx ) { const auto & Op = Opr [ Idx ] ; if ( ! Op . isSupported ( STI ) ) continue ; UsedOprMask |= Op . getMask ( ) ; unsigned Val = Op . decode ( Code ) ; if ( ! Op . isValid ( Val ) ) return false ; HasNonDefaultVal |= ( Val != Op . Default ) ; }

return ( Code & ~ UsedOprMask ) == <NUM_LIT> ;


<s> UsedOprMask |= Op . getMask ( ) ; unsigned Val = Op . decode ( Code ) ; if ( ! Op . isValid ( Val ) ) return false ; HasNonDefaultVal |= ( Val != Op . Default ) ; } return ( Code & ~ UsedOprMask ) == <NUM_LIT> ;


<s> bool isSymbolicDepCtrEncoding ( unsigned Code , bool & HasNonDefaultVal , const MCSubtargetInfo & STI ) { return isSymbolicCustomOperandEncoding ( DepCtrInfo , DEP_CTR_<ISA_LIT>ZE , Code , HasNonDefaultVal , STI ) ;


<s> memset ( & Header , <NUM_LIT> , sizeof ( Header ) ) ; Header . amd_kernel_code_version_major = <NUM_LIT> ; Header . amd_kernel_code_version_minor = <NUM_LIT> ; Header . amd_machine_kind = <NUM_LIT> ; Header . amd_machine_version_major = ISA . Major ; Header . amd_machine_version_minor = ISA . Minor ; Header . amd_machine_version_stepping = ISA . Stepping ; Header . kernel_code_entry_byte_offset = sizeof ( Header ) ; Header . wavefront_size = <NUM_LIT> ; Header . call_convention = - <NUM_LIT> ; Header . kernarg_segment_alignment = <NUM_LIT> ; Header . group_segment_alignment = <NUM_LIT> ; Header . private_segment_alignment = <NUM_LIT> ;


<s> switch ( OpType ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> } } else { for ( int Id = UfmtGFX10 :: UFMT_FIRST ; Id <= UfmtGFX10 :: UFMT_LAST ; ++ Id ) { if ( Fmt == DfmtNfmt2UFmtGFX10 [ Id ] ) return Id ; } } return UFMT_UNDEF ;


<s> unsigned decodeExpcnt ( const IsaVersion & Version , unsigned Waitcnt ) {

return unpackBits ( Waitcnt , getExpcntBitShift ( Version . Major ) , getExpcntBitWidth ( Version . Major ) ) ;


<s> return unpackBits ( Waitcnt , getExpcntBitShift ( Version . Major ) , getExpcntBitWidth ( Version . Major ) ) ;


<s> unsigned decodeLgkmcnt ( const IsaVersion & Version , unsigned Waitcnt ) { return unpackBits ( Waitcnt , getLgkmcntBitShift ( Version . Major ) , getLgkmcntBitWidth ( Version . Major ) ) ;


<s> OpId = <NUM_LIT> ; StreamId = <NUM_LIT> ; } else { OpId = ( Val & OP_MASK_ ) >> OP_SHIFT_ ; StreamId = ( Val & STREAM_ID_MASK_ ) >> STREAM_ID_SHIFT_ ; }


<s> unsigned encodeExpcnt ( const IsaVersion & Version , unsigned Waitcnt , unsigned Expcnt ) { return packBits ( Expcnt , Waitcnt , getExpcntBitShift ( Version . Major ) , getExpcntBitWidth ( Version . Major ) ) ;


<s> return packBits ( Lgkmcnt , Waitcnt , getLgkmcntBitShift ( Version . Major ) , getLgkmcntBitWidth ( Version . Major ) ) ;


<s> unsigned encodeLgkmcnt ( const IsaVersion & Version , unsigned Waitcnt , unsigned Lgkmcnt ) { return packBits ( Lgkmcnt , Waitcnt , getLgkmcntBitShift ( Version . Major ) , getLgkmcntBitWidth ( Version . Major ) ) ;


<s> uint64_t encodeMsg ( uint64_t MsgId , uint64_t OpId , uint64_t StreamId ) {

return MsgId | ( OpId << OP_SHIFT_ ) | ( StreamId << STREAM_ID_SHIFT_ ) ;


<s> Waitcnt = packBits ( Vmcnt , Waitcnt , getVmcntBitShiftLo ( Version . Major ) , getVmcntBitWidthLo ( Version . Major ) ) ;

return packBits ( Vmcnt >> getVmcntBitWidthLo ( Version . Major ) , Waitcnt , getVmcntBitShiftHi ( Version . Major ) , getVmcntBitWidthHi ( Version . Major ) ) ;


<s> unsigned encodeVmcnt ( const IsaVersion & Version , unsigned Waitcnt , unsigned Vmcnt ) {

Waitcnt = packBits ( Vmcnt , Waitcnt , getVmcntBitShiftLo ( Version . Major ) , getVmcntBitWidthLo ( Version . Major ) ) ;


<s> unsigned getExpcntBitMask ( const IsaVersion & Version ) {

return ( <NUM_LIT> << getExpcntBitWidth ( Version . Major ) ) - <NUM_LIT> ;


<s> return VersionMajor >= <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ;


<s> unsigned getExpcntBitShift ( unsigned VersionMajor ) {


<s> unsigned getExpcntBitWidth ( unsigned VersionMajor ) {


<s> const GcnBufferFormatInfo * getGcnBufferFormatInfo ( uint8_t Format , const MCSubtargetInfo & STI ) { return isGFX11Plus ( STI ) ? getGfx11PlusBufferFormatInfo ( Format ) : isGFX10 ( STI ) ? getGfx10BufferFormatInfo ( Format ) : getGfx9BufferFormatInfo ( Format ) ;


<s> const GcnBufferFormatInfo * getGcnBufferFormatInfo ( uint8_t Format , const MCSubtargetInfo & STI ) {

return isGFX11Plus ( STI ) ? getGfx11PlusBufferFormatInfo ( Format ) : isGFX10 ( STI ) ? getGfx10BufferFormatInfo ( Format ) : getGfx9BufferFormatInfo ( Format ) ;


<s> return isGFX11Plus ( STI ) ? ID_MASK_GFX11Plus_ : ID_MASK_PreGFX11_ ;


<s> assert ( msgRequiresOp ( MsgId , STI ) ) ;

return ( MsgId == ID_SYSMSG ) ? OpSysSymbolic [ OpId ] : OpGsSymbolic [ OpId ] ;


<s> for ( int Id = UfmtGFX10 :: UFMT_FIRST ; Id <= UfmtGFX10 :: UFMT_LAST ; ++ Id ) { if ( Name == UfmtSymbolicGFX10 [ Id ] ) return Id ; } }


<s> for ( int Id = UfmtGFX11 :: UFMT_FIRST ; Id <= UfmtGFX11 :: UFMT_LAST ; ++ Id ) { if ( Name == UfmtSymbolicGFX11 [ Id ] ) return Id ;


<s> StringRef getUnifiedFormatName ( unsigned Id , const MCSubtargetInfo & STI ) { if ( isValidUnifiedFormat ( Id , STI ) ) return isGFX10 ( STI ) ? UfmtSymbolicGFX10 [ Id ] : UfmtSymbolicGFX11 [ Id ] ; return <STR_LIT> ;


<s> if ( isValidUnifiedFormat ( Id , STI ) ) return isGFX10 ( STI ) ? UfmtSymbolicGFX10 [ Id ] : UfmtSymbolicGFX11 [ Id ] ;


<s> unsigned getVmcntBitMask ( const IsaVersion & Version ) {

return ( <NUM_LIT> << ( getVmcntBitWidthLo ( Version . Major ) + getVmcntBitWidthHi ( Version . Major ) ) ) - <NUM_LIT> ;


<s> unsigned getVmcntBitShiftHi ( unsigned VersionMajor ) {


<s> unsigned getVmcntBitShiftHi ( unsigned VersionMajor ) { return <NUM_LIT> ;


<s> unsigned getVmcntBitShiftLo ( unsigned VersionMajor ) {


<s> return ( VersionMajor == <NUM_LIT> || VersionMajor == <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> ;


<s> unsigned getVmcntBitWidthLo ( unsigned VersionMajor ) {


<s> unsigned VmcntLo = getBitMask ( getVmcntBitShiftLo ( Version . Major ) , getVmcntBitWidthLo ( Version . Major ) ) ;

unsigned Expcnt = getBitMask ( getExpcntBitShift ( Version . Major ) , getExpcntBitWidth ( Version . Major ) ) ;


<s> bool hasPackedD16 ( const MCSubtargetInfo & STI ) {

return ! STI . getFeatureBits ( ) [ <STR_LIT> ] && ! isCI ( STI ) && ! is<ISA_LIT> ( STI ) ;


<s> bool isGFX10Plus ( const MCSubtargetInfo & STI ) { return isGFX10 ( STI ) || isGFX11Plus ( STI ) ;


<s> bool isGFX11 ( const MCSubtargetInfo & STI ) { return STI . getFeatureBits ( ) [ <STR_LIT> ] ;


<s> bool isGFX8_GFX9_GFX10 ( const MCSubtargetInfo & STI ) {

return isVI ( STI ) || isGFX9 ( STI ) || isGFX10 ( STI ) ;


<s> switch ( Id ) { case ET_NULL : return ! isGFX11Plus ( STI ) ; case ET_POS4 : case ET_PRIM : return isGFX10Plus ( STI ) ; case ET_DUAL_SRC_BLEND0 : case ET_DUAL_SRC_BLEND1 : return isGFX11Plus ( STI ) ; default :

if ( Id >= ET_PARAM0 && Id <= ET_PARAM31 ) return ! isGFX11Plus ( STI ) ;


<s> return ( MsgId & ~ ( getMsgIdMask ( STI ) ) ) == <NUM_LIT> ;


<s> switch ( MsgId ) { case ID_GS_PreGFX11 : return ( OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_ ) && OpId != OP_GS_NOP ; case ID_GS_DONE_PreGFX11 :

return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_ ;


<s> bool isValidMsgStream ( int64_t MsgId , int64_t OpId , int64_t StreamId , const MCSubtargetInfo & STI , bool Strict ) { assert ( isValidMsgOp ( MsgId , OpId , STI , Strict ) ) ; if ( ! Strict ) return <NUM_LIT> <= StreamId && isUInt < STREAM_ID_WIDTH_ > ( StreamId ) ; if ( ! isGFX11Plus ( STI ) ) { switch ( MsgId ) { case ID_GS_PreGFX11 : return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_ ;


<s> bool isValidUnifiedFormat ( unsigned Id , const MCSubtargetInfo & STI ) { return isGFX10 ( STI ) ? Id <= UfmtGFX10 :: UFMT_LAST : Id <= UfmtGFX11 :: UFMT_LAST ;


<s> bool msgRequiresOp ( int64_t MsgId , const MCSubtargetInfo & STI ) { return MsgId == ID_SYSMSG || ( ! isGFX11Plus ( STI ) && ( MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11 ) ) ;


<s> unsigned packBits ( unsigned Src , unsigned Dst , unsigned Shift , unsigned Width ) { unsigned Mask = getBitMask ( Shift , Width ) ; return ( ( Src << Shift ) & Mask ) | ( Dst & ~ Mask ) ;


<s> if ( isGFX10 ( STI ) ) return UFMT_DEFAULT ;


<s> unsigned getDefaultFormatEncoding ( const MCSubtargetInfo & STI ) { if ( isGFX10 ( STI ) ) return UFMT_DEFAULT ;


<s> return isGFX10 ( STI ) ? ( Val <= UFMT_MAX ) : ( Val <= DFMT_NFMT_MAX ) ;


<s> return <STR_LIT> :: HOSTCALL_PTR_OFFSET ; default : llvm_unreachable ( <STR_LIT> ) ;


<s> case <NUM_LIT> : case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <STR_LIT> :: HOSTCALL_PTR_OFFSET ; default :


<s> int Idx = getOprIdx < const MCSubtargetInfo & > ( Id , Opr , OPR_<ISA_LIT>ZE , STI ) ;

return ( Idx < <NUM_LIT> ) ? <STR_LIT> : Opr [ Idx ] . Name ;


<s> StringRef getHwreg ( unsigned Id , const MCSubtargetInfo & STI ) {

int Idx = getOprIdx < const MCSubtargetInfo & > ( Id , Opr , OPR_<ISA_LIT>ZE , STI ) ;


<s> return ( Idx < <NUM_LIT> ) ? Idx : Opr [ Idx ] . Encoding ;


<s> int64_t getHwregId ( const StringRef Name , const MCSubtargetInfo & STI ) { int Idx = getOprIdx < const MCSubtargetInfo & > ( Name , Opr , OPR_<ISA_LIT>ZE , STI ) ; return ( Idx < <NUM_LIT> ) ? Idx : Opr [ Idx ] . Encoding ;


<s> bool getMAIIsDGEMM ( unsigned Opc ) { const MAIInstInfo * Info = getMAIInstInfoHelper ( Opc ) ;

return Info ? Info -> is_dgemm : false ;


<s> bool getMAIIsDGEMM ( unsigned Opc ) {

const MAIInstInfo * Info = getMAIInstInfoHelper ( Opc ) ;


<s> const MAIInstInfo * Info = getMAIInstInfoHelper ( Opc ) ; return Info ? Info -> is_gfx940_xdl : false ;


<s> return Info ? Info -> is_gfx940_xdl : false ;


<s> int64_t getMsgId ( const StringRef Name , const MCSubtargetInfo & STI ) { int Idx = getOprIdx < const MCSubtargetInfo & > ( Name , Msg , MSG_<ISA_LIT>ZE , STI ) ; return ( Idx < <NUM_LIT> ) ? Idx : Msg [ Idx ] . Encoding ;


<s> int Idx = getOprIdx < const MCSubtargetInfo & > ( Name , Msg , MSG_<ISA_LIT>ZE , STI ) ;

return ( Idx < <NUM_LIT> ) ? Idx : Msg [ Idx ] . Encoding ;


<s> int Idx = getOprIdx < const MCSubtargetInfo & > ( MsgId , Msg , MSG_<ISA_LIT>ZE , STI ) ;

return ( Idx < <NUM_LIT> ) ? <STR_LIT> : Msg [ Idx ] . Name ;


<s> return ( Idx < <NUM_LIT> ) ? <STR_LIT> : Msg [ Idx ] . Name ;


<s> static int getOprIdx ( int Id , const CustomOperand < T > OpInfo [ ] , int OpInfoSize , T Context , bool QuickCheck = true ) { auto Test = [ = ] ( const CustomOperand < T > & Op ) { return Op . Encoding == Id && ! Op . Name . empty ( ) ; } ;

if ( QuickCheck && isValidOpr < T > ( Id , OpInfo , OpInfoSize , Context ) && OpInfo [ Id ] . Encoding == Id ) {


<s> if ( has90AInsts && ArgNumAGPR ) return alignTo ( ArgNumVGPR , <NUM_LIT> ) + ArgNumAGPR ; return std :: max ( ArgNumVGPR , ArgNumAGPR ) ;


<s> bool hasMAIInsts ( const MCSubtargetInfo & STI ) {

return STI . getFeatureBits ( ) [ <STR_LIT> ] ;


<s> return STI . getFeatureBits ( ) [ <STR_LIT> ] ;


<s> bool isGFX8Plus ( const MCSubtargetInfo & STI ) {

return isVI ( STI ) || isGFX9Plus ( STI ) ;


<s> bool isGFX940 ( const MCSubtargetInfo & STI ) { return STI . getFeatureBits ( ) [ <STR_LIT> ] ;


<s> return isGFX9 ( STI ) || isGFX10 ( STI ) ;


<s> bool isKernelCC ( const Function * Func ) { return <STR_LIT> ( Func -> getCallingConv ( ) ) ;


<s> return is<ISA_LIT> ( STI ) || isCI ( STI ) || isVI ( STI ) || isGFX9 ( STI ) ;


<s> bool isNotGFX10Plus ( const MCSubtargetInfo & STI ) {

return is<ISA_LIT> ( STI ) || isCI ( STI ) || isVI ( STI ) || isGFX9 ( STI ) ;


<s> bool isValidMsgId ( int64_t MsgId ) { return <NUM_LIT> <= MsgId && isUInt < ID_WIDTH_ > ( MsgId ) ;


<s> return <NUM_LIT> <= MsgId && isUInt < ID_WIDTH_ > ( MsgId ) ;


<s> if ( ! Strict ) return <NUM_LIT> <= OpId && isUInt < OP_WIDTH_ > ( OpId ) ; switch ( MsgId ) { case ID_GS : return ( OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_ ) && OpId != OP_GS_NOP ; case ID_GS_DONE : return OP_GS_FIRST_ <= OpId && OpId < OP_GS_LAST_ ; case ID_SYSMSG :

return OP_SYS_FIRST_ <= OpId && OpId < OP_SYS_LAST_ ;


<s> * this = getDefaultForCallingConv ( F . getCallingConv ( ) ) ; StringRef IEEEAttr = F . getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) ; if ( ! IEEEAttr . empty ( ) ) IEEE = IEEEAttr == <STR_LIT> ; StringRef DX10ClampAttr = F . getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) ; if ( ! DX10ClampAttr . empty ( ) ) DX10Clamp = DX10ClampAttr == <STR_LIT> ;

FP32Denormals = ST . hasFP32Denormals ( F ) ;


<s> <STR_LIT> ( const Function & F , const GCNSubtarget & ST ) { * this = getDefaultForCallingConv ( F . getCallingConv ( ) ) ;

StringRef IEEEAttr = F . getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) ;


<s> if ( DX10Clamp != CalleeMode . DX10Clamp ) return false ; if ( IEEE != CalleeMode . IEEE ) return false ;

return oneWayCompatible ( FP64FP16Denormals , CalleeMode . FP64FP16Denormals ) && oneWayCompatible ( FP32Denormals , CalleeMode . FP32Denormals ) ;


<s> if ( IEEE != CalleeMode . IEEE ) return false ;


<s> static bool oneWayCompatible ( bool CallerMode , bool CalleeMode ) { return CallerMode == CalleeMode || ( CallerMode && ! CalleeMode ) ;


<s> return CallerMode == CalleeMode || ( CallerMode && ! CalleeMode ) ;


<s> <ISA_LIT>ModeRegisterDefaults Mode ; Mode . DX10Clamp = true ; Mode . IEEE = IsCompute ; Mode . FP32InputDenormals = false ; Mode . FP32OutputDenormals = false ;


<s> <ISA_LIT>ModeRegisterDefaults Mode ; Mode . DX10Clamp = true ; Mode . IEEE = IsCompute ; Mode . FP32InputDenormals = false ;


<s> case CallingConv :: <ISA_LIT>_KERNEL : case CallingConv :: SPIR_KERNEL : return true ;


<s> } unsigned SubReg0 = TRI -> getSubReg ( Reg0 , <NUM_LIT> ) ; if ( SubReg0 == <NUM_LIT> ) { return TRI -> getSubRegIndex ( Reg1 , Reg0 ) > <NUM_LIT> ; } for ( unsigned Idx = <NUM_LIT> ; SubReg0 > <NUM_LIT> ; ++ Idx ) {

if ( isRegIntersect ( Reg1 , SubReg0 , TRI ) ) {


<s> bool isRegIntersect ( unsigned Reg0 , unsigned Reg1 , const MCRegisterInfo * TRI ) { if ( Reg0 == Reg1 ) { return true ; } unsigned SubReg0 = TRI -> getSubReg ( Reg0 , <NUM_LIT> ) ; if ( SubReg0 == <NUM_LIT> ) { return TRI -> getSubRegIndex ( Reg1 , Reg0 ) > <NUM_LIT> ; } for ( unsigned Idx = <NUM_LIT> ; SubReg0 > <NUM_LIT> ; ++ Idx ) { if ( isRegIntersect ( Reg1 , SubReg0 , TRI ) ) { return true ; } SubReg0 = TRI -> getSubReg ( Reg0 , Idx ) ;


AMDHSA_BITS_SET ( KD . compute_pgm_rsrc1 , amdhsa :: COMPUTE_PGM_RSRC1_WGP_MODE , STI -> getFeatureBits ( ) . test ( FeatureCuMode ) ? <NUM_LIT> : <NUM_LIT> ) ;


<s> Header . amd_kernel_code_version_minor = <NUM_LIT> ; Header . amd_machine_kind = <NUM_LIT> ; Header . amd_machine_version_major = Version . Major ; Header . amd_machine_version_minor = Version . Minor ; Header . amd_machine_version_stepping = Version . Stepping ; Header . kernel_code_entry_byte_offset = sizeof ( Header ) ; Header . wavefront_size = <NUM_LIT> ; Header . call_convention = - <NUM_LIT> ;

Header . kernarg_segment_alignment = <NUM_LIT> ;


<s> if ( Features . test ( FeatureISAVersion8_0_3 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_1_0 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureVolcanicIslands ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_0 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_2 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_4 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_6 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureGFX9 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureSouthernIslands ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ;


<s> if ( Features . test ( FeatureISAVersion6_0_0 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion6_0_1 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion7_0_0 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion7_0_1 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion7_0_2 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion7_0_3 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion7_0_4 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureSeaIslands ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_0_1 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_0_2 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_0_3 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_1_0 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureVolcanicIslands ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_0 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_2 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_4 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion9_0_6 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ;

if ( Features . test ( FeatureGFX9 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ;


<s> if ( WavesPerEU >= getMaxWavesPerEU ( ) ) return <NUM_LIT> ; unsigned MinNumSGPRs = getTotalNumSGPRs ( Features ) / ( WavesPerEU + <NUM_LIT> ) ; if ( Features . test ( FeatureTrapHandler ) ) MinNumSGPRs -= std :: min ( MinNumSGPRs , ( unsigned ) TRAP_NUM_SGPRS ) ;

MinNumSGPRs = alignDown ( MinNumSGPRs , getSGPRAllocGranule ( Features ) ) + <NUM_LIT> ;


<s> unsigned getMinNumVGPRs ( const FeatureBitset & Features , unsigned WavesPerEU ) { assert ( WavesPerEU != <NUM_LIT> ) ; if ( WavesPerEU >= getMaxWavesPerEU ( ) ) return <NUM_LIT> ;

unsigned MinNumVGPRs = alignDown ( getTotalNumVGPRs ( Features ) / ( WavesPerEU + <NUM_LIT> ) , getVGPRAllocGranule ( Features ) ) + <NUM_LIT> ;


<s> return getNumExtraSGPRs ( Features , VCCUsed , FlatScrUsed , Features [ <STR_LIT> ] ) ;


<s> unsigned getNumSGPRBlocks ( const FeatureBitset & Features , unsigned NumSGPRs ) { NumSGPRs = alignTo ( std :: max ( <NUM_LIT> , NumSGPRs ) , getSGPREncodingGranule ( Features ) ) ; return NumSGPRs / getSGPREncodingGranule ( Features ) - <NUM_LIT> ;


<s> NumVGPRs = alignTo ( std :: max ( <NUM_LIT> , NumVGPRs ) , getVGPREncodingGranule ( Features ) ) ; return NumVGPRs / getVGPREncodingGranule ( Features ) - <NUM_LIT> ;


<s> return STI -> getFeatureBits ( ) . test ( FeatureCodeObjectV3 ) ;


<s> auto TargetTriple = STI -> getTargetTriple ( ) ; auto ISAVersion = IsaInfo :: getIsaVersion ( STI -> getFeatureBits ( ) ) ;

Stream << TargetTriple . getArchName ( ) << '-' << TargetTriple . getVendorName ( ) << '-' << TargetTriple . getOSName ( ) << '-' << TargetTriple . getEnvironmentName ( ) << '-' << <STR_LIT> << ISAVersion . Major << ISAVersion . Minor << ISAVersion . Stepping ;


<s> void streamIsaVersion ( const MCSubtargetInfo * STI , raw_ostream & Stream ) { auto TargetTriple = STI -> getTargetTriple ( ) ;

auto ISAVersion = IsaInfo :: getIsaVersion ( STI -> getFeatureBits ( ) ) ;


<s> return getRegBitWidth ( MRI -> getRegClass ( RCID ) ) / <NUM_LIT> ;


<s> unsigned RCID = Desc . OpInfo [ OpNo ] . RegClass ; return getRegBitWidth ( MRI -> getRegClass ( RCID ) ) / <NUM_LIT> ;


<s> if ( Literal >= - <NUM_LIT> && Literal <= <NUM_LIT> ) return true ;

uint16_t Val = static_cast < uint16_t > ( Literal ) ;


<s> uint16_t Val = static_cast < uint16_t > ( Literal ) ; return Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : return false ;


<s> unsigned OpType = Desc . OpInfo [ OpNo ] . OperandType ; return OpType >= <STR_LIT> && OpType <= <STR_LIT> ;


<s> bool is<ISA_LIT>SrcInlinableOperand ( const MCInstrDesc & Desc , unsigned OpNo ) { unsigned OpType = Desc . OpInfo [ OpNo ] . OperandType ;

return OpType >= <STR_LIT> && OpType <= <STR_LIT> ;


<s> bool is<ISA_LIT>SrcOperand ( const MCInstrDesc & Desc , unsigned OpNo ) { unsigned OpType = Desc . OpInfo [ OpNo ] . OperandType ;


<s> bool isGFX9Plus ( const MCSubtargetInfo & STI ) {

return isGFX9 ( STI ) || isGFX10 ( STI ) ;


<s> else { AS . FLAT_ADDRESS = <NUM_LIT> ; AS . CONSTANT_ADDRESS = <NUM_LIT> ; AS . PRIVATE_ADDRESS = <NUM_LIT> ;


<s> bool isInlinableIntLiteral ( int64_t Literal ) { return Literal >= - <NUM_LIT> && Literal <= <NUM_LIT> ;


<s> Stream << TargetTriple . getArchName ( ) << '-' << TargetTriple . getVendorName ( ) << '-' << TargetTriple . getOSName ( ) << '-' << TargetTriple . getEnvironmentName ( ) << '-' << <STR_LIT> << Version . Major << Version . Minor << Version . Stepping ;

if ( hasXNACK ( * STI ) ) Stream << <STR_LIT> ;


<s> int64_t EncodedOffset = convertSMRDOffsetUnits ( ST , ByteOffset ) ; return isUInt < <NUM_LIT> > ( EncodedOffset ) ? Optional < int64_t > ( EncodedOffset ) : None ;


<s> if ( ! isDwordAligned ( ByteOffset ) && ! hasSMEMByteOffset ( ST ) ) return None ; assert ( isCI ( ST ) ) ;

int64_t EncodedOffset = convertSMRDOffsetUnits ( ST , ByteOffset ) ;


<s> return isLegalSMRDEncodedImmOffset ( ST , EncodedOffset ) ? Optional < int64_t > ( EncodedOffset ) : None ;


<s> Optional < int64_t > getSMRDEncodedOffset ( const MCSubtargetInfo & ST , int64_t ByteOffset ) {

if ( ! isDwordAligned ( ByteOffset ) && ! hasSMEMByteOffset ( ST ) ) return None ;


<s> return hasSMEMByteOffset ( ST ) ? isUInt < <NUM_LIT> > ( EncodedOffset ) : isUInt < <NUM_LIT> > ( EncodedOffset ) ;


<s> StringRef DX10ClampAttr = F . getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) ; if ( ! DX10ClampAttr . empty ( ) ) DX10Clamp = DX10ClampAttr == <STR_LIT> ; FP32InputDenormals = ST . hasFP32Denormals ( F ) ;


<s> return unpackBits ( Waitcnt , getVmcntBitShift ( ) , getVmcntBitWidth ( ) ) ;


<s> unsigned decodeVmcnt ( const IsaInfo :: IsaVersion & Version , unsigned Waitcnt ) { return unpackBits ( Waitcnt , getVmcntBitShift ( ) , getVmcntBitWidth ( ) ) ;


<s> unsigned encodeVmcnt ( const IsaInfo :: IsaVersion & Version , unsigned Waitcnt , unsigned Vmcnt ) { return packBits ( Vmcnt , Waitcnt , getVmcntBitShift ( ) , getVmcntBitWidth ( ) ) ;


<s> if ( Features . test ( FeatureISAVersion8_0_1 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_0_2 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_0_3 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_0_4 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ;

if ( Features . test ( FeatureISAVersion8_1_0 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ;


<s> if ( Features . test ( FeatureISAVersion8_0_3 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_0_4 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ; if ( Features . test ( FeatureISAVersion8_1_0 ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ;

if ( ! Features . test ( FeatureGCN ) || Features . test ( FeatureSouthernIslands ) ) return { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } ;


<s> unsigned getVmcntBitMask ( const IsaInfo :: IsaVersion & Version ) {

return ( <NUM_LIT> << getVmcntBitWidth ( ) ) - <NUM_LIT> ;


<s> unsigned Lgkmcnt = getBitMask ( getLgkmcntBitShift ( ) , getLgkmcntBitWidth ( ) ) ;


<s> unsigned getWaitcntBitMask ( const IsaInfo :: IsaVersion & Version ) {

unsigned Vmcnt = getBitMask ( getVmcntBitShift ( ) , getVmcntBitWidth ( ) ) ;


<s> AS . FLAT_ADDRESS = <NUM_LIT> ; AS . PRIVATE_ADDRESS = <NUM_LIT> ; AS . REGION_ADDRESS = <NUM_LIT> ; return AS ;


<s> AS . FLAT_ADDRESS = <NUM_LIT> ; AS . PRIVATE_ADDRESS = <NUM_LIT> ;


<s> return unpackBits ( Waitcnt , getExpcntBitShift ( ) , getExpcntBitWidth ( ) ) ;


<s> unsigned decodeLgkmcnt ( const IsaVersion & Version , unsigned Waitcnt ) { return unpackBits ( Waitcnt , getLgkmcntBitShift ( ) , getLgkmcntBitWidth ( ) ) ;


<s> unsigned VmcntLo = unpackBits ( Waitcnt , getVmcntBitShiftLo ( ) , getVmcntBitWidthLo ( ) ) ; if ( Version . Major < <NUM_LIT> ) return VmcntLo ; unsigned VmcntHi = unpackBits ( Waitcnt , getVmcntBitShiftHi ( ) , getVmcntBitWidthHi ( ) ) ; VmcntHi <<= getVmcntBitWidthLo ( ) ;


<s> Decoded . VmCnt = decodeVmcnt ( Version , Encoded ) ; Decoded . ExpCnt = decodeExpcnt ( Version , Encoded ) ; Decoded . LgkmCnt = decodeLgkmcnt ( Version , Encoded ) ; return Decoded ;


<s> Waitcnt decodeWaitcnt ( const IsaVersion & Version , unsigned Encoded ) { Waitcnt Decoded ; Decoded . VmCnt = decodeVmcnt ( Version , Encoded ) ; Decoded . ExpCnt = decodeExpcnt ( Version , Encoded ) ; Decoded . LgkmCnt = decodeLgkmcnt ( Version , Encoded ) ; return Decoded ;


<s> unsigned encodeExpcnt ( const IsaVersion & Version , unsigned Waitcnt , unsigned Expcnt ) { return packBits ( Expcnt , Waitcnt , getExpcntBitShift ( ) , getExpcntBitWidth ( ) ) ;


<s> return packBits ( Expcnt , Waitcnt , getExpcntBitShift ( ) , getExpcntBitWidth ( ) ) ;


<s> return packBits ( Lgkmcnt , Waitcnt , getLgkmcntBitShift ( ) , getLgkmcntBitWidth ( ) ) ;


<s> Waitcnt = packBits ( Vmcnt , Waitcnt , getVmcntBitShiftLo ( ) , getVmcntBitWidthLo ( ) ) ;

if ( Version . Major < <NUM_LIT> ) return Waitcnt ;


<s> unsigned encodeWaitcnt ( const IsaVersion & Version , const Waitcnt & Decoded ) {

return encodeWaitcnt ( Version , Decoded . VmCnt , Decoded . ExpCnt , Decoded . LgkmCnt ) ;


<s> if ( STI -> getFeatureBits ( ) . test ( FeatureSGPRInitBug ) ) return FIXED_NUM_SGPRS_FOR_INIT_BUG ; IsaVersion Version = getIsaVersion ( STI -> getCPU ( ) ) ; if ( Version . Major >= <NUM_LIT> ) return <NUM_LIT> ;


<s> unsigned getAddressableNumVGPRs ( const MCSubtargetInfo * STI ) {


<s> unsigned getBitMask ( unsigned Shift , unsigned Width ) { return ( ( <NUM_LIT> << Width ) - <NUM_LIT> ) << Shift ;


<s> unsigned getBitMask ( unsigned Shift , unsigned Width ) {

return ( ( <NUM_LIT> << Width ) - <NUM_LIT> ) << Shift ;


AMDHSA_BITS_SET ( KD . compute_pgm_rsrc1 , amdhsa :: COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP , <NUM_LIT> ) ;


<s> unsigned getEUsPerCU ( const MCSubtargetInfo * STI ) {


<s> return ( <NUM_LIT> << getExpcntBitWidth ( ) ) - <NUM_LIT> ;


<s> unsigned getInitialP<ISA_LIT>nputAddr ( const Function & F ) {

return getIntegerAttribute ( F , <STR_LIT> , <NUM_LIT> ) ;


<s> StringRef Str = A . getValueAsString ( ) ; if ( Str . getAsInteger ( <NUM_LIT> , Result ) ) { LLVMContext & Ctx = F . getContext ( ) ; Ctx . emitError ( <STR_LIT> + Name ) ; }


<s> if ( ! A . isStringAttribute ( ) ) return Default ; LLVMContext & Ctx = F . getContext ( ) ; std :: pair < int , int > Ints = Default ; std :: pair < StringRef , StringRef > Strs = A . getValueAsString ( ) . split ( ',' ) ;

if ( Strs . first . trim ( ) . getAsInteger ( <NUM_LIT> , Ints . first ) ) {


<s> unsigned getLgkmcntBitMask ( const IsaVersion & Version ) {

return ( <NUM_LIT> << getLgkmcntBitWidth ( ) ) - <NUM_LIT> ;


<s> unsigned getLgkmcntBitShift ( ) { return <NUM_LIT> ;


<s> if ( STI -> getFeatureBits ( ) . test ( FeatureLocalMemorySize32768 ) ) return <NUM_LIT> ; if ( STI -> getFeatureBits ( ) . test ( FeatureLocalMemorySize65536 ) ) return <NUM_LIT> ;


<s> const MIMGInfo * NewInfo = getMIMGOpcodeHelper ( OrigInfo -> BaseOpcode , OrigInfo -> MIMGEncoding , NewChannels , OrigInfo -> VAddrDwords ) ;

return NewInfo ? NewInfo -> Opcode : - <NUM_LIT> ;


<s> const MIMGInfo * NewInfo = getMIMGOpcodeHelper ( OrigInfo -> BaseOpcode , OrigInfo -> MIMGEncoding , NewChannels , OrigInfo -> VAddrDwords ) ; return NewInfo ? NewInfo -> Opcode : - <NUM_LIT> ;


<s> if ( STI -> getFeatureBits ( ) . test ( FeatureTrapHandler ) ) MaxNumSGPRs -= std :: min ( MaxNumSGPRs , ( unsigned ) TRAP_NUM_SGPRS ) ; MaxNumSGPRs = alignDown ( MaxNumSGPRs , getSGPRAllocGranule ( STI ) ) ; return std :: min ( MaxNumSGPRs , AddressableNumSGPRs ) ;


<s> unsigned MaxNumVGPRs = alignDown ( getTotalNumVGPRs ( STI ) / WavesPerEU , getVGPRAllocGranule ( STI ) ) ; unsigned AddressableNumVGPRs = getAddressableNumVGPRs ( STI ) ; return std :: min ( MaxNumVGPRs , AddressableNumVGPRs ) ;


<s> unsigned getMaxNumVGPRs ( const MCSubtargetInfo * STI , unsigned WavesPerEU ) { assert ( WavesPerEU != <NUM_LIT> ) ; unsigned MaxNumVGPRs = alignDown ( getTotalNumVGPRs ( STI ) / WavesPerEU , getVGPRAllocGranule ( STI ) ) ; unsigned AddressableNumVGPRs = getAddressableNumVGPRs ( STI ) ;

return std :: min ( MaxNumVGPRs , AddressableNumVGPRs ) ;


<s> return getWavesPerWorkGroup ( STI , FlatWorkGroupSize ) ;


<s> return alignTo ( getMaxWavesPerCU ( STI , FlatWorkGroupSize ) , getEUsPerCU ( STI ) ) / getEUsPerCU ( STI ) ;


<s> if ( ! STI -> getFeatureBits ( ) . test ( FeatureGCN ) ) return <NUM_LIT> ; unsigned N = getWavesPerWorkGroup ( STI , FlatWorkGroupSize ) ; if ( N == <NUM_LIT> ) return <NUM_LIT> ; N = <NUM_LIT> / N ; return std :: min ( N , <NUM_LIT> ) ;


<s> if ( ! STI -> getFeatureBits ( ) . test ( FeatureGCN ) ) return <NUM_LIT> ; unsigned N = getWavesPerWorkGroup ( STI , FlatWorkGroupSize ) ;

if ( N == <NUM_LIT> ) return <NUM_LIT> ;


<s> return getMCOpcodeGen ( Opcode , static_cast < Subtarget > ( Gen ) ) ;


<s> const MIMGInfo * Info = getMIMGOpcodeHelper ( BaseOpcode , MIMGEncoding , VDataDwords , VAddrDwords ) ;

return Info ? Info -> Opcode : - <NUM_LIT> ;


<s> int getMIMGOpcode ( unsigned BaseOpcode , unsigned MIMGEncoding , unsigned VDataDwords , unsigned VAddrDwords ) {

const MIMGInfo * Info = getMIMGOpcodeHelper ( BaseOpcode , MIMGEncoding , VDataDwords , VAddrDwords ) ;


<s> unsigned getMinFlatWorkGroupSize ( const MCSubtargetInfo * STI ) { return <NUM_LIT> ;


<s> unsigned MinNumSGPRs = getTotalNumSGPRs ( STI ) / ( WavesPerEU + <NUM_LIT> ) ; if ( STI -> getFeatureBits ( ) . test ( FeatureTrapHandler ) ) MinNumSGPRs -= std :: min ( MinNumSGPRs , ( unsigned ) TRAP_NUM_SGPRS ) ; MinNumSGPRs = alignDown ( MinNumSGPRs , getSGPRAllocGranule ( STI ) ) + <NUM_LIT> ; return std :: min ( MinNumSGPRs , getAddressableNumSGPRs ( STI ) ) ;


<s> unsigned getMinNumSGPRs ( const MCSubtargetInfo * STI , unsigned WavesPerEU ) { assert ( WavesPerEU != <NUM_LIT> ) ; if ( WavesPerEU >= getMaxWavesPerEU ( ) ) return <NUM_LIT> ; unsigned MinNumSGPRs = getTotalNumSGPRs ( STI ) / ( WavesPerEU + <NUM_LIT> ) ;

if ( STI -> getFeatureBits ( ) . test ( FeatureTrapHandler ) ) MinNumSGPRs -= std :: min ( MinNumSGPRs , ( unsigned ) TRAP_NUM_SGPRS ) ;


<s> unsigned MinNumVGPRs = alignDown ( getTotalNumVGPRs ( STI ) / ( WavesPerEU + <NUM_LIT> ) , getVGPRAllocGranule ( STI ) ) + <NUM_LIT> ; return std :: min ( MinNumVGPRs , getAddressableNumVGPRs ( STI ) ) ;


<s> unsigned getMinWavesPerEU ( const MCSubtargetInfo * STI ) { return <NUM_LIT> ;


<s> int getMUBUFBaseOpcode ( unsigned Opc ) { const MUBUFInfo * Info = getMUBUFInfoFromOpcode ( Opc ) ; return Info ? Info -> BaseOpcode : - <NUM_LIT> ;


<s> const MUBUFInfo * Info = getMUBUFInfoFromOpcode ( Opc ) ;

return Info ? Info -> BaseOpcode : - <NUM_LIT> ;


<s> const MUBUFInfo * Info = getMUBUFOpcodeHelper ( Opc ) ; return Info ? Info -> dwords : <NUM_LIT> ;


<s> return Info ? Info -> dwords : <NUM_LIT> ;


<s> const MUBUFInfo * Info = getMUBUFOpcodeHelper ( Opc ) ;

return Info ? Info -> has_soffset : false ;


<s> const MUBUFInfo * Info = getMUBUFOpcodeHelper ( Opc ) ; return Info ? Info -> has_vaddr : false ;


<s> return Info ? Info -> has_vaddr : false ;


<s> return Info ? Info -> Opcode : - <NUM_LIT> ;


<s> unsigned getNumVGPRBlocks ( const MCSubtargetInfo * STI , unsigned NumVGPRs ) { NumVGPRs = alignTo ( std :: max ( <NUM_LIT> , NumVGPRs ) , getVGPREncodingGranule ( STI ) ) ; return NumVGPRs / getVGPREncodingGranule ( STI ) - <NUM_LIT> ;


<s> unsigned getRegBitWidth ( const MCRegisterClass & RC ) { return getRegBitWidth ( RC . getID ( ) ) ;


<s> unsigned getRegOperandSize ( const MCRegisterInfo * MRI , const MCInstrDesc & Desc , unsigned OpNo ) { assert ( OpNo < Desc . NumOperands ) ; unsigned RCID = Desc . OpInfo [ OpNo ] . RegClass ; return getRegBitWidth ( MRI -> getRegClass ( RCID ) ) / <NUM_LIT> ;


<s> unsigned getRegOperandSize ( const MCRegisterInfo * MRI , const MCInstrDesc & Desc , unsigned OpNo ) {


<s> if ( Version . Major >= <NUM_LIT> ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> unsigned getSGPREncodingGranule ( const MCSubtargetInfo * STI ) {


<s> if ( isGCN3Encoding ( ST ) ) return ByteOffset ;


<s> unsigned getTotalNumVGPRs ( const MCSubtargetInfo * STI ) { return <NUM_LIT> ;


<s> unsigned getVGPRAllocGranule ( const MCSubtargetInfo * STI ) { return <NUM_LIT> ;


<s> unsigned getVGPREncodingGranule ( const MCSubtargetInfo * STI ) { return getVGPRAllocGranule ( STI ) ;


<s> unsigned getVmcntBitMask ( const IsaVersion & Version ) { unsigned VmcntLo = ( <NUM_LIT> << getVmcntBitWidthLo ( ) ) - <NUM_LIT> ; if ( Version . Major < <NUM_LIT> ) return VmcntLo ; unsigned VmcntHi = ( ( <NUM_LIT> << getVmcntBitWidthHi ( ) ) - <NUM_LIT> ) << getVmcntBitWidthLo ( ) ; return VmcntLo | VmcntHi ;


<s> unsigned getVmcntBitMask ( const IsaVersion & Version ) { unsigned VmcntLo = ( <NUM_LIT> << getVmcntBitWidthLo ( ) ) - <NUM_LIT> ;

if ( Version . Major < <NUM_LIT> ) return VmcntLo ;


<s> unsigned getVmcntBitShiftHi ( ) { return <NUM_LIT> ;


<s> unsigned Expcnt = getBitMask ( getExpcntBitShift ( ) , getExpcntBitWidth ( ) ) ; unsigned Lgkmcnt = getBitMask ( getLgkmcntBitShift ( ) , getLgkmcntBitWidth ( ) ) ; unsigned Waitcnt = VmcntLo | Expcnt | Lgkmcnt ; if ( Version . Major < <NUM_LIT> ) return Waitcnt ; unsigned VmcntHi = getBitMask ( getVmcntBitShiftHi ( ) , getVmcntBitWidthHi ( ) ) ;


<s> unsigned getWavefrontSize ( const MCSubtargetInfo * STI ) { if ( STI -> getFeatureBits ( ) . test ( FeatureWavefrontSize16 ) ) return <NUM_LIT> ; if ( STI -> getFeatureBits ( ) . test ( FeatureWavefrontSize32 ) ) return <NUM_LIT> ;


<s> return alignTo ( FlatWorkGroupSize , getWavefrontSize ( STI ) ) / getWavefrontSize ( STI ) ;


<s> return ! STI . getFeatureBits ( ) [ <STR_LIT> ] ;


<s> bool hasSRAMECC ( const MCSubtargetInfo & STI ) {

return STI . getFeatureBits ( ) [ <STR_LIT> ] ;


<s> bool hasSRAMECC ( const MCSubtargetInfo & STI ) { return STI . getFeatureBits ( ) [ <STR_LIT> ] ;


<s> return STI . getFeatureBits ( ) [ <STR_LIT> ] ;


<s> Header . amd_machine_version_stepping = Version . Stepping ; Header . kernel_code_entry_byte_offset = sizeof ( Header ) ; Header . wavefront_size = <NUM_LIT> ; Header . call_convention = - <NUM_LIT> ; Header . kernarg_segment_alignment = <NUM_LIT> ; Header . group_segment_alignment = <NUM_LIT> ;

Header . private_segment_alignment = <NUM_LIT> ;


<s> case CallingConv :: <ISA_LIT>_PS : case CallingConv :: <ISA_LIT>_CS : return F -> getAttributes ( ) . hasParamAttribute ( A -> getArgNo ( ) , Attribute :: InReg ) || F -> getAttributes ( ) . hasParamAttribute ( A -> getArgNo ( ) , Attribute :: ByVal ) ; default : return false ; }


<s> bool isCompute ( CallingConv :: ID cc ) {

return ! isShader ( cc ) || cc == CallingConv :: <ISA_LIT>_CS ;


<s> bool isGCN3Encoding ( const MCSubtargetInfo & STI ) {


<s> bool isInlinableLiteral16 ( int16_t Literal , bool HasInv2Pi ) { if ( ! HasInv2Pi ) return false ; if ( Literal >= - <NUM_LIT> && Literal <= <NUM_LIT> ) return true ; uint16_t Val = static_cast < uint16_t > ( Literal ) ; return Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> ;


<s> if ( ! HasInv2Pi ) return false ; if ( Literal >= - <NUM_LIT> && Literal <= <NUM_LIT> ) return true ; uint16_t Val = static_cast < uint16_t > ( Literal ) ; return Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> || Val == <NUM_LIT> ;


<s> if ( Literal >= - <NUM_LIT> && Literal <= <NUM_LIT> ) return true ; uint64_t Val = static_cast < uint64_t > ( Literal ) ;

return ( Val == DoubleToBits ( <NUM_LIT> ) ) || ( Val == DoubleToBits ( <NUM_LIT> ) ) || ( Val == DoubleToBits ( - <NUM_LIT> ) ) || ( Val == DoubleToBits ( <NUM_LIT> ) ) || ( Val == DoubleToBits ( - <NUM_LIT> ) ) || ( Val == DoubleToBits ( <NUM_LIT> ) ) || ( Val == DoubleToBits ( - <NUM_LIT> ) ) || ( Val == DoubleToBits ( <NUM_LIT> ) ) || ( Val == DoubleToBits ( - <NUM_LIT> ) ) || ( Val == <NUM_LIT> && HasInv2Pi ) ;


<s> bool isIntrinsicSourceOfDivergence ( unsigned IntrID ) { return lookupSourceOfDivergence ( IntrID ) ;


<s> bool isLegalSMRDImmOffset ( const MCSubtargetInfo & ST , int64_t ByteOffset ) { int64_t EncodedOffset = getSMRDEncodedOffset ( ST , ByteOffset ) ; return isGCN3Encoding ( ST ) ? isUInt < <NUM_LIT> > ( EncodedOffset ) : isUInt < <NUM_LIT> > ( EncodedOffset ) ;


<s> return GV -> getType ( ) -> getAddressSpace ( ) == <ISA_LIT>AS :: CONSTANT_ADDRESS || GV -> getType ( ) -> getAddressSpace ( ) == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ;


<s> for ( MCRegAliasIterator R ( Reg0 , TRI , true ) ; R . isValid ( ) ; ++ R ) {


<s> for ( MCRegAliasIterator R ( Reg0 , TRI , true ) ; R . isValid ( ) ; ++ R ) { if ( * R == Reg1 ) return true ; }


<s> switch ( cc ) { case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_LS :


<s> bool isShader ( CallingConv :: ID cc ) { switch ( cc ) { case CallingConv :: <ISA_LIT>_VS : case CallingConv :: <ISA_LIT>_LS :


<s> bool is<ISA_LIT>SrcInlinableOperand ( const MCInstrDesc & Desc , unsigned OpNo ) {


<s> unsigned OpType = Desc . OpInfo [ OpNo ] . OperandType ;

return OpType >= <STR_LIT> && OpType <= <STR_LIT> ;


<s> Dst |= ( Src << Shift ) & getBitMask ( Shift , Width ) ; return Dst ;


<s> return TT . getOS ( ) != Triple :: AMDHSA ;


<s> Imm = MaxImm ; } else { uint32_t High = ( Imm + Align ) & ~ <NUM_LIT> ; uint32_t Low = ( Imm + Align ) & <NUM_LIT> ; Imm = Low ; Overflow = High - Align ;


<s> auto Version = getIsaVersion ( STI -> getCPU ( ) ) ; Stream << TargetTriple . getArchName ( ) << '-' << TargetTriple . getVendorName ( ) << '-' << TargetTriple . getOSName ( ) << '-' << TargetTriple . getEnvironmentName ( ) << '-' << <STR_LIT> << Version . Major << Version . Minor << Version . Stepping ;

if ( hasXNACK ( * STI ) ) Stream << <STR_LIT> ;


<s> unsigned unpackBits ( unsigned Src , unsigned Shift , unsigned Width ) { return ( Src & getBitMask ( Shift , Width ) ) >> Shift ;


<s> if ( ! SPReg ) { const GCNSubtarget & ST = MIRBuilder . getMF ( ) . getSubtarget < GCNSubtarget > ( ) ; if ( ST . enableFlatScratch ( ) ) { SPReg = MIRBuilder . buildCopy ( PtrTy , MFI -> getStackPtrOffsetReg ( ) ) . getReg ( <NUM_LIT> ) ; } else { SPReg = MIRBuilder . buildInstr ( <STR_LIT> , { PtrTy } , { MFI -> getStackPtrOffsetReg ( ) } ) . getReg ( <NUM_LIT> ) ;


<s> return false ; } if ( any_of ( CallerF . args ( ) , [ ] ( const Argument & A ) { return A . hasByValAttr ( ) || A . hasSwiftErrorAttr ( ) ; } ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } if ( MF . getTarget ( ) . Options . GuaranteedTailCallOpt ) return canGuaranteeTCO ( CalleeCC ) && CalleeCC == CallerF . getCallingConv ( ) ; if ( ! doCallerAndCalleePassArgsTheSameWay ( Info , MF , InArgs ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; }

if ( ! areCalleeOutgoingArgsTailCallable ( Info , MF , OutArgs ) ) return false ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } MachineFunction & MF = MIRBuilder . getMF ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const Function & F = MF . getFunction ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; SmallVector < ArgInfo , <NUM_LIT> > OutArgs ; for ( auto & OrigArg : Info . OrigArgs ) splitToValueTypes ( OrigArg , OutArgs , DL , Info . CallConv ) ; SmallVector < ArgInfo , <NUM_LIT> > InArgs ; if ( Info . CanLowerReturn && ! Info . OrigRet . Ty -> isVoidTy ( ) ) splitToValueTypes ( Info . OrigRet , InArgs , DL , Info . CallConv ) ; bool CanTailCallOpt = isEligibleForTailCallOptimization ( MIRBuilder , Info , InArgs , OutArgs ) ; if ( Info . IsMustTailCall && ! CanTailCallOpt ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } Info . IsTailCall = CanTailCallOpt ; if ( CanTailCallOpt ) return lowerTailCall ( MIRBuilder , Info , OutArgs ) ; CCAssignFn * AssignFnFixed ; CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( Info . CallConv , TLI ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , false ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; MIB . addDef ( TRI -> getReturnAddressReg ( MF ) ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , Info . CallConv ) ;


<s> splitToValueTypes ( OrigArg , SplitArgs , DL , CC ) ; ++ Idx ; } if ( CC == CallingConv :: <ISA_LIT>_PS ) { if ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> || ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> && Info -> isP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ) ) { CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ; Info -> markP<ISA_LIT>nputEnabled ( <NUM_LIT> ) ; } if ( Subtarget . isAmdPalOS ( ) ) { unsigned PsInputBits = Info -> getP<ISA_LIT>nputAddr ( ) & Info -> getP<ISA_LIT>nputEnable ( ) ; if ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> || ( ( PsInputBits & <NUM_LIT> ) == <NUM_LIT> && ( PsInputBits >> <NUM_LIT> & <NUM_LIT> ) ) ) Info -> markP<ISA_LIT>nputEnabled ( countTrailingZeros ( Info -> getP<ISA_LIT>nputAddr ( ) , ZB_Undefined ) ) ; } } const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForCall ( CC , F . isVarArg ( ) ) ; if ( ! MBB . empty ( ) ) B . setInstr ( * MBB . begin ( ) ) ; if ( ! IsEntryFunc && ! IsGraphics ) { TLI . allocateSpecialInputVGPRsFixed ( CCInfo , MF , * TRI , * Info ) ; } IncomingValueAssigner Assigner ( AssignFn ) ; if ( ! determineAssignments ( Assigner , SplitArgs , CCInfo ) ) return false ; FormalArgHandler Handler ( B , MRI ) ; if ( ! handleAssignments ( Handler , SplitArgs , CCInfo , ArgLocs , B ) ) return false ; uint64_t StackOffset = Assigner . StackOffset ; if ( IsEntryFunc ) { TLI . allocateSystemSGPRs ( CCInfo , MF , * Info , CC , IsGraphics ) ; } else { if ( ! Subtarget . enableFlatScratch ( ) ) CCInfo . AllocateReg ( Info -> getScratchRSrcReg ( ) ) ; TLI . allocateSpecialInputSGPRs ( CCInfo , MF , * TRI , * Info ) ; } Info -> setBytesInStackArgArea ( StackOffset ) ; B . setMBB ( MBB ) ; return true ;


<s> MFI -> setIfReturnsVoid ( ! Val ) ; assert ( ! Val == VRegs . empty ( ) && <STR_LIT> ) ; CallingConv :: ID CC = B . getMF ( ) . getFunction ( ) . getCallingConv ( ) ; const bool IsShader = <STR_LIT> ( CC ) ; const bool IsWaveEnd = ( IsShader && MFI -> returnsVoid ( ) ) || <STR_LIT> ( CC ) ; if ( IsWaveEnd ) { B . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) ; return true ; } auto const & ST = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned ReturnOpc = <NUM_LIT> ; if ( IsShader ) ReturnOpc = <STR_LIT> ; else if ( CC == CallingConv :: <ISA_LIT>_Gfx ) ReturnOpc = <STR_LIT> ; else ReturnOpc = <STR_LIT> ; auto Ret = B . buildInstrNoInsert ( ReturnOpc ) ; Register ReturnAddrVReg ; if ( ReturnOpc == <STR_LIT> ) { ReturnAddrVReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Ret . addUse ( ReturnAddrVReg ) ; } else if ( ReturnOpc == <STR_LIT> ) { ReturnAddrVReg = MRI . createVirtualRegister ( & <STR_LIT> ) ;


<s> if ( ! IsSibCall ) { unsigned NumReusableBytes = FuncInfo -> getBytesInStackArgArea ( ) ; SmallVector < CCValAssign , <NUM_LIT> > OutLocs ; CCState OutInfo ( CalleeCC , false , MF , OutLocs , F . getContext ( ) ) ; OutgoingValueAssigner CalleeAssigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( CalleeAssigner , OutArgs , OutInfo ) ) return false ; NumBytes = alignTo ( OutInfo . getNextStackOffset ( ) , ST . getStackAlignment ( ) ) ; FPDiff = NumReusableBytes - NumBytes ; assert ( isAligned ( ST . getStackAlignment ( ) , FPDiff ) && <STR_LIT> ) ; } SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ; SmallVector < std :: pair < MCRegister , Register > , <NUM_LIT> > ImplicitArgRegs ; if ( Info . CallConv != CallingConv :: <ISA_LIT>_Gfx ) { if ( ! passSpecialInputs ( MIRBuilder , CCInfo , ImplicitArgRegs , Info ) ) return false ; } OutgoingValueAssigner Assigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( Assigner , OutArgs , CCInfo ) ) return false ; <ISA_LIT>OutgoingArgHandler Handler ( MIRBuilder , MRI , MIB , true , FPDiff ) ; if ( ! handleAssignments ( Handler , OutArgs , CCInfo , ArgLocs , MIRBuilder ) ) return false ; handleImplicitCallArguments ( MIRBuilder , MIB , ST , * FuncInfo , ImplicitArgRegs ) ; if ( ! IsSibCall ) { MIB -> getOperand ( <NUM_LIT> ) . setImm ( FPDiff ) ; CallSeqStart . addImm ( NumBytes ) . addImm ( <NUM_LIT> ) ;

MIRBuilder . buildInstr ( <STR_LIT> ) . addImm ( NumBytes ) . addImm ( <NUM_LIT> ) ;


<s> CCAssignFn * AssignFnFixed ; CCAssignFn * AssignFnVarArg ; std :: tie ( AssignFnFixed , AssignFnVarArg ) = getAssignFnsForCC ( CalleeCC , TLI ) ; MachineInstrBuilder CallSeqStart ; if ( ! IsSibCall ) CallSeqStart = MIRBuilder . buildInstr ( <STR_LIT> ) ; unsigned Opc = getCallOpcode ( MF , Info . Callee . isReg ( ) , true ) ; auto MIB = MIRBuilder . buildInstrNoInsert ( Opc ) ; if ( ! addCallTargetOperands ( MIB , MIRBuilder , Info ) ) return false ; MIB . addImm ( <NUM_LIT> ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const uint32_t * Mask = TRI -> getCallPreservedMask ( MF , CalleeCC ) ; MIB . addRegMask ( Mask ) ; int FPDiff = <NUM_LIT> ; unsigned NumBytes = <NUM_LIT> ; if ( ! IsSibCall ) { unsigned NumReusableBytes = FuncInfo -> getBytesInStackArgArea ( ) ; SmallVector < CCValAssign , <NUM_LIT> > OutLocs ; CCState OutInfo ( CalleeCC , false , MF , OutLocs , F . getContext ( ) ) ; OutgoingValueAssigner CalleeAssigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( CalleeAssigner , OutArgs , OutInfo ) ) return false ; NumBytes = alignTo ( OutInfo . getNextStackOffset ( ) , ST . getStackAlignment ( ) ) ; FPDiff = NumReusableBytes - NumBytes ; assert ( isAligned ( ST . getStackAlignment ( ) , FPDiff ) && <STR_LIT> ) ; } SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( Info . CallConv , Info . IsVarArg , MF , ArgLocs , F . getContext ( ) ) ; SmallVector < std :: pair < MCRegister , Register > , <NUM_LIT> > ImplicitArgRegs ; if ( Info . CallConv != CallingConv :: <ISA_LIT>_Gfx ) { if ( ! passSpecialInputs ( MIRBuilder , CCInfo , ImplicitArgRegs , Info ) ) return false ; } OutgoingValueAssigner Assigner ( AssignFnFixed , AssignFnVarArg ) ; if ( ! determineAssignments ( Assigner , OutArgs , CCInfo ) ) return false ; <ISA_LIT>OutgoingArgHandler Handler ( MIRBuilder , MRI , MIB , true , FPDiff ) ; if ( ! handleAssignments ( Handler , OutArgs , CCInfo , ArgLocs , MIRBuilder ) ) return false ; handleImplicitCallArguments ( MIRBuilder , MIB , ST , * FuncInfo , ImplicitArgRegs ) ; if ( ! IsSibCall ) { MIB -> getOperand ( <NUM_LIT> ) . setImm ( FPDiff ) ;

CallSeqStart . addImm ( NumBytes ) . addImm ( <NUM_LIT> ) ;


<s> } if ( IncomingArgY && ! IncomingArgY -> isMasked ( ) && CalleeArgInfo -> WorkItemIDY && NeedWorkItemIDY && ST . getMaxWorkitemID ( MF . getFunction ( ) , <NUM_LIT> ) != <NUM_LIT> ) { Register Y = MRI . createGenericVirtualRegister ( S32 ) ; LI -> loadInputValue ( Y , MIRBuilder , IncomingArgY , std :: get < <NUM_LIT> > ( WorkitemIDY ) , std :: get < <NUM_LIT> > ( WorkitemIDY ) ) ; Y = MIRBuilder . buildShl ( S32 , Y , MIRBuilder . buildConstant ( S32 , <NUM_LIT> ) ) . getReg ( <NUM_LIT> ) ; InputReg = InputReg ? MIRBuilder . buildOr ( S32 , InputReg , Y ) . getReg ( <NUM_LIT> ) : Y ; } if ( IncomingArgZ && ! IncomingArgZ -> isMasked ( ) && CalleeArgInfo -> WorkItemIDZ && NeedWorkItemIDZ && ST . getMaxWorkitemID ( MF . getFunction ( ) , <NUM_LIT> ) != <NUM_LIT> ) { Register Z = MRI . createGenericVirtualRegister ( S32 ) ; LI -> loadInputValue ( Z , MIRBuilder , IncomingArgZ , std :: get < <NUM_LIT> > ( WorkitemIDZ ) , std :: get < <NUM_LIT> > ( WorkitemIDZ ) ) ; Z = MIRBuilder . buildShl ( S32 , Z , MIRBuilder . buildConstant ( S32 , <NUM_LIT> ) ) . getReg ( <NUM_LIT> ) ; InputReg = InputReg ? MIRBuilder . buildOr ( S32 , InputReg , Z ) . getReg ( <NUM_LIT> ) : Z ; } if ( ! InputReg && ( NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ ) ) { InputReg = MRI . createGenericVirtualRegister ( S32 ) ; if ( ! IncomingArgX && ! IncomingArgY && ! IncomingArgZ ) { MIRBuilder . buildUndef ( InputReg ) ; } else { ArgDescriptor IncomingArg = ArgDescriptor :: createArg ( IncomingArgX ? * IncomingArgX : IncomingArgY ? * IncomingArgY : * IncomingArgZ , ~ <NUM_LIT> ) ; LI -> loadInputValue ( InputReg , MIRBuilder , & IncomingArg , & <STR_LIT> , S32 ) ; } } if ( OutgoingArg -> isRegister ( ) ) { if ( InputReg ) ArgRegs . emplace_back ( OutgoingArg -> getRegister ( ) , InputReg ) ; if ( ! CCInfo . AllocateReg ( OutgoingArg -> getRegister ( ) ) ) report_fatal_error ( <STR_LIT> ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } return true ;


<s> if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , ArgTy ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) std :: tie ( OutgoingArg , ArgRC , ArgTy ) = CalleeArgInfo -> getPreloadedValue ( <STR_LIT> ) ; if ( ! OutgoingArg ) return false ; auto WorkitemIDX = CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ; auto WorkitemIDY = CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ; auto WorkitemIDZ = CallerArgInfo . getPreloadedValue ( <STR_LIT> ) ; const ArgDescriptor * IncomingArgX = std :: get < <NUM_LIT> > ( WorkitemIDX ) ; const ArgDescriptor * IncomingArgY = std :: get < <NUM_LIT> > ( WorkitemIDY ) ; const ArgDescriptor * IncomingArgZ = std :: get < <NUM_LIT> > ( WorkitemIDZ ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const bool NeedWorkItemIDX = ! Info . CB -> hasFnAttr ( <STR_LIT> ) ; const bool NeedWorkItemIDY = ! Info . CB -> hasFnAttr ( <STR_LIT> ) ; const bool NeedWorkItemIDZ = ! Info . CB -> hasFnAttr ( <STR_LIT> ) ; Register InputReg ; if ( IncomingArgX && ! IncomingArgX -> isMasked ( ) && CalleeArgInfo -> WorkItemIDX && NeedWorkItemIDX ) { InputReg = MRI . createGenericVirtualRegister ( S32 ) ; LI -> loadInputValue ( InputReg , MIRBuilder , IncomingArgX , std :: get < <NUM_LIT> > ( WorkitemIDX ) , std :: get < <NUM_LIT> > ( WorkitemIDX ) ) ; } if ( IncomingArgY && ! IncomingArgY -> isMasked ( ) && CalleeArgInfo -> WorkItemIDY && NeedWorkItemIDY ) { Register Y = MRI . createGenericVirtualRegister ( S32 ) ; LI -> loadInputValue ( Y , MIRBuilder , IncomingArgY , std :: get < <NUM_LIT> > ( WorkitemIDY ) , std :: get < <NUM_LIT> > ( WorkitemIDY ) ) ; Y = MIRBuilder . buildShl ( S32 , Y , MIRBuilder . buildConstant ( S32 , <NUM_LIT> ) ) . getReg ( <NUM_LIT> ) ; InputReg = InputReg ? MIRBuilder . buildOr ( S32 , InputReg , Y ) . getReg ( <NUM_LIT> ) : Y ; } if ( IncomingArgZ && ! IncomingArgZ -> isMasked ( ) && CalleeArgInfo -> WorkItemIDZ && NeedWorkItemIDZ ) { Register Z = MRI . createGenericVirtualRegister ( S32 ) ; LI -> loadInputValue ( Z , MIRBuilder , IncomingArgZ , std :: get < <NUM_LIT> > ( WorkitemIDZ ) , std :: get < <NUM_LIT> > ( WorkitemIDZ ) ) ; Z = MIRBuilder . buildShl ( S32 , Z , MIRBuilder . buildConstant ( S32 , <NUM_LIT> ) ) . getReg ( <NUM_LIT> ) ; InputReg = InputReg ? MIRBuilder . buildOr ( S32 , InputReg , Z ) . getReg ( <NUM_LIT> ) : Z ; } if ( ! InputReg && ( NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ ) ) {


<s> <ISA_LIT>CallLowering :: <ISA_LIT>CallLowering ( const <ISA_LIT>TargetLowering & TLI ) : CallLowering ( & TLI ) , AMDGPUA<ISA_LIT> ( TLI . get<ISA_LIT>AS ( ) ) {


<s> unsigned DispatchIDReg = Info -> addDispatchID ( * TRI ) ; CCInfo . AllocateReg ( DispatchIDReg ) ; } if ( Info -> hasFlatScratchInit ( ) ) { unsigned FlatScratchInitReg = Info -> addFlatScratchInit ( * TRI ) ; CCInfo . AllocateReg ( FlatScratchInitReg ) ; } unsigned NumArgs = F . arg_size ( ) ; Function :: const_arg_iterator CurOrigArg = F . arg_begin ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ CurOrigArg ) { EVT ValEVT = TLI . getValueType ( DL , CurOrigArg -> getType ( ) ) ; if ( ! ValEVT . isSimple ( ) ) return false ; MVT ValVT = ValEVT . getSimpleVT ( ) ; <STR_LIT> Flags ; ArgInfo OrigArg { VRegs [ i ] , CurOrigArg -> getType ( ) } ; setArgFlags ( OrigArg , i + <NUM_LIT> , DL , F ) ; Flags . setOrigAlign ( DL . getABITypeAlignment ( CurOrigArg -> getType ( ) ) ) ; CCAssignFn * AssignFn = CCAssignFnForCall ( F . getCallingConv ( ) , false ) ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( Res ) return false ; } Function :: const_arg_iterator Arg = F . arg_begin ( ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_VS ) { for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ Arg ) { CCValAssign & VA = ArgLocs [ i ] ; MRI . addLiveIn ( VA . getLocReg ( ) , VRegs [ i ] ) ;

MIRBuilder . getMBB ( ) . addLiveIn ( VA . getLocReg ( ) ) ;


<s> unsigned Align = DL . getABITypeAlignment ( ParamTy ) ; unsigned PtrReg = lowerParameterPtr ( MIRBuilder , ParamTy , Offset ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MONonTemporal | MachineMemOperand :: MOInvariant , TypeSize , Align ) ; MIRBuilder . buildLoad ( DstReg , PtrReg , * MMO ) ;


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = * MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; unsigned KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ;

unsigned OffsetReg = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ;


<s> } if ( Info -> hasFlatScratchInit ( ) ) { unsigned FlatScratchInitReg = Info -> addFlatScratchInit ( * TRI ) ; CCInfo . AllocateReg ( FlatScratchInitReg ) ; } unsigned NumArgs = F . arg_size ( ) ; Function :: const_arg_iterator CurOrigArg = F . arg_begin ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; unsigned P<ISA_LIT>nputNum = <NUM_LIT> ; BitVector Skipped ( NumArgs ) ; for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ CurOrigArg ) { EVT ValEVT = TLI . getValueType ( DL , CurOrigArg -> getType ( ) ) ; <STR_LIT> Flags ; ArgInfo OrigArg { VRegs [ i ] , CurOrigArg -> getType ( ) } ; setArgFlags ( OrigArg , i + <NUM_LIT> , DL , F ) ; Flags . setOrigAlign ( DL . getABITypeAlignment ( CurOrigArg -> getType ( ) ) ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && ! OrigArg . Flags . isInReg ( ) && ! OrigArg . Flags . isByVal ( ) && P<ISA_LIT>nputNum <= <NUM_LIT> ) { if ( CurOrigArg -> use_empty ( ) && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ) { Skipped . set ( i ) ; ++ P<ISA_LIT>nputNum ; continue ; } Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! CurOrigArg -> use_empty ( ) ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; ++ P<ISA_LIT>nputNum ; } CCAssignFn * AssignFn = CCAssignFnForCall ( F . getCallingConv ( ) , false ) ; if ( ValEVT . isVector ( ) ) { EVT ElemVT = ValEVT . getVectorElementType ( ) ; if ( ! ValEVT . isSimple ( ) ) return false ; MVT ValVT = ElemVT . getSimpleVT ( ) ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( ! Res ) return false ; } else { MVT ValVT = ValEVT . getSimpleVT ( ) ; if ( ! ValEVT . isSimple ( ) ) return false ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ;


<s> const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ; unsigned Align = DL . getABITypeAlignment ( ParamTy ) ;

unsigned PtrReg = lowerParameterPtr ( MIRBuilder , ParamTy , Offset ) ;


<s> PointerType * PtrTy = PointerType :: get ( ParamTy , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; unsigned KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; unsigned OffsetReg = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildConstant ( OffsetReg , Offset ) ; MIRBuilder . buildGEP ( DstReg , KernArgSegmentVReg , OffsetReg ) ;


<s> unsigned KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; unsigned OffsetReg = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildConstant ( OffsetReg , Offset ) ; MIRBuilder . buildGEP ( DstReg , KernArgSegmentVReg , OffsetReg ) ; return DstReg ;


<s> bool <ISA_LIT>CallLowering :: lowerReturn ( MachineIRBuilder & MIRBuilder , const Value * Val , unsigned VReg ) const { if ( Val ) return false ;


<s> const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ;


<s> void <ISA_LIT>CallLowering :: lowerParameter ( MachineIRBuilder & MIRBuilder , Type * ParamTy , uint64_t Offset , unsigned Align , unsigned DstReg ) const { MachineFunction & MF = MIRBuilder . getMF ( ) ; const Function & F = MF . getFunction ( ) ;

const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ;


<s> const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; unsigned DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; unsigned KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ;


<s> if ( Val ) return false ; MIRBuilder . buildInstr ( <STR_LIT> ) ; return true ;


<s> if ( Val ) return false ; MIRBuilder . buildInstr ( <STR_LIT> ) ;


<s> for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ CurOrigArg ) { EVT ValEVT = TLI . getValueType ( DL , CurOrigArg -> getType ( ) ) ; if ( ! ValEVT . isSimple ( ) ) return false ; MVT ValVT = ValEVT . getSimpleVT ( ) ; <STR_LIT> Flags ; ArgInfo OrigArg { VRegs [ i ] , CurOrigArg -> getType ( ) } ; setArgFlags ( OrigArg , i + <NUM_LIT> , DL , F ) ; Flags . setOrigAlign ( DL . getABITypeAlignment ( CurOrigArg -> getType ( ) ) ) ; CCAssignFn * AssignFn = CCAssignFnForCall ( F . getCallingConv ( ) , false ) ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( Res ) return false ; } Function :: const_arg_iterator Arg = F . arg_begin ( ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_VS ) { for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ Arg ) { CCValAssign & VA = ArgLocs [ i ] ; MRI . addLiveIn ( VA . getLocReg ( ) , VRegs [ i ] ) ; MIRBuilder . getMBB ( ) . addLiveIn ( VA . getLocReg ( ) ) ; MIRBuilder . buildCopy ( VRegs [ i ] , VA . getLocReg ( ) ) ; } return true ; } for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ Arg ) { CCValAssign & VA = ArgLocs [ i ] ;

lowerParameter ( MIRBuilder , Arg -> getType ( ) , VA . getLocMemOffset ( ) + Subtarget -> getExplicitKernelArgOffset ( MF ) , VRegs [ i ] ) ;


<s> MIRBuilder . getMBB ( ) . addLiveIn ( InputPtrReg ) ; MIRBuilder . buildCopy ( VReg , InputPtrReg ) ; CCInfo . AllocateReg ( InputPtrReg ) ; } if ( Info -> hasDispatchID ( ) ) { unsigned DispatchIDReg = Info -> addDispatchID ( * TRI ) ; CCInfo . AllocateReg ( DispatchIDReg ) ; } if ( Info -> hasFlatScratchInit ( ) ) { unsigned FlatScratchInitReg = Info -> addFlatScratchInit ( * TRI ) ; CCInfo . AllocateReg ( FlatScratchInitReg ) ; } unsigned NumArgs = F . arg_size ( ) ; Function :: const_arg_iterator CurOrigArg = F . arg_begin ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ CurOrigArg ) { EVT ValEVT = TLI . getValueType ( DL , CurOrigArg -> getType ( ) ) ; if ( ! ValEVT . isSimple ( ) ) return false ; MVT ValVT = ValEVT . getSimpleVT ( ) ; <STR_LIT> Flags ; ArgInfo OrigArg { VRegs [ i ] , CurOrigArg -> getType ( ) } ; setArgFlags ( OrigArg , i + <NUM_LIT> , DL , F ) ; Flags . setOrigAlign ( DL . getABITypeAlignment ( CurOrigArg -> getType ( ) ) ) ; CCAssignFn * AssignFn = CCAssignFnForCall ( F . getCallingConv ( ) , false ) ; bool Res = AssignFn ( i , ValVT , ValVT , CCValAssign :: Full , OrigArg . Flags , CCInfo ) ; if ( Res ) return false ; } Function :: const_arg_iterator Arg = F . arg_begin ( ) ; if ( F . getCallingConv ( ) == CallingConv :: <ISA_LIT>_VS ) { for ( unsigned i = <NUM_LIT> ; i != NumArgs ; ++ i , ++ Arg ) { CCValAssign & VA = ArgLocs [ i ] ; MRI . addLiveIn ( VA . getLocReg ( ) , VRegs [ i ] ) ; MIRBuilder . getMBB ( ) . addLiveIn ( VA . getLocReg ( ) ) ; MIRBuilder . buildCopy ( VRegs [ i ] , VA . getLocReg ( ) ) ; } return true ; }


<s> return AssignFn ( ValNo , ValVT , LocVT , LocInfo , Flags , State ) ;


<s> void assignValueToAddress ( Register ValVReg , Register Addr , uint64_t Size , MachinePointerInfo & MPO , CCValAssign & VA ) override {

MachineFunction & MF = MIRBuilder . getMF ( ) ;


<s> void assignValueToReg ( Register ValVReg , Register PhysReg , CCValAssign & VA ) override { Register ExtReg ; if ( VA . getLocVT ( ) . getSizeInBits ( ) < <NUM_LIT> ) {

ExtReg = MIRBuilder . buildAnyExt ( LLT :: scalar ( <NUM_LIT> ) , ValVReg ) . getReg ( <NUM_LIT> ) ;


<s> } else ExtReg = extendRegister ( ValVReg , VA ) ; const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; if ( TRI -> isSGPRReg ( MRI , PhysReg ) ) { auto ToSGPR = MIRBuilder . buildIntrinsic ( <STR_LIT> , { MRI . getType ( ExtReg ) } , false ) . addReg ( ExtReg ) ; ExtReg = ToSGPR . getReg ( <NUM_LIT> ) ; }

MIRBuilder . buildCopy ( PhysReg , ExtReg ) ;


<s> FormalArgHandler ( MachineIRBuilder & B , MachineRegisterInfo & MRI , CCAssignFn * AssignFn ) : IncomingArgHandler ( B , MRI , AssignFn ) {


<s> auto & MFI = MIRBuilder . getMF ( ) . getFrameInfo ( ) ; int FI = MFI . CreateFixedObject ( Size , Offset , true ) ; MPO = MachinePointerInfo :: getFixedStack ( MIRBuilder . getMF ( ) , FI ) ; auto AddrReg = MIRBuilder . buildFrameIndex ( LLT :: pointer ( <ISA_LIT>AS :: PRIVATE_ADDRESS , <NUM_LIT> ) , FI ) ; StackUsed = std :: max ( StackUsed , Size + Offset ) ; return AddrReg . getReg ( <NUM_LIT> ) ;


<s> auto AddrReg = MIRBuilder . buildFrameIndex ( LLT :: pointer ( <ISA_LIT>AS :: PRIVATE_ADDRESS , <NUM_LIT> ) , FI ) ;

StackUsed = std :: max ( StackUsed , Size + Offset ) ;


<s> IncomingArgHandler ( MachineIRBuilder & B , MachineRegisterInfo & MRI , CCAssignFn * AssignFn ) : ValueHandler ( B , MRI , AssignFn ) {


<s> bool isIncomingArgumentHandler ( ) const override {


<s> for ( auto & Arg : F . args ( ) ) { if ( DL . getTypeStoreSize ( Arg . getType ( ) ) == <NUM_LIT> ) continue ; const bool InReg = Arg . hasAttribute ( Attribute :: InReg ) ; if ( ! IsShader && InReg ) return false ; if ( Arg . hasAttribute ( Attribute :: SwiftSelf ) || Arg . hasAttribute ( Attribute :: SwiftError ) || Arg . hasAttribute ( Attribute :: Nest ) ) return false ; if ( CC == CallingConv :: <ISA_LIT>_PS && ! InReg && P<ISA_LIT>nputNum <= <NUM_LIT> ) { const bool ArgUsed = ! Arg . use_empty ( ) ; bool SkipArg = ! ArgUsed && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! SkipArg ) { Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ArgUsed ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; } ++ P<ISA_LIT>nputNum ; if ( SkipArg ) { for ( int I = <NUM_LIT> , E = VRegs [ Idx ] . size ( ) ; I != E ; ++ I ) B . buildUndef ( VRegs [ Idx ] [ I ] ) ; ++ Idx ; continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg . getType ( ) ) ; const unsigned OrigArgIdx = Idx + AttributeList :: FirstArgIndex ; setArgFlags ( OrigArg , OrigArgIdx , DL , F ) ; splitToValueTypes ( B , OrigArg , OrigArgIdx , SplitArgs , DL , CC , [ & ] ( ArrayRef < Register > Regs , Register DstReg , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { assert ( DstReg == VRegs [ Idx ] [ VTSplitIdx ] ) ; packSplitRegsToOrigType ( B , VRegs [ Idx ] [ VTSplitIdx ] , Regs , LLTy , PartLLT ) ; } ) ; ++ Idx ; } if ( CC == CallingConv :: <ISA_LIT>_PS ) { if ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> || ( ( Info -> getP<ISA_LIT>nputAddr ( ) & <NUM_LIT> ) == <NUM_LIT> && Info -> isP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ) ) { CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ;

Info -> markP<ISA_LIT>nputAllocated ( <NUM_LIT> ) ;


<s> const Align KernArgBaseAlign ( <NUM_LIT> ) ; const unsigned BaseOffset = Subtarget -> getExplicitKernelArgOffset ( F ) ; uint64_t ExplicitArgOffset = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { Type * ArgTy = Arg . getType ( ) ; unsigned AllocSize = DL . getTypeAllocSize ( ArgTy ) ; if ( AllocSize == <NUM_LIT> ) continue ; unsigned ABIAlign = DL . getABITypeAlignment ( ArgTy ) ; uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; ArrayRef < Register > OrigArgRegs = VRegs [ i ] ; Register ArgReg = OrigArgRegs . size ( ) == <NUM_LIT> ? OrigArgRegs [ <NUM_LIT> ] : MRI . createGenericVirtualRegister ( getLLTForType ( * ArgTy , DL ) ) ; Align Alignment = commonAlignment ( KernArgBaseAlign , ArgOffset ) ; ArgOffset = alignTo ( ArgOffset , DL . getABITypeAlignment ( ArgTy ) ) ; lowerParameter ( B , ArgTy , ArgOffset , Alignment , ArgReg ) ; if ( OrigArgRegs . size ( ) > <NUM_LIT> ) unpackRegs ( OrigArgRegs , ArgReg , ArgTy , B ) ;


<s> const Align KernArgBaseAlign ( <NUM_LIT> ) ; const unsigned BaseOffset = Subtarget -> getExplicitKernelArgOffset ( F ) ; uint64_t ExplicitArgOffset = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { Type * ArgTy = Arg . getType ( ) ; unsigned AllocSize = DL . getTypeAllocSize ( ArgTy ) ; if ( AllocSize == <NUM_LIT> ) continue ; unsigned ABIAlign = DL . getABITypeAlignment ( ArgTy ) ; uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; ArrayRef < Register > OrigArgRegs = VRegs [ i ] ; Register ArgReg = OrigArgRegs . size ( ) == <NUM_LIT> ? OrigArgRegs [ <NUM_LIT> ] : MRI . createGenericVirtualRegister ( getLLTForType ( * ArgTy , DL ) ) ; Align Alignment = commonAlignment ( KernArgBaseAlign , ArgOffset ) ; ArgOffset = alignTo ( ArgOffset , DL . getABITypeAlignment ( ArgTy ) ) ; lowerParameter ( B , ArgTy , ArgOffset , Alignment , ArgReg ) ; if ( OrigArgRegs . size ( ) > <NUM_LIT> ) unpackRegs ( OrigArgRegs , ArgReg , ArgTy , B ) ; ++ i ; } TLI . allocateSpecialEntryInputVGPRs ( CCInfo , MF , * TRI , * Info ) ; TLI . allocateSystemSGPRs ( CCInfo , MF , * Info , F . getCallingConv ( ) , false ) ;


<s> const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ;

unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ;


<s> MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , TypeSize , Alignment ) ; B . buildLoad ( DstReg , PtrReg , * MMO ) ;


<s> Register <ISA_LIT>CallLowering :: lowerParameterPtr ( MachineIRBuilder & B , Type * ParamTy , uint64_t Offset ) const { MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; Register KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ; Register KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ; auto OffsetReg = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , Offset ) ;

return B . buildPtrAdd ( PtrType , KernArgSegmentVReg , OffsetReg ) . getReg ( <NUM_LIT> ) ;


<s> auto & MF = B . getMF ( ) ; const auto & F = MF . getFunction ( ) ; const DataLayout & DL = MF . getDataLayout ( ) ; MachineRegisterInfo * MRI = B . getMRI ( ) ; CallingConv :: ID CC = F . getCallingConv ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; ArgInfo OrigRetInfo ( VRegs , Val -> getType ( ) ) ; setArgFlags ( OrigRetInfo , AttributeList :: ReturnIndex , DL , F ) ; SmallVector < ArgInfo , <NUM_LIT> > SplitRetInfos ; splitToValueTypes ( B , OrigRetInfo , AttributeList :: ReturnIndex , SplitRetInfos , DL , CC , [ & ] ( ArrayRef < Register > Regs , Register SrcReg , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { unpackRegsToOrigType ( B , Regs , SrcReg , SplitRetInfos [ VTSplitIdx ] , LLTy , PartLLT ) ; } ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForReturn ( CC , F . isVarArg ( ) ) ; OutgoingValueHandler RetHandler ( B , * MRI , Ret , AssignFn ) ;

return handleAssignments ( B , SplitRetInfos , RetHandler ) ;


<s> int NumDst = LCMTy . getSizeInBits ( ) / LLTy . getSizeInBits ( ) ; SmallVector < Register , <NUM_LIT> > PadDstRegs ( NumDst ) ; std :: copy ( DstRegs . begin ( ) , DstRegs . end ( ) , PadDstRegs . begin ( ) ) ; for ( int I = DstRegs . size ( ) ; I != NumDst ; ++ I ) PadDstRegs [ I ] = MRI . createGenericVirtualRegister ( LLTy ) ;

return B . buildUnmerge ( PadDstRegs , Widened ) ;


<s> OutgoingValueHandler ( MachineIRBuilder & B , MachineRegisterInfo & MRI , MachineInstrBuilder MIB , CCAssignFn * AssignFn ) : ValueHandler ( B , MRI , AssignFn ) , MIB ( MIB ) {


<s> B . buildTrunc ( OrigRegs [ <NUM_LIT> ] , Widened ) ; } return ; } if ( LLTy . isVector ( ) && PartLLT . isVector ( ) ) { assert ( OrigRegs . size ( ) == <NUM_LIT> ) ; assert ( LLTy . getElementType ( ) == PartLLT . getElementType ( ) ) ; mergeVectorRegsToResultRegs ( B , OrigRegs , Regs ) ; return ; } assert ( LLTy . isVector ( ) && ! PartLLT . isVector ( ) ) ; LLT DstEltTy = LLTy . getElementType ( ) ; LLT RealDstEltTy = MRI . getType ( OrigRegs [ <NUM_LIT> ] ) . getElementType ( ) ; assert ( DstEltTy . getSizeInBits ( ) == RealDstEltTy . getSizeInBits ( ) ) ; if ( DstEltTy == PartLLT ) { if ( RealDstEltTy . isPointer ( ) ) { for ( Register Reg : Regs ) MRI . setType ( Reg , RealDstEltTy ) ; } B . buildBuildVector ( OrigRegs [ <NUM_LIT> ] , Regs ) ; } else if ( DstEltTy . getSizeInBits ( ) > PartLLT . getSizeInBits ( ) ) { SmallVector < Register , <NUM_LIT> > EltMerges ; int PartsPerElt = DstEltTy . getSizeInBits ( ) / PartLLT . getSizeInBits ( ) ; assert ( DstEltTy . getSizeInBits ( ) % PartLLT . getSizeInBits ( ) == <NUM_LIT> ) ; for ( int I = <NUM_LIT> , NumElts = LLTy . getNumElements ( ) ; I != NumElts ; ++ I ) { auto Merge = B . buildMerge ( RealDstEltTy , Regs . take_front ( PartsPerElt ) ) ; MRI . setType ( Merge . getReg ( <NUM_LIT> ) , RealDstEltTy ) ;

EltMerges . push_back ( Merge . getReg ( <NUM_LIT> ) ) ;


<s> static void packSplitRegsToOrigType ( MachineIRBuilder & B , ArrayRef < Register > OrigRegs , ArrayRef < Register > Regs , LLT LLTy , LLT PartLLT ) { MachineRegisterInfo & MRI = * B . getMRI ( ) ; if ( ! LLTy . isVector ( ) && ! PartLLT . isVector ( ) ) { assert ( OrigRegs . size ( ) == <NUM_LIT> ) ; LLT OrigTy = MRI . getType ( OrigRegs [ <NUM_LIT> ] ) ; unsigned SrcSize = PartLLT . getSizeInBits ( ) * Regs . size ( ) ; if ( SrcSize == OrigTy . getSizeInBits ( ) ) B . buildMerge ( OrigRegs [ <NUM_LIT> ] , Regs ) ; else { auto Widened = B . buildMerge ( LLT :: scalar ( SrcSize ) , Regs ) ; B . buildTrunc ( OrigRegs [ <NUM_LIT> ] , Widened ) ; } return ; } if ( LLTy . isVector ( ) && PartLLT . isVector ( ) ) { assert ( OrigRegs . size ( ) == <NUM_LIT> ) ; assert ( LLTy . getElementType ( ) == PartLLT . getElementType ( ) ) ; mergeVectorRegsToResultRegs ( B , OrigRegs , Regs ) ; return ; } assert ( LLTy . isVector ( ) && ! PartLLT . isVector ( ) ) ; LLT DstEltTy = LLTy . getElementType ( ) ; LLT RealDstEltTy = MRI . getType ( OrigRegs [ <NUM_LIT> ] ) . getElementType ( ) ; assert ( DstEltTy . getSizeInBits ( ) == RealDstEltTy . getSizeInBits ( ) ) ; if ( DstEltTy == PartLLT ) { if ( RealDstEltTy . isPointer ( ) ) { for ( Register Reg : Regs ) MRI . setType ( Reg , RealDstEltTy ) ; } B . buildBuildVector ( OrigRegs [ <NUM_LIT> ] , Regs ) ; } else if ( DstEltTy . getSizeInBits ( ) > PartLLT . getSizeInBits ( ) ) { SmallVector < Register , <NUM_LIT> > EltMerges ; int PartsPerElt = DstEltTy . getSizeInBits ( ) / PartLLT . getSizeInBits ( ) ; assert ( DstEltTy . getSizeInBits ( ) % PartLLT . getSizeInBits ( ) == <NUM_LIT> ) ; for ( int I = <NUM_LIT> , NumElts = LLTy . getNumElements ( ) ; I != NumElts ; ++ I ) {

auto Merge = B . buildMerge ( RealDstEltTy , Regs . take_front ( PartsPerElt ) ) ;


<s> ComputeValueVTs ( TLI , DL , OrigArg . Ty , SplitVTs ) ; assert ( OrigArg . Regs . size ( ) == SplitVTs . size ( ) ) ; int SplitIdx = <NUM_LIT> ; for ( EVT VT : SplitVTs ) { Register Reg = OrigArg . Regs [ SplitIdx ] ; Type * Ty = VT . getTypeForEVT ( Ctx ) ; LLT LLTy = getLLTForType ( * Ty , DL ) ; if ( OrigArgIdx == AttributeList :: ReturnIndex && VT . isScalarInteger ( ) ) { unsigned ExtendOp = TargetOpcode :: G_ANYEXT ; if ( OrigArg . Flags [ <NUM_LIT> ] . isSExt ( ) ) { assert ( OrigArg . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_SEXT ; } else if ( OrigArg . Flags [ <NUM_LIT> ] . isZExt ( ) ) { assert ( OrigArg . Regs . size ( ) == <NUM_LIT> && <STR_LIT> ) ; ExtendOp = TargetOpcode :: G_ZEXT ; } EVT ExtVT = TLI . getTypeForExtReturn ( Ctx , VT , extOpcodeToISDExtOpcode ( ExtendOp ) ) ; if ( ExtVT != VT ) { VT = ExtVT ; Ty = ExtVT . getTypeForEVT ( Ctx ) ; LLTy = getLLTForType ( * Ty , DL ) ; Reg = B . buildInstr ( ExtendOp , { LLTy } , { Reg } ) . getReg ( <NUM_LIT> ) ; }


<s> const unsigned SrcSize = SrcTy . getSizeInBits ( ) ; const unsigned PartSize = PartTy . getSizeInBits ( ) ; if ( SrcTy . isVector ( ) && ! PartTy . isVector ( ) && PartSize > SrcTy . getElementType ( ) . getSizeInBits ( ) ) { auto UnmergeToEltTy = B . buildUnmerge ( SrcTy . getElementType ( ) , SrcReg ) ; for ( int i = <NUM_LIT> , e = DstRegs . size ( ) ; i != e ; ++ i ) B . buildAnyExt ( DstRegs [ i ] , UnmergeToEltTy . getReg ( i ) ) ; return ; } if ( SrcSize % PartSize == <NUM_LIT> ) {


<s> const LLT P4 = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register VReg = MRI . createGenericVirtualRegister ( P4 ) ; MRI . addLiveIn ( InputPtrReg , VReg ) ; B . getMBB ( ) . addLiveIn ( InputPtrReg ) ; B . buildCopy ( VReg , InputPtrReg ) ; CCInfo . AllocateReg ( InputPtrReg ) ; } if ( Info . hasDispatchID ( ) ) { Register DispatchIDReg = Info . addDispatchID ( TRI ) ; MF . addLiveIn ( DispatchIDReg , & <STR_LIT> ) ; CCInfo . AllocateReg ( DispatchIDReg ) ;


<s> const bool IsGraphics = <STR_LIT> ( CC ) ; const bool IsEntryFunc = <STR_LIT> ( CC ) ; MachineFunction & MF = B . getMF ( ) ; MachineBasicBlock & MBB = B . getMBB ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const GCNSubtarget & Subtarget = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget . getRegisterInfo ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; Info -> allocateModuleLDSGlobal ( F . getParent ( ) ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( CC , F . isVarArg ( ) , MF , ArgLocs , F . getContext ( ) ) ; if ( Info -> hasImplicitBufferPtr ( ) ) { Register ImplicitBufferPtrReg = Info -> addImplicitBufferPtr ( * TRI ) ; MF . addLiveIn ( ImplicitBufferPtrReg , & <STR_LIT> ) ; CCInfo . AllocateReg ( ImplicitBufferPtrReg ) ; } if ( Info -> hasFlatScratchInit ( ) && ! Subtarget . isAmdPalOS ( ) ) { Register FlatScratchInitReg = Info -> addFlatScratchInit ( * TRI ) ; MF . addLiveIn ( FlatScratchInitReg , & <STR_LIT> ) ; CCInfo . AllocateReg ( FlatScratchInitReg ) ; } SmallVector < ArgInfo , <NUM_LIT> > SplitArgs ; unsigned Idx = <NUM_LIT> ; unsigned P<ISA_LIT>nputNum = <NUM_LIT> ; if ( ! FLI . CanLowerReturn ) insertSRetIncomingArgument ( F , SplitArgs , FLI . DemoteRegister , MRI , DL ) ; for ( auto & Arg : F . args ( ) ) { if ( DL . getTypeStoreSize ( Arg . getType ( ) ) == <NUM_LIT> ) continue ; const bool InReg = Arg . hasAttribute ( Attribute :: InReg ) ; if ( ! IsGraphics && InReg ) return false ; if ( Arg . hasAttribute ( Attribute :: SwiftSelf ) || Arg . hasAttribute ( Attribute :: SwiftError ) || Arg . hasAttribute ( Attribute :: Nest ) ) return false ; if ( CC == CallingConv :: <ISA_LIT>_PS && ! InReg && P<ISA_LIT>nputNum <= <NUM_LIT> ) { const bool ArgUsed = ! Arg . use_empty ( ) ; bool SkipArg = ! ArgUsed && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! SkipArg ) { Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ArgUsed ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ; } ++ P<ISA_LIT>nputNum ; if ( SkipArg ) { for ( Register R : VRegs [ Idx ] ) B . buildUndef ( R ) ; ++ Idx ; continue ; } } ArgInfo OrigArg ( VRegs [ Idx ] , Arg , Idx ) ; const unsigned OrigArgIdx = Idx + AttributeList :: FirstArgIndex ; setArgFlags ( OrigArg , OrigArgIdx , DL , F ) ; splitToValueTypes ( OrigArg , SplitArgs , DL , CC ) ; ++ Idx ;


<s> CallingConv :: ID CC = B . getMF ( ) . getFunction ( ) . getCallingConv ( ) ; const bool IsShader = <STR_LIT> ( CC ) ; const bool IsWaveEnd = ( IsShader && MFI -> returnsVoid ( ) ) || <STR_LIT> ( CC ) ; if ( IsWaveEnd ) { B . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) ; return true ; } unsigned ReturnOpc = IsShader ? <STR_LIT> : <STR_LIT> ; auto Ret = B . buildInstrNoInsert ( ReturnOpc ) ; if ( ! FLI . CanLowerReturn ) insertSRetStores ( B , Val -> getType ( ) , VRegs , FLI . DemoteRegister ) ; else if ( ! lowerReturnVal ( B , Val , VRegs , Ret ) ) return false ; B . insertInstr ( Ret ) ;


<s> assert ( ! Val == VRegs . empty ( ) && <STR_LIT> ) ; CallingConv :: ID CC = B . getMF ( ) . getFunction ( ) . getCallingConv ( ) ; const bool IsShader = <STR_LIT> ( CC ) ; const bool IsWaveEnd = ( IsShader && MFI -> returnsVoid ( ) ) || <STR_LIT> ( CC ) ; if ( IsWaveEnd ) { B . buildInstr ( <STR_LIT> ) . addImm ( <NUM_LIT> ) ; return true ; } unsigned ReturnOpc = IsShader ? <STR_LIT> : <STR_LIT> ;

auto Ret = B . buildInstrNoInsert ( ReturnOpc ) ;


<s> bool assignArg ( unsigned ValNo , MVT ValVT , MVT LocVT , CCValAssign :: LocInfo LocInfo , const CallLowering :: ArgInfo & Info , CCState & State ) override { return AssignFn ( ValNo , ValVT , LocVT , LocInfo , Info . Flags , State ) ;


<s> return AssignFn ( ValNo , ValVT , LocVT , LocInfo , Info . Flags , State ) ;


<s> void assignValueToAddress ( Register ValVReg , Register Addr , uint64_t Size , MachinePointerInfo & MPO , CCValAssign & VA ) override { auto MMO = MIRBuilder . getMF ( ) . getMachineMemOperand ( MPO , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant , Size , <NUM_LIT> ) ; MIRBuilder . buildLoad ( ValVReg , Addr , * MMO ) ;


<s> case CCValAssign :: LocInfo :: SExt : case CCValAssign :: LocInfo :: ZExt : case CCValAssign :: LocInfo :: AExt : { auto Copy = MIRBuilder . buildCopy ( LLT { VA . getLocVT ( ) } , PhysReg ) ; MIRBuilder . buildTrunc ( ValVReg , Copy ) ; break ; } default :

MIRBuilder . buildCopy ( ValVReg , PhysReg ) ;


<s> FormalArgHandler ( MachineIRBuilder & MIRBuilder , MachineRegisterInfo & MRI , CCAssignFn * AssignFn ) : IncomingArgHandler ( MIRBuilder , MRI , AssignFn ) {


<s> if ( OrigTy . isVector ( ) ) { return LLT :: vector ( OrigTy . getNumElements ( ) * Factor , OrigTy . getElementType ( ) ) ; } return LLT :: scalar ( OrigTy . getSizeInBits ( ) * Factor ) ;


<s> MPO = MachinePointerInfo :: getFixedStack ( MIRBuilder . getMF ( ) , FI ) ; Register AddrReg = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: PRIVATE_ADDRESS , <NUM_LIT> ) ) ; MIRBuilder . buildFrameIndex ( AddrReg , FI ) ;


<s> IncomingArgHandler ( MachineIRBuilder & MIRBuilder , MachineRegisterInfo & MRI , CCAssignFn * AssignFn ) : ValueHandler ( MIRBuilder , MRI , AssignFn ) {


<s> bool isArgumentHandler ( ) const override { return true ;


<s> bool isArgumentHandler ( ) const override {


<s> const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( CC , F . isVarArg ( ) , MF , ArgLocs , F . getContext ( ) ) ; if ( ! IsEntryFunc ) { Register ReturnAddrReg = TRI -> getReturnAddressReg ( MF ) ; Register LiveInReturn = MF . addLiveIn ( ReturnAddrReg , & <STR_LIT> ) ; MBB . addLiveIn ( ReturnAddrReg ) ; MIRBuilder . buildCopy ( LiveInReturn , ReturnAddrReg ) ; } if ( Info -> hasImplicitBufferPtr ( ) ) { Register ImplicitBufferPtrReg = Info -> addImplicitBufferPtr ( * TRI ) ; MF . addLiveIn ( ImplicitBufferPtrReg , & <STR_LIT> ) ; CCInfo . AllocateReg ( ImplicitBufferPtrReg ) ; } SmallVector < ArgInfo , <NUM_LIT> > SplitArgs ; unsigned Idx = <NUM_LIT> ; unsigned P<ISA_LIT>nputNum = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { if ( DL . getTypeStoreSize ( Arg . getType ( ) ) == <NUM_LIT> ) continue ; const bool InReg = Arg . hasAttribute ( Attribute :: InReg ) ; if ( ! IsShader && InReg ) return false ; if ( Arg . hasAttribute ( Attribute :: SwiftSelf ) || Arg . hasAttribute ( Attribute :: SwiftError ) || Arg . hasAttribute ( Attribute :: Nest ) ) return false ; if ( CC == CallingConv :: <ISA_LIT>_PS && ! InReg && P<ISA_LIT>nputNum <= <NUM_LIT> ) { const bool ArgUsed = ! Arg . use_empty ( ) ; bool SkipArg = ! ArgUsed && ! Info -> isP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ! SkipArg ) { Info -> markP<ISA_LIT>nputAllocated ( P<ISA_LIT>nputNum ) ; if ( ArgUsed ) Info -> markP<ISA_LIT>nputEnabled ( P<ISA_LIT>nputNum ) ;


<s> const GCNSubtarget * Subtarget = & MF . getSubtarget < GCNSubtarget > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget -> getRegisterInfo ( ) ; const <ISA_LIT>TargetLowering & TLI = * getTLI < <ISA_LIT>TargetLowering > ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( F . getCallingConv ( ) , F . isVarArg ( ) , MF , ArgLocs , F . getContext ( ) ) ; allocateHSAUserSGPRs ( CCInfo , MIRBuilder , MF , * TRI , * Info ) ; unsigned i = <NUM_LIT> ; const unsigned KernArgBaseAlign = <NUM_LIT> ; const unsigned BaseOffset = Subtarget -> getExplicitKernelArgOffset ( F ) ; uint64_t ExplicitArgOffset = <NUM_LIT> ; for ( auto & Arg : F . args ( ) ) { Type * ArgTy = Arg . getType ( ) ; unsigned AllocSize = DL . getTypeAllocSize ( ArgTy ) ; if ( AllocSize == <NUM_LIT> ) continue ; unsigned ABIAlign = DL . getABITypeAlignment ( ArgTy ) ; uint64_t ArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + BaseOffset ; ExplicitArgOffset = alignTo ( ExplicitArgOffset , ABIAlign ) + AllocSize ; ArrayRef < Register > OrigArgRegs = VRegs [ i ] ; Register ArgReg = OrigArgRegs . size ( ) == <NUM_LIT> ? OrigArgRegs [ <NUM_LIT> ] : MRI . createGenericVirtualRegister ( getLLTForType ( * ArgTy , DL ) ) ; unsigned Align = MinAlign ( KernArgBaseAlign , ArgOffset ) ; ArgOffset = alignTo ( ArgOffset , DL . getABITypeAlignment ( ArgTy ) ) ; lowerParameter ( MIRBuilder , ArgTy , ArgOffset , Align , ArgReg ) ; if ( OrigArgRegs . size ( ) > <NUM_LIT> ) unpackRegs ( OrigArgRegs , ArgReg , ArgTy , MIRBuilder ) ; ++ i ; } TLI . allocateSpecialEntryInputVGPRs ( CCInfo , MF , * TRI , * Info ) ;

TLI . allocateSystemSGPRs ( CCInfo , MF , * Info , F . getCallingConv ( ) , false ) ;


<s> const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned TypeSize = DL . getTypeStoreSize ( ParamTy ) ;

Register PtrReg = lowerParameterPtr ( MIRBuilder , ParamTy , Offset ) ;


<s> const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = MF . getFunction ( ) ; const DataLayout & DL = F . getParent ( ) -> getDataLayout ( ) ; PointerType * PtrTy = PointerType :: get ( ParamTy , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; LLT PtrType = getLLTForType ( * PtrTy , DL ) ; Register DstReg = MRI . createGenericVirtualRegister ( PtrType ) ; Register KernArgSegmentPtr = MFI -> getPreloadedReg ( <STR_LIT> ) ;

Register KernArgSegmentVReg = MRI . getLiveInVirtReg ( KernArgSegmentPtr ) ;


<s> auto const & ST = MIRBuilder . getMF ( ) . getSubtarget < GCNSubtarget > ( ) ; unsigned ReturnOpc = IsShader ? <STR_LIT> : <STR_LIT> ; auto Ret = MIRBuilder . buildInstrNoInsert ( ReturnOpc ) ; Register ReturnAddrVReg ; if ( ReturnOpc == <STR_LIT> ) { ReturnAddrVReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Ret . addUse ( ReturnAddrVReg ) ; } if ( ! lowerReturnVal ( MIRBuilder , Val , VRegs , Ret ) ) return false ; if ( ReturnOpc == <STR_LIT> ) { const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ;

Register LiveInReturn = MF . addLiveIn ( TRI -> getReturnAddressReg ( MF ) , & <STR_LIT> ) ;


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; ArgInfo OrigRetInfo ( VRegs , Val -> getType ( ) ) ; setArgFlags ( OrigRetInfo , AttributeList :: ReturnIndex , DL , F ) ; SmallVector < ArgInfo , <NUM_LIT> > SplitRetInfos ; splitToValueTypes ( OrigRetInfo , SplitRetInfos , DL , MRI , CC , [ & ] ( ArrayRef < Register > Regs , LLT LLTy , LLT PartLLT , int VTSplitIdx ) {

unpackRegsToOrigType ( MIRBuilder , Regs , VRegs [ VTSplitIdx ] , LLTy , PartLLT ) ;


<s> splitToValueTypes ( OrigRetInfo , SplitRetInfos , DL , MRI , CC , [ & ] ( ArrayRef < Register > Regs , LLT LLTy , LLT PartLLT , int VTSplitIdx ) { unpackRegsToOrigType ( MIRBuilder , Regs , VRegs [ VTSplitIdx ] , LLTy , PartLLT ) ; } ) ; CCAssignFn * AssignFn = TLI . CCAssignFnForReturn ( CC , F . isVarArg ( ) ) ; OutgoingValueHandler RetHandler ( MIRBuilder , MF . getRegInfo ( ) , Ret , AssignFn ) ;

return handleAssignments ( MIRBuilder , SplitRetInfos , RetHandler ) ;


<s> OutgoingValueHandler ( MachineIRBuilder & MIRBuilder , MachineRegisterInfo & MRI , MachineInstrBuilder MIB , CCAssignFn * AssignFn ) : ValueHandler ( MIRBuilder , MRI , AssignFn ) , MIB ( MIB ) {


<s> for ( EVT VT : SplitVTs ) { unsigned NumParts = TLI . getNumRegistersForCallingConv ( Ctx , CallConv , VT ) ; Type * Ty = VT . getTypeForEVT ( Ctx ) ; if ( NumParts == <NUM_LIT> ) { SplitArgs . emplace_back ( OrigArg . Regs [ SplitIdx ] , Ty , OrigArg . Flags , OrigArg . IsFixed ) ; ++ SplitIdx ; continue ; } LLT LLTy = getLLTForType ( * Ty , DL ) ; SmallVector < Register , <NUM_LIT> > SplitRegs ; EVT PartVT = TLI . getRegisterTypeForCallingConv ( Ctx , CallConv , VT ) ; Type * PartTy = PartVT . getTypeForEVT ( Ctx ) ; LLT PartLLT = getLLTForType ( * PartTy , DL ) ; for ( unsigned i = <NUM_LIT> ; i < NumParts ; ++ i ) { Register PartReg = MRI . createGenericVirtualRegister ( PartLLT ) ; SplitRegs . push_back ( PartReg ) ; SplitArgs . emplace_back ( ArrayRef < Register > ( PartReg ) , PartTy , OrigArg . Flags ) ; } PerformArgSplit ( SplitRegs , LLTy , PartLLT , SplitIdx ) ; ++ SplitIdx ;


<s> Type * Ty = VT . getTypeForEVT ( Ctx ) ; if ( NumParts == <NUM_LIT> ) { SplitArgs . emplace_back ( OrigArg . Regs [ SplitIdx ] , Ty , OrigArg . Flags , OrigArg . IsFixed ) ; ++ SplitIdx ; continue ; } LLT LLTy = getLLTForType ( * Ty , DL ) ; SmallVector < Register , <NUM_LIT> > SplitRegs ; EVT PartVT = TLI . getRegisterTypeForCallingConv ( Ctx , CallConv , VT ) ; Type * PartTy = PartVT . getTypeForEVT ( Ctx ) ; LLT PartLLT = getLLTForType ( * PartTy , DL ) ; for ( unsigned i = <NUM_LIT> ; i < NumParts ; ++ i ) { Register PartReg = MRI . createGenericVirtualRegister ( PartLLT ) ; SplitRegs . push_back ( PartReg ) ; SplitArgs . emplace_back ( ArrayRef < Register > ( PartReg ) , PartTy , OrigArg . Flags ) ; } PerformArgSplit ( SplitRegs , LLTy , PartLLT , SplitIdx ) ; ++ SplitIdx ; }


<s> const int NumRoundedParts = ( SrcSize + PartSize - <NUM_LIT> ) / PartSize ; LLT BigTy = getMultipleType ( PartTy , NumRoundedParts ) ; auto ImpDef = MIRBuilder . buildUndef ( BigTy ) ; Register BigReg = MRI . createGenericVirtualRegister ( BigTy ) ; MIRBuilder . buildInsert ( BigReg , ImpDef . getReg ( <NUM_LIT> ) , SrcReg , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; int64_t Offset = <NUM_LIT> ;

for ( unsigned i = <NUM_LIT> , e = DstRegs . size ( ) ; i != e ; ++ i , Offset += PartSize ) MIRBuilder . buildExtract ( DstRegs [ i ] , BigReg , Offset ) ;


<s> <ISA_LIT>IncomingArgHandler ( MachineIRBuilder & B , MachineRegisterInfo & MRI , CCAssignFn * AssignFn ) : <ISA_LIT>ValueHandler ( true , B , MRI , AssignFn ) {


<s> <ISA_LIT>OutgoingArgHandler ( MachineIRBuilder & MIRBuilder , MachineRegisterInfo & MRI , MachineInstrBuilder MIB , CCAssignFn * AssignFn , CCAssignFn * AssignFnVarArg , bool IsTailCall = false , int FPDiff = <NUM_LIT> ) : <ISA_LIT>ValueHandler ( false , MIRBuilder , MRI , AssignFn ) , MIB ( MIB ) , AssignFnVarArg ( AssignFnVarArg ) , FPDiff ( FPDiff ) , IsTailCall ( IsTailCall ) {


<s> <ISA_LIT>OutgoingValueHandler ( MachineIRBuilder & B , MachineRegisterInfo & MRI , MachineInstrBuilder MIB , CCAssignFn * AssignFn ) : <ISA_LIT>ValueHandler ( false , B , MRI , AssignFn ) , MIB ( MIB ) {


<s> <ISA_LIT>ValueHandler ( bool IsIncoming , MachineIRBuilder & B , MachineRegisterInfo & MRI , CCAssignFn * AssignFn ) : ValueHandler ( IsIncoming , B , MRI , AssignFn ) {


<s> void assignValueToAddress ( const CallLowering :: ArgInfo & Arg , Register Addr , uint64_t MemSize , MachinePointerInfo & MPO , CCValAssign & VA ) override {

Register ValVReg = VA . getLocInfo ( ) != CCValAssign :: LocInfo :: FPExt ? extendRegister ( Arg . Regs [ <NUM_LIT> ] , VA ) : Arg . Regs [ <NUM_LIT> ] ;


<s> Register ExtReg = extendRegisterMin32 ( ValVReg , VA ) ; MIRBuilder . buildCopy ( PhysReg , ExtReg ) ;


<s> Register ExtReg = extendRegisterMin32 ( ValVReg , VA ) ;

MIRBuilder . buildCopy ( PhysReg , ExtReg ) ;


<s> CallReturnHandler ( MachineIRBuilder & MIRBuilder , MachineRegisterInfo & MRI , MachineInstrBuilder MIB , CCAssignFn * AssignFn ) : <ISA_LIT>IncomingArgHandler ( MIRBuilder , MRI , AssignFn ) , MIB ( MIB ) {


<s> if ( VA . getLocVT ( ) . getSizeInBits ( ) < <NUM_LIT> ) { return MIRBuilder . buildAnyExt ( LLT :: scalar ( <NUM_LIT> ) , ValVReg ) . getReg ( <NUM_LIT> ) ;


<s> if ( VA . getLocVT ( ) . getSizeInBits ( ) < <NUM_LIT> ) {

return MIRBuilder . buildAnyExt ( LLT :: scalar ( <NUM_LIT> ) , ValVReg ) . getReg ( <NUM_LIT> ) ;


<s> FormalArgHandler ( MachineIRBuilder & B , MachineRegisterInfo & MRI , CCAssignFn * AssignFn ) : <ISA_LIT>IncomingArgHandler ( B , MRI , AssignFn ) {


<s> static unsigned getCallOpcode ( const MachineFunction & CallerF , bool IsIndirect , bool IsTailCall ) {


<s> static unsigned getCallOpcode ( const MachineFunction & CallerF , bool IsIndirect , bool IsTailCall ) { return <STR_LIT> ;


<s> } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! SPReg ) SPReg = MIRBuilder . buildCopy ( PtrTy , MFI -> getStackPtrOffsetReg ( ) ) . getReg ( <NUM_LIT> ) ; auto OffsetReg = MIRBuilder . buildConstant ( S32 , Offset ) ; auto AddrReg = MIRBuilder . buildPtrAdd ( PtrTy , SPReg , OffsetReg ) ;

MPO = MachinePointerInfo :: getStack ( MF , Offset ) ;


<s> Register getStackAddress ( uint64_t Size , int64_t Offset , MachinePointerInfo & MPO ) override { MachineFunction & MF = MIRBuilder . getMF ( ) ; const LLT PtrTy = LLT :: pointer ( <ISA_LIT>AS :: PRIVATE_ADDRESS , <NUM_LIT> ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; if ( IsTailCall ) { llvm_unreachable ( <STR_LIT> ) ; } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! SPReg ) SPReg = MIRBuilder . buildCopy ( PtrTy , MFI -> getStackPtrOffsetReg ( ) ) . getReg ( <NUM_LIT> ) ; auto OffsetReg = MIRBuilder . buildConstant ( S32 , Offset ) ; auto AddrReg = MIRBuilder . buildPtrAdd ( PtrTy , SPReg , OffsetReg ) ; MPO = MachinePointerInfo :: getStack ( MF , Offset ) ;