<s> switch ( MI . getOpcode ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : { DstDivReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; TmpDivReg = MRI . createGenericVirtualRegister ( Ty ) ; break ; } case <STR_LIT> : { DstRemReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; TmpRemReg = MRI . createGenericVirtualRegister ( Ty ) ; break ; } case <STR_LIT> : { DstDivReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; DstRemReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; TmpDivReg = MRI . createGenericVirtualRegister ( Ty ) ; TmpRemReg = MRI . createGenericVirtualRegister ( Ty ) ; break ; } } if ( Ty == S32 ) legalizeUnsignedDIV_REM32Impl ( B , TmpDivReg , TmpRemReg , LHS , RHS ) ; else legalizeUnsignedDIV_REM64Impl ( B , TmpDivReg , TmpRemReg , LHS , RHS ) ; if ( DstDivReg ) { auto Sign = B . buildXor ( Ty , LHSign , RHSign ) . getReg ( <NUM_LIT> ) ; auto SignXor = B . buildXor ( Ty , TmpDivReg , Sign ) . getReg ( <NUM_LIT> ) ; B . buildSub ( DstDivReg , SignXor , Sign ) ; } if ( DstRemReg ) { auto Sign = LHSign . getReg ( <NUM_LIT> ) ; auto SignXor = B . buildXor ( Ty , TmpRemReg , Sign ) . getReg ( <NUM_LIT> ) ; B . buildSub ( DstRemReg , SignXor , Sign ) ; } MI . eraseFromParent ( ) ;


<s> llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : { DstDivReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; TmpDivReg = MRI . createGenericVirtualRegister ( Ty ) ; break ; } case <STR_LIT> : { DstRemReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; TmpRemReg = MRI . createGenericVirtualRegister ( Ty ) ; break ; } case <STR_LIT> : { DstDivReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; DstRemReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; TmpDivReg = MRI . createGenericVirtualRegister ( Ty ) ; TmpRemReg = MRI . createGenericVirtualRegister ( Ty ) ; break ; } } if ( Ty == S32 ) legalizeUnsignedDIV_REM32Impl ( B , TmpDivReg , TmpRemReg , LHS , RHS ) ; else legalizeUnsignedDIV_REM64Impl ( B , TmpDivReg , TmpRemReg , LHS , RHS ) ; if ( DstDivReg ) { auto Sign = B . buildXor ( Ty , LHSign , RHSign ) . getReg ( <NUM_LIT> ) ; auto SignXor = B . buildXor ( Ty , TmpDivReg , Sign ) . getReg ( <NUM_LIT> ) ;

B . buildSub ( DstDivReg , SignXor , Sign ) ;


<s> if ( ST . hasTrigReducedRange ( ) ) { auto MulVal = B . buildFMul ( Ty , SrcReg , OneOver2Pi , Flags ) ; TrigVal = B . buildIntrinsic ( <STR_LIT> , { Ty } , false ) . addUse ( MulVal . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) . getReg ( <NUM_LIT> ) ; } else TrigVal = B . buildFMul ( Ty , SrcReg , OneOver2Pi , Flags ) . getReg ( <NUM_LIT> ) ; <STR_LIT> TrigIntrin = MI . getOpcode ( ) == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; B . buildIntrinsic ( TrigIntrin , makeArrayRef < Register > ( DstReg ) , false ) . addUse ( TrigVal ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ;


<s> bool <ISA_LIT>LegalizerInfo :: legalizeSinCos ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT Ty = MRI . getType ( DstReg ) ; unsigned Flags = MI . getFlags ( ) ; Register TrigVal ; auto OneOver2Pi = B . buildFConstant ( Ty , <NUM_LIT> * numbers :: inv_pi ) ; if ( ST . hasTrigReducedRange ( ) ) { auto MulVal = B . buildFMul ( Ty , SrcReg , OneOver2Pi , Flags ) ; TrigVal = B . buildIntrinsic ( <STR_LIT> , { Ty } , false ) . addUse ( MulVal . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) . getReg ( <NUM_LIT> ) ; } else TrigVal = B . buildFMul ( Ty , SrcReg , OneOver2Pi , Flags ) . getReg ( <NUM_LIT> ) ; <STR_LIT> TrigIntrin = MI . getOpcode ( ) == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; B . buildIntrinsic ( TrigIntrin , makeArrayRef < Register > ( DstReg ) , false ) . addUse ( TrigVal ) . setMIFlags ( Flags ) ;


<s> B . buildInstr ( <STR_LIT> ) . addImm ( static_cast < unsigned > ( GCNSubtarget :: TrapID :: LLVMAMDHSATrap ) ) ; MI . eraseFromParent ( ) ;


<s> Register LiveIn = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; if ( ! loadInputValue ( LiveIn , B , <STR_LIT> ) ) return false ; Register SGPR01 ( <STR_LIT> ) ;


<s> switch ( * HsaAbiVer ) { case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V2 : case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V3 : return legalizeTrapHsaQueuePtr ( MI , MRI , B ) ; case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V4 : return ST . supportsGetDoorbellID ( ) ? legalizeTrapHsa ( MI , MRI , B ) : legalizeTrapHsaQueuePtr ( MI , MRI , B ) ; }


<s> break ; } case <STR_LIT> : { DstDivReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; DstRemReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; break ; } } const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const unsigned FirstSrcOpIdx = MI . getNumExplicitDefs ( ) ; Register Num = MI . getOperand ( FirstSrcOpIdx ) . getReg ( ) ; Register Den = MI . getOperand ( FirstSrcOpIdx + <NUM_LIT> ) . getReg ( ) ;

LLT Ty = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> auto ScaledY = B . buildFMul ( S32 , RcpIFlag , Scale ) ; auto Z = B . buildFPTOUI ( S32 , ScaledY ) ; auto NegY = B . buildSub ( S32 , B . buildConstant ( S32 , <NUM_LIT> ) , Y ) ; auto NegYZ = B . buildMul ( S32 , NegY , Z ) ; Z = B . buildAdd ( S32 , Z , B . buildUMulH ( S32 , Z , NegYZ ) ) ; auto Q = B . buildUMulH ( S32 , X , Z ) ; auto R = B . buildSub ( S32 , X , B . buildMul ( S32 , Q , Y ) ) ; auto One = B . buildConstant ( S32 , <NUM_LIT> ) ; auto Cond = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , R , Y ) ; if ( DstDivReg ) Q = B . buildSelect ( S32 , Cond , B . buildAdd ( S32 , Q , One ) , Q ) ; R = B . buildSelect ( S32 , Cond , B . buildSub ( S32 , R , Y ) , R ) ; Cond = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , R , Y ) ; if ( DstDivReg ) B . buildSelect ( DstDivReg , Cond , B . buildAdd ( S32 , Q , One ) , Q ) ;

if ( DstRemReg ) B . buildSelect ( DstRemReg , Cond , B . buildSub ( S32 , R , Y ) , R ) ;


<s> Register Mul3_Hi = UnmergeMul3 . getReg ( <NUM_LIT> ) ; auto Sub1_Lo = B . buildUSubo ( S32 , S1 , NumerLo , Mul3_Lo ) ; auto Sub1_Hi = B . buildUSube ( S32 , S1 , NumerHi , Mul3_Hi , Sub1_Lo . getReg ( <NUM_LIT> ) ) ; auto Sub1_Mi = B . buildSub ( S32 , NumerHi , Mul3_Hi ) ; auto Sub1 = B . buildMerge ( S64 , { Sub1_Lo , Sub1_Hi } ) ; auto UnmergeDenom = B . buildUnmerge ( S32 , Denom ) ; Register DenomLo = UnmergeDenom . getReg ( <NUM_LIT> ) ; Register DenomHi = UnmergeDenom . getReg ( <NUM_LIT> ) ; auto CmpHi = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , Sub1_Hi , DenomHi ) ; auto C1 = B . buildSExt ( S32 , CmpHi ) ; auto CmpLo = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , Sub1_Lo , DenomLo ) ; auto C2 = B . buildSExt ( S32 , CmpLo ) ; auto CmpEq = B . buildICmp ( CmpInst :: ICMP_EQ , S1 , Sub1_Hi , DenomHi ) ; auto C3 = B . buildSelect ( S32 , CmpEq , C2 , C1 ) ; auto Sub2_Lo = B . buildUSubo ( S32 , S1 , Sub1_Lo , DenomLo ) ; auto Sub2_Mi = B . buildUSube ( S32 , S1 , Sub1_Mi , DenomHi , Sub1_Lo . getReg ( <NUM_LIT> ) ) ; auto Sub2_Hi = B . buildUSube ( S32 , S1 , Sub2_Mi , Zero32 , Sub2_Lo . getReg ( <NUM_LIT> ) ) ; auto Sub2 = B . buildMerge ( S64 , { Sub2_Lo , Sub2_Hi } ) ; auto One64 = B . buildConstant ( S64 , <NUM_LIT> ) ; auto Add3 = B . buildAdd ( S64 , MulHi3 , One64 ) ; auto C4 = B . buildSExt ( S32 , B . buildICmp ( CmpInst :: ICMP_UGE , S1 , Sub2_Hi , DenomHi ) ) ; auto C5 = B . buildSExt ( S32 , B . buildICmp ( CmpInst :: ICMP_UGE , S1 , Sub2_Lo , DenomLo ) ) ; auto C6 = B . buildSelect ( S32 , B . buildICmp ( CmpInst :: ICMP_EQ , S1 , Sub2_Hi , DenomHi ) , C5 , C4 ) ; auto Add4 = B . buildAdd ( S64 , Add3 , One64 ) ; auto Sub3_Lo = B . buildUSubo ( S32 , S1 , Sub2_Lo , DenomLo ) ; auto Sub3_Mi = B . buildUSube ( S32 , S1 , Sub2_Mi , DenomHi , Sub2_Lo . getReg ( <NUM_LIT> ) ) ; auto Sub3_Hi = B . buildUSube ( S32 , S1 , Sub3_Mi , Zero32 , Sub3_Lo . getReg ( <NUM_LIT> ) ) ; auto Sub3 = B . buildMerge ( S64 , { Sub3_Lo , Sub3_Hi } ) ; if ( DstDivReg ) {

auto Sel1 = B . buildSelect ( S64 , B . buildICmp ( CmpInst :: ICMP_NE , S1 , C6 , Zero32 ) , Add4 , Add3 ) ;


<s> bool <ISA_LIT>LegalizerInfo :: loadInputValue ( Register DstReg , MachineIRBuilder & B , <STR_LIT> ArgType ) const { const <ISA_LIT>MachineFunctionInfo * MFI = B . getMF ( ) . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const ArgDescriptor * Arg ; const TargetRegisterClass * ArgRC ; LLT ArgTy ; std :: tie ( Arg , ArgRC , ArgTy ) = MFI -> getPreloadedValue ( ArgType ) ; if ( ! Arg ) { assert ( ArgType == <STR_LIT> ) ;

B . buildConstant ( DstReg , <NUM_LIT> ) ;


<s> const <ISA_LIT>MachineFunctionInfo * MFI = B . getMF ( ) . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const ArgDescriptor * Arg ; const TargetRegisterClass * ArgRC ; LLT ArgTy ; std :: tie ( Arg , ArgRC , ArgTy ) = MFI -> getPreloadedValue ( ArgType ) ; if ( ! Arg ) { assert ( ArgType == <STR_LIT> ) ;


<s> if ( Size <= <NUM_LIT> ) return false ; if ( ! Ty . isVector ( ) ) return true ; LLT EltTy = Ty . getElementType ( ) ; if ( EltTy . isPointer ( ) ) return true ; unsigned EltSize = EltTy . getSizeInBits ( ) ; return EltSize != <NUM_LIT> && EltSize != <NUM_LIT> ;


<s> if ( EltTy . isPointer ( ) ) return true ; unsigned EltSize = EltTy . getSizeInBits ( ) ; return EltSize != <NUM_LIT> && EltSize != <NUM_LIT> ;


<s> return ST . enableFlatScratch ( ) ? <NUM_LIT> : <NUM_LIT> ; case <ISA_LIT>AS :: LOCAL_ADDRESS :

return ST . useDS128 ( ) ? <NUM_LIT> : <NUM_LIT> ;


<s> const LLT Ty = Query . Types [ TypeIdx ] ; const LLT EltTy = Ty . getElementType ( ) ; const int Size = Ty . getSizeInBits ( ) ;

const int EltSize = EltTy . getSizeInBits ( ) ;


<s> assert ( EltSize < <NUM_LIT> ) ; const int NewNumElts = ( <NUM_LIT> * NextMul32 + EltSize - <NUM_LIT> ) / EltSize ; return std :: make_pair ( TypeIdx , LLT :: fixed_vector ( NewNumElts , EltTy ) ) ;


<s> static LegalityPredicate numElementsNotEven ( unsigned TypeIdx ) { return [ = ] ( const LegalityQuery & Query ) {

const LLT QueryTy = Query . Types [ TypeIdx ] ;


<s> return QueryTy . isVector ( ) && QueryTy . getNumElements ( ) % <NUM_LIT> != <NUM_LIT> ; } ;


<s> static LegalizeMutation oneMoreElement ( unsigned TypeIdx ) { return [ = ] ( const LegalityQuery & Query ) { const LLT Ty = Query . Types [ TypeIdx ] ;

const LLT EltTy = Ty . getElementType ( ) ;


<s> static LegalizeMutation oneMoreElement ( unsigned TypeIdx ) { return [ = ] ( const LegalityQuery & Query ) {

const LLT Ty = Query . Types [ TypeIdx ] ;


<s> auto EndIdx = Intr -> VAddrEnd ; for ( unsigned I = Intr -> VAddrStart ; I < EndIdx ; I ++ ) { MachineOperand & SrcOp = MI . getOperand ( ArgOffset + I ) ; if ( ! SrcOp . isReg ( ) ) continue ; Register AddrReg = SrcOp . getReg ( ) ; if ( ( I < Intr -> GradientStart ) || ( I >= Intr -> GradientStart && I < Intr -> CoordStart && ! IsG16 ) || ( I >= Intr -> CoordStart && ! IsA16 ) ) { AddrReg = B . buildBitcast ( V2S16 , AddrReg ) . getReg ( <NUM_LIT> ) ; PackedAddrs . push_back ( AddrReg ) ; } else { if ( ( ( I + <NUM_LIT> ) >= EndIdx ) || ( ( Intr -> NumGradients / <NUM_LIT> ) % <NUM_LIT> == <NUM_LIT> && ( I == static_cast < unsigned > ( Intr -> GradientStart + ( Intr -> NumGradients / <NUM_LIT> ) - <NUM_LIT> ) || I == static_cast < unsigned > ( Intr -> GradientStart + Intr -> NumGradients - <NUM_LIT> ) ) ) || ! MI . getOperand ( ArgOffset + I + <NUM_LIT> ) . isReg ( ) ) { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) } ) . getReg ( <NUM_LIT> ) ) ; } else { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , MI . getOperand ( ArgOffset + I + <NUM_LIT> ) . getReg ( ) } ) . getReg ( <NUM_LIT> ) ) ; ++ I ; }


<s> const LLT S16 = LLT :: scalar ( <NUM_LIT> ) ; const LLT V2S16 = LLT :: fixed_vector ( <NUM_LIT> , <NUM_LIT> ) ; auto EndIdx = Intr -> VAddrEnd ; for ( unsigned I = Intr -> VAddrStart ; I < EndIdx ; I ++ ) { MachineOperand & SrcOp = MI . getOperand ( ArgOffset + I ) ; if ( ! SrcOp . isReg ( ) ) continue ; Register AddrReg = SrcOp . getReg ( ) ; if ( ( I < Intr -> GradientStart ) || ( I >= Intr -> GradientStart && I < Intr -> CoordStart && ! IsG16 ) || ( I >= Intr -> CoordStart && ! IsA16 ) ) { AddrReg = B . buildBitcast ( V2S16 , AddrReg ) . getReg ( <NUM_LIT> ) ; PackedAddrs . push_back ( AddrReg ) ; } else { if ( ( ( I + <NUM_LIT> ) >= EndIdx ) || ( ( Intr -> NumGradients / <NUM_LIT> ) % <NUM_LIT> == <NUM_LIT> && ( I == static_cast < unsigned > ( Intr -> GradientStart + ( Intr -> NumGradients / <NUM_LIT> ) - <NUM_LIT> ) || I == static_cast < unsigned > ( Intr -> GradientStart + Intr -> NumGradients - <NUM_LIT> ) ) ) || ! MI . getOperand ( ArgOffset + I + <NUM_LIT> ) . isReg ( ) ) { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) } ) . getReg ( <NUM_LIT> ) ) ; } else { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , MI . getOperand ( ArgOffset + I + <NUM_LIT> ) . getReg ( ) } ) . getReg ( <NUM_LIT> ) ) ; ++ I ;


<s> if ( Size != MemSizeInBits ) return Size <= <NUM_LIT> && Ty . isVector ( ) ; if ( loadStoreBitcastWorkaround ( Ty ) && isRegisterType ( Ty ) ) return true ;

return Ty . isVector ( ) && ( ! MemTy . isVector ( ) || MemTy == Ty ) && ( Size <= <NUM_LIT> || isRegisterSize ( Size ) ) && ! isRegisterVectorElementType ( Ty . getElementType ( ) ) ;


<s> if ( loadStoreBitcastWorkaround ( Ty ) && isRegisterType ( Ty ) ) return true ;


<s> if ( Query . MMODescrs [ <NUM_LIT> ] . Ordering != AtomicOrdering :: NotAtomic ) return false ; return shouldWidenLoad ( ST , Query . MMODescrs [ <NUM_LIT> ] . MemoryTy , Query . MMODescrs [ <NUM_LIT> ] . AlignInBits , Query . Types [ <NUM_LIT> ] . getAddressSpace ( ) , Opcode ) ;


<s> static bool shouldWidenLoad ( const GCNSubtarget & ST , const LegalityQuery & Query , unsigned Opcode ) {

if ( Query . MMODescrs [ <NUM_LIT> ] . Ordering != AtomicOrdering :: NotAtomic ) return false ;


<s> return Ty . getSizeInBits ( ) % <NUM_LIT> == <NUM_LIT> ; } ;


<s> MachineRegisterInfo & MRI = * B . getMRI ( ) ; std :: tie ( BaseReg , ImmOffset ) = <STR_LIT> ( MRI , OrigOffset ) ; if ( MRI . getType ( BaseReg ) . isPointer ( ) ) BaseReg = B . buildPtrToInt ( MRI . getType ( OrigOffset ) , BaseReg ) . getReg ( <NUM_LIT> ) ; unsigned Overflow = ImmOffset & ~ MaxImm ; ImmOffset -= Overflow ; if ( ( int32_t ) Overflow < <NUM_LIT> ) { Overflow += ImmOffset ; ImmOffset = <NUM_LIT> ; } if ( Overflow != <NUM_LIT> ) { if ( ! BaseReg ) { BaseReg = B . buildConstant ( S32 , Overflow ) . getReg ( <NUM_LIT> ) ; } else {

auto OverflowVal = B . buildConstant ( S32 , Overflow ) ;


<s> if ( MachineInstr * SrcFNeg = getOpcodeDef ( <STR_LIT> , ModSrc , MRI ) ) { ModSrc = SrcFNeg -> getOperand ( <NUM_LIT> ) . getReg ( ) ;

if ( MachineInstr * SrcFAbs = getOpcodeDef ( <STR_LIT> , ModSrc , MRI ) ) ModSrc = SrcFAbs -> getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> unsigned SPDenormMode = Enable ? FP_DENORM_FLUSH_NONE : Mode . fpDenormModeSPValue ( ) ; if ( ST . hasDenormModeInst ( ) ) { uint32_t DPDenormModeDefault = Mode . fpDenormModeDPValue ( ) ; uint32_t NewDenormModeValue = SPDenormMode | ( DPDenormModeDefault << <NUM_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addImm ( NewDenormModeValue ) ; } else { unsigned SPDenormModeBitField = <STR_LIT> :: ID_MODE | ( <NUM_LIT> << <STR_LIT> :: OFFSET_SHIFT_ ) | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ;

B . buildInstr ( <STR_LIT> ) . addImm ( SPDenormMode ) . addImm ( SPDenormModeBitField ) ;


<s> static void toggleSPDenormMode ( bool Enable , MachineIRBuilder & B , const GCNSubtarget & ST , <STR_LIT> Mode ) { unsigned SPDenormMode = Enable ? FP_DENORM_FLUSH_NONE : Mode . fpDenormModeSPValue ( ) ; if ( ST . hasDenormModeInst ( ) ) { uint32_t DPDenormModeDefault = Mode . fpDenormModeDPValue ( ) ; uint32_t NewDenormModeValue = SPDenormMode | ( DPDenormModeDefault << <NUM_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addImm ( NewDenormModeValue ) ; } else { unsigned SPDenormModeBitField = <STR_LIT> :: ID_MODE | ( <NUM_LIT> << <STR_LIT> :: OFFSET_SHIFT_ ) | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ;


<s> if ( MaybeVOffsetVal && MaybeSOffsetVal && MaybeVIndexVal && MaybeVIndexVal -> Value == <NUM_LIT> ) { uint64_t TotalOffset = MaybeVOffsetVal -> Value . getZExtValue ( ) + MaybeSOffsetVal -> Value . getZExtValue ( ) + ImmOffset ;


<s> return [ = ] ( const LegalityQuery & Query ) { const LLT QueryTy = Query . Types [ TypeIdx ] ; return QueryTy . isVector ( ) && QueryTy . getSizeInBits ( ) < Size ; } ;


<s> return QueryTy . isVector ( ) && QueryTy . getSizeInBits ( ) > Size ;


<s> static LegalityPredicate vectorWiderThan ( unsigned TypeIdx , unsigned Size ) { return [ = ] ( const LegalityQuery & Query ) {


<s> Register NegatedCond = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! MRI . hasOneNonDBGUse ( NegatedCond ) ) return nullptr ; eraseInstr ( * UseMI , MRI ) ; UseMI = & * MRI . use_instr_nodbg_begin ( NegatedCond ) ; Negated = true ; } if ( UseMI -> getParent ( ) != Parent || UseMI -> getOpcode ( ) != <STR_LIT> ) return nullptr ; MachineBasicBlock :: iterator Next = std :: next ( UseMI -> getIterator ( ) ) ; if ( Next == Parent -> end ( ) ) { MachineFunction :: iterator NextMBB = std :: next ( Parent -> getIterator ( ) ) ; if ( NextMBB == Parent -> getParent ( ) -> end ( ) ) return nullptr ; UncondBrTarget = & * NextMBB ; } else { if ( Next -> getOpcode ( ) != <STR_LIT> ) return nullptr ;


<s> if ( Ty . isVector ( ) ) return Ty . changeElementCount ( ElementCount :: getFixed ( PowerOf2Ceil ( Ty . getNumElements ( ) ) ) ) ;

return LLT :: scalar ( PowerOf2Ceil ( Ty . getSizeInBits ( ) ) ) ;


<s> return LLT :: scalar ( PowerOf2Ceil ( Ty . getSizeInBits ( ) ) ) ;


<s> setAction ( { G_CONSTANT , S1 } , Legal ) ; setAction ( { G_CONSTANT , S32 } , Legal ) ; setAction ( { G_CONSTANT , S64 } , Legal ) ; setAction ( { G_FCONSTANT , S32 } , Legal ) ; setAction ( { G_FADD , S32 } , Legal ) ; setAction ( { G_FMUL , S32 } , Legal ) ; setAction ( { G_FPTOUI , S32 } , Legal ) ; setAction ( { G_FPTOUI , <NUM_LIT> , S32 } , Legal ) ; setAction ( { G_GEP , P1 } , Legal ) ; setAction ( { G_GEP , P2 } , Legal ) ; setAction ( { G_GEP , <NUM_LIT> , S64 } , Legal ) ; setAction ( { G_ICMP , S1 } , Legal ) ; setAction ( { G_ICMP , <NUM_LIT> , S32 } , Legal ) ; setAction ( { G_LOAD , P1 } , Legal ) ; setAction ( { G_LOAD , P2 } , Legal ) ; setAction ( { G_LOAD , S32 } , Legal ) ; setAction ( { G_LOAD , <NUM_LIT> , P1 } , Legal ) ; setAction ( { G_LOAD , <NUM_LIT> , P2 } , Legal ) ; setAction ( { G_OR , S32 } , Legal ) ; setAction ( { G_SELECT , S32 } , Legal ) ; setAction ( { G_SELECT , <NUM_LIT> , S1 } , Legal ) ;


<s> case TargetOpcode :: G_FDIV : return legalizeFDIV ( MI , MRI , B ) ; case TargetOpcode :: G_UDIV : case TargetOpcode :: G_UREM : return legalizeUDIV_UREM ( MI , MRI , B ) ; case TargetOpcode :: G_SDIV : case TargetOpcode :: G_SREM : return legalizeSDIV_SREM ( MI , MRI , B ) ; case TargetOpcode :: G_ATOMIC_CMPXCHG : return legalizeAtomicCmpXChg ( MI , MRI , B ) ; case TargetOpcode :: G_FLOG : return legalizeFlog ( MI , B , numbers :: ln2f ) ; case TargetOpcode :: G_FLOG10 : return legalizeFlog ( MI , B , numbers :: ln2f / numbers :: ln10f ) ; case TargetOpcode :: G_FEXP : return legalizeFExp ( MI , B ) ; case TargetOpcode :: G_FPOW : return legalizeFPow ( MI , B ) ; case TargetOpcode :: G_FFLOOR : return legalizeFFloor ( MI , MRI , B ) ; case TargetOpcode :: G_BUILD_VECTOR : return legalizeBuildVector ( MI , MRI , B ) ;


<s> auto Fma0 = B . buildFMA ( S32 , NegDivScale0 , ApproxRcp , One , Flags ) ; auto Fma1 = B . buildFMA ( S32 , Fma0 , ApproxRcp , ApproxRcp , Flags ) ; auto Mul = B . buildFMul ( S32 , NumeratorScaled , Fma1 , Flags ) ; auto Fma2 = B . buildFMA ( S32 , NegDivScale0 , Mul , NumeratorScaled , Flags ) ; auto Fma3 = B . buildFMA ( S32 , Fma2 , Fma1 , Mul , Flags ) ; auto Fma4 = B . buildFMA ( S32 , NegDivScale0 , Fma3 , NumeratorScaled , Flags ) ; if ( ! Mode . allFP32Denormals ( ) ) toggleSPDenormMode ( false , B , ST , Mode ) ;

auto Fmas = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( Fma4 . getReg ( <NUM_LIT> ) ) . addUse ( Fma1 . getReg ( <NUM_LIT> ) ) . addUse ( Fma3 . getReg ( <NUM_LIT> ) ) . addUse ( NumeratorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ;


<s> if ( ! Mode . allFP32Denormals ( ) ) toggleSPDenormMode ( true , B , ST , Mode ) ; auto Fma0 = B . buildFMA ( S32 , NegDivScale0 , ApproxRcp , One , Flags ) ; auto Fma1 = B . buildFMA ( S32 , Fma0 , ApproxRcp , ApproxRcp , Flags ) ; auto Mul = B . buildFMul ( S32 , NumeratorScaled , Fma1 , Flags ) ; auto Fma2 = B . buildFMA ( S32 , NegDivScale0 , Mul , NumeratorScaled , Flags ) ; auto Fma3 = B . buildFMA ( S32 , Fma2 , Fma1 , Mul , Flags ) ; auto Fma4 = B . buildFMA ( S32 , NegDivScale0 , Fma3 , NumeratorScaled , Flags ) ; if ( ! Mode . allFP32Denormals ( ) ) toggleSPDenormMode ( false , B , ST , Mode ) ; auto Fmas = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( Fma4 . getReg ( <NUM_LIT> ) ) . addUse ( Fma1 . getReg ( <NUM_LIT> ) ) . addUse ( Fma3 . getReg ( <NUM_LIT> ) ) . addUse ( NumeratorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ;

B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( Fmas . getReg ( <NUM_LIT> ) ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ;


<s> Register RHS = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; uint16_t Flags = MI . getFlags ( ) ; LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; LLT S1 = LLT :: scalar ( <NUM_LIT> ) ; auto One = B . buildFConstant ( S64 , <NUM_LIT> ) ; auto DivScale0 = B . buildIntrinsic ( <STR_LIT> , { S64 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto NegDivScale0 = B . buildFNeg ( S64 , DivScale0 . getReg ( <NUM_LIT> ) , Flags ) ; auto Rcp = B . buildIntrinsic ( <STR_LIT> , { S64 } , false ) . addUse ( DivScale0 . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto Fma0 = B . buildFMA ( S64 , NegDivScale0 , Rcp , One , Flags ) ; auto Fma1 = B . buildFMA ( S64 , Rcp , Fma0 , Rcp , Flags ) ; auto Fma2 = B . buildFMA ( S64 , NegDivScale0 , Fma1 , One , Flags ) ; auto DivScale1 = B . buildIntrinsic ( <STR_LIT> , { S64 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto Fma3 = B . buildFMA ( S64 , Fma1 , Fma2 , Fma1 , Flags ) ; auto Mul = B . buildFMul ( S64 , DivScale1 . getReg ( <NUM_LIT> ) , Fma3 , Flags ) ; auto Fma4 = B . buildFMA ( S64 , NegDivScale0 , Mul , DivScale1 . getReg ( <NUM_LIT> ) , Flags ) ; Register Scale ; if ( ! ST . hasUsableDivScaleConditionOutput ( ) ) { LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto NumUnmerge = B . buildUnmerge ( S32 , LHS ) ; auto DenUnmerge = B . buildUnmerge ( S32 , RHS ) ;

auto Scale0Unmerge = B . buildUnmerge ( S32 , DivScale0 ) ;


<s> if ( ST . hasTrigReducedRange ( ) ) { auto MulVal = B . buildFMul ( Ty , SrcReg , OneOver2Pi , Flags ) ; TrigVal = B . buildIntrinsic ( <STR_LIT> , { Ty } , false ) . addUse ( MulVal . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) . getReg ( <NUM_LIT> ) ; } else TrigVal = B . buildFMul ( Ty , SrcReg , OneOver2Pi , Flags ) . getReg ( <NUM_LIT> ) ; <STR_LIT> TrigIntrin = MI . getOpcode ( ) == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; B . buildIntrinsic ( TrigIntrin , makeArrayRef < Register > ( DstReg ) , false ) . addUse ( TrigVal ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ;


<s> case TargetOpcode :: G_ADDRSPACE_CAST : return legalizeAddrSpaceCast ( MI , MRI , B ) ; case TargetOpcode :: G_FRINT : return legalizeFrint ( MI , MRI , B ) ; case TargetOpcode :: G_FCEIL : return legalizeFceil ( MI , MRI , B ) ; case TargetOpcode :: G_INTRIN<ISA_LIT>C_TRUNC : return legalizeIntrinsicTrunc ( MI , MRI , B ) ; case TargetOpcode :: G_<ISA_LIT>TOFP : return legalizeITOFP ( MI , MRI , B , true ) ; case TargetOpcode :: G_UITOFP : return legalizeITOFP ( MI , MRI , B , false ) ; case TargetOpcode :: G_FMINNUM : case TargetOpcode :: G_FMAXNUM : case TargetOpcode :: G_FMINNUM_IEEE : case TargetOpcode :: G_FMAXNUM_IEEE : return legalizeMinNumMaxNum ( MI , MRI , B ) ; case TargetOpcode :: G_EXTRACT_VECTOR_ELT :

return legalizeExtractVectorElt ( MI , MRI , B ) ;


<s> LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; LLT S1 = LLT :: scalar ( <NUM_LIT> ) ; auto Abs = B . buildFAbs ( S32 , RHS , Flags ) ; const APFloat C0Val ( <NUM_LIT> ) ; auto C0 = B . buildConstant ( S32 , <NUM_LIT> ) ; auto C1 = B . buildConstant ( S32 , <NUM_LIT> ) ; auto C2 = B . buildConstant ( S32 , FloatToBits ( <NUM_LIT> ) ) ; auto CmpRes = B . buildFCmp ( CmpInst :: FCMP_OGT , S1 , Abs , C0 , Flags ) ; auto Sel = B . buildSelect ( S32 , CmpRes , C1 , C2 , Flags ) ;

auto Mul0 = B . buildFMul ( S32 , RHS , Sel , Flags ) ;


<s> } return false ; } case <STR_LIT> : { if ( MachineInstr * BrCond = verifyCFIntrinsic ( MI , MRI ) ) { const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; B . setInstr ( * BrCond ) ; Register Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; B . buildInstr ( <STR_LIT> ) . addUse ( Reg ) . addMBB ( BrCond -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MI . eraseFromParent ( ) ; BrCond -> eraseFromParent ( ) ; MRI . setRegClass ( Reg , TRI -> getWaveMaskRegClass ( ) ) ; return true ; } return false ; } case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeImplicitArgPtr ( MI , MRI , B ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> :


<s> return LLT :: scalar ( Size ) ; } return LLT :: scalarOrVector ( ElementCount :: getFixed ( Size / <NUM_LIT> ) , <NUM_LIT> ) ;


<s> const unsigned Size = Ty . getSizeInBits ( ) ; if ( Size <= <NUM_LIT> ) { return LLT :: scalar ( Size ) ; } return LLT :: scalarOrVector ( ElementCount :: getFixed ( Size / <NUM_LIT> ) , <NUM_LIT> ) ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; assert ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; if ( ST . hasApertureRegs ( ) ) { unsigned Offset = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: OFFSET_SRC_SHARED_BASE : <STR_LIT> :: OFFSET_SRC_PRIVATE_BASE ; unsigned WidthM1 = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: WIDTH_M1_SRC_SHARED_BASE : <STR_LIT> :: WIDTH_M1_SRC_PRIVATE_BASE ; unsigned Encoding = <STR_LIT> :: ID_MEM_BASES << <STR_LIT> :: ID_SHIFT_ | Offset << <STR_LIT> :: OFFSET_SHIFT_ | WidthM1 << <STR_LIT> :: WIDTH_M1_SHIFT_ ; Register GetReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( GetReg ) . addImm ( Encoding ) ; MRI . setType ( GetReg , S32 ) ; auto ShiftAmt = B . buildConstant ( S32 , WidthM1 + <NUM_LIT> ) ; return B . buildShl ( S32 , GetReg , ShiftAmt ) . getReg ( <NUM_LIT> ) ; } MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; Register LoadAddr = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ;


<s> Register KernargPtrReg = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; if ( ! loadInputValue ( KernargPtrReg , B , <STR_LIT> ) ) return Register ( ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , LLT :: scalar ( <NUM_LIT> ) , commonAlignment ( Align ( <NUM_LIT> ) , Offset ) ) ; B . buildPtrAdd ( LoadAddr , KernargPtrReg , B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , Offset ) . getReg ( <NUM_LIT> ) ) ; return B . buildLoad ( S32 , LoadAddr , * MMO ) . getReg ( <NUM_LIT> ) ; } Register QueuePtr = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; if ( ! loadInputValue ( QueuePtr , B , <STR_LIT> ) ) return Register ( ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ;

MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , LLT :: scalar ( <NUM_LIT> ) , commonAlignment ( Align ( <NUM_LIT> ) , StructOffset ) ) ;


<s> case <STR_LIT> : { const ConstantInt * CI = Def -> getOperand ( <NUM_LIT> ) . getCImm ( ) ; return CI -> getSExtValue ( ) != TM . getNullPointerValue ( AddrSpace ) ;


<s> unsigned RegSize = Ty . getSizeInBits ( ) ; uint64_t MemSize = Query . MMODescrs [ <NUM_LIT> ] . MemoryTy . getSizeInBits ( ) ; uint64_t AlignBits = Query . MMODescrs [ <NUM_LIT> ] . AlignInBits ; unsigned AS = Query . Types [ <NUM_LIT> ] . getAddressSpace ( ) ; if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) return false ; if ( Ty . isVector ( ) && MemSize != RegSize ) return false ; if ( IsLoad && MemSize < Size ) MemSize = std :: max ( MemSize , Align ) ; if ( MemSize != RegSize && RegSize != <NUM_LIT> ) return false ; if ( MemSize > maxSizeForAddrSpace ( ST , AS , IsLoad ) ) return false ; switch ( MemSize ) { case <NUM_LIT> : case <NUM_LIT> : case <NUM_LIT> : case <NUM_LIT> : case <NUM_LIT> : break ; case <NUM_LIT> : if ( ! ST . hasDwordx3LoadStores ( ) ) return false ; break ; case <NUM_LIT> : case <NUM_LIT> : break ; default : return false ; }


<s> auto BuildPtr = B . buildMerge ( DstTy , { SrcAsInt , ApertureReg } ) ; if ( isKnownNonNull ( Src , MRI , TM , SrcAS ) ) { B . buildCopy ( Dst , BuildPtr ) ; MI . eraseFromParent ( ) ; return true ; } auto SegmentNull = B . buildConstant ( SrcTy , TM . getNullPointerValue ( SrcAS ) ) ; auto FlatNull = B . buildConstant ( DstTy , TM . getNullPointerValue ( DestAS ) ) ; auto CmpRes = B . buildICmp ( CmpInst :: ICMP_NE , LLT :: scalar ( <NUM_LIT> ) , Src , SegmentNull . getReg ( <NUM_LIT> ) ) ; B . buildSelect ( Dst , CmpRes , BuildPtr , FlatNull ) ; MI . eraseFromParent ( ) ; return true ; } if ( DestAS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT && SrcTy . getSizeInBits ( ) == <NUM_LIT> ) { B . buildExtract ( Dst , Src , <NUM_LIT> ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT && DstTy . getSizeInBits ( ) == <NUM_LIT> ) { const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; uint32_t AddrHiVal = Info -> get32BitAddressHighBits ( ) ; auto HighAddr = B . buildConstant ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT , <NUM_LIT> ) , AddrHiVal ) ;

B . buildMerge ( Dst , { Src , HighAddr } ) ;


<s> return legalizeITOFP ( MI , MRI , B , false ) ; case TargetOpcode :: G_FPTO<ISA_LIT> : return legalizeFPTOI ( MI , MRI , B , true ) ; case TargetOpcode :: G_FPTOUI : return legalizeFPTOI ( MI , MRI , B , false ) ; case TargetOpcode :: G_FMINNUM : case TargetOpcode :: G_FMAXNUM : case TargetOpcode :: G_FMINNUM_IEEE : case TargetOpcode :: G_FMAXNUM_IEEE : return legalizeMinNumMaxNum ( Helper , MI ) ; case TargetOpcode :: G_EXTRACT_VECTOR_ELT : return legalizeExtractVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_INSERT_VECTOR_ELT : return legalizeInsertVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return legalizeShuffleVector ( MI , MRI , B ) ; case TargetOpcode :: G_F<ISA_LIT>N : case TargetOpcode :: G_FCOS : return legalizeSinCos ( MI , MRI , B ) ; case TargetOpcode :: G_GLOBAL_VALUE : return legalizeGlobalValue ( MI , MRI , B ) ; case TargetOpcode :: G_LOAD : case TargetOpcode :: G_SEXTLOAD : case TargetOpcode :: G_ZEXTLOAD : return legalizeLoad ( Helper , MI ) ; case TargetOpcode :: G_FMAD : return legalizeFMad ( MI , MRI , B ) ;


<s> case TargetOpcode :: G_ADDRSPACE_CAST : return legalizeAddrSpaceCast ( MI , MRI , B ) ; case TargetOpcode :: G_FRINT : return legalizeFrint ( MI , MRI , B ) ; case TargetOpcode :: G_FCEIL : return legalizeFceil ( MI , MRI , B ) ; case TargetOpcode :: G_FREM : return legalizeFrem ( MI , MRI , B ) ; case TargetOpcode :: G_INTRIN<ISA_LIT>C_TRUNC : return legalizeIntrinsicTrunc ( MI , MRI , B ) ; case TargetOpcode :: G_<ISA_LIT>TOFP : return legalizeITOFP ( MI , MRI , B , true ) ; case TargetOpcode :: G_UITOFP : return legalizeITOFP ( MI , MRI , B , false ) ; case TargetOpcode :: G_FPTO<ISA_LIT> : return legalizeFPTOI ( MI , MRI , B , true ) ; case TargetOpcode :: G_FPTOUI : return legalizeFPTOI ( MI , MRI , B , false ) ; case TargetOpcode :: G_FMINNUM : case TargetOpcode :: G_FMAXNUM : case TargetOpcode :: G_FMINNUM_IEEE : case TargetOpcode :: G_FMAXNUM_IEEE : return legalizeMinNumMaxNum ( Helper , MI ) ; case TargetOpcode :: G_EXTRACT_VECTOR_ELT : return legalizeExtractVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_INSERT_VECTOR_ELT : return legalizeInsertVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return legalizeShuffleVector ( MI , MRI , B ) ; case TargetOpcode :: G_F<ISA_LIT>N : case TargetOpcode :: G_FCOS : return legalizeSinCos ( MI , MRI , B ) ; case TargetOpcode :: G_GLOBAL_VALUE : return legalizeGlobalValue ( MI , MRI , B ) ; case TargetOpcode :: G_LOAD : case TargetOpcode :: G_SEXTLOAD : case TargetOpcode :: G_ZEXTLOAD : return legalizeLoad ( Helper , MI ) ; case TargetOpcode :: G_FMAD : return legalizeFMad ( MI , MRI , B ) ; case TargetOpcode :: G_FDIV : return legalizeFDIV ( MI , MRI , B ) ; case TargetOpcode :: G_UDIV : case TargetOpcode :: G_UREM : case TargetOpcode :: G_UDIVREM : return legalizeUnsignedDIV_REM ( MI , MRI , B ) ; case TargetOpcode :: G_SDIV : case TargetOpcode :: G_SREM : case TargetOpcode :: G_SDIVREM : return legalizeSignedDIV_REM ( MI , MRI , B ) ; case TargetOpcode :: G_ATOMIC_CMPXCHG : return legalizeAtomicCmpXChg ( MI , MRI , B ) ; case TargetOpcode :: G_FLOG : return legalizeFlog ( MI , B , numbers :: ln2f ) ; case TargetOpcode :: G_FLOG10 : return legalizeFlog ( MI , B , numbers :: ln2f / numbers :: ln10f ) ; case TargetOpcode :: G_FEXP : return legalizeFExp ( MI , B ) ; case TargetOpcode :: G_FPOW : return legalizeFPow ( MI , B ) ; case TargetOpcode :: G_FFLOOR : return legalizeFFloor ( MI , MRI , B ) ; case TargetOpcode :: G_BUILD_VECTOR :

return legalizeBuildVector ( MI , MRI , B ) ;


<s> bool <ISA_LIT>LegalizerInfo :: legalizeDSAtomicFPIntrinsic ( LegalizerHelper & Helper , MachineInstr & MI , <STR_LIT> IID ) const { GISelChangeObserver & Observer = Helper . Observer ;


<s> Observer . changingInstr ( MI ) ; MI . setDesc ( ST . getInstrInfo ( ) -> get ( getDSFPAtomicOpcode ( IID ) ) ) ; for ( int I = <NUM_LIT> ; I > <NUM_LIT> ; -- I ) MI . removeOperand ( I ) ; MI . removeOperand ( <NUM_LIT> ) ; Observer . changedInstr ( MI ) ; return true ;


<s> Optional < ValueAndVReg > MaybeIdxVal = getIConstantVRegValWithLookThrough ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI ) ; if ( ! MaybeIdxVal ) return true ; const int64_t IdxVal = MaybeIdxVal -> Value . getSExtValue ( ) ; Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Vec = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT VecTy = MRI . getType ( Vec ) ; LLT EltTy = VecTy . getElementType ( ) ; assert ( EltTy == MRI . getType ( Dst ) ) ; if ( IdxVal < VecTy . getNumElements ( ) ) { auto Unmerge = B . buildUnmerge ( EltTy , Vec ) ; B . buildCopy ( Dst , Unmerge . getReg ( IdxVal ) ) ; } else { B . buildUndef ( Dst ) ; } MI . eraseFromParent ( ) ;


<s> const int64_t IdxVal = MaybeIdxVal -> Value . getSExtValue ( ) ; Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Vec = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT VecTy = MRI . getType ( Vec ) ; LLT EltTy = VecTy . getElementType ( ) ; assert ( EltTy == MRI . getType ( Dst ) ) ; if ( IdxVal < VecTy . getNumElements ( ) ) { auto Unmerge = B . buildUnmerge ( EltTy , Vec ) ; B . buildCopy ( Dst , Unmerge . getReg ( IdxVal ) ) ; } else { B . buildUndef ( Dst ) ; }


<s> const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; Register Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( MRI . getType ( Src ) == S64 ) ; auto Trunc = B . buildIntrinsicTrunc ( S64 , Src ) ; const auto Zero = B . buildFConstant ( S64 , <NUM_LIT> ) ; const auto One = B . buildFConstant ( S64 , <NUM_LIT> ) ; auto Lt0 = B . buildFCmp ( CmpInst :: FCMP_OGT , S1 , Src , Zero ) ; auto NeTrunc = B . buildFCmp ( CmpInst :: FCMP_ONE , S1 , Src , Trunc ) ; auto And = B . buildAnd ( S1 , Lt0 , NeTrunc ) ; auto Add = B . buildSelect ( S64 , And , One , Zero ) ; B . buildFAdd ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Trunc , Add ) ;


<s> int RoundMode = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( RoundMode == ( int ) RoundingMode :: TowardPositive ) Opc = <STR_LIT> ; else if ( RoundMode == ( int ) RoundingMode :: TowardNegative ) Opc = <STR_LIT> ; else return false ; B . buildInstr ( Opc ) . addDef ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . addUse ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MI . eraseFromParent ( ) ; return true ;


<s> Register Vec = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Ins = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT VecTy = MRI . getType ( Vec ) ; LLT EltTy = VecTy . getElementType ( ) ; assert ( EltTy == MRI . getType ( Ins ) ) ; ( void ) Ins ; unsigned NumElts = VecTy . getNumElements ( ) ; if ( IdxVal < NumElts ) { SmallVector < Register , <NUM_LIT> > SrcRegs ; for ( unsigned i = <NUM_LIT> ; i < NumElts ; ++ i ) SrcRegs . push_back ( MRI . createGenericVirtualRegister ( EltTy ) ) ; B . buildUnmerge ( SrcRegs , Vec ) ; SrcRegs [ IdxVal ] = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; B . buildMerge ( Dst , SrcRegs ) ; } else { B . buildUndef ( Dst ) ; }


<s> if ( IdxVal < NumElts ) { SmallVector < Register , <NUM_LIT> > SrcRegs ; for ( unsigned i = <NUM_LIT> ; i < NumElts ; ++ i ) SrcRegs . push_back ( MRI . createGenericVirtualRegister ( EltTy ) ) ; B . buildUnmerge ( SrcRegs , Vec ) ; SrcRegs [ IdxVal ] = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; B . buildMerge ( Dst , SrcRegs ) ;


<s> LLT PtrTy = MRI . getType ( PtrReg ) ; unsigned AddrSpace = PtrTy . getAddressSpace ( ) ; if ( AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { LLT ConstPtr = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; auto Cast = B . buildAddrSpaceCast ( ConstPtr , PtrReg ) ; Observer . changingInstr ( MI ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( Cast . getReg ( <NUM_LIT> ) ) ; Observer . changedInstr ( MI ) ; return true ; } if ( MI . getOpcode ( ) != <STR_LIT> ) return false ; Register ValReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT ValTy = MRI . getType ( ValReg ) ; MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; const unsigned ValSize = ValTy . getSizeInBits ( ) ; const LLT MemTy = MMO -> getMemoryType ( ) ; const Align MemAlign = MMO -> getAlign ( ) ; const unsigned MemSize = MemTy . getSizeInBits ( ) ; const uint64_t AlignInBits = <NUM_LIT> * MemAlign . value ( ) ; if ( shouldWidenLoad ( ST , MemTy , AlignInBits , AddrSpace , MI . getOpcode ( ) ) ) { const unsigned WideMemSize = PowerOf2Ceil ( MemSize ) ; if ( WideMemSize == ValSize ) { MachineFunction & MF = B . getMF ( ) ; MachineMemOperand * WideMMO = MF . getMachineMemOperand ( MMO , <NUM_LIT> , WideMemSize / <NUM_LIT> ) ; Observer . changingInstr ( MI ) ; MI . setMemRefs ( MF , { WideMMO } ) ; Observer . changedInstr ( MI ) ; return true ; } if ( ValSize > WideMemSize ) return false ; LLT WideTy = widenToNextPowerOf2 ( ValTy ) ; Register WideLoad ; if ( ! WideTy . isVector ( ) ) { WideLoad = B . buildLoadFromOffset ( WideTy , PtrReg , * MMO , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . buildTrunc ( ValReg , WideLoad ) . getReg ( <NUM_LIT> ) ; } else { if ( isRegisterType ( ValTy ) ) { WideLoad = B . buildLoadFromOffset ( WideTy , PtrReg , * MMO , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . buildExtract ( ValReg , WideLoad , <NUM_LIT> ) ; } else { WideLoad = B . buildLoadFromOffset ( WideTy , PtrReg , * MMO , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . buildDeleteTrailingVectorElements ( ValReg , WideLoad ) ; }


<s> MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , MemSize , MemAlign ) ; MI . addMemOperand ( MF , MMO ) ; if ( ! isPowerOf2_32 ( Size ) ) { if ( Ty . isVector ( ) ) Helper . moreElementsVectorDst ( MI , getPow2VectorType ( Ty ) , <NUM_LIT> ) ; else Helper . widenScalarDst ( MI , getPow2ScalarType ( Ty ) , <NUM_LIT> ) ; } Observer . changedInstr ( MI ) ;


<s> Register KernargPtrReg = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; if ( ! loadInputValue ( KernargPtrReg , B , <STR_LIT> ) ) return false ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , LLT :: scalar ( <NUM_LIT> ) , commonAlignment ( Align ( <NUM_LIT> ) , Offset ) ) ; Register LoadAddr = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; B . buildPtrAdd ( LoadAddr , KernargPtrReg , B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , Offset ) . getReg ( <NUM_LIT> ) ) ; Register Temp = B . buildLoad ( S64 , LoadAddr , * MMO ) . getReg ( <NUM_LIT> ) ; B . buildCopy ( SGPR01 , Temp ) ;

B . buildInstr ( <STR_LIT> ) . addImm ( static_cast < unsigned > ( GCNSubtarget :: TrapID :: LLVMAMDHSATrap ) ) . addReg ( SGPR01 , RegState :: Implicit ) ;


<s> if ( Optional < uint8_t > HsaAbiVer = <STR_LIT> ( & ST ) ) { switch ( * HsaAbiVer ) { case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V2 : case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V3 : return legalizeTrapHsaQueuePtr ( MI , MRI , B ) ; case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V4 :

case ELF :: ELFABIVER<ISA_LIT>ON_AMDGPU_HSA_V5 :


<s> const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const unsigned Mask = Arg -> getMask ( ) ; const unsigned Shift = countTrailingZeros < unsigned > ( Mask ) ; Register AndMaskSrc = LiveIn ; if ( Shift != <NUM_LIT> ) { auto ShiftAmt = B . buildConstant ( S32 , Shift ) ; AndMaskSrc = B . buildLShr ( S32 , LiveIn , ShiftAmt ) . getReg ( <NUM_LIT> ) ; }

B . buildAnd ( DstReg , AndMaskSrc , B . buildConstant ( S32 , Mask >> Shift ) ) ;


<s> for ( unsigned I = Intr -> VAddrStart ; I < EndIdx ; I ++ ) { MachineOperand & SrcOp = MI . getOperand ( ArgOffset + I ) ; if ( ! SrcOp . isReg ( ) ) continue ; Register AddrReg = SrcOp . getReg ( ) ; if ( ( I < Intr -> GradientStart ) || ( I >= Intr -> GradientStart && I < Intr -> CoordStart && ! IsG16 ) || ( I >= Intr -> CoordStart && ! IsA16 ) ) { if ( ( I < Intr -> GradientStart ) && IsA16 && ( B . getMRI ( ) -> getType ( AddrReg ) == S16 ) ) { assert ( I == Intr -> BiasIndex && <STR_LIT> ) ; PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) } ) . getReg ( <NUM_LIT> ) ) ; } else { assert ( ( ! IsA16 || Intr -> NumBiasArgs == <NUM_LIT> || I != Intr -> BiasIndex ) && <STR_LIT> ) ;

AddrReg = B . buildBitcast ( V2S16 , AddrReg ) . getReg ( <NUM_LIT> ) ;


<s> if ( ( I < Intr -> GradientStart ) && IsA16 && ( B . getMRI ( ) -> getType ( AddrReg ) == S16 ) ) { assert ( I == Intr -> BiasIndex && <STR_LIT> ) ; PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) } ) . getReg ( <NUM_LIT> ) ) ; } else { assert ( ( ! IsA16 || Intr -> NumBiasArgs == <NUM_LIT> || I != Intr -> BiasIndex ) && <STR_LIT> ) ; AddrReg = B . buildBitcast ( V2S16 , AddrReg ) . getReg ( <NUM_LIT> ) ; PackedAddrs . push_back ( AddrReg ) ; } } else { if ( ( ( I + <NUM_LIT> ) >= EndIdx ) || ( ( Intr -> NumGradients / <NUM_LIT> ) % <NUM_LIT> == <NUM_LIT> && ( I == static_cast < unsigned > ( Intr -> GradientStart + ( Intr -> NumGradients / <NUM_LIT> ) - <NUM_LIT> ) || I == static_cast < unsigned > ( Intr -> GradientStart + Intr -> NumGradients - <NUM_LIT> ) ) ) || ! MI . getOperand ( ArgOffset + I + <NUM_LIT> ) . isReg ( ) ) { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) } ) . getReg ( <NUM_LIT> ) ) ; } else { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , MI . getOperand ( ArgOffset + I + <NUM_LIT> ) . getReg ( ) } ) . getReg ( <NUM_LIT> ) ) ; ++ I ; } }


<s> B . buildConstant ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , C ) ; MI . eraseFromParent ( ) ;


<s> B . buildConstant ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , C ) ;


<s> unsigned SizeInBits = MemoryTy . getSizeInBits ( ) ; if ( isPowerOf2_32 ( SizeInBits ) ) return false ; if ( SizeInBits == <NUM_LIT> && ST . hasDwordx3LoadStores ( ) ) return false ; if ( SizeInBits >= maxSizeForAddrSpace ( ST , AddrSpace , Opcode ) ) return false ; unsigned RoundedSize = NextPowerOf2 ( SizeInBits ) ; if ( AlignInBits < RoundedSize ) return false ; const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; bool Fast = false ; return TLI -> allowsMisalignedMemoryAccessesImpl ( RoundedSize , AddrSpace , Align ( AlignInBits / <NUM_LIT> ) , MachineMemOperand :: MOLoad , & Fast ) && Fast ;


<s> if ( isPowerOf2_32 ( SizeInBits ) ) return false ; if ( SizeInBits == <NUM_LIT> && ST . hasDwordx3LoadStores ( ) ) return false ; if ( SizeInBits >= maxSizeForAddrSpace ( ST , AddrSpace , Opcode ) ) return false ;

unsigned RoundedSize = NextPowerOf2 ( SizeInBits ) ;


<s> unsigned Size = Ty . getSizeInBits ( ) ; assert ( Size % <NUM_LIT> == <NUM_LIT> ) ; return std :: make_pair ( TypeIdx , LLT :: scalarOrVector ( Size / <NUM_LIT> , <NUM_LIT> ) ) ; } ;


<s> return [ = ] ( const LegalityQuery & Query ) { const LLT Ty = Query . Types [ TypeIdx ] ; unsigned Size = Ty . getSizeInBits ( ) ; assert ( Size % <NUM_LIT> == <NUM_LIT> ) ;

return std :: make_pair ( TypeIdx , LLT :: scalarOrVector ( Size / <NUM_LIT> , <NUM_LIT> ) ) ;


<s> return LLT :: scalar ( Size ) ; } return LLT :: scalarOrVector ( Size / <NUM_LIT> , <NUM_LIT> ) ;


<s> case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ;


<s> unsigned Encoding = <STR_LIT> :: ID_MEM_BASES << <STR_LIT> :: ID_SHIFT_ | Offset << <STR_LIT> :: OFFSET_SHIFT_ | WidthM1 << <STR_LIT> :: WIDTH_M1_SHIFT_ ; Register GetReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( GetReg ) . addImm ( Encoding ) ; MRI . setType ( GetReg , S32 ) ; auto ShiftAmt = B . buildConstant ( S32 , WidthM1 + <NUM_LIT> ) ; return B . buildShl ( S32 , GetReg , ShiftAmt ) . getReg ( <NUM_LIT> ) ; }

Register QueuePtr = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ;


<s> Register <ISA_LIT>LegalizerInfo :: handleD16VData ( MachineIRBuilder & B , MachineRegisterInfo & MRI , Register Reg , bool ImageStore ) const { const LLT S16 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; LLT StoreVT = MRI . getType ( Reg ) ; assert ( StoreVT . isVector ( ) && StoreVT . getElementType ( ) == S16 ) ; if ( ST . hasUnpackedD16VMem ( ) ) { auto Unmerge = B . buildUnmerge ( S16 , Reg ) ; SmallVector < Register , <NUM_LIT> > WideRegs ; for ( int I = <NUM_LIT> , E = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; I != E ; ++ I ) WideRegs . push_back ( B . buildAnyExt ( S32 , Unmerge . getReg ( I ) ) . getReg ( <NUM_LIT> ) ) ; int NumElts = StoreVT . getNumElements ( ) ; return B . buildBuildVector ( LLT :: vector ( NumElts , S32 ) , WideRegs ) . getReg ( <NUM_LIT> ) ; } if ( ImageStore && ST . hasImageStoreD16Bug ( ) ) { if ( StoreVT . getNumElements ( ) == <NUM_LIT> ) { SmallVector < Register , <NUM_LIT> > PackedRegs ; Reg = B . buildBitcast ( S32 , Reg ) . getReg ( <NUM_LIT> ) ; PackedRegs . push_back ( Reg ) ;

PackedRegs . resize ( <NUM_LIT> , B . buildUndef ( S32 ) . getReg ( <NUM_LIT> ) ) ;


<s> SmallVector < Register , <NUM_LIT> > WideRegs ; for ( int I = <NUM_LIT> , E = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; I != E ; ++ I ) WideRegs . push_back ( B . buildAnyExt ( S32 , Unmerge . getReg ( I ) ) . getReg ( <NUM_LIT> ) ) ; int NumElts = StoreVT . getNumElements ( ) ; return B . buildBuildVector ( LLT :: vector ( NumElts , S32 ) , WideRegs ) . getReg ( <NUM_LIT> ) ; } if ( ImageStore && ST . hasImageStoreD16Bug ( ) ) { if ( StoreVT . getNumElements ( ) == <NUM_LIT> ) { SmallVector < Register , <NUM_LIT> > PackedRegs ; Reg = B . buildBitcast ( S32 , Reg ) . getReg ( <NUM_LIT> ) ; PackedRegs . push_back ( Reg ) ; PackedRegs . resize ( <NUM_LIT> , B . buildUndef ( S32 ) . getReg ( <NUM_LIT> ) ) ; return B . buildBuildVector ( LLT :: vector ( <NUM_LIT> , S32 ) , PackedRegs ) . getReg ( <NUM_LIT> ) ; } if ( StoreVT . getNumElements ( ) == <NUM_LIT> ) { SmallVector < Register , <NUM_LIT> > PackedRegs ; auto Unmerge = B . buildUnmerge ( S16 , Reg ) ; for ( int I = <NUM_LIT> , E = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; I != E ; ++ I ) PackedRegs . push_back ( Unmerge . getReg ( I ) ) ; PackedRegs . resize ( <NUM_LIT> , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) ) ; Reg = B . buildBuildVector ( LLT :: vector ( <NUM_LIT> , S16 ) , PackedRegs ) . getReg ( <NUM_LIT> ) ; return B . buildBitcast ( LLT :: vector ( <NUM_LIT> , S32 ) , Reg ) . getReg ( <NUM_LIT> ) ; } if ( StoreVT . getNumElements ( ) == <NUM_LIT> ) { SmallVector < Register , <NUM_LIT> > PackedRegs ; Reg = B . buildBitcast ( LLT :: vector ( <NUM_LIT> , S32 ) , Reg ) . getReg ( <NUM_LIT> ) ; auto Unmerge = B . buildUnmerge ( S32 , Reg ) ; for ( int I = <NUM_LIT> , E = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; I != E ; ++ I ) PackedRegs . push_back ( Unmerge . getReg ( I ) ) ;


<s> Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register CmpVal = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register NewVal = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;

assert ( <STR_LIT> ( MRI . getType ( PtrReg ) . getAddressSpace ( ) ) && <STR_LIT> ) ;


<s> LLT VecTy = LLT :: vector ( <NUM_LIT> , ValTy ) ; Register PackedVal = B . buildBuildVector ( VecTy , { NewVal , CmpVal } ) . getReg ( <NUM_LIT> ) ;

B . buildInstr ( <STR_LIT> ) . addDef ( DstReg ) . addUse ( PtrReg ) . addUse ( PackedVal ) . setMemRefs ( MI . memoperands ( ) ) ;


<s> const bool HasVIndex = MI . getNumOperands ( ) == NumVIndexOps ; Register VIndex ; if ( HasVIndex ) { VIndex = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; ++ OpOffset ; } Register VOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; Register SOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; unsigned AuxiliaryData = MI . getOperand ( <NUM_LIT> + OpOffset ) . getImm ( ) ; MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; unsigned ImmOffset ; unsigned TotalOffset ; std :: tie ( VOffset , ImmOffset , TotalOffset ) = splitBufferOffsets ( B , VOffset ) ; if ( TotalOffset != <NUM_LIT> ) MMO = B . getMF ( ) . getMachineMemOperand ( MMO , TotalOffset , MMO -> getSize ( ) ) ; if ( ! VIndex ) VIndex = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; auto MIB = B . buildInstr ( getBufferAtomicPseudo ( IID ) ) ;


<s> } Register VData = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; Register CmpVal ; if ( IsCmpSwap ) { CmpVal = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; ++ OpOffset ; } Register RSrc = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; const unsigned NumVIndexOps = ( IsCmpSwap ? <NUM_LIT> : <NUM_LIT> ) + HasReturn ; const bool HasVIndex = MI . getNumOperands ( ) == NumVIndexOps ; Register VIndex ; if ( HasVIndex ) { VIndex = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; ++ OpOffset ; } Register VOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; Register SOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; unsigned AuxiliaryData = MI . getOperand ( <NUM_LIT> + OpOffset ) . getImm ( ) ; MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; unsigned ImmOffset ; unsigned TotalOffset ; std :: tie ( VOffset , ImmOffset , TotalOffset ) = splitBufferOffsets ( B , VOffset ) ; if ( TotalOffset != <NUM_LIT> ) MMO = B . getMF ( ) . getMachineMemOperand ( MMO , TotalOffset , MMO -> getSize ( ) ) ; if ( ! VIndex ) VIndex = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; auto MIB = B . buildInstr ( getBufferAtomicPseudo ( IID ) ) ;


<s> } ; packLanes ( RayOrigin ) ; if ( IsA16 ) { auto UnmergeRayDir = B . buildUnmerge ( { S16 , S16 , S16 , S16 } , RayDir ) ; auto UnmergeRayInvDir = B . buildUnmerge ( { S16 , S16 , S16 , S16 } , RayInvDir ) ; Register R1 = MRI . createGenericVirtualRegister ( S32 ) ; Register R2 = MRI . createGenericVirtualRegister ( S32 ) ; Register R3 = MRI . createGenericVirtualRegister ( S32 ) ; B . buildMerge ( R1 , { UnmergeRayDir . getReg ( <NUM_LIT> ) , UnmergeRayDir . getReg ( <NUM_LIT> ) } ) ; B . buildMerge ( R2 , { UnmergeRayDir . getReg ( <NUM_LIT> ) , UnmergeRayInvDir . getReg ( <NUM_LIT> ) } ) ; B . buildMerge ( R3 , { UnmergeRayInvDir . getReg ( <NUM_LIT> ) , UnmergeRayInvDir . getReg ( <NUM_LIT> ) } ) ; Ops . push_back ( R1 ) ; Ops . push_back ( R2 ) ; Ops . push_back ( R3 ) ; } else { packLanes ( RayDir ) ; packLanes ( RayInvDir ) ; }

auto MIB = B . buildInstr ( <STR_LIT> ) . addDef ( DstReg ) . addImm ( Opcode ) ;


<s> return legalizeExtractVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_INSERT_VECTOR_ELT : return legalizeInsertVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return legalizeShuffleVector ( MI , MRI , B ) ; case TargetOpcode :: G_F<ISA_LIT>N : case TargetOpcode :: G_FCOS : return legalizeSinCos ( MI , MRI , B ) ; case TargetOpcode :: G_GLOBAL_VALUE : return legalizeGlobalValue ( MI , MRI , B ) ; case TargetOpcode :: G_LOAD : return legalizeLoad ( Helper , MI ) ; case TargetOpcode :: G_FMAD : return legalizeFMad ( MI , MRI , B ) ; case TargetOpcode :: G_FDIV : return legalizeFDIV ( MI , MRI , B ) ; case TargetOpcode :: G_UDIV : case TargetOpcode :: G_UREM : return legalizeUDIV_UREM ( MI , MRI , B ) ; case TargetOpcode :: G_SDIV : case TargetOpcode :: G_SREM : return legalizeSDIV_SREM ( MI , MRI , B ) ; case TargetOpcode :: G_ATOMIC_CMPXCHG : return legalizeAtomicCmpXChg ( MI , MRI , B ) ; case TargetOpcode :: G_FLOG : return legalizeFlog ( MI , B , numbers :: ln2f ) ; case TargetOpcode :: G_FLOG10 : return legalizeFlog ( MI , B , numbers :: ln2f / numbers :: ln10f ) ; case TargetOpcode :: G_FEXP : return legalizeFExp ( MI , B ) ; case TargetOpcode :: G_FPOW : return legalizeFPow ( MI , B ) ;


<s> case TargetOpcode :: G_FMAD : return legalizeFMad ( MI , MRI , B ) ; case TargetOpcode :: G_FDIV : return legalizeFDIV ( MI , MRI , B ) ; case TargetOpcode :: G_UDIV : case TargetOpcode :: G_UREM : return legalizeUDIV_UREM ( MI , MRI , B ) ; case TargetOpcode :: G_SDIV : case TargetOpcode :: G_SREM : return legalizeSDIV_SREM ( MI , MRI , B ) ; case TargetOpcode :: G_ATOMIC_CMPXCHG : return legalizeAtomicCmpXChg ( MI , MRI , B ) ; case TargetOpcode :: G_FLOG : return legalizeFlog ( MI , B , numbers :: ln2f ) ; case TargetOpcode :: G_FLOG10 : return legalizeFlog ( MI , B , numbers :: ln2f / numbers :: ln10f ) ; case TargetOpcode :: G_FEXP : return legalizeFExp ( MI , B ) ; case TargetOpcode :: G_FPOW : return legalizeFPow ( MI , B ) ; case TargetOpcode :: G_FFLOOR : return legalizeFFloor ( MI , MRI , B ) ;


<s> bool <ISA_LIT>LegalizerInfo :: legalizeExtractVectorElt ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { Optional < ValueAndVReg > MaybeIdxVal = getConstantVRegValWithLookThrough ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI ) ; if ( ! MaybeIdxVal ) return true ; const int64_t IdxVal = MaybeIdxVal -> Value . getSExtValue ( ) ; Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Vec = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT VecTy = MRI . getType ( Vec ) ;

LLT EltTy = VecTy . getElementType ( ) ;


<s> assert ( EltTy == MRI . getType ( Dst ) ) ; if ( IdxVal < VecTy . getNumElements ( ) ) B . buildExtract ( Dst , Vec , IdxVal * EltTy . getSizeInBits ( ) ) ; else B . buildUndef ( Dst ) ; MI . eraseFromParent ( ) ;


<s> unsigned AS = Ty . getAddressSpace ( ) ; const GlobalValue * GV = MI . getOperand ( <NUM_LIT> ) . getGlobal ( ) ; MachineFunction & MF = B . getMF ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) { if ( ! MFI -> isModuleEntryFunction ( ) ) { const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadLDSDecl ( Fn , <STR_LIT> , MI . getDebugLoc ( ) , DS_Warning ) ; Fn . getContext ( ) . diagnose ( BadLDSDecl ) ; B . buildIntrinsic ( <STR_LIT> , ArrayRef < Register > ( ) , true ) ; B . buildUndef ( DstReg ) ; MI . eraseFromParent ( ) ; return true ; } if ( ! <ISA_LIT>TargetLowering :: hasDefinedInitializer ( GV ) ) { const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( ! TLI -> shouldUseLDSConstAddress ( GV ) ) { MI . getOperand ( <NUM_LIT> ) . setTargetFlags ( <ISA_LIT>InstrInfo :: MO_ABS32_LO ) ; return true ; } if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS && GV -> hasExternalLinkage ( ) ) { Type * Ty = GV -> getValueType ( ) ; if ( B . getDataLayout ( ) . getTypeAllocSize ( Ty ) . isZero ( ) ) { MFI -> setDynLDSAlign ( B . getDataLayout ( ) , * cast < GlobalVariable > ( GV ) ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto Sz = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) ; B . buildIntToPtr ( DstReg , Sz ) ; MI . eraseFromParent ( ) ; return true ; } } B . buildConstant ( DstReg , MFI -> allocateLDSGlobal ( B . getDataLayout ( ) , * cast < GlobalVariable > ( GV ) ) ) ; MI . eraseFromParent ( ) ; return true ; } const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadInit ( Fn , <STR_LIT> , MI . getDebugLoc ( ) ) ; Fn . getContext ( ) . diagnose ( BadInit ) ; return true ; } const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( TLI -> shouldEmitFixup ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> ) ; MI . eraseFromParent ( ) ; return true ; } if ( TLI -> shouldEmitPCReloc ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_REL32 ) ; MI . eraseFromParent ( ) ; return true ;


<s> Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Vec = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Ins = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT VecTy = MRI . getType ( Vec ) ; LLT EltTy = VecTy . getElementType ( ) ; assert ( EltTy == MRI . getType ( Ins ) ) ; if ( IdxVal < VecTy . getNumElements ( ) ) B . buildInsert ( Dst , Vec , Ins , IdxVal * EltTy . getSizeInBits ( ) ) ; else B . buildUndef ( Dst ) ; MI . eraseFromParent ( ) ;


<s> MI . getOperand ( <NUM_LIT> ) . setReg ( Cast . getReg ( <NUM_LIT> ) ) ; Observer . changedInstr ( MI ) ; return true ; } Register ValReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT ValTy = MRI . getType ( ValReg ) ; MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; const unsigned ValSize = ValTy . getSizeInBits ( ) ; const unsigned MemSize = <NUM_LIT> * MMO -> getSize ( ) ; const Align MemAlign = MMO -> getAlign ( ) ; const unsigned AlignInBits = <NUM_LIT> * MemAlign . value ( ) ; if ( shouldWidenLoad ( ST , MemSize , AlignInBits , AddrSpace , MI . getOpcode ( ) ) ) { const unsigned WideMemSize = PowerOf2Ceil ( MemSize ) ; if ( WideMemSize == ValSize ) { MachineFunction & MF = B . getMF ( ) ; MachineMemOperand * WideMMO = MF . getMachineMemOperand ( MMO , <NUM_LIT> , WideMemSize / <NUM_LIT> ) ; Observer . changingInstr ( MI ) ; MI . setMemRefs ( MF , { WideMMO } ) ; Observer . changedInstr ( MI ) ; return true ; } if ( ValSize > WideMemSize ) return false ; LLT WideTy = widenToNextPowerOf2 ( ValTy ) ; Register WideLoad ; if ( ! WideTy . isVector ( ) ) { WideLoad = B . buildLoadFromOffset ( WideTy , PtrReg , * MMO , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ;

B . buildTrunc ( ValReg , WideLoad ) . getReg ( <NUM_LIT> ) ;


<s> Observer . changingInstr ( MI ) ; MI . setMemRefs ( MF , { WideMMO } ) ; Observer . changedInstr ( MI ) ; return true ; } if ( ValSize > WideMemSize ) return false ; LLT WideTy = widenToNextPowerOf2 ( ValTy ) ; Register WideLoad ; if ( ! WideTy . isVector ( ) ) { WideLoad = B . buildLoadFromOffset ( WideTy , PtrReg , * MMO , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . buildTrunc ( ValReg , WideLoad ) . getReg ( <NUM_LIT> ) ; } else { if ( isRegisterType ( ValTy ) ) { WideLoad = B . buildLoadFromOffset ( WideTy , PtrReg , * MMO , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . buildExtract ( ValReg , WideLoad , <NUM_LIT> ) ; } else { B . setInsertPt ( B . getMBB ( ) , ++ B . getInsertPt ( ) ) ; WideLoad = Helper . widenWithUnmerge ( WideTy , ValReg ) ; B . setInsertPt ( B . getMBB ( ) , MI . getIterator ( ) ) ; B . buildLoadFromOffset ( WideLoad , PtrReg , * MMO , <NUM_LIT> ) ; } } MI . eraseFromParent ( ) ; return true ;


<s> B . setInsertPt ( B . getMBB ( ) , MI ) ; } MI . setDesc ( B . getTII ( ) . get ( <STR_LIT> ) ) ; MI . RemoveOperand ( <NUM_LIT> ) ; const unsigned MemSize = ( Size + <NUM_LIT> ) / <NUM_LIT> ; const Align MemAlign ( <NUM_LIT> ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , MemSize , MemAlign ) ; MI . addMemOperand ( MF , MMO ) ; if ( ! isPowerOf2_32 ( Size ) ) {

if ( Ty . isVector ( ) ) Helper . moreElementsVectorDst ( MI , getPow2VectorType ( Ty ) , <NUM_LIT> ) ;


<s> MachineFunction & MF = B . getMF ( ) ; Observer . changingInstr ( MI ) ; if ( shouldBitcastLoadStoreType ( ST , Ty , Size ) ) { Ty = getBitcastRegisterType ( Ty ) ; Helper . bitcastDst ( MI , Ty , <NUM_LIT> ) ; Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; B . setInsertPt ( B . getMBB ( ) , MI ) ; } MI . setDesc ( B . getTII ( ) . get ( <STR_LIT> ) ) ; MI . RemoveOperand ( <NUM_LIT> ) ; const unsigned MemSize = ( Size + <NUM_LIT> ) / <NUM_LIT> ; const Align MemAlign ( <NUM_LIT> ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , MemSize , MemAlign ) ; MI . addMemOperand ( MF , MMO ) ; if ( ! isPowerOf2_32 ( Size ) ) { if ( Ty . isVector ( ) ) Helper . moreElementsVectorDst ( MI , getPow2VectorType ( Ty ) , <NUM_LIT> ) ; else Helper . widenScalarDst ( MI , getPow2ScalarType ( Ty ) , <NUM_LIT> ) ; } Observer . changedInstr ( MI ) ; return true ;


<s> } else { MachineRegisterInfo & MRI = * B . getMRI ( ) ; Register LiveIn = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ;

if ( ! loadInputValue ( LiveIn , B , <STR_LIT> ) ) return false ;


<s> MachineRegisterInfo & MRI = * B . getMRI ( ) ; Register LiveIn = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; if ( ! loadInputValue ( LiveIn , B , <STR_LIT> ) ) return false ; Register SGPR01 ( <STR_LIT> ) ; B . buildCopy ( SGPR01 , LiveIn ) ; B . buildInstr ( <STR_LIT> ) . addImm ( GCNSubtarget :: TrapIDLLVMTrap ) . addReg ( SGPR01 , RegState :: Implicit ) ; } MI . eraseFromParent ( ) ; return true ;


<s> const ArgDescriptor * Arg ; const TargetRegisterClass * ArgRC ; LLT ArgTy ; std :: tie ( Arg , ArgRC , ArgTy ) = MFI -> getPreloadedValue ( ArgType ) ;

if ( ! Arg -> isRegister ( ) || ! Arg -> getRegister ( ) . isValid ( ) ) return false ;


<s> Register AddrReg = SrcOp . getReg ( ) ; if ( I < Intr -> GradientStart ) { AddrReg = B . buildBitcast ( V2S16 , AddrReg ) . getReg ( <NUM_LIT> ) ; PackedAddrs . push_back ( AddrReg ) ; } else { if ( ( ( I + <NUM_LIT> ) >= EndIdx ) || ( ( Intr -> NumGradients / <NUM_LIT> ) % <NUM_LIT> == <NUM_LIT> && ( I == static_cast < unsigned > ( Intr -> GradientStart + ( Intr -> NumGradients / <NUM_LIT> ) - <NUM_LIT> ) || I == static_cast < unsigned > ( Intr -> GradientStart + Intr -> NumGradients - <NUM_LIT> ) ) ) || ! MI . getOperand ( ArgOffset + I + <NUM_LIT> ) . isReg ( ) ) { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) } ) . getReg ( <NUM_LIT> ) ) ; } else { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , MI . getOperand ( ArgOffset + I + <NUM_LIT> ) . getReg ( ) } ) . getReg ( <NUM_LIT> ) ) ; ++ I ; }


<s> static bool shouldBitcastLoadStoreType ( const GCNSubtarget & ST , const LLT Ty , const unsigned MemSizeInBits ) { const unsigned Size = Ty . getSizeInBits ( ) ; if ( Size != MemSizeInBits ) return Size <= <NUM_LIT> && Ty . isVector ( ) ; if ( loadStoreBitcastWorkaround ( Ty ) && isRegisterType ( Ty ) ) return true ;

return Ty . isVector ( ) && ( Size <= <NUM_LIT> || isRegisterSize ( Size ) ) && ! isRegisterVectorElementType ( Ty . getElementType ( ) ) ;


<s> if ( Size != MemSizeInBits ) return Size <= <NUM_LIT> && Ty . isVector ( ) ; if ( loadStoreBitcastWorkaround ( Ty ) && isRegisterType ( Ty ) ) return true ; return Ty . isVector ( ) && ( Size <= <NUM_LIT> || isRegisterSize ( Size ) ) && ! isRegisterVectorElementType ( Ty . getElementType ( ) ) ;


<s> return shouldWidenLoad ( ST , Query . MMODescrs [ <NUM_LIT> ] . SizeInBits , Query . MMODescrs [ <NUM_LIT> ] . AlignInBits , Query . Types [ <NUM_LIT> ] . getAddressSpace ( ) , Opcode ) ;


<s> UseMI = & * MRI . use_instr_nodbg_begin ( NegatedCond ) ; Negated = true ; } if ( UseMI -> getParent ( ) != Parent || UseMI -> getOpcode ( ) != <STR_LIT> ) return nullptr ; MachineBasicBlock :: iterator Next = std :: next ( UseMI -> getIterator ( ) ) ; if ( Next == Parent -> end ( ) ) { MachineFunction :: iterator NextMBB = std :: next ( Parent -> getIterator ( ) ) ;

if ( NextMBB == Parent -> getParent ( ) -> end ( ) ) return nullptr ;


<s> return LLT :: scalar ( PowerOf2Ceil ( Ty . getSizeInBits ( ) ) ) ;


<s> static LLT widenToNextPowerOf2 ( LLT Ty ) { if ( Ty . isVector ( ) ) return Ty . changeNumElements ( PowerOf2Ceil ( Ty . getNumElements ( ) ) ) ;

return LLT :: scalar ( PowerOf2Ceil ( Ty . getSizeInBits ( ) ) ) ;


<s> const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadLDSDecl ( Fn , <STR_LIT> , MI . getDebugLoc ( ) , DS_Warning ) ; Fn . getContext ( ) . diagnose ( BadLDSDecl ) ; B . buildIntrinsic ( <STR_LIT> , ArrayRef < Register > ( ) , true ) ; B . buildUndef ( DstReg ) ; MI . eraseFromParent ( ) ; return true ; } if ( ! <ISA_LIT>TargetLowering :: hasDefinedInitializer ( GV ) ) { const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( ! TLI -> shouldUseLDSConstAddress ( GV ) ) { MI . getOperand ( <NUM_LIT> ) . setTargetFlags ( <ISA_LIT>InstrInfo :: MO_ABS32_LO ) ; return true ; } if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS && GV -> hasExternalLinkage ( ) ) { Type * Ty = GV -> getValueType ( ) ; if ( B . getDataLayout ( ) . getTypeAllocSize ( Ty ) . isZero ( ) ) { MFI -> setDynLDSAlign ( B . getDataLayout ( ) , * cast < GlobalVariable > ( GV ) ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto Sz = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) ; B . buildIntToPtr ( DstReg , Sz ) ; MI . eraseFromParent ( ) ; return true ; } } B . buildConstant ( DstReg , MFI -> allocateLDSGlobal ( B . getDataLayout ( ) , * cast < GlobalVariable > ( GV ) ) ) ; MI . eraseFromParent ( ) ; return true ; } const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadInit ( Fn , <STR_LIT> , MI . getDebugLoc ( ) ) ;

Fn . getContext ( ) . diagnose ( BadInit ) ;


<s> <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) { if ( ! MFI -> isModuleEntryFunction ( ) && ! GV -> getName ( ) . equals ( <STR_LIT> ) ) { const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadLDSDecl ( Fn , <STR_LIT> , MI . getDebugLoc ( ) , DS_Warning ) ; Fn . getContext ( ) . diagnose ( BadLDSDecl ) ; B . buildIntrinsic ( <STR_LIT> , ArrayRef < Register > ( ) , true ) ; B . buildUndef ( DstReg ) ; MI . eraseFromParent ( ) ; return true ; } if ( ! <ISA_LIT>TargetLowering :: hasDefinedInitializer ( GV ) ) { const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( ! TLI -> shouldUseLDSConstAddress ( GV ) ) { MI . getOperand ( <NUM_LIT> ) . setTargetFlags ( <ISA_LIT>InstrInfo :: MO_ABS32_LO ) ; return true ; } if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS && GV -> hasExternalLinkage ( ) ) { Type * Ty = GV -> getValueType ( ) ; if ( B . getDataLayout ( ) . getTypeAllocSize ( Ty ) . isZero ( ) ) { MFI -> setDynLDSAlign ( B . getDataLayout ( ) , * cast < GlobalVariable > ( GV ) ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto Sz = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) ; B . buildIntToPtr ( DstReg , Sz ) ; MI . eraseFromParent ( ) ; return true ; } } B . buildConstant ( DstReg , MFI -> allocateLDSGlobal ( B . getDataLayout ( ) , * cast < GlobalVariable > ( GV ) ) ) ; MI . eraseFromParent ( ) ; return true ; } const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadInit ( Fn , <STR_LIT> , MI . getDebugLoc ( ) ) ; Fn . getContext ( ) . diagnose ( BadInit ) ; return true ; } const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( TLI -> shouldEmitFixup ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> ) ; MI . eraseFromParent ( ) ; return true ; } if ( TLI -> shouldEmitPCReloc ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_REL32 ) ;


<s> Optional < ValueAndVReg > MaybeSOffsetVal = getConstantVRegValWithLookThrough ( SOffset , MRI ) ; Optional < ValueAndVReg > MaybeVIndexVal = getConstantVRegValWithLookThrough ( VIndex , MRI ) ; if ( MaybeVOffsetVal && MaybeSOffsetVal && MaybeVIndexVal && MaybeVIndexVal -> Value == <NUM_LIT> ) { uint64_t TotalOffset = MaybeVOffsetVal -> Value . getZExtValue ( ) + MaybeSOffsetVal -> Value . getZExtValue ( ) + ImmOffset ; MMO -> setOffset ( TotalOffset ) ;


<s> Register KernArgReg = B . getMRI ( ) -> createGenericVirtualRegister ( PtrTy ) ; if ( ! loadInputValue ( KernArgReg , B , <STR_LIT> ) ) llvm_unreachable ( <STR_LIT> ) ; auto COffset = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , Offset ) ; return B . buildPtrAdd ( PtrTy , KernArgReg , COffset ) . getReg ( <NUM_LIT> ) ;


<s> Register <ISA_LIT>LegalizerInfo :: getKernargParameterPtr ( MachineIRBuilder & B , int64_t Offset ) const { LLT PtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register KernArgReg = B . getMRI ( ) -> createGenericVirtualRegister ( PtrTy ) ; if ( ! loadInputValue ( KernArgReg , B , <STR_LIT> ) ) llvm_unreachable ( <STR_LIT> ) ;

auto COffset = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , Offset ) ;


<s> Register Ptr = getKernargParameterPtr ( B , Offset ) ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; B . buildLoad ( DstReg , Ptr , PtrInfo , Align ( <NUM_LIT> ) , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ; MI . eraseFromParent ( ) ; return true ;


<s> bool <ISA_LIT>LegalizerInfo :: legalizeWorkitemIDIntrinsic ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B , unsigned Dim , <STR_LIT> ArgType ) const { unsigned MaxID = ST . getMaxWorkitemID ( B . getMF ( ) . getFunction ( ) , Dim ) ; if ( MaxID == <NUM_LIT> ) return replaceWithConstant ( B , MI , <NUM_LIT> ) ; const <ISA_LIT>MachineFunctionInfo * MFI = B . getMF ( ) . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const ArgDescriptor * Arg ; const TargetRegisterClass * ArgRC ; LLT ArgTy ; std :: tie ( Arg , ArgRC , ArgTy ) = MFI -> getPreloadedValue ( ArgType ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! Arg ) { B . buildUndef ( DstReg ) ;


<s> unsigned MaxID = ST . getMaxWorkitemID ( B . getMF ( ) . getFunction ( ) , Dim ) ; if ( MaxID == <NUM_LIT> ) return replaceWithConstant ( B , MI , <NUM_LIT> ) ; const <ISA_LIT>MachineFunctionInfo * MFI = B . getMF ( ) . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const ArgDescriptor * Arg ; const TargetRegisterClass * ArgRC ; LLT ArgTy ; std :: tie ( Arg , ArgRC , ArgTy ) = MFI -> getPreloadedValue ( ArgType ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> } if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS && GV -> hasExternalLinkage ( ) ) { Type * Ty = GV -> getValueType ( ) ; if ( B . getDataLayout ( ) . getTypeAllocSize ( Ty ) . isZero ( ) ) { MFI -> setDynLDSAlign ( B . getDataLayout ( ) , * cast < GlobalVariable > ( GV ) ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto Sz = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) ; B . buildIntToPtr ( DstReg , Sz ) ; MI . eraseFromParent ( ) ; return true ; } } B . buildConstant ( DstReg , MFI -> allocateLDSGlobal ( B . getDataLayout ( ) , * cast < GlobalVariable > ( GV ) ) ) ; MI . eraseFromParent ( ) ; return true ; } const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadInit ( Fn , <STR_LIT> , MI . getDebugLoc ( ) ) ; Fn . getContext ( ) . diagnose ( BadInit ) ; return true ; }

const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ;


<s> MachineFunction & MF = B . getMF ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) { if ( ! MFI -> isModuleEntryFunction ( ) && ! GV -> getName ( ) . equals ( <STR_LIT> ) ) { const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadLDSDecl ( Fn , <STR_LIT> , MI . getDebugLoc ( ) , DS_Warning ) ; Fn . getContext ( ) . diagnose ( BadLDSDecl ) ; B . buildIntrinsic ( <STR_LIT> , ArrayRef < Register > ( ) , true ) ; B . buildUndef ( DstReg ) ; MI . eraseFromParent ( ) ; return true ; } if ( ! <ISA_LIT>TargetLowering :: hasDefinedInitializer ( GV ) ) { const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( ! TLI -> shouldUseLDSConstAddress ( GV ) ) { MI . getOperand ( <NUM_LIT> ) . setTargetFlags ( <ISA_LIT>InstrInfo :: MO_ABS32_LO ) ; return true ; } if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS && GV -> hasExternalLinkage ( ) ) { Type * Ty = GV -> getValueType ( ) ; if ( B . getDataLayout ( ) . getTypeAllocSize ( Ty ) . isZero ( ) ) { MFI -> setDynLDSAlign ( B . getDataLayout ( ) , * cast < GlobalVariable > ( GV ) ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto Sz = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) ; B . buildIntToPtr ( DstReg , Sz ) ; MI . eraseFromParent ( ) ; return true ; } } B . buildConstant ( DstReg , MFI -> allocateLDSGlobal ( B . getDataLayout ( ) , * cast < GlobalVariable > ( GV ) ) ) ; MI . eraseFromParent ( ) ; return true ; }


<s> unsigned Overflow = ImmOffset & ~ MaxImm ; ImmOffset -= Overflow ; if ( ( int32_t ) Overflow < <NUM_LIT> ) { Overflow += ImmOffset ; ImmOffset = <NUM_LIT> ; } if ( Overflow != <NUM_LIT> ) { if ( ! BaseReg ) { BaseReg = B . buildConstant ( S32 , Overflow ) . getReg ( <NUM_LIT> ) ; } else { auto OverflowVal = B . buildConstant ( S32 , Overflow ) ; BaseReg = B . buildAdd ( S32 , BaseReg , OverflowVal ) . getReg ( <NUM_LIT> ) ; } }

if ( ! BaseReg ) BaseReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ;


<s> SmallVector < Register , <NUM_LIT> > Ops ; if ( Is64 ) { auto Unmerge = B . buildUnmerge ( { S32 , S32 } , NodePtr ) ; Ops . push_back ( Unmerge . getReg ( <NUM_LIT> ) ) ; Ops . push_back ( Unmerge . getReg ( <NUM_LIT> ) ) ; } else { Ops . push_back ( NodePtr ) ; } Ops . push_back ( RayExtent ) ; auto packLanes = [ & Ops , & S32 , & B ] ( Register Src ) { auto Unmerge = B . buildUnmerge ( { S32 , S32 , S32 , S32 } , Src ) ; Ops . push_back ( Unmerge . getReg ( <NUM_LIT> ) ) ; Ops . push_back ( Unmerge . getReg ( <NUM_LIT> ) ) ; Ops . push_back ( Unmerge . getReg ( <NUM_LIT> ) ) ; } ; packLanes ( RayOrigin ) ; if ( IsA16 ) { auto UnmergeRayDir = B . buildUnmerge ( { S16 , S16 , S16 , S16 } , RayDir ) ; auto UnmergeRayInvDir = B . buildUnmerge ( { S16 , S16 , S16 , S16 } , RayInvDir ) ; Register R1 = MRI . createGenericVirtualRegister ( S32 ) ; Register R2 = MRI . createGenericVirtualRegister ( S32 ) ; Register R3 = MRI . createGenericVirtualRegister ( S32 ) ; B . buildMerge ( R1 , { UnmergeRayDir . getReg ( <NUM_LIT> ) , UnmergeRayDir . getReg ( <NUM_LIT> ) } ) ; B . buildMerge ( R2 , { UnmergeRayDir . getReg ( <NUM_LIT> ) , UnmergeRayInvDir . getReg ( <NUM_LIT> ) } ) ; B . buildMerge ( R3 , { UnmergeRayInvDir . getReg ( <NUM_LIT> ) , UnmergeRayInvDir . getReg ( <NUM_LIT> ) } ) ;


<s> case TargetOpcode :: G_EXTRACT_VECTOR_ELT : return legalizeExtractVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_INSERT_VECTOR_ELT : return legalizeInsertVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return legalizeShuffleVector ( MI , MRI , B ) ; case TargetOpcode :: G_F<ISA_LIT>N : case TargetOpcode :: G_FCOS : return legalizeSinCos ( MI , MRI , B ) ; case TargetOpcode :: G_GLOBAL_VALUE : return legalizeGlobalValue ( MI , MRI , B ) ; case TargetOpcode :: G_LOAD : case TargetOpcode :: G_SEXTLOAD : case TargetOpcode :: G_ZEXTLOAD : return legalizeLoad ( Helper , MI ) ; case TargetOpcode :: G_FMAD : return legalizeFMad ( MI , MRI , B ) ; case TargetOpcode :: G_FDIV : return legalizeFDIV ( MI , MRI , B ) ; case TargetOpcode :: G_UDIV : case TargetOpcode :: G_UREM : case TargetOpcode :: G_UDIVREM : return legalizeUnsignedDIV_REM ( MI , MRI , B ) ; case TargetOpcode :: G_SDIV : case TargetOpcode :: G_SREM : case TargetOpcode :: G_SDIVREM : return legalizeSignedDIV_REM ( MI , MRI , B ) ;


<s> case TargetOpcode :: G_FCEIL : return legalizeFceil ( MI , MRI , B ) ; case TargetOpcode :: G_FREM : return legalizeFrem ( MI , MRI , B ) ; case TargetOpcode :: G_INTRIN<ISA_LIT>C_TRUNC : return legalizeIntrinsicTrunc ( MI , MRI , B ) ; case TargetOpcode :: G_<ISA_LIT>TOFP : return legalizeITOFP ( MI , MRI , B , true ) ; case TargetOpcode :: G_UITOFP : return legalizeITOFP ( MI , MRI , B , false ) ; case TargetOpcode :: G_FPTO<ISA_LIT> : return legalizeFPTOI ( MI , MRI , B , true ) ; case TargetOpcode :: G_FPTOUI : return legalizeFPTOI ( MI , MRI , B , false ) ; case TargetOpcode :: G_FMINNUM : case TargetOpcode :: G_FMAXNUM : case TargetOpcode :: G_FMINNUM_IEEE : case TargetOpcode :: G_FMAXNUM_IEEE : return legalizeMinNumMaxNum ( Helper , MI ) ; case TargetOpcode :: G_EXTRACT_VECTOR_ELT : return legalizeExtractVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_INSERT_VECTOR_ELT : return legalizeInsertVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return legalizeShuffleVector ( MI , MRI , B ) ; case TargetOpcode :: G_F<ISA_LIT>N : case TargetOpcode :: G_FCOS : return legalizeSinCos ( MI , MRI , B ) ; case TargetOpcode :: G_GLOBAL_VALUE : return legalizeGlobalValue ( MI , MRI , B ) ; case TargetOpcode :: G_LOAD : case TargetOpcode :: G_SEXTLOAD : case TargetOpcode :: G_ZEXTLOAD : return legalizeLoad ( Helper , MI ) ; case TargetOpcode :: G_FMAD : return legalizeFMad ( MI , MRI , B ) ; case TargetOpcode :: G_FDIV : return legalizeFDIV ( MI , MRI , B ) ; case TargetOpcode :: G_UDIV : case TargetOpcode :: G_UREM : case TargetOpcode :: G_UDIVREM : return legalizeUnsignedDIV_REM ( MI , MRI , B ) ; case TargetOpcode :: G_SDIV : case TargetOpcode :: G_SREM : case TargetOpcode :: G_SDIVREM : return legalizeSignedDIV_REM ( MI , MRI , B ) ; case TargetOpcode :: G_ATOMIC_CMPXCHG : return legalizeAtomicCmpXChg ( MI , MRI , B ) ;


<s> const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; if ( ST . hasApertureRegs ( ) ) { unsigned Offset = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: OFFSET_SRC_SHARED_BASE : <STR_LIT> :: OFFSET_SRC_PRIVATE_BASE ; unsigned WidthM1 = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: WIDTH_M1_SRC_SHARED_BASE : <STR_LIT> :: WIDTH_M1_SRC_PRIVATE_BASE ; unsigned Encoding = <STR_LIT> :: ID_MEM_BASES << <STR_LIT> :: ID_SHIFT_ | Offset << <STR_LIT> :: OFFSET_SHIFT_ | WidthM1 << <STR_LIT> :: WIDTH_M1_SHIFT_ ; unsigned ApertureReg = MRI . createGenericVirtualRegister ( S32 ) ; unsigned GetReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addDef ( GetReg ) . addImm ( Encoding ) ; MRI . setType ( GetReg , S32 ) ; auto ShiftAmt = MIRBuilder . buildConstant ( S32 , WidthM1 + <NUM_LIT> ) ; MIRBuilder . buildInstr ( TargetOpcode :: G_SHL ) . addDef ( ApertureReg ) . addUse ( GetReg ) . addUse ( ShiftAmt . getReg ( <NUM_LIT> ) ) ; return ApertureReg ; } unsigned QueuePtr = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; MIRBuilder . buildConstant ( QueuePtr , <NUM_LIT> ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; Value * V = UndefValue :: get ( PointerType :: get ( Type :: getInt8Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; MachinePointerInfo PtrInfo ( V , StructOffset ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , <NUM_LIT> , MinAlign ( <NUM_LIT> , StructOffset ) ) ; Register LoadResult = MRI . createGenericVirtualRegister ( S32 ) ; Register LoadAddr ; MIRBuilder . materializeGEP ( LoadAddr , QueuePtr , LLT :: scalar ( <NUM_LIT> ) , StructOffset ) ; MIRBuilder . buildLoad ( LoadResult , LoadAddr , * MMO ) ; return LoadResult ;


<s> } unsigned QueuePtr = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; MIRBuilder . buildConstant ( QueuePtr , <NUM_LIT> ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; Value * V = UndefValue :: get ( PointerType :: get ( Type :: getInt8Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; MachinePointerInfo PtrInfo ( V , StructOffset ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , <NUM_LIT> , MinAlign ( <NUM_LIT> , StructOffset ) ) ;

Register LoadResult = MRI . createGenericVirtualRegister ( S32 ) ;


<s> MRI . setType ( GetReg , S32 ) ; auto ShiftAmt = B . buildConstant ( S32 , WidthM1 + <NUM_LIT> ) ; B . buildInstr ( TargetOpcode :: G_SHL ) . addDef ( ApertureReg ) . addUse ( GetReg ) . addUse ( ShiftAmt . getReg ( <NUM_LIT> ) ) ; return ApertureReg ; } Register QueuePtr = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! loadInputValue ( QueuePtr , B , & MFI -> getArgInfo ( ) . QueuePtr ) ) return Register ( ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; Value * V = UndefValue :: get ( PointerType :: get ( Type :: getInt8Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; MachinePointerInfo PtrInfo ( V , StructOffset ) ;

MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , <NUM_LIT> , MinAlign ( <NUM_LIT> , StructOffset ) ) ;


<s> auto NumeratorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; auto ApproxRcp = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( DenominatorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto NegDivScale0 = B . buildFNeg ( S32 , DenominatorScaled , Flags ) ; if ( ! Mode . FP32Denormals ) toggleSPDenormMode ( true , B , ST , Mode ) ; auto Fma0 = B . buildFMA ( S32 , NegDivScale0 , ApproxRcp , One , Flags ) ; auto Fma1 = B . buildFMA ( S32 , Fma0 , ApproxRcp , ApproxRcp , Flags ) ; auto Mul = B . buildFMul ( S32 , NumeratorScaled , Fma1 , Flags ) ; auto Fma2 = B . buildFMA ( S32 , NegDivScale0 , Mul , NumeratorScaled , Flags ) ; auto Fma3 = B . buildFMA ( S32 , Fma2 , Fma1 , Mul , Flags ) ; auto Fma4 = B . buildFMA ( S32 , NegDivScale0 , Fma3 , NumeratorScaled , Flags ) ;

if ( ! Mode . FP32Denormals ) toggleSPDenormMode ( false , B , ST , Mode ) ;


<s> auto Fma0 = B . buildFMA ( S64 , NegDivScale0 , Rcp , One , Flags ) ; auto Fma1 = B . buildFMA ( S64 , Rcp , Fma0 , Rcp , Flags ) ; auto Fma2 = B . buildFMA ( S64 , NegDivScale0 , Fma1 , One , Flags ) ; auto DivScale1 = B . buildIntrinsic ( <STR_LIT> , { S64 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; auto Fma3 = B . buildFMA ( S64 , Fma1 , Fma2 , Fma1 , Flags ) ; auto Mul = B . buildMul ( S64 , DivScale1 . getReg ( <NUM_LIT> ) , Fma3 , Flags ) ; auto Fma4 = B . buildFMA ( S64 , NegDivScale0 , Mul , DivScale1 . getReg ( <NUM_LIT> ) , Flags ) ; Register Scale ; if ( ! ST . hasUsableDivScaleConditionOutput ( ) ) { Scale = MRI . createGenericVirtualRegister ( S1 ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto NumUnmerge = B . buildUnmerge ( S32 , LHS ) ; auto DenUnmerge = B . buildUnmerge ( S32 , RHS ) ; auto Scale0Unmerge = B . buildUnmerge ( S32 , DivScale0 ) ; auto Scale1Unmerge = B . buildUnmerge ( S32 , DivScale1 ) ; auto CmpNum = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , NumUnmerge . getReg ( <NUM_LIT> ) , Scale1Unmerge . getReg ( <NUM_LIT> ) ) ; auto CmpDen = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , DenUnmerge . getReg ( <NUM_LIT> ) , Scale0Unmerge . getReg ( <NUM_LIT> ) ) ; B . buildXor ( Scale , CmpNum , CmpDen ) ; } else { Scale = DivScale1 . getReg ( <NUM_LIT> ) ;


<s> const LLT P2 = LLT :: pointer ( <NUM_LIT> , <NUM_LIT> ) ; setAction ( { G_CONSTANT , S64 } , Legal ) ; setAction ( { G_GEP , P1 } , Legal ) ; setAction ( { G_GEP , P2 } , Legal ) ; setAction ( { G_GEP , <NUM_LIT> , S64 } , Legal ) ; setAction ( { G_LOAD , P1 } , Legal ) ; setAction ( { G_LOAD , P2 } , Legal ) ; setAction ( { G_LOAD , S32 } , Legal ) ; setAction ( { G_LOAD , <NUM_LIT> , P1 } , Legal ) ; setAction ( { G_LOAD , <NUM_LIT> , P2 } , Legal ) ; setAction ( { G_STORE , S32 } , Legal ) ; setAction ( { G_STORE , <NUM_LIT> , P1 } , Legal ) ;


<s> static bool isNot ( const MachineRegisterInfo & MRI , const MachineInstr & MI ) { if ( MI . getOpcode ( ) != TargetOpcode :: G_XOR ) return false ;

auto ConstVal = getConstantVRegVal ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI ) ;


<s> assert ( DstReg . isVirtual ( ) && <STR_LIT> ) ; Register LiveIn = getFunctionLiveInPhysReg ( B . getMF ( ) , B . getTII ( ) , SrcReg , * ArgRC , ArgTy ) ; if ( Arg -> isMasked ( ) ) { const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const unsigned Mask = Arg -> getMask ( ) ; const unsigned Shift = countTrailingZeros < unsigned > ( Mask ) ; Register AndMaskSrc = LiveIn ; if ( Shift != <NUM_LIT> ) { auto ShiftAmt = B . buildConstant ( S32 , Shift ) ;

AndMaskSrc = B . buildLShr ( S32 , LiveIn , ShiftAmt ) . getReg ( <NUM_LIT> ) ;


<s> Register AndMaskSrc = LiveIn ; if ( Shift != <NUM_LIT> ) { auto ShiftAmt = B . buildConstant ( S32 , Shift ) ; AndMaskSrc = B . buildLShr ( S32 , LiveIn , ShiftAmt ) . getReg ( <NUM_LIT> ) ; } B . buildAnd ( DstReg , AndMaskSrc , B . buildConstant ( S32 , Mask >> Shift ) ) ; } else { B . buildCopy ( DstReg , LiveIn ) ; }


<s> auto Mul = B . buildMul ( S64 , DivScale1 . getReg ( <NUM_LIT> ) , Fma3 , Flags ) ; auto Fma4 = B . buildFMA ( S64 , NegDivScale0 , Mul , DivScale1 . getReg ( <NUM_LIT> ) , Flags ) ; Register Scale ; if ( ! ST . hasUsableDivScaleConditionOutput ( ) ) { Scale = MRI . createGenericVirtualRegister ( S1 ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto NumUnmerge = B . buildUnmerge ( S32 , LHS ) ; auto DenUnmerge = B . buildUnmerge ( S32 , RHS ) ; auto Scale0Unmerge = B . buildUnmerge ( S32 , DivScale0 ) ; auto Scale1Unmerge = B . buildUnmerge ( S32 , DivScale1 ) ; auto CmpNum = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , NumUnmerge . getReg ( <NUM_LIT> ) , Scale1Unmerge . getReg ( <NUM_LIT> ) ) ; auto CmpDen = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , DenUnmerge . getReg ( <NUM_LIT> ) , Scale0Unmerge . getReg ( <NUM_LIT> ) ) ;

B . buildXor ( Scale , CmpNum , CmpDen ) ;


<s> Scale = MRI . createGenericVirtualRegister ( S1 ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto NumUnmerge = B . buildUnmerge ( S32 , LHS ) ; auto DenUnmerge = B . buildUnmerge ( S32 , RHS ) ; auto Scale0Unmerge = B . buildUnmerge ( S32 , DivScale0 ) ; auto Scale1Unmerge = B . buildUnmerge ( S32 , DivScale1 ) ; auto CmpNum = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , NumUnmerge . getReg ( <NUM_LIT> ) , Scale1Unmerge . getReg ( <NUM_LIT> ) ) ; auto CmpDen = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , DenUnmerge . getReg ( <NUM_LIT> ) , Scale0Unmerge . getReg ( <NUM_LIT> ) ) ; B . buildXor ( Scale , CmpNum , CmpDen ) ; } else { Scale = DivScale1 . getReg ( <NUM_LIT> ) ; } auto Fmas = B . buildIntrinsic ( <STR_LIT> , { S64 } , false ) . addUse ( Fma4 . getReg ( <NUM_LIT> ) ) . addUse ( Fma3 . getReg ( <NUM_LIT> ) ) . addUse ( Mul . getReg ( <NUM_LIT> ) ) . addUse ( Scale ) . setMIFlags ( Flags ) ; B . buildIntrinsic ( <STR_LIT> , { S64 } , false ) . addDef ( Res ) . addUse ( Fmas . getReg ( <NUM_LIT> ) ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ;


<s> } else { B . buildInstr ( <STR_LIT> ) . addDef ( Def ) . addUse ( Use ) . addMBB ( BrCond -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) . addImm ( <NUM_LIT> ) ; } MRI . setRegClass ( Def , TRI -> getWaveMaskRegClass ( ) ) ; MRI . setRegClass ( Use , TRI -> getWaveMaskRegClass ( ) ) ; MI . eraseFromParent ( ) ; BrCond -> eraseFromParent ( ) ; return true ; } return false ; } case <STR_LIT> : { if ( MachineInstr * BrCond = verifyCFIntrinsic ( MI , MRI ) ) { const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; B . setInstr ( * BrCond ) ; Register Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; B . buildInstr ( <STR_LIT> ) . addUse ( Reg ) . addMBB ( BrCond -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MI . eraseFromParent ( ) ; BrCond -> eraseFromParent ( ) ; MRI . setRegClass ( Reg , TRI -> getWaveMaskRegClass ( ) ) ; return true ; } return false ; } case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeImplicitArgPtr ( MI , MRI , B ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeFDIVFastIntrin ( MI , MRI , B ) ; case <STR_LIT> : return legalizeIsAddrSpace ( MI , MRI , B , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; case <STR_LIT> : return legalizeIsAddrSpace ( MI , MRI , B , <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; case <STR_LIT> : { B . setInstr ( MI ) ; B . buildConstant ( MI . getOperand ( <NUM_LIT> ) , ST . getWavefrontSize ( ) ) ; MI . eraseFromParent ( ) ; return true ; } case <STR_LIT> : return legalizeRawBufferStore ( MI , MRI , B , false ) ; case <STR_LIT> :

return legalizeRawBufferStore ( MI , MRI , B , true ) ;


<s> if ( ST . hasApertureRegs ( ) ) { unsigned Offset = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: OFFSET_SRC_SHARED_BASE : <STR_LIT> :: OFFSET_SRC_PRIVATE_BASE ; unsigned WidthM1 = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: WIDTH_M1_SRC_SHARED_BASE : <STR_LIT> :: WIDTH_M1_SRC_PRIVATE_BASE ; unsigned Encoding = <STR_LIT> :: ID_MEM_BASES << <STR_LIT> :: ID_SHIFT_ | Offset << <STR_LIT> :: OFFSET_SHIFT_ | WidthM1 << <STR_LIT> :: WIDTH_M1_SHIFT_ ; unsigned ShiftAmt = MRI . createGenericVirtualRegister ( S32 ) ; unsigned ApertureReg = MRI . createGenericVirtualRegister ( S32 ) ; unsigned GetReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addDef ( GetReg ) . addImm ( Encoding ) ; MRI . setType ( GetReg , S32 ) ; MIRBuilder . buildConstant ( ShiftAmt , WidthM1 + <NUM_LIT> ) ; MIRBuilder . buildInstr ( TargetOpcode :: G_SHL ) . addDef ( ApertureReg ) . addUse ( GetReg ) . addUse ( ShiftAmt ) ; return ApertureReg ; } unsigned QueuePtr = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; MIRBuilder . buildConstant ( QueuePtr , <NUM_LIT> ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ;

Value * V = UndefValue :: get ( PointerType :: get ( Type :: getInt8Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ;


<s> MIRBuilder . setInstr ( MI ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( Dst ) ; LLT SrcTy = MRI . getType ( Src ) ; unsigned DestAS = DstTy . getAddressSpace ( ) ; unsigned SrcAS = SrcTy . getAddressSpace ( ) ; assert ( ! DstTy . isVector ( ) ) ; const <ISA_LIT>TargetMachine & TM = static_cast < const <ISA_LIT>TargetMachine & > ( MF . getTarget ( ) ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; if ( ST . getTargetLowering ( ) -> isNoopAddrSpaceCast ( SrcAS , DestAS ) ) { MI . setDesc ( MIRBuilder . getTII ( ) . get ( TargetOpcode :: G_BITCAST ) ) ; return true ; } if ( SrcAS == <ISA_LIT>AS :: FLAT_ADDRESS ) { assert ( DestAS == <ISA_LIT>AS :: LOCAL_ADDRESS || DestAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; unsigned NullVal = TM . getNullPointerValue ( DestAS ) ; unsigned SegmentNullReg = MRI . createGenericVirtualRegister ( DstTy ) ; unsigned FlatNullReg = MRI . createGenericVirtualRegister ( SrcTy ) ; MIRBuilder . buildConstant ( SegmentNullReg , NullVal ) ; MIRBuilder . buildConstant ( FlatNullReg , <NUM_LIT> ) ; unsigned PtrLo32 = MRI . createGenericVirtualRegister ( DstTy ) ; MIRBuilder . buildExtract ( PtrLo32 , Src , <NUM_LIT> ) ; unsigned CmpRes = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildICmp ( CmpInst :: ICMP_NE , CmpRes , Src , FlatNullReg ) ; MIRBuilder . buildSelect ( Dst , CmpRes , PtrLo32 , SegmentNullReg ) ; MI . eraseFromParent ( ) ; return true ; } assert ( SrcAS == <ISA_LIT>AS :: LOCAL_ADDRESS || SrcAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; unsigned FlatNullReg = MRI . createGenericVirtualRegister ( DstTy ) ; unsigned SegmentNullReg = MRI . createGenericVirtualRegister ( SrcTy ) ; MIRBuilder . buildConstant ( SegmentNullReg , TM . getNullPointerValue ( SrcAS ) ) ; MIRBuilder . buildConstant ( FlatNullReg , TM . getNullPointerValue ( DestAS ) ) ; unsigned ApertureReg = getSegmentAperture ( DestAS , MRI , MIRBuilder ) ; unsigned CmpRes = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ;

MIRBuilder . buildICmp ( CmpInst :: ICMP_NE , CmpRes , Src , SegmentNullReg ) ;


<s> unsigned SegmentNullReg = MRI . createGenericVirtualRegister ( SrcTy ) ; MIRBuilder . buildConstant ( SegmentNullReg , TM . getNullPointerValue ( SrcAS ) ) ; MIRBuilder . buildConstant ( FlatNullReg , TM . getNullPointerValue ( DestAS ) ) ; unsigned ApertureReg = getSegmentAperture ( DestAS , MRI , MIRBuilder ) ; unsigned CmpRes = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildICmp ( CmpInst :: ICMP_NE , CmpRes , Src , SegmentNullReg ) ; unsigned BuildPtr = MRI . createGenericVirtualRegister ( DstTy ) ; unsigned SrcAsInt = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildInstr ( TargetOpcode :: G_PTRTOINT ) . addDef ( SrcAsInt ) . addUse ( Src ) ; MIRBuilder . buildMerge ( BuildPtr , { SrcAsInt , ApertureReg } ) ; MIRBuilder . buildSelect ( Dst , CmpRes , BuildPtr , FlatNullReg ) ; MI . eraseFromParent ( ) ;


<s> switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_ADDRSPACE_CAST : return legalizeAddrSpaceCast ( MI , MRI , MIRBuilder ) ; default : return false ; } llvm_unreachable ( <STR_LIT> ) ;


<s> switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_ADDRSPACE_CAST : return legalizeAddrSpaceCast ( MI , MRI , MIRBuilder ) ; default : return false ; }


<s> const LLT Ty = Query . Types [ TypeIdx ] ; unsigned Size = Ty . getSizeInBits ( ) ; LLT CoercedTy ; if ( Size < <NUM_LIT> ) { assert ( Size == <NUM_LIT> ) ; CoercedTy = LLT :: scalar ( <NUM_LIT> ) ; } else CoercedTy = LLT :: scalarOrVector ( Size / <NUM_LIT> , <NUM_LIT> ) ; return std :: make_pair ( TypeIdx , CoercedTy ) ; } ;


<s> return [ = ] ( const LegalityQuery & Query ) { const LLT Ty = Query . Types [ TypeIdx ] ;

unsigned Size = Ty . getSizeInBits ( ) ;


<s> const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadInit ( Fn , <STR_LIT> , MI . getDebugLoc ( ) ) ; Fn . getContext ( ) . diagnose ( BadInit ) ; return true ; } const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( TLI -> shouldEmitFixup ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> ) ; MI . eraseFromParent ( ) ; return true ; } if ( TLI -> shouldEmitPCReloc ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_REL32 ) ; MI . eraseFromParent ( ) ; return true ; } LLT PtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register GOTAddr = MRI . createGenericVirtualRegister ( PtrTy ) ; MachineMemOperand * GOTMMO = MF . getMachineMemOperand ( MachinePointerInfo :: getGOT ( MF ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , <NUM_LIT> , Align ( <NUM_LIT> ) ) ; buildPCRelGlobalAddress ( GOTAddr , PtrTy , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; if ( Ty . getSizeInBits ( ) == <NUM_LIT> ) { auto Load = B . buildLoad ( PtrTy , GOTAddr , * GOTMMO ) ; B . buildExtract ( DstReg , Load , <NUM_LIT> ) ; } else B . buildLoad ( DstReg , GOTAddr , * GOTMMO ) ;