<s> if ( ! Arg -> isRegister ( ) || ! Arg -> getRegister ( ) . isValid ( ) ) return false ; Register SrcReg = Arg -> getRegister ( ) ; assert ( SrcReg . isPhysical ( ) && <STR_LIT> ) ; assert ( DstReg . isVirtual ( ) && <STR_LIT> ) ; MachineRegisterInfo & MRI = * B . getMRI ( ) ; LLT Ty = MRI . getType ( DstReg ) ; Register LiveIn = getLiveInRegister ( B , MRI , SrcReg , Ty ) ; if ( Arg -> isMasked ( ) ) { const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const unsigned Mask = Arg -> getMask ( ) ; const unsigned Shift = countTrailingZeros < unsigned > ( Mask ) ; Register AndMaskSrc = LiveIn ; if ( Shift != <NUM_LIT> ) { auto ShiftAmt = B . buildConstant


<s> AddrReg = B . buildBitcast ( V2S16 , AddrReg ) . getReg ( <NUM_LIT> ) ; PackedAddrs . push_back ( AddrReg ) ; } else { if ( ( ( I + <NUM_LIT> ) >= ( AddrIdx + NumVAddrs ) ) || ( ( NumGradients / <NUM_LIT> ) % <NUM_LIT> == <NUM_LIT> && ( I == DimIdx + ( NumGradients / <NUM_LIT> ) - <NUM_LIT> || I == DimIdx + NumGradients - <NUM_LIT> ) ) || ! MI . getOperand ( I + <NUM_LIT> ) . isReg ( ) ) { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , B . buildUndef ( S16 ) . getReg (

<NUM_LIT> ) } ) . getReg ( <NUM_LIT> ) ) ;


<s> MachineBasicBlock :: iterator Next = std :: next ( UseMI . getIterator ( ) ) ; if ( Next != MI . getParent ( ) -> end ( ) ) { if ( Next -> getOpcode ( ) != <STR_LIT> ) return nullptr ; Br = & * Next ; } return &


<s> static MachineInstr * verifyCFIntrinsic ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineInstr * & Br ) { Register CondDef = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! MRI . hasOneNonDBGUse ( CondDef ) ) return nullptr ; MachineInstr & UseMI = * MRI . use_instr_nodbg_begin ( CondDef ) ; if ( UseMI . getParent ( ) != MI . getParent ( ) || UseMI . getOpcode ( ) !=


<s> return [ = ] ( const LegalityQuery & Query ) { return Query . Types [ TypeIdx ] . getElementType


<s> Register LiveIn = MRI . getLiveInVirtReg ( Reg ) ; if ( LiveIn ) return LiveIn ; Register NewReg = MRI . createGenericVirtualRegister ( Ty ) ; MRI . addLiveIn ( Reg


<s> Register NewReg = MRI . createGenericVirtualRegister ( Ty ) ; MRI . addLiveIn ( Reg ,


<s> MRI . setType ( GetReg , S32 ) ; auto ShiftAmt = B . buildConstant ( S32 , WidthM1 + <NUM_LIT> ) ; B . buildInstr ( TargetOpcode :: G_SHL ) . addDef ( ApertureReg ) . addUse ( GetReg ) . addUse ( ShiftAmt . getReg ( <NUM_LIT> ) ) ; return ApertureReg ; } Register QueuePtr = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! loadInputValue ( QueuePtr , B , & MFI -> getArgInfo ( ) . QueuePtr ) ) return Register ( ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; Value * V = UndefValue :: get ( PointerType :: get ( Type :: getInt8Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; MachinePointerInfo PtrInfo ( V ,


<s> B . setInstr ( MI ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( Dst ) ; LLT SrcTy = MRI . getType ( Src ) ; unsigned DestAS = DstTy . getAddressSpace ( ) ; unsigned SrcAS = SrcTy . getAddressSpace ( ) ; assert ( ! DstTy . isVector ( ) ) ; const <ISA_LIT>TargetMachine & TM = static_cast < const <ISA_LIT>TargetMachine & > ( MF . getTarget ( ) ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; if ( ST . getTargetLowering ( ) -> isNoopAddrSpaceCast ( SrcAS , DestAS ) ) { MI . setDesc ( B . getTII ( ) . get ( TargetOpcode :: G_BITCAST ) ) ; return true ; } if ( DestAS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { B . buildExtract ( Dst , Src , <NUM_LIT> ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo <


<s> auto FlatNull = B . buildConstant ( SrcTy , <NUM_LIT> ) ; Register PtrLo32 = MRI . createGenericVirtualRegister ( DstTy ) ; B . buildExtract ( PtrLo32 , Src , <NUM_LIT> ) ; Register CmpRes = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; B . buildICmp ( CmpInst :: ICMP_NE , CmpRes , Src , FlatNull . getReg ( <NUM_LIT> ) ) ; B . buildSelect ( Dst , CmpRes , PtrLo32 , SegmentNull . getReg ( <NUM_LIT> ) ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS != <ISA_LIT>AS :: LOCAL_ADDRESS && SrcAS != <ISA_LIT>AS :: PRIVATE_ADDRESS ) return false ; if ( ! ST . hasFlatAddressSpace ( ) ) return false ; auto SegmentNull = B . buildConstant ( SrcTy , TM . getNullPointerValue ( SrcAS ) ) ; auto FlatNull = B . buildConstant ( DstTy , TM . getNullPointerValue ( DestAS ) ) ; Register ApertureReg = getSegmentAperture ( SrcAS , MRI , B ) ; if ( ! ApertureReg . isValid ( ) ) return false ; Register CmpRes = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; B . buildICmp ( CmpInst :: ICMP_NE , CmpRes , Src , SegmentNull . getReg ( <NUM_LIT> ) ) ; Register BuildPtr = MRI . createGenericVirtualRegister (


<s> switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_ADDRSPACE_CAST : return legalizeAddrSpaceCast ( MI , MRI , B ) ; case TargetOpcode :: G_FRINT : return legalizeFrint ( MI , MRI , B ) ; case TargetOpcode :: G_FCEIL : return legalizeFceil ( MI , MRI , B ) ; case TargetOpcode :: G_INTRIN<ISA_LIT>C_TRUNC : return legalizeIntrinsicTrunc ( MI , MRI , B ) ; case TargetOpcode :: G_<ISA_LIT>TOFP : return legalizeITOFP ( MI , MRI , B , true ) ; case TargetOpcode :: G_UITOFP : return legalizeITOFP ( MI , MRI , B , false ) ; case TargetOpcode :: G_FMINNUM : case TargetOpcode :: G_FMAXNUM : case TargetOpcode :: G_FMINNUM_IEEE : case TargetOpcode ::


<s> assert ( EltTy == MRI . getType ( Dst ) ) ; B . setInstr ( MI ) ; if ( IdxVal . getValue ( ) < VecTy . getNumElements ( ) ) B . buildExtract ( Dst , Vec , IdxVal . getValue ( ) * EltTy . getSizeInBits ( ) ) ; else B . buildUndef ( Dst ) ; MI . eraseFromParent (


<s> if ( IdxVal . getValue ( ) < VecTy . getNumElements ( ) ) B . buildExtract ( Dst , Vec , IdxVal . getValue ( ) * EltTy . getSizeInBits ( ) ) ; else B . buildUndef ( Dst ) ; MI . eraseFromParent ( ) ; return true


<s> if ( ! Unsafe && ResTy == S32 && ST . hasFP32Denormals ( ) ) return false ; if ( auto CLHS = getConstantFPVRegVal ( LHS , MRI ) ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( RHS ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { auto FNeg = B . buildFNeg ( ResTy , RHS , Flags ) ; B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( FNeg . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ; } } if ( Unsafe ) { auto RCP = B . buildIntrinsic ( <STR_LIT> , { ResTy } , false ) . addUse ( RHS ) . setMIFlags


<s> const auto One = B . buildFConstant ( S64 , <NUM_LIT> ) ; auto Lt0 = B . buildFCmp ( CmpInst :: FCMP_OGT , S1 , Src , Zero ) ; auto NeTrunc = B . buildFCmp ( CmpInst :: FCMP_ONE , S1 , Src , Trunc ) ; auto And = B . buildAnd ( S1 , Lt0 , NeTrunc ) ; auto Add = B . buildSelect ( S64 , And , One ,


<s> LLT DstTy = MRI . getType ( Dst ) ; LLT S16 = LLT :: scalar ( <NUM_LIT> ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; if ( legalizeFastUnsafeFDIV ( MI , MRI , B ) ) return true ; if ( DstTy == S16 ) return legalizeFDIV16 (


<s> Register RHS = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; uint16_t Flags = MI . getFlags ( ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; LLT S1 = LLT :: scalar ( <NUM_LIT> ) ; auto One = B . buildFConstant ( S32 , <NUM_LIT> ) ; auto DenominatorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( RHS ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; auto NumeratorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; auto ApproxRcp = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( DenominatorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto NegDivScale0 = B . buildFNeg ( S32 , DenominatorScaled , Flags ) ; if ( ! ST . hasFP32Denormals ( ) ) toggleSPDenormMode ( true , ST , B ) ; auto Fma0 = B . buildFMA ( S32 , NegDivScale0 , ApproxRcp , One , Flags ) ; auto Fma1 = B . buildFMA ( S32 , Fma0 , ApproxRcp , ApproxRcp , Flags ) ; auto Mul = B . buildFMul ( S32 , NumeratorScaled ,


<s> Register Res = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register LHS = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register RHS = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; uint16_t Flags = MI . getFlags ( ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; LLT S1 = LLT :: scalar ( <NUM_LIT> ) ; auto One = B . buildFConstant ( S32 , <NUM_LIT> ) ; auto DenominatorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( RHS ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; auto NumeratorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; auto ApproxRcp = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( DenominatorScaled . getReg (


<s> if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! ST . hasFP32Denormals ( ) ) return true ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! ST . hasFP16Denormals ( ) ) return true ; MachineFunction & MF = B . getMF ( ) ; MachineIRBuilder HelperBuilder ( MI ) ; GISelObserverWrapper DummyObserver ; LegalizerHelper Helper ( MF , DummyObserver , HelperBuilder ) ; HelperBuilder . setMBB ( * MI . getParent ( ) ) ; return Helper . lowerFMad ( MI ) == LegalizerHelper


<s> LLT Ty = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; assert ( Ty . isScalar ( ) ) ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! ST . hasFP32Denormals ( ) ) return true ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! ST . hasFP16Denormals ( ) ) return true ; MachineFunction & MF = B . getMF (


<s> if ( TLI -> shouldEmitPCReloc ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_REL32 ) ; MI . eraseFromParent ( ) ; return true ; } LLT PtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register GOTAddr = MRI . createGenericVirtualRegister ( PtrTy ) ; MachineMemOperand * GOTMMO = MF . getMachineMemOperand ( MachinePointerInfo :: getGOT ( MF ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , <NUM_LIT> , <NUM_LIT> ) ; buildPCRelGlobalAddress ( GOTAddr , PtrTy , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; if ( Ty . getSizeInBits ( ) == <NUM_LIT> ) { auto Load = B . buildLoad ( PtrTy , GOTAddr , * GOTMMO ) ; B . buildExtract ( DstReg , Load , <NUM_LIT> ) ; } else B . buildLoad ( DstReg , GOTAddr , * GOTMMO ) ; MI . eraseFromParent (


<s> const <ISA_LIT>MachineFunctionInfo * MFI = B . getMF ( ) . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! MFI -> isEntryFunction ( ) ) { return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; } B . setInstr ( MI ) ; uint64_t Offset = ST . getTargetLowering ( ) -> getImplicitParameterOffset ( B . getMF ( ) , <ISA_LIT>TargetLowering :: FIRST_IMPLICIT ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType


<s> Register Ins = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT VecTy = MRI . getType ( Vec ) ; LLT EltTy = VecTy . getElementType ( ) ; assert ( EltTy == MRI . getType ( Ins ) ) ; B . setInstr ( MI ) ; if ( IdxVal . getValue ( ) < VecTy . getNumElements ( ) ) B . buildInsert ( Dst , Vec , Ins , IdxVal . getValue ( ) * EltTy . getSizeInBits ( ) ) ; else B . buildUndef ( Dst ) ; MI . eraseFromParent ( ) ; return


<s> const auto SignBitMask = B . buildConstant ( S32 , UINT32_C ( <NUM_LIT> ) << <NUM_LIT> ) ; auto SignBit = B . buildAnd ( S32 , Hi , SignBitMask ) ; const auto FractMask = B . buildConstant ( S64 , ( UINT64_C ( <NUM_LIT> ) << FractBits ) - <NUM_LIT> ) ; const auto Zero32 = B . buildConstant ( S32 , <NUM_LIT> ) ; auto SignBit64 = B . buildMerge ( S64 , { Zero32 . getReg ( <NUM_LIT> ) , SignBit . getReg ( <NUM_LIT> ) } ) ; auto Shr = B . buildAShr ( S64 , FractMask , Exp ) ; auto Not = B . buildNot ( S64 , Shr ) ; auto Tmp0 = B . buildAnd ( S64 , Src


<s> const LLT S1 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; Register Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( MRI . getType ( Src ) == S64 ) ; auto Unmerge = B . buildUnmerge ( { S32 , S32 } , Src ) ; Register Hi = Unmerge . getReg ( <NUM_LIT> ) ; auto Exp = extractF64Exponent ( Hi , B ) ; const unsigned FractBits = <NUM_LIT> ; const auto SignBitMask = B . buildConstant ( S32 , UINT32_C ( <NUM_LIT> ) << <NUM_LIT> ) ; auto SignBit = B . buildAnd ( S32 , Hi , SignBitMask ) ; const auto FractMask = B . buildConstant ( S64 , ( UINT64_C ( <NUM_LIT> ) << FractBits ) - <NUM_LIT> ) ; const auto Zero32 = B . buildConstant ( S32 , <NUM_LIT> ) ; auto SignBit64 = B . buildMerge ( S64 , { Zero32 . getReg ( <NUM_LIT> ) , SignBit . getReg (


<s> bool <ISA_LIT>LegalizerInfo :: legalizePreloadedArgIntrin ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B , <STR_LIT> ArgType ) const { B . setInstr ( MI ) ; const <ISA_LIT>MachineFunctionInfo * MFI = B . getMF ( ) . getInfo <


<s> } if ( Ty . isVector ( ) ) { if ( Ty . getElementType ( ) == S16 && Ty . getNumElements ( ) <= <NUM_LIT> ) { if ( IsFormat ) MI . getOperand ( <NUM_LIT> ) . setReg ( handleD16VData ( B , MRI , VData ) ) ; return true ; } return Ty . getElementType ( ) == S32 && Ty .


<s> assert ( Arg -> getRegister ( ) . isPhysical ( ) ) ; MachineRegisterInfo & MRI = * B . getMRI ( ) ; LLT Ty = MRI . getType ( DstReg ) ; Register LiveIn = getLiveInRegister ( MRI , Arg -> getRegister ( ) , Ty ) ; if ( Arg -> isMasked ( ) ) { const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const unsigned Mask = Arg -> getMask ( ) ; const unsigned Shift = countTrailingZeros < unsigned > ( Mask ) ; Register AndMaskSrc = LiveIn ; if ( Shift != <NUM_LIT> ) { auto ShiftAmt = B . buildConstant ( S32 , Shift ) ; AndMaskSrc = B . buildLShr ( S32 , LiveIn , ShiftAmt ) . getReg ( <NUM_LIT> ) ; } B . buildAnd ( DstReg , AndMaskSrc , B . buildConstant ( S32 , Mask >> Shift ) ) ; } else B . buildCopy ( DstReg , LiveIn ) ; if ( ! MRI . getVRegDef ( LiveIn ) ) { MachineBasicBlock & OrigInsBB = B . getMBB ( ) ; auto OrigInsPt = B . getInsertPt ( ) ; MachineBasicBlock & EntryMBB = B . getMF ( ) . front ( ) ; EntryMBB . addLiveIn ( Arg -> getRegister ( ) ) ; B . setInsertPt ( EntryMBB , EntryMBB . begin ( ) ) ; B . buildCopy ( LiveIn , Arg -> getRegister ( ) ) ; B . setInsertPt ( OrigInsBB , OrigInsPt ) ; } return


<s> <STR_LIT> :: OFFSET_SRC_PRIVATE_BASE ; unsigned WidthM1 = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: WIDTH_M1_SRC_SHARED_BASE : <STR_LIT> :: WIDTH_M1_SRC_PRIVATE_BASE ; unsigned Encoding = <STR_LIT> :: ID_MEM_BASES << <STR_LIT> :: ID_SHIFT_ | Offset << <STR_LIT> :: OFFSET_SHIFT_ | WidthM1 << <STR_LIT> :: WIDTH_M1_SHIFT_ ; Register ApertureReg = MRI . createGenericVirtualRegister ( S32 ) ; Register GetReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( GetReg ) . addImm ( Encoding ) ; MRI . setType ( GetReg , S32 ) ; auto ShiftAmt = B . buildConstant ( S32 , WidthM1 + <NUM_LIT> ) ; B . buildInstr ( TargetOpcode :: G_SHL ) . addDef ( ApertureReg ) . addUse ( GetReg ) . addUse ( ShiftAmt . getReg ( <NUM_LIT> ) ) ; return ApertureReg ; } Register QueuePtr = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! loadInputValue ( QueuePtr , B , & MFI -> getArgInfo ( ) . QueuePtr ) ) return Register ( ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS


<s> B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( RHS ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { auto FNeg = B . buildFNeg ( ResTy , RHS , Flags ) ; B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( FNeg . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ; } } if ( Unsafe ) { auto RCP = B . buildIntrinsic ( <STR_LIT> , { ResTy } , false ) . addUse ( RHS ) . setMIFlags ( Flags ) ; B . buildFMul ( Res , LHS , RCP , Flags ) ; MI . eraseFromParent ( ) ; return true


<s> LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; const MachineFunction & MF = B . getMF ( ) ; bool Unsafe = MF . getTarget ( ) . Options . UnsafeFPMath || MI . getFlag ( MachineInstr :: FmArcp ) ; if ( ! MF . getTarget ( ) . Options . UnsafeFPMath && ResTy == S64 ) return false ; if ( ! Unsafe && ResTy == S32 && MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> getMode ( ) . FP32Denormals ) return false ; if ( auto CLHS = getConstantFPVRegVal ( LHS , MRI ) ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( RHS ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { auto FNeg = B . buildFNeg ( ResTy , RHS , Flags ) ; B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( FNeg . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ; } } if ( Unsafe ) { auto RCP = B . buildIntrinsic ( <STR_LIT> , { ResTy } , false ) . addUse ( RHS ) . setMIFlags ( Flags ) ; B . buildFMul ( Res , LHS , RCP , Flags ) ; MI . eraseFromParent


<s> <STR_LIT> Mode = MFI -> getMode ( ) ; uint16_t Flags = MI . getFlags ( ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; LLT S1 = LLT :: scalar ( <NUM_LIT> ) ; auto One = B . buildFConstant ( S32 , <NUM_LIT> ) ; auto DenominatorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( RHS ) . addUse ( LHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto NumeratorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto ApproxRcp = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( DenominatorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto NegDivScale0 = B . buildFNeg ( S32 , DenominatorScaled , Flags ) ; if ( ! Mode . FP32Denormals ) toggleSPDenormMode ( true , B , ST , Mode ) ; auto Fma0 = B . buildFMA ( S32 , NegDivScale0 , ApproxRcp , One , Flags ) ; auto Fma1 = B . buildFMA ( S32 , Fma0 , ApproxRcp , ApproxRcp , Flags ) ; auto Mul = B . buildFMul ( S32 , NumeratorScaled , Fma1 , Flags ) ; auto Fma2 = B . buildFMA ( S32 , NegDivScale0 , Mul , NumeratorScaled , Flags ) ; auto Fma3 = B . buildFMA ( S32 , Fma2 , Fma1 , Mul , Flags ) ; auto Fma4 = B . buildFMA ( S32 , NegDivScale0 , Fma3 , NumeratorScaled , Flags ) ; if ( ! Mode . FP32Denormals ) toggleSPDenormMode ( false , B , ST , Mode ) ; auto Fmas = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( Fma4 . getReg ( <NUM_LIT> ) ) . addUse ( Fma1 . getReg ( <NUM_LIT> ) ) . addUse ( Fma3 . getReg ( <NUM_LIT> ) ) . addUse ( NumeratorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( Fmas . getReg ( <NUM_LIT> ) ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true


<s> auto Fma2 = B . buildFMA ( S64 , NegDivScale0 , Fma1 , One , Flags ) ; auto DivScale1 = B . buildIntrinsic ( <STR_LIT> , { S64 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto Fma3 = B . buildFMA ( S64 , Fma1 , Fma2 , Fma1 , Flags ) ; auto Mul = B . buildMul ( S64 , DivScale1 . getReg ( <NUM_LIT> ) , Fma3 , Flags ) ; auto Fma4 = B . buildFMA ( S64 , NegDivScale0 , Mul , DivScale1 . getReg ( <NUM_LIT> ) , Flags ) ; Register Scale ; if ( ! ST . hasUsableDivScaleConditionOutput ( ) ) { Scale = MRI . createGenericVirtualRegister ( S1 ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto NumUnmerge = B . buildUnmerge ( S32 , LHS ) ; auto DenUnmerge = B . buildUnmerge ( S32 , RHS ) ; auto Scale0Unmerge = B . buildUnmerge ( S32 , DivScale0 ) ; auto Scale1Unmerge = B . buildUnmerge ( S32 , DivScale1 ) ; auto CmpNum = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , NumUnmerge . getReg ( <NUM_LIT> ) , Scale1Unmerge . getReg ( <NUM_LIT> ) ) ; auto CmpDen = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , DenUnmerge . getReg ( <NUM_LIT> ) , Scale0Unmerge . getReg ( <NUM_LIT> ) ) ; B . buildXor ( Scale , CmpNum , CmpDen ) ; } else { Scale = DivScale1 . getReg ( <NUM_LIT> ) ; } auto Fmas = B . buildIntrinsic ( <STR_LIT> , { S64 } , false ) . addUse ( Fma4 . getReg ( <NUM_LIT> ) ) . addUse ( Fma3 . getReg ( <NUM_LIT> ) ) . addUse ( Mul . getReg ( <NUM_LIT> ) ) . addUse ( Scale ) . setMIFlags ( Flags ) ; B . buildIntrinsic ( <STR_LIT> , makeArrayRef ( Res ) , false ) . addUse ( Fmas . getReg ( <NUM_LIT> ) ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; MI . eraseFromParent


<s> auto Fma4 = B . buildFMA ( S64 , NegDivScale0 , Mul , DivScale1 . getReg ( <NUM_LIT> ) , Flags ) ; Register Scale ; if ( ! ST . hasUsableDivScaleConditionOutput ( ) ) { Scale = MRI . createGenericVirtualRegister ( S1 ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto NumUnmerge = B . buildUnmerge ( S32 , LHS ) ; auto DenUnmerge = B . buildUnmerge ( S32 , RHS ) ; auto Scale0Unmerge = B . buildUnmerge ( S32 , DivScale0 ) ; auto Scale1Unmerge = B . buildUnmerge ( S32 , DivScale1 ) ; auto CmpNum = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , NumUnmerge . getReg ( <NUM_LIT> ) , Scale1Unmerge . getReg ( <NUM_LIT> ) ) ; auto CmpDen = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , DenUnmerge . getReg ( <NUM_LIT> ) , Scale0Unmerge . getReg ( <NUM_LIT> ) ) ; B . buildXor ( Scale , CmpNum , CmpDen ) ; } else { Scale = DivScale1 . getReg ( <NUM_LIT> ) ; } auto Fmas = B . buildIntrinsic ( <STR_LIT> , { S64 } , false ) . addUse ( Fma4 . getReg ( <NUM_LIT> ) ) . addUse ( Fma3 . getReg ( <NUM_LIT> ) ) . addUse ( Mul .

getReg ( <NUM_LIT> ) ) . addUse ( Scale ) . setMIFlags ( Flags ) ;


<s> bool <ISA_LIT>LegalizerInfo :: legalizeFMad ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { LLT Ty = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; assert ( Ty . isScalar ( ) ) ; MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . FP32Denormals ) return true ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . FP64FP16Denormals ) return true ; MachineIRBuilder HelperBuilder ( MI ) ; GISelObserverWrapper DummyObserver


<s> if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . FP32Denormals ) return true ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . FP64FP16Denormals ) return true ; MachineIRBuilder HelperBuilder ( MI ) ; GISelObserverWrapper DummyObserver ; LegalizerHelper Helper ( MF , DummyObserver , HelperBuilder ) ; HelperBuilder . setMBB ( * MI . getParent ( ) ) ; return Helper . lowerFMad ( MI ) == LegalizerHelper ::


<s> return true ; } return false ; } case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeImplicitArgPtr ( MI , MRI , B ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeFDIVFastIntrin ( MI , MRI , B ) ; case <STR_LIT> : return legalizeIsAddrSpace ( MI , MRI , B , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; case <STR_LIT> : return legalizeIsAddrSpace ( MI , MRI , B , <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; case <STR_LIT> : { B . setInstr ( MI ) ; B . buildConstant ( MI . getOperand ( <NUM_LIT> ) , ST . getWavefrontSize ( ) ) ; MI . eraseFromParent ( ) ; return true ; } case <STR_LIT> : return legalizeRawBufferStore ( MI , MRI , B , false ) ; case <STR_LIT> : return legalizeRawBufferStore ( MI , MRI , B , true ) ; case <STR_LIT> : return legalizeAtomicIncDec ( MI , B , true ) ; case


<s> return false ; } case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeImplicitArgPtr ( MI , MRI , B ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeFDIVFastIntrin ( MI , MRI , B ) ; case <STR_LIT> : return legalizeIsAddrSpace ( MI , MRI , B , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; case <STR_LIT> : return legalizeIsAddrSpace ( MI , MRI , B , <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; case <STR_LIT> : { B . setInstr ( MI ) ; B . buildConstant ( MI . getOperand ( <NUM_LIT> ) , ST . getWavefrontSize ( ) ) ; MI . eraseFromParent ( ) ; return


<s> unsigned DPDenormModeDefault = Mode . FP64FP16Denormals ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT ; unsigned NewDenormModeValue = SPDenormMode | ( DPDenormModeDefault << <NUM_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addImm ( NewDenormModeValue ) ; } else { unsigned SPDenormModeBitField = <STR_LIT> :: ID_MODE | ( <NUM_LIT> << <STR_LIT> :: OFFSET_SHIFT_

) | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ;


<s> setAction ( { G_GEP , <NUM_LIT> , S64 } , Legal ) ; setAction ( { G_ICMP , S1 } , Legal ) ; setAction ( { G_ICMP , <NUM_LIT> , S32 } , Legal ) ; setAction ( { G_LOAD , P1 } , Legal ) ; setAction ( { G_LOAD , P2 } , Legal ) ; setAction ( { G_LOAD , S32 } , Legal ) ; setAction ( { G_LOAD , <NUM_LIT> , P1 } , Legal ) ; setAction ( { G_LOAD , <NUM_LIT> , P2 } , Legal ) ; setAction ( { G_SELECT , S32 } , Legal ) ; setAction ( { G_SELECT , <NUM_LIT> , S1 } , Legal ) ; setAction ( { G_STORE , S32 } , Legal ) ; setAction ( { G_STORE , <NUM_LIT> , P1 } , Legal ) ; setAction ( { G_GEP , S64 } , Legal ) ; setAction ( { G_LOAD , <NUM_LIT> , S64 } , Legal ) ; setAction ( { G_STORE , <NUM_LIT> , S64 } , Legal ) ; computeTables ( )


<s> auto FlatNull = MIRBuilder . buildConstant ( SrcTy , <NUM_LIT> ) ; Register PtrLo32 = MRI . createGenericVirtualRegister ( DstTy ) ; MIRBuilder . buildExtract ( PtrLo32 , Src , <NUM_LIT> ) ; Register CmpRes = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildICmp ( CmpInst :: ICMP_NE , CmpRes , Src , FlatNull . getReg ( <NUM_LIT> ) ) ; MIRBuilder . buildSelect ( Dst , CmpRes , PtrLo32 , SegmentNull . getReg ( <NUM_LIT> ) ) ; MI . eraseFromParent ( ) ; return true ; } assert ( SrcAS == <ISA_LIT>AS :: LOCAL_ADDRESS || SrcAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; auto SegmentNull = MIRBuilder . buildConstant ( SrcTy , TM . getNullPointerValue ( SrcAS ) ) ; auto FlatNull = MIRBuilder . buildConstant ( DstTy , TM . getNullPointerValue ( DestAS ) ) ; Register ApertureReg = getSegmentAperture ( DestAS , MRI , MIRBuilder ) ; Register CmpRes = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildICmp ( CmpInst :: ICMP_NE , CmpRes , Src , SegmentNull . getReg ( <NUM_LIT> ) ) ; Register BuildPtr = MRI . createGenericVirtualRegister ( DstTy ) ; Register SrcAsInt = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildInstr ( TargetOpcode :: G_PTRTOINT ) . addDef ( SrcAsInt ) . addUse ( Src ) ; MIRBuilder . buildMerge ( BuildPtr , { SrcAsInt , ApertureReg } ) ; MIRBuilder . buildSelect ( Dst , CmpRes , BuildPtr , FlatNull . getReg ( <NUM_LIT> ) ) ; MI . eraseFromParent ( )


<s> case TargetOpcode :: G_ADDRSPACE_CAST : return legalizeAddrSpaceCast ( MI , MRI , MIRBuilder ) ; case TargetOpcode :: G_FRINT : return legalizeFrint ( MI , MRI , MIRBuilder ) ; case TargetOpcode :: G_FCEIL : return legalizeFceil ( MI , MRI


<s> LLT Ty = MRI . getType ( Src ) ; assert ( Ty . isScalar ( ) && Ty . getSizeInBits ( ) == <NUM_LIT> ) ; APFloat C1Val ( APFloat :: IEEEdouble ( ) , <STR_LIT> ) ; APFloat C2Val ( APFloat :: IEEEdouble ( ) , <STR_LIT> ) ; auto C1 = MIRBuilder . buildFConstant ( Ty , C1Val ) ; auto CopySign = MIRBuilder . buildFCopysign ( Ty , C1 , Src ) ; auto Tmp1 = MIRBuilder . buildFAdd ( Ty , Src , CopySign ) ; auto Tmp2 = MIRBuilder . buildFSub ( Ty , Tmp1 , CopySign ) ; auto C2 = MIRBuilder . buildFConstant ( Ty , C2Val ) ; auto Fabs = MIRBuilder . buildFAbs ( Ty , Src ) ; auto Cond = MIRBuilder . buildFCmp ( CmpInst :: FCMP_OGT , LLT :: scalar ( <NUM_LIT> ) , Fabs , C2 ) ; MIRBuilder . buildSelect ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Cond , Src , Tmp2 ) ; return true


<s> BrCond -> eraseFromParent ( ) ; return true ; } return false ; } case <STR_LIT> : { if ( MachineInstr * BrCond = verifyCFIntrinsic ( MI , MRI ) ) { const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; B . setInstr ( * BrCond ) ; Register Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; B . buildInstr ( <STR_LIT> ) . addUse ( Reg ) . addMBB ( BrCond -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MI . eraseFromParent ( ) ; BrCond -> eraseFromParent ( ) ; MRI . setRegClass ( Reg , TRI -> getWaveMaskRegClass ( ) ) ; return true ; } return false ; } case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeImplicitArgPtr ( MI , MRI , B ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case


<s> const unsigned Shift = countTrailingZeros < unsigned > ( Mask ) ; auto ShiftAmt = B . buildConstant ( S32 , Shift ) ; auto LShr = B . buildLShr ( S32 , LiveIn , ShiftAmt ) ; B . buildAnd ( DstReg , LShr , B . buildConstant ( S32 , Mask >> Shift ) ) ; } else B . buildCopy ( DstReg , LiveIn ) ; if ( ! MRI . getVRegDef ( LiveIn )


<s> switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_ADDRSPACE_CAST : return legalizeAddrSpaceCast ( MI , MRI , B ) ; case TargetOpcode :: G_FRINT : return legalizeFrint ( MI , MRI , B ) ; case TargetOpcode :: G_FCEIL : return legalizeFceil ( MI , MRI , B ) ; case TargetOpcode :: G_INTRIN<ISA_LIT>C_TRUNC : return legalizeIntrinsicTrunc ( MI , MRI , B ) ; case TargetOpcode :: G_<ISA_LIT>TOFP : return legalizeITOFP ( MI , MRI , B , true ) ; case TargetOpcode :: G_UITOFP : return legalizeITOFP ( MI , MRI , B , false ) ; case TargetOpcode :: G_FMINNUM : case TargetOpcode :: G_FMAXNUM : case TargetOpcode :: G_FMINNUM_IEEE : case TargetOpcode :: G_FMAXNUM_IEEE : return legalizeMinNumMaxNum ( MI , MRI , B ) ; case TargetOpcode :: G_EXTRACT_VECTOR_ELT : return legalizeExtractVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_INSERT_VECTOR_ELT : return legalizeInsertVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_F<ISA_LIT>N : case TargetOpcode :: G_FCOS : return legalizeSinCos ( MI , MRI , B ) ; case TargetOpcode :: G_GLOBAL_VALUE : return legalizeGlobalValue ( MI , MRI , B ) ; case TargetOpcode :: G_LOAD : return legalizeLoad ( MI , MRI , B , Observer ) ; case TargetOpcode ::


<s> return legalizeInsertVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_F<ISA_LIT>N : case TargetOpcode :: G_FCOS : return legalizeSinCos ( MI , MRI , B ) ; case TargetOpcode :: G_GLOBAL_VALUE : return legalizeGlobalValue ( MI , MRI , B ) ; case TargetOpcode :: G_LOAD : return legalizeLoad ( MI , MRI , B , Observer ) ; case TargetOpcode :: G_FMAD : return legalizeFMad ( MI , MRI , B ) ; case TargetOpcode :: G_FDIV : return legalizeFDIV ( MI , MRI , B ) ; default : return false ; } llvm_unreachable ( <STR_LIT>


<s> bool <ISA_LIT>LegalizerInfo :: legalizeFDIV ( MachineInstr & MI , MachineRegisterInfo & MRI


<s> if ( legalizeFastUnsafeFDIV ( MI , MRI , B


<s> static LegalizeMutation bitcastToRegisterType ( unsigned TypeIdx ) { return [ = ] ( const LegalityQuery & Query ) { const LLT Ty = Query . Types [ TypeIdx ] ; unsigned Size = Ty . getSizeInBits ( ) ; LLT CoercedTy


<s> const ArgDescriptor * Arg ; const TargetRegisterClass * RC ; LLT ArgTy ; std :: tie ( Arg , RC , ArgTy ) = MFI -> getPreloadedValue ( ArgType ) ; if ( ! Arg ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return nullptr


<s> const ArgDescriptor * <ISA_LIT>LegalizerInfo :: getArgDescriptor ( MachineIRBuilder & B , <STR_LIT> ArgType ) const { const <ISA_LIT>MachineFunctionInfo * MFI = B . getMF ( ) . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const ArgDescriptor * Arg ; const TargetRegisterClass * RC ; LLT ArgTy ; std :: tie ( Arg , RC , ArgTy ) = MFI -> getPreloadedValue ( ArgType ) ; if ( ! Arg ) { LLVM_DEBUG ( dbgs ( ) <<


<s> static bool isLoadStoreLegal ( const GCNSubtarget & ST , const LegalityQuery & Query , unsigned Opcode ) { const LLT Ty = Query . Types [ <NUM_LIT> ] ; return isRegisterType ( Ty ) && isLoadStoreSizeLegal ( ST , Query

, Opcode ) && ! loadStoreBitcastWorkaround ( Ty ) ;


<s> unsigned RegSize = Ty . getSizeInBits ( ) ; unsigned MemSize = Query . MMODescrs [ <NUM_LIT> ] . SizeInBits ; unsigned Align = Query . MMODescrs [ <NUM_LIT> ] . AlignInBits ; unsigned AS = Query . Types [ <NUM_LIT> ] . getAddressSpace ( ) ; if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) return false ; if ( IsLoad && MemSize < Size ) MemSize = std :: max ( MemSize , Align ) ; if ( MemSize != RegSize && RegSize != <NUM_LIT> ) return false ; if ( MemSize > maxSizeForAddrSpace ( ST , AS , IsLoad ) ) return false ; switch ( MemSize ) { case <NUM_LIT> : case <NUM_LIT> : case <NUM_LIT> : case <NUM_LIT>


<s> MI . eraseFromParent ( ) ; return true ; } if ( SrcAS == <ISA_LIT>AS :: FLAT_ADDRESS ) { assert ( DestAS == <ISA_LIT>AS :: LOCAL_ADDRESS || DestAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; unsigned NullVal = TM . getNullPointerValue ( DestAS ) ; auto SegmentNull = B . buildConstant ( DstTy , NullVal ) ; auto FlatNull = B . buildConstant ( SrcTy , <NUM_LIT> ) ; auto PtrLo32 = B . buildExtract ( DstTy , Src , <NUM_LIT> ) ; auto CmpRes = B . buildICmp ( CmpInst :: ICMP_NE , LLT :: scalar ( <NUM_LIT> ) , Src , FlatNull . getReg ( <NUM_LIT> ) ) ; B . buildSelect ( Dst , CmpRes , PtrLo32 , SegmentNull . getReg ( <NUM_LIT> ) ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS != <ISA_LIT>AS :: LOCAL_ADDRESS && SrcAS != <ISA_LIT>AS :: PRIVATE_ADDRESS ) return false ; if ( ! ST . hasFlatAddressSpace ( ) ) return false ; auto SegmentNull = B . buildConstant ( SrcTy , TM . getNullPointerValue ( SrcAS


<s> LLT VecTy = LLT :: vector ( <NUM_LIT> , ValTy ) ; Register PackedVal = B . buildBuildVector ( VecTy , { NewVal , CmpVal } ) . getReg ( <NUM_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( DstReg ) . addUse ( PtrReg )

. addUse ( PackedVal ) . setMemRefs ( MI . memoperands ( ) ) ;


<s> LLT VecTy = LLT :: vector ( <NUM_LIT> , ValTy ) ; Register PackedVal = B . buildBuildVector ( VecTy , { NewVal , CmpVal } ) . getReg ( <NUM_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( DstReg ) . addUse ( PtrReg ) . addUse ( PackedVal ) . setMemRefs ( MI . memoperands ( ) ) ; MI . eraseFromParent


<s> Register VIndex ; if ( HasVIndex ) { VIndex = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; ++ OpOffset ; } Register VOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; Register SOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; unsigned AuxiliaryData = MI . getOperand ( <NUM_LIT> + OpOffset ) . getImm ( ) ; MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; unsigned ImmOffset ; unsigned TotalOffset ; std :: tie ( VOffset , ImmOffset , TotalOffset ) = splitBufferOffsets ( B , VOffset ) ; if ( TotalOffset != <NUM_LIT> ) MMO = B . getMF ( ) . getMachineMemOperand ( MMO , TotalOffset , MMO -> getSize ( ) ) ; if ( ! VIndex ) VIndex = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; auto MIB = B . buildInstr ( getBufferAtomicPseudo ( IID ) ) . addDef ( Dst ) . addUse ( VData ) ; if ( IsCmpSwap ) MIB . addReg ( CmpVal ) ; MIB . addUse ( RSrc ) . addUse ( VIndex ) . addUse ( VOffset ) . addUse ( SOffset ) . addImm ( ImmOffset ) . addImm ( AuxiliaryData ) . addImm ( HasVIndex ? - <NUM_LIT> : <NUM_LIT> ) . addMemOperand ( MMO ) ; MI . eraseFromParent


<s> ++ OpOffset ; } Register RSrc = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; const unsigned NumVIndexOps = IsCmpSwap ? <NUM_LIT> : <NUM_LIT> ; const bool HasVIndex = MI . getNumOperands ( ) == NumVIndexOps ; Register VIndex ; if ( HasVIndex ) { VIndex = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; ++ OpOffset ; } Register VOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; Register SOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; unsigned AuxiliaryData = MI . getOperand ( <NUM_LIT> + OpOffset ) . getImm ( ) ; MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; unsigned ImmOffset ; unsigned TotalOffset ; std :: tie ( VOffset , ImmOffset , TotalOffset ) = splitBufferOffsets ( B , VOffset ) ; if ( TotalOffset != <NUM_LIT> ) MMO = B . getMF ( ) . getMachineMemOperand ( MMO , TotalOffset , MMO ->


<s> } unsigned AuxiliaryData = MI . getOperand ( <NUM_LIT> + OpOffset ) . getImm ( ) ; unsigned ImmOffset ; unsigned TotalOffset ; LLT Ty = MRI . getType ( Dst ) ; LLT EltTy = Ty . getScalarType ( ) ; const bool IsD16 = IsFormat && ( EltTy . getSizeInBits ( ) == <NUM_LIT> ) ; const bool Unpacked = ST . hasUnpackedD16VMem ( ) ; std :: tie ( VOffset , ImmOffset , TotalOffset ) = splitBufferOffsets ( B , VOffset ) ; if ( TotalOffset != <NUM_LIT> ) MMO = B . getMF ( ) . getMachineMemOperand ( MMO , TotalOffset , MemSize ) ; unsigned Opc ; if ( IsTyped ) { Opc = IsD16 ? <STR_LIT> : <STR_LIT> ; } else if ( IsFormat ) { Opc = IsD16 ? <STR_LIT> : <STR_LIT> ; } else { switch ( MemSize ) { case <NUM_LIT> : Opc = <STR_LIT> ; break ; case <NUM_LIT> : Opc = <STR_LIT> ; break ; default : Opc = <STR_LIT> ; break ; } } Register LoadDstReg ; bool IsExtLoad = ( ! IsD16 && MemSize < <NUM_LIT> ) || ( IsD16 && ! Ty . isVector ( ) ) ; LLT UnpackedTy = Ty . changeElementSize ( <NUM_LIT> ) ; if ( IsExtLoad ) LoadDstReg = B . getMRI ( ) -> createGenericVirtualRegister ( S32 ) ; else if ( Unpacked && IsD16 && Ty . isVector ( ) ) LoadDstReg = B . getMRI ( ) -> createGenericVirtualRegister ( UnpackedTy ) ; else LoadDstReg = Dst ; if ( ! VIndex ) VIndex = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; auto MIB = B . buildInstr ( Opc ) . addDef ( LoadDstReg ) . addUse ( RSrc ) . addUse ( VIndex ) . addUse ( VOffset ) . addUse ( SOffset ) . addImm ( ImmOffset ) ; if ( IsTyped ) MIB . addImm ( Format ) ; MIB . addImm ( AuxiliaryData ) . addImm ( HasVIndex ? - <NUM_LIT> : <NUM_LIT> ) . addMemOperand ( MMO ) ; if ( LoadDstReg != Dst ) { B . setInsertPt ( B . getMBB ( ) , ++ B . getInsertPt ( ) ) ; if ( IsExtLoad ) B . buildTrunc ( Dst , LoadDstReg ) ; else { auto Unmerge = B . buildUnmerge ( S32 , LoadDstReg ) ; SmallVector < Register , <NUM_LIT> > Repack ; for ( unsigned I = <NUM_LIT> , N = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; I != N ; ++ I ) Repack . push_back ( B . buildTrunc ( EltTy , Unmerge . getReg ( I ) ) . getReg ( <NUM_LIT> ) ) ; B . buildMerge (


<s> unsigned AuxiliaryData = MI . getOperand ( <NUM_LIT> + OpOffset ) . getImm ( ) ; std :: tie ( VOffset , ImmOffset , TotalOffset ) = splitBufferOffsets ( B , VOffset ) ; if ( TotalOffset != <NUM_LIT> ) MMO = B . getMF ( ) . getMachineMemOperand ( MMO , TotalOffset , MemSize ) ; unsigned Opc ; if ( IsTyped ) { Opc = IsD16 ? <STR_LIT> : <STR_LIT> ; } else if ( IsFormat ) { Opc = IsD16 ? <STR_LIT> : <STR_LIT> ; } else { switch ( MemSize ) { case <NUM_LIT> : Opc = <STR_LIT> ; break ; case <NUM_LIT> : Opc = <STR_LIT> ; break ; default : Opc = <STR_LIT> ; break ; } } if ( ! VIndex ) VIndex = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; auto MIB = B . buildInstr ( Opc ) . addUse ( VData ) . addUse ( RSrc ) . addUse ( VIndex ) . addUse ( VOffset ) . addUse ( SOffset ) . addImm ( ImmOffset ) ; if ( IsTyped ) MIB . addImm ( Format ) ; MIB . addImm ( AuxiliaryData ) . addImm ( HasVIndex ? - <NUM_LIT> : <NUM_LIT> ) . addMemOperand (


<s> bool <ISA_LIT>LegalizerInfo :: legalizeBuildVector ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; assert ( MRI . getType ( Dst ) == LLT :: vector ( <NUM_LIT> , <NUM_LIT> ) ) ; Register Src0 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src1 = MI . getOperand ( <NUM_LIT> ) . getReg (


<s> return legalizeFPTOI ( MI , MRI , B , false ) ; case TargetOpcode :: G_FMINNUM : case TargetOpcode :: G_FMAXNUM : case TargetOpcode :: G_FMINNUM_IEEE : case TargetOpcode :: G_FMAXNUM_IEEE : return legalizeMinNumMaxNum ( Helper , MI ) ; case TargetOpcode :: G_EXTRACT_VECTOR_ELT : return legalizeExtractVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_INSERT_VECTOR_ELT : return legalizeInsertVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return legalizeShuffleVector ( MI , MRI , B ) ; case TargetOpcode :: G_F<ISA_LIT>N : case TargetOpcode :: G_FCOS : return legalizeSinCos ( MI , MRI , B ) ; case TargetOpcode :: G_GLOBAL_VALUE : return legalizeGlobalValue ( MI , MRI , B ) ; case TargetOpcode :: G_LOAD : return legalizeLoad ( MI , MRI , B , Observer ) ; case TargetOpcode :: G_FMAD : return legalizeFMad ( MI , MRI , B ) ; case TargetOpcode :: G_FDIV : return legalizeFDIV ( MI , MRI , B ) ; case TargetOpcode :: G_UDIV : case TargetOpcode :: G_UREM : return legalizeUDIV_UREM ( MI , MRI , B ) ; case TargetOpcode :: G_SDIV : case TargetOpcode :: G_SREM : return legalizeSDIV_SREM ( MI , MRI , B ) ; case TargetOpcode :: G_ATOMIC_CMPXCHG : return legalizeAtomicCmpXChg ( MI , MRI , B ) ; case TargetOpcode :: G_FLOG : return legalizeFlog ( MI , B , numbers :: ln2f ) ; case TargetOpcode :: G_FLOG10 : return legalizeFlog ( MI , B , numbers :: ln2f / numbers :: ln10f ) ; case TargetOpcode :: G_FEXP : return legalizeFExp ( MI , B ) ; case TargetOpcode :: G_FPOW : return legalizeFPow ( MI , B ) ; case TargetOpcode :: G_FFLOOR : return legalizeFFloor ( MI , MRI , B ) ; case TargetOpcode :: G_BUILD_VECTOR : return legalizeBuildVector ( MI , MRI , B ) ; default : return false ; } llvm_unreachable ( <STR_LIT>


<s> case TargetOpcode :: G_EXTRACT_VECTOR_ELT : return legalizeExtractVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_INSERT_VECTOR_ELT : return legalizeInsertVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return legalizeShuffleVector ( MI , MRI , B ) ; case TargetOpcode :: G_F<ISA_LIT>N : case TargetOpcode :: G_FCOS : return legalizeSinCos ( MI , MRI , B ) ; case TargetOpcode :: G_GLOBAL_VALUE : return legalizeGlobalValue ( MI , MRI , B ) ; case TargetOpcode :: G_LOAD : return legalizeLoad ( MI , MRI , B , Observer ) ; case TargetOpcode :: G_FMAD : return legalizeFMad ( MI , MRI , B ) ; case TargetOpcode :: G_FDIV : return legalizeFDIV ( MI , MRI , B ) ; case TargetOpcode :: G_UDIV : case TargetOpcode :: G_UREM : return legalizeUDIV_UREM ( MI , MRI , B ) ; case TargetOpcode :: G_SDIV : case TargetOpcode :: G_SREM : return legalizeSDIV_SREM ( MI , MRI , B ) ; case TargetOpcode :: G_ATOMIC_CMPXCHG : return legalizeAtomicCmpXChg ( MI , MRI , B ) ; case TargetOpcode :: G_FLOG : return legalizeFlog ( MI , B , numbers :: ln2f ) ; case TargetOpcode :: G_FLOG10 : return legalizeFlog ( MI , B , numbers :: ln2f / numbers :: ln10f ) ; case TargetOpcode :: G_FEXP : return legalizeFExp ( MI , B ) ; case TargetOpcode :: G_FPOW : return legalizeFPow ( MI , B ) ; case TargetOpcode :: G_FFLOOR : return legalizeFFloor ( MI , MRI , B ) ; case TargetOpcode :: G_BUILD_VECTOR : return legalizeBuildVector ( MI , MRI


<s> if ( ST . getTrapHandlerAbi ( ) != GCNSubtarget :: TrapHandlerAbiHsa || ! ST . isTrapHandlerEnabled ( ) ) { DiagnosticInfoUnsupported NoTrap ( B . getMF ( ) . getFunction ( ) , <STR_LIT> , MI . getDebugLoc ( ) , DS_Warning ) ; LLVMContext & Ctx = B . getMF ( ) . getFunction ( ) . getContext ( ) ; Ctx . diagnose ( NoTrap ) ; } else { B . buildInstr ( <STR_LIT> ) . addImm ( GCNSubtarget :: TrapIDLLVMDebugTrap ) ; } MI . eraseFromParent ( ) ; return


<s> Optional < ValueAndVReg > IdxVal = getConstantVRegValWithLookThrough ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI ) ; if ( ! IdxVal ) return true ; Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Vec = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT VecTy = MRI . getType ( Vec ) ; LLT EltTy = VecTy . getElementType ( ) ; assert ( EltTy == MRI . getType ( Dst ) ) ; if ( IdxVal -> Value < VecTy . getNumElements ( ) ) B . buildExtract ( Dst ,

Vec , IdxVal -> Value * EltTy . getSizeInBits ( ) ) ;


<s> LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; if ( legalizeFastUnsafeFDIV ( MI , MRI , B ) ) return true ; if ( DstTy == S16 ) return legalizeFDIV16 ( MI , MRI , B ) ; if ( DstTy == S32 ) return legalizeFDIV32 ( MI , MRI , B


<s> bool <ISA_LIT>LegalizerInfo :: legalizeFDIV ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( Dst ) ; LLT S16 = LLT :: scalar ( <NUM_LIT> ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; if ( legalizeFastUnsafeFDIV ( MI , MRI , B ) ) return true ; if ( DstTy == S16 ) return legalizeFDIV16 ( MI


<s> LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto LHSExt = B . buildFPExt ( S32 , LHS , Flags ) ; auto RHSExt = B . buildFPExt ( S32 , RHS , Flags ) ; auto RCP = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( RHSExt . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto QUOT = B . buildFMul ( S32 , LHSExt ,


<s> auto One = B . buildFConstant ( S32 , <NUM_LIT> ) ; auto DenominatorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto NumeratorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto ApproxRcp = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( DenominatorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto NegDivScale0 = B . buildFNeg ( S32 , DenominatorScaled , Flags ) ; if ( ! Mode . allFP32Denormals ( ) ) toggleSPDenormMode ( true , B , ST , Mode ) ; auto Fma0 = B . buildFMA ( S32 , NegDivScale0 , ApproxRcp , One , Flags ) ; auto Fma1 = B . buildFMA ( S32 , Fma0 , ApproxRcp , ApproxRcp , Flags ) ; auto Mul = B . buildFMul ( S32 , NumeratorScaled , Fma1 ,


<s> auto DenominatorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto NumeratorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto ApproxRcp = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( DenominatorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto NegDivScale0 = B . buildFNeg ( S32 , DenominatorScaled , Flags ) ; if ( ! Mode . allFP32Denormals ( ) ) toggleSPDenormMode ( true , B , ST , Mode ) ; auto Fma0 = B . buildFMA ( S32 , NegDivScale0 , ApproxRcp , One , Flags ) ; auto Fma1 = B . buildFMA ( S32 , Fma0 , ApproxRcp , ApproxRcp , Flags ) ; auto Mul = B . buildFMul ( S32 , NumeratorScaled , Fma1 , Flags ) ; auto Fma2 = B . buildFMA ( S32 , NegDivScale0 , Mul , NumeratorScaled , Flags ) ; auto Fma3 = B . buildFMA ( S32 , Fma2 , Fma1 , Mul , Flags ) ; auto Fma4 = B . buildFMA ( S32 , NegDivScale0 , Fma3 , NumeratorScaled , Flags ) ; if ( ! Mode . allFP32Denormals ( ) ) toggleSPDenormMode ( false , B , ST , Mode ) ; auto Fmas = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( Fma4 . getReg ( <NUM_LIT> ) ) . addUse ( Fma1 . getReg ( <NUM_LIT> ) ) . addUse

( Fma3 . getReg ( <NUM_LIT> ) ) . addUse ( NumeratorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ;


<s> auto K1 = B . buildFConstant ( S64 , BitsToDouble ( UINT64_C ( <NUM_LIT> ) ) ) ; auto Mul = B . buildFMul ( S64 , Trunc , K0 , Flags ) ; auto FloorMul = B . buildFFloor ( S64 , Mul , Flags ) ; auto Fma = B . buildFMA ( S64 , FloorMul , K1 , Trunc , Flags ) ; auto Hi = Signed ? B . buildFPTO<ISA_LIT> ( S32 , FloorMul ) : B . buildFPTOUI ( S32 , FloorMul ) ; auto Lo = B . buildFPTOUI ( S32 , Fma ) ; B . buildMerge ( Dst , { Lo , Hi }


<s> auto CopySign = B . buildFCopysign ( Ty , C1 , Src ) ; auto Tmp1 = B . buildFAdd ( Ty , Src , CopySign ) ; auto Tmp2 = B . buildFSub ( Ty , Tmp1 , CopySign ) ; auto C2 = B . buildFConstant ( Ty , C2Val ) ; auto Fabs = B . buildFAbs ( Ty


<s> bool <ISA_LIT>LegalizerInfo :: legalizeFrint ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { Register Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT Ty = MRI . getType ( Src ) ; assert ( Ty . isScalar ( ) && Ty . getSizeInBits ( ) == <NUM_LIT> ) ; APFloat C1Val ( APFloat :: IEEEdouble ( ) , <STR_LIT> ) ; APFloat C2Val ( APFloat :: IEEEdouble ( ) , <STR_LIT> ) ; auto C1 = B . buildFConstant ( Ty , C1Val ) ; auto CopySign = B . buildFCopysign ( Ty , C1 , Src ) ; auto Tmp1 = B . buildFAdd ( Ty , Src , CopySign ) ; auto Tmp2 = B . buildFSub ( Ty , Tmp1 , CopySign ) ; auto C2 = B . buildFConstant ( Ty , C2Val ) ; auto Fabs = B . buildFAbs ( Ty , Src ) ; auto Cond = B . buildFCmp ( CmpInst :: FCMP_OGT , LLT :: scalar ( <NUM_LIT> ) , Fabs , C2 ) ; B . buildSelect ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Cond , Src ,


<s> MachineFunction & MF = B . getMF ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) { if ( ! MFI -> isEntryFunction ( ) ) { const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadLDSDecl ( Fn , <STR_LIT> , MI . getDebugLoc ( ) , DS_Warning ) ; Fn . getContext ( ) . diagnose ( BadLDSDecl ) ; B . buildIntrinsic ( <STR_LIT> , ArrayRef < Register > ( ) , true ) ; B . buildUndef ( DstReg ) ; MI . eraseFromParent ( ) ; return true ; } if ( ! <ISA_LIT>TargetLowering :: hasDefinedInitializer ( GV ) ) { const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( ! TLI -> shouldUseLDSConstAddress ( GV ) ) { MI . getOperand ( <NUM_LIT> ) . setTargetFlags ( <ISA_LIT>InstrInfo :: MO_ABS32_LO ) ; return true ; } B . buildConstant ( DstReg , MFI -> allocateLDSGlobal ( B . getDataLayout ( ) , * cast < GlobalVariable > ( GV ) ) ) ; MI . eraseFromParent ( ) ; return true ; } const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadInit ( Fn , <STR_LIT> , MI . getDebugLoc ( ) ) ; Fn . getContext ( ) . diagnose ( BadInit ) ; return true ; } const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( TLI -> shouldEmitFixup ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> ) ; MI . eraseFromParent ( ) ; return true ; } if ( TLI -> shouldEmitPCReloc ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_REL32 ) ; MI . eraseFromParent ( ) ; return true ; } LLT PtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register GOTAddr = MRI . createGenericVirtualRegister ( PtrTy ) ; MachineMemOperand * GOTMMO = MF . getMachineMemOperand ( MachinePointerInfo :: getGOT ( MF ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , <NUM_LIT> , Align ( <NUM_LIT> ) ) ; buildPCRelGlobalAddress ( GOTAddr , PtrTy , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; if ( Ty . getSizeInBits ( ) == <NUM_LIT> ) { auto Load = B . buildLoad ( PtrTy , GOTAddr


<s> if ( ! MFI -> isEntryFunction ( ) ) { return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; } uint64_t Offset = ST . getTargetLowering ( ) -> getImplicitParameterOffset ( B . getMF ( ) , <ISA_LIT>TargetLowering :: FIRST_IMPLICIT ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( DstReg ) ; LLT IdxTy = LLT :: scalar ( DstTy . getSizeInBits ( ) ) ; const ArgDescriptor * Arg ; const TargetRegisterClass


<s> if ( ! IdxVal ) return true ; Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Vec = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Ins = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT VecTy = MRI . getType ( Vec ) ; LLT EltTy = VecTy . getElementType ( ) ; assert ( EltTy == MRI .


<s> Register Vec = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Ins = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT VecTy = MRI . getType ( Vec ) ; LLT EltTy = VecTy . getElementType ( ) ; assert ( EltTy == MRI . getType ( Ins ) ) ; if ( IdxVal -> Value < VecTy . getNumElements ( ) ) B . buildInsert ( Dst , Vec , Ins , IdxVal -> Value * EltTy . getSizeInBits ( ) ) ; else B . buildUndef ( Dst ) ; MI . eraseFromParent (


<s> Register Def = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Use = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock * CondBrTarget = BrCond -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; B . setInsertPt ( B . getMBB ( ) , BrCond -> getIterator ( ) ) ; if ( IntrID == <STR_LIT> ) { B . buildInstr ( <STR_LIT> ) . addDef ( Def ) . addUse ( Use ) . addMBB ( UncondBrTarget ) ; } else { B . buildInstr ( <STR_LIT> ) . addDef ( Def ) . addUse ( Use ) . addMBB ( UncondBrTarget ) . addImm ( <NUM_LIT> ) ; } if ( Br ) { Br -> getOperand ( <NUM_LIT> ) . setMBB ( CondBrTarget ) ; } else { B . buildBr ( * CondBrTarget ) ; } MRI . setRegClass ( Def , TRI -> getWaveMaskRegClass ( ) ) ; MRI . setRegClass ( Use , TRI -> getWaveMaskRegClass ( ) ) ; MI . eraseFromParent ( ) ; BrCond -> eraseFromParent ( ) ; return true ; } return false ; } case <STR_LIT> : { MachineInstr * Br = nullptr ; MachineBasicBlock * UncondBrTarget = nullptr ; if ( MachineInstr * BrCond = verifyCFIntrinsic ( MI , MRI , Br , UncondBrTarget ) ) { const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; MachineBasicBlock * CondBrTarget = BrCond -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; Register Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; B . setInsertPt ( B . getMBB ( ) , BrCond -> getIterator ( ) ) ; B . buildInstr ( <STR_LIT> ) . addUse ( Reg ) . addMBB ( UncondBrTarget ) ; if ( Br ) Br -> getOperand ( <NUM_LIT> ) . setMBB ( CondBrTarget ) ; else B . buildBr ( * CondBrTarget ) ; MI . eraseFromParent ( ) ; BrCond -> eraseFromParent ( ) ; MRI . setRegClass ( Reg , TRI -> getWaveMaskRegClass ( ) ) ; return true ; } return false ; } case <STR_LIT> : if ( ! <STR_LIT> ( B . getMF ( ) . getFunction ( ) . getCallingConv ( ) ) ) { B . buildConstant ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> ) ; MI . eraseFromParent ( ) ; return true ; } return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeImplicitArgPtr ( MI , MRI ,


<s> auto Not = B . buildNot ( S64 , Shr ) ; auto Tmp0 = B . buildAnd ( S64 , Src , Not ) ; auto FiftyOne = B . buildConstant ( S32 , FractBits - <NUM_LIT> ) ; auto ExpLt0 = B . buildICmp ( CmpInst :: ICMP_SLT , S1 , Exp , Zero32 ) ; auto ExpGt51 = B . buildICmp ( CmpInst :: ICMP_SGT , S1 , Exp , FiftyOne ) ; auto Tmp1 = B . buildSelect ( S64 , ExpLt0 , SignBit64 , Tmp0 ) ; B . buildSelect ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , ExpGt51 ,


<s> auto Exp = extractF64Exponent ( Hi , B ) ; const unsigned FractBits = <NUM_LIT> ; const auto SignBitMask = B . buildConstant ( S32 , UINT32_C ( <NUM_LIT> ) << <NUM_LIT> ) ; auto SignBit = B . buildAnd ( S32 , Hi , SignBitMask ) ; const auto FractMask = B . buildConstant ( S64 , ( UINT64_C ( <NUM_LIT> ) << FractBits ) - <NUM_LIT> ) ; const auto Zero32 = B . buildConstant ( S32 , <NUM_LIT> ) ; auto SignBit64 = B . buildMerge ( S64 , { Zero32 , SignBit } ) ; auto Shr = B . buildAShr ( S64 , FractMask , Exp ) ; auto Not = B . buildNot ( S64 , Shr ) ; auto Tmp0 = B . buildAnd ( S64 , Src , Not ) ; auto FiftyOne = B . buildConstant ( S32 , FractBits - <NUM_LIT> ) ; auto ExpLt0 = B . buildICmp ( CmpInst :: ICMP_SLT , S1 , Exp , Zero32 ) ; auto ExpGt51 = B . buildICmp ( CmpInst :: ICMP_SGT , S1 , Exp , FiftyOne ) ; auto Tmp1 = B . buildSelect ( S64


<s> auto Hi32 = B . buildExtract ( LLT :: scalar ( <NUM_LIT> ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> ) ; B . buildICmp ( ICmpInst :: ICMP_EQ , MI . getOperand ( <NUM_LIT> ) , Hi32 , ApertureReg ) ; MI . eraseFromParent (


<s> Register Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; assert ( MRI . getType ( Src ) == S64 && MRI . getType ( Dst ) == S64 ) ; auto Unmerge = B . buildUnmerge ( { S32 , S32 } , Src ) ; auto CvtHi = Signed ? B . build<ISA_LIT>TOFP ( S64 , Unmerge . getReg ( <NUM_LIT>


<s> Observer . changingInstr ( MI ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( Cast . getReg ( <NUM_LIT> ) ) ; Observer . changedInstr ( MI ) ; return true


<s> bool <ISA_LIT>LegalizerInfo :: legalizePreloadedArgIntrin ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B , <STR_LIT> ArgType ) const { const ArgDescriptor * Arg = getArgDescriptor ( B


<s> bool <ISA_LIT>LegalizerInfo :: legalizeShuffleVector ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { const LLT V2S16 = LLT :: vector ( <NUM_LIT> , <NUM_LIT> ) ; Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src0 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( Dst ) ; LLT SrcTy = MRI . getType ( Src0 ) ; if ( SrcTy == V2S16 && DstTy == V2S16 && <STR_LIT> ( MI . getOperand ( <NUM_LIT> ) .


<s> const ArgDescriptor * Arg = getArgDescriptor ( B , <STR_LIT> ) ; if ( ! Arg ) return false ; MachineRegisterInfo & MRI = * B . getMRI ( ) ; Register SGPR01 ( <STR_LIT> ) ; Register LiveIn = getLiveInRegister ( B , MRI , SGPR01 , LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) , false ) ; if ( ! loadInputValue ( LiveIn , B , Arg ) ) return false ; B . buildCopy (


<s> const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const bool IsDiv = MI . getOpcode ( ) == <STR_LIT> ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Num = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Den = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT Ty = MRI . getType ( DstReg ) ; if ( Ty == S32 ) legalizeUDIV_UREM32Impl ( B , DstReg , Num , Den , IsDiv ) ; else if ( Ty == S64 ) legalizeUDIV_UREM64Impl ( B , DstReg , Num , Den , IsDiv ) ; else return false ; MI . eraseFromParent ( ) ; return


<s> const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const bool IsDiv = MI . getOpcode ( ) == <STR_LIT> ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Num = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Den = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT Ty = MRI . getType ( DstReg ) ; if ( Ty == S32 ) legalizeUDIV_UREM32Impl ( B , DstReg , Num , Den , IsDiv ) ; else if ( Ty == S64 ) legalizeUDIV_UREM64Impl ( B , DstReg


<s> bool <ISA_LIT>LegalizerInfo :: legalizeUDIV_UREM32 ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { const bool IsDiv = MI . getOpcode ( ) == <STR_LIT> ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Num = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Den = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; legalizeUDIV_UREM32Impl ( B , DstReg , Num , Den , IsDiv ) ; MI . eraseFromParent ( ) ; return


<s> auto One = B . buildConstant ( S32 , <NUM_LIT> ) ; auto Cond = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , R , Y ) ; if ( IsDiv ) Q = B . buildSelect ( S32 , Cond , B . buildAdd ( S32 , Q , One ) , Q ) ; R = B . buildSelect ( S32 , Cond , B . buildSub ( S32 , R , Y ) , R ) ; Cond = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , R , Y ) ; if ( IsDiv ) B . buildSelect ( DstReg , Cond , B . buildAdd (


<s> auto FloatY = B . buildUITOFP ( S32 , Y ) ; auto RcpIFlag = B . buildInstr ( <STR_LIT> , { S32 } , { FloatY } ) ; auto Scale = B . buildFConstant ( S32 , BitsToFloat ( <NUM_LIT> ) ) ; auto ScaledY = B . buildFMul ( S32 , RcpIFlag , Scale ) ; auto Z = B . buildFPTOUI ( S32 , ScaledY ) ; auto NegY = B . buildSub ( S32 , B . buildConstant ( S32 , <NUM_LIT> ) , Y ) ; auto NegYZ = B . buildMul ( S32 , NegY , Z ) ; Z = B . buildAdd ( S32 , Z , B . buildUMulH ( S32 , Z , NegYZ ) ) ; auto Q = B . buildUMulH ( S32 , X , Z ) ; auto R = B . buildSub ( S32 , X , B . buildMul ( S32 , Q , Y ) ) ; auto One = B . buildConstant ( S32 , <NUM_LIT> ) ; auto Cond = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , R , Y ) ; if ( IsDiv ) Q = B . buildSelect ( S32 , Cond , B . buildAdd ( S32 , Q , One ) , Q ) ; R = B . buildSelect ( S32 , Cond , B . buildSub ( S32 , R , Y ) , R ) ; Cond = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , R , Y ) ; if ( IsDiv ) B . buildSelect ( DstReg , Cond , B . buildAdd ( S32


<s> auto CmpHi = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , Sub1_Hi , DenomHi ) ; auto C1 = B . buildSExt ( S32 , CmpHi ) ; auto CmpLo = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , Sub1_Lo , DenomLo ) ; auto C2 = B . buildSExt ( S32 , CmpLo ) ; auto CmpEq = B . buildICmp ( CmpInst :: ICMP_EQ , S1 , Sub1_Hi , DenomHi ) ; auto C3 = B . buildSelect ( S32 , CmpEq , C2 , C1 ) ; auto Sub2_Lo = B . buildUSubo ( S32 , S1 , Sub1_Lo , DenomLo ) ; auto Sub2_Mi = B . buildUSube ( S32 , S1 , Sub1_Mi , DenomHi , Sub1_Lo . getReg ( <NUM_LIT> ) ) ; auto Sub2_Hi = B . buildUSube ( S32 , S1 , Sub2_Mi , Zero32 , Sub2_Lo . getReg ( <NUM_LIT> ) ) ; auto Sub2 = B . buildMerge ( S64 , { Sub2_Lo , Sub2_Hi } ) ; auto One64 = B . buildConstant ( S64 , <NUM_LIT> ) ; auto Add3 = B . buildAdd ( S64 , MulHi3 , One64 ) ; auto C4 = B . buildSExt ( S32 , B . buildICmp ( CmpInst :: ICMP_UGE , S1 , Sub2_Hi , DenomHi ) ) ; auto C5 = B . buildSExt ( S32 , B . buildICmp ( CmpInst :: ICMP_UGE , S1 , Sub2_Lo , DenomLo ) ) ; auto C6 = B . buildSelect ( S32 , B . buildICmp ( CmpInst :: ICMP_EQ , S1 , Sub2_Hi , DenomHi ) , C5 , C4 ) ; auto Add4 = B . buildAdd ( S64 , Add3 , One64 ) ; auto Sub3_Lo = B . buildUSubo ( S32 , S1 , Sub2_Lo , DenomLo ) ; auto Sub3_Mi = B . buildUSube ( S32 , S1 , Sub2_Mi , DenomHi , Sub2_Lo . getReg ( <NUM_LIT> ) ) ; auto Sub3_Hi = B . buildUSube ( S32 , S1 , Sub3_Mi , Zero32 , Sub3_Lo . getReg ( <NUM_LIT> ) ) ; auto Sub3 = B . buildMerge ( S64 , { Sub3_Lo , Sub3_Hi } ) ; if ( IsDiv ) { auto Sel1 = B . buildSelect ( S64 , B . buildICmp ( CmpInst :: ICMP_NE , S1 , C6 , Zero32 ) , Add4 , Add3 ) ; B . buildSelect ( DstReg , B . buildICmp ( CmpInst :: ICMP_NE , S1 , C3 , Zero32 ) , Sel1 , MulHi3 ) ; } else { auto Sel2 = B . buildSelect ( S64 , B . buildICmp ( CmpInst :: ICMP_NE , S1 ,


<s> switch ( AS ) { case <ISA_LIT>AS :: PRIVATE_ADDRESS : return <NUM_LIT> ; case <ISA_LIT>AS :: LOCAL_ADDRESS : return ST . useDS128 ( ) ? <NUM_LIT> : <NUM_LIT> ; case <ISA_LIT>AS :: GLOBAL_ADDRESS


<s> static unsigned maxSizeForAddrSpace ( const GCNSubtarget & ST , unsigned AS , bool IsLoad ) { switch ( AS ) { case <ISA_LIT>AS :: PRIVATE_ADDRESS : return <NUM_LIT> ; case <ISA_LIT>AS :: LOCAL_ADDRESS : return ST . useDS128 ( ) ? <NUM_LIT> : <NUM_LIT> ; case <ISA_LIT>AS :: GLOBAL_ADDRESS : case <ISA_LIT>AS :: CONSTANT_ADDRESS : case <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT : return IsLoad ? <NUM_LIT> : <NUM_LIT> ; default : return


<s> AddrReg = B . buildBitcast ( V2S16 , AddrReg ) . getReg ( <NUM_LIT> ) ; PackedAddrs . push_back ( AddrReg ) ; } else { if ( ( ( I + <NUM_LIT> ) >= EndIdx ) || ( ( NumGradients / <NUM_LIT> ) % <NUM_LIT> == <NUM_LIT> && ( I == DimIdx + ( NumGradients / <NUM_LIT> ) - <NUM_LIT> || I == DimIdx + NumGradients - <NUM_LIT> ) ) || ! MI . getOperand ( I + <NUM_LIT> ) . isReg ( ) ) { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) } ) . getReg ( <NUM_LIT> ) ) ; } else


<s> if ( ( ( I + <NUM_LIT> ) >= EndIdx ) || ( ( NumGradients / <NUM_LIT> ) % <NUM_LIT> == <NUM_LIT> && ( I == DimIdx + ( NumGradients / <NUM_LIT> ) - <NUM_LIT> || I == DimIdx + NumGradients - <NUM_LIT> ) ) || ! MI . getOperand ( I + <NUM_LIT> ) . isReg ( ) ) { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) } ) . getReg ( <NUM_LIT> ) ) ; } else


<s> static MachineInstr * verifyCFIntrinsic ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineInstr * & Br , MachineBasicBlock * & UncondBrTarget ) { Register CondDef = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! MRI . hasOneNonDBGUse ( CondDef ) ) return nullptr ; MachineBasicBlock * Parent = MI . getParent ( ) ; MachineInstr & UseMI = * MRI


<s> const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; const LLT P1 = LLT :: pointer ( <NUM_LIT> , <NUM_LIT> ) ; const LLT P2 = LLT :: pointer ( <NUM_LIT> , <NUM_LIT> ) ; setAction ( { G_ADD , S32 } , Legal ) ; setAction ( { G_AND , S32 } , Legal ) ; setAction ( { G_BITCAST , V2S16 } , Legal ) ; setAction ( { G_BITCAST , <NUM_LIT> , S32 } , Legal ) ; setAction ( { G_BITCAST , S32 } , Legal ) ; setAction ( { G_BITCAST , <NUM_LIT> , V2S16 } , Legal ) ; setAction ( { G_CONSTANT , S1 } , Legal ) ; setAction ( { G_CONSTANT , S32 } , Legal ) ; setAction ( { G_CONSTANT , S64 } , Legal ) ; setAction ( { G_FCONSTANT , S32 } , Legal ) ; setAction ( { G_FMUL , S32 } , Legal ) ; setAction ( { G_GEP , P1 } , Legal ) ; setAction ( { G_GEP , P2 } , Legal ) ; setAction ( { G_GEP , <NUM_LIT> , S64 } , Legal ) ; setAction ( { G_ICMP , S1 } , Legal


<s> const LLT Ty = Query . Types [ TypeIdx ] ; unsigned Size = Ty . getSizeInBits ( ) ; assert ( Size % <NUM_LIT> == <NUM_LIT> ) ; return std :: make_pair ( TypeIdx , LLT :: scalarOrVector ( ElementCount :: getFixed ( Size / <NUM_LIT>


<s> Register PCReg = PtrTy . getSizeInBits ( ) != <NUM_LIT> ? DstReg : B . getMRI ( ) -> createGenericVirtualRegister ( ConstPtrTy ) ; MachineInstrBuilder MIB = B . buildInstr ( <STR_LIT> ) . addDef ( PCReg ) ; MIB . addGlobalAddress ( GV , Offset + <NUM_LIT> , GAFlags ) ; if ( GAFlags == <ISA_LIT>InstrInfo :: MO_NONE ) MIB . addImm ( <NUM_LIT> ) ; else MIB . addGlobalAddress ( GV , Offset + <NUM_LIT> , GAFlags + <NUM_LIT> ) ; B . getMRI ( ) -> setRegClass


<s> return [ = ] ( const LegalityQuery & Query ) { const LLT QueryTy = Query . Types [ TypeIdx ] ; if ( ! QueryTy . isVector (


<s> const LLT QueryTy = Query . Types [ TypeIdx ] ; if ( ! QueryTy . isVector ( ) ) return false ; const LLT EltTy = QueryTy . getElementType ( ) ; return EltTy == LLT :: scalar ( <NUM_LIT> ) ||

EltTy . getSizeInBits ( ) >= <NUM_LIT> ;


<s> auto Mul1 = B . buildFMul ( S32 , Rcp , B . buildFConstant ( S32 , BitsToFloat ( <NUM_LIT> ) ) ) ; auto Mul2 = B . buildFMul ( S32 , Mul1 , B . buildFConstant ( S32 , BitsToFloat ( <NUM_LIT> ) ) ) ; auto Trunc = B . buildIntrinsicTrunc ( S32 , Mul2 ) ; auto Mad2 = B . buildFMAD ( S32 , Trunc , B . buildFConstant ( S32 , BitsToFloat ( <NUM_LIT> ) ) , Mul1 ) ; auto ResultLo = B . buildFPTOUI ( S32 , Mad2 ) ; auto ResultHi = B . buildFPTOUI ( S32 , Trunc ) ; return { ResultLo . getReg ( <NUM_LIT> ) , ResultHi


<s> auto Mad = B . buildFMAD ( S32 , CvtHi , B . buildFConstant ( S32 , BitsToFloat ( <NUM_LIT> ) ) , CvtLo ) ; auto Rcp = B . buildInstr ( <STR_LIT> , { S32 } , { Mad } ) ; auto Mul1 = B . buildFMul ( S32 , Rcp , B . buildFConstant ( S32 , BitsToFloat ( <NUM_LIT> ) ) ) ; auto Mul2 = B . buildFMul ( S32 , Mul1 , B . buildFConstant ( S32 , BitsToFloat ( <NUM_LIT> ) ) ) ; auto Trunc = B . buildIntrinsicTrunc ( S32 , Mul2 ) ; auto Mad2 = B . buildFMAD ( S32 , Trunc , B . buildFConstant ( S32 , BitsToFloat ( <NUM_LIT> ) ) , Mul1 ) ; auto ResultLo = B . buildFPTOUI


<s> } return LLT :: scalarOrVector ( ElementCount :: getFixed (


<s> return LLT :: scalarOrVector ( ElementCount :: getFixed ( Size


<s> case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case


<s> case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT>


<s> static unsigned getDSFPAtomicOpcode ( <STR_LIT> IID ) { switch ( IID ) { case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT>


<s> unsigned Bits = Ty . getSizeInBits ( ) ; unsigned Pow2Bits = <NUM_LIT> << Log2_32_Ceil ( Bits ) ; return LLT :: scalar ( Pow2Bits


<s> unsigned Bits = Ty . getSizeInBits ( ) ; unsigned Pow2Bits = <NUM_LIT> << Log2_32_Ceil (


<s> static LLT getPow2VectorType ( LLT Ty ) { unsigned NElts = Ty .


<s> unsigned NElts = Ty . getNumElements ( ) ; unsigned Pow2NElts = <NUM_LIT> << Log2_32_Ceil ( NElts ) ; return Ty . changeElementCount ( ElementCount ::


<s> return B . buildBuildVector ( LLT :: fixed_vector ( <NUM_LIT> , S32 ) , PackedRegs ) . getReg ( <NUM_LIT> ) ; } if ( StoreVT . getNumElements ( ) == <NUM_LIT> ) { SmallVector < Register , <NUM_LIT> > PackedRegs ; auto Unmerge = B . buildUnmerge ( S16 , Reg ) ; for ( int I = <NUM_LIT> , E = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; I != E ; ++ I ) PackedRegs . push_back ( Unmerge . getReg ( I ) ) ; PackedRegs . resize ( <NUM_LIT> , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) ) ; Reg = B . buildBuildVector ( LLT :: fixed_vector ( <NUM_LIT> , S16 ) , PackedRegs ) . getReg ( <NUM_LIT> ) ; return B . buildBitcast ( LLT :: fixed_vector ( <NUM_LIT> , S32 ) , Reg ) . getReg ( <NUM_LIT> ) ; } if ( StoreVT . getNumElements ( ) == <NUM_LIT> ) { SmallVector < Register , <NUM_LIT> > PackedRegs ; Reg = B . buildBitcast ( LLT :: fixed_vector ( <NUM_LIT> , S32 ) , Reg ) . getReg ( <NUM_LIT> ) ; auto Unmerge = B . buildUnmerge ( S32 , Reg ) ; for ( int I = <NUM_LIT> , E = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; I != E ; ++ I ) PackedRegs . push_back ( Unmerge . getReg ( I ) ) ; PackedRegs . resize ( <NUM_LIT> , B . buildUndef ( S32 ) . getReg ( <NUM_LIT> ) ) ; return B . buildBuildVector ( LLT :: fixed_vector ( <NUM_LIT> , S32 ) , PackedRegs ) . getReg ( <NUM_LIT> ) ; } llvm_unreachable ( <STR_LIT> )


<s> if ( MI . getOpcode ( ) != TargetOpcode :: G_XOR ) return false ; auto ConstVal = getIConstantVRegSExtVal ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ,


<s> static bool isRegisterSize ( unsigned


<s> return EltSize == <NUM_LIT> || EltSize == <NUM_LIT> || ( EltSize == <NUM_LIT> && Ty . getNumElements ( ) % <NUM_LIT> == <NUM_LIT> ) || EltSize == <NUM_LIT> ||


<s> static bool isRegisterVectorType ( LLT Ty ) { const int EltSize = Ty . getElementType ( ) .


<s> static LegalityPredicate isSmallOddVector ( unsigned TypeIdx ) { return [ = ] ( const LegalityQuery & Query ) { const LLT Ty = Query .


<s> return [ = ] ( const LegalityQuery & Query ) { const LLT Ty = Query . Types [ TypeIdx ] ; const LLT EltTy = Ty . getScalarType ( ) ; return EltTy . getSizeInBits ( ) == <NUM_LIT> &&


<s> auto HighAddr = B . buildConstant ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT , <NUM_LIT> ) , AddrHiVal ) ; B . buildMerge ( Dst , { Src , HighAddr } ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS == <ISA_LIT>AS :: FLAT_ADDRESS ) { assert ( DestAS == <ISA_LIT>AS :: LOCAL_ADDRESS || DestAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; unsigned NullVal = TM . getNullPointerValue ( DestAS ) ; auto SegmentNull = B . buildConstant ( DstTy , NullVal ) ; auto FlatNull = B . buildConstant ( SrcTy , <NUM_LIT> ) ; auto PtrLo32 = B . buildExtract ( DstTy , Src , <NUM_LIT> ) ; auto CmpRes = B . buildICmp ( CmpInst :: ICMP_NE , LLT :: scalar ( <NUM_LIT> ) , Src , FlatNull . getReg ( <NUM_LIT> ) ) ; B . buildSelect ( Dst , CmpRes , PtrLo32 , SegmentNull . getReg ( <NUM_LIT> ) ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS != <ISA_LIT>AS :: LOCAL_ADDRESS &&

SrcAS != <ISA_LIT>AS :: PRIVATE_ADDRESS ) return false ;


<s> Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register CmpVal = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register NewVal = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( <STR_LIT> ( MRI . getType ( PtrReg ) . getAddressSpace ( ) ) && <STR_LIT> ) ; LLT ValTy = MRI . getType ( CmpVal ) ; LLT VecTy = LLT :: fixed_vector ( <NUM_LIT> , ValTy ) ; Register PackedVal = B . buildBuildVector ( VecTy , { NewVal , CmpVal } ) . getReg ( <NUM_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( DstReg ) . addUse ( PtrReg ) . addUse ( PackedVal ) . setMemRefs ( MI . memoperands ( ) ) ; MI . eraseFromParent


<s> Register PtrReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register CmpVal = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register NewVal = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( <STR_LIT> ( MRI . getType ( PtrReg ) . getAddressSpace ( ) ) && <STR_LIT> ) ; LLT ValTy = MRI . getType ( CmpVal


<s> Register VOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; Register SOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; unsigned AuxiliaryData = MI . getOperand ( <NUM_LIT> + OpOffset ) . getImm ( ) ; MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; unsigned ImmOffset ; std :: tie ( VOffset , ImmOffset ) = splitBufferOffsets ( B , VOffset ) ; updateBufferMMO ( MMO , VOffset , SOffset , ImmOffset , VIndex , * B . getMRI ( ) ) ; auto MIB = B . buildInstr ( getBufferAtomicPseudo ( IID ) ) ; if ( HasReturn ) MIB . addDef ( Dst ) ; MIB . addUse ( VData ) ; if ( IsCmpSwap ) MIB . addReg ( CmpVal ) ; MIB . addUse ( RSrc ) . addUse ( VIndex ) . addUse ( VOffset ) . addUse ( SOffset ) . addImm ( ImmOffset ) . addImm ( AuxiliaryData

) . addImm ( HasVIndex ? - <NUM_LIT> : <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> Opc = IsD16 ? <STR_LIT> : <STR_LIT> ; } else if ( IsFormat ) { Opc = IsD16 ? <STR_LIT> : <STR_LIT> ; } else { switch ( MemTy . getSizeInBits ( ) ) { case <NUM_LIT> : Opc = <STR_LIT> ; break ; case <NUM_LIT> : Opc = <STR_LIT> ; break ; default : Opc = <STR_LIT> ; break ; } } Register LoadDstReg ; bool IsExtLoad = ( ! IsD16 && MemTy . getSizeInBits ( ) < <NUM_LIT> ) || ( IsD16 && ! Ty . isVector ( ) ) ; LLT UnpackedTy = Ty . changeElementSize ( <NUM_LIT> ) ; if ( IsExtLoad ) LoadDstReg = B . getMRI ( ) -> createGenericVirtualRegister ( S32 ) ; else if ( Unpacked && IsD16 && Ty . isVector ( ) ) LoadDstReg = B . getMRI ( ) -> createGenericVirtualRegister ( UnpackedTy ) ; else LoadDstReg = Dst ; auto MIB = B . buildInstr ( Opc ) . addDef ( LoadDstReg ) . addUse ( RSrc ) . addUse ( VIndex ) . addUse ( VOffset ) . addUse ( SOffset ) . addImm ( ImmOffset ) ; if ( IsTyped ) MIB . addImm ( Format ) ; MIB . addImm ( AuxiliaryData ) . addImm ( HasVIndex ? - <NUM_LIT> : <NUM_LIT> ) . addMemOperand ( MMO ) ; if ( LoadDstReg != Dst ) { B . setInsertPt ( B . getMBB ( ) , ++ B . getInsertPt ( ) ) ; if ( IsExtLoad ) B . buildTrunc ( Dst , LoadDstReg ) ; else { auto Unmerge = B . buildUnmerge ( S32 , LoadDstReg ) ; SmallVector < Register , <NUM_LIT> >


<s> } else { VIndex = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; } Register VOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; Register SOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; unsigned Format = <NUM_LIT> ; if ( IsTyped ) { Format = MI . getOperand ( <NUM_LIT> + OpOffset ) . getImm ( ) ; ++ OpOffset ; } unsigned AuxiliaryData = MI . getOperand ( <NUM_LIT> + OpOffset ) . getImm ( ) ; std :: tie ( VOffset , ImmOffset ) = splitBufferOffsets ( B , VOffset ) ; updateBufferMMO ( MMO , VOffset , SOffset , ImmOffset , VIndex , MRI ) ; unsigned Opc ; if ( IsTyped ) { Opc = IsD16 ? <STR_LIT> : <STR_LIT> ; } else if ( IsFormat ) { Opc = IsD16 ? <STR_LIT> : <STR_LIT> ; } else { switch ( MemSize ) { case <NUM_LIT> : Opc = <STR_LIT> ; break ; case <NUM_LIT> : Opc = <STR_LIT> ; break ; default : Opc =


<s> Register Src0 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src1 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( MRI . getType ( Src0 ) == LLT ::


<s> Opcode = <STR_LIT> ( BaseOpcodes [ Is64 ] [ IsA16 ] , <STR_LIT> , NumVDataDwords , PowerOf2Ceil ( NumVAddrDwords ) ) ; } assert ( Opcode != - <NUM_LIT> ) ; SmallVector < Register , <NUM_LIT> > Ops ; if ( Is64 ) { auto Unmerge = B . buildUnmerge ( { S32 , S32 } , NodePtr ) ; Ops . push_back ( Unmerge . getReg ( <NUM_LIT> ) ) ; Ops . push_back ( Unmerge . getReg ( <NUM_LIT> ) ) ; } else { Ops . push_back ( NodePtr ) ; } Ops . push_back ( RayExtent ) ; auto packLanes = [ & Ops , & S32 , & B ] ( Register Src ) { auto Unmerge = B . buildUnmerge ( { S32 , S32 , S32 , S32 } , Src ) ; Ops . push_back ( Unmerge . getReg ( <NUM_LIT> ) ) ; Ops . push_back ( Unmerge . getReg ( <NUM_LIT> ) ) ; Ops . push_back ( Unmerge . getReg ( <NUM_LIT> ) ) ; } ; packLanes ( RayOrigin ) ; if ( IsA16 ) { auto UnmergeRayDir = B . buildUnmerge ( { S16 , S16 , S16 , S16 } , RayDir ) ; auto UnmergeRayInvDir = B . buildUnmerge ( { S16 , S16 , S16 , S16 } , RayInvDir ) ; Register R1 = MRI . createGenericVirtualRegister ( S32 ) ; Register R2 = MRI . createGenericVirtualRegister ( S32 ) ; Register R3 = MRI . createGenericVirtualRegister ( S32 ) ; B . buildMerge ( R1 , { UnmergeRayDir . getReg ( <NUM_LIT> ) , UnmergeRayDir . getReg ( <NUM_LIT> ) } ) ; B . buildMerge ( R2 , { UnmergeRayDir . getReg ( <NUM_LIT> ) , UnmergeRayInvDir . getReg ( <NUM_LIT> ) } ) ; B . buildMerge ( R3 , { UnmergeRayInvDir . getReg ( <NUM_LIT> ) , UnmergeRayInvDir . getReg ( <NUM_LIT> ) } ) ; Ops . push_back ( R1 ) ; Ops . push_back (


<s> LLT SrcTy = MRI . getType ( Src ) ; unsigned NewOpc = MI . getOpcode ( ) == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; auto Tmp = B . buildInstr ( NewOpc , { DstTy } , { Src } ) ; B . buildUMin ( Dst , Tmp , B . buildConstant ( DstTy ,


<s> MachineIRBuilder & B = Helper . MIRBuilder ; MachineRegisterInfo & MRI = * B . getMRI ( ) ; switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_ADDRSPACE_CAST : return legalizeAddrSpaceCast ( MI , MRI , B ) ; case TargetOpcode :: G_FRINT : return legalizeFrint ( MI , MRI , B ) ; case TargetOpcode :: G_FCEIL : return legalizeFceil ( MI , MRI , B ) ; case TargetOpcode :: G_FREM : return legalizeFrem ( MI , MRI , B ) ; case TargetOpcode :: G_INTRIN<ISA_LIT>C_TRUNC : return legalizeIntrinsicTrunc ( MI , MRI , B ) ; case TargetOpcode :: G_<ISA_LIT>TOFP : return legalizeITOFP ( MI , MRI , B , true ) ; case TargetOpcode :: G_UITOFP : return legalizeITOFP ( MI , MRI , B , false ) ; case TargetOpcode :: G_FPTO<ISA_LIT> : return legalizeFPTOI ( MI , MRI , B , true ) ; case TargetOpcode :: G_FPTOUI : return legalizeFPTOI ( MI , MRI , B , false ) ; case TargetOpcode :: G_FMINNUM : case TargetOpcode :: G_FMAXNUM : case TargetOpcode :: G_FMINNUM_IEEE : case TargetOpcode :: G_FMAXNUM_IEEE : return legalizeMinNumMaxNum ( Helper , MI ) ; case TargetOpcode :: G_EXTRACT_VECTOR_ELT : return legalizeExtractVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_INSERT_VECTOR_ELT : return legalizeInsertVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return legalizeShuffleVector ( MI , MRI , B ) ; case TargetOpcode :: G_F<ISA_LIT>N : case TargetOpcode :: G_FCOS : return legalizeSinCos ( MI , MRI , B ) ; case TargetOpcode :: G_GLOBAL_VALUE : return legalizeGlobalValue ( MI , MRI , B ) ; case TargetOpcode :: G_LOAD : case TargetOpcode :: G_SEXTLOAD : case TargetOpcode :: G_ZEXTLOAD : return legalizeLoad ( Helper , MI ) ; case TargetOpcode ::


<s> case TargetOpcode :: G_FMINNUM_IEEE : case TargetOpcode :: G_FMAXNUM_IEEE : return legalizeMinNumMaxNum ( Helper , MI ) ; case TargetOpcode :: G_EXTRACT_VECTOR_ELT : return legalizeExtractVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_INSERT_VECTOR_ELT : return legalizeInsertVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return legalizeShuffleVector ( MI , MRI , B ) ; case TargetOpcode :: G_F<ISA_LIT>N : case TargetOpcode :: G_FCOS : return legalizeSinCos ( MI , MRI , B ) ; case TargetOpcode :: G_GLOBAL_VALUE : return legalizeGlobalValue ( MI , MRI , B ) ; case TargetOpcode :: G_LOAD : case TargetOpcode :: G_SEXTLOAD : case TargetOpcode :: G_ZEXTLOAD : return legalizeLoad ( Helper , MI ) ; case TargetOpcode :: G_FMAD : return legalizeFMad ( MI , MRI , B ) ; case TargetOpcode :: G_FDIV : return legalizeFDIV ( MI , MRI , B ) ; case TargetOpcode :: G_UDIV : case TargetOpcode :: G_UREM : case TargetOpcode :: G_UDIVREM : return legalizeUnsignedDIV_REM ( MI , MRI , B ) ; case TargetOpcode :: G_SDIV : case TargetOpcode :: G_SREM : case TargetOpcode :: G_SDIVREM : return legalizeSignedDIV_REM ( MI , MRI , B ) ; case TargetOpcode :: G_ATOMIC_CMPXCHG : return legalizeAtomicCmpXChg ( MI , MRI , B ) ; case TargetOpcode :: G_FLOG : return legalizeFlog ( MI , B , numbers :: ln2f ) ; case TargetOpcode :: G_FLOG10 : return legalizeFlog ( MI , B , numbers :: ln2f / numbers :: ln10f ) ; case TargetOpcode :: G_FEXP : return legalizeFExp ( MI , B ) ; case TargetOpcode :: G_FPOW : return legalizeFPow ( MI , B ) ; case TargetOpcode :: G_FFLOOR : return legalizeFFloor ( MI , MRI , B ) ; case TargetOpcode :: G_BUILD_VECTOR : return legalizeBuildVector ( MI , MRI


<s> DiagnosticInfoUnsupported NoTrap ( B . getMF ( ) . getFunction ( ) , <STR_LIT> , MI . getDebugLoc ( ) , DS_Warning ) ; LLVMContext & Ctx = B . getMF ( ) . getFunction ( ) . getContext ( ) ; Ctx . diagnose ( NoTrap ) ; } else { B . buildInstr ( <STR_LIT> ) . addImm ( static_cast < unsigned > ( GCNSubtarget :: TrapID :: LLVMAMDHSADebugTrap ) ) ; } MI . eraseFromParent ( ) ; return true


<s> if ( ! ST . isTrapHandlerEnabled ( ) || ST . getTrapHandlerAbi ( ) != GCNSubtarget :: TrapHandlerAbi :: AMDHSA ) { DiagnosticInfoUnsupported NoTrap ( B . getMF ( ) . getFunction ( ) , <STR_LIT> , MI . getDebugLoc ( ) , DS_Warning ) ; LLVMContext & Ctx = B . getMF ( ) . getFunction ( ) . getContext ( ) ; Ctx . diagnose ( NoTrap ) ; } else { B . buildInstr ( <STR_LIT> ) . addImm ( static_cast < unsigned > ( GCNSubtarget :: TrapID :: LLVMAMDHSADebugTrap ) ) ; } MI . eraseFromParent


<s> for ( int I = <NUM_LIT> ; I > <NUM_LIT> ; -- I ) MI . RemoveOperand ( I ) ; MI . RemoveOperand ( <NUM_LIT> ) ; Observer . changedInstr ( MI


<s> LLT VecTy = MRI . getType ( Vec ) ; LLT EltTy = VecTy . getElementType ( ) ; assert ( EltTy == MRI . getType ( Dst ) ) ; if ( IdxVal < VecTy . getNumElements ( ) ) B . buildExtract ( Dst , Vec , IdxVal * EltTy . getSizeInBits ( ) ) ; else B . buildUndef ( Dst ) ; MI . eraseFromParent ( ) ; return true


<s> MI . eraseFromParent ( ) ; return true ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { auto FNeg = B . buildFNeg ( ResTy , RHS , Flags ) ; B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( FNeg . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ; } } auto RCP = B . buildIntrinsic ( <STR_LIT> , { ResTy } , false ) . addUse ( RHS ) . setMIFlags ( Flags ) ; B . buildFMul ( Res , LHS , RCP , Flags ) ; MI . eraseFromParent ( ) ; return true


<s> Register Res = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register X = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Y = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; uint16_t Flags = MI . getFlags ( ) ; LLT ResTy = MRI . getType ( Res ) ; const MachineFunction & MF = B . getMF ( ) ; bool AllowInaccurateRcp = MF . getTarget ( ) . Options . UnsafeFPMath || MI . getFlag ( MachineInstr :: FmAfn ) ; if ( ! AllowInaccurateRcp ) return false ; auto NegY = B . buildFNeg ( ResTy , Y ) ; auto One = B . buildFConstant ( ResTy , <NUM_LIT> ) ; auto R = B . buildIntrinsic ( <STR_LIT> , { ResTy } , false ) .


<s> auto Tmp0 = B . buildFMA ( ResTy , NegY , R , One ) ; R = B . buildFMA ( ResTy , Tmp0 , R , R ) ; auto Tmp1 = B . buildFMA ( ResTy , NegY , R , One ) ; R = B . buildFMA ( ResTy , Tmp1 , R , R ) ; auto Ret = B . buildFMul ( ResTy , X , R ) ; auto Tmp2 = B . buildFMA ( ResTy , NegY , Ret , X ) ; B . buildFMA ( Res , Tmp2


<s> const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; Register Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( MRI . getType ( Src ) == S64 ) ; auto Trunc = B . buildIntrinsicTrunc ( S64 , Src ) ; const auto Zero = B . buildFConstant ( S64 , <NUM_LIT> ) ; const auto One = B . buildFConstant ( S64 , <NUM_LIT> ) ; auto Lt0 = B . buildFCmp ( CmpInst :: FCMP_OGT , S1 , Src , Zero ) ; auto NeTrunc = B . buildFCmp ( CmpInst :: FCMP_ONE , S1 , Src , Trunc ) ; auto And = B . buildAnd ( S1 , Lt0 , NeTrunc ) ; auto Add = B . buildSelect ( S64 , And , One , Zero


<s> const auto Zero = B . buildFConstant ( S64 , <NUM_LIT> ) ; const auto One = B . buildFConstant ( S64 , <NUM_LIT> ) ; auto Lt0 = B . buildFCmp ( CmpInst :: FCMP_OGT , S1 , Src , Zero ) ; auto NeTrunc = B . buildFCmp ( CmpInst :: FCMP_ONE , S1 , Src , Trunc ) ; auto And = B . buildAnd ( S1 , Lt0


<s> LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; if ( DstTy == S16 ) return legalizeFDIV16 ( MI , MRI , B ) ; if ( DstTy == S32 ) return legalizeFDIV32 ( MI , MRI , B ) ; if ( DstTy == S64 ) return legalizeFDIV64 ( MI , MRI , B


<s> if ( legalizeFastUnsafeFDIV ( MI , MRI , B ) ) return true ; Register Res = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register LHS = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register RHS = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; uint16_t Flags = MI . getFlags ( ) ; LLT S16 = LLT :: scalar ( <NUM_LIT> ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto LHSExt = B . buildFPExt ( S32 , LHS , Flags


<s> auto LHSExt = B . buildFPExt ( S32 , LHS , Flags ) ; auto RHSExt = B . buildFPExt ( S32 , RHS , Flags ) ; auto RCP = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( RHSExt . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto QUOT = B . buildFMul ( S32 , LHSExt , RCP , Flags ) ; auto RDst = B . buildFPTrunc ( S16 , QUOT , Flags ) ; B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( RDst . getReg ( <NUM_LIT> ) ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; MI . eraseFromParent (


<s> auto Fma0 = B . buildFMA ( S32 , NegDivScale0 , ApproxRcp , One , Flags ) ; auto Fma1 = B . buildFMA ( S32 , Fma0 , ApproxRcp , ApproxRcp , Flags ) ; auto Mul = B . buildFMul ( S32 , NumeratorScaled , Fma1 , Flags ) ; auto Fma2 = B . buildFMA ( S32 , NegDivScale0 , Mul , NumeratorScaled , Flags ) ; auto Fma3 = B . buildFMA ( S32 , Fma2 , Fma1 , Mul , Flags ) ; auto Fma4 = B . buildFMA ( S32 , NegDivScale0 , Fma3 , NumeratorScaled , Flags ) ; if ( ! Mode . allFP32Denormals ( ) ) toggleSPDenormMode ( false , B , ST , Mode ) ; auto Fmas = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( Fma4 . getReg ( <NUM_LIT> ) ) . addUse ( Fma1 . getReg ( <NUM_LIT> ) ) . addUse ( Fma3 . getReg ( <NUM_LIT> ) ) . addUse ( NumeratorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( Fmas . getReg ( <NUM_LIT> ) ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; MI . eraseFromParent (


<s> auto Fma2 = B . buildFMA ( S64 , NegDivScale0 , Fma1 , One , Flags ) ; auto DivScale1 = B . buildIntrinsic ( <STR_LIT> , { S64 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto Fma3 = B . buildFMA ( S64 , Fma1 , Fma2 , Fma1 , Flags ) ; auto Mul = B . buildFMul ( S64 , DivScale1 . getReg ( <NUM_LIT> ) , Fma3 , Flags ) ; auto Fma4 = B . buildFMA ( S64 , NegDivScale0 , Mul , DivScale1 . getReg ( <NUM_LIT> ) , Flags ) ; Register Scale ; if ( ! ST . hasUsableDivScaleConditionOutput ( ) ) { LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto NumUnmerge = B . buildUnmerge ( S32 , LHS ) ; auto DenUnmerge = B . buildUnmerge ( S32 , RHS ) ; auto Scale0Unmerge = B . buildUnmerge ( S32 , DivScale0 ) ; auto Scale1Unmerge = B . buildUnmerge ( S32 , DivScale1 ) ; auto CmpNum = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , NumUnmerge . getReg ( <NUM_LIT> ) , Scale1Unmerge . getReg ( <NUM_LIT> ) ) ; auto CmpDen = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , DenUnmerge . getReg ( <NUM_LIT> ) , Scale0Unmerge . getReg ( <NUM_LIT> ) ) ; Scale = B . buildXor ( S1 , CmpNum , CmpDen ) . getReg ( <NUM_LIT> ) ; } else { Scale = DivScale1 . getReg


<s> auto One = B . buildFConstant ( S64 , <NUM_LIT> ) ; auto DivScale0 = B . buildIntrinsic ( <STR_LIT> , { S64 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto NegDivScale0 = B . buildFNeg ( S64 , DivScale0 . getReg ( <NUM_LIT> ) , Flags ) ; auto Rcp = B . buildIntrinsic ( <STR_LIT> , { S64 } , false ) . addUse ( DivScale0 . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto Fma0 = B . buildFMA ( S64 , NegDivScale0 , Rcp , One , Flags ) ; auto Fma1 = B . buildFMA ( S64 , Rcp , Fma0 , Rcp , Flags ) ; auto Fma2 = B . buildFMA ( S64 , NegDivScale0 , Fma1 , One , Flags ) ; auto DivScale1 = B . buildIntrinsic ( <STR_LIT> , { S64 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addImm ( <NUM_LIT> ) .


<s> Register RHS = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; uint16_t Flags = MI . getFlags ( ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; LLT S1 = LLT :: scalar ( <NUM_LIT> ) ; auto Abs = B . buildFAbs ( S32 , RHS , Flags ) ; const APFloat C0Val ( <NUM_LIT> ) ; auto C0 = B . buildConstant ( S32 , <NUM_LIT> ) ; auto C1 = B . buildConstant ( S32 , <NUM_LIT> ) ; auto C2 = B . buildConstant ( S32 , FloatToBits ( <NUM_LIT> ) ) ; auto CmpRes = B . buildFCmp ( CmpInst :: FCMP_OGT , S1 , Abs ,


<s> auto CmpRes = B . buildFCmp ( CmpInst :: FCMP_OGT , S1 , Abs , C0 , Flags ) ; auto Sel = B . buildSelect ( S32 , CmpRes , C1 , C2 , Flags ) ; auto Mul0 = B . buildFMul ( S32 , RHS , Sel , Flags ) ; auto RCP = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( Mul0 . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto Mul1 = B . buildFMul ( S32


<s> Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Flags = MI . getFlags ( ) ; LLT Ty = B . getMRI ( ) -> getType ( Dst ) ; auto K = B . buildFConstant ( Ty , numbers :: log2e ) ; auto Mul = B . buildFMul ( Ty , Src , K , Flags ) ; B . buildFExp2 ( Dst ,


<s> Register ModSrc = stripAnySourceMods ( OrigSrc , MRI ) ; auto Const = B . buildFConstant ( S64 , BitsToDouble ( <NUM_LIT> ) ) ; Register Min = MRI . createGenericVirtualRegister ( S64 ) ; const <ISA_LIT>MachineFunctionInfo * MFI = B . getMF ( ) . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( MFI -> getMode ( ) . IEEE ) B . buildFMinNumIEEE ( Min , Fract , Const , Flags ) ; else B . buildFMinNum ( Min , Fract , Const , Flags ) ; Register CorrectedFract = Min ; if ( ! MI . getFlag ( MachineInstr :: FmNoNans ) ) { auto IsNan = B . buildFCmp ( CmpInst :: FCMP_ORD , S1 , ModSrc , ModSrc , Flags ) ; CorrectedFract = B . buildSelect ( S64 , IsNan , ModSrc , Min , Flags ) . getReg ( <NUM_LIT> ) ; } auto NegFract = B . buildFNeg ( S64 , CorrectedFract , Flags ) ; B . buildFAdd ( Dst , OrigSrc , NegFract , Flags


<s> Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT Ty = B . getMRI ( ) -> getType ( Dst ) ; unsigned Flags = MI . getFlags ( ) ; auto Log2Operand = B . buildFLog2 ( Ty


<s> bool <ISA_LIT>LegalizerInfo :: legalizeFMad ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { LLT Ty = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; assert ( Ty . isScalar ( ) ) ; MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . allFP32Denormals ( ) ) return true ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . allFP64FP16Denormals ( ) ) return true ; MachineIRBuilder HelperBuilder ( MI ) ; GISelObserverWrapper


<s> LLT Ty = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; assert ( Ty . isScalar ( ) ) ; MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . allFP32Denormals


<s> Register Src0 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src1 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Flags = MI . getFlags ( ) ; LLT Ty = B . getMRI ( ) -> getType ( Dst ) ; const LLT S16 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; if ( Ty == S32 ) { auto Log = B . buildFLog2 ( S32 , Src0 , Flags ) ; auto Mul = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( Log

. getReg ( <NUM_LIT> ) ) . addUse ( Src1 ) . setMIFlags ( Flags ) ;


<s> const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const LLT SrcLT = MRI . getType ( Src ) ; assert ( ( SrcLT == S32 || SrcLT == S64 ) && MRI . getType ( Dst ) == S64 ) ; unsigned Flags = MI . getFlags ( ) ; auto Trunc = B . buildIntrinsicTrunc ( SrcLT , Src , Flags ) ; MachineInstrBuilder Sign ; if ( Signed && SrcLT == S32 ) { Sign = B . buildAShr ( S32 , Src , B . buildConstant ( S32 , <NUM_LIT> ) ) ; Trunc = B . buildFAbs ( S32 , Trunc , Flags ) ; } MachineInstrBuilder K0 , K1 ; if ( SrcLT == S64 ) { K0 = B . buildFConstant ( S64 , BitsToDouble ( UINT64_C ( <NUM_LIT> ) ) ) ; K1 = B . buildFConstant ( S64 , BitsToDouble ( UINT64_C ( <NUM_LIT> ) ) ) ; } else { K0 = B . buildFConstant ( S32 , BitsToFloat ( UINT32_C ( <NUM_LIT> ) ) ) ; K1 = B . buildFConstant ( S32 , BitsToFloat ( UINT32_C ( <NUM_LIT> ) ) ) ; } auto Mul = B . buildFMul ( SrcLT , Trunc , K0 , Flags ) ; auto FloorMul = B . buildFFloor ( SrcLT , Mul , Flags ) ; auto Fma = B . buildFMA ( SrcLT , FloorMul , K1


<s> bool <ISA_LIT>LegalizerInfo :: legalizeFrem ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src0Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src1Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; auto Flags = MI . getFlags ( ) ; LLT Ty = MRI . getType ( DstReg ) ; auto Div = B . buildFDiv ( Ty , Src0Reg , Src1Reg , Flags ) ; auto Trunc = B . buildIntrinsicTrunc ( Ty , Div , Flags ) ; auto Neg = B . buildFNeg ( Ty , Trunc , Flags ) ; B . buildFMA ( DstReg , Neg , Src1Reg , Src0Reg , Flags ) ; MI . eraseFromParent (


<s> assert ( Ty . isScalar ( ) && Ty . getSizeInBits ( ) == <NUM_LIT> ) ; APFloat C1Val ( APFloat :: IEEEdouble ( ) , <STR_LIT> ) ; APFloat C2Val ( APFloat :: IEEEdouble ( ) , <STR_LIT> ) ; auto C1 = B . buildFConstant ( Ty , C1Val ) ; auto CopySign = B . buildFCopysign ( Ty


<s> MI . eraseFromParent ( ) ; return true ; } const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( TLI -> shouldEmitFixup ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> ) ; MI . eraseFromParent ( ) ; return true ; } if ( TLI -> shouldEmitPCReloc ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_REL32 ) ; MI . eraseFromParent ( ) ; return true ; } LLT PtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register GOTAddr = MRI . createGenericVirtualRegister ( PtrTy ) ; LLT LoadTy = Ty . getSizeInBits ( ) == <NUM_LIT> ? PtrTy : Ty ; MachineMemOperand * GOTMMO = MF . getMachineMemOperand ( MachinePointerInfo :: getGOT ( MF ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , LoadTy , Align ( <NUM_LIT> ) ) ; buildPCRelGlobalAddress ( GOTAddr , PtrTy , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; if ( Ty . getSizeInBits ( ) == <NUM_LIT> ) { auto Load = B . buildLoad ( PtrTy , GOTAddr


<s> const <ISA_LIT>MachineFunctionInfo * MFI = B . getMF ( ) . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! MFI -> isEntryFunction ( ) ) { return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; } Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! getImplicitArgPtr ( DstReg , MRI , B ) ) return false ; MI . eraseFromParent ( ) ; return true


<s> int64_t IdxVal = MaybeIdxVal -> Value . getSExtValue ( ) ; Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Vec = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Ins = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT VecTy = MRI . getType ( Vec ) ; LLT EltTy = VecTy . getElementType ( ) ; assert ( EltTy == MRI . getType ( Ins ) ) ; if ( IdxVal < VecTy . getNumElements ( ) ) B . buildInsert ( Dst , Vec , Ins , IdxVal * EltTy . getSizeInBits ( ) ) ; else B . buildUndef ( Dst