<s> const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadLDSDecl ( Fn , <STR_LIT> , MI . getDebugLoc ( ) , DS_Warning ) ; Fn . getContext ( ) . diagnose ( BadLDSDecl ) ; B . buildIntrinsic ( <STR_LIT> , ArrayRef < Register > ( ) , true ) ; B . buildUndef ( DstReg ) ; MI . eraseFromParent ( ) ; return true ; } if ( ! <ISA_LIT>TargetLowering :: hasDefinedInitializer ( GV ) ) { const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( ! TLI -> shouldUseLDSConstAddress ( GV ) ) { MI . getOperand ( <NUM_LIT> ) . setTargetFlags ( <ISA_LIT>InstrInfo :: MO_ABS32_LO ) ; return true ; } B . buildConstant ( DstReg , MFI -> allocateLDSGlobal ( B . getDataLayout ( ) , * GV ) ) ; MI . eraseFromParent ( ) ; return true ; } const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadInit ( Fn , <STR_LIT> , MI . getDebugLoc ( ) ) ; Fn . getContext ( ) . diagnose ( BadInit ) ; return true ; } const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( TLI -> shouldEmitFixup ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> ) ; MI . eraseFromParent ( ) ; return true ; } if ( TLI -> shouldEmitPCReloc ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_REL32 ) ; MI . eraseFromParent ( ) ; return true ; } LLT PtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register GOTAddr = MRI . createGenericVirtualRegister ( PtrTy ) ; MachineMemOperand * GOTMMO = MF . getMachineMemOperand ( MachinePointerInfo :: getGOT ( MF ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , <NUM_LIT> , Align ( <NUM_LIT> ) ) ; buildPCRelGlobalAddress ( GOTAddr , PtrTy , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; if ( Ty . getSizeInBits ( ) == <NUM_LIT> ) { auto Load = B . buildLoad ( PtrTy , GOTAddr , * GOTMMO ) ; B . buildExtract ( DstReg , Load , <NUM_LIT> ) ;

} else B . buildLoad ( DstReg , GOTAddr , * GOTMMO ) ;


<s> if ( ! MFI -> isEntryFunction ( ) ) { return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; } uint64_t Offset = ST . getTargetLowering ( ) -> getImplicitParameterOffset ( B . getMF ( ) , <ISA_LIT>TargetLowering :: FIRST_IMPLICIT ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( DstReg ) ; LLT IdxTy = LLT :: scalar ( DstTy . getSizeInBits ( ) ) ; const ArgDescriptor * Arg ; const TargetRegisterClass * RC ; std :: tie ( Arg , RC ) = MFI -> getPreloadedValue ( <STR_LIT> ) ; if ( ! Arg ) return false ; Register KernargPtrReg = MRI . createGenericVirtualRegister ( DstTy ) ; if ( ! loadInputValue ( KernargPtrReg , B , Arg ) ) return false ; B . buildPtrAdd ( DstReg , KernargPtrReg , B . buildConstant ( IdxTy , Offset ) . getReg ( <NUM_LIT> ) ) ;


<s> uint64_t Offset = ST . getTargetLowering ( ) -> getImplicitParameterOffset ( B . getMF ( ) , <ISA_LIT>TargetLowering :: FIRST_IMPLICIT ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( DstReg ) ; LLT IdxTy = LLT :: scalar ( DstTy . getSizeInBits ( ) ) ; const ArgDescriptor * Arg ; const TargetRegisterClass * RC ;

std :: tie ( Arg , RC ) = MFI -> getPreloadedValue ( <STR_LIT> ) ;


<s> Register Den = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; legalizeUDIV_UREM32Impl ( B , DstReg , Num , Den , IsRem ) ; MI . eraseFromParent ( ) ; return true ;


<s> auto Remainder_GE_Den = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , Remainder , Den ) ; auto Remainder_GE_Zero = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , Num , Num_S_Remainder ) ; auto Tmp1 = B . buildAnd ( S1 , Remainder_GE_Den , Remainder_GE_Zero ) ; auto One = B . buildConstant ( S32 , <NUM_LIT> ) ; auto Quotient_A_One = B . buildAdd ( S32 , Quotient , One ) ; auto Quotient_S_One = B . buildSub ( S32 , Quotient , One ) ; auto Div = B . buildSelect ( S32 , Tmp1 , Quotient_A_One , Quotient ) ; if ( IsRem ) { Div = B . buildSelect ( S32 , Remainder_GE_Zero , Div , Quotient_S_One ) ; auto Remainder_S_Den = B . buildSub ( S32 , Remainder , Den ) ; auto Remainder_A_Den = B . buildAdd ( S32 , Remainder , Den ) ; auto Rem = B . buildSelect ( S32 , Tmp1 , Remainder_S_Den , Remainder ) ; B . buildSelect ( DstReg , Remainder_GE_Zero , Rem , Remainder_A_Den ) ; } else { B . buildSelect ( DstReg , Remainder_GE_Zero , Div , Quotient_S_One ) ;


<s> AddrRegs [ I ] = MI . getOperand ( DimIdx + I ) . getReg ( ) ; assert ( B . getMRI ( ) -> getType ( AddrRegs [ I ] ) == LLT :: scalar ( <NUM_LIT> ) ) ; }

auto VAddr = B . buildBuildVector ( LLT :: vector ( NumVAddrs , <NUM_LIT> ) , AddrRegs ) ;


return NumDefs + <NUM_LIT> + ( BaseOpcode -> Store ? <NUM_LIT> : <NUM_LIT> ) ;


<s> assert ( ! BaseOpcode -> Atomic ) ; return NumDefs + <NUM_LIT> + ( BaseOpcode -> Store ? <NUM_LIT> : <NUM_LIT> ) ;


<s> const <STR_LIT> * DimInfo = <STR_LIT> ( ImageDimIntr -> Dim ) ; int NumGradients = BaseOpcode -> Gradients ? DimInfo -> NumGradients : <NUM_LIT> ;

int NumCoords = BaseOpcode -> Coordinates ? DimInfo -> NumCoords : <NUM_LIT> ;


<s> int NumGradients = BaseOpcode -> Gradients ? DimInfo -> NumGradients : <NUM_LIT> ; int NumCoords = BaseOpcode -> Coordinates ? DimInfo -> NumCoords : <NUM_LIT> ; int NumLCM = BaseOpcode -> LodOrClampOrMip ? <NUM_LIT> : <NUM_LIT> ; int NumVAddr = BaseOpcode -> NumExtraArgs + NumGradients + NumCoords + NumLCM ;


<s> static int getImageVAddrIdxBegin ( const <STR_LIT> * BaseOpcode , int NumDefs ) { if ( BaseOpcode -> Atomic ) return NumDefs + <NUM_LIT> + ( BaseOpcode -> AtomicX2 ? <NUM_LIT> : <NUM_LIT> ) ;

return getDMaskIdx ( BaseOpcode , NumDefs ) + <NUM_LIT> ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; assert ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; if ( ST . hasApertureRegs ( ) ) { unsigned Offset = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: OFFSET_SRC_SHARED_BASE : <STR_LIT> :: OFFSET_SRC_PRIVATE_BASE ; unsigned WidthM1 = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: WIDTH_M1_SRC_SHARED_BASE : <STR_LIT> :: WIDTH_M1_SRC_PRIVATE_BASE ; unsigned Encoding = <STR_LIT> :: ID_MEM_BASES << <STR_LIT> :: ID_SHIFT_ | Offset << <STR_LIT> :: OFFSET_SHIFT_ | WidthM1 << <STR_LIT> :: WIDTH_M1_SHIFT_ ; Register GetReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( GetReg ) . addImm ( Encoding ) ; MRI . setType ( GetReg , S32 ) ; auto ShiftAmt = B . buildConstant ( S32 , WidthM1 + <NUM_LIT> ) ; return B . buildShl ( S32 , GetReg , ShiftAmt ) . getReg ( <NUM_LIT> ) ; } Register QueuePtr = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! loadInputValue ( QueuePtr , B , & MFI -> getArgInfo ( ) . QueuePtr ) ) return Register ( ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , <NUM_LIT> , MinAlign ( <NUM_LIT> , StructOffset ) ) ; Register LoadAddr ; B . materializePtrAdd ( LoadAddr , QueuePtr , LLT :: scalar ( <NUM_LIT> ) , StructOffset ) ; return B . buildLoad ( S32 , LoadAddr , * MMO ) . getReg ( <NUM_LIT> ) ;


<s> assert ( MRI . getType ( Src0 ) == LLT :: scalar ( <NUM_LIT> ) ) ; B . setInstr ( MI ) ; auto Merge = B . buildMerge ( S32 , { Src0 , Src1 } ) ; B . buildBitcast ( Dst , Merge ) ; MI . eraseFromParent ( ) ; return true ;


<s> assert ( DstTy == V2S16 ) ; Register Src0 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src1 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( MRI . getType ( Src0 ) == LLT :: scalar ( <NUM_LIT> ) ) ;


<s> const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . allFP32Denormals ( ) ) return true ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . allFP64FP16Denormals ( ) ) return true ; MachineIRBuilder HelperBuilder ( MI ) ; GISelObserverWrapper DummyObserver ; LegalizerHelper Helper ( MF , DummyObserver , HelperBuilder ) ; HelperBuilder . setMBB ( * MI . getParent ( ) ) ; return Helper . lowerFMad ( MI ) == LegalizerHelper :: Legalized ;


<s> assert ( Ty . isScalar ( ) ) ; MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . allFP32Denormals ( ) ) return true ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . allFP64FP16Denormals ( ) ) return true ; MachineIRBuilder HelperBuilder ( MI ) ; GISelObserverWrapper DummyObserver ; LegalizerHelper Helper ( MF , DummyObserver , HelperBuilder ) ; HelperBuilder . setMBB ( * MI . getParent ( ) ) ;

return Helper . lowerFMad ( MI ) == LegalizerHelper :: Legalized ;


<s> return true ; } const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( TLI -> shouldEmitFixup ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> ) ; MI . eraseFromParent ( ) ; return true ; } if ( TLI -> shouldEmitPCReloc ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_REL32 ) ; MI . eraseFromParent ( ) ; return true ; } LLT PtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register GOTAddr = MRI . createGenericVirtualRegister ( PtrTy ) ; MachineMemOperand * GOTMMO = MF . getMachineMemOperand ( MachinePointerInfo :: getGOT ( MF ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , <NUM_LIT> , <NUM_LIT> ) ; buildPCRelGlobalAddress ( GOTAddr , PtrTy , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; if ( Ty . getSizeInBits ( ) == <NUM_LIT> ) { auto Load = B . buildLoad ( PtrTy , GOTAddr , * GOTMMO ) ; B . buildExtract ( DstReg , Load , <NUM_LIT> ) ;


<s> MI . setDesc ( B . getTII ( ) . get ( <STR_LIT> ) ) ; MI . RemoveOperand ( <NUM_LIT> ) ; const unsigned MemSize = ( Size + <NUM_LIT> ) / <NUM_LIT> ; const unsigned MemAlign = <NUM_LIT> ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , MemSize , MemAlign ) ; MI . addMemOperand ( MF , MMO ) ; if ( ! isPowerOf2_32 ( Size ) ) {

LegalizerHelper Helper ( MF , * this , Observer , B ) ;


<s> } else { if ( ( ( I + <NUM_LIT> ) >= ( AddrIdx + NumVAddrs ) ) || ( ( NumGradients / <NUM_LIT> ) % <NUM_LIT> == <NUM_LIT> && ( I == DimIdx + ( NumGradients / <NUM_LIT> ) - <NUM_LIT> || I == DimIdx + NumGradients - <NUM_LIT> ) ) ) { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) } ) . getReg ( <NUM_LIT> ) ) ; } else {

PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , MI . getOperand ( I + <NUM_LIT> ) . getReg ( ) } ) . getReg ( <NUM_LIT> ) ) ;


<s> static void packImageA16AddressToDwords ( MachineIRBuilder & B , MachineInstr & MI , SmallVectorImpl < Register > & PackedAddrs , int AddrIdx , int DimIdx , int NumVAddrs , int NumGradients ) { const LLT S16 = LLT :: scalar ( <NUM_LIT> ) ; const LLT V2S16 = LLT :: vector ( <NUM_LIT> , <NUM_LIT> ) ; for ( int I = AddrIdx ; I < AddrIdx + NumVAddrs ; ++ I ) { Register AddrReg = MI . getOperand ( I ) . getReg ( ) ; if ( I < DimIdx ) { AddrReg = B . buildBitcast ( V2S16 , AddrReg ) . getReg ( <NUM_LIT> ) ; PackedAddrs . push_back ( AddrReg ) ; } else { if ( ( ( I + <NUM_LIT> ) >= ( AddrIdx + NumVAddrs ) ) || ( ( NumGradients / <NUM_LIT> ) % <NUM_LIT> == <NUM_LIT> && ( I == DimIdx + ( NumGradients / <NUM_LIT> ) - <NUM_LIT> || I == DimIdx + NumGradients - <NUM_LIT> ) ) ) { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) } ) . getReg ( <NUM_LIT> ) ) ;


<s> static void bitcastToS16Vector ( MachineIRBuilder & B , Register DstReg , ArrayRef < Register > UnmergeParts ) { MachineRegisterInfo & MRI = * B . getMRI ( ) ; const LLT V2S16 = LLT :: vector ( <NUM_LIT> , <NUM_LIT> ) ; LLT TargetTy = MRI . getType ( DstReg ) ; int NumElts = UnmergeParts . size ( ) ; if ( NumElts == <NUM_LIT> ) { assert ( TargetTy == V2S16 ) ; B . buildBitcast ( DstReg , UnmergeParts [ <NUM_LIT> ] ) ; return ; } SmallVector < Register , <NUM_LIT> > RemergeParts ( NumElts ) ; for ( int I = <NUM_LIT> ; I != NumElts ; ++ I ) RemergeParts [ I ] = B . buildBitcast ( V2S16 , UnmergeParts [ I ] ) . getReg ( <NUM_LIT> ) ; if ( TargetTy . getSizeInBits ( ) == <NUM_LIT> * NumElts ) { B . buildConcatVectors ( DstReg , RemergeParts ) ; return ; } const LLT V3S16 = LLT :: vector ( <NUM_LIT> , <NUM_LIT> ) ;

const LLT V6S16 = LLT :: vector ( <NUM_LIT> , <NUM_LIT> ) ;


<s> MachineFunction & MF = B . getMF ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; B . setInstr ( MI ) ; if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) { if ( ! MFI -> isEntryFunction ( ) ) { const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadLDSDecl ( Fn , <STR_LIT> , MI . getDebugLoc ( ) ) ; Fn . getContext ( ) . diagnose ( BadLDSDecl ) ; } if ( ! <ISA_LIT>TargetLowering :: hasDefinedInitializer ( GV ) ) { const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( ! TLI -> shouldUseLDSConstAddress ( GV ) ) { MI . getOperand ( <NUM_LIT> ) . setTargetFlags ( <ISA_LIT>InstrInfo :: MO_ABS32_LO ) ; return true ; } B . buildConstant ( DstReg , MFI -> allocateLDSGlobal ( B . getDataLayout ( ) , * GV ) ) ; MI . eraseFromParent ( ) ; return true ; } const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadInit ( Fn , <STR_LIT> , MI . getDebugLoc ( ) ) ;

Fn . getContext ( ) . diagnose ( BadInit ) ;


<s> MRI . setRegClass ( Def , TRI -> getWaveMaskRegClass ( ) ) ; MRI . setRegClass ( Use , TRI -> getWaveMaskRegClass ( ) ) ; MI . eraseFromParent ( ) ; BrCond -> eraseFromParent ( ) ; return true ; } return false ; } case <STR_LIT> : { MachineInstr * Br = nullptr ; if ( MachineInstr * BrCond = verifyCFIntrinsic ( MI , MRI , Br ) ) { const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; B . setInstr ( * BrCond ) ; MachineBasicBlock * BrTarget = BrCond -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( Br ) BrTarget = Br -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; Register Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; B . buildInstr ( <STR_LIT> ) . addUse ( Reg ) . addMBB ( BrTarget ) ; if ( Br ) Br -> getOperand ( <NUM_LIT> ) . setMBB ( BrCond -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MI . eraseFromParent ( ) ; BrCond -> eraseFromParent ( ) ; MRI . setRegClass ( Reg , TRI -> getWaveMaskRegClass ( ) ) ; return true ; } return false ; } case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeImplicitArgPtr ( MI , MRI , B ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeFDIVFastIntrin ( MI , MRI , B ) ; case <STR_LIT> : return legalizeIsAddrSpace ( MI , MRI , B , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; case <STR_LIT> : return legalizeIsAddrSpace ( MI , MRI , B , <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; case <STR_LIT> : { B . setInstr ( MI ) ; B . buildConstant ( MI . getOperand ( <NUM_LIT> ) , ST . getWavefrontSize ( ) ) ; MI . eraseFromParent ( ) ; return true ; } case <STR_LIT> : return legalizeSBufferLoad ( MI , B , Observer ) ; case <STR_LIT> : case <STR_LIT> :

return legalizeBufferStore ( MI , MRI , B , false , false ) ;


<s> auto Unmerge = B . buildUnmerge ( S32 , WideDstReg ) ; int NumOps = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; SmallVector < Register , <NUM_LIT> > RemergeParts ( NumOps ) ; for ( int I = <NUM_LIT> ; I != NumOps ; ++ I ) RemergeParts [ I ] = B . buildTrunc ( S16 , Unmerge . getReg ( I ) ) . getReg ( <NUM_LIT> ) ;

B . buildBuildVector ( DstReg , RemergeParts ) ;


<s> static void truncToS16Vector ( MachineIRBuilder & B , Register DstReg , ArrayRef < Register > UnmergeParts ) { const LLT S16 = LLT :: scalar ( <NUM_LIT> ) ; SmallVector < Register , <NUM_LIT> > RemergeParts ( UnmergeParts . size ( ) ) ;

for ( int I = <NUM_LIT> , E = UnmergeParts . size ( ) ; I != E ; ++ I ) RemergeParts [ I ] = B . buildTrunc ( S16 , UnmergeParts [ I ] ) . getReg ( <NUM_LIT> ) ;


<s> SmallVector < Register , <NUM_LIT> > RemergeParts ( UnmergeParts . size ( ) ) ; for ( int I = <NUM_LIT> , E = UnmergeParts . size ( ) ; I != E ; ++ I ) RemergeParts [ I ] = B . buildTrunc ( S16 , UnmergeParts [ I ] ) . getReg ( <NUM_LIT> ) ; B . buildBuildVector ( DstReg , RemergeParts ) ;


<s> return true ; } if ( ! <ISA_LIT>TargetLowering :: hasDefinedInitializer ( GV ) ) { const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( ! TLI -> shouldUseLDSConstAddress ( GV ) ) { MI . getOperand ( <NUM_LIT> ) . setTargetFlags ( <ISA_LIT>InstrInfo :: MO_ABS32_LO ) ; return true ; } if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS && GV -> hasExternalLinkage ( ) ) { Type * Ty = GV -> getValueType ( ) ; if ( B . getDataLayout ( ) . getTypeAllocSize ( Ty ) . isZero ( ) ) { MFI -> setDynLDSAlign ( B . getDataLayout ( ) , * cast < GlobalVariable > ( GV ) ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto Sz = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) ; B . buildIntToPtr ( DstReg , Sz ) ; MI . eraseFromParent ( ) ; return true ; } } B . buildConstant ( DstReg , MFI -> allocateLDSGlobal ( B . getDataLayout ( ) , * cast < GlobalVariable > ( GV ) ) ) ; MI . eraseFromParent ( ) ; return true ; } const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadInit ( Fn , <STR_LIT> , MI . getDebugLoc ( ) ) ; Fn . getContext ( ) . diagnose ( BadInit ) ; return true ; }


<s> MachineIRBuilder & B = Helper . MIRBuilder ; MachineRegisterInfo & MRI = * B . getMRI ( ) ; auto IntrID = MI . getIntrinsicID ( ) ; switch ( IntrID ) { case <STR_LIT> : case <STR_LIT> : { MachineInstr * Br = nullptr ; MachineBasicBlock * UncondBrTarget = nullptr ; bool Negated = false ; if ( MachineInstr * BrCond = verifyCFIntrinsic ( MI , MRI , Br , UncondBrTarget , Negated ) ) { const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; Register Def = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Use = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock * CondBrTarget = BrCond -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( Negated ) std :: swap ( CondBrTarget , UncondBrTarget ) ; B . setInsertPt ( B . getMBB ( ) , BrCond -> getIterator ( ) ) ; if ( IntrID == <STR_LIT> ) { B . buildInstr ( <STR_LIT> ) . addDef ( Def ) . addUse ( Use ) . addMBB ( UncondBrTarget ) ; } else { B . buildInstr ( <STR_LIT> ) . addDef ( Def ) . addUse ( Use ) . addMBB ( UncondBrTarget ) . addImm ( <NUM_LIT> ) ; } if ( Br ) { Br -> getOperand ( <NUM_LIT> ) . setMBB ( CondBrTarget ) ; } else { B . buildBr ( * CondBrTarget ) ; } MRI . setRegClass ( Def , TRI -> getWaveMaskRegClass ( ) ) ; MRI . setRegClass ( Use , TRI -> getWaveMaskRegClass ( ) ) ; MI . eraseFromParent ( ) ; BrCond -> eraseFromParent ( ) ; return true ; } return false ; } case <STR_LIT> : { MachineInstr * Br = nullptr ; MachineBasicBlock * UncondBrTarget = nullptr ; bool Negated = false ; if ( MachineInstr * BrCond = verifyCFIntrinsic ( MI , MRI , Br , UncondBrTarget , Negated ) ) { const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; MachineBasicBlock * CondBrTarget = BrCond -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; Register Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( Negated ) std :: swap ( CondBrTarget , UncondBrTarget ) ; B . setInsertPt ( B . getMBB ( ) , BrCond -> getIterator ( ) ) ; B . buildInstr ( <STR_LIT> ) . addUse ( Reg ) . addMBB ( UncondBrTarget ) ; if ( Br ) Br -> getOperand ( <NUM_LIT> ) . setMBB ( CondBrTarget ) ; else B . buildBr ( * CondBrTarget ) ;


<s> LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto NumUnmerge = B . buildUnmerge ( S32 , LHS ) ; auto DenUnmerge = B . buildUnmerge ( S32 , RHS ) ; auto Scale0Unmerge = B . buildUnmerge ( S32 , DivScale0 ) ; auto Scale1Unmerge = B . buildUnmerge ( S32 , DivScale1 ) ; auto CmpNum = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , NumUnmerge . getReg ( <NUM_LIT> ) , Scale1Unmerge . getReg ( <NUM_LIT> ) ) ; auto CmpDen = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , DenUnmerge . getReg ( <NUM_LIT> ) , Scale0Unmerge . getReg ( <NUM_LIT> ) ) ; Scale = B . buildXor ( S1 , CmpNum , CmpDen ) . getReg ( <NUM_LIT> ) ; } else { Scale = DivScale1 . getReg ( <NUM_LIT> ) ; }

auto Fmas = B . buildIntrinsic ( <STR_LIT> , { S64 } , false ) . addUse ( Fma4 . getReg ( <NUM_LIT> ) ) . addUse ( Fma3 . getReg ( <NUM_LIT> ) ) . addUse ( Mul . getReg ( <NUM_LIT> ) ) . addUse ( Scale ) . setMIFlags ( Flags ) ;


<s> auto Mul = B . buildMul ( S64 , DivScale1 . getReg ( <NUM_LIT> ) , Fma3 , Flags ) ; auto Fma4 = B . buildFMA ( S64 , NegDivScale0 , Mul , DivScale1 . getReg ( <NUM_LIT> ) , Flags ) ; Register Scale ; if ( ! ST . hasUsableDivScaleConditionOutput ( ) ) { LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto NumUnmerge = B . buildUnmerge ( S32 , LHS ) ; auto DenUnmerge = B . buildUnmerge ( S32 , RHS ) ; auto Scale0Unmerge = B . buildUnmerge ( S32 , DivScale0 ) ; auto Scale1Unmerge = B . buildUnmerge ( S32 , DivScale1 ) ; auto CmpNum = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , NumUnmerge . getReg ( <NUM_LIT> ) , Scale1Unmerge . getReg ( <NUM_LIT> ) ) ; auto CmpDen = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , DenUnmerge . getReg ( <NUM_LIT> ) , Scale0Unmerge . getReg ( <NUM_LIT> ) ) ; Scale = B . buildXor ( S1 , CmpNum , CmpDen ) . getReg ( <NUM_LIT> ) ; } else { Scale = DivScale1 . getReg ( <NUM_LIT> ) ;


<s> } Register QueuePtr = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! loadInputValue ( QueuePtr , MIRBuilder , & MFI -> getArgInfo ( ) . QueuePtr ) ) return Register ( ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; Value * V = UndefValue :: get ( PointerType :: get ( Type :: getInt8Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; MachinePointerInfo PtrInfo ( V , StructOffset ) ;

MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , <NUM_LIT> , MinAlign ( <NUM_LIT> , StructOffset ) ) ;


<s> Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( Dst ) ; LLT SrcTy = MRI . getType ( Src ) ; unsigned DestAS = DstTy . getAddressSpace ( ) ; unsigned SrcAS = SrcTy . getAddressSpace ( ) ; assert ( ! DstTy . isVector ( ) ) ; const <ISA_LIT>TargetMachine & TM = static_cast < const <ISA_LIT>TargetMachine & > ( MF . getTarget ( ) ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; if ( ST . getTargetLowering ( ) -> isNoopAddrSpaceCast ( SrcAS , DestAS ) ) { MI . setDesc ( MIRBuilder . getTII ( ) . get ( TargetOpcode :: G_BITCAST ) ) ; return true ; } if ( DestAS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { MIRBuilder . buildExtract ( Dst , Src , <NUM_LIT> ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; uint32_t AddrHiVal = Info -> get32BitAddressHighBits ( ) ; auto HighAddr = MIRBuilder . buildConstant ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT , <NUM_LIT> ) , AddrHiVal ) ; MIRBuilder . buildMerge ( Dst , { Src , HighAddr . getReg ( <NUM_LIT> ) } ) ; MI . eraseFromParent ( ) ;


<s> } if ( SrcAS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; uint32_t AddrHiVal = Info -> get32BitAddressHighBits ( ) ; auto HighAddr = MIRBuilder . buildConstant ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT , <NUM_LIT> ) , AddrHiVal ) ; MIRBuilder . buildMerge ( Dst , { Src , HighAddr . getReg ( <NUM_LIT> ) } ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS == <ISA_LIT>AS :: FLAT_ADDRESS ) { assert ( DestAS == <ISA_LIT>AS :: LOCAL_ADDRESS || DestAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; unsigned NullVal = TM . getNullPointerValue ( DestAS ) ; auto SegmentNull = MIRBuilder . buildConstant ( DstTy , NullVal ) ; auto FlatNull = MIRBuilder . buildConstant ( SrcTy , <NUM_LIT> ) ; Register PtrLo32 = MRI . createGenericVirtualRegister ( DstTy ) ; MIRBuilder . buildExtract ( PtrLo32 , Src , <NUM_LIT> ) ; Register CmpRes = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildICmp ( CmpInst :: ICMP_NE , CmpRes , Src , FlatNull . getReg ( <NUM_LIT> ) ) ; MIRBuilder . buildSelect ( Dst , CmpRes , PtrLo32 , SegmentNull . getReg ( <NUM_LIT> ) ) ; MI . eraseFromParent ( ) ; return true ; }

if ( SrcAS != <ISA_LIT>AS :: LOCAL_ADDRESS && SrcAS != <ISA_LIT>AS :: PRIVATE_ADDRESS ) return false ;


<s> bool <ISA_LIT>LegalizerInfo :: legalizeCustom ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & MIRBuilder , GISelChangeObserver & Observer ) const { switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_ADDRSPACE_CAST : return legalizeAddrSpaceCast ( MI , MRI , MIRBuilder ) ; case TargetOpcode :: G_FRINT : return legalizeFrint ( MI , MRI , MIRBuilder ) ; case TargetOpcode :: G_FCEIL : return legalizeFceil ( MI , MRI , MIRBuilder ) ; case TargetOpcode :: G_INTRIN<ISA_LIT>C_TRUNC : return legalizeIntrinsicTrunc ( MI , MRI , MIRBuilder ) ; case TargetOpcode :: G_<ISA_LIT>TOFP : return legalizeITOFP ( MI , MRI , MIRBuilder , true ) ; case TargetOpcode :: G_UITOFP : return legalizeITOFP ( MI , MRI , MIRBuilder , false ) ; case TargetOpcode :: G_FMINNUM : case TargetOpcode :: G_FMAXNUM : case TargetOpcode :: G_FMINNUM_IEEE : case TargetOpcode :: G_FMAXNUM_IEEE : return legalizeMinNumMaxNum ( MI , MRI , MIRBuilder ) ; case TargetOpcode :: G_EXTRACT_VECTOR_ELT :

return legalizeExtractVectorElt ( MI , MRI , MIRBuilder ) ;


<s> return false ; } case <STR_LIT> : { if ( MachineInstr * BrCond = verifyCFIntrinsic ( MI , MRI ) ) { const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; B . setInstr ( * BrCond ) ; Register Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; B . buildInstr ( <STR_LIT> ) . addUse ( Reg ) . addMBB ( BrCond -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MI . eraseFromParent ( ) ; BrCond -> eraseFromParent ( ) ; MRI . setRegClass ( Reg , TRI -> getWaveMaskRegClass ( ) ) ; return true ; } return false ; } case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeImplicitArgPtr ( MI , MRI , B ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> :

return legalizeFDIVFast ( MI , MRI , B ) ;


<s> LLT Ty = MRI . getType ( DstReg ) ; Register LiveIn = getLiveInRegister ( MRI , Arg -> getRegister ( ) , Ty ) ; if ( Arg -> isMasked ( ) ) { const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const unsigned Mask = Arg -> getMask ( ) ; const unsigned Shift = countTrailingZeros < unsigned > ( Mask ) ; auto ShiftAmt = B . buildConstant ( S32 , Shift ) ; auto LShr = B . buildLShr ( S32 , LiveIn , ShiftAmt ) ; B . buildAnd ( DstReg , LShr , B . buildConstant ( S32 , Mask >> Shift ) ) ; } else B . buildCopy ( DstReg , LiveIn ) ; if ( ! MRI . getVRegDef ( LiveIn ) ) { MachineBasicBlock & OrigInsBB = B . getMBB ( ) ;


<s> bool <ISA_LIT>LegalizerInfo :: loadInputValue ( Register DstReg , MachineIRBuilder & B , const ArgDescriptor * Arg ) const { if ( ! Arg -> isRegister ( ) || ! Arg -> getRegister ( ) . isValid ( ) ) return false ; assert ( Arg -> getRegister ( ) . isPhysical ( ) ) ; MachineRegisterInfo & MRI = * B . getMRI ( ) ; LLT Ty = MRI . getType ( DstReg ) ; Register LiveIn = getLiveInRegister ( MRI , Arg -> getRegister ( ) , Ty ) ; if ( Arg -> isMasked ( ) ) { const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const unsigned Mask = Arg -> getMask ( ) ; const unsigned Shift = countTrailingZeros < unsigned > ( Mask ) ; auto ShiftAmt = B . buildConstant ( S32 , Shift ) ; auto LShr = B . buildLShr ( S32 , LiveIn , ShiftAmt ) ; B . buildAnd ( DstReg , LShr , B . buildConstant ( S32 , Mask >> Shift ) ) ; } else B . buildCopy ( DstReg , LiveIn ) ; if ( ! MRI . getVRegDef ( LiveIn ) ) { MachineBasicBlock & OrigInsBB = B . getMBB ( ) ; auto OrigInsPt = B . getInsertPt ( ) ;

MachineBasicBlock & EntryMBB = B . getMF ( ) . front ( ) ;


<s> setAction ( { G_CONSTANT , S1 } , Legal ) ; setAction ( { G_FADD , S32 } , Legal ) ; setAction ( { G_FCMP , S1 } , Legal ) ; setAction ( { G_FCMP , <NUM_LIT> , S32 } , Legal ) ; setAction ( { G_FCMP , <NUM_LIT> , S64 } , Legal ) ; setAction ( { G_FMUL , S32 } , Legal ) ; setAction ( { G_ZEXT , S64 } , Legal ) ; setAction ( { G_ZEXT , <NUM_LIT> , S32 } , Legal ) ; setAction ( { G_FPTO<ISA_LIT> , S32 } , Legal ) ; setAction ( { G_FPTO<ISA_LIT> , <NUM_LIT> , S32 } , Legal ) ; setAction ( { G_FPTOUI , S32 } , Legal ) ; setAction ( { G_FPTOUI , <NUM_LIT> , S32 } , Legal ) ; for ( LLT PtrTy : AddrSpaces ) { LLT IdxTy = LLT :: scalar ( PtrTy . getSizeInBits ( ) ) ; setAction ( { G_GEP , PtrTy } , Legal ) ; setAction ( { G_GEP , <NUM_LIT> , IdxTy } , Legal ) ; } setAction ( { G_ICMP , S1 } , Legal ) ; setAction ( { G_ICMP , <NUM_LIT> , S32 } , Legal ) ; getActionDefinitionsBuilder ( { G_LOAD , G_STORE } ) . legalIf ( [ = , & ST ] ( const LegalityQuery & Query ) { const LLT & Ty0 = Query . Types [ <NUM_LIT> ] ; switch ( Ty0 . getSizeInBits ( ) ) { case <NUM_LIT> : case <NUM_LIT> : case <NUM_LIT> : return true ; case <NUM_LIT> : return ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: SEA_ISLANDS ) ; case <NUM_LIT> : case <NUM_LIT> : default : return false ; } } ) ; setAction ( { G_SELECT , S32 } , Legal ) ; setAction ( { G_SELECT , <NUM_LIT> , S1 } , Legal ) ; setAction ( { G_SHL , S32 } , Legal ) ; setAction ( { G_GEP , S64 } , Legal ) ; for ( unsigned Op : { G_EXTRACT_VECTOR_ELT , G_INSERT_VECTOR_ELT } ) { getActionDefinitionsBuilder ( Op ) . legalIf ( [ = ] ( const LegalityQuery & Query ) { const LLT & VecTy = Query . Types [ <NUM_LIT> ] ; const LLT & IdxTy = Query . Types [ <NUM_LIT> ] ;

return VecTy . getSizeInBits ( ) % <NUM_LIT> == <NUM_LIT> && VecTy . getSizeInBits ( ) <= <NUM_LIT> && IdxTy . getSizeInBits ( ) == <NUM_LIT> ;


<s> case <STR_LIT> : return legalizeImplicitArgPtr ( MI , MRI , B ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeFDIVFastIntrin ( MI , MRI , B ) ; case <STR_LIT> : return legalizeIsAddrSpace ( MI , MRI , B , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; case <STR_LIT> : return legalizeIsAddrSpace ( MI , MRI , B , <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; case <STR_LIT> : { B . setInstr ( MI ) ; B . buildConstant ( MI . getOperand ( <NUM_LIT> ) , ST . getWavefrontSize ( ) ) ; MI . eraseFromParent ( ) ;


<s> const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S512 = LLT :: scalar ( <NUM_LIT> ) ; const LLT GlobalPtr = GetAddrSpacePtr ( <ISA_LIT>AS :: GLOBAL_ADDRESS ) ; const LLT ConstantPtr = GetAddrSpacePtr ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; const LLT LocalPtr = GetAddrSpacePtr ( <ISA_LIT>AS :: LOCAL_ADDRESS ) ; const LLT FlatPtr = GetAddrSpacePtr ( <ISA_LIT>AS . FLAT_ADDRESS ) ; const LLT PrivatePtr = GetAddrSpacePtr ( <ISA_LIT>AS . PRIVATE_ADDRESS ) ; const LLT AddrSpaces [ ] = { GlobalPtr , ConstantPtr , LocalPtr , FlatPtr , PrivatePtr } ; setAction ( { G_ADD , S32 } , Legal ) ; setAction ( { G_ASHR , S32 } , Legal ) ; setAction ( { G_SUB , S32 } , Legal ) ; setAction ( { G_MUL , S32 } , Legal ) ; setAction ( { G_AND , S32 } , Legal ) ; setAction ( { G_OR , S32 } , Legal ) ; setAction ( { G_XOR , S32 } , Legal ) ; setAction ( { G_BITCAST , V2S16 } , Legal ) ; setAction ( { G_BITCAST , <NUM_LIT> , S32 } , Legal ) ; setAction ( { G_BITCAST , S32 } , Legal ) ; setAction ( { G_BITCAST , <NUM_LIT> , V2S16 } , Legal ) ; getActionDefinitionsBuilder ( G_FCONSTANT ) . legalFor ( { S32 , S64 } ) ; getActionDefinitionsBuilder ( G_IMPLICIT_DEF ) . legalIf ( [ = ] ( const LegalityQuery & Query ) { return Query . Types [ <NUM_LIT> ] . getSizeInBits ( ) <= <NUM_LIT> ; } ) . clampScalar ( <NUM_LIT> , S1 , S512 ) ; getActionDefinitionsBuilder ( G_CONSTANT ) . legalFor ( { S1 , S32 , S64 } ) ; setAction ( { G_CONSTANT , S1 } , Legal ) ; setAction ( { G_FADD , S32 } , Legal ) ; setAction ( { G_FCMP , S1 } , Legal ) ; setAction ( { G_FCMP , <NUM_LIT> , S32 } , Legal ) ; setAction ( { G_FCMP , <NUM_LIT> , S64 } , Legal ) ; setAction ( { G_FMUL , S32 } , Legal ) ; setAction ( { G_ZEXT , S64 } , Legal ) ; setAction ( { G_ZEXT , <NUM_LIT> , S32 } , Legal ) ; setAction ( { G_FPTO<ISA_LIT> , S32 } , Legal ) ; setAction ( { G_FPTO<ISA_LIT> , <NUM_LIT> , S32 } , Legal ) ; setAction ( { G_<ISA_LIT>TOFP , S32 } , Legal ) ; setAction ( { G_<ISA_LIT>TOFP , <NUM_LIT> , S32 } , Legal ) ; setAction ( { G_FPTOUI , S32 } , Legal ) ; setAction ( { G_FPTOUI , <NUM_LIT> , S32 } , Legal ) ; for ( LLT PtrTy : AddrSpaces ) { LLT IdxTy = LLT :: scalar ( PtrTy . getSizeInBits ( ) ) ; setAction ( { G_GEP , PtrTy } , Legal ) ; setAction ( { G_GEP , <NUM_LIT> , IdxTy } , Legal ) ; }


<s> static bool isLegalVOP3PShuffleMask ( ArrayRef < int > Mask ) { assert ( Mask . size ( ) == <NUM_LIT> ) ; if ( Mask [ <NUM_LIT> ] == - <NUM_LIT> || Mask [ <NUM_LIT> ] == - <NUM_LIT> ) return true ;

return ( ( Mask [ <NUM_LIT> ] == <NUM_LIT> || Mask [ <NUM_LIT> ] == <NUM_LIT> ) && ( Mask [ <NUM_LIT> ] == <NUM_LIT> || Mask [ <NUM_LIT> ] == <NUM_LIT> ) ) || ( ( Mask [ <NUM_LIT> ] == <NUM_LIT> || Mask [ <NUM_LIT> ] == <NUM_LIT> ) && ( Mask [ <NUM_LIT> ] == <NUM_LIT> || Mask [ <NUM_LIT> ] == <NUM_LIT> ) ) ;


<s> assert ( Mask . size ( ) == <NUM_LIT> ) ; if ( Mask [ <NUM_LIT> ] == - <NUM_LIT> || Mask [ <NUM_LIT> ] == - <NUM_LIT> ) return true ; return ( ( Mask [ <NUM_LIT> ] == <NUM_LIT> || Mask [ <NUM_LIT> ] == <NUM_LIT> ) && ( Mask [ <NUM_LIT> ] == <NUM_LIT> || Mask [ <NUM_LIT> ] == <NUM_LIT> ) ) || ( ( Mask [ <NUM_LIT> ] == <NUM_LIT> || Mask [ <NUM_LIT> ] == <NUM_LIT> ) && ( Mask [ <NUM_LIT> ] == <NUM_LIT> || Mask [ <NUM_LIT> ] == <NUM_LIT> ) ) ;


<s> case TargetOpcode :: G_FRINT : return legalizeFrint ( MI , MRI , B ) ; case TargetOpcode :: G_FCEIL : return legalizeFceil ( MI , MRI , B ) ; case TargetOpcode :: G_INTRIN<ISA_LIT>C_TRUNC : return legalizeIntrinsicTrunc ( MI , MRI , B ) ; case TargetOpcode :: G_<ISA_LIT>TOFP : return legalizeITOFP ( MI , MRI , B , true ) ; case TargetOpcode :: G_UITOFP : return legalizeITOFP ( MI , MRI , B , false ) ; case TargetOpcode :: G_FPTO<ISA_LIT> : return legalizeFPTOI ( MI , MRI , B , true ) ; case TargetOpcode :: G_FPTOUI : return legalizeFPTOI ( MI , MRI , B , false ) ; case TargetOpcode :: G_FMINNUM : case TargetOpcode :: G_FMAXNUM : case TargetOpcode :: G_FMINNUM_IEEE : case TargetOpcode :: G_FMAXNUM_IEEE : return legalizeMinNumMaxNum ( MI , MRI , B ) ; case TargetOpcode :: G_EXTRACT_VECTOR_ELT : return legalizeExtractVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_INSERT_VECTOR_ELT : return legalizeInsertVectorElt ( MI , MRI , B ) ;


<s> MI . eraseFromParent ( ) ; BrCond -> eraseFromParent ( ) ; MRI . setRegClass ( Reg , TRI -> getWaveMaskRegClass ( ) ) ; return true ; } return false ; } case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeImplicitArgPtr ( MI , MRI , B ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeFDIVFastIntrin ( MI , MRI , B ) ; case <STR_LIT> : return legalizeIsAddrSpace ( MI , MRI , B , <ISA_LIT>AS :: LOCAL_ADDRESS ) ; case <STR_LIT> : return legalizeIsAddrSpace ( MI , MRI , B , <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; case <STR_LIT> : { B . setInstr ( MI ) ; B . buildConstant ( MI . getOperand ( <NUM_LIT> ) , ST . getWavefrontSize ( ) ) ; MI . eraseFromParent ( ) ; return true ; } case <STR_LIT> : return legalizeSBufferLoad ( MI , B , Observer ) ; case <STR_LIT> : case <STR_LIT> : return legalizeBufferStore ( MI , MRI , B , false , false ) ; case <STR_LIT> : case <STR_LIT> : return legalizeBufferStore ( MI , MRI , B , false , true ) ; case <STR_LIT> : case <STR_LIT> :

return legalizeBufferStore ( MI , MRI , B , true , true ) ;


<s> if ( SrcTy == V2S16 && DstTy == V2S16 && isLegalVOP3PShuffleMask ( MI . getOperand ( <NUM_LIT> ) . getShuffleMask ( ) ) ) return true ; MachineIRBuilder HelperBuilder ( MI ) ; GISelObserverWrapper DummyObserver ; LegalizerHelper Helper ( B . getMF ( ) , DummyObserver , HelperBuilder ) ;


<s> bool <ISA_LIT>LegalizerInfo :: legalizeIntrinsic ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { switch ( MI . getOperand ( MI . getNumExplicitDefs ( ) ) . getIntrinsicID ( ) ) { case <STR_LIT> : { if ( MachineInstr * BrCond = verifyCFIntrinsic ( MI , MRI ) ) { const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; B . setInstr ( * BrCond ) ; Register Def = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Use = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; B . buildInstr ( <STR_LIT> ) . addDef ( Def ) . addUse ( Use ) . addMBB ( BrCond -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MRI . setRegClass ( Def , TRI -> getWaveMaskRegClass ( ) ) ; MRI . setRegClass ( Use , TRI -> getWaveMaskRegClass ( ) ) ; MI . eraseFromParent ( ) ; BrCond -> eraseFromParent ( ) ; return true ; } return false ; } case <STR_LIT> : { if ( MachineInstr * BrCond = verifyCFIntrinsic ( MI , MRI ) ) { const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; B . setInstr ( * BrCond ) ; Register Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; B . buildInstr ( <STR_LIT> ) . addUse ( Reg ) . addMBB ( BrCond -> getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MI . eraseFromParent ( ) ; BrCond -> eraseFromParent ( ) ; MRI . setRegClass ( Reg , TRI -> getWaveMaskRegClass ( ) ) ; return true ; } return false ; } case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizeImplicitArgPtr ( MI , MRI , B ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> : return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; case <STR_LIT> :


<s> MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const bool IsIEEEOp = MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ; if ( ! MFI -> getMode ( ) . IEEE ) return ! IsIEEEOp ; if ( IsIEEEOp ) return true ; MachineIRBuilder HelperBuilder ( MI ) ; GISelObserverWrapper DummyObserver ; LegalizerHelper Helper ( MF , DummyObserver , HelperBuilder ) ; HelperBuilder . setMBB ( * MI . getParent ( ) ) ; return Helper . lowerFMinNumMaxNum ( MI ) == LegalizerHelper :: Legalized ;


<s> bool <ISA_LIT>LegalizerInfo :: legalizeMinNumMaxNum ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const bool IsIEEEOp = MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ; if ( ! MFI -> getMode ( ) . IEEE ) return ! IsIEEEOp ; if ( IsIEEEOp ) return true ; MachineIRBuilder HelperBuilder ( MI ) ;


<s> } if ( DestAS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { B . buildExtract ( Dst , Src , <NUM_LIT> ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; uint32_t AddrHiVal = Info -> get32BitAddressHighBits ( ) ; auto HighAddr = B . buildConstant ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT , <NUM_LIT> ) , AddrHiVal ) ; B . buildMerge ( Dst , { Src , HighAddr } ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS == <ISA_LIT>AS :: FLAT_ADDRESS ) { assert ( DestAS == <ISA_LIT>AS :: LOCAL_ADDRESS || DestAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; if ( isKnownNonNull ( Src , MRI , TM , SrcAS ) ) { B . buildExtract ( Dst , Src , <NUM_LIT> ) ; MI . eraseFromParent ( ) ; return true ; } unsigned NullVal = TM . getNullPointerValue ( DestAS ) ; auto SegmentNull = B . buildConstant ( DstTy , NullVal ) ; auto FlatNull = B . buildConstant ( SrcTy , <NUM_LIT> ) ; auto PtrLo32 = B . buildExtract ( DstTy , Src , <NUM_LIT> ) ; auto CmpRes = B . buildICmp ( CmpInst :: ICMP_NE , LLT :: scalar ( <NUM_LIT> ) , Src , FlatNull . getReg ( <NUM_LIT> ) ) ; B . buildSelect ( Dst , CmpRes , PtrLo32 , SegmentNull . getReg ( <NUM_LIT> ) ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS != <ISA_LIT>AS :: LOCAL_ADDRESS && SrcAS != <ISA_LIT>AS :: PRIVATE_ADDRESS ) return false ; if ( ! ST . hasFlatAddressSpace ( ) ) return false ; Register ApertureReg = getSegmentAperture ( SrcAS , MRI , B ) ; if ( ! ApertureReg . isValid ( ) ) return false ; Register SrcAsInt = B . buildPtrToInt ( S32 , Src ) . getReg ( <NUM_LIT> ) ; auto BuildPtr = B . buildMerge ( DstTy , { SrcAsInt , ApertureReg } ) ; if ( isKnownNonNull ( Src , MRI , TM , SrcAS ) ) { B . buildCopy ( Dst , BuildPtr ) ; MI . eraseFromParent ( ) ; return true ; }

auto SegmentNull = B . buildConstant ( SrcTy , TM . getNullPointerValue ( SrcAS ) ) ;


<s> unsigned WidthM1 = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: WIDTH_M1_SRC_SHARED_BASE : <STR_LIT> :: WIDTH_M1_SRC_PRIVATE_BASE ; unsigned Encoding = <STR_LIT> :: ID_MEM_BASES << <STR_LIT> :: ID_SHIFT_ | Offset << <STR_LIT> :: OFFSET_SHIFT_ | WidthM1 << <STR_LIT> :: WIDTH_M1_SHIFT_ ; unsigned ApertureReg = MRI . createGenericVirtualRegister ( S32 ) ; unsigned GetReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; MIRBuilder . buildInstr ( <STR_LIT> ) . addDef ( GetReg ) . addImm ( Encoding ) ; MRI . setType ( GetReg , S32 ) ; auto ShiftAmt = MIRBuilder . buildConstant ( S32 , WidthM1 + <NUM_LIT> ) ; MIRBuilder . buildInstr ( TargetOpcode :: G_SHL ) . addDef ( ApertureReg ) . addUse ( GetReg ) . addUse ( ShiftAmt . getReg ( <NUM_LIT> ) ) ; return ApertureReg ; } unsigned QueuePtr = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; MIRBuilder . buildConstant ( QueuePtr , <NUM_LIT> ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; Value * V = UndefValue :: get ( PointerType :: get ( Type :: getInt8Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ; MachinePointerInfo PtrInfo ( V , StructOffset ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , <NUM_LIT> , MinAlign ( <NUM_LIT> , StructOffset ) ) ; unsigned LoadResult = MRI . createGenericVirtualRegister ( S32 ) ; unsigned LoadAddr = <STR_LIT> ; MIRBuilder . materializeGEP ( LoadAddr , QueuePtr , LLT :: scalar ( <NUM_LIT> ) , StructOffset ) ;

MIRBuilder . buildLoad ( LoadResult , LoadAddr , * MMO ) ;


<s> unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( Dst ) ; LLT SrcTy = MRI . getType ( Src ) ; unsigned DestAS = DstTy . getAddressSpace ( ) ; unsigned SrcAS = SrcTy . getAddressSpace ( ) ; assert ( ! DstTy . isVector ( ) ) ; const <ISA_LIT>TargetMachine & TM = static_cast < const <ISA_LIT>TargetMachine & > ( MF . getTarget ( ) ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; if ( ST . getTargetLowering ( ) -> isNoopAddrSpaceCast ( SrcAS , DestAS ) ) { MI . setDesc ( MIRBuilder . getTII ( ) . get ( TargetOpcode :: G_BITCAST ) ) ; return true ; } if ( SrcAS == <ISA_LIT>AS :: FLAT_ADDRESS ) { assert ( DestAS == <ISA_LIT>AS :: LOCAL_ADDRESS || DestAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; unsigned NullVal = TM . getNullPointerValue ( DestAS ) ; auto SegmentNull = MIRBuilder . buildConstant ( DstTy , NullVal ) ; auto FlatNull = MIRBuilder . buildConstant ( SrcTy , <NUM_LIT> ) ; unsigned PtrLo32 = MRI . createGenericVirtualRegister ( DstTy ) ; MIRBuilder . buildExtract ( PtrLo32 , Src , <NUM_LIT> ) ; unsigned CmpRes = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildICmp ( CmpInst :: ICMP_NE , CmpRes , Src , FlatNull . getReg ( <NUM_LIT> ) ) ; MIRBuilder . buildSelect ( Dst , CmpRes , PtrLo32 , SegmentNull . getReg ( <NUM_LIT> ) ) ; MI . eraseFromParent ( ) ; return true ; } assert ( SrcAS == <ISA_LIT>AS :: LOCAL_ADDRESS || SrcAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; auto SegmentNull = MIRBuilder . buildConstant ( SrcTy , TM . getNullPointerValue ( SrcAS ) ) ; auto FlatNull = MIRBuilder . buildConstant ( DstTy , TM . getNullPointerValue ( DestAS ) ) ; unsigned ApertureReg = getSegmentAperture ( DestAS , MRI , MIRBuilder ) ; unsigned CmpRes = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildICmp ( CmpInst :: ICMP_NE , CmpRes , Src , SegmentNull . getReg ( <NUM_LIT> ) ) ; unsigned BuildPtr = MRI . createGenericVirtualRegister ( DstTy ) ; unsigned SrcAsInt = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildInstr ( TargetOpcode :: G_PTRTOINT ) . addDef ( SrcAsInt ) . addUse ( Src ) ; MIRBuilder . buildMerge ( BuildPtr , { SrcAsInt , ApertureReg } ) ; MIRBuilder . buildSelect ( Dst , CmpRes , BuildPtr , FlatNull . getReg ( <NUM_LIT> ) ) ; MI . eraseFromParent ( ) ; return true ;


<s> const auto One = B . buildFConstant ( S64 , <NUM_LIT> ) ; auto Lt0 = B . buildFCmp ( CmpInst :: FCMP_OGT , S1 , Src , Zero ) ; auto NeTrunc = B . buildFCmp ( CmpInst :: FCMP_ONE , S1 , Src , Trunc ) ; auto And = B . buildAnd ( S1 , Lt0 , NeTrunc ) ; auto Add = B . buildSelect ( S64 , And , One , Zero ) ; B . buildFAdd ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Trunc , Add ) ;


<s> LLT Ty = MRI . getType ( Src ) ; assert ( Ty . isScalar ( ) && Ty . getSizeInBits ( ) == <NUM_LIT> ) ; APFloat C1Val ( APFloat :: IEEEdouble ( ) , <STR_LIT> ) ; APFloat C2Val ( APFloat :: IEEEdouble ( ) , <STR_LIT> ) ; auto C1 = MIRBuilder . buildFConstant ( Ty , C1Val ) ; auto CopySign = MIRBuilder . buildFCopysign ( Ty , C1 , Src ) ; auto Tmp1 = MIRBuilder . buildFAdd ( Ty , Src , CopySign ) ; auto Tmp2 = MIRBuilder . buildFSub ( Ty , Tmp1 , CopySign ) ; auto C2 = MIRBuilder . buildFConstant ( Ty , C2Val ) ;

auto Fabs = MIRBuilder . buildFAbs ( Ty , Src ) ;


<s> APFloat C2Val ( APFloat :: IEEEdouble ( ) , <STR_LIT> ) ; auto C1 = MIRBuilder . buildFConstant ( Ty , C1Val ) ; auto CopySign = MIRBuilder . buildFCopysign ( Ty , C1 , Src ) ; auto Tmp1 = MIRBuilder . buildFAdd ( Ty , Src , CopySign ) ; auto Tmp2 = MIRBuilder . buildFSub ( Ty , Tmp1 , CopySign ) ; auto C2 = MIRBuilder . buildFConstant ( Ty , C2Val ) ; auto Fabs = MIRBuilder . buildFAbs ( Ty , Src ) ; auto Cond = MIRBuilder . buildFCmp ( CmpInst :: FCMP_OGT , LLT :: scalar ( <NUM_LIT> ) , Fabs , C2 ) ; MIRBuilder . buildSelect ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Cond , Src , Tmp2 ) ;


<s> assert ( MRI . getType ( Src ) == S64 ) ; auto Unmerge = B . buildUnmerge ( { S32 , S32 } , Src ) ; unsigned Hi = Unmerge . getReg ( <NUM_LIT> ) ; auto Exp = extractF64Exponent ( Hi , B ) ; const unsigned FractBits = <NUM_LIT> ; const auto SignBitMask = B . buildConstant ( S32 , UINT32_C ( <NUM_LIT> ) << <NUM_LIT> ) ; auto SignBit = B . buildAnd ( S32 , Hi , SignBitMask ) ; const auto FractMask = B . buildConstant ( S64 , ( UINT64_C ( <NUM_LIT> ) << FractBits ) - <NUM_LIT> ) ; const auto Zero32 = B . buildConstant ( S32 , <NUM_LIT> ) ; auto SignBit64 = B . buildMerge ( S64 , { Zero32 . getReg ( <NUM_LIT> ) , SignBit . getReg ( <NUM_LIT> ) } ) ;

auto Shr = B . buildAShr ( S64 , FractMask , Exp ) ;


<s> unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; assert ( MRI . getType ( Src ) == S64 && MRI . getType ( Dst ) == S64 ) ; auto Unmerge = B . buildUnmerge ( { S32 , S32 } , Src ) ; auto CvtHi = Signed ? B . build<ISA_LIT>TOFP ( S64 , Unmerge . getReg ( <NUM_LIT> ) ) : B . buildUITOFP ( S64 , Unmerge . getReg ( <NUM_LIT> ) ) ; auto CvtLo = B . buildUITOFP ( S64 , Unmerge . getReg ( <NUM_LIT> ) ) ; auto ThirtyTwo = B . buildConstant ( S32 , <NUM_LIT> ) ; auto LdExp = B . buildIntrinsic ( <STR_LIT> , { S64 } , false ) . addUse ( CvtHi . getReg ( <NUM_LIT> ) ) . addUse ( ThirtyTwo . getReg ( <NUM_LIT> ) ) ; B . buildFAdd ( Dst , LdExp , CvtLo ) ;


<s> LLT StoreVT = MRI . getType ( Reg ) ; assert ( StoreVT . isVector ( ) && StoreVT . getElementType ( ) == S16 ) ; if ( ST . hasUnpackedD16VMem ( ) ) { auto Unmerge = B . buildUnmerge ( S16 , Reg ) ; SmallVector < Register , <NUM_LIT> > WideRegs ; for ( int I = <NUM_LIT> , E = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; I != E ; ++ I ) WideRegs . push_back ( B . buildAnyExt ( S32 , Unmerge . getReg ( I ) ) . getReg ( <NUM_LIT> ) ) ; int NumElts = StoreVT . getNumElements ( ) ; return B . buildBuildVector ( LLT :: vector ( NumElts , S32 ) , WideRegs ) . getReg ( <NUM_LIT> ) ; } if ( ImageStore && ST . hasImageStoreD16Bug ( ) ) { if ( StoreVT . getNumElements ( ) == <NUM_LIT> ) { SmallVector < Register , <NUM_LIT> > PackedRegs ; Reg = B . buildBitcast ( S32 , Reg ) . getReg ( <NUM_LIT> ) ; PackedRegs . push_back ( Reg ) ; PackedRegs . resize ( <NUM_LIT> , B . buildUndef ( S32 ) . getReg ( <NUM_LIT> ) ) ; return B . buildBuildVector ( LLT :: vector ( <NUM_LIT> , S32 ) , PackedRegs ) . getReg ( <NUM_LIT> ) ;


<s> } if ( ImageStore && ST . hasImageStoreD16Bug ( ) ) { if ( StoreVT . getNumElements ( ) == <NUM_LIT> ) { SmallVector < Register , <NUM_LIT> > PackedRegs ; Reg = B . buildBitcast ( S32 , Reg ) . getReg ( <NUM_LIT> ) ; PackedRegs . push_back ( Reg ) ; PackedRegs . resize ( <NUM_LIT> , B . buildUndef ( S32 ) . getReg ( <NUM_LIT> ) ) ; return B . buildBuildVector ( LLT :: vector ( <NUM_LIT> , S32 ) , PackedRegs ) . getReg ( <NUM_LIT> ) ; } if ( StoreVT . getNumElements ( ) == <NUM_LIT> ) { SmallVector < Register , <NUM_LIT> > PackedRegs ; auto Unmerge = B . buildUnmerge ( S16 , Reg ) ; for ( int I = <NUM_LIT> , E = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; I != E ; ++ I ) PackedRegs . push_back ( Unmerge . getReg ( I ) ) ; PackedRegs . resize ( <NUM_LIT> , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) ) ; Reg = B . buildBuildVector ( LLT :: vector ( <NUM_LIT> , S16 ) , PackedRegs ) . getReg ( <NUM_LIT> ) ; return B . buildBitcast ( LLT :: vector ( <NUM_LIT> , S32 ) , Reg ) . getReg ( <NUM_LIT> ) ; }


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; if ( ST . getTargetLowering ( ) -> isNoopAddrSpaceCast ( SrcAS , DestAS ) ) { MI . setDesc ( MIRBuilder . getTII ( ) . get ( TargetOpcode :: G_BITCAST ) ) ; return true ; } if ( DestAS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { MIRBuilder . buildExtract ( Dst , Src , <NUM_LIT> ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; uint32_t AddrHiVal = Info -> get32BitAddressHighBits ( ) ; auto HighAddr = MIRBuilder . buildConstant ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT , <NUM_LIT> ) , AddrHiVal ) ; MIRBuilder . buildMerge ( Dst , { Src , HighAddr . getReg ( <NUM_LIT> ) } ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS == <ISA_LIT>AS :: FLAT_ADDRESS ) { assert ( DestAS == <ISA_LIT>AS :: LOCAL_ADDRESS || DestAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; unsigned NullVal = TM . getNullPointerValue ( DestAS ) ; auto SegmentNull = MIRBuilder . buildConstant ( DstTy , NullVal ) ; auto FlatNull = MIRBuilder . buildConstant ( SrcTy , <NUM_LIT> ) ; Register PtrLo32 = MRI . createGenericVirtualRegister ( DstTy ) ; MIRBuilder . buildExtract ( PtrLo32 , Src , <NUM_LIT> ) ; Register CmpRes = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; MIRBuilder . buildICmp ( CmpInst :: ICMP_NE , CmpRes , Src , FlatNull . getReg ( <NUM_LIT> ) ) ; MIRBuilder . buildSelect ( Dst , CmpRes , PtrLo32 , SegmentNull . getReg ( <NUM_LIT> ) ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS != <ISA_LIT>AS :: LOCAL_ADDRESS && SrcAS != <ISA_LIT>AS :: PRIVATE_ADDRESS ) return false ; if ( ! ST . hasFlatAddressSpace ( ) ) return false ; auto SegmentNull = MIRBuilder . buildConstant ( SrcTy , TM . getNullPointerValue ( SrcAS ) ) ; auto FlatNull = MIRBuilder . buildConstant ( DstTy , TM . getNullPointerValue ( DestAS ) ) ; Register ApertureReg = getSegmentAperture ( DestAS , MRI , MIRBuilder ) ;


<s> setAction ( { G_GEP , S64 } , Legal ) ; for ( unsigned Op : { G_EXTRACT_VECTOR_ELT , G_INSERT_VECTOR_ELT } ) { getActionDefinitionsBuilder ( Op ) . legalIf ( [ = ] ( const LegalityQuery & Query ) { const LLT & VecTy = Query . Types [ <NUM_LIT> ] ; const LLT & IdxTy = Query . Types [ <NUM_LIT> ] ; return VecTy . getSizeInBits ( ) % <NUM_LIT> == <NUM_LIT> && VecTy . getSizeInBits ( ) <= <NUM_LIT> && IdxTy . getSizeInBits ( ) == <NUM_LIT> ; } ) ; } getActionDefinitionsBuilder ( { G_EXTRACT , G_INSERT } ) . legalIf ( [ = ] ( const LegalityQuery & Query ) { const LLT & Ty0 = Query . Types [ <NUM_LIT> ] ; const LLT & Ty1 = Query . Types [ <NUM_LIT> ] ; return ( Ty0 . getSizeInBits ( ) % <NUM_LIT> == <NUM_LIT> ) && ( Ty1 . getSizeInBits ( ) % <NUM_LIT> == <NUM_LIT> ) ; } ) ; for ( unsigned Op : { G_MERGE_VALUES , G_UNMERGE_VALUES } ) { unsigned BigTyIdx = Op == G_MERGE_VALUES ? <NUM_LIT> : <NUM_LIT> ; unsigned LitTyIdx = Op == G_MERGE_VALUES ? <NUM_LIT> : <NUM_LIT> ; getActionDefinitionsBuilder ( Op ) . legalIf ( [ = ] ( const LegalityQuery & Query ) { const LLT & BigTy = Query . Types [ BigTyIdx ] ; const LLT & LitTy = Query . Types [ LitTyIdx ] ; return BigTy . getSizeInBits ( ) % <NUM_LIT> == <NUM_LIT> && LitTy . getSizeInBits ( ) % <NUM_LIT> == <NUM_LIT> && BigTy . getSizeInBits ( ) <= <NUM_LIT> ; } ) . fewerElementsIf ( [ ] ( const LegalityQuery & Query ) { return true ; } , [ ] ( const LegalityQuery & Query ) { return std :: make_pair ( <NUM_LIT> , Query . Types [ <NUM_LIT> ] . getElementType ( ) ) ; } ) . fewerElementsIf ( [ ] ( const LegalityQuery & Query ) { return true ; } , [ ] ( const LegalityQuery & Query ) { return std :: make_pair ( <NUM_LIT> , Query . Types [ <NUM_LIT> ] . getElementType ( ) ) ; } ) ; } computeTables ( ) ; verify ( * ST . getInstrInfo ( ) ) ;


<s> const unsigned ValSize = ValTy . getSizeInBits ( ) ; const LLT MemTy = MMO -> getMemoryType ( ) ; const Align MemAlign = MMO -> getAlign ( ) ; const unsigned MemSize = MemTy . getSizeInBits ( ) ; const unsigned AlignInBits = <NUM_LIT> * MemAlign . value ( ) ; if ( shouldWidenLoad ( ST , MemTy , AlignInBits , AddrSpace , MI . getOpcode ( ) ) ) { const unsigned WideMemSize = PowerOf2Ceil ( MemSize ) ; if ( WideMemSize == ValSize ) { MachineFunction & MF = B . getMF ( ) ; MachineMemOperand * WideMMO = MF . getMachineMemOperand ( MMO , <NUM_LIT> , WideMemSize / <NUM_LIT> ) ; Observer . changingInstr ( MI ) ; MI . setMemRefs ( MF , { WideMMO } ) ; Observer . changedInstr ( MI ) ; return true ; } if ( ValSize > WideMemSize ) return false ; LLT WideTy = widenToNextPowerOf2 ( ValTy ) ; Register WideLoad ; if ( ! WideTy . isVector ( ) ) { WideLoad = B . buildLoadFromOffset ( WideTy , PtrReg , * MMO , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; B . buildTrunc ( ValReg , WideLoad ) . getReg ( <NUM_LIT> ) ;


<s> for ( unsigned I = Intr -> VAddrStart ; I < EndIdx ; I ++ ) { MachineOperand & SrcOp = MI . getOperand ( ArgOffset + I ) ; if ( ! SrcOp . isReg ( ) ) continue ; Register AddrReg = SrcOp . getReg ( ) ; if ( ( I < Intr -> GradientStart ) || ( I >= Intr -> GradientStart && I < Intr -> CoordStart && ! IsG16 ) || ( I >= Intr -> CoordStart && ! IsA16 ) ) { if ( ( I < Intr -> GradientStart ) && IsA16 && ( B . getMRI ( ) -> getType ( AddrReg ) == S16 ) ) { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) } ) . getReg ( <NUM_LIT> ) ) ; } else { AddrReg = B . buildBitcast ( V2S16 , AddrReg ) . getReg ( <NUM_LIT> ) ; PackedAddrs . push_back ( AddrReg ) ; } } else { if ( ( ( I + <NUM_LIT> ) >= EndIdx ) || ( ( Intr -> NumGradients / <NUM_LIT> ) % <NUM_LIT> == <NUM_LIT> && ( I == static_cast < unsigned > ( Intr -> GradientStart + ( Intr -> NumGradients / <NUM_LIT> ) - <NUM_LIT> ) || I == static_cast < unsigned > ( Intr -> GradientStart + Intr -> NumGradients - <NUM_LIT> ) ) ) || ! MI . getOperand ( ArgOffset + I + <NUM_LIT> ) . isReg ( ) ) { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) } ) . getReg ( <NUM_LIT> ) ) ; } else { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , MI . getOperand ( ArgOffset + I + <NUM_LIT> ) . getReg ( ) } ) . getReg ( <NUM_LIT> ) ) ;


<s> MachineOperand & SrcOp = MI . getOperand ( ArgOffset + I ) ; if ( ! SrcOp . isReg ( ) ) continue ; Register AddrReg = SrcOp . getReg ( ) ; if ( ( I < Intr -> GradientStart ) || ( I >= Intr -> GradientStart && I < Intr -> CoordStart && ! IsG16 ) || ( I >= Intr -> CoordStart && ! IsA16 ) ) { if ( ( I < Intr -> GradientStart ) && IsA16 && ( B . getMRI ( ) -> getType ( AddrReg ) == S16 ) ) { PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) } ) . getReg ( <NUM_LIT> ) ) ; } else { AddrReg = B . buildBitcast ( V2S16 , AddrReg ) . getReg ( <NUM_LIT> ) ; PackedAddrs . push_back ( AddrReg ) ; } } else {

if ( ( ( I + <NUM_LIT> ) >= EndIdx ) || ( ( Intr -> NumGradients / <NUM_LIT> ) % <NUM_LIT> == <NUM_LIT> && ( I == static_cast < unsigned > ( Intr -> GradientStart + ( Intr -> NumGradients / <NUM_LIT> ) - <NUM_LIT> ) || I == static_cast < unsigned > ( Intr -> GradientStart + Intr -> NumGradients - <NUM_LIT> ) ) ) || ! MI . getOperand ( ArgOffset + I + <NUM_LIT> ) . isReg ( ) ) {


<s> using namespace TargetOpcode ; const LLT S1 = LLT :: scalar ( <NUM_LIT> ) ; const LLT V2S16 = LLT :: vector ( <NUM_LIT> , <NUM_LIT> ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; const LLT P1 = LLT :: pointer ( <NUM_LIT> , <NUM_LIT> ) ; const LLT P2 = LLT :: pointer ( <NUM_LIT> , <NUM_LIT> ) ; setAction ( { G_ADD , S32 } , Legal ) ; setAction ( { G_AND , S32 } , Legal ) ; setAction ( { G_BITCAST , V2S16 } , Legal ) ; setAction ( { G_BITCAST , <NUM_LIT> , S32 } , Legal ) ; setAction ( { G_BITCAST , S32 } , Legal ) ; setAction ( { G_BITCAST , <NUM_LIT> , V2S16 } , Legal ) ; setAction ( { G_CONSTANT , S1 } , Legal ) ; setAction ( { G_CONSTANT , S32 } , Legal ) ;

setAction ( { G_CONSTANT , S64 } , Legal ) ;


<s> <ISA_LIT>SimplifyLibCalls ( const TargetOptions & Opt = TargetOptions ( ) ) : FunctionPass ( ID ) , Options ( Opt ) {

initialize<ISA_LIT>SimplifyLibCallsPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>SimplifyLibCallsPass ( const TargetOptions & Opt ) {

return new <ISA_LIT>SimplifyLibCalls ( Opt ) ;


<s> CallInst * CreateCallEx ( IRB & B , Value * Callee , Value * Arg , const Twine & Name = <STR_LIT> ) { CallInst * R = B . CreateCall ( Callee , Arg , Name ) ;

if ( Function * F = dyn_cast < Function > ( Callee ) ) R -> setCallingConv ( F -> getCallingConv ( ) ) ;


<s> CallInst * R = B . CreateCall ( Callee , { Arg1 , Arg2 } , Name ) ;


<s> return ( getVecSize ( FInfo ) != <NUM_LIT> ) ? false : fold_recip ( CI , B , FInfo ) ; case <ISA_LIT>LibFunc :: EI_DIVIDE : assert ( ( FInfo . getPrefix ( ) == <ISA_LIT>LibFunc :: NATIVE || FInfo . getPrefix ( ) == <ISA_LIT>LibFunc :: HALF ) && <STR_LIT> ) ; return ( getVecSize ( FInfo ) != <NUM_LIT> ) ? false : fold_divide ( CI , B , FInfo ) ; case <ISA_LIT>LibFunc :: EI_POW : case <ISA_LIT>LibFunc :: EI_POWR : case <ISA_LIT>LibFunc :: EI_POWN : return fold_pow ( CI , B , FInfo ) ; case <ISA_LIT>LibFunc :: EI_ROOTN : return ( getVecSize ( FInfo ) != <NUM_LIT> ) ? false : fold_rootn ( CI , B , FInfo ) ; case <ISA_LIT>LibFunc :: EI_FMA : case <ISA_LIT>LibFunc :: EI_MAD : case <ISA_LIT>LibFunc :: EI_NFMA :

return ( getVecSize ( FInfo ) != <NUM_LIT> ) ? false : fold_fma_mad ( CI , B , FInfo ) ;


<s> for ( unsigned I = <NUM_LIT> ; I != PtrArgLoc ; ++ I ) ArgTys . push_back ( CI -> getArgOperand ( I ) -> getType ( ) ) ; ArgTys . push_back ( PtrTy ) ; Name = Name + <STR_LIT> + std :: to_string ( Size ) ; auto * FTy = FunctionType :: get ( Callee -> getReturnType ( ) , ArrayRef < Type * > ( ArgTys ) , false ) ; <ISA_LIT>LibFunc NewLibFunc ( Name , FTy ) ; auto * F = <ISA_LIT>LibFunc :: getOrInsertFunction ( M , NewLibFunc ) ; if ( ! F ) return false ; auto * BCast = B . CreatePointerCast ( PtrArg , PtrTy ) ; SmallVector < Value * , <NUM_LIT> > Args ;

for ( unsigned I = <NUM_LIT> ; I != PtrArgLoc ; ++ I ) Args . push_back ( CI -> getArgOperand ( I ) ) ;


<s> auto & Ctx = M -> getContext ( ) ; std :: string Name = Callee -> getName ( ) ; auto NumArg = CI -> getNumArgOperands ( ) ; if ( NumArg != <NUM_LIT> && NumArg != <NUM_LIT> ) return false ; auto * PacketSize = CI -> getArgOperand ( NumArg - <NUM_LIT> ) ; auto * PacketAlign = CI -> getArgOperand ( NumArg - <NUM_LIT> ) ; if ( ! isa < ConstantInt > ( PacketSize ) || ! isa < ConstantInt > ( PacketAlign ) ) return false ; unsigned Size = cast < ConstantInt > ( PacketSize ) -> getZExtValue ( ) ; unsigned Align = cast < ConstantInt > ( PacketAlign ) -> getZExtValue ( ) ; if ( Size != Align || ! isPowerOf2_32 ( Size ) ) return false ; Type * PtrElemTy ; if ( Size <= <NUM_LIT> ) PtrElemTy = Type :: getIntNTy ( Ctx , Size * <NUM_LIT> ) ; else PtrElemTy = VectorType :: get ( Type :: getInt64Ty ( Ctx ) , Size / <NUM_LIT> ) ; unsigned PtrArgLoc = CI -> getNumArgOperands ( ) - <NUM_LIT> ; auto PtrArg = CI -> getArgOperand ( PtrArgLoc ) ; unsigned PtrArgAS = PtrArg -> getType ( ) -> getPointerAddressSpace ( ) ; auto * PtrTy = llvm :: PointerType :: get ( PtrElemTy , PtrArgAS ) ; SmallVector < llvm :: Type * , <NUM_LIT> > ArgTys ; for ( unsigned I = <NUM_LIT> ; I != PtrArgLoc ; ++ I ) ArgTys . push_back ( CI -> getArgOperand ( I ) -> getType ( ) ) ; ArgTys . push_back ( PtrTy ) ; Name = Name + <STR_LIT> + std :: to_string ( Size ) ; auto * FTy = FunctionType :: get ( Callee -> getReturnType ( ) , ArrayRef < Type * > ( ArgTys ) , false ) ; <ISA_LIT>LibFunc NewLibFunc ( Name , FTy ) ; auto * F = <ISA_LIT>LibFunc :: getOrInsertFunction ( M , NewLibFunc ) ; if ( ! F ) return false ; auto * BCast = B . CreatePointerCast ( PtrArg , PtrTy ) ; SmallVector < Value * , <NUM_LIT> > Args ; for ( unsigned I = <NUM_LIT> ; I != PtrArgLoc ; ++ I ) Args . push_back ( CI -> getArgOperand ( I ) ) ; Args . push_back ( BCast ) ;

auto * NCI = B . CreateCall ( F , Args ) ;


<s> Value * opr0 = CI -> getArgOperand ( <NUM_LIT> ) ; if ( ConstantFP * CF = dyn_cast < ConstantFP > ( opr0 ) ) { Value * nval = B . CreateFDiv ( ConstantFP :: get ( CF -> getType ( ) , <NUM_LIT> ) , opr0 , <STR_LIT> ) ; DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * nval << <STR_LIT> ) ;


<s> bool <ISA_LIT>LibCalls :: fold_recip ( CallInst * CI , IRBuilder < > & B , const FuncInfo & FInfo ) { Value * opr0 = CI -> getArgOperand ( <NUM_LIT> ) ; if ( ConstantFP * CF = dyn_cast < ConstantFP > ( opr0 ) ) { Value * nval = B . CreateFDiv ( ConstantFP :: get ( CF -> getType ( ) , <NUM_LIT> ) , opr0 , <STR_LIT> ) ; DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * nval << <STR_LIT> ) ; replaceCall ( nval ) ; return true ; } return false ;


<s> DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * opr0 << <STR_LIT> ) ; replaceCall ( opr0 ) ; return true ; } if ( ci_opr1 == <NUM_LIT> ) { std :: vector < const Type * > ParamsTys ; ParamsTys . push_back ( opr0 -> getType ( ) ) ; Module * M = CI -> getModule ( ) ; if ( Constant * FPExpr = getFunction ( M , <ISA_LIT>LibFunc ( <ISA_LIT>LibFunc :: EI_SQRT , FInfo ) ) ) { DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * opr0 << <STR_LIT> ) ; Value * nval = CreateCallEx ( B , FPExpr , opr0 , <STR_LIT> ) ; replaceCall ( nval ) ; return true ; } } else if ( ci_opr1 == <NUM_LIT> ) {


<s> int ci_opr1 = ( int ) CINT -> getSExtValue ( ) ; if ( ci_opr1 == <NUM_LIT> ) { DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * opr0 << <STR_LIT> ) ; replaceCall ( opr0 ) ; return true ; } if ( ci_opr1 == <NUM_LIT> ) { std :: vector < const Type * > ParamsTys ; ParamsTys . push_back ( opr0 -> getType ( ) ) ; Module * M = CI -> getModule ( ) ; if ( Constant * FPExpr = getFunction ( M , <ISA_LIT>LibFunc ( <ISA_LIT>LibFunc :: EI_SQRT , FInfo ) ) ) { DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * opr0 << <STR_LIT> ) ; Value * nval = CreateCallEx ( B , FPExpr , opr0 , <STR_LIT> ) ; replaceCall ( nval ) ; return true ; } } else if ( ci_opr1 == <NUM_LIT> ) { Module * M = CI -> getModule ( ) ; if ( Constant * FPExpr = getFunction ( M , <ISA_LIT>LibFunc ( <ISA_LIT>LibFunc :: EI_CBRT , FInfo ) ) ) { DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * opr0 << <STR_LIT> ) ;


<s> CArgVal -> replaceAllUsesWith ( AvailableVal ) ; if ( CArgVal -> getNumUses ( ) == <NUM_LIT> ) LI -> eraseFromParent ( ) ; CArgVal = CI -> getArgOperand ( <NUM_LIT> ) ; } } } Module * M = CI -> getModule ( ) ; fInfo . setId ( isSin ? <ISA_LIT>LibFunc :: EI_COS : <ISA_LIT>LibFunc :: EI_<ISA_LIT>N ) ; std :: string const PairName = fInfo . mangle ( ) ; CallInst * UI = nullptr ; for ( User * U : CArgVal -> users ( ) ) { CallInst * XI = dyn_cast_or_null < CallInst > ( U ) ; if ( ! XI || XI == CI || XI -> getParent ( ) != CBB ) continue ; Function * UCallee = XI -> getCalledFunction ( ) ; if ( ! UCallee || ! UCallee -> getName ( ) . equals ( PairName ) ) continue ; BasicBlock :: iterator BBI = CI -> getIterator ( ) ; if ( BBI == CI -> getParent ( ) -> begin ( ) ) break ; -- BBI ; for ( int I = MaxScan ; I > <NUM_LIT> && BBI != CBB -> begin ( ) ; -- BBI , -- I ) { if ( cast < Instruction > ( BBI ) == XI ) { UI = XI ; break ; } } if ( UI ) break ; } if ( ! UI ) return false ; <ISA_LIT>LibFunc nf ( <ISA_LIT>LibFunc :: EI_<ISA_LIT>NCOS , fInfo ) ; nf . getLeads ( ) [ <NUM_LIT> ] . PtrKind = <ISA_LIT>LibFunc :: GENERIC ; Function * Fsincos = dyn_cast_or_null < Function > ( getFunction ( M , nf ) ) ; if ( ! Fsincos ) return false ; BasicBlock :: iterator ItOld = B . GetInsertPoint ( ) ; AllocaInst * Alloc = insertAlloca ( UI , B , <STR_LIT> ) ; B . SetInsertPoint ( UI ) ; Value * P = Alloc ; Type * PTy = Fsincos -> getFunctionType ( ) -> getParamType ( <NUM_LIT> ) ;

const <ISA_LIT>AS AS = <STR_LIT> ( * M ) ;


<s> if ( getArgType ( FInfo ) == <ISA_LIT>LibFunc :: F32 && ( getVecSize ( FInfo ) == <NUM_LIT> ) && ( FInfo . getPrefix ( ) != <ISA_LIT>LibFunc :: NATIVE ) ) { if ( Constant * FPExpr = getNativeFunction ( CI -> getModule ( ) , <ISA_LIT>LibFunc ( <ISA_LIT>LibFunc :: EI_SQRT , FInfo ) ) ) { Value * opr0 = CI -> getArgOperand ( <NUM_LIT> ) ; DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << <STR_LIT> << * opr0 << <STR_LIT> ) ; Value * nval = CreateCallEx ( B , FPExpr , opr0 , <STR_LIT> ) ; replaceCall ( nval ) ; return true ; }


<s> Constant * <ISA_LIT>LibCalls :: getFunction ( Module * M , const FuncInfo & fInfo ) { return EnablePreLink ? <ISA_LIT>LibFunc :: getOrInsertFunction ( M , fInfo ) : <ISA_LIT>LibFunc :: getFunction ( M , fInfo ) ;


<s> return EnablePreLink ? <ISA_LIT>LibFunc :: getOrInsertFunction ( M , fInfo ) : <ISA_LIT>LibFunc :: getFunction ( M , fInfo ) ;


<s> if ( getArgType ( FInfo ) == <ISA_LIT>LibFunc :: F64 || ! HasNative ( FInfo . getId ( ) ) ) return nullptr ; FuncInfo nf = FInfo ; nf . setPrefix ( <ISA_LIT>LibFunc :: NATIVE ) ; return getFunction ( M , nf ) ;


<s> AllocaInst * Alloc = B . CreateAlloca ( RetType , <NUM_LIT> , std :: string ( prefix ) + UI -> getName ( ) ) ;

Alloc -> setAlignment ( UCallee -> getParent ( ) -> getDataLayout ( ) . getTypeAllocSize ( RetType ) ) ;


<s> AllocaInst * Alloc = B . CreateAlloca ( RetType , <NUM_LIT> , std :: string ( prefix ) + UI -> getName ( ) ) ; Alloc -> setAlignment ( UCallee -> getParent ( ) -> getDataLayout ( ) . getTypeAllocSize ( RetType ) ) ; return Alloc ;


<s> bool <ISA_LIT>LibCalls :: isUnsafeMath ( const CallInst * CI ) const {

if ( auto Op = dyn_cast < FPMathOperator > ( CI ) ) if ( Op -> hasUnsafeAlgebra ( ) ) return true ;


<s> if ( auto Op = dyn_cast < FPMathOperator > ( CI ) ) if ( Op -> hasUnsafeAlgebra ( ) ) return true ; const Function * F = CI -> getParent ( ) -> getParent ( ) ; Attribute Attr = F -> getFnAttribute ( <STR_LIT> ) ;

return Attr . getValueAsString ( ) == <STR_LIT> ;


<s> static double log2 ( double V ) { return :: log2 ( V ) ; return log ( V ) / <NUM_LIT> ;


<s> <ISA_LIT>LibFunc nf = FInfo ; nf . setPrefix ( <ISA_LIT>LibFunc :: NATIVE ) ; if ( Constant * FPExpr = getFunction ( M , nf ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * CI << <STR_LIT> ) ;


<s> if ( ! EnablePreLink ) Changed |= setFastFlags ( F , Options ) ; for ( auto & BB : F ) { for ( BasicBlock :: iterator I = BB . begin ( ) , E = BB . end ( ) ; I != E ; ) { CallInst * CI = dyn_cast < CallInst > ( I ) ; ++ I ;


<s> B . addAttribute ( <STR_LIT> , <STR_LIT> ) ; } if ( ! B . hasAttributes ( ) ) return false ; F . addAttributes ( AttributeList :: FunctionIndex , B ) ;


<s> nf . getLeads ( ) [ <NUM_LIT> ] . ArgType = FInfo . getLeads ( ) [ <NUM_LIT> ] . ArgType ; nf . getLeads ( ) [ <NUM_LIT> ] . VectorSize = FInfo . getLeads ( ) [ <NUM_LIT> ] . VectorSize ; nf . setPrefix ( <ISA_LIT>LibFunc :: NATIVE ) ; nf . setId ( <ISA_LIT>LibFunc :: EI_<ISA_LIT>N ) ; Constant * sinExpr = getFunction ( M , nf ) ; nf . setPrefix ( <ISA_LIT>LibFunc :: NATIVE ) ; nf . setId ( <ISA_LIT>LibFunc :: EI_COS ) ; Constant * cosExpr = getFunction ( M , nf ) ; if ( sinExpr && cosExpr ) { Value * sinval = CallInst :: Create ( sinExpr , opr0 , <STR_LIT> , aCI ) ; Value * cosval = CallInst :: Create ( cosExpr , opr0 , <STR_LIT> , aCI ) ; new StoreInst ( cosval , aCI -> getArgOperand ( <NUM_LIT> ) , aCI ) ; DEBUG_WITH_TYPE ( <STR_LIT> , dbgs ( ) << <STR_LIT> << * aCI << <STR_LIT> ) ; replaceCall ( sinval ) ; return true ; } }


<s> assert ( eltval && <STR_LIT> ) ; bool found = false ; for ( int i = <NUM_LIT> ; i < sz ; ++ i ) { if ( eltval -> isExactlyValue ( ftbl [ i ] . input ) ) { DVal . push_back ( ftbl [ i ] . result ) ; found = true ; break ; } } if ( ! found ) { return false ; } } LLVMContext & context = CI -> getParent ( ) -> getParent ( ) -> getContext ( ) ; Constant * nval ; if ( getArgType ( FInfo ) == <ISA_LIT>LibFunc :: F32 ) { SmallVector < float , <NUM_LIT> > FVal ; for ( unsigned i = <NUM_LIT> ; i < DVal . size ( ) ; ++ i ) { FVal . push_back ( ( float ) DVal [ i ] ) ; } ArrayRef < float > tmp ( FVal ) ; nval = ConstantDataVector :: get ( context , tmp ) ; } else { ArrayRef < double > tmp ( DVal ) ; nval = ConstantDataVector :: get ( context , tmp ) ; } DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * nval << <STR_LIT> ) ; replaceCall ( nval ) ; return true ; } } else { if ( ConstantFP * CF = dyn_cast < ConstantFP > ( opr0 ) ) { for ( int i = <NUM_LIT> ; i < sz ; ++ i ) { if ( CF -> isExactlyValue ( ftbl [ i ] . input ) ) { Value * nval = ConstantFP :: get ( CF -> getType ( ) , ftbl [ i ] . result ) ; DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * nval << <STR_LIT> ) ; replaceCall ( nval ) ; return true ;


<s> Function * Callee = aCI -> getCalledFunction ( ) ; FuncInfo FInfo ; if ( ! parseFunctionName ( Callee -> getName ( ) , & FInfo ) || ! FInfo . isMangled ( ) || FInfo . getPrefix ( ) != <ISA_LIT>LibFunc :: NOPFX || getArgType ( FInfo ) == <ISA_LIT>LibFunc :: F64 || ! HasNative ( FInfo . getId ( ) ) || ! ( AllNative || useNativeFunc ( FInfo . getName ( ) ) ) ) {


<s> return false ; } if ( FInfo . getId ( ) == <ISA_LIT>LibFunc :: EI_<ISA_LIT>NCOS ) return sincosUseNative ( aCI , FInfo ) ; FInfo . setPrefix ( <ISA_LIT>LibFunc :: NATIVE ) ; Constant * F = getFunction ( aCI -> getModule ( ) , FInfo ) ; if ( ! F ) return false ; aCI -> setCalledFunction ( F ) ;

DEBUG_WITH_TYPE ( <STR_LIT> , dbgs ( ) << <STR_LIT> << * aCI << <STR_LIT> ) ;


<s> CallInst * R = B . CreateCall ( Callee , Arg , Name ) ; if ( Function * F = dyn_cast < Function > ( Callee ) ) R -> setCallingConv ( F -> getCallingConv ( ) ) ; return R ;


<s> if ( Function * F = dyn_cast < Function > ( Callee ) ) R -> setCallingConv ( F -> getCallingConv ( ) ) ; return R ;


<s> CallInst * R = B . CreateCall ( Callee , { Arg1 , Arg2 } , Name ) ;


<s> } if ( ci_opr1 == <NUM_LIT> ) { std :: vector < const Type * > ParamsTys ; ParamsTys . push_back ( opr0 -> getType ( ) ) ; Module * M = CI -> getModule ( ) ; if ( Constant * FPExpr = getFunction ( M , <ISA_LIT>LibFunc ( <ISA_LIT>LibFunc :: EI_SQRT , FInfo ) ) ) { LLVM_DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * opr0 << <STR_LIT> ) ; Value * nval = CreateCallEx ( B , FPExpr , opr0 , <STR_LIT> ) ; replaceCall ( nval ) ; return true ; } } else if ( ci_opr1 == <NUM_LIT> ) { Module * M = CI -> getModule ( ) ; if ( Constant * FPExpr = getFunction ( M , <ISA_LIT>LibFunc ( <ISA_LIT>LibFunc :: EI_CBRT , FInfo ) ) ) { LLVM_DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * opr0 << <STR_LIT> ) ; Value * nval = CreateCallEx ( B , FPExpr , opr0 , <STR_LIT> ) ; replaceCall ( nval ) ; return true ; } } else if ( ci_opr1 == - <NUM_LIT> ) {


<s> nf . getLeads ( ) [ <NUM_LIT> ] . PtrKind = <ISA_LIT>LibFunc :: getEPtrKindFromAddrSpace ( <ISA_LIT>AS :: FLAT_ADDRESS ) ; Function * Fsincos = dyn_cast_or_null < Function > ( getFunction ( M , nf ) ) ; if ( ! Fsincos ) return false ; BasicBlock :: iterator ItOld = B . GetInsertPoint ( ) ; AllocaInst * Alloc = insertAlloca ( UI , B , <STR_LIT> ) ; B . SetInsertPoint ( UI ) ; Value * P = Alloc ; Type * PTy = Fsincos -> getFunctionType ( ) -> getParamType ( <NUM_LIT> ) ; if ( PTy -> getPointerAddressSpace ( ) != <ISA_LIT>AS :: PRIVATE_ADDRESS ) P = B . CreateAddrSpaceCast ( Alloc , PTy ) ; CallInst * Call = CreateCallEx2 ( B , Fsincos , UI -> getArgOperand ( <NUM_LIT> ) , P ) ; LLVM_DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * UI << <STR_LIT> << * Call << <STR_LIT> ) ; if ( ! isSin ) { B . SetInsertPoint ( & * ItOld ) ; UI -> replaceAllUsesWith ( & * Call ) ; Instruction * Reload = B . CreateLoad ( Alloc ) ; CI -> replaceAllUsesWith ( Reload ) ; UI -> eraseFromParent ( ) ; CI -> eraseFromParent ( ) ; } else { Instruction * Reload = B . CreateLoad ( Alloc ) ; UI -> replaceAllUsesWith ( Reload ) ;


<s> if ( getArgType ( FInfo ) == <ISA_LIT>LibFunc :: F32 && ( getVecSize ( FInfo ) == <NUM_LIT> ) && ( FInfo . getPrefix ( ) != <ISA_LIT>LibFunc :: NATIVE ) ) { if ( Constant * FPExpr = getNativeFunction ( CI -> getModule ( ) , <ISA_LIT>LibFunc ( <ISA_LIT>LibFunc :: EI_SQRT , FInfo ) ) ) { Value * opr0 = CI -> getArgOperand ( <NUM_LIT> ) ; LLVM_DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << <STR_LIT> << * opr0 << <STR_LIT> ) ;


<s> if ( getArgType ( FInfo ) != <ISA_LIT>LibFunc :: F32 || FInfo . getPrefix ( ) != <ISA_LIT>LibFunc :: NOPFX || ! HasNative ( FInfo . getId ( ) ) ) return false ; <ISA_LIT>LibFunc nf = FInfo ; nf . setPrefix ( <ISA_LIT>LibFunc :: NATIVE ) ;

if ( Constant * FPExpr = getFunction ( M , nf ) ) {


<s> if ( getArgType ( FInfo ) != <ISA_LIT>LibFunc :: F32 || FInfo . getPrefix ( ) != <ISA_LIT>LibFunc :: NOPFX || ! HasNative ( FInfo . getId ( ) ) ) return false ; <ISA_LIT>LibFunc nf = FInfo ; nf . setPrefix ( <ISA_LIT>LibFunc :: NATIVE ) ; if ( Constant * FPExpr = getFunction ( M , nf ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * CI << <STR_LIT> ) ; CI -> setCalledFunction ( FPExpr ) ; LLVM_DEBUG ( dbgs ( ) << * CI << '\n' ) ; return true ;


<s> bool Changed = false ; auto AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; F . printAsOperand ( dbgs ( ) , false , F . getParent ( ) ) ; dbgs ( ) << '\n' ; ) ; if ( ! EnablePreLink ) Changed |= setFastFlags ( F , Options ) ; for ( auto & BB : F ) { for ( BasicBlock :: iterator I = BB . begin ( ) , E = BB . end ( ) ; I != E ; ) { CallInst * CI = dyn_cast < CallInst > ( I ) ; ++ I ; if ( ! CI ) continue ; Function * Callee = CI -> getCalledFunction ( ) ; if ( Callee == <NUM_LIT> ) continue ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * CI << <STR_LIT> ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; F . printAsOperand ( dbgs ( ) , false , F . getParent ( ) ) ; dbgs ( ) << '\n' ; ) ; if ( ! EnablePreLink ) Changed |= setFastFlags ( F , Options ) ; for ( auto & BB : F ) { for ( BasicBlock :: iterator I = BB . begin ( ) , E = BB . end ( ) ; I != E ; ) { CallInst * CI = dyn_cast < CallInst > ( I ) ; ++ I ; if ( ! CI ) continue ; Function * Callee = CI -> getCalledFunction ( ) ; if ( Callee == <NUM_LIT> ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * CI << <STR_LIT> ; dbgs ( ) . flush ( ) ) ; if ( Simplifier . fold ( CI , AA ) ) Changed = true ;


<s> return new <ISA_LIT>SimplifyLibCalls ( Opt , TM ) ;


<s> unsigned PtrArgAS = PtrArg -> getType ( ) -> getPointerAddressSpace ( ) ; auto * PtrTy = llvm :: PointerType :: get ( PtrElemTy , PtrArgAS ) ; SmallVector < llvm :: Type * , <NUM_LIT> > ArgTys ; for ( unsigned I = <NUM_LIT> ; I != PtrArgLoc ; ++ I ) ArgTys . push_back ( CI -> getArgOperand ( I ) -> getType ( ) ) ; ArgTys . push_back ( PtrTy ) ; Name = Name + <STR_LIT> + std :: to_string ( Size ) ; auto * FTy = FunctionType :: get ( Callee -> getReturnType ( ) , ArrayRef < Type * > ( ArgTys ) , false ) ; <ISA_LIT>LibFunc NewLibFunc ( Name , FTy ) ; FunctionCallee F = <ISA_LIT>LibFunc :: getOrInsertFunction ( M , NewLibFunc ) ; if ( ! F ) return false ; auto * BCast = B . CreatePointerCast ( PtrArg , PtrTy ) ; SmallVector < Value * , <NUM_LIT> > Args ;


<s> for ( unsigned I = <NUM_LIT> ; I != PtrArgLoc ; ++ I ) ArgTys . push_back ( CI -> getArgOperand ( I ) -> getType ( ) ) ; ArgTys . push_back ( PtrTy ) ; Name = Name + <STR_LIT> + std :: to_string ( Size ) ; auto * FTy = FunctionType :: get ( Callee -> getReturnType ( ) , ArrayRef < Type * > ( ArgTys ) , false ) ; <ISA_LIT>LibFunc NewLibFunc ( Name , FTy ) ; FunctionCallee F = <ISA_LIT>LibFunc :: getOrInsertFunction ( M , NewLibFunc ) ; if ( ! F ) return false ; auto * BCast = B . CreatePointerCast ( PtrArg , PtrTy ) ; SmallVector < Value * , <NUM_LIT> > Args ; for ( unsigned I = <NUM_LIT> ; I != PtrArgLoc ; ++ I ) Args . push_back ( CI -> getArgOperand ( I ) ) ; Args . push_back ( BCast ) ; auto * NCI = B . CreateCall ( F , Args ) ; NCI -> setAttributes ( CI -> getAttributes ( ) ) ; CI -> replaceAllUsesWith ( NCI ) ; CI -> dropAllReferences ( ) ;


<s> replaceCall ( opr0 ) ; return true ; } if ( ci_opr1 == <NUM_LIT> ) { std :: vector < const Type * > ParamsTys ; ParamsTys . push_back ( opr0 -> getType ( ) ) ; Module * M = CI -> getModule ( ) ; if ( FunctionCallee FPExpr = getFunction ( M , <ISA_LIT>LibFunc ( <ISA_LIT>LibFunc :: EI_SQRT , FInfo ) ) ) { LLVM_DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * opr0 << <STR_LIT> ) ; Value * nval = CreateCallEx ( B , FPExpr , opr0 , <STR_LIT> ) ; replaceCall ( nval ) ; return true ; } } else if ( ci_opr1 == <NUM_LIT> ) { Module * M = CI -> getModule ( ) ;

if ( FunctionCallee FPExpr = getFunction ( M , <ISA_LIT>LibFunc ( <ISA_LIT>LibFunc :: EI_CBRT , FInfo ) ) ) {


<s> if ( ! CINT ) { return false ; } int ci_opr1 = ( int ) CINT -> getSExtValue ( ) ; if ( ci_opr1 == <NUM_LIT> ) { LLVM_DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * opr0 << <STR_LIT> ) ; replaceCall ( opr0 ) ; return true ; } if ( ci_opr1 == <NUM_LIT> ) { std :: vector < const Type * > ParamsTys ; ParamsTys . push_back ( opr0 -> getType ( ) ) ; Module * M = CI -> getModule ( ) ; if ( FunctionCallee FPExpr = getFunction ( M , <ISA_LIT>LibFunc ( <ISA_LIT>LibFunc :: EI_SQRT , FInfo ) ) ) { LLVM_DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * opr0 << <STR_LIT> ) ; Value * nval = CreateCallEx ( B , FPExpr , opr0 , <STR_LIT> ) ; replaceCall ( nval ) ; return true ; } } else if ( ci_opr1 == <NUM_LIT> ) { Module * M = CI -> getModule ( ) ; if ( FunctionCallee FPExpr = getFunction ( M , <ISA_LIT>LibFunc ( <ISA_LIT>LibFunc :: EI_CBRT , FInfo ) ) ) { LLVM_DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * opr0 << <STR_LIT> ) ; Value * nval = CreateCallEx ( B , FPExpr , opr0 , <STR_LIT> ) ; replaceCall ( nval ) ; return true ; } } else if ( ci_opr1 == - <NUM_LIT> ) { LLVM_DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * opr0 << <STR_LIT> ) ; Value * nval = B . CreateFDiv ( ConstantFP :: get ( opr0 -> getType ( ) , <NUM_LIT> ) , opr0 , <STR_LIT> ) ; replaceCall ( nval ) ; return true ; } else if ( ci_opr1 == - <NUM_LIT> ) { std :: vector < const Type * > ParamsTys ; ParamsTys . push_back ( opr0 -> getType ( ) ) ; Module * M = CI -> getModule ( ) ; if ( FunctionCallee FPExpr = getFunction ( M , <ISA_LIT>LibFunc ( <ISA_LIT>LibFunc :: EI_RSQRT , FInfo ) ) ) { LLVM_DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * opr0 << <STR_LIT> ) ; Value * nval = CreateCallEx ( B , FPExpr , opr0 , <STR_LIT> ) ; replaceCall ( nval ) ;


<s> AllocaInst * Alloc = B . CreateAlloca ( RetType , <NUM_LIT> , std :: string ( prefix ) + UI -> getName ( ) ) ; Alloc -> setAlignment ( MaybeAlign ( UCallee -> getParent ( ) -> getDataLayout ( ) . getTypeAllocSize ( RetType ) ) ) ;


<s> BasicBlock :: iterator ItNew = getEntryIns ( UI ) ; Function * UCallee = UI -> getCalledFunction ( ) ; Type * RetType = UCallee -> getReturnType ( ) ; B . SetInsertPoint ( & * ItNew ) ;

AllocaInst * Alloc = B . CreateAlloca ( RetType , <NUM_LIT> , std :: string ( prefix ) + UI -> getName ( ) ) ;


<s> auto * M = Callee -> getParent ( ) ; auto & Ctx = M -> getContext ( ) ; std :: string Name = std :: string ( Callee -> getName ( ) ) ; auto NumArg = CI -> getNumArgOperands ( ) ; if ( NumArg != <NUM_LIT> && NumArg != <NUM_LIT> ) return false ; auto * PacketSize = CI -> getArgOperand ( NumArg - <NUM_LIT> ) ; auto * PacketAlign = CI -> getArgOperand ( NumArg - <NUM_LIT> ) ; if ( ! isa < ConstantInt > ( PacketSize ) || ! isa < ConstantInt > ( PacketAlign ) ) return false ; unsigned Size = cast < ConstantInt > ( PacketSize ) -> getZExtValue ( ) ; Align Alignment = cast < ConstantInt > ( PacketAlign ) -> getAlignValue ( ) ; if ( Alignment != Size ) return false ; Type * PtrElemTy ; if ( Size <= <NUM_LIT> ) PtrElemTy = Type :: getIntNTy ( Ctx , Size * <NUM_LIT> ) ; else PtrElemTy = FixedVectorType :: get ( Type :: getInt64Ty ( Ctx ) , Size / <NUM_LIT> ) ; unsigned PtrArgLoc = CI -> getNumArgOperands ( ) - <NUM_LIT> ; auto PtrArg = CI -> getArgOperand ( PtrArgLoc ) ; unsigned PtrArgAS = PtrArg -> getType ( ) -> getPointerAddressSpace ( ) ; auto * PtrTy = llvm :: PointerType :: get ( PtrElemTy , PtrArgAS ) ; SmallVector < llvm :: Type * , <NUM_LIT> > ArgTys ; for ( unsigned I = <NUM_LIT> ; I != PtrArgLoc ; ++ I ) ArgTys . push_back ( CI -> getArgOperand ( I ) -> getType ( ) ) ; ArgTys . push_back ( PtrTy ) ; Name = Name + <STR_LIT> + std :: to_string ( Size ) ; auto * FTy = FunctionType :: get ( Callee -> getReturnType ( ) , ArrayRef < Type * > ( ArgTys ) , false ) ; <ISA_LIT>LibFunc NewLibFunc ( Name , FTy ) ; FunctionCallee F = <ISA_LIT>LibFunc :: getOrInsertFunction ( M , NewLibFunc ) ; if ( ! F ) return false ; auto * BCast = B . CreatePointerCast ( PtrArg , PtrTy ) ; SmallVector < Value * , <NUM_LIT> > Args ; for ( unsigned I = <NUM_LIT> ; I != PtrArgLoc ; ++ I ) Args . push_back ( CI -> getArgOperand ( I ) ) ; Args . push_back ( BCast ) ; auto * NCI = B . CreateCall ( F , Args ) ; NCI -> setAttributes ( CI -> getAttributes ( ) ) ;


<s> Function * F = CI -> getParent ( ) -> getParent ( ) ; const GCNSubtarget & ST = TM -> getSubtarget < GCNSubtarget > ( * F ) ; unsigned N = ST . getWavefrontSize ( ) ; LLVM_DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << N << <STR_LIT> ) ; CI -> replaceAllUsesWith ( ConstantInt :: get ( B . getInt32Ty ( ) , N ) ) ;


<s> StringRef Features = TM -> getTargetFeatureString ( ) ; if ( ( CPU . empty ( ) || CPU . equals_insensitive ( <STR_LIT> ) ) && ( Features . empty ( ) || Features . find_insensitive ( <STR_LIT> ) == StringRef :: npos ) ) return false ; Function * F = CI -> getParent ( ) -> getParent ( ) ; const GCNSubtarget & ST = TM -> getSubtarget < GCNSubtarget > ( * F ) ; unsigned N = ST . getWavefrontSize ( ) ; LLVM_DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << N << <STR_LIT> ) ; CI -> replaceAllUsesWith ( ConstantInt :: get ( B . getInt32Ty ( ) , N ) ) ;


<s> unsigned PtrArgAS = PtrArg -> getType ( ) -> getPointerAddressSpace ( ) ; auto * PtrTy = llvm :: PointerType :: get ( PtrElemTy , PtrArgAS ) ; SmallVector < llvm :: Type * , <NUM_LIT> > ArgTys ; for ( unsigned I = <NUM_LIT> ; I != PtrArgLoc ; ++ I ) ArgTys . push_back ( CI -> getArgOperand ( I ) -> getType ( ) ) ; ArgTys . push_back ( PtrTy ) ; Name = Name + <STR_LIT> + std :: to_string ( Size ) ; auto * FTy = FunctionType :: get ( Callee -> getReturnType ( ) , ArrayRef < Type * > ( ArgTys ) , false ) ; <ISA_LIT>LibFunc NewLibFunc ( Name , FTy ) ; FunctionCallee F = <ISA_LIT>LibFunc :: getOrInsertFunction ( M , NewLibFunc ) ; if ( ! F ) return false ; auto * BCast = B . CreatePointerCast ( PtrArg , PtrTy ) ; SmallVector < Value * , <NUM_LIT> > Args ; for ( unsigned I = <NUM_LIT> ; I != PtrArgLoc ; ++ I ) Args . push_back ( CI -> getArgOperand ( I ) ) ; Args . push_back ( BCast ) ; auto * NCI = B . CreateCall ( F , Args ) ; NCI -> setAttributes ( CI -> getAttributes ( ) ) ; CI -> replaceAllUsesWith ( NCI ) ; CI -> dropAllReferences ( ) ; CI -> eraseFromParent ( ) ;


<s> if ( NumArg != <NUM_LIT> && NumArg != <NUM_LIT> ) return false ; auto * PacketSize = CI -> getArgOperand ( NumArg - <NUM_LIT> ) ; auto * PacketAlign = CI -> getArgOperand ( NumArg - <NUM_LIT> ) ; if ( ! isa < ConstantInt > ( PacketSize ) || ! isa < ConstantInt > ( PacketAlign ) ) return false ; unsigned Size = cast < ConstantInt > ( PacketSize ) -> getZExtValue ( ) ; unsigned Align = cast < ConstantInt > ( PacketAlign ) -> getZExtValue ( ) ; if ( Size != Align || ! isPowerOf2_32 ( Size ) ) return false ; Type * PtrElemTy ; if ( Size <= <NUM_LIT> ) PtrElemTy = Type :: getIntNTy ( Ctx , Size * <NUM_LIT> ) ; else PtrElemTy = VectorType :: get ( Type :: getInt64Ty ( Ctx ) , Size / <NUM_LIT> ) ; unsigned PtrArgLoc = CI -> getNumArgOperands ( ) - <NUM_LIT> ; auto PtrArg = CI -> getArgOperand ( PtrArgLoc ) ; unsigned PtrArgAS = PtrArg -> getType ( ) -> getPointerAddressSpace ( ) ; auto * PtrTy = llvm :: PointerType :: get ( PtrElemTy , PtrArgAS ) ; SmallVector < llvm :: Type * , <NUM_LIT> > ArgTys ; for ( unsigned I = <NUM_LIT> ; I != PtrArgLoc ; ++ I ) ArgTys . push_back ( CI -> getArgOperand ( I ) -> getType ( ) ) ; ArgTys . push_back ( PtrTy ) ; Name = Name + <STR_LIT> + std :: to_string ( Size ) ; auto * FTy = FunctionType :: get ( Callee -> getReturnType ( ) , ArrayRef < Type * > ( ArgTys ) , false ) ; <ISA_LIT>LibFunc NewLibFunc ( Name , FTy ) ; FunctionCallee F = <ISA_LIT>LibFunc :: getOrInsertFunction ( M , NewLibFunc ) ; if ( ! F ) return false ; auto * BCast = B . CreatePointerCast ( PtrArg , PtrTy ) ; SmallVector < Value * , <NUM_LIT> > Args ; for ( unsigned I = <NUM_LIT> ; I != PtrArgLoc ; ++ I ) Args . push_back ( CI -> getArgOperand ( I ) ) ;


<s> int numArgs = ( int ) aCI -> getNumArgOperands ( ) ; if ( numArgs > <NUM_LIT> ) return false ; Constant * copr0 = nullptr ; Constant * copr1 = nullptr ; Constant * copr2 = nullptr ; if ( numArgs > <NUM_LIT> ) { if ( ( copr0 = dyn_cast < Constant > ( aCI -> getArgOperand ( <NUM_LIT> ) ) ) == nullptr ) return false ; } if ( numArgs > <NUM_LIT> ) { if ( ( copr1 = dyn_cast < Constant > ( aCI -> getArgOperand ( <NUM_LIT> ) ) ) == nullptr ) { if ( FInfo . getId ( ) != <ISA_LIT>LibFunc :: EI_<ISA_LIT>NCOS ) return false ; } } if ( numArgs > <NUM_LIT> ) { if ( ( copr2 = dyn_cast < Constant > ( aCI -> getArgOperand ( <NUM_LIT> ) ) ) == nullptr ) return false ; } double DVal0 [ <NUM_LIT> ] , DVal1 [ <NUM_LIT> ] ; bool hasTwoResults = ( FInfo . getId ( ) == <ISA_LIT>LibFunc :: EI_<ISA_LIT>NCOS ) ; if ( getVecSize ( FInfo ) == <NUM_LIT> ) { if ( ! evaluateScalarMathFunc ( FInfo , DVal0 [ <NUM_LIT> ] , DVal1 [ <NUM_LIT> ] , copr0 , copr1 , copr2 ) ) { return false ; } } else { ConstantDataVector * CDV0 = dyn_cast_or_null < ConstantDataVector > ( copr0 ) ; ConstantDataVector * CDV1 = dyn_cast_or_null < ConstantDataVector > ( copr1 ) ; ConstantDataVector * CDV2 = dyn_cast_or_null < ConstantDataVector > ( copr2 ) ; for ( int i = <NUM_LIT> ; i < getVecSize ( FInfo ) ; ++ i ) { Constant * celt0 = CDV0 ? CDV0 -> getElementAsConstant ( i ) : nullptr ; Constant * celt1 = CDV1 ? CDV1 -> getElementAsConstant ( i ) : nullptr ; Constant * celt2 = CDV2 ? CDV2 -> getElementAsConstant ( i ) : nullptr ; if ( ! evaluateScalarMathFunc ( FInfo , DVal0 [ i ] , DVal1 [ i ] , celt0 , celt1 , celt2 ) ) { return false ; } } } LLVMContext & context = CI -> getParent ( ) -> getParent ( ) -> getContext ( ) ; Constant * nval0 , * nval1 ; if ( getVecSize ( FInfo ) == <NUM_LIT> ) { nval0 = ConstantFP :: get ( CI -> getType ( ) , DVal0 [ <NUM_LIT> ] ) ; if ( hasTwoResults ) nval1 = ConstantFP :: get ( CI -> getType ( ) , DVal1 [ <NUM_LIT> ] ) ; } else { if ( getArgType ( FInfo ) == <ISA_LIT>LibFunc :: F32 ) { SmallVector < float , <NUM_LIT> > FVal0 , FVal1 ; for ( int i = <NUM_LIT> ; i < getVecSize ( FInfo ) ; ++ i ) FVal0 . push_back ( ( float ) DVal0 [ i ] ) ; ArrayRef < float > tmp0 ( FVal0 ) ;

nval0 = ConstantDataVector :: get ( context , tmp0 ) ;


<s> return ! EnablePreLink && fold_wavefrontsize ( CI , B ) ; } FuncInfo FInfo ; if ( ! parseFunctionName ( Callee -> getName ( ) , & FInfo ) ) return false ; if ( CI -> getNumArgOperands ( ) != FInfo . getNumArgs ( ) ) return false ; if ( TDOFold ( CI , FInfo ) ) return true ; if ( isUnsafeMath ( CI ) && evaluateCall ( CI , FInfo ) ) return true ; switch ( FInfo . getId ( ) ) { case <ISA_LIT>LibFunc :: EI_RECIP : assert ( ( FInfo . getPrefix ( ) == <ISA_LIT>LibFunc :: NATIVE || FInfo . getPrefix ( ) == <ISA_LIT>LibFunc :: HALF ) && <STR_LIT> ) ; return ( getVecSize ( FInfo ) != <NUM_LIT> ) ? false : fold_recip ( CI , B , FInfo ) ; case <ISA_LIT>LibFunc :: EI_DIVIDE : assert ( ( FInfo . getPrefix ( ) == <ISA_LIT>LibFunc :: NATIVE || FInfo . getPrefix ( ) == <ISA_LIT>LibFunc :: HALF ) && <STR_LIT> ) ; return ( getVecSize ( FInfo ) != <NUM_LIT> ) ? false : fold_divide ( CI , B , FInfo ) ; case <ISA_LIT>LibFunc :: EI_POW : case <ISA_LIT>LibFunc :: EI_POWR : case <ISA_LIT>LibFunc :: EI_POWN : return fold_pow ( CI , B , FInfo ) ; case <ISA_LIT>LibFunc :: EI_ROOTN : return ( getVecSize ( FInfo ) != <NUM_LIT> ) ? false : fold_rootn ( CI , B , FInfo ) ; case <ISA_LIT>LibFunc :: EI_FMA : case <ISA_LIT>LibFunc :: EI_MAD : case <ISA_LIT>LibFunc :: EI_NFMA : return ( getVecSize ( FInfo ) != <NUM_LIT> ) ? false : fold_fma_mad ( CI , B , FInfo ) ; case <ISA_LIT>LibFunc :: EI_SQRT : return isUnsafeMath ( CI ) && fold_sqrt ( CI , B , FInfo ) ; case <ISA_LIT>LibFunc :: EI_COS : case <ISA_LIT>LibFunc :: EI_<ISA_LIT>N : if ( ( getArgType ( FInfo ) == <ISA_LIT>LibFunc :: F32 || getArgType ( FInfo ) == <ISA_LIT>LibFunc :: F64 ) && ( FInfo . getPrefix ( ) == <ISA_LIT>LibFunc :: NOPFX ) ) return fold_sincos ( CI , B , AA ) ; break ; case <ISA_LIT>LibFunc :: EI_READ_PIPE_2 : case <ISA_LIT>LibFunc :: EI_READ_PIPE_4 :


<s> bool <ISA_LIT>LibCalls :: fold_read_write_pipe ( CallInst * CI , IRBuilder < > & B , FuncInfo & FInfo ) { auto * Callee = CI -> getCalledFunction ( ) ; if ( ! Callee -> isDeclaration ( ) ) return false ; assert ( Callee -> hasName ( ) && <STR_LIT> ) ; auto * M = Callee -> getParent ( ) ; auto & Ctx = M -> getContext ( ) ; std :: string Name = std :: string ( Callee -> getName ( ) ) ; auto NumArg = CI -> getNumArgOperands ( ) ; if ( NumArg != <NUM_LIT> && NumArg != <NUM_LIT> ) return false ; auto * PacketSize = CI -> getArgOperand ( NumArg - <NUM_LIT> ) ; auto * PacketAlign = CI -> getArgOperand ( NumArg - <NUM_LIT> ) ; if ( ! isa < ConstantInt > ( PacketSize ) || ! isa < ConstantInt > ( PacketAlign ) ) return false ; unsigned Size = cast < ConstantInt > ( PacketSize ) -> getZExtValue ( ) ; unsigned Align = cast < ConstantInt > ( PacketAlign ) -> getZExtValue ( ) ; if ( Size != Align || ! isPowerOf2_32 ( Size ) ) return false ; Type * PtrElemTy ;

if ( Size <= <NUM_LIT> ) PtrElemTy = Type :: getIntNTy ( Ctx , Size * <NUM_LIT> ) ;


<s> if ( cast < Instruction > ( BBI ) == XI ) { UI = XI ; break ; } } if ( UI ) break ; } if ( ! UI ) return false ; <ISA_LIT>LibFunc nf ( <ISA_LIT>LibFunc :: EI_<ISA_LIT>NCOS , fInfo ) ; nf . getLeads ( ) [ <NUM_LIT> ] . PtrKind = <ISA_LIT>LibFunc :: getEPtrKindFromAddrSpace ( <ISA_LIT>AS :: FLAT_ADDRESS ) ; FunctionCallee Fsincos = getFunction ( M , nf ) ; if ( ! Fsincos ) return false ; BasicBlock :: iterator ItOld = B . GetInsertPoint ( ) ; AllocaInst * Alloc = insertAlloca ( UI , B , <STR_LIT> ) ; B . SetInsertPoint ( UI ) ; Value * P = Alloc ; Type * PTy = Fsincos . getFunctionType ( ) -> getParamType ( <NUM_LIT> ) ; if ( PTy -> getPointerAddressSpace ( ) != <ISA_LIT>AS :: PRIVATE_ADDRESS ) P = B . CreateAddrSpaceCast ( Alloc , PTy ) ; CallInst * Call = CreateCallEx2 ( B , Fsincos , UI -> getArgOperand ( <NUM_LIT> ) , P ) ; LLVM_DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << * UI << <STR_LIT> << * Call << <STR_LIT> ) ; if ( ! isSin ) { B . SetInsertPoint ( & * ItOld ) ; UI -> replaceAllUsesWith ( & * Call ) ;

Instruction * Reload = B . CreateLoad ( Alloc -> getAllocatedType ( ) , Alloc ) ;


<s> Function * F = CI -> getParent ( ) -> getParent ( ) ; const GCNSubtarget & ST = TM -> getSubtarget < GCNSubtarget > ( * F ) ; unsigned N = ST . getWavefrontSize ( ) ; LLVM_DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << N << <STR_LIT> ) ; CI -> replaceAllUsesWith ( ConstantInt :: get ( B . getInt32Ty ( ) , N ) ) ; CI -> eraseFromParent ( ) ; return true ;


<s> StringRef Features = TM -> getTargetFeatureString ( ) ; if ( ( CPU . empty ( ) || CPU . equals_lower ( <STR_LIT> ) ) && ( Features . empty ( ) || Features . find_lower ( <STR_LIT> ) == StringRef :: npos ) ) return false ; Function * F = CI -> getParent ( ) -> getParent ( ) ; const GCNSubtarget & ST = TM -> getSubtarget < GCNSubtarget > ( * F ) ; unsigned N = ST . getWavefrontSize ( ) ; LLVM_DEBUG ( errs ( ) << <STR_LIT> << * CI << <STR_LIT> << N << <STR_LIT> ) ; CI -> replaceAllUsesWith ( ConstantInt :: get ( B . getInt32Ty ( ) , N ) ) ;


<s> bool <ISA_LIT>LibCalls :: isUnsafeMath ( const CallInst * CI ) const { if ( auto Op = dyn_cast < FPMathOperator > ( CI ) ) if ( Op -> isFast ( ) ) return true ; const Function * F = CI -> getParent ( ) -> getParent ( ) ;

Attribute Attr = F -> getFnAttribute ( <STR_LIT> ) ;


<s> const Function * F = CI -> getParent ( ) -> getParent ( ) ; Attribute Attr = F -> getFnAttribute ( <STR_LIT> ) ;


<s> return <ISA_LIT>LibFunc :: parse ( FMangledName , * FInfo ) ;


<s> FInfo . setPrefix ( <ISA_LIT>LibFunc :: NATIVE ) ; FunctionCallee F = getFunction ( aCI -> getModule ( ) , FInfo ) ; if ( ! F ) return false ; aCI -> setCalledFunction ( F ) ; DEBUG_WITH_TYPE ( <STR_LIT> , dbgs ( ) << <STR_LIT> << * aCI << <STR_LIT> ) ;


<s> return AllNative || std :: find ( UseNative . begin ( ) , UseNative . end ( ) , F ) != UseNative . end ( ) ;