<s> auto SrcPtrInfo = MachinePointerInfo :: getStack ( MF , Offset ) ; SDValue Ptr = DAG . getFrameIndex ( FI , <STR_LIT> ) ;

return DAG . getLoad ( VT , SL , DAG . getEntryNode ( ) , Ptr , SrcPtrInfo , <NUM_LIT> , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;


<s> SDValue <ISA_LIT>TargetLowering :: loadStackInputValue ( SelectionDAG & DAG , EVT VT , const SDLoc & SL , int64_t Offset ) const { MachineFunction & MF = DAG . getMachineFunction ( ) ; MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; int FI = MFI . CreateFixedObject ( VT . getStoreSize ( ) , Offset , true ) ;

auto SrcPtrInfo = MachinePointerInfo :: getStack ( MF , Offset ) ;


<s> DiagnosticInfoUnsupported BadLDSDecl ( Fn , <STR_LIT> , SDLoc ( Op ) . getDebugLoc ( ) ) ; DAG . getContext ( ) -> diagnose ( BadLDSDecl ) ; } assert ( G -> getOffset ( ) == <NUM_LIT> && <STR_LIT> ) ; if ( ! hasDefinedInitializer ( GV ) ) { unsigned Offset = MFI -> allocateLDSGlobal ( DL , * GV ) ; return DAG . getConstant ( Offset , SDLoc ( Op ) , Op . getValueType ( ) ) ; } } const Function & Fn = DAG . getMachineFunction ( ) . getFunction ( ) ; DiagnosticInfoUnsupported BadInit ( Fn , <STR_LIT> , SDLoc ( Op ) . getDebugLoc ( ) ) ; DAG . getContext ( ) -> diagnose ( BadInit ) ;


<s> case <STR_LIT> : return LowerFLOG ( Op , DAG , <NUM_LIT> / <ISA_LIT>_LOG2E_F ) ; case <STR_LIT> : return LowerFLOG ( Op , DAG , <ISA_LIT>_LN2_F / <ISA_LIT>_LN10_F ) ; case <STR_LIT> : return Lower<ISA_LIT>NT_TO_FP ( Op , DAG ) ; case <STR_LIT> : return LowerUINT_TO_FP ( Op , DAG ) ; case <STR_LIT> : return LowerFP_TO_FP16 ( Op , DAG ) ; case <STR_LIT> : return LowerFP_TO_<ISA_LIT>NT ( Op , DAG ) ; case <STR_LIT> : return LowerFP_TO_UINT ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return LowerCTLZ_CTTZ ( Op , DAG ) ; case <STR_LIT> : return LowerDYNAMIC_STACKALLOC ( Op , DAG ) ; } return Op ;


<s> if ( Flags . isDefined ( ) ) return Flags . hasNoSignedZeros ( ) ;


<s> const auto Flags = Op . getNode ( ) -> getFlags ( ) ; if ( Flags . isDefined ( ) ) return Flags . hasNoSignedZeros ( ) ; return false ;


<s> bool mergeStoresAfterLegalization ( ) const override { return false ;


<s> KnownBits Known ; EVT VT = Op . getValueType ( ) ; DAG . computeKnownBits ( Op , Known ) ; return VT . getSizeInBits ( ) - Known . countMinLeadingZeros ( ) ;


<s> case <STR_LIT> : { if ( DCI . getDAGCombineLevel ( ) < AfterLegalizeDAG ) break ; return performShlCombine ( N , DCI ) ; } case <STR_LIT> : { if ( DCI . getDAGCombineLevel ( ) < AfterLegalizeDAG ) break ; return performSrlCombine ( N , DCI ) ; } case <STR_LIT> : { if ( DCI . getDAGCombineLevel ( ) < AfterLegalizeDAG ) break ; return performSraCombine ( N , DCI ) ; } case <STR_LIT> : return performTruncateCombine ( N , DCI ) ; case <STR_LIT> : return performMulCombine ( N , DCI ) ; case <STR_LIT> : return performMulhsCombine ( N , DCI ) ; case <STR_LIT> : return performMulhuCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { simplifyI24 ( N , <NUM_LIT> , DCI ) || simplifyI24 ( N , <NUM_LIT> , DCI ) ; return SDValue ( ) ; } case <STR_LIT> : case <STR_LIT> : return performMulLoHi24Combine ( N , DCI ) ; case <STR_LIT> : return performSelectCombine ( N , DCI ) ; case <STR_LIT> : return performFNegCombine ( N , DCI ) ; case <STR_LIT> : return performFAbsCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : { assert ( ! N -> getValueType ( <NUM_LIT> ) . isVector ( ) && <STR_LIT> ) ; ConstantSDNode * Width = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( ! Width ) break ; uint32_t WidthVal = Width -> getZExtValue ( ) & <NUM_LIT> ; if ( WidthVal == <NUM_LIT> ) return DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; ConstantSDNode * Offset = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( ! Offset ) break ; SDValue BitsFrom = N -> getOperand ( <NUM_LIT> ) ; uint32_t OffsetVal = Offset -> getZExtValue ( ) & <NUM_LIT> ; bool Signed = N -> getOpcode ( ) == <STR_LIT> ; if ( OffsetVal == <NUM_LIT> ) { unsigned SignBits = Signed ? ( <NUM_LIT> - WidthVal + <NUM_LIT> ) : ( <NUM_LIT> - WidthVal ) ; unsigned OpSignBits = DAG . ComputeNumSignBits ( BitsFrom ) ;

if ( OpSignBits >= SignBits ) return BitsFrom ;


<s> } case <STR_LIT> : case <STR_LIT> : { if ( ! mayIgnoreSignedZero ( N0 ) ) return SDValue ( ) ; SDValue LHS = N0 . getOperand ( <NUM_LIT> ) ; SDValue MHS = N0 . getOperand ( <NUM_LIT> ) ; SDValue RHS = N0 . getOperand ( <NUM_LIT> ) ; if ( LHS . getOpcode ( ) == <STR_LIT> ) LHS = LHS . getOperand ( <NUM_LIT> ) ; else if ( MHS . getOpcode ( ) == <STR_LIT> ) MHS = MHS . getOperand ( <NUM_LIT> ) ; else MHS = DAG . getNode ( <STR_LIT> , SL , VT , MHS ) ; if ( RHS . getOpcode ( ) != <STR_LIT> ) RHS = DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; else RHS = RHS . getOperand ( <NUM_LIT> ) ; SDValue Res = DAG . getNode ( Opc , SL , VT , LHS , MHS , RHS ) ; if ( ! N0 . hasOneUse ( ) ) DAG . ReplaceAllUsesWith ( N0 , DAG . getNode ( <STR_LIT> , SL , VT , Res ) ) ; return Res ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue LHS = N0 . getOperand ( <NUM_LIT> ) ; SDValue RHS = N0 . getOperand ( <NUM_LIT> ) ; if ( isConstantFPZero ( RHS ) ) return SDValue ( ) ; SDValue NegLHS = DAG . getNode ( <STR_LIT> , SL , VT , LHS ) ; SDValue NegRHS = DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; unsigned Opposite = inverseMinMax ( Opc ) ; SDValue Res = DAG . getNode ( Opposite , SL , VT , NegLHS , NegRHS , N0 -> getFlags ( ) ) ; if ( ! N0 . hasOneUse ( ) ) DAG . ReplaceAllUsesWith ( N0 , DAG . getNode ( <STR_LIT> , SL , VT , Res ) ) ; return Res ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue CvtSrc = N0 . getOperand ( <NUM_LIT> ) ; if ( CvtSrc . getOpcode ( ) == <STR_LIT> ) { return DAG . getNode ( Opc , SL , VT , CvtSrc . getOperand ( <NUM_LIT> ) ) ; } if ( ! N0 . hasOneUse ( ) ) return SDValue ( ) ; SDValue Neg = DAG . getNode ( <STR_LIT> , SL , CvtSrc . getValueType ( ) , CvtSrc ) ; return DAG . getNode ( Opc , SL , VT , Neg , N0 -> getFlags ( ) ) ; } case <STR_LIT> : { SDValue CvtSrc = N0 . getOperand ( <NUM_LIT> ) ;


<s> std :: tie ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) = expandUnalignedLoad ( LN , DAG ) ; return DAG . getMergeValues ( Ops , SDLoc ( N ) ) ; } if ( ! IsFast ) return SDValue ( ) ; } if ( ! shouldCombineMemoryType ( VT ) ) return SDValue ( ) ; EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VT ) ; SDValue NewLoad = DAG . getLoad ( NewVT , SL , LN -> getChain ( ) , LN -> getBasePtr ( ) , LN -> getMemOperand ( ) ) ;

SDValue BC = DAG . getNode ( <STR_LIT> , SL , VT , NewLoad ) ;


<s> SDLoc SL ( N ) ; SDValue MulLo = DAG . getNode ( MulLoOpc , SL , <STR_LIT> , N0 , N1 ) ; SDValue MulHi = DAG . getNode ( MulHiOpc , SL , <STR_LIT> , N0 , N1 ) ;

return DAG . getMergeValues ( { MulLo , MulHi } , SL ) ;


<s> SDValue <ISA_LIT>TargetLowering :: performMulLoHi24Combine ( SDNode * N , DAGCombinerInfo & DCI ) const { SelectionDAG & DAG = DCI . DAG ; if ( simplifyI24 ( N , <NUM_LIT> , DCI ) || simplifyI24 ( N , <NUM_LIT> , DCI ) ) return SDValue ( ) ; SDValue N0 = N -> getOperand ( <NUM_LIT> ) ; SDValue N1 = N -> getOperand ( <NUM_LIT> ) ; bool Signed = ( N -> getOpcode ( ) == <STR_LIT> ) ; unsigned MulLoOpc = Signed ? <STR_LIT> : <STR_LIT> ; unsigned MulHiOpc = Signed ? <STR_LIT> : <STR_LIT> ;


<s> if ( Cond . getOpcode ( ) != <STR_LIT> ) return SDValue ( ) ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; SDValue LHS = Cond . getOperand ( <NUM_LIT> ) ; SDValue RHS = Cond . getOperand ( <NUM_LIT> ) ; SDValue CC = Cond . getOperand ( <NUM_LIT> ) ; SDValue True = N -> getOperand ( <NUM_LIT> ) ; SDValue False = N -> getOperand ( <NUM_LIT> ) ; if ( Cond . hasOneUse ( ) ) { SelectionDAG & DAG = DCI . DAG ; if ( ( DAG . isConstantValueOfAnyType ( True ) || DAG . isConstantValueOfAnyType ( True ) ) && ( ! DAG . isConstantValueOfAnyType ( False ) && ! DAG . isConstantValueOfAnyType ( False ) ) ) { SDLoc SL ( N ) ; <STR_LIT> NewCC = getSetCCInverse ( cast < CondCodeSDNode > ( CC ) -> get ( ) , LHS . getValueType ( ) . isInteger ( ) ) ; SDValue NewCond = DAG . getSetCC ( SL , Cond . getValueType ( ) , LHS , RHS , NewCC ) ; return DAG . getNode ( <STR_LIT> , SL , VT , NewCond , False , True ) ;


<s> EVT VT = N -> getValueType ( <NUM_LIT> ) ; ConstantSDNode * RHS = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( ! RHS ) return SDValue ( ) ; SDValue LHS = N -> getOperand ( <NUM_LIT> ) ; unsigned RHSVal = RHS -> getZExtValue ( ) ; if ( ! RHSVal ) return LHS ; SDLoc SL ( N ) ; SelectionDAG & DAG = DCI . DAG ; switch ( LHS -> getOpcode ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue X = LHS -> getOperand ( <NUM_LIT> ) ; if ( VT == <STR_LIT> && RHSVal == <NUM_LIT> && X . getValueType ( ) == <STR_LIT> && isOperationLegal ( <STR_LIT> , <STR_LIT> ) ) { SDValue Vec = DAG . getBuildVector ( <STR_LIT> , SL , { DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) , LHS -> getOperand ( <NUM_LIT> ) } ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Vec ) ; } if ( VT != <STR_LIT> ) break ; KnownBits Known ; DAG . computeKnownBits ( X , Known ) ; unsigned LZ = Known . countMinLeadingZeros ( ) ; if ( LZ < RHSVal ) break ; EVT XVT = X . getValueType ( ) ; SDValue Shl = DAG . getNode ( <STR_LIT> , SL , XVT , X , SDValue ( RHS , <NUM_LIT> ) ) ; return DAG . getZExtOrTrunc ( Shl , SL , VT ) ; } }

if ( VT != <STR_LIT> ) return SDValue ( ) ;


<s> ConstantSDNode * RHS = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( ! RHS ) return SDValue ( ) ; SDValue LHS = N -> getOperand ( <NUM_LIT> ) ; unsigned RHSVal = RHS -> getZExtValue ( ) ; if ( ! RHSVal ) return LHS ; SDLoc SL ( N ) ; SelectionDAG & DAG = DCI . DAG ; switch ( LHS -> getOpcode ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue X = LHS -> getOperand ( <NUM_LIT> ) ; if ( VT == <STR_LIT> && RHSVal == <NUM_LIT> && X . getValueType ( ) == <STR_LIT> && isOperationLegal ( <STR_LIT> , <STR_LIT> ) ) {

SDValue Vec = DAG . getBuildVector ( <STR_LIT> , SL , { DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) , LHS -> getOperand ( <NUM_LIT> ) } ) ;


<s> SDValue <ISA_LIT>TargetLowering :: performSrlCombine ( SDNode * N , DAGCombinerInfo & DCI ) const { if ( N -> getValueType ( <NUM_LIT> ) != <STR_LIT> ) return SDValue ( ) ; const ConstantSDNode * RHS = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( ! RHS ) return SDValue ( ) ; unsigned ShiftAmt = RHS -> getZExtValue ( ) ; if ( ShiftAmt < <NUM_LIT> ) return SDValue ( ) ; SelectionDAG & DAG = DCI . DAG ; SDLoc SL ( N ) ; SDValue One = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; SDValue Zero = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; SDValue VecOp = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , N -> getOperand ( <NUM_LIT> ) ) ; SDValue Hi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , VecOp , One ) ; SDValue NewConst = DAG . getConstant ( ShiftAmt - <NUM_LIT> , SL , <STR_LIT> ) ; SDValue NewShift = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Hi , NewConst ) ; SDValue BuildPair = DAG . getBuildVector ( <STR_LIT> , SL , { NewShift , Zero } ) ;

return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , BuildPair ) ;


<s> SDValue One = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; SDValue Zero = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; SDValue VecOp = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , N -> getOperand ( <NUM_LIT> ) ) ; SDValue Hi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , VecOp , One ) ; SDValue NewConst = DAG . getConstant ( ShiftAmt - <NUM_LIT> , SL , <STR_LIT> ) ; SDValue NewShift = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Hi , NewConst ) ; SDValue BuildPair = DAG . getBuildVector ( <STR_LIT> , SL , { NewShift , Zero } ) ;


<s> if ( SN -> isVolatile ( ) || ! <STR_LIT> ( SN ) ) return SDValue ( ) ; EVT VT = SN -> getMemoryVT ( ) ; unsigned Size = VT . getStoreSize ( ) ; SDLoc SL ( N ) ; SelectionDAG & DAG = DCI . DAG ; unsigned Align = SN -> getAlignment ( ) ; if ( Align < Size && isTypeLegal ( VT ) ) { bool IsFast ; unsigned AS = SN -> getAddressSpace ( ) ; if ( ! allowsMisalignedMemoryAccesses ( VT , AS , Align , & IsFast ) ) { if ( VT . isVector ( ) ) return scalarizeVectorStore ( SN , DAG ) ;

return expandUnalignedStore ( SN , DAG ) ;


<s> unsigned AS = SN -> getAddressSpace ( ) ; if ( ! allowsMisalignedMemoryAccesses ( VT , AS , Align , & IsFast ) ) { if ( VT . isVector ( ) ) return scalarizeVectorStore ( SN , DAG ) ; return expandUnalignedStore ( SN , DAG ) ; } if ( ! IsFast ) return SDValue ( ) ; } if ( ! shouldCombineMemoryType ( VT ) ) return SDValue ( ) ; EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VT ) ;


<s> } } } } if ( VT . getScalarSizeInBits ( ) < <NUM_LIT> ) { EVT SrcVT = Src . getValueType ( ) ; if ( SrcVT . getScalarSizeInBits ( ) > <NUM_LIT> && ( Src . getOpcode ( ) == <STR_LIT> || Src . getOpcode ( ) == <STR_LIT> || Src . getOpcode ( ) == <STR_LIT> ) ) { SDValue Amt = Src . getOperand ( <NUM_LIT> ) ; KnownBits Known ; DAG . computeKnownBits ( Amt , Known ) ; unsigned Size = VT . getScalarSizeInBits ( ) ; if ( ( Known . isConstant ( ) && Known . getConstant ( ) . ule ( Size ) ) || ( Known . getBitWidth ( ) - Known . countMinLeadingZeros ( ) <= Log2_32 ( Size ) ) ) { EVT MidVT = VT . isVector ( ) ? EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , VT . getVectorNumElements ( ) ) : <STR_LIT> ; EVT NewShiftVT = getShiftAmountTy ( MidVT , DAG . getDataLayout ( ) ) ; SDValue Trunc = DAG . getNode ( <STR_LIT> , SL , MidVT , Src . getOperand ( <NUM_LIT> ) ) ; DCI . AddToWorklist ( Trunc . getNode ( ) ) ; if ( Amt . getValueType ( ) != NewShiftVT ) { Amt = DAG . getZExtOrTrunc ( Amt , SL , NewShiftVT ) ; DCI . AddToWorklist ( Amt . getNode ( ) ) ; } SDValue ShrunkShift = DAG . getNode ( Src . getOpcode ( ) , SL , MidVT , Trunc , Amt ) ; return DAG . getNode ( <STR_LIT> , SL , VT , ShrunkShift ) ;


<s> SDValue Vec = Src . getOperand ( <NUM_LIT> ) ; if ( Vec . getOpcode ( ) == <STR_LIT> ) { SDValue Elt0 = Vec . getOperand ( <NUM_LIT> ) ; EVT EltVT = Elt0 . getValueType ( ) ; if ( VT . getSizeInBits ( ) <= EltVT . getSizeInBits ( ) ) { if ( EltVT . isFloatingPoint ( ) ) { Elt0 = DAG . getNode ( <STR_LIT> , SL , EltVT . changeTypeToInteger ( ) , Elt0 ) ; } return DAG . getNode ( <STR_LIT> , SL , VT , Elt0 ) ; } } } if ( Src . getOpcode ( ) == <STR_LIT> && ! VT . isVector ( ) ) { if ( auto K = isConstOrConstSplat ( Src . getOperand ( <NUM_LIT> ) ) ) { if ( <NUM_LIT> * K -> getZExtValue ( ) == Src . getValueType ( ) . getScalarSizeInBits ( ) ) { SDValue BV = stripBitcast ( Src . getOperand ( <NUM_LIT> ) ) ; if ( BV . getOpcode ( ) == <STR_LIT> && BV . getValueType ( ) . getVectorNumElements ( ) == <NUM_LIT> ) {

SDValue SrcElt = BV . getOperand ( <NUM_LIT> ) ;


<s> APInt Demanded = APInt :: getLowBitsSet ( VT . getSizeInBits ( ) , <NUM_LIT> ) ; APInt KnownZero , KnownOne ; TargetLowering :: TargetLoweringOpt TLO ( DAG , true , true ) ; if ( TLI . SimplifyDemandedBits ( Node24 , OpIdx , Demanded , DCI , TLO ) ) return true ; return false ;


<s> EVT VT = Op . getValueType ( ) ; APInt Demanded = APInt :: getLowBitsSet ( VT . getSizeInBits ( ) , <NUM_LIT> ) ; APInt KnownZero , KnownOne ; TargetLowering :: TargetLoweringOpt TLO ( DAG , true , true ) ; if ( TLI . SimplifyDemandedBits ( Node24 , OpIdx , Demanded , DCI , TLO ) ) return true ;


<s> SDValue <ISA_LIT>TargetLowering :: SplitVectorLoad ( const SDValue Op , SelectionDAG & DAG ) const { LoadSDNode * Load = cast < LoadSDNode > ( Op ) ; EVT VT = Op . getValueType ( ) ; if ( VT . getVectorNumElements ( ) == <NUM_LIT> ) return scalarizeVectorLoad ( Load , DAG ) ; SDValue BasePtr = Load -> getBasePtr ( ) ; EVT MemVT = Load -> getMemoryVT ( ) ; SDLoc SL ( Op ) ; const MachinePointerInfo & SrcValue = Load -> getMemOperand ( ) -> getPointerInfo ( ) ; EVT LoVT , HiVT ; EVT LoMemVT , HiMemVT ; SDValue Lo , Hi ; std :: tie ( LoVT , HiVT ) = DAG . GetSplitDestVTs ( VT ) ; std :: tie ( LoMemVT , HiMemVT ) = DAG . GetSplitDestVTs ( MemVT ) ;

std :: tie ( Lo , Hi ) = DAG . SplitVector ( Op , SL , LoVT , HiVT ) ;


<s> unsigned Size = LoMemVT . getStoreSize ( ) ; unsigned BaseAlign = Load -> getAlignment ( ) ; unsigned HiAlign = MinAlign ( BaseAlign , Size ) ; SDValue LoLoad = DAG . getExtLoad ( Load -> getExtensionType ( ) , SL , LoVT , Load -> getChain ( ) , BasePtr , SrcValue , LoMemVT , BaseAlign , Load -> getMemOperand ( ) -> getFlags ( ) ) ; SDValue HiPtr = DAG . getObjectPtrOffset ( SL , BasePtr , Size ) ; SDValue HiLoad = DAG . getExtLoad ( Load -> getExtensionType ( ) , SL , HiVT , Load -> getChain ( ) , HiPtr , SrcValue . getWithOffset ( LoMemVT . getStoreSize ( ) ) , HiMemVT , HiAlign , Load -> getMemOperand ( ) -> getFlags ( ) ) ; SDValue Ops [ ] = { DAG . getNode ( <STR_LIT> , SL , VT , LoLoad , HiLoad ) , DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LoLoad . getValue ( <NUM_LIT> ) , HiLoad . getValue ( <NUM_LIT> ) ) } ;

return DAG . getMergeValues ( Ops , SL ) ;


<s> std :: tie ( LoVT , HiVT ) = DAG . GetSplitDestVTs ( VT ) ; std :: tie ( LoMemVT , HiMemVT ) = DAG . GetSplitDestVTs ( MemVT ) ; std :: tie ( Lo , Hi ) = DAG . SplitVector ( Val , SL , LoVT , HiVT ) ; SDValue HiPtr = DAG . getObjectPtrOffset ( SL , BasePtr , LoMemVT . getStoreSize ( ) ) ; const MachinePointerInfo & SrcValue = Store -> getMemOperand ( ) -> getPointerInfo ( ) ; unsigned BaseAlign = Store -> getAlignment ( ) ; unsigned Size = LoMemVT . getStoreSize ( ) ; unsigned HiAlign = MinAlign ( BaseAlign , Size ) ; SDValue LoStore = DAG . getTruncStore ( Chain , SL , Lo , BasePtr , SrcValue , LoMemVT , BaseAlign , Store -> getMemOperand ( ) -> getFlags ( ) ) ;

SDValue HiStore = DAG . getTruncStore ( Chain , SL , Hi , HiPtr , SrcValue . getWithOffset ( Size ) , HiMemVT , HiAlign , Store -> getMemOperand ( ) -> getFlags ( ) ) ;


<s> return Val . getOpcode ( ) == <STR_LIT> ? Val . getOperand ( <NUM_LIT> ) : Val ;


<s> static inline SDValue stripBitcast ( SDValue Val ) {

return Val . getOpcode ( ) == <STR_LIT> ? Val . getOperand ( <NUM_LIT> ) : Val ;


<s> SmallVector < MachineBasicBlock * , <NUM_LIT> > Stack ; Stack . push_back ( A ) ; do { MachineBasicBlock * MBB = Stack . back ( ) ; Stack . pop_back ( ) ; for ( MachineBasicBlock * Succ : MBB -> successors ( ) ) { if ( Succ == A ) return false ; if ( Succ != B && Reachable . insert ( Succ ) . second ) Stack . push_back ( Succ ) ; }


<s> bool <ISA_LIT>LateCodeGenPrepare :: runOnFunction ( Function & F ) { if ( skipFunction ( F ) ) return false ;

AC = & getAnalysis < AssumptionCacheTracker > ( ) . getAssumptionCache ( F ) ;


<s> PointerType * Int8PtrTy = Type :: getInt8PtrTy ( LI . getContext ( ) , AS ) ; auto * NewPtr = IRB . CreateBitCast ( IRB . CreateConstGEP1_64 ( IRB . CreatePointerBitCastOrAddrSpaceCast ( Base , Int8PtrTy ) , Offset - Adjust ) , Int32PtrTy ) ; LoadInst * NewLd = IRB . CreateAlignedLoad ( IRB . getInt32Ty ( ) , NewPtr , Align ( <NUM_LIT> ) ) ; NewLd -> copyMetadata ( LI ) ; NewLd -> setMetadata ( LLVMContext :: MD_range , nullptr ) ; unsigned ShAmt = Adjust * <NUM_LIT> ; auto * NewVal = IRB . CreateBitCast ( IRB . CreateTrunc ( IRB . CreateLShr ( NewLd , ShAmt ) , IntNTy ) , LI . getType ( ) ) ; LI . replaceAllUsesWith ( NewVal ) ;


<s> } IRBuilder < > IRB ( & LI ) ; IRB . SetCurrentDebugLocation ( LI . getDebugLoc ( ) ) ; unsigned AS = LI . getPointerAddressSpace ( ) ; unsigned LdBits = DL -> getTypeStoreSize ( LI . getType ( ) ) * <NUM_LIT> ; auto IntNTy = Type :: getIntNTy ( LI . getContext ( ) , LdBits ) ; PointerType * Int32PtrTy = Type :: getInt32PtrTy ( LI . getContext ( ) , AS ) ; PointerType * Int8PtrTy = Type :: getInt8PtrTy ( LI . getContext ( ) , AS ) ; auto * NewPtr = IRB . CreateBitCast ( IRB . CreateConstGEP1_64 ( IRB . CreatePointerBitCastOrAddrSpaceCast ( Base , Int8PtrTy ) , Offset - Adjust ) , Int32PtrTy ) ; LoadInst * NewLd = IRB . CreateAlignedLoad ( IRB . getInt32Ty ( ) , NewPtr , Align ( <NUM_LIT> ) ) ; NewLd -> copyMetadata ( LI ) ;

NewLd -> setMetadata ( LLVMContext :: MD_range , nullptr ) ;


<s> if ( LI . getAlign ( ) >= <NUM_LIT> ) return false ; if ( ! canWidenScalarExtLoad ( LI ) ) return false ; int64_t Offset = <NUM_LIT> ; auto * Base = GetPointerBaseWithConstantOffset ( LI . getPointerOperand ( ) , Offset , * DL ) ; if ( ! isDWORDAligned ( Base ) ) return false ; int64_t Adjust = Offset & <NUM_LIT> ; if ( Adjust == <NUM_LIT> ) { LI . setAlignment ( Align ( <NUM_LIT> ) ) ; return true ; } IRBuilder < > IRB ( & LI ) ; IRB . SetCurrentDebugLocation ( LI . getDebugLoc ( ) ) ;

unsigned AS = LI . getPointerAddressSpace ( ) ;


<s> <ISA_LIT>LateCodeGenPrepare ( ) : FunctionPass ( ID ) {


<s> if ( AS != <ISA_LIT>AS :: CONSTANT_ADDRESS && AS != <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) return false ; if ( ! LI . isSimple ( ) ) return false ; auto * Ty = LI . getType ( ) ; if ( Ty -> isAggregateType ( ) ) return false ; unsigned TySize = DL -> getTypeStoreSize ( Ty ) ;

if ( TySize >= <NUM_LIT> ) return false ;


<s> FunctionPass * llvm :: create<ISA_LIT>LateCodeGenPreparePass ( ) { return new <ISA_LIT>LateCodeGenPrepare ( ) ;


<s> return new <ISA_LIT>LateCodeGenPrepare ( ) ;


<s> StringRef getPassName ( ) const override {


<s> KnownBits Known = computeKnownBits ( V , * DL , <NUM_LIT> , AC ) ; return Known . countMinTrailingZeros ( ) >= <NUM_LIT> ;


<s> if ( skipFunction ( F ) ) return false ; AC = & getAnalysis < AssumptionCacheTracker > ( ) . getAssumptionCache ( F ) ; DA = & getAnalysis < LegacyDivergenceAnalysis > ( ) ; bool Changed = false ; for ( auto & BB : F ) for ( Instruction & I : llvm :: make_early_inc_range ( BB ) ) Changed |= visit ( I ) ;


<s> auto * Base = GetPointerBaseWithConstantOffset ( LI . getPointerOperand ( ) , Offset , * DL ) ; if ( ! isDWORDAligned ( Base ) ) return false ; int64_t Adjust = Offset & <NUM_LIT> ; if ( Adjust == <NUM_LIT> ) { LI . setAlignment ( Align ( <NUM_LIT> ) ) ; return true ; } IRBuilder < > IRB ( & LI ) ; IRB . SetCurrentDebugLocation ( LI . getDebugLoc ( ) ) ; unsigned AS = LI . getPointerAddressSpace ( ) ; unsigned LdBits = DL -> getTypeStoreSize ( LI . getType ( ) ) * <NUM_LIT> ; auto IntNTy = Type :: getIntNTy ( LI . getContext ( ) , LdBits ) ; PointerType * Int32PtrTy = Type :: getInt32PtrTy ( LI . getContext ( ) , AS ) ; PointerType * Int8PtrTy = Type :: getInt8PtrTy ( LI . getContext ( ) , AS ) ; auto * NewPtr = IRB . CreateBitCast ( IRB . CreateConstGEP1_64 ( IRB . getInt8Ty ( ) , IRB . CreatePointerBitCastOrAddrSpaceCast ( Base , Int8PtrTy ) , Offset - Adjust ) , Int32PtrTy ) ; LoadInst * NewLd = IRB . CreateAlignedLoad ( IRB . getInt32Ty ( ) , NewPtr , Align ( <NUM_LIT> ) ) ; NewLd -> copyMetadata ( LI ) ; NewLd -> setMetadata ( LLVMContext :: MD_range , nullptr ) ; unsigned ShAmt = Adjust * <NUM_LIT> ; auto * NewVal = IRB . CreateBitCast ( IRB . CreateTrunc ( IRB . CreateLShr ( NewLd , ShAmt ) , IntNTy ) , LI . getType ( ) ) ;


<s> if ( ! canWidenScalarExtLoad ( LI ) ) return false ; int64_t Offset = <NUM_LIT> ; auto * Base = GetPointerBaseWithConstantOffset ( LI . getPointerOperand ( ) , Offset , * DL ) ; if ( ! isDWORDAligned ( Base ) ) return false ; int64_t Adjust = Offset & <NUM_LIT> ; if ( Adjust == <NUM_LIT> ) { LI . setAlignment ( Align ( <NUM_LIT> ) ) ; return true ; } IRBuilder < > IRB ( & LI ) ; IRB . SetCurrentDebugLocation ( LI . getDebugLoc ( ) ) ; unsigned AS = LI . getPointerAddressSpace ( ) ; unsigned LdBits = DL -> getTypeStoreSize ( LI . getType ( ) ) * <NUM_LIT> ; auto IntNTy = Type :: getIntNTy ( LI . getContext ( ) , LdBits ) ; PointerType * Int32PtrTy = Type :: getInt32PtrTy ( LI . getContext ( ) , AS ) ; PointerType * Int8PtrTy = Type :: getInt8PtrTy ( LI . getContext ( ) , AS ) ; auto * NewPtr = IRB . CreateBitCast ( IRB . CreateConstGEP1_64 ( IRB . getInt8Ty ( ) , IRB . CreatePointerBitCastOrAddrSpaceCast ( Base , Int8PtrTy ) , Offset - Adjust ) , Int32PtrTy ) ; LoadInst * NewLd = IRB . CreateAlignedLoad ( IRB . getInt32Ty ( ) , NewPtr , Align ( <NUM_LIT> ) ) ; NewLd -> copyMetadata ( LI ) ; NewLd -> setMetadata ( LLVMContext :: MD_range , nullptr ) ; unsigned ShAmt = Adjust * <NUM_LIT> ; auto * NewVal = IRB . CreateBitCast ( IRB . CreateTrunc ( IRB . CreateLShr ( NewLd , ShAmt ) , IntNTy ) , LI . getType ( ) ) ; LI . replaceAllUsesWith ( NewVal ) ;


<s> } if ( auto * I = dyn_cast < Instruction > ( V ) ) { const Function * UF = I -> getFunction ( ) ; if ( UF == F ) { Ret = true ; } else if ( ! F ) { Ret |= ! isKernelCC ( UF ) ; } continue ; } assert ( isa < Constant > ( V ) && <STR_LIT> ) ;

append_range ( Stack , V -> users ( ) ) ;


<s> if ( ! GV . hasInitializer ( ) ) { continue ; } if ( ! isa < UndefValue > ( GV . getInitializer ( ) ) ) { continue ; } if ( GV . isConstant ( ) ) { continue ; } if ( ! shouldLowerLDSToStruct ( UsedList , GV , F ) ) { continue ; }


<s> if ( ! GV . hasInitializer ( ) ) { continue ; } if ( ! isa < UndefValue > ( GV . getInitializer ( ) ) ) { continue ; } if ( GV . isConstant ( ) ) { continue ; } if ( ! shouldLowerLDSToStruct ( UsedList , GV , F ) ) { continue ; } LocalVars . push_back ( & GV ) ; }


<s> if ( auto * I = dyn_cast < Instruction > ( U ) ) { return I -> getFunction ( ) == F ; }


<s> Ret = true ; } else if ( ! F ) { Ret |= ! isKernelCC ( UF ) ; } continue ; } if ( auto * E = dyn_cast < ConstantExpr > ( V ) ) { if ( F ) { if ( ! isUsedOnlyFromFunction ( E , F ) ) return false ; } for ( const User * U : E -> users ( ) ) { if ( Visited . insert ( U ) . second ) { Stack . push_back ( U ) ; } } continue ; } return ! F ;


<s> assert ( ! F || isKernelCC ( F ) ) ; while ( ! Stack . empty ( ) ) { const User * V = Stack . pop_back_val ( ) ; Visited . insert ( V ) ; if ( auto * G = dyn_cast < GlobalValue > ( V -> stripPointerCasts ( ) ) ) { if ( UsedList . contains ( G ) ) { continue ; } } if ( auto * I = dyn_cast < Instruction > ( V ) ) { const Function * UF = I -> getFunction ( ) ; if ( UF == F ) { Ret = true ; } else if ( ! F ) { Ret |= ! isKernelCC ( UF ) ; } continue ; } if ( auto * E = dyn_cast < ConstantExpr > ( V ) ) { if ( F ) { if ( ! isUsedOnlyFromFunction ( E , F ) ) return false ; } for ( const User * U : E -> users ( ) ) { if ( Visited . insert ( U ) . second ) { Stack . push_back ( U ) ; } } continue ; }


<s> SmallVector < CallGraphNode * , <NUM_LIT> > CGNStack ( df_begin ( KCGN ) , df_end ( KCGN ) ) ; SmallPtrSet < CallGraphNode * , <NUM_LIT> > VisitedCGNodes ; while ( ! CGNStack . empty ( ) ) { auto * CGN = CGNStack . pop_back_val ( ) ; if ( ! VisitedCGNodes . insert ( CGN ) . second ) continue ; for ( auto GI = CGN -> begin ( ) , GE = CGN -> end ( ) ; GI != GE ; ++ GI ) { auto * RCB = cast < CallBase > ( GI -> first . getValue ( ) ) ; auto * RCGN = GI -> second ; if ( auto * DCallee = RCGN -> getFunction ( ) ) { ReachableCallees . insert ( DCallee ) ; } else if ( RCB -> isIndirectCall ( ) ) { auto * RCBFTy = RCB -> getFunctionType ( ) ; for ( auto * ACGN : AddressTakenFunctions ) { auto * ACallee = ACGN -> getFunction ( ) ; if ( ACallee -> getFunctionType ( ) == RCBFTy ) { ReachableCallees . insert ( ACallee ) ; CGNStack . append ( df_begin ( ACGN ) , df_end ( ACGN ) ) ; }


<s> for ( auto & GV : M . globals ( ) ) { if ( GV . getType ( ) -> getPointerAddressSpace ( ) != <ISA_LIT>AS :: LOCAL_ADDRESS ) { continue ; } if ( ! GV . hasInitializer ( ) ) { continue ; } if ( ! isa < UndefValue > ( GV . getInitializer ( ) ) ) { continue ; } if ( GV . isConstant ( ) ) { continue ; } if ( std :: none_of ( GV . user_begin ( ) , GV . user_end ( ) , [ & ] ( User * U ) { return userRequiresLowering ( UsedList , U ) ; } ) ) {


<s> continue ; } if ( std :: none_of ( GV . user_begin ( ) , GV . user_end ( ) , [ & ] ( User * U ) { return userRequiresLowering ( UsedList , U ) ; } ) ) { continue ; } LocalVars . push_back ( & GV ) ; }


<s> bool isKernelCC ( Function * Func ) {

return <STR_LIT> ( Func -> getCallingConv ( ) ) ;


<s> bool isKernelCC ( Function * Func ) { return <STR_LIT> ( Func -> getCallingConv ( ) ) ;


<s> User * V = Stack . pop_back_val ( ) ; Visited . insert ( V ) ; if ( auto * G = dyn_cast < GlobalValue > ( V -> stripPointerCasts ( ) ) ) { if ( UsedList . contains ( G ) ) { continue ; } } if ( auto * I = dyn_cast < Instruction > ( V ) ) { if ( isKernelCC ( I -> getFunction ( ) ) ) { continue ; } } if ( auto * E = dyn_cast < ConstantExpr > ( V ) ) { for ( Value :: user_iterator EU = E -> user_begin ( ) ; EU != E -> user_end ( ) ; ++ EU ) { if ( Visited . insert ( * EU ) . second ) { Stack . push_back ( * EU ) ;


<s> if ( ! F || F -> isDeclaration ( ) || <STR_LIT> ( F ) ) continue ; AddressTakenFunctions . insert ( CGN ) ; }


<s> auto * F = CGN -> getFunction ( ) ; if ( ! F || F -> isDeclaration ( ) || <STR_LIT> ( F ) ) continue ; AddressTakenFunctions . insert ( CGN ) ;


<s> static void collectFunctionUses ( User * U , const Function * F , SetVector < Instruction * > & InstUsers ) { SmallVector < User * > Stack { U } ; while ( ! Stack . empty ( ) ) { U = Stack . pop_back_val ( ) ; if ( auto * I = dyn_cast < Instruction > ( U ) ) { if ( I -> getFunction ( ) == F ) InstUsers . insert ( I ) ; continue ;


<s> auto * U = UserStack . pop_back_val ( ) ; if ( ! VisitedUsers . insert ( U ) . second ) continue ; if ( isa < GlobalValue > ( U ) ) return SmallPtrSet < Function * , <NUM_LIT> > ( ) ; if ( isa < Constant > ( U ) ) { append_range ( UserStack , U -> users ( ) ) ;


<s> void collectReachableCallees ( Module & M , DenseMap < Function * , SmallPtrSet < Function * , <NUM_LIT> >> & KernelToCallees ) { CollectReachableCallees CRC { M } ;

CRC . collectReachableCallees ( KernelToCallees ) ;


<s> void collectReachableCallees ( Module & M , DenseMap < Function * , SmallPtrSet < Function * , <NUM_LIT> >> & KernelToCallees ) {


<s> return DL . getValueOrABITypeAlignment ( GV -> getPointerAlignment ( DL ) , GV -> getValueType ( ) ) ;


<s> SmallVector < GlobalValue * , <NUM_LIT> > TmpVec ; collectUsedGlobalVariables ( M , TmpVec , true ) ; UsedList . insert ( TmpVec . begin ( ) , TmpVec . end ( ) ) ; TmpVec . clear ( ) ; collectUsedGlobalVariables ( M , TmpVec , false ) ; UsedList . insert ( TmpVec . begin ( ) , TmpVec . end ( ) ) ;


<s> SmallVector < const User * , <NUM_LIT> > Stack ( GV -> users ( ) ) ; while ( ! Stack . empty ( ) ) {

const User * U = Stack . pop_back_val ( ) ;


<s> for ( Instruction * I : InstUsers ) {

convertConstantExprsToInstructions ( I , C ) ;


<s> void replaceConstantUsesInFunction ( ConstantExpr * C , const Function * F ) {


<s> SmallPtrSet < const User * , <NUM_LIT> > Visited ; SmallVector < const User * , <NUM_LIT> > Stack ( GV . users ( ) ) ; SmallPtrSet < const GlobalValue * , <NUM_LIT> > GlobalUsers ; assert ( ! F || isKernelCC ( F ) ) ; while ( ! Stack . empty ( ) ) { const User * V = Stack . pop_back_val ( ) ; Visited . insert ( V ) ; if ( auto * G = dyn_cast < GlobalValue > ( V ) ) { StringRef GName = G -> getName ( ) ; if ( F && GName != <STR_LIT> && GName != <STR_LIT> ) {


<s> if ( auto * G = dyn_cast < GlobalValue > ( V ) ) { StringRef GName = G -> getName ( ) ; if ( F && GName != <STR_LIT> && GName != <STR_LIT> ) { return false ; } GlobalUsers . insert ( G ) ; continue ; } if ( auto * I = dyn_cast < Instruction > ( V ) ) { const Function * UF = I -> getFunction ( ) ; if ( UF == F ) { Ret = true ; } else if ( ! F ) {


<s> static Register buildDivRCP ( MachineIRBuilder & B , Register Src ) { const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto Cvt0 = B . buildUITOFP ( S32 , Src ) ;

auto RcpIFlag = B . buildInstr ( <STR_LIT> , { S32 } , { Cvt0 } ) ;


<s> const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto Cvt0 = B . buildUITOFP ( S32 , Src ) ; auto RcpIFlag = B . buildInstr ( <STR_LIT> , { S32 } , { Cvt0 } ) ;

auto FPUIntMaxPlus1 = B . buildFConstant ( S32 , BitsToFloat ( <NUM_LIT> ) ) ;


<s> MachineInstrBuilder MIB = B . buildInstr ( <STR_LIT> ) . addDef ( PCReg ) ; MIB . addGlobalAddress ( GV , Offset + <NUM_LIT> , GAFlags ) ; if ( GAFlags == <ISA_LIT>InstrInfo :: MO_NONE ) MIB . addImm ( <NUM_LIT> ) ; else MIB . addGlobalAddress ( GV , Offset + <NUM_LIT> , GAFlags + <NUM_LIT> ) ; B . getMRI ( ) -> setRegClass ( PCReg , & <STR_LIT> ) ;

if ( PtrTy . getSizeInBits ( ) == <NUM_LIT> ) B . buildExtract ( DstReg , PCReg , <NUM_LIT> ) ;


<s> assert ( B . getMRI ( ) -> getType ( SrcOp . getReg ( ) ) == S32 ) ; } } int NumAddrRegs = AddrRegs . size ( ) ; if ( NumAddrRegs != <NUM_LIT> ) { if ( NumAddrRegs > <NUM_LIT> && ! isPowerOf2_32 ( NumAddrRegs ) ) { const int RoundedNumRegs = NextPowerOf2 ( NumAddrRegs ) ; auto Undef = B . buildUndef ( S32 ) ; AddrRegs . append ( RoundedNumRegs - NumAddrRegs , Undef . getReg ( <NUM_LIT> ) ) ; NumAddrRegs = RoundedNumRegs ; } auto VAddr = B . buildBuildVector ( LLT :: vector ( NumAddrRegs , <NUM_LIT> ) , AddrRegs ) ; MI . getOperand ( DimIdx ) . setReg ( VAddr . getReg ( <NUM_LIT> ) ) ; } for ( int I = <NUM_LIT> ; I != NumVAddrs ; ++ I ) { MachineOperand & SrcOp = MI . getOperand ( DimIdx + I ) ;

if ( SrcOp . isReg ( ) ) MI . getOperand ( DimIdx + I ) . setReg ( <STR_LIT> ) ;


<s> return [ = ] ( const LegalityQuery & Query ) { const LLT QueryTy = Query . Types [ TypeIdx ] ; return QueryTy . isVector ( ) && QueryTy . getElementType ( ) == Type ; } ;


<s> static MachineInstrBuilder extractF64Exponent ( unsigned Hi , MachineIRBuilder & B ) { const unsigned FractBits = <NUM_LIT> ; const unsigned ExpBits = <NUM_LIT> ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto Const0 = B . buildConstant ( S32 , FractBits - <NUM_LIT> ) ; auto Const1 = B . buildConstant ( S32 , ExpBits ) ; auto ExpPart = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( Const0 . getReg ( <NUM_LIT> ) ) . addUse ( Const1 . getReg ( <NUM_LIT> ) ) ; return B . buildSub ( S32 , ExpPart , B . buildConstant ( S32 , <NUM_LIT> ) ) ;


<s> const unsigned ExpBits = <NUM_LIT> ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto Const0 = B . buildConstant ( S32 , FractBits - <NUM_LIT> ) ; auto Const1 = B . buildConstant ( S32 , ExpBits ) ; auto ExpPart = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( Const0 . getReg ( <NUM_LIT> ) ) . addUse ( Const1 . getReg ( <NUM_LIT> ) ) ;

return B . buildSub ( S32 , ExpPart , B . buildConstant ( S32 , <NUM_LIT> ) ) ;


<s> unsigned Size = Ty . getSizeInBits ( ) ; unsigned Pieces = ( Size + <NUM_LIT> ) / <NUM_LIT> ; unsigned NewNumElts = ( Ty . getNumElements ( ) + <NUM_LIT> ) / Pieces ;

return std :: make_pair ( TypeIdx , LLT :: scalarOrVector ( NewNumElts , EltTy ) ) ;


<s> const ArgDescriptor * <ISA_LIT>LegalizerInfo :: getArgDescriptor ( MachineIRBuilder & B , <STR_LIT> ArgType ) const { const <ISA_LIT>MachineFunctionInfo * MFI = B . getMF ( ) . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const ArgDescriptor * Arg ; const TargetRegisterClass * RC ;

std :: tie ( Arg , RC ) = MFI -> getPreloadedValue ( ArgType ) ;


<s> return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> :


<s> Register LiveIn = MRI . getLiveInVirtReg ( PhyReg ) ; if ( ! LiveIn ) { LiveIn = MRI . createGenericVirtualRegister ( Ty ) ; MRI . addLiveIn ( PhyReg , LiveIn ) ; }

if ( ! InsertLiveInCopy ) return LiveIn ;


<s> unsigned Pow2NElts = <NUM_LIT> << Log2_32_Ceil ( NElts ) ; return Ty . changeNumElements ( Pow2NElts ) ;


<s> MachineFunction & MF = B . getMF ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; assert ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) ; if ( ST . hasApertureRegs ( ) ) { unsigned Offset = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: OFFSET_SRC_SHARED_BASE : <STR_LIT> :: OFFSET_SRC_PRIVATE_BASE ; unsigned WidthM1 = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: WIDTH_M1_SRC_SHARED_BASE : <STR_LIT> :: WIDTH_M1_SRC_PRIVATE_BASE ; unsigned Encoding = <STR_LIT> :: ID_MEM_BASES << <STR_LIT> :: ID_SHIFT_ | Offset << <STR_LIT> :: OFFSET_SHIFT_ | WidthM1 << <STR_LIT> :: WIDTH_M1_SHIFT_ ; Register GetReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( GetReg ) . addImm ( Encoding ) ; MRI . setType ( GetReg , S32 ) ; auto ShiftAmt = B . buildConstant ( S32 , WidthM1 + <NUM_LIT> ) ; return B . buildShl ( S32 , GetReg , ShiftAmt ) . getReg ( <NUM_LIT> ) ;


<s> return Query . Types [ TypeIdx0 ] . getSizeInBits ( ) > Query . Types [ TypeIdx1 ] . getSizeInBits ( ) ;


<s> SmallVector < Register , <NUM_LIT> > WideRegs ;

for ( int I = <NUM_LIT> , E = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; I != E ; ++ I ) WideRegs . push_back ( B . buildAnyExt ( S32 , Unmerge . getReg ( I ) ) . getReg ( <NUM_LIT> ) ) ;


<s> assert ( StoreVT . isVector ( ) && StoreVT . getElementType ( ) == S16 ) ; auto Unmerge = B . buildUnmerge ( S16 , Reg ) ; SmallVector < Register , <NUM_LIT> > WideRegs ; for ( int I = <NUM_LIT> , E = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; I != E ; ++ I ) WideRegs . push_back ( B . buildAnyExt ( S32 , Unmerge . getReg ( I ) ) . getReg ( <NUM_LIT> ) ) ;

int NumElts = StoreVT . getNumElements ( ) ;


<s> if ( ! MRI . getVRegDef ( LiveIn ) ) { MachineBasicBlock & OrigInsBB = B . getMBB ( ) ; auto OrigInsPt = B . getInsertPt ( ) ;

MachineBasicBlock & EntryMBB = B . getMF ( ) . front ( ) ;


<s> Register <ISA_LIT>LegalizerInfo :: insertLiveInCopy ( MachineIRBuilder & B , MachineRegisterInfo & MRI , Register LiveIn , Register PhyReg ) const { assert ( PhyReg . isPhysical ( ) && <STR_LIT> ) ; if ( ! MRI . getVRegDef ( LiveIn ) ) { MachineBasicBlock & OrigInsBB = B . getMBB ( ) ; auto OrigInsPt = B . getInsertPt ( ) ; MachineBasicBlock & EntryMBB = B . getMF ( ) . front ( ) ; EntryMBB . addLiveIn ( PhyReg ) ; B . setInsertPt ( EntryMBB , EntryMBB . begin ( ) ) ; B . buildCopy ( LiveIn , PhyReg ) ;

B . setInsertPt ( OrigInsBB , OrigInsPt ) ;


<s> return Ty . getSizeInBits ( ) <= MaxSize && EltTy . getSizeInBits ( ) % <NUM_LIT> == <NUM_LIT> ; } ;


<s> const LLT EltTy = Ty . getScalarType ( ) ;

return Ty . getSizeInBits ( ) <= MaxSize && EltTy . getSizeInBits ( ) % <NUM_LIT> == <NUM_LIT> ;


<s> if ( Ty . isVector ( ) ) { const int EltSize = Ty . getElementType ( ) . getSizeInBits ( ) ; return EltSize == <NUM_LIT> || EltSize == <NUM_LIT> || ( EltSize == <NUM_LIT> && Ty . getNumElements ( ) % <NUM_LIT> == <NUM_LIT> ) || EltSize == <NUM_LIT> || EltSize == <NUM_LIT> ;


<s> return [ = ] ( const LegalityQuery & Query ) { const LLT Ty = Query . Types [ TypeIdx ] ; return Ty . isVector ( ) && Ty . getNumElements ( ) % <NUM_LIT> != <NUM_LIT> && Ty . getElementType ( ) . getSizeInBits ( ) < <NUM_LIT> && Ty . getSizeInBits ( ) % <NUM_LIT> != <NUM_LIT> ;


<s> static LegalityPredicate isSmallOddVector ( unsigned TypeIdx ) { return [ = ] ( const LegalityQuery & Query ) { const LLT Ty = Query . Types [ TypeIdx ] ; return Ty . isVector ( ) && Ty . getNumElements ( ) % <NUM_LIT> != <NUM_LIT> && Ty . getElementType ( ) . getSizeInBits ( ) < <NUM_LIT> && Ty . getSizeInBits ( ) % <NUM_LIT> != <NUM_LIT> ;


<s> const LLT Ty = Query . Types [ TypeIdx ] ; return ! Ty . isVector ( ) && Ty . getSizeInBits ( ) > <NUM_LIT> && Query . MMODescrs [ <NUM_LIT> ] . SizeInBits < Ty . getSizeInBits ( ) ; } ;


<s> B . setInstr ( MI ) ; Register PackedVal = B . buildBuildVector ( VecTy , { NewVal , CmpVal } ) . getReg ( <NUM_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( DstReg ) . addUse ( PtrReg ) . addUse ( PackedVal ) . setMemRefs ( MI . memoperands ( ) ) ;


<s> unsigned Opc = IsInc ? <STR_LIT> : <STR_LIT> ; B . buildInstr ( Opc ) . addDef ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . addUse ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . addUse ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . cloneMemRefs ( MI ) ; MI . eraseFromParent ( ) ;


<s> <STR_LIT> ; B . buildInstr ( Opc ) . addDef ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . addUse ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . addUse ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . cloneMemRefs ( MI ) ;


<s> const unsigned NumVIndexOps = IsCmpSwap ? <NUM_LIT> : <NUM_LIT> ; const bool HasVIndex = MI . getNumOperands ( ) == NumVIndexOps ; Register VIndex ; if ( HasVIndex ) { VIndex = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; ++ OpOffset ; } Register VOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; Register SOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; unsigned AuxiliaryData = MI . getOperand ( <NUM_LIT> + OpOffset ) . getImm ( ) ; MachineMemOperand * MMO = * MI . memoperands_begin ( ) ; unsigned ImmOffset ; unsigned TotalOffset ; std :: tie ( VOffset , ImmOffset , TotalOffset ) = splitBufferOffsets ( B , VOffset ) ; if ( TotalOffset != <NUM_LIT> ) MMO = B . getMF ( ) . getMachineMemOperand ( MMO , TotalOffset , MMO -> getSize ( ) ) ; if ( ! VIndex ) VIndex = B . buildConstant ( LLT :: scalar ( <NUM_LIT> ) , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; auto MIB = B . buildInstr ( getBufferAtomicPseudo ( IID ) ) . addDef ( Dst ) . addUse ( VData ) ;

if ( IsCmpSwap ) MIB . addReg ( CmpVal ) ;


<s> <STR_LIT> ; } else if ( IsFormat ) { Opc = IsD16 ? <STR_LIT> : <STR_LIT> ; } else { switch ( MemSize ) { case <NUM_LIT> : Opc = <STR_LIT> ; break ; case <NUM_LIT> : Opc = <STR_LIT> ; break ; default : Opc = <STR_LIT> ; break ; } } Register LoadDstReg ; bool IsExtLoad = ( ! IsD16 && MemSize < <NUM_LIT> ) || ( IsD16 && ! Ty . isVector ( ) ) ; LLT UnpackedTy = Ty . changeElementSize ( <NUM_LIT> ) ; if ( IsExtLoad ) LoadDstReg = B . getMRI ( ) -> createGenericVirtualRegister ( S32 ) ; else if ( Unpacked && IsD16 && Ty . isVector ( ) ) LoadDstReg = B . getMRI ( ) -> createGenericVirtualRegister ( UnpackedTy ) ; else LoadDstReg = Dst ; if ( ! VIndex ) VIndex = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; auto MIB = B . buildInstr ( Opc ) . addDef ( LoadDstReg ) . addUse ( RSrc ) . addUse ( VIndex ) . addUse ( VOffset ) . addUse ( SOffset ) . addImm ( ImmOffset ) ; if ( IsTyped ) MIB . addImm ( Format ) ; MIB . addImm ( AuxiliaryData ) . addImm ( HasVIndex ? - <NUM_LIT> : <NUM_LIT> ) . addMemOperand ( MMO ) ; if ( LoadDstReg != Dst ) { B . setInsertPt ( B . getMBB ( ) , ++ B . getInsertPt ( ) ) ; if ( IsExtLoad ) B . buildTrunc ( Dst , LoadDstReg ) ; else { auto Unmerge = B . buildUnmerge ( S32 , LoadDstReg ) ; SmallVector < Register , <NUM_LIT> > Repack ; for ( unsigned I = <NUM_LIT> , N = Unmerge -> getNumOperands ( ) - <NUM_LIT> ; I != N ; ++ I ) Repack . push_back ( B . buildTrunc ( EltTy , Unmerge . getReg ( I ) ) . getReg ( <NUM_LIT> ) ) ;


<s> Register VOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; Register SOffset = MI . getOperand ( <NUM_LIT> + OpOffset ) . getReg ( ) ; unsigned Format = <NUM_LIT> ; if ( IsTyped ) { Format = MI . getOperand ( <NUM_LIT> + OpOffset ) . getImm ( ) ; ++ OpOffset ; } unsigned AuxiliaryData = MI . getOperand ( <NUM_LIT> + OpOffset ) . getImm ( ) ; std :: tie ( VOffset , ImmOffset , TotalOffset ) = splitBufferOffsets ( B , VOffset ) ; if ( TotalOffset != <NUM_LIT> ) MMO = B . getMF ( ) . getMachineMemOperand ( MMO , TotalOffset , MemSize ) ; unsigned Opc ; if ( IsTyped ) { Opc = IsD16 ? <STR_LIT> : <STR_LIT> ; } else if ( IsFormat ) { Opc = IsD16 ? <STR_LIT> : <STR_LIT> ; } else { switch ( MemSize ) { case <NUM_LIT> : Opc = <STR_LIT> ; break ; case <NUM_LIT> : Opc = <STR_LIT> ; break ; default :


<s> Register Src0 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src1 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( MRI . getType ( Src0 ) == LLT :: scalar ( <NUM_LIT> ) ) ; B . setInstr ( MI ) ; auto Merge = B . buildMerge ( S32 , { Src0 , Src1 } ) ; B . buildBitcast ( Dst , Merge ) ; MI . eraseFromParent ( ) ;


<s> case TargetOpcode :: G_FMINNUM : case TargetOpcode :: G_FMAXNUM : case TargetOpcode :: G_FMINNUM_IEEE : case TargetOpcode :: G_FMAXNUM_IEEE : return legalizeMinNumMaxNum ( MI , MRI , B ) ; case TargetOpcode :: G_EXTRACT_VECTOR_ELT : return legalizeExtractVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_INSERT_VECTOR_ELT : return legalizeInsertVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return legalizeShuffleVector ( MI , MRI , B ) ; case TargetOpcode :: G_F<ISA_LIT>N : case TargetOpcode :: G_FCOS : return legalizeSinCos ( MI , MRI , B ) ; case TargetOpcode :: G_GLOBAL_VALUE : return legalizeGlobalValue ( MI , MRI , B ) ; case TargetOpcode :: G_LOAD : return legalizeLoad ( MI , MRI , B , Observer ) ; case TargetOpcode :: G_FMAD : return legalizeFMad ( MI , MRI , B ) ; case TargetOpcode :: G_FDIV : return legalizeFDIV ( MI , MRI , B ) ; case TargetOpcode :: G_UDIV : case TargetOpcode :: G_UREM : return legalizeUDIV_UREM ( MI , MRI , B ) ; case TargetOpcode :: G_SDIV : case TargetOpcode :: G_SREM : return legalizeSDIV_SREM ( MI , MRI , B ) ; case TargetOpcode :: G_ATOMIC_CMPXCHG : return legalizeAtomicCmpXChg ( MI , MRI , B ) ; case TargetOpcode :: G_FLOG : return legalizeFlog ( MI , B , <NUM_LIT> / numbers :: log2ef ) ; case TargetOpcode :: G_FLOG10 : return legalizeFlog ( MI , B , numbers :: ln2f / numbers :: ln10f ) ; case TargetOpcode :: G_FEXP : return legalizeFExp ( MI , B ) ;


<s> return legalizeInsertVectorElt ( MI , MRI , B ) ; case TargetOpcode :: G_SHUFFLE_VECTOR : return legalizeShuffleVector ( MI , MRI , B ) ; case TargetOpcode :: G_F<ISA_LIT>N : case TargetOpcode :: G_FCOS : return legalizeSinCos ( MI , MRI , B ) ; case TargetOpcode :: G_GLOBAL_VALUE : return legalizeGlobalValue ( MI , MRI , B ) ; case TargetOpcode :: G_LOAD : return legalizeLoad ( MI , MRI , B , Observer ) ; case TargetOpcode :: G_FMAD : return legalizeFMad ( MI , MRI , B ) ; case TargetOpcode :: G_FDIV : return legalizeFDIV ( MI , MRI , B ) ; case TargetOpcode :: G_UDIV : case TargetOpcode :: G_UREM : return legalizeUDIV_UREM ( MI , MRI , B ) ; case TargetOpcode :: G_SDIV : case TargetOpcode :: G_SREM : return legalizeSDIV_SREM ( MI , MRI , B ) ; case TargetOpcode :: G_ATOMIC_CMPXCHG : return legalizeAtomicCmpXChg ( MI , MRI , B ) ; case TargetOpcode :: G_FLOG : return legalizeFlog ( MI , B , <NUM_LIT> / numbers :: log2ef ) ; case TargetOpcode :: G_FLOG10 : return legalizeFlog ( MI , B , numbers :: ln2f / numbers :: ln10f ) ; case TargetOpcode :: G_FEXP : return legalizeFExp ( MI , B ) ; case TargetOpcode :: G_FPOW : return legalizeFPow ( MI , B ) ; case TargetOpcode :: G_FFLOOR :

return legalizeFFloor ( MI , MRI , B ) ;


<s> if ( ST . getTrapHandlerAbi ( ) != GCNSubtarget :: TrapHandlerAbiHsa || ! ST . isTrapHandlerEnabled ( ) ) { DiagnosticInfoUnsupported NoTrap ( B . getMF ( ) . getFunction ( ) , <STR_LIT> , MI . getDebugLoc ( ) , DS_Warning ) ; LLVMContext & Ctx = B . getMF ( ) . getFunction ( ) . getContext ( ) ;


<s> LLT EltTy = VecTy . getElementType ( ) ; assert ( EltTy == MRI . getType ( Dst ) ) ; B . setInstr ( MI ) ; if ( IdxVal -> Value < VecTy . getNumElements ( ) ) B . buildExtract ( Dst , Vec , IdxVal -> Value * EltTy . getSizeInBits ( ) ) ; else B . buildUndef ( Dst ) ;


<s> LLT EltTy = VecTy . getElementType ( ) ; assert ( EltTy == MRI . getType ( Dst ) ) ; B . setInstr ( MI ) ; if ( IdxVal -> Value < VecTy . getNumElements ( ) ) B . buildExtract ( Dst , Vec , IdxVal -> Value * EltTy . getSizeInBits ( ) ) ; else B . buildUndef ( Dst ) ; MI . eraseFromParent ( ) ;


<s> LLT ResTy = MRI . getType ( Res ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; const MachineFunction & MF = B . getMF ( ) ; bool Unsafe = MF . getTarget ( ) . Options . UnsafeFPMath || MI . getFlag ( MachineInstr :: FmArcp ) ; if ( ! MF . getTarget ( ) . Options . UnsafeFPMath && ResTy == S64 ) return false ; if ( ! Unsafe && ResTy == S32 && MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> getMode ( ) . allFP32Denormals ( ) ) return false ;

if ( auto CLHS = getConstantFPVRegVal ( LHS , MRI ) ) {


<s> B . setInstr ( MI ) ; const LLT S1 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; Register Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( MRI . getType ( Src ) == S64 ) ; auto Trunc = B . buildIntrinsicTrunc ( S64 , Src ) ; const auto Zero = B . buildFConstant ( S64 , <NUM_LIT> ) ; const auto One = B . buildFConstant ( S64 , <NUM_LIT> ) ; auto Lt0 = B . buildFCmp ( CmpInst :: FCMP_OGT , S1 , Src , Zero ) ; auto NeTrunc = B . buildFCmp ( CmpInst :: FCMP_ONE , S1 , Src , Trunc ) ;

auto And = B . buildAnd ( S1 , Lt0 , NeTrunc ) ;


<s> const auto Zero = B . buildFConstant ( S64 , <NUM_LIT> ) ; const auto One = B . buildFConstant ( S64 , <NUM_LIT> ) ; auto Lt0 = B . buildFCmp ( CmpInst :: FCMP_OGT , S1 , Src , Zero ) ; auto NeTrunc = B . buildFCmp ( CmpInst :: FCMP_ONE , S1 , Src , Trunc ) ; auto And = B . buildAnd ( S1 , Lt0 , NeTrunc ) ; auto Add = B . buildSelect ( S64 , And , One , Zero ) ; B . buildFAdd ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Trunc , Add ) ; return true ;


<s> Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( Dst ) ; LLT S16 = LLT :: scalar ( <NUM_LIT> ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; if ( legalizeFastUnsafeFDIV ( MI , MRI , B ) ) return true ; if ( DstTy == S16 ) return legalizeFDIV16 ( MI , MRI , B ) ;

if ( DstTy == S32 ) return legalizeFDIV32 ( MI , MRI , B ) ;


<s> auto LHSExt = B . buildFPExt ( S32 , LHS , Flags ) ; auto RHSExt = B . buildFPExt ( S32 , RHS , Flags ) ; auto RCP = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( RHSExt . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto QUOT = B . buildFMul ( S32 , LHSExt , RCP , Flags ) ; auto RDst = B . buildFPTrunc ( S16 , QUOT , Flags ) ; B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( RDst . getReg ( <NUM_LIT> ) ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ;


<s> LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto LHSExt = B . buildFPExt ( S32 , LHS , Flags ) ; auto RHSExt = B . buildFPExt ( S32 , RHS , Flags ) ; auto RCP = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( RHSExt . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto QUOT = B . buildFMul ( S32 , LHSExt , RCP , Flags ) ; auto RDst = B . buildFPTrunc ( S16 , QUOT , Flags ) ; B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( RDst . getReg ( <NUM_LIT> ) ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ;


<s> auto NumeratorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto ApproxRcp = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( DenominatorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto NegDivScale0 = B . buildFNeg ( S32 , DenominatorScaled , Flags ) ; if ( ! Mode . allFP32Denormals ( ) ) toggleSPDenormMode ( true , B , ST , Mode ) ; auto Fma0 = B . buildFMA ( S32 , NegDivScale0 , ApproxRcp , One , Flags ) ; auto Fma1 = B . buildFMA ( S32 , Fma0 , ApproxRcp , ApproxRcp , Flags ) ; auto Mul = B . buildFMul ( S32 , NumeratorScaled , Fma1 , Flags ) ; auto Fma2 = B . buildFMA ( S32 , NegDivScale0 , Mul , NumeratorScaled , Flags ) ; auto Fma3 = B . buildFMA ( S32 , Fma2 , Fma1 , Mul , Flags ) ; auto Fma4 = B . buildFMA ( S32 , NegDivScale0 , Fma3 , NumeratorScaled , Flags ) ; if ( ! Mode . allFP32Denormals ( ) ) toggleSPDenormMode ( false , B , ST , Mode ) ; auto Fmas = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( Fma4 . getReg ( <NUM_LIT> ) ) . addUse ( Fma1 . getReg ( <NUM_LIT> ) ) . addUse ( Fma3 . getReg ( <NUM_LIT> ) ) . addUse ( NumeratorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( Fmas . getReg ( <NUM_LIT> ) ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ;


<s> auto NumeratorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto ApproxRcp = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( DenominatorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto NegDivScale0 = B . buildFNeg ( S32 , DenominatorScaled , Flags ) ; if ( ! Mode . allFP32Denormals ( ) ) toggleSPDenormMode ( true , B , ST , Mode ) ; auto Fma0 = B . buildFMA ( S32 , NegDivScale0 , ApproxRcp , One , Flags ) ; auto Fma1 = B . buildFMA ( S32 , Fma0 , ApproxRcp , ApproxRcp , Flags ) ; auto Mul = B . buildFMul ( S32 , NumeratorScaled , Fma1 , Flags ) ; auto Fma2 = B . buildFMA ( S32 , NegDivScale0 , Mul , NumeratorScaled , Flags ) ;

auto Fma3 = B . buildFMA ( S32 , Fma2 , Fma1 , Mul , Flags ) ;


<s> auto NumUnmerge = B . buildUnmerge ( S32 , LHS ) ; auto DenUnmerge = B . buildUnmerge ( S32 , RHS ) ; auto Scale0Unmerge = B . buildUnmerge ( S32 , DivScale0 ) ; auto Scale1Unmerge = B . buildUnmerge ( S32 , DivScale1 ) ; auto CmpNum = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , NumUnmerge . getReg ( <NUM_LIT> ) , Scale1Unmerge . getReg ( <NUM_LIT> ) ) ; auto CmpDen = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , DenUnmerge . getReg ( <NUM_LIT> ) , Scale0Unmerge . getReg ( <NUM_LIT> ) ) ; Scale = B . buildXor ( S1 , CmpNum , CmpDen ) . getReg ( <NUM_LIT> ) ; } else { Scale = DivScale1 . getReg ( <NUM_LIT> ) ; }

auto Fmas = B . buildIntrinsic ( <STR_LIT> , { S64 } , false ) . addUse ( Fma4 . getReg ( <NUM_LIT> ) ) . addUse ( Fma3 . getReg ( <NUM_LIT> ) ) . addUse ( Mul . getReg ( <NUM_LIT> ) ) . addUse ( Scale ) . setMIFlags ( Flags ) ;


<s> auto Abs = B . buildFAbs ( S32 , RHS , Flags ) ; const APFloat C0Val ( <NUM_LIT> ) ; auto C0 = B . buildConstant ( S32 , <NUM_LIT> ) ; auto C1 = B . buildConstant ( S32 , <NUM_LIT> ) ; auto C2 = B . buildConstant ( S32 , FloatToBits ( <NUM_LIT> ) ) ; auto CmpRes = B . buildFCmp ( CmpInst :: FCMP_OGT , S1 , Abs , C0 , Flags ) ; auto Sel = B . buildSelect ( S32 , CmpRes , C1 , C2 , Flags ) ; auto Mul0 = B . buildFMul ( S32 , RHS , Sel , Flags ) ; auto RCP = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( Mul0 . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto Mul1 = B . buildFMul ( S32 , LHS , RCP , Flags ) ; B . buildFMul ( Res , Sel , Mul1 , Flags ) ; MI . eraseFromParent ( ) ; return true ;


<s> unsigned Flags = MI . getFlags ( ) ; LLT Ty = B . getMRI ( ) -> getType ( Dst ) ; B . setInstr ( MI ) ; auto K = B . buildFConstant ( Ty , numbers :: log2e ) ; auto Mul = B . buildFMul ( Ty , Src , K , Flags ) ; B . buildFExp2 ( Dst , Mul , Flags ) ;


<s> unsigned Flags = MI . getFlags ( ) ; LLT Ty = B . getMRI ( ) -> getType ( Dst ) ; B . setInstr ( MI ) ; auto K = B . buildFConstant ( Ty , numbers :: log2e ) ; auto Mul = B . buildFMul ( Ty , Src , K , Flags ) ; B . buildFExp2 ( Dst , Mul , Flags ) ; MI . eraseFromParent ( ) ; return true ;


<s> const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register OrigSrc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Flags = MI . getFlags ( ) ; assert ( ST . hasFractBug ( ) && MRI . getType ( Dst ) == S64 && <STR_LIT> ) ; auto Fract = B . buildIntrinsic ( <STR_LIT> , { S64 } , false ) . addUse ( OrigSrc ) . setMIFlags ( Flags ) ; Register ModSrc = stripAnySourceMods ( OrigSrc , MRI ) ; auto Const = B . buildFConstant ( S64 , BitsToDouble ( <NUM_LIT> ) ) ; Register Min = MRI . createGenericVirtualRegister ( S64 ) ; const <ISA_LIT>MachineFunctionInfo * MFI = B . getMF ( ) . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( MFI -> getMode ( ) . IEEE ) B . buildFMinNumIEEE ( Min , Fract , Const , Flags ) ; else B . buildFMinNum ( Min , Fract , Const , Flags ) ; Register CorrectedFract = Min ; if ( ! MI . getFlag ( MachineInstr :: FmNoNans ) ) { auto IsNan = B . buildFCmp ( CmpInst :: FCMP_ORD , S1 , ModSrc , ModSrc , Flags ) ; CorrectedFract = B . buildSelect ( S64 , IsNan , ModSrc , Min , Flags ) . getReg ( <NUM_LIT> ) ; } auto NegFract = B . buildFNeg ( S64 , CorrectedFract , Flags ) ; B . buildFAdd ( Dst , OrigSrc , NegFract , Flags ) ; MI . eraseFromParent ( ) ;


<s> Register Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT Ty = B . getMRI ( ) -> getType ( Dst ) ; unsigned Flags = MI . getFlags ( ) ; B . setInstr ( MI ) ; auto Log2Operand = B . buildFLog2 ( Ty , Src , Flags ) ; auto Log2BaseInvertedOperand = B . buildFConstant ( Ty , Log2BaseInverted ) ;

B . buildFMul ( Dst , Log2Operand , Log2BaseInvertedOperand , Flags ) ;


<s> if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . allFP32Denormals ( ) ) return true ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . allFP64FP16Denormals ( ) ) return true ; MachineIRBuilder HelperBuilder ( MI ) ;


<s> unsigned Flags = MI . getFlags ( ) ; LLT Ty = B . getMRI ( ) -> getType ( Dst ) ; B . setInstr ( MI ) ; const LLT S16 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; if ( Ty == S32 ) { auto Log = B . buildFLog2 ( S32 , Src0 , Flags ) ; auto Mul = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( Log . getReg ( <NUM_LIT> ) ) . addUse ( Src1 ) . setMIFlags ( Flags ) ;


<s> const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; assert ( MRI . getType ( Src ) == S64 && MRI . getType ( Dst ) == S64 ) ; unsigned Flags = MI . getFlags ( ) ; auto Trunc = B . buildIntrinsicTrunc ( S64 , Src , Flags ) ; auto K0 = B . buildFConstant ( S64 , BitsToDouble ( UINT64_C ( <NUM_LIT> ) ) ) ; auto K1 = B . buildFConstant ( S64 , BitsToDouble ( UINT64_C ( <NUM_LIT> ) ) ) ; auto Mul = B . buildFMul ( S64 , Trunc , K0 , Flags ) ; auto FloorMul = B . buildFFloor ( S64 , Mul , Flags ) ;

auto Fma = B . buildFMA ( S64 , FloorMul , K1 , Trunc , Flags ) ;


<s> auto CopySign = B . buildFCopysign ( Ty , C1 , Src ) ; auto Tmp1 = B . buildFAdd ( Ty , Src , CopySign ) ; auto Tmp2 = B . buildFSub ( Ty , Tmp1 , CopySign ) ; auto C2 = B . buildFConstant ( Ty , C2Val ) ; auto Fabs = B . buildFAbs ( Ty , Src ) ; auto Cond = B . buildFCmp ( CmpInst :: FCMP_OGT , LLT :: scalar ( <NUM_LIT> ) , Fabs , C2 ) ; B . buildSelect ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Cond , Src , Tmp2 ) ;


<s> <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; B . setInstr ( MI ) ; if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) { if ( ! MFI -> isEntryFunction ( ) ) { const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadLDSDecl ( Fn , <STR_LIT> , MI . getDebugLoc ( ) , DS_Warning ) ; Fn . getContext ( ) . diagnose ( BadLDSDecl ) ; B . buildIntrinsic ( <STR_LIT> , ArrayRef < Register > ( ) , true ) ; B . buildUndef ( DstReg ) ; MI . eraseFromParent ( ) ; return true ; } if ( ! <ISA_LIT>TargetLowering :: hasDefinedInitializer ( GV ) ) { const <ISA_LIT>TargetLowering * TLI = ST . getTargetLowering ( ) ; if ( ! TLI -> shouldUseLDSConstAddress ( GV ) ) { MI . getOperand ( <NUM_LIT> ) . setTargetFlags ( <ISA_LIT>InstrInfo :: MO_ABS32_LO ) ; return true ; } B . buildConstant ( DstReg , MFI -> allocateLDSGlobal ( B . getDataLayout ( ) , * GV ) ) ; MI . eraseFromParent ( ) ; return true ; } const Function & Fn = MF . getFunction ( ) ; DiagnosticInfoUnsupported BadInit ( Fn , <STR_LIT> , MI . getDebugLoc ( ) ) ; Fn . getContext ( ) . diagnose ( BadInit ) ;


<s> Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( DstReg ) ; LLT IdxTy = LLT :: scalar ( DstTy . getSizeInBits ( ) ) ; const ArgDescriptor * Arg ; const TargetRegisterClass * RC ; std :: tie ( Arg , RC ) = MFI -> getPreloadedValue ( <STR_LIT> ) ; if ( ! Arg ) return false ; Register KernargPtrReg = MRI . createGenericVirtualRegister ( DstTy ) ; if ( ! loadInputValue ( KernargPtrReg , B , Arg ) ) return false ;

B . buildPtrAdd ( DstReg , KernargPtrReg , B . buildConstant ( IdxTy , Offset ) . getReg ( <NUM_LIT> ) ) ;


<s> if ( ! MFI -> isEntryFunction ( ) ) { return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; } B . setInstr ( MI ) ; uint64_t Offset = ST . getTargetLowering ( ) -> getImplicitParameterOffset ( B . getMF ( ) , <ISA_LIT>TargetLowering :: FIRST_IMPLICIT ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( DstReg ) ; LLT IdxTy = LLT :: scalar ( DstTy . getSizeInBits ( ) ) ; const ArgDescriptor * Arg ; const TargetRegisterClass * RC ; std :: tie ( Arg , RC ) = MFI -> getPreloadedValue ( <STR_LIT> ) ; if ( ! Arg ) return false ; Register KernargPtrReg = MRI . createGenericVirtualRegister ( DstTy ) ; if ( ! loadInputValue ( KernargPtrReg , B , Arg ) ) return false ; B . buildPtrAdd ( DstReg , KernargPtrReg , B . buildConstant ( IdxTy , Offset ) . getReg ( <NUM_LIT> ) ) ;


<s> Optional < ValueAndVReg > IdxVal = getConstantVRegValWithLookThrough ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MRI ) ; if ( ! IdxVal ) return true ; Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Vec = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Ins = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT VecTy = MRI . getType ( Vec ) ; LLT EltTy = VecTy . getElementType ( ) ; assert ( EltTy == MRI . getType ( Ins ) ) ;


<s> auto Shr = B . buildAShr ( S64 , FractMask , Exp ) ; auto Not = B . buildNot ( S64 , Shr ) ; auto Tmp0 = B . buildAnd ( S64 , Src , Not ) ; auto FiftyOne = B . buildConstant ( S32 , FractBits - <NUM_LIT> ) ; auto ExpLt0 = B . buildICmp ( CmpInst :: ICMP_SLT , S1 , Exp , Zero32 ) ; auto ExpGt51 = B . buildICmp ( CmpInst :: ICMP_SGT , S1 , Exp , FiftyOne ) ; auto Tmp1 = B . buildSelect ( S64 , ExpLt0 , SignBit64 , Tmp0 ) ; B . buildSelect ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , ExpGt51 , Src , Tmp1 ) ;


<s> const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; Register Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( MRI . getType ( Src ) == S64 ) ; auto Unmerge = B . buildUnmerge ( { S32 , S32 } , Src ) ; Register Hi = Unmerge . getReg ( <NUM_LIT> ) ; auto Exp = extractF64Exponent ( Hi , B ) ; const unsigned FractBits = <NUM_LIT> ; const auto SignBitMask = B . buildConstant ( S32 , UINT32_C ( <NUM_LIT> ) << <NUM_LIT> ) ; auto SignBit = B . buildAnd ( S32 , Hi , SignBitMask ) ; const auto FractMask = B . buildConstant ( S64 , ( UINT64_C ( <NUM_LIT> ) << FractBits ) - <NUM_LIT> ) ; const auto Zero32 = B . buildConstant ( S32 , <NUM_LIT> ) ; auto SignBit64 = B . buildMerge ( S64 , { Zero32 , SignBit } ) ; auto Shr = B . buildAShr ( S64 , FractMask , Exp ) ; auto Not = B . buildNot ( S64 , Shr ) ; auto Tmp0 = B . buildAnd ( S64 , Src , Not ) ; auto FiftyOne = B . buildConstant ( S32 , FractBits - <NUM_LIT> ) ; auto ExpLt0 = B . buildICmp ( CmpInst :: ICMP_SLT , S1 , Exp , Zero32 ) ; auto ExpGt51 = B . buildICmp ( CmpInst :: ICMP_SGT , S1 , Exp , FiftyOne ) ;

auto Tmp1 = B . buildSelect ( S64 , ExpLt0 , SignBit64 , Tmp0 ) ;


<s> auto Hi32 = B . buildExtract ( LLT :: scalar ( <NUM_LIT> ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> ) ; B . buildICmp ( ICmpInst :: ICMP_EQ , MI . getOperand ( <NUM_LIT> ) , Hi32 , ApertureReg ) ; MI . eraseFromParent ( ) ; return true ;


<s> const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; assert ( MRI . getType ( Src ) == S64 && MRI . getType ( Dst ) == S64 ) ; auto Unmerge = B . buildUnmerge ( { S32 , S32 } , Src ) ; auto CvtHi = Signed ? B . build<ISA_LIT>TOFP ( S64 , Unmerge . getReg ( <NUM_LIT> ) ) : B . buildUITOFP ( S64 , Unmerge . getReg ( <NUM_LIT> ) ) ; auto CvtLo = B . buildUITOFP ( S64 , Unmerge . getReg ( <NUM_LIT> ) ) ;

auto ThirtyTwo = B . buildConstant ( S32 , <NUM_LIT> ) ;


<s> auto Unmerge = B . buildUnmerge ( { S32 , S32 } , Src ) ; auto CvtHi = Signed ? B . build<ISA_LIT>TOFP ( S64 , Unmerge . getReg ( <NUM_LIT> ) ) : B . buildUITOFP ( S64 , Unmerge . getReg ( <NUM_LIT> ) ) ; auto CvtLo = B . buildUITOFP ( S64 , Unmerge . getReg ( <NUM_LIT> ) ) ;


<s> LLT ConstPtr = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ;

auto Cast = B . buildAddrSpaceCast ( ConstPtr , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> if ( IsIEEEOp ) return true ; MachineIRBuilder HelperBuilder ( MI ) ; GISelObserverWrapper DummyObserver ; LegalizerHelper Helper ( MF , DummyObserver , HelperBuilder ) ; HelperBuilder . setInstr ( MI ) ; return Helper . lowerFMinNumMaxNum ( MI ) == LegalizerHelper :: Legalized ;


<s> if ( ! loadInputValue ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , B , Arg ) ) return false ;


<s> B . setInstr ( MI ) ; const ArgDescriptor * Arg = getArgDescriptor ( B , ArgType ) ;


<s> Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT Ty = B . getMRI ( ) -> getType ( Dst ) ; unsigned Size = Ty . getSizeInBits ( ) ; MachineFunction & MF = B . getMF ( ) ; Observer . changingInstr ( MI ) ; MI . setDesc ( B . getTII ( ) . get ( <STR_LIT> ) ) ;


<s> bool <ISA_LIT>LegalizerInfo :: legalizeSDIV_SREM ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { if ( MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) == LLT :: scalar ( <NUM_LIT> ) ) return legalizeSDIV_SREM32 ( MI , MRI , B ) ;


<s> LHS = B . buildAdd ( S32 , LHS , LHSign ) . getReg ( <NUM_LIT> ) ; RHS = B . buildAdd ( S32 , RHS , RHSign ) . getReg ( <NUM_LIT> ) ; LHS = B . buildXor ( S32 , LHS , LHSign ) . getReg ( <NUM_LIT> ) ; RHS = B . buildXor ( S32 , RHS , RHSign ) . getReg ( <NUM_LIT> ) ; Register UDivRem = MRI . createGenericVirtualRegister ( S32 ) ; legalizeUDIV_UREM32Impl ( B , UDivRem , LHS , RHS , IsRem ) ; if ( IsRem ) {


<s> auto LHSign = B . buildAShr ( S32 , LHS , ThirtyOne ) ; auto RHSign = B . buildAShr ( S32 , LHS , ThirtyOne ) ; LHS = B . buildAdd ( S32 , LHS , LHSign ) . getReg ( <NUM_LIT> ) ; RHS = B . buildAdd ( S32 , RHS , RHSign ) . getReg ( <NUM_LIT> ) ; LHS = B . buildXor ( S32 , LHS , LHSign ) . getReg ( <NUM_LIT> ) ; RHS = B . buildXor ( S32 , RHS , RHSign ) . getReg ( <NUM_LIT> ) ; Register UDivRem = MRI . createGenericVirtualRegister ( S32 ) ; legalizeUDIV_UREM32Impl ( B , UDivRem , LHS , RHS , IsRem ) ; if ( IsRem ) { auto RSign = LHSign ; UDivRem = B . buildXor ( S32 , UDivRem , RSign ) . getReg ( <NUM_LIT> ) ; B . buildSub ( DstReg , UDivRem , RSign ) ; } else { auto DSign = B . buildXor ( S32 , LHSign , RHSign ) ;

UDivRem = B . buildXor ( S32 , UDivRem , DSign ) . getReg ( <NUM_LIT> ) ;


<s> LLT Ty = MRI . getType ( DstReg ) ; unsigned Flags = MI . getFlags ( ) ; Register TrigVal ; auto OneOver2Pi = B . buildFConstant ( Ty , <NUM_LIT> / M_PI ) ; if ( ST . hasTrigReducedRange ( ) ) { auto MulVal = B . buildFMul ( Ty , SrcReg , OneOver2Pi , Flags ) ; TrigVal = B . buildIntrinsic ( <STR_LIT> , { Ty } , false ) . addUse ( MulVal . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) . getReg ( <NUM_LIT> ) ; } else TrigVal = B . buildFMul ( Ty , SrcReg , OneOver2Pi , Flags ) . getReg ( <NUM_LIT> ) ; <STR_LIT> TrigIntrin = MI . getOpcode ( ) == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; B . buildIntrinsic ( TrigIntrin , makeArrayRef < Register > ( DstReg ) , false ) . addUse ( TrigVal ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ;


<s> Register LiveIn = getLiveInRegister ( B , MRI , SGPR01 , LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) , false ) ; if ( ! loadInputValue ( LiveIn , B , Arg ) ) return false ; B . buildCopy ( SGPR01 , LiveIn ) ; B . buildInstr ( <STR_LIT> ) . addImm ( GCNSubtarget :: TrapIDLLVMTrap ) . addReg ( SGPR01 , RegState :: Implicit ) ; } MI . eraseFromParent ( ) ; return true ;


<s> if ( ! Arg ) return false ; MachineRegisterInfo & MRI = * B . getMRI ( ) ; Register SGPR01 ( <STR_LIT> ) ; Register LiveIn = getLiveInRegister ( B , MRI , SGPR01 , LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) , false ) ; if ( ! loadInputValue ( LiveIn , B , Arg ) ) return false ; B . buildCopy ( SGPR01 , LiveIn ) ; B . buildInstr ( <STR_LIT> ) . addImm ( GCNSubtarget :: TrapIDLLVMTrap ) . addReg ( SGPR01 , RegState :: Implicit ) ;


<s> LLT Ty = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ;

if ( Ty == LLT :: scalar ( <NUM_LIT> ) ) return legalizeUDIV_UREM32 ( MI , MRI , B ) ;


<s> bool <ISA_LIT>LegalizerInfo :: legalizeUDIV_UREM ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { LLT Ty = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) ) return legalizeUDIV_UREM32 ( MI , MRI , B ) ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) ) return legalizeUDIV_UREM64 ( MI , MRI , B ) ;


<s> B . setInstr ( MI ) ; const bool IsRem = MI . getOpcode ( ) == <STR_LIT> ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Num = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Den = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; legalizeUDIV_UREM32Impl ( B , DstReg , Num , Den , IsRem ) ;


<s> Register Num = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Den = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; legalizeUDIV_UREM32Impl ( B , DstReg , Num , Den , IsRem ) ;


<s> auto RCP_S_E = B . buildSub ( S32 , RCP , E ) ; auto Tmp0 = B . buildSelect ( S32 , CmpRcpHiZero , RCP_A_E , RCP_S_E ) ; auto Quotient = B . buildUMulH ( S32 , Tmp0 , Num ) ; auto Num_S_Remainder = B . buildMul ( S32 , Quotient , Den ) ; auto Remainder = B . buildSub ( S32 , Num , Num_S_Remainder ) ; auto Remainder_GE_Den = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , Remainder , Den ) ; auto Remainder_GE_Zero = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , Num , Num_S_Remainder ) ; auto Tmp1 = B . buildAnd ( S1 , Remainder_GE_Den , Remainder_GE_Zero ) ; auto One = B . buildConstant ( S32 , <NUM_LIT> ) ; auto Quotient_A_One = B . buildAdd ( S32 , Quotient , One ) ; auto Quotient_S_One = B . buildSub ( S32 , Quotient , One ) ; auto Div = B . buildSelect ( S32 , Tmp1 , Quotient , Quotient_A_One ) ;


<s> auto NEG_RCP_LO = B . buildSub ( S32 , Zero , RCP_LO ) ; auto CmpRcpHiZero = B . buildICmp ( CmpInst :: ICMP_EQ , S1 , RCP_HI , Zero ) ; auto ABS_RCP_LO = B . buildSelect ( S32 , CmpRcpHiZero , NEG_RCP_LO , RCP_LO ) ; auto E = B . buildUMulH ( S32 , ABS_RCP_LO , RCP ) ; auto RCP_A_E = B . buildAdd ( S32 , RCP , E ) ; auto RCP_S_E = B . buildSub ( S32 , RCP , E ) ; auto Tmp0 = B . buildSelect ( S32 , CmpRcpHiZero , RCP_A_E , RCP_S_E ) ; auto Quotient = B . buildUMulH ( S32 , Tmp0 , Num ) ; auto Num_S_Remainder = B . buildMul ( S32 , Quotient , Den ) ; auto Remainder = B . buildSub ( S32 , Num , Num_S_Remainder ) ; auto Remainder_GE_Den = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , Remainder , Den ) ; auto Remainder_GE_Zero = B . buildICmp ( CmpInst :: ICMP_UGE , S1 , Num , Num_S_Remainder ) ; auto Tmp1 = B . buildAnd ( S1 , Remainder_GE_Den , Remainder_GE_Zero ) ; auto One = B . buildConstant ( S32 , <NUM_LIT> ) ; auto Quotient_A_One = B . buildAdd ( S32 , Quotient , One ) ; auto Quotient_S_One = B . buildSub ( S32 , Quotient , One ) ; auto Div = B . buildSelect ( S32 , Tmp1 , Quotient , Quotient_A_One ) ; if ( IsRem ) { Div = B . buildSelect ( S32 , Remainder_GE_Zero , Div , Quotient_S_One ) ; auto Remainder_S_Den = B . buildSub ( S32 , Remainder , Den ) ; auto Remainder_A_Den = B . buildAdd ( S32 , Remainder , Den ) ; auto Rem = B . buildSelect ( S32 , Tmp1 , Remainder_S_Den , Remainder ) ; B . buildSelect ( DstReg , Remainder_GE_Zero , Rem , Remainder_A_Den ) ; } else { B . buildSelect ( DstReg , Remainder_GE_Zero , Div , Quotient_S_One ) ; }


<s> if ( ! Arg -> isRegister ( ) || ! Arg -> getRegister ( ) . isValid ( ) ) return false ; Register SrcReg = Arg -> getRegister ( ) ; assert ( SrcReg . isPhysical ( ) && <STR_LIT> ) ; assert ( DstReg . isVirtual ( ) && <STR_LIT> ) ; MachineRegisterInfo & MRI = * B . getMRI ( ) ; LLT Ty = MRI . getType ( DstReg ) ; Register LiveIn = getLiveInRegister ( B , MRI , SrcReg , Ty ) ; if ( Arg -> isMasked ( ) ) { const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const unsigned Mask = Arg -> getMask ( ) ; const unsigned Shift = countTrailingZeros < unsigned > ( Mask ) ; Register AndMaskSrc = LiveIn ; if ( Shift != <NUM_LIT> ) {

auto ShiftAmt = B . buildConstant ( S32 , Shift ) ;


<s> return [ = ] ( const LegalityQuery & Query ) { const LLT Ty = Query . Types [ TypeIdx ] ; const LLT EltTy = Ty . getElementType ( ) ; const int Size = Ty . getSizeInBits ( ) ; const int EltSize = EltTy . getSizeInBits ( ) ; const int NextMul32 = ( Size + <NUM_LIT> ) / <NUM_LIT> ; assert ( EltSize < <NUM_LIT> ) ; const int NewNumElts = ( <NUM_LIT> * NextMul32 + EltSize - <NUM_LIT> ) / EltSize ; return std :: make_pair ( TypeIdx , LLT :: vector ( NewNumElts , EltTy ) ) ; } ;


<s> static LegalizeMutation oneMoreElement ( unsigned TypeIdx ) { return [ = ] ( const LegalityQuery & Query ) { const LLT Ty = Query . Types [ TypeIdx ] ; const LLT EltTy = Ty . getElementType ( ) ; return std :: make_pair ( TypeIdx , LLT :: vector ( Ty . getNumElements ( ) + <NUM_LIT> , EltTy ) ) ; } ;


<s> AddrReg = B . buildBitcast ( V2S16 , AddrReg ) . getReg ( <NUM_LIT> ) ; PackedAddrs . push_back ( AddrReg ) ; } else { if ( ( ( I + <NUM_LIT> ) >= ( AddrIdx + NumVAddrs ) ) || ( ( NumGradients / <NUM_LIT> ) % <NUM_LIT> == <NUM_LIT> && ( I == DimIdx + ( NumGradients / <NUM_LIT> ) - <NUM_LIT> || I == DimIdx + NumGradients - <NUM_LIT> ) ) || ! MI . getOperand ( I + <NUM_LIT> ) . isReg ( ) ) {

PackedAddrs . push_back ( B . buildBuildVector ( V2S16 , { AddrReg , B . buildUndef ( S16 ) . getReg ( <NUM_LIT> ) } ) . getReg ( <NUM_LIT> ) ) ;


<s> return [ = ] ( const LegalityQuery & Query ) { return Query . Types [ TypeIdx ] . getSizeInBits ( ) == Size ; } ;


<s> static LegalityPredicate smallerThan ( unsigned TypeIdx0 , unsigned TypeIdx1 ) { return [ = ] ( const LegalityQuery & Query ) { return Query . Types [ TypeIdx0 ] . getSizeInBits ( ) < Query . Types [ TypeIdx1 ] . getSizeInBits ( ) ;


<s> std :: tie ( BaseReg , TotalConstOffset , OffsetDef ) = <STR_LIT> ( * B . getMRI ( ) , OrigOffset ) ; unsigned ImmOffset = TotalConstOffset ; unsigned Overflow = ImmOffset & ~ MaxImm ; ImmOffset -= Overflow ; if ( ( int32_t ) Overflow < <NUM_LIT> ) { Overflow += ImmOffset ; ImmOffset = <NUM_LIT> ; } if ( Overflow != <NUM_LIT> ) { if ( ! BaseReg ) { BaseReg = B . buildConstant ( S32 , Overflow ) . getReg ( <NUM_LIT> ) ; } else { auto OverflowVal = B . buildConstant ( S32 , Overflow ) ; BaseReg = B . buildAdd ( S32 , BaseReg , OverflowVal ) . getReg ( <NUM_LIT> ) ;


<s> ImmOffset -= Overflow ; if ( ( int32_t ) Overflow < <NUM_LIT> ) { Overflow += ImmOffset ; ImmOffset = <NUM_LIT> ; } if ( Overflow != <NUM_LIT> ) { if ( ! BaseReg ) { BaseReg = B . buildConstant ( S32 , Overflow ) . getReg ( <NUM_LIT> ) ; } else { auto OverflowVal = B . buildConstant ( S32 , Overflow ) ; BaseReg = B . buildAdd ( S32 , BaseReg , OverflowVal ) . getReg ( <NUM_LIT> ) ; } } if ( ! BaseReg ) BaseReg = B . buildConstant ( S32 , <NUM_LIT> ) . getReg ( <NUM_LIT> ) ; return std :: make_tuple ( BaseReg , ImmOffset , TotalConstOffset ) ;


<s> MachineBasicBlock :: iterator Next = std :: next ( UseMI . getIterator ( ) ) ; if ( Next != MI . getParent ( ) -> end ( ) ) { if ( Next -> getOpcode ( ) != <STR_LIT> ) return nullptr ; Br = & * Next ; }


<s> static MachineInstr * verifyCFIntrinsic ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineInstr * & Br ) { Register CondDef = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! MRI . hasOneNonDBGUse ( CondDef ) ) return nullptr ; MachineInstr & UseMI = * MRI . use_instr_nodbg_begin ( CondDef ) ;

if ( UseMI . getParent ( ) != MI . getParent ( ) || UseMI . getOpcode ( ) != <STR_LIT> ) return nullptr ;


<s> return [ = ] ( const LegalityQuery & Query ) {

return Query . Types [ TypeIdx ] . getElementType ( ) == Type ;


<s> Register LiveIn = MRI . getLiveInVirtReg ( Reg ) ; if ( LiveIn ) return LiveIn ; Register NewReg = MRI . createGenericVirtualRegister ( Ty ) ;


<s> Register NewReg = MRI . createGenericVirtualRegister ( Ty ) ;


<s> MRI . setType ( GetReg , S32 ) ; auto ShiftAmt = B . buildConstant ( S32 , WidthM1 + <NUM_LIT> ) ; B . buildInstr ( TargetOpcode :: G_SHL ) . addDef ( ApertureReg ) . addUse ( GetReg ) . addUse ( ShiftAmt . getReg ( <NUM_LIT> ) ) ; return ApertureReg ; } Register QueuePtr = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! loadInputValue ( QueuePtr , B , & MFI -> getArgInfo ( ) . QueuePtr ) ) return Register ( ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; Value * V = UndefValue :: get ( PointerType :: get ( Type :: getInt8Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ) ;

MachinePointerInfo PtrInfo ( V , StructOffset ) ;


<s> B . setInstr ( MI ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( Dst ) ; LLT SrcTy = MRI . getType ( Src ) ; unsigned DestAS = DstTy . getAddressSpace ( ) ; unsigned SrcAS = SrcTy . getAddressSpace ( ) ; assert ( ! DstTy . isVector ( ) ) ; const <ISA_LIT>TargetMachine & TM = static_cast < const <ISA_LIT>TargetMachine & > ( MF . getTarget ( ) ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; if ( ST . getTargetLowering ( ) -> isNoopAddrSpaceCast ( SrcAS , DestAS ) ) { MI . setDesc ( B . getTII ( ) . get ( TargetOpcode :: G_BITCAST ) ) ; return true ; } if ( DestAS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { B . buildExtract ( Dst , Src , <NUM_LIT> ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) {

const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;


<s> auto FlatNull = B . buildConstant ( SrcTy , <NUM_LIT> ) ; Register PtrLo32 = MRI . createGenericVirtualRegister ( DstTy ) ; B . buildExtract ( PtrLo32 , Src , <NUM_LIT> ) ; Register CmpRes = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; B . buildICmp ( CmpInst :: ICMP_NE , CmpRes , Src , FlatNull . getReg ( <NUM_LIT> ) ) ; B . buildSelect ( Dst , CmpRes , PtrLo32 , SegmentNull . getReg ( <NUM_LIT> ) ) ; MI . eraseFromParent ( ) ; return true ; } if ( SrcAS != <ISA_LIT>AS :: LOCAL_ADDRESS && SrcAS != <ISA_LIT>AS :: PRIVATE_ADDRESS ) return false ; if ( ! ST . hasFlatAddressSpace ( ) ) return false ; auto SegmentNull = B . buildConstant ( SrcTy , TM . getNullPointerValue ( SrcAS ) ) ; auto FlatNull = B . buildConstant ( DstTy , TM . getNullPointerValue ( DestAS ) ) ; Register ApertureReg = getSegmentAperture ( SrcAS , MRI , B ) ; if ( ! ApertureReg . isValid ( ) ) return false ; Register CmpRes = MRI . createGenericVirtualRegister ( LLT :: scalar ( <NUM_LIT> ) ) ; B . buildICmp ( CmpInst :: ICMP_NE , CmpRes , Src , SegmentNull . getReg ( <NUM_LIT> ) ) ;

Register BuildPtr = MRI . createGenericVirtualRegister ( DstTy ) ;


<s> switch ( MI . getOpcode ( ) ) { case TargetOpcode :: G_ADDRSPACE_CAST : return legalizeAddrSpaceCast ( MI , MRI , B ) ; case TargetOpcode :: G_FRINT : return legalizeFrint ( MI , MRI , B ) ; case TargetOpcode :: G_FCEIL : return legalizeFceil ( MI , MRI , B ) ; case TargetOpcode :: G_INTRIN<ISA_LIT>C_TRUNC : return legalizeIntrinsicTrunc ( MI , MRI , B ) ; case TargetOpcode :: G_<ISA_LIT>TOFP : return legalizeITOFP ( MI , MRI , B , true ) ; case TargetOpcode :: G_UITOFP : return legalizeITOFP ( MI , MRI , B , false ) ; case TargetOpcode :: G_FMINNUM : case TargetOpcode :: G_FMAXNUM : case TargetOpcode :: G_FMINNUM_IEEE :


<s> assert ( EltTy == MRI . getType ( Dst ) ) ; B . setInstr ( MI ) ; if ( IdxVal . getValue ( ) < VecTy . getNumElements ( ) ) B . buildExtract ( Dst , Vec , IdxVal . getValue ( ) * EltTy . getSizeInBits ( ) ) ; else B . buildUndef ( Dst ) ;


<s> if ( IdxVal . getValue ( ) < VecTy . getNumElements ( ) ) B . buildExtract ( Dst , Vec , IdxVal . getValue ( ) * EltTy . getSizeInBits ( ) ) ; else B . buildUndef ( Dst ) ; MI . eraseFromParent ( ) ;


<s> if ( ! Unsafe && ResTy == S32 && ST . hasFP32Denormals ( ) ) return false ; if ( auto CLHS = getConstantFPVRegVal ( LHS , MRI ) ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( RHS ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { auto FNeg = B . buildFNeg ( ResTy , RHS , Flags ) ; B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( FNeg . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ; } } if ( Unsafe ) {

auto RCP = B . buildIntrinsic ( <STR_LIT> , { ResTy } , false ) . addUse ( RHS ) . setMIFlags ( Flags ) ;


<s> const auto One = B . buildFConstant ( S64 , <NUM_LIT> ) ; auto Lt0 = B . buildFCmp ( CmpInst :: FCMP_OGT , S1 , Src , Zero ) ; auto NeTrunc = B . buildFCmp ( CmpInst :: FCMP_ONE , S1 , Src , Trunc ) ; auto And = B . buildAnd ( S1 , Lt0 , NeTrunc ) ;

auto Add = B . buildSelect ( S64 , And , One , Zero ) ;


<s> bool <ISA_LIT>LegalizerInfo :: legalizeFDIV ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { B . setInstr ( MI ) ; Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT DstTy = MRI . getType ( Dst ) ; LLT S16 = LLT :: scalar ( <NUM_LIT> ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; if ( legalizeFastUnsafeFDIV ( MI , MRI , B ) ) return true ; if ( DstTy == S16 ) return legalizeFDIV16 ( MI , MRI , B ) ; if ( DstTy == S32 ) return legalizeFDIV32 ( MI , MRI , B ) ; return false ;


<s> LLT DstTy = MRI . getType ( Dst ) ; LLT S16 = LLT :: scalar ( <NUM_LIT> ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; if ( legalizeFastUnsafeFDIV ( MI , MRI , B ) ) return true ;

if ( DstTy == S16 ) return legalizeFDIV16 ( MI , MRI , B ) ;


<s> Register RHS = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; uint16_t Flags = MI . getFlags ( ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; LLT S1 = LLT :: scalar ( <NUM_LIT> ) ; auto One = B . buildFConstant ( S32 , <NUM_LIT> ) ; auto DenominatorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( RHS ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; auto NumeratorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; auto ApproxRcp = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( DenominatorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto NegDivScale0 = B . buildFNeg ( S32 , DenominatorScaled , Flags ) ; if ( ! ST . hasFP32Denormals ( ) ) toggleSPDenormMode ( true , ST , B ) ; auto Fma0 = B . buildFMA ( S32 , NegDivScale0 , ApproxRcp , One , Flags ) ; auto Fma1 = B . buildFMA ( S32 , Fma0 , ApproxRcp , ApproxRcp , Flags ) ;

auto Mul = B . buildFMul ( S32 , NumeratorScaled , Fma1 , Flags ) ;


<s> Register Res = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register LHS = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register RHS = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; uint16_t Flags = MI . getFlags ( ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; LLT S1 = LLT :: scalar ( <NUM_LIT> ) ; auto One = B . buildFConstant ( S32 , <NUM_LIT> ) ; auto DenominatorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( RHS ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; auto NumeratorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ;

auto ApproxRcp = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( DenominatorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ;


<s> if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! ST . hasFP32Denormals ( ) ) return true ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! ST . hasFP16Denormals ( ) ) return true ; MachineFunction & MF = B . getMF ( ) ; MachineIRBuilder HelperBuilder ( MI ) ; GISelObserverWrapper DummyObserver ; LegalizerHelper Helper ( MF , DummyObserver , HelperBuilder ) ; HelperBuilder . setMBB ( * MI . getParent ( ) ) ;

return Helper . lowerFMad ( MI ) == LegalizerHelper :: Legalized ;


<s> LLT Ty = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; assert ( Ty . isScalar ( ) ) ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! ST . hasFP32Denormals ( ) ) return true ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! ST . hasFP16Denormals ( ) ) return true ;


<s> if ( TLI -> shouldEmitPCReloc ( GV ) ) { buildPCRelGlobalAddress ( DstReg , Ty , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_REL32 ) ; MI . eraseFromParent ( ) ; return true ; } LLT PtrTy = LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ; Register GOTAddr = MRI . createGenericVirtualRegister ( PtrTy ) ; MachineMemOperand * GOTMMO = MF . getMachineMemOperand ( MachinePointerInfo :: getGOT ( MF ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , <NUM_LIT> , <NUM_LIT> ) ; buildPCRelGlobalAddress ( GOTAddr , PtrTy , B , GV , <NUM_LIT> , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; if ( Ty . getSizeInBits ( ) == <NUM_LIT> ) { auto Load = B . buildLoad ( PtrTy , GOTAddr , * GOTMMO ) ; B . buildExtract ( DstReg , Load , <NUM_LIT> ) ; } else B . buildLoad ( DstReg , GOTAddr , * GOTMMO ) ;


<s> const <ISA_LIT>MachineFunctionInfo * MFI = B . getMF ( ) . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! MFI -> isEntryFunction ( ) ) { return legalizePreloadedArgIntrin ( MI , MRI , B , <STR_LIT> ) ; } B . setInstr ( MI ) ; uint64_t Offset = ST . getTargetLowering ( ) -> getImplicitParameterOffset ( B . getMF ( ) , <ISA_LIT>TargetLowering :: FIRST_IMPLICIT ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> Register Ins = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; LLT VecTy = MRI . getType ( Vec ) ; LLT EltTy = VecTy . getElementType ( ) ; assert ( EltTy == MRI . getType ( Ins ) ) ; B . setInstr ( MI ) ; if ( IdxVal . getValue ( ) < VecTy . getNumElements ( ) ) B . buildInsert ( Dst , Vec , Ins , IdxVal . getValue ( ) * EltTy . getSizeInBits ( ) ) ; else B . buildUndef ( Dst ) ; MI . eraseFromParent ( ) ;


<s> const auto SignBitMask = B . buildConstant ( S32 , UINT32_C ( <NUM_LIT> ) << <NUM_LIT> ) ; auto SignBit = B . buildAnd ( S32 , Hi , SignBitMask ) ; const auto FractMask = B . buildConstant ( S64 , ( UINT64_C ( <NUM_LIT> ) << FractBits ) - <NUM_LIT> ) ; const auto Zero32 = B . buildConstant ( S32 , <NUM_LIT> ) ; auto SignBit64 = B . buildMerge ( S64 , { Zero32 . getReg ( <NUM_LIT> ) , SignBit . getReg ( <NUM_LIT> ) } ) ; auto Shr = B . buildAShr ( S64 , FractMask , Exp ) ; auto Not = B . buildNot ( S64 , Shr ) ;

auto Tmp0 = B . buildAnd ( S64 , Src , Not ) ;


<s> const LLT S1 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; Register Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( MRI . getType ( Src ) == S64 ) ; auto Unmerge = B . buildUnmerge ( { S32 , S32 } , Src ) ; Register Hi = Unmerge . getReg ( <NUM_LIT> ) ; auto Exp = extractF64Exponent ( Hi , B ) ; const unsigned FractBits = <NUM_LIT> ; const auto SignBitMask = B . buildConstant ( S32 , UINT32_C ( <NUM_LIT> ) << <NUM_LIT> ) ; auto SignBit = B . buildAnd ( S32 , Hi , SignBitMask ) ; const auto FractMask = B . buildConstant ( S64 , ( UINT64_C ( <NUM_LIT> ) << FractBits ) - <NUM_LIT> ) ; const auto Zero32 = B . buildConstant ( S32 , <NUM_LIT> ) ;

auto SignBit64 = B . buildMerge ( S64 , { Zero32 . getReg ( <NUM_LIT> ) , SignBit . getReg ( <NUM_LIT> ) } ) ;


<s> bool <ISA_LIT>LegalizerInfo :: legalizePreloadedArgIntrin ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B , <STR_LIT> ArgType ) const { B . setInstr ( MI ) ;

const <ISA_LIT>MachineFunctionInfo * MFI = B . getMF ( ) . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;


<s> } if ( Ty . isVector ( ) ) { if ( Ty . getElementType ( ) == S16 && Ty . getNumElements ( ) <= <NUM_LIT> ) { if ( IsFormat ) MI . getOperand ( <NUM_LIT> ) . setReg ( handleD16VData ( B , MRI , VData ) ) ; return true ; }

return Ty . getElementType ( ) == S32 && Ty . getNumElements ( ) <= <NUM_LIT> ;


<s> assert ( Arg -> getRegister ( ) . isPhysical ( ) ) ; MachineRegisterInfo & MRI = * B . getMRI ( ) ; LLT Ty = MRI . getType ( DstReg ) ; Register LiveIn = getLiveInRegister ( MRI , Arg -> getRegister ( ) , Ty ) ; if ( Arg -> isMasked ( ) ) { const LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; const unsigned Mask = Arg -> getMask ( ) ; const unsigned Shift = countTrailingZeros < unsigned > ( Mask ) ; Register AndMaskSrc = LiveIn ; if ( Shift != <NUM_LIT> ) { auto ShiftAmt = B . buildConstant ( S32 , Shift ) ; AndMaskSrc = B . buildLShr ( S32 , LiveIn , ShiftAmt ) . getReg ( <NUM_LIT> ) ; } B . buildAnd ( DstReg , AndMaskSrc , B . buildConstant ( S32 , Mask >> Shift ) ) ; } else B . buildCopy ( DstReg , LiveIn ) ; if ( ! MRI . getVRegDef ( LiveIn ) ) { MachineBasicBlock & OrigInsBB = B . getMBB ( ) ; auto OrigInsPt = B . getInsertPt ( ) ; MachineBasicBlock & EntryMBB = B . getMF ( ) . front ( ) ; EntryMBB . addLiveIn ( Arg -> getRegister ( ) ) ; B . setInsertPt ( EntryMBB , EntryMBB . begin ( ) ) ; B . buildCopy ( LiveIn , Arg -> getRegister ( ) ) ; B . setInsertPt ( OrigInsBB , OrigInsPt ) ; }


<s> if ( ST . hasDenormModeInst ( ) ) { unsigned DPDenormModeDefault = ST . hasFP64Denormals ( ) ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT ; unsigned NewDenormModeValue = SPDenormMode | ( DPDenormModeDefault << <NUM_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addImm ( NewDenormModeValue ) ; } else { unsigned SPDenormModeBitField = <STR_LIT> :: ID_MODE | ( <NUM_LIT> << <STR_LIT> :: OFFSET_SHIFT_ ) | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ; B . buildInstr ( <STR_LIT> ) . addImm ( SPDenormMode ) . addImm ( SPDenormModeBitField ) ;


<s> B . buildInstr ( <STR_LIT> ) . addImm ( NewDenormModeValue ) ; } else { unsigned SPDenormModeBitField = <STR_LIT> :: ID_MODE | ( <NUM_LIT> << <STR_LIT> :: OFFSET_SHIFT_ ) | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ; B . buildInstr ( <STR_LIT> ) . addImm ( SPDenormMode ) . addImm ( SPDenormModeBitField ) ;


<s> MachineInstr & UseMI = * MRI . use_instr_nodbg_begin ( CondDef ) ; return UseMI . getParent ( ) == MI . getParent ( ) && UseMI . getOpcode ( ) == <STR_LIT> ? & UseMI : nullptr ;


<s> <STR_LIT> :: OFFSET_SRC_PRIVATE_BASE ; unsigned WidthM1 = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: WIDTH_M1_SRC_SHARED_BASE : <STR_LIT> :: WIDTH_M1_SRC_PRIVATE_BASE ; unsigned Encoding = <STR_LIT> :: ID_MEM_BASES << <STR_LIT> :: ID_SHIFT_ | Offset << <STR_LIT> :: OFFSET_SHIFT_ | WidthM1 << <STR_LIT> :: WIDTH_M1_SHIFT_ ; Register ApertureReg = MRI . createGenericVirtualRegister ( S32 ) ; Register GetReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; B . buildInstr ( <STR_LIT> ) . addDef ( GetReg ) . addImm ( Encoding ) ; MRI . setType ( GetReg , S32 ) ; auto ShiftAmt = B . buildConstant ( S32 , WidthM1 + <NUM_LIT> ) ; B . buildInstr ( TargetOpcode :: G_SHL ) . addDef ( ApertureReg ) . addUse ( GetReg ) . addUse ( ShiftAmt . getReg ( <NUM_LIT> ) ) ; return ApertureReg ; } Register QueuePtr = MRI . createGenericVirtualRegister ( LLT :: pointer ( <ISA_LIT>AS :: CONSTANT_ADDRESS , <NUM_LIT> ) ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! loadInputValue ( QueuePtr , B , & MFI -> getArgInfo ( ) . QueuePtr ) ) return Register ( ) ;


<s> B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( RHS ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { auto FNeg = B . buildFNeg ( ResTy , RHS , Flags ) ; B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( FNeg . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ; } } if ( Unsafe ) { auto RCP = B . buildIntrinsic ( <STR_LIT> , { ResTy } , false ) . addUse ( RHS ) . setMIFlags ( Flags ) ; B . buildFMul ( Res , LHS , RCP , Flags ) ; MI . eraseFromParent ( ) ;


<s> LLT S64 = LLT :: scalar ( <NUM_LIT> ) ; const MachineFunction & MF = B . getMF ( ) ; bool Unsafe = MF . getTarget ( ) . Options . UnsafeFPMath || MI . getFlag ( MachineInstr :: FmArcp ) ; if ( ! MF . getTarget ( ) . Options . UnsafeFPMath && ResTy == S64 ) return false ; if ( ! Unsafe && ResTy == S32 && MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> getMode ( ) . FP32Denormals ) return false ; if ( auto CLHS = getConstantFPVRegVal ( LHS , MRI ) ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( RHS ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { auto FNeg = B . buildFNeg ( ResTy , RHS , Flags ) ; B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( FNeg . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ; return true ; } } if ( Unsafe ) { auto RCP = B . buildIntrinsic ( <STR_LIT> , { ResTy } , false ) . addUse ( RHS ) . setMIFlags ( Flags ) ; B . buildFMul ( Res , LHS , RCP , Flags ) ;


<s> <STR_LIT> Mode = MFI -> getMode ( ) ; uint16_t Flags = MI . getFlags ( ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; LLT S1 = LLT :: scalar ( <NUM_LIT> ) ; auto One = B . buildFConstant ( S32 , <NUM_LIT> ) ; auto DenominatorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( RHS ) . addUse ( LHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto NumeratorScaled = B . buildIntrinsic ( <STR_LIT> , { S32 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto ApproxRcp = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( DenominatorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; auto NegDivScale0 = B . buildFNeg ( S32 , DenominatorScaled , Flags ) ; if ( ! Mode . FP32Denormals ) toggleSPDenormMode ( true , B , ST , Mode ) ; auto Fma0 = B . buildFMA ( S32 , NegDivScale0 , ApproxRcp , One , Flags ) ; auto Fma1 = B . buildFMA ( S32 , Fma0 , ApproxRcp , ApproxRcp , Flags ) ; auto Mul = B . buildFMul ( S32 , NumeratorScaled , Fma1 , Flags ) ; auto Fma2 = B . buildFMA ( S32 , NegDivScale0 , Mul , NumeratorScaled , Flags ) ; auto Fma3 = B . buildFMA ( S32 , Fma2 , Fma1 , Mul , Flags ) ; auto Fma4 = B . buildFMA ( S32 , NegDivScale0 , Fma3 , NumeratorScaled , Flags ) ; if ( ! Mode . FP32Denormals ) toggleSPDenormMode ( false , B , ST , Mode ) ; auto Fmas = B . buildIntrinsic ( <STR_LIT> , { S32 } , false ) . addUse ( Fma4 . getReg ( <NUM_LIT> ) ) . addUse ( Fma1 . getReg ( <NUM_LIT> ) ) . addUse ( Fma3 . getReg ( <NUM_LIT> ) ) . addUse ( NumeratorScaled . getReg ( <NUM_LIT> ) ) . setMIFlags ( Flags ) ; B . buildIntrinsic ( <STR_LIT> , Res , false ) . addUse ( Fmas . getReg ( <NUM_LIT> ) ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ; MI . eraseFromParent ( ) ;


<s> auto Fma2 = B . buildFMA ( S64 , NegDivScale0 , Fma1 , One , Flags ) ; auto DivScale1 = B . buildIntrinsic ( <STR_LIT> , { S64 , S1 } , false ) . addUse ( LHS ) . addUse ( RHS ) . addImm ( <NUM_LIT> ) . setMIFlags ( Flags ) ; auto Fma3 = B . buildFMA ( S64 , Fma1 , Fma2 , Fma1 , Flags ) ; auto Mul = B . buildMul ( S64 , DivScale1 . getReg ( <NUM_LIT> ) , Fma3 , Flags ) ; auto Fma4 = B . buildFMA ( S64 , NegDivScale0 , Mul , DivScale1 . getReg ( <NUM_LIT> ) , Flags ) ; Register Scale ; if ( ! ST . hasUsableDivScaleConditionOutput ( ) ) { Scale = MRI . createGenericVirtualRegister ( S1 ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto NumUnmerge = B . buildUnmerge ( S32 , LHS ) ; auto DenUnmerge = B . buildUnmerge ( S32 , RHS ) ; auto Scale0Unmerge = B . buildUnmerge ( S32 , DivScale0 ) ; auto Scale1Unmerge = B . buildUnmerge ( S32 , DivScale1 ) ; auto CmpNum = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , NumUnmerge . getReg ( <NUM_LIT> ) , Scale1Unmerge . getReg ( <NUM_LIT> ) ) ; auto CmpDen = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , DenUnmerge . getReg ( <NUM_LIT> ) , Scale0Unmerge . getReg ( <NUM_LIT> ) ) ; B . buildXor ( Scale , CmpNum , CmpDen ) ; } else { Scale = DivScale1 . getReg ( <NUM_LIT> ) ; } auto Fmas = B . buildIntrinsic ( <STR_LIT> , { S64 } , false ) . addUse ( Fma4 . getReg ( <NUM_LIT> ) ) . addUse ( Fma3 . getReg ( <NUM_LIT> ) ) . addUse ( Mul . getReg ( <NUM_LIT> ) ) . addUse ( Scale ) . setMIFlags ( Flags ) ; B . buildIntrinsic ( <STR_LIT> , makeArrayRef ( Res ) , false ) . addUse ( Fmas . getReg ( <NUM_LIT> ) ) . addUse ( RHS ) . addUse ( LHS ) . setMIFlags ( Flags ) ;


<s> auto Fma4 = B . buildFMA ( S64 , NegDivScale0 , Mul , DivScale1 . getReg ( <NUM_LIT> ) , Flags ) ; Register Scale ; if ( ! ST . hasUsableDivScaleConditionOutput ( ) ) { Scale = MRI . createGenericVirtualRegister ( S1 ) ; LLT S32 = LLT :: scalar ( <NUM_LIT> ) ; auto NumUnmerge = B . buildUnmerge ( S32 , LHS ) ; auto DenUnmerge = B . buildUnmerge ( S32 , RHS ) ; auto Scale0Unmerge = B . buildUnmerge ( S32 , DivScale0 ) ; auto Scale1Unmerge = B . buildUnmerge ( S32 , DivScale1 ) ; auto CmpNum = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , NumUnmerge . getReg ( <NUM_LIT> ) , Scale1Unmerge . getReg ( <NUM_LIT> ) ) ; auto CmpDen = B . buildICmp ( ICmpInst :: ICMP_EQ , S1 , DenUnmerge . getReg ( <NUM_LIT> ) , Scale0Unmerge . getReg ( <NUM_LIT> ) ) ; B . buildXor ( Scale , CmpNum , CmpDen ) ; } else { Scale = DivScale1 . getReg ( <NUM_LIT> ) ; }

auto Fmas = B . buildIntrinsic ( <STR_LIT> , { S64 } , false ) . addUse ( Fma4 . getReg ( <NUM_LIT> ) ) . addUse ( Fma3 . getReg ( <NUM_LIT> ) ) . addUse ( Mul . getReg ( <NUM_LIT> ) ) . addUse ( Scale ) . setMIFlags ( Flags ) ;


<s> bool <ISA_LIT>LegalizerInfo :: legalizeFMad ( MachineInstr & MI , MachineRegisterInfo & MRI , MachineIRBuilder & B ) const { LLT Ty = MRI . getType ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; assert ( Ty . isScalar ( ) ) ; MachineFunction & MF = B . getMF ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . FP32Denormals ) return true ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . FP64FP16Denormals ) return true ; MachineIRBuilder HelperBuilder ( MI ) ;


<s> if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . FP32Denormals ) return true ; if ( Ty == LLT :: scalar ( <NUM_LIT> ) && ! MFI -> getMode ( ) . FP64FP16Denormals ) return true ; MachineIRBuilder HelperBuilder ( MI ) ; GISelObserverWrapper DummyObserver ; LegalizerHelper Helper ( MF , DummyObserver , HelperBuilder ) ; HelperBuilder . setMBB ( * MI . getParent ( ) ) ;