<s> unsigned Opcode = Node -> getMachineOpcode ( ) ; if ( TII -> isMIMG ( Opcode ) && ! TII -> get ( Opcode ) . mayStore ( ) ) adjustWritemask ( Node , DAG ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { legalizeTargetIndependentNode ( Node , DAG ) ;


<s> Lane = SubIdx2Lane ( I -> getConstantOperandVal ( <NUM_LIT> ) ) ; unsigned Comp ; for ( unsigned i = <NUM_LIT> , Dmask = OldDmask ; i <= Lane ; i ++ ) { Comp = countTrailingZeros ( Dmask ) ; Dmask &= ~ ( <NUM_LIT> << Comp ) ; } if ( Users [ Lane ] ) return Node ; Users [ Lane ] = * I ; NewDmask |= <NUM_LIT> << Comp ; } if ( NewDmask == OldDmask ) return Node ; unsigned BitsSet = countPopulation ( NewDmask ) ; int NewOpcode = <STR_LIT> ( Node -> getMachineOpcode ( ) , BitsSet ) ; assert ( NewOpcode != - <NUM_LIT> && NewOpcode != static_cast < int > ( Node -> getMachineOpcode ( ) ) && <STR_LIT> ) ; SmallVector < SDValue , <NUM_LIT> > Ops ; Ops . insert ( Ops . end ( ) , Node -> op_begin ( ) , Node -> op_begin ( ) + DmaskIdx ) ; Ops . push_back ( DAG . getTargetConstant ( NewDmask , SDLoc ( Node ) , <STR_LIT> ) ) ; Ops . insert ( Ops . end ( ) , Node -> op_begin ( ) + DmaskIdx + <NUM_LIT> , Node -> op_end ( ) ) ; MVT SVT = Node -> getValueType ( <NUM_LIT> ) . getVectorElementType ( ) . getSimpleVT ( ) ; MVT ResultVT = BitsSet == <NUM_LIT> ? SVT : <STR_LIT> ( SVT , BitsSet == <NUM_LIT> ? <NUM_LIT> : BitsSet ) ; SDVTList NewVTList = HasChain ? DAG . getVTList ( ResultVT , <STR_LIT> ) : DAG . getVTList ( ResultVT ) ; MachineSDNode * NewNode = DAG . getMachineNode ( NewOpcode , SDLoc ( Node ) , NewVTList , Ops ) ; if ( HasChain ) { DAG . setNodeMemRefs ( NewNode , Node -> memoperands ( ) ) ; DAG . ReplaceAllUsesOfValueWith ( SDValue ( Node , <NUM_LIT> ) , SDValue ( NewNode , <NUM_LIT> ) ) ; } if ( BitsSet == <NUM_LIT> ) { assert ( Node -> hasNUsesOfValue ( <NUM_LIT> , <NUM_LIT> ) ) ; SDNode * Copy = DAG . getMachineNode ( TargetOpcode :: COPY , SDLoc ( Node ) , Users [ Lane ] -> getValueType ( <NUM_LIT> ) , SDValue ( NewNode , <NUM_LIT> ) ) ; DAG . ReplaceAllUsesWith ( Users [ Lane ] , Copy ) ; return nullptr ; } for ( unsigned i = <NUM_LIT> , Idx = <STR_LIT> ; i < <NUM_LIT> ; ++ i ) { SDNode * User = Users [ i ] ; if ( ! User ) continue ; SDValue Op = DAG . getTargetConstant ( Idx , SDLoc ( User ) , <STR_LIT> ) ; DAG . UpdateNodeOperands ( User , SDValue ( NewNode , <NUM_LIT> ) , Op ) ; switch ( Idx ) {


<s> else if ( ! <STR_LIT> ( FLI -> Fn -> getCallingConv ( ) ) ) return true ; } return ! KDA || KDA -> isDivergent ( FLI -> getValueFromVirtualReg ( Reg ) ) ; } } break ; case <STR_LIT> : { const LoadSDNode * L = cast < LoadSDNode > ( N ) ; unsigned AS = L -> getAddressSpace ( ) ; return AS == <ISA_LIT>AS :: PRIVATE_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ; } break ; case <STR_LIT> : return true ; break ; case <STR_LIT> : { } return <STR_LIT> ( cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; case <STR_LIT> : return <STR_LIT> ( cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> R = dyn_cast < RegisterSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; } if ( R ) { const MachineFunction * MF = FLI -> MF ; const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; const MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; const <ISA_LIT>RegisterInfo & TRI = ST . getInstrInfo ( ) -> getRegisterInfo ( ) ; unsigned Reg = R -> getReg ( ) ; if ( TRI . isPhysicalRegister ( Reg ) ) return TRI . isVGPR ( MRI , Reg ) ; if ( MRI . isLiveIn ( Reg ) ) { if ( TRI . isVGPR ( MRI , Reg ) ) return true ; else if ( ! <STR_LIT> ( FLI -> Fn -> getCallingConv ( ) ) ) return true ; } return ! KDA || KDA -> isDivergent ( FLI -> getValueFromVirtualReg ( Reg ) ) ; }


<s> SDValue <ISA_LIT>TargetLowering :: LowerSTORE ( SDValue Op , SelectionDAG & DAG ) const { SDLoc DL ( Op ) ; StoreSDNode * Store = cast < StoreSDNode > ( Op ) ; EVT VT = Store -> getMemoryVT ( ) ; if ( VT == <STR_LIT> ) { return DAG . getTruncStore ( Store -> getChain ( ) , DL , DAG . getSExtOrTrunc ( Store -> getValue ( ) , DL , <STR_LIT> ) , Store -> getBasePtr ( ) , <STR_LIT> , Store -> getMemOperand ( ) ) ; } assert ( VT . isVector ( ) && Store -> getValue ( ) . getValueType ( ) . getScalarType ( ) == <STR_LIT> ) ; unsigned AS = Store -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , AS , Store -> getAlignment ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; default :


<s> case <STR_LIT> : { SDValue Elt0 = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; SDValue Elt1 = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; DCI . AddToWorklist ( Elt0 . getNode ( ) ) ; DCI . AddToWorklist ( Elt1 . getNode ( ) ) ; return DAG . getNode ( Opc , SL , EltVT , Elt0 , Elt1 , Vec -> getFlags ( ) ) ; } } } if ( ! DCI . isBeforeLegalize ( ) ) return SDValue ( ) ; unsigned VecSize = VecVT . getSizeInBits ( ) ; unsigned EltSize = EltVT . getSizeInBits ( ) ; auto * Idx = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( isa < MemSDNode > ( Vec ) && EltSize <= <NUM_LIT> && EltVT . isByteSized ( ) && VecSize > <NUM_LIT> && VecSize % <NUM_LIT> == <NUM_LIT> && Idx ) { EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VecVT ) ; unsigned BitIndex = Idx -> getZExtValue ( ) * EltSize ; unsigned EltIdx = BitIndex / <NUM_LIT> ; unsigned LeftoverBitIdx = BitIndex % <NUM_LIT> ; SDLoc SL ( N ) ;

SDValue Cast = DAG . getNode ( <STR_LIT> , SL , NewVT , Vec ) ;


<s> if ( ConstantFPSDNode * CFP = isConstOrConstSplatFP ( N0 ) ) { EVT VT = N -> getValueType ( <NUM_LIT> ) ; return getCanonicalConstantFP ( DAG , SDLoc ( N ) , VT , CFP -> getValueAPF ( ) ) ; } if ( N0 . getOpcode ( ) == <STR_LIT> && VT == <STR_LIT> && isTypeLegal ( <STR_LIT> ) ) { SDLoc SL ( N ) ; SDValue NewElts [ <NUM_LIT> ] ; SDValue Lo = N0 . getOperand ( <NUM_LIT> ) ; SDValue Hi = N0 . getOperand ( <NUM_LIT> ) ; EVT EltVT = Lo . getValueType ( ) ; if ( vectorEltWillFoldAway ( Lo ) || vectorEltWillFoldAway ( Hi ) ) { for ( unsigned I = <NUM_LIT> ; I != <NUM_LIT> ; ++ I ) { SDValue Op = N0 . getOperand ( I ) ; if ( ConstantFPSDNode * CFP = dyn_cast < ConstantFPSDNode > ( Op ) ) { NewElts [ I ] = getCanonicalConstantFP ( DAG , SL , EltVT , CFP -> getValueAPF ( ) ) ;


<s> SelectionDAG & DAG = DCI . DAG ; SDValue N0 = N -> getOperand ( <NUM_LIT> ) ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; if ( N0 . isUndef ( ) ) { APFloat QNaN = APFloat :: getQNaN ( SelectionDAG :: EVTToAPFloatSemantics ( VT ) ) ; return DAG . getConstantFP ( QNaN , SDLoc ( N ) , VT ) ; } if ( ConstantFPSDNode * CFP = isConstOrConstSplatFP ( N0 ) ) { EVT VT = N -> getValueType ( <NUM_LIT> ) ; return getCanonicalConstantFP ( DAG , SDLoc ( N ) , VT , CFP -> getValueAPF ( ) ) ; } if ( N0 . getOpcode ( ) == <STR_LIT> && VT == <STR_LIT> && isTypeLegal ( <STR_LIT> ) ) { SDLoc SL ( N ) ; SDValue NewElts [ <NUM_LIT> ] ; SDValue Lo = N0 . getOperand ( <NUM_LIT> ) ; SDValue Hi = N0 . getOperand ( <NUM_LIT> ) ; EVT EltVT = Lo . getValueType ( ) ; if ( vectorEltWillFoldAway ( Lo ) || vectorEltWillFoldAway ( Hi ) ) { for ( unsigned I = <NUM_LIT> ; I != <NUM_LIT> ; ++ I ) { SDValue Op = N0 . getOperand ( I ) ;

if ( ConstantFPSDNode * CFP = dyn_cast < ConstantFPSDNode > ( Op ) ) {


<s> if ( auto C = dyn_cast < ConstantSDNode > ( CombinedOffset ) ) { uint32_t Imm = C -> getZExtValue ( ) ; uint32_t SOffset , ImmOffset ; if ( <STR_LIT> ( Imm , SOffset , ImmOffset , Subtarget ) ) { Offsets [ <NUM_LIT> ] = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; Offsets [ <NUM_LIT> ] = DAG . getConstant ( SOffset , DL , <STR_LIT> ) ; Offsets [ <NUM_LIT> ] = DAG . getConstant ( ImmOffset , DL , <STR_LIT> ) ; return ; }


<s> ConstantSDNode * C1 = nullptr ; if ( N0 . getOpcode ( ) == <STR_LIT> ) { if ( ( C1 = dyn_cast < ConstantSDNode > ( N0 . getOperand ( <NUM_LIT> ) ) ) ) N0 = N0 . getOperand ( <NUM_LIT> ) ; } else if ( ( C1 = dyn_cast < ConstantSDNode > ( N0 ) ) ) N0 = SDValue ( ) ; if ( C1 ) { unsigned ImmOffset = C1 -> getZExtValue ( ) ; unsigned Overflow = ImmOffset & ~ MaxImm ;


<s> auto OverflowVal = DAG . getConstant ( Overflow , DL , <STR_LIT> ) ; if ( ! N0 ) N0 = OverflowVal ; else { SDValue Ops [ ] = { N0 , OverflowVal } ; N0 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Ops ) ; } } } if ( ! N0 ) N0 = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; if ( ! C1 ) C1 = cast < ConstantSDNode > ( DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) ; return { N0 , SDValue ( C1 , <NUM_LIT> ) } ;


<s> case <STR_LIT> : { if ( getTargetMachine ( ) . getOptLevel ( ) > CodeGenOpt :: None ) return performMinMaxCombine ( N , DCI ) ; break ; } case <STR_LIT> : return performFMACombine ( N , DCI ) ; case <STR_LIT> : { if ( SDValue Widended = widenLoad ( cast < LoadSDNode > ( N ) , DCI ) ) return Widended ; LLVM_FALLTHROUGH ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> : return performXorCombine ( N , DCI ) ; case <STR_LIT> : return performZeroExtendCombine ( N , DCI ) ; case <STR_LIT> : return performClassCombine ( N , DCI ) ; case <STR_LIT> : return performFCanonicalizeCombine ( N , DCI ) ; case <STR_LIT> : return performRcpCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src = N -> getOperand ( <NUM_LIT> ) ; if ( Src . isUndef ( ) ) return Src ; break ; } case <STR_LIT> : case <STR_LIT> : return performUCharToFloatCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return performCvtF32UByteNCombine ( N , DCI ) ; case <STR_LIT> : return performFMed3Combine ( N , DCI ) ; case <STR_LIT> : return performCvtPkRTZCombine ( N , DCI ) ; case <STR_LIT> : return performClampCombine ( N , DCI ) ; case <STR_LIT> : { SelectionDAG & DAG = DCI . DAG ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; if ( VT == <STR_LIT> || VT == <STR_LIT> ) { SDLoc SL ( N ) ; SDValue Src = N -> getOperand ( <NUM_LIT> ) ; EVT EltVT = Src . getValueType ( ) ;

if ( EltVT == <STR_LIT> ) Src = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ;


<s> case <STR_LIT> : return performAddCombine ( N , DCI ) ; case <STR_LIT> : return performSubCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : return performAddCarrySubCarryCombine ( N , DCI ) ; case <STR_LIT> : return performFAddCombine ( N , DCI ) ; case <STR_LIT> : return performFSubCombine ( N , DCI ) ; case <STR_LIT> : return performSetCCCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( getTargetMachine ( ) . getOptLevel ( ) > CodeGenOpt :: None ) return performMinMaxCombine ( N , DCI ) ; break ; } case <STR_LIT> : return performFMACombine ( N , DCI ) ; case <STR_LIT> : { if ( SDValue Widended = widenLoad ( cast < LoadSDNode > ( N ) , DCI ) ) return Widended ; LLVM_FALLTHROUGH ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> :


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Elt0 = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; SDValue Elt1 = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; DCI . AddToWorklist ( Elt0 . getNode ( ) ) ; DCI . AddToWorklist ( Elt1 . getNode ( ) ) ; return DAG . getNode ( Opc , SL , EltVT , Elt0 , Elt1 , Vec -> getFlags ( ) ) ; } } } if ( ! DCI . isBeforeLegalize ( ) ) return SDValue ( ) ; unsigned VecSize = VecVT . getSizeInBits ( ) ; unsigned EltSize = EltVT . getSizeInBits ( ) ; auto * Idx = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( isa < MemSDNode > ( Vec ) && EltSize <= <NUM_LIT> && EltVT . isByteSized ( ) && VecSize > <NUM_LIT> && VecSize % <NUM_LIT> == <NUM_LIT> && Idx ) { EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VecVT ) ; unsigned BitIndex = Idx -> getZExtValue ( ) * EltSize ; unsigned EltIdx = BitIndex / <NUM_LIT> ;

unsigned LeftoverBitIdx = BitIndex % <NUM_LIT> ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Elt0 = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; SDValue Elt1 = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; DCI . AddToWorklist ( Elt0 . getNode ( ) ) ; DCI . AddToWorklist ( Elt1 . getNode ( ) ) ; return DAG . getNode ( Opc , SL , EltVT , Elt0 , Elt1 , Vec -> getFlags ( ) ) ; } } } if ( ! DCI . isBeforeLegalize ( ) ) return SDValue ( ) ; unsigned VecSize = VecVT . getSizeInBits ( ) ; unsigned EltSize = EltVT . getSizeInBits ( ) ; auto * Idx = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( isa < MemSDNode > ( Vec ) && EltSize <= <NUM_LIT> && EltVT . isByteSized ( ) && VecSize > <NUM_LIT> && VecSize % <NUM_LIT> == <NUM_LIT> && Idx ) {

EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VecVT ) ;


<s> } Info . flags |= MachineMemOperand :: MOLoad ; } else if ( Attr . hasFnAttribute ( Attribute :: WriteOnly ) ) { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getArgOperand ( <NUM_LIT> ) -> getType ( ) ) ; Info . flags |= MachineMemOperand :: MOStore ; } else { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable ; Info . flags |= MachineMemOperand :: MOVolatile ; } return true ; } switch ( IntrID ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align = <NUM_LIT> ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align = <NUM_LIT> ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . ptrVal = MFI -> getGWSPSV ( * MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) ) ;


<s> unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned MovExecOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , TmpExec ) ; BuildMI ( MBB , I , DL , TII -> get ( MovExecOpc ) , SaveExec ) . addReg ( Exec ) ; MachineBasicBlock * LoopBB = MF -> CreateMachineBasicBlock ( ) ; MachineBasicBlock * RemainderBB = MF -> CreateMachineBasicBlock ( ) ; MachineFunction :: iterator MBBI ( MBB ) ; ++ MBBI ; MF -> insert ( MBBI , LoopBB ) ; MF -> insert ( MBBI , RemainderBB ) ; LoopBB -> addSuccessor ( LoopBB ) ; LoopBB -> addSuccessor ( RemainderBB ) ; RemainderBB -> transferSuccessorsAndUpdatePHIs ( & MBB ) ; RemainderBB -> splice ( RemainderBB -> begin ( ) , & MBB , I , MBB . end ( ) ) ; MBB . addSuccessor ( LoopBB ) ; const MachineOperand * Idx = TII -> getNamedOperand ( MI , <STR_LIT> :: idx ) ; auto InsPt = emitLoadM0FromVGPRLoop ( TII , MRI , MBB , * LoopBB , DL , * Idx , InitResultReg , DstReg , PhiReg , TmpExec , Offset , UseGPRIdxMode , IsIndirectSrc ) ; MachineBasicBlock :: iterator First = RemainderBB -> begin ( ) ;

BuildMI ( * RemainderBB , First , DL , TII -> get ( MovExecOpc ) , Exec ) . addReg ( SaveExec ) ;


<s> void <ISA_LIT>TargetLowering :: AdjustInstrPostInstrSelection ( MachineInstr & MI , SDNode * Node ) const { const <ISA_LIT>InstrInfo * TII = getSubtarget ( ) -> getInstrInfo ( ) ; MachineRegisterInfo & MRI = MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; if ( TII -> isVOP3 ( MI . getOpcode ( ) ) ) { TII -> legalizeOperandsVOP3 ( MRI , MI ) ; if ( const MCOperandInfo * OpInfo = MI . getDesc ( ) . OpInfo ) { unsigned Opc = MI . getOpcode ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget -> getRegisterInfo ( ) ; for ( auto I : { <STR_LIT> ( Opc , <STR_LIT> :: src0 ) , <STR_LIT> ( Opc , <STR_LIT> :: src1 ) } ) { if ( I == - <NUM_LIT> ) break ; MachineOperand & Op = MI . getOperand ( I ) ; if ( ( OpInfo [ I ] . RegClass != llvm :: <STR_LIT> && OpInfo [ I ] . RegClass != llvm :: <STR_LIT> ) || ! Op . getReg ( ) . isVirtual ( ) || ! TRI -> isAGPR ( MRI , Op . getReg ( ) ) ) continue ; auto * Src = MRI . getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Src || ! Src -> isCopy ( ) || ! TRI -> isSGPRReg ( MRI , Src -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; auto * RC = TRI -> getRegClassForReg ( MRI , Op . getReg ( ) ) ; auto * NewRC = TRI -> getEquivalentVGPRClass ( RC ) ; MRI . setRegClass ( Op . getReg ( ) , NewRC ) ; } } return ; } int NoRetAtomicOp = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( NoRetAtomicOp != - <NUM_LIT> ) { if ( ! Node -> hasAnyUseOfValue ( <NUM_LIT> ) ) { int Glc1Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: glc1 ) ; if ( Glc1Idx != - <NUM_LIT> ) MI . RemoveOperand ( Glc1Idx ) ; MI . RemoveOperand ( <NUM_LIT> ) ; MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; return ; } if ( ( Node -> hasNUsesOfValue ( <NUM_LIT> , <NUM_LIT> ) && Node -> use_begin ( ) -> isMachineOpcode ( ) && Node -> use_begin ( ) -> getMachineOpcode ( ) == <STR_LIT> && ! Node -> use_begin ( ) -> hasAnyUseOfValue ( <NUM_LIT> ) ) ) { Register Def = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; MI . RemoveOperand ( <NUM_LIT> ) ; BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Def ) ;


<s> int Glc1Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: glc1 ) ; if ( Glc1Idx != - <NUM_LIT> ) MI . RemoveOperand ( Glc1Idx ) ; MI . RemoveOperand ( <NUM_LIT> ) ; MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; return ; } if ( ( Node -> hasNUsesOfValue ( <NUM_LIT> , <NUM_LIT> ) && Node -> use_begin ( ) -> isMachineOpcode ( ) && Node -> use_begin ( ) -> getMachineOpcode ( ) == <STR_LIT> && ! Node -> use_begin ( ) -> hasAnyUseOfValue ( <NUM_LIT> ) ) ) { Register Def = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; MI . RemoveOperand ( <NUM_LIT> ) ; BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Def ) ; } return ;


<s> if ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) { if ( Subtarget -> hasUnalignedDSAccessEnabled ( ) && ! Subtarget -> hasLDSMisalignedBug ( ) ) { if ( IsFast ) * IsFast = Alignment != Align ( <NUM_LIT> ) ; return true ; } if ( Size == <NUM_LIT> ) { bool AlignedBy4 = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( Size == <NUM_LIT> ) { bool Aligned = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = Aligned ; return Aligned ; } if ( Size == <NUM_LIT> ) { bool Aligned = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = Aligned ; return Aligned ; } } if ( AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { bool AlignedBy4 = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 || Subtarget -> enableFlatScratch ( ) || Subtarget -> hasUnalignedScratchAccess ( ) ;


<s> if ( Info -> isEntryFunction ( ) ) { reservePrivateMemoryRegs ( getTargetMachine ( ) , MF , * TRI , * Info ) ; } assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ; if ( Info -> getStackPtrOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getStackPtrOffsetReg ( ) ) ;

if ( Info -> getScratchRSrcReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getScratchRSrcReg ( ) ) ;


<s> case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getOperand ( <NUM_LIT> ) -> getType ( ) ) ; Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = dyn_cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol || ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getOperand ( <NUM_LIT> ) -> getType ( ) -> getPointerElementType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; return true ; } case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; }


<s> return true ; } case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getOperand ( <NUM_LIT> ) -> getType ( ) ) ; Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = dyn_cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; if ( ! Vol || ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getOperand ( <NUM_LIT> ) -> getType ( ) -> getPointerElementType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; return true ; } case <STR_LIT> : case <STR_LIT> : { Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align . reset ( ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore ; const ConstantInt * Vol = cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ;

if ( ! Vol -> isZero ( ) ) Info . flags |= MachineMemOperand :: MOVolatile ;


<s> unsigned NumLoads = <NUM_LIT> ; MVT LoadVT = VT . getSimpleVT ( ) ; unsigned NumElts = LoadVT . isVector ( ) ? LoadVT . getVectorNumElements ( ) : <NUM_LIT> ; assert ( ( LoadVT . getScalarType ( ) == <STR_LIT> || LoadVT . getScalarType ( ) == <STR_LIT> ) ) ; if ( NumElts == <NUM_LIT> || NumElts == <NUM_LIT> ) { NumLoads = NumElts / <NUM_LIT> ; LoadVT = <STR_LIT> ( LoadVT . getScalarType ( ) , <NUM_LIT> ) ; } SDVTList VTList = DAG . getVTList ( { LoadVT , <STR_LIT> } ) ; SDValue Ops [ ] = { DAG . getEntryNode ( ) , Rsrc , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , { } , { } , { } , CachePolicy , DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) , } ; setBufferOffsets ( Offset , DAG , & Ops [ <NUM_LIT> ] , NumLoads > <NUM_LIT> ? <NUM_LIT> * NumLoads : <NUM_LIT> ) ; uint64_t InstOffset = cast < ConstantSDNode > ( Ops [ <NUM_LIT> ] ) -> getZExtValue ( ) ; for ( unsigned i = <NUM_LIT> ; i < NumLoads ; ++ i ) { Ops [ <NUM_LIT> ] = DAG . getTargetConstant ( InstOffset + <NUM_LIT> * i , DL , <STR_LIT> ) ; Loads . push_back ( getMemIntrinsicNode ( <STR_LIT> , DL , VTList , Ops , LoadVT , MMO , DAG ) ) ;


<s> MachineFunction & MF = DAG . getMachineFunction ( ) ; const DataLayout & DataLayout = DAG . getDataLayout ( ) ; unsigned Align = DataLayout . getABITypeAlignment ( VT . getTypeForEVT ( * DAG . getContext ( ) ) ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , VT . getStoreSize ( ) , Align ) ; if ( ! Offset -> isDivergent ( ) ) { SDValue Ops [ ] = { Rsrc , Offset , CachePolicy } ; if ( VT . isVector ( ) && VT . getVectorNumElements ( ) == <NUM_LIT> ) { EVT WidenedVT = EVT :: getVectorVT ( * DAG . getContext ( ) , VT . getVectorElementType ( ) , <NUM_LIT> ) ; auto WidenedOp = DAG . getMemIntrinsicNode ( <STR_LIT> , DL , DAG . getVTList ( WidenedVT ) , Ops , WidenedVT , MF . getMachineMemOperand ( MMO , <NUM_LIT> , WidenedVT . getStoreSize ( ) ) ) ; auto Subvector = DAG . getNode ( <STR_LIT> , DL , VT , WidenedOp , DAG . getVectorIdxConstant ( <NUM_LIT> , DL ) ) ; return Subvector ; } return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , DAG . getVTList ( VT ) , Ops , VT , MMO ) ; } SmallVector < SDValue , <NUM_LIT> > Loads ; unsigned NumLoads = <NUM_LIT> ; MVT LoadVT = VT . getSimpleVT ( ) ; unsigned NumElts = LoadVT . isVector ( ) ? LoadVT . getVectorNumElements ( ) : <NUM_LIT> ; assert ( ( LoadVT . getScalarType ( ) == <STR_LIT> || LoadVT . getScalarType ( ) == <STR_LIT> ) ) ; if ( NumElts == <NUM_LIT> || NumElts == <NUM_LIT> ) { NumLoads = NumElts / <NUM_LIT> ; LoadVT = <STR_LIT> ( LoadVT . getScalarType ( ) , <NUM_LIT> ) ; } SDVTList VTList = DAG . getVTList ( { LoadVT , <STR_LIT> } ) ;


<s> if ( ! ST ) return EVT :: getEVT ( Ty , true ) ; if ( ST -> getNumContainedTypes ( ) != <NUM_LIT> || ! ST -> getContainedType ( <NUM_LIT> ) -> isIntegerTy ( <NUM_LIT> ) ) return EVT ( ) ; return EVT :: getEVT ( ST -> getContainedType ( <NUM_LIT> ) ) ;


<s> EVT EltVT = N -> getValueType ( <NUM_LIT> ) ; SDValue Idx = N -> getOperand ( <NUM_LIT> ) ; unsigned Opc = Vec . getOpcode ( ) ; switch ( Opc ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Elt0 = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; SDValue Elt1 = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; DCI . AddToWorklist ( Elt0 . getNode ( ) ) ; DCI . AddToWorklist ( Elt1 . getNode ( ) ) ; return DAG . getNode ( Opc , SL , EltVT , Elt0 , Elt1 , Vec -> getFlags ( ) ) ; } } } unsigned VecSize = VecVT . getSizeInBits ( ) ; unsigned EltSize = EltVT . getSizeInBits ( ) ; if ( VecSize <= <NUM_LIT> && ( VecSize > <NUM_LIT> || EltSize >= <NUM_LIT> ) && ! isa < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ) { SDLoc SL ( N ) ; SDValue Idx = N -> getOperand ( <NUM_LIT> ) ; SDValue V ; for ( unsigned I = <NUM_LIT> , E = VecVT . getVectorNumElements ( ) ; I < E ; ++ I ) { SDValue IC = DAG . getVectorIdxConstant ( I , SL ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec , IC ) ; if ( I == <NUM_LIT> ) V = Elt ; else V = DAG . getSelectCC ( SL , Idx , IC , Elt , V , <STR_LIT> ) ; } return V ;


<s> unsigned EltSize = EltVT . getSizeInBits ( ) ; if ( VecSize <= <NUM_LIT> && ( VecSize > <NUM_LIT> || EltSize >= <NUM_LIT> ) && ! isa < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ) { SDLoc SL ( N ) ; SDValue Idx = N -> getOperand ( <NUM_LIT> ) ; SDValue V ; for ( unsigned I = <NUM_LIT> , E = VecVT . getVectorNumElements ( ) ; I < E ; ++ I ) { SDValue IC = DAG . getVectorIdxConstant ( I , SL ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec , IC ) ; if ( I == <NUM_LIT> ) V = Elt ; else V = DAG . getSelectCC ( SL , Idx , IC , Elt , V , <STR_LIT> ) ; } return V ; } if ( ! DCI . isBeforeLegalize ( ) ) return SDValue ( ) ; auto * Idx = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( isa < MemSDNode > ( Vec ) && EltSize <= <NUM_LIT> && EltVT . isByteSized ( ) && VecSize > <NUM_LIT> && VecSize % <NUM_LIT> == <NUM_LIT> && Idx ) { EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VecVT ) ; unsigned BitIndex = Idx -> getZExtValue ( ) * EltSize ; unsigned EltIdx = BitIndex / <NUM_LIT> ; unsigned LeftoverBitIdx = BitIndex % <NUM_LIT> ; SDLoc SL ( N ) ; SDValue Cast = DAG . getNode ( <STR_LIT> , SL , NewVT , Vec ) ; DCI . AddToWorklist ( Cast . getNode ( ) ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Cast , DAG . getConstant ( EltIdx , SL , <STR_LIT> ) ) ; DCI . AddToWorklist ( Elt . getNode ( ) ) ; SDValue Srl = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Elt , DAG . getConstant ( LeftoverBitIdx , SL , <STR_LIT> ) ) ; DCI . AddToWorklist ( Srl . getNode ( ) ) ; SDValue Trunc = DAG . getNode ( <STR_LIT> , SL , EltVT . changeTypeToInteger ( ) , Srl ) ; DCI . AddToWorklist ( Trunc . getNode ( ) ) ; return DAG . getNode ( <STR_LIT> , SL , EltVT , Trunc ) ; }


<s> } } } } } if ( const CallInst * CI = dyn_cast < CallInst > ( V ) ) { if ( isa < InlineAsm > ( CI -> getCalledValue ( ) ) ) { const <ISA_LIT>RegisterInfo * <ISA_LIT>RI = Subtarget -> getRegisterInfo ( ) ; ImmutableCallSite CS ( CI ) ; TargetLowering :: AsmOperandInfoVector TargetConstraints = ParseConstraints ( MF . getDataLayout ( ) , Subtarget -> getRegisterInfo ( ) , CS ) ; for ( auto & TC : TargetConstraints ) { if ( TC . Type == InlineAsm :: isOutput ) { ComputeConstraintToUse ( TC , SDValue ( ) ) ; unsigned AssignedReg ; const TargetRegisterClass * RC ; std :: tie ( AssignedReg , RC ) = getRegForInlineAsmConstraint ( <ISA_LIT>RI , TC . ConstraintCode , TC . ConstraintVT ) ; if ( RC ) { MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; if ( AssignedReg != <NUM_LIT> && <ISA_LIT>RI -> isSGPRReg ( MRI , AssignedReg ) ) return true ; else if ( <ISA_LIT>RI -> isSGPRClass ( RC ) ) return true ; } } } } } SmallPtrSet < const Value * , <NUM_LIT> > Visited ; return hasCFUser ( V , Visited , Subtarget -> getWavefrontSize ( ) ) ;


<s> if ( V == U -> getOperand ( <NUM_LIT> ) ) { switch ( Intrinsic -> getIntrinsicID ( ) ) { default : Result = false ; break ; case <STR_LIT> : case <STR_LIT> : Result = true ; break ; } } } else { Result = hasCFUser ( U , Visited ) ; } if ( Result ) break ;


<s> for ( auto U : V -> users ( ) ) { if ( const IntrinsicInst * Intrinsic = dyn_cast < IntrinsicInst > ( U ) ) { if ( V == U -> getOperand ( <NUM_LIT> ) ) { switch ( Intrinsic -> getIntrinsicID ( ) ) { default : Result = false ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Result = true ; break ; } }


<s> unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { if ( ! Op -> isDivergent ( ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && ! Op -> isDivergent ( ) && ! Load -> isVolatile ( ) && isMemOpHasNoClobberedMemOperand ( Load ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return WidenVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return WidenVectorLoad ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable ( <STR_LIT> ) ; }

} else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: REGION_ADDRESS ) {


<s> Opc = ( Opc == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; } else { Args [ <NUM_LIT> ] = LHS ; Args [ <NUM_LIT> ] = Zero ; Opc = ( Opc == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; } return DAG . getNode ( Opc , SL , VTList , Args ) ; } } if ( LHS . getOpcode ( ) == <STR_LIT> ) { auto C = dyn_cast < ConstantSDNode > ( LHS . getOperand ( <NUM_LIT> ) ) ; if ( ! C || ! C -> isNullValue ( ) ) return SDValue ( ) ; SDValue Args [ ] = { LHS . getOperand ( <NUM_LIT> ) , RHS , LHS . getOperand ( <NUM_LIT> ) } ;

return DAG . getNode ( <STR_LIT> , SDLoc ( N ) , LHS -> getVTList ( ) , Args ) ;


<s> MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant , VT . getStoreSize ( ) , Align ) ; if ( ! Offset -> isDivergent ( ) ) { SDValue Ops [ ] = { Rsrc , Offset , GLC , DLC , } ; if ( VT . isVector ( ) && VT . getVectorNumElements ( ) == <NUM_LIT> ) { EVT WidenedVT = EVT :: getVectorVT ( * DAG . getContext ( ) , VT . getVectorElementType ( ) , <NUM_LIT> ) ; auto WidenedOp = DAG . getMemIntrinsicNode ( <STR_LIT> , DL , DAG . getVTList ( WidenedVT ) , Ops , WidenedVT , MF . getMachineMemOperand ( MMO , <NUM_LIT> , WidenedVT . getStoreSize ( ) ) ) ; auto Subvector = DAG . getNode ( <STR_LIT> , DL , VT , WidenedOp , DAG . getVectorIdxConstant ( <NUM_LIT> , DL ) ) ; return Subvector ; } return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , DAG . getVTList ( VT ) , Ops , VT , MMO ) ; } SmallVector < SDValue , <NUM_LIT> > Loads ; unsigned NumLoads = <NUM_LIT> ; MVT LoadVT = VT . getSimpleVT ( ) ; unsigned NumElts = LoadVT . isVector ( ) ? LoadVT . getVectorNumElements ( ) : <NUM_LIT> ; assert ( ( LoadVT . getScalarType ( ) == <STR_LIT> || LoadVT . getScalarType ( ) == <STR_LIT> ) ) ; if ( NumElts == <NUM_LIT> || NumElts == <NUM_LIT> ) { NumLoads = NumElts / <NUM_LIT> ; LoadVT = <STR_LIT> ; } SDVTList VTList = DAG . getVTList ( { LoadVT , <STR_LIT> } ) ; unsigned CachePolicy = cast < ConstantSDNode > ( GLC ) -> getZExtValue ( ) ; SDValue Ops [ ] = {

DAG . getEntryNode ( ) , Rsrc , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , { } , { } , { } , DAG . getTargetConstant ( CachePolicy , DL , <STR_LIT> ) , DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) , } ;


<s> EVT WidenedVT = EVT :: getVectorVT ( * DAG . getContext ( ) , VT . getVectorElementType ( ) , <NUM_LIT> ) ; auto WidenedOp = DAG . getMemIntrinsicNode ( <STR_LIT> , DL , DAG . getVTList ( WidenedVT ) , Ops , WidenedVT , MF . getMachineMemOperand ( MMO , <NUM_LIT> , WidenedVT . getStoreSize ( ) ) ) ; auto Subvector = DAG . getNode ( <STR_LIT> , DL , VT , WidenedOp , DAG . getVectorIdxConstant ( <NUM_LIT> , DL ) ) ; return Subvector ; } return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , DAG . getVTList ( VT ) , Ops , VT , MMO ) ; } SmallVector < SDValue , <NUM_LIT> > Loads ; unsigned NumLoads = <NUM_LIT> ; MVT LoadVT = VT . getSimpleVT ( ) ; unsigned NumElts = LoadVT . isVector ( ) ? LoadVT . getVectorNumElements ( ) : <NUM_LIT> ; assert ( ( LoadVT . getScalarType ( ) == <STR_LIT> || LoadVT . getScalarType ( ) == <STR_LIT> ) ) ; if ( NumElts == <NUM_LIT> || NumElts == <NUM_LIT> ) {


<s> return BB ; case <STR_LIT> : { const <ISA_LIT>InstrInfo * TII = getSubtarget ( ) -> getInstrInfo ( ) ; MachineFunction * MF = BB -> getParent ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( MFI -> LDSSize ) ; MI . eraseFromParent ( ) ; return BB ; } case <STR_LIT> : return splitKillBlock ( MI , BB ) ; default : return <ISA_LIT>TargetLowering :: EmitInstrWithCustomInserter ( MI , BB ) ; }


<s> case <STR_LIT> : return BB ; case <STR_LIT> : { const <ISA_LIT>InstrInfo * TII = getSubtarget ( ) -> getInstrInfo ( ) ; MachineFunction * MF = BB -> getParent ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( MFI -> LDSSize ) ; MI . eraseFromParent ( ) ; return BB ; }


<s> EVT PtrVT = Op . getValueType ( ) ; if ( ! shouldEmitGOTReloc ( GV , getTargetMachine ( ) ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT ) ; SDValue GOTAddr = buildPCRelGlobalAddress ( DAG , GV , DL , <NUM_LIT> , PtrVT , <ISA_LIT>InstrInfo :: MO_GOTPCREL ) ; Type * Ty = PtrVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; const DataLayout & DataLayout = DAG . getDataLayout ( ) ; unsigned Align = DataLayout . getABITypeAlignment ( PtrTy ) ;

MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ;


<s> SDValue Glue = Chain . getValue ( <NUM_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Chain , Op . getOperand ( <NUM_LIT> ) , Glue ) ; } case <STR_LIT> : { SDValue Ops [ ] = { Chain , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; EVT VT = Op . getOperand ( <NUM_LIT> ) . getValueType ( ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOStore , VT . getStoreSize ( ) , <NUM_LIT> ) ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Ops , VT , MMO ) ; }


<s> SDValue <ISA_LIT>TargetLowering :: LowerINTRIN<ISA_LIT>C_VOID ( SDValue Op , SelectionDAG & DAG ) const { MachineFunction & MF = DAG . getMachineFunction ( ) ; SDLoc DL ( Op ) ; SDValue Chain = Op . getOperand ( <NUM_LIT> ) ; unsigned IntrinsicID = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; switch ( IntrinsicID ) { case <STR_LIT> : { Chain = copyToM0 ( DAG , Chain , DL , Op . getOperand ( <NUM_LIT> ) ) ; SDValue Glue = Chain . getValue ( <NUM_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Chain , Op . getOperand ( <NUM_LIT> ) , Glue ) ; } case <STR_LIT> : { SDValue Ops [ ] = { Chain , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; EVT VT = Op . getOperand ( <NUM_LIT> ) . getValueType ( ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOStore , VT . getStoreSize ( ) , <NUM_LIT> ) ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Ops , VT , MMO ) ; } case <STR_LIT> : { if ( const ConstantFPSDNode * K = dyn_cast < ConstantFPSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ) { if ( ! K -> isNegative ( ) ) return Chain ;


<s> MachineBasicBlock * <ISA_LIT>TargetLowering :: splitKillBlock ( MachineInstr & MI , MachineBasicBlock * BB ) const { const <ISA_LIT>InstrInfo * TII = getSubtarget ( ) -> getInstrInfo ( ) ; MachineBasicBlock :: iterator SplitPoint ( & MI ) ; ++ SplitPoint ; if ( SplitPoint == BB -> end ( ) ) { MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; return BB ; } MachineFunction * MF = BB -> getParent ( ) ; MachineBasicBlock * SplitBB = MF -> CreateMachineBasicBlock ( BB -> getBasicBlock ( ) ) ; for ( MachineBasicBlock * Succ : BB -> successors ( ) ) { for ( MachineInstr & MI : * Succ ) { if ( ! MI . isPHI ( ) ) break ; for ( unsigned I = <NUM_LIT> , E = MI . getNumOperands ( ) ; I != E ; I += <NUM_LIT> ) { MachineOperand & FromBB = MI . getOperand ( I ) ; if ( BB == FromBB . getMBB ( ) ) { FromBB . setMBB ( SplitBB ) ; break ; } } } } MF -> insert ( ++ MachineFunction :: iterator ( BB ) , SplitBB ) ; SplitBB -> splice ( SplitBB -> begin ( ) , BB , SplitPoint , BB -> end ( ) ) ; SplitBB -> transferSuccessors ( BB ) ; BB -> addSuccessor ( SplitBB ) ; MI . setDesc ( TII -> get ( <STR_LIT> ) ) ;


<s> ++ SplitPoint ; if ( SplitPoint == BB -> end ( ) ) { MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; return BB ; } MachineFunction * MF = BB -> getParent ( ) ; MachineBasicBlock * SplitBB = MF -> CreateMachineBasicBlock ( BB -> getBasicBlock ( ) ) ; for ( MachineBasicBlock * Succ : BB -> successors ( ) ) { for ( MachineInstr & MI : * Succ ) { if ( ! MI . isPHI ( ) ) break ; for ( unsigned I = <NUM_LIT> , E = MI . getNumOperands ( ) ; I != E ; I += <NUM_LIT> ) { MachineOperand & FromBB = MI . getOperand ( I ) ; if ( BB == FromBB . getMBB ( ) ) { FromBB . setMBB ( SplitBB ) ; break ; } } } } MF -> insert ( ++ MachineFunction :: iterator ( BB ) , SplitBB ) ; SplitBB -> splice ( SplitBB -> begin ( ) , BB , SplitPoint , BB -> end ( ) ) ; SplitBB -> transferSuccessors ( BB ) ;


<s> MI . eraseFromParent ( ) ; return & MBB ; } if ( Val -> isReg ( ) ) MRI . clearKillFlags ( Val -> getReg ( ) ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { MachineBasicBlock :: iterator I ( & MI ) ; MachineInstr * SetOn = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( VGPRIndexMode :: DST_ENABLE ) ; SetOn -> getOperand ( <NUM_LIT> ) . setIsUndef ( ) ; BuildMI ( MBB , std :: next ( I ) , DL , TII -> get ( <STR_LIT> ) ) ; } unsigned PhiReg = MRI . createVirtualRegister ( VecRC ) ; auto InsPt = loadM0FromVGPR ( TII , MBB , MI , SrcVec -> getReg ( ) , PhiReg , Offset , UseGPRIdxMode ) ; MachineBasicBlock * LoopBB = InsPt -> getParent ( ) ; if ( UseGPRIdxMode ) { BuildMI ( * LoopBB , InsPt , DL , TII -> get ( <STR_LIT> ) ) . addReg ( PhiReg , RegState :: Undef , SubReg ) . add ( * Val ) . addReg ( Dst , RegState :: ImplicitDefine ) . addReg ( PhiReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; } else { const MCInstrDesc & MovRelDesc = TII -> get ( getMOVRELDPseudo ( VecRC ) ) ;

BuildMI ( * LoopBB , InsPt , DL , MovRelDesc ) . addReg ( Dst , RegState :: Define ) . addReg ( PhiReg ) . add ( * Val ) . addImm ( SubReg - <STR_LIT> ) ;


<s> case <STR_LIT> : { const DebugLoc & DL = MI . getDebugLoc ( ) ; const int TrapType = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( Subtarget -> getTrapHandlerAbi ( ) == <ISA_LIT>Subtarget :: TrapHandlerAbiHsa && Subtarget -> isTrapHandlerEnabled ( ) ) { MachineFunction * MF = BB -> getParent ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; if ( ! BB -> isLiveIn ( UserSGPR ) ) BB -> addLiveIn ( UserSGPR ) ; BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( UserSGPR ) ; BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( TrapType ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; } else { switch ( TrapType ) { case <ISA_LIT>Subtarget :: TrapCodeLLVMTrap : BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) ) ; break ; case <ISA_LIT>Subtarget :: TrapCodeLLVMDebugTrap : { DiagnosticInfoUnsupported NoTrap ( * MF -> getFunction ( ) , <STR_LIT> , DL , DS_Warning ) ; LLVMContext & C = MF -> getFunction ( ) -> getContext ( ) ; C . diagnose ( NoTrap ) ; BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; break ; } default : llvm_unreachable ( <STR_LIT> ) ; } } MI . eraseFromParent ( ) ; return BB ; } case <STR_LIT> : BuildMI ( * BB , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; MI . eraseFromParent ( ) ; return BB ; case <STR_LIT> : { DebugLoc DL = MI . getDebugLoc ( ) ;

BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( MFI -> getLDSSize ( ) ) ;


<s> } else if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SEA_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; } else llvm_unreachable ( <STR_LIT> ) ; if ( AM . Scale == <NUM_LIT> ) return true ; if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ; return false ; case <ISA_LIT>AS :: PRIVATE_ADDRESS : return isLegalMUBUFAddressingMode ( AM ) ; case <ISA_LIT>AS :: LOCAL_ADDRESS : case <ISA_LIT>AS :: REGION_ADDRESS : if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ;


<s> SDValue NewLD = DAG . getExtLoad ( <STR_LIT> , DL , <STR_LIT> , Chain , BasePtr , RealMemVT , MMO ) ; SDValue Ops [ ] = { DAG . getNode ( <STR_LIT> , DL , MemVT , NewLD ) , NewLD . getValue ( <NUM_LIT> ) } ; return DAG . getMergeValues ( Ops , DL ) ; } if ( ! MemVT . isVector ( ) ) return SDValue ( ) ; assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ; unsigned AS = Load -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , MemVT , AS , Load -> getAlignment ( ) ) ) { SDValue Ops [ <NUM_LIT> ] ; std :: tie ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) = expandUnalignedLoad ( Load , DAG ) ; return DAG . getMergeValues ( Ops , DL ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ; switch ( AS ) { case <ISA_LIT>AS :: CONSTANT_ADDRESS : if ( isMemOpUniform ( Load ) ) return SDValue ( ) ; LLVM_FALLTHROUGH ; case <ISA_LIT>AS :: GLOBAL_ADDRESS : if ( Subtarget -> getScalarizeGlobalBehavior ( ) && isMemOpUniform ( Load ) && isMemOpHasNoClobberedMemOperand ( Load ) ) return SDValue ( ) ; LLVM_FALLTHROUGH ; case <ISA_LIT>AS :: FLAT_ADDRESS : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <ISA_LIT>AS :: PRIVATE_ADDRESS : switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; default :


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> : return performXorCombine ( N , DCI ) ; case <STR_LIT> : return performClassCombine ( N , DCI ) ; case <STR_LIT> : return performFCanonicalizeCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src = N -> getOperand ( <NUM_LIT> ) ; if ( Src . isUndef ( ) ) return Src ; break ; } case <STR_LIT> : case <STR_LIT> : return performUCharToFloatCombine ( N , DCI ) ; case <STR_LIT> :


<s> case <STR_LIT> : { if ( SDValue Res = lowerINSERT_VECTOR_ELT ( SDValue ( N , <NUM_LIT> ) , DAG ) ) Results . push_back ( Res ) ; return ; } case <STR_LIT> : { if ( SDValue Res = lowerEXTRACT_VECTOR_ELT ( SDValue ( N , <NUM_LIT> ) , DAG ) ) Results . push_back ( Res ) ; return ; } default :


<s> MachineInstr * SetOn = BuildMI ( * MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( Tmp , RegState :: Kill ) . addImm ( IdxMode ) ; SetOn -> getOperand ( <NUM_LIT> ) . setIsUndef ( ) ; } return true ; } if ( Offset == <NUM_LIT> ) { BuildMI ( * MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . add ( * Idx ) ; } else { BuildMI ( * MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . add ( * Idx ) . addImm ( Offset ) ;


<s> BuildMI ( * MBB , I , DL , TII -> get ( <STR_LIT> ) , Tmp ) . add ( * Idx ) . addImm ( Offset ) ; MachineInstr * SetOn = BuildMI ( * MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( Tmp , RegState :: Kill ) . addImm ( IdxMode ) ; SetOn -> getOperand ( <NUM_LIT> ) . setIsUndef ( ) ; } return true ; } if ( Offset == <NUM_LIT> ) { BuildMI ( * MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . add ( * Idx ) ; } else { BuildMI ( * MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . add ( * Idx ) . addImm ( Offset ) ; } return true ;


<s> case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; unsigned Opc = getImageOpcode ( IID ) ; Res = DAG . getMemIntrinsicNode ( Opc , DL , VTList , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; Chain = Res . getValue ( <NUM_LIT> ) ; return adjustLoadValueType ( Res , LoadVT , DL , DAG , Unpacked ) ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; unsigned Opc = getImageOpcode ( IID ) ; Res = DAG . getMemIntrinsicNode ( Opc , DL , VTList , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; Chain = Res . getValue ( <NUM_LIT> ) ; return adjustLoadValueType ( Res , LoadVT , DL , DAG , Unpacked ) ; }


<s> SDLoc DL ( Op ) ; LoadSDNode * Load = cast < LoadSDNode > ( Op ) ; <STR_LIT> ExtType = Load -> getExtensionType ( ) ; EVT MemVT = Load -> getMemoryVT ( ) ; if ( ExtType == <STR_LIT> && MemVT . getSizeInBits ( ) < <NUM_LIT> ) { if ( MemVT == <STR_LIT> && isTypeLegal ( <STR_LIT> ) ) return SDValue ( ) ; SDValue Chain = Load -> getChain ( ) ; SDValue BasePtr = Load -> getBasePtr ( ) ; MachineMemOperand * MMO = Load -> getMemOperand ( ) ; EVT RealMemVT = ( MemVT == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; SDValue NewLD = DAG . getExtLoad ( <STR_LIT> , DL , <STR_LIT> , Chain , BasePtr , RealMemVT , MMO ) ; SDValue Ops [ ] = { DAG . getNode ( <STR_LIT> , DL , MemVT , NewLD ) , NewLD . getValue ( <NUM_LIT> ) } ; return DAG . getMergeValues ( Ops , DL ) ; } if ( ! MemVT . isVector ( ) ) return SDValue ( ) ; assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ; unsigned AS = Load -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , MemVT , AS , Load -> getAlignment ( ) ) ) { SDValue Ops [ <NUM_LIT> ] ; std :: tie ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) = expandUnalignedLoad ( Load , DAG ) ; return DAG . getMergeValues ( Ops , DL ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT ) { if ( isMemOpUniform ( Load ) ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && isMemOpUniform ( Load ) && ! Load -> isVolatile ( ) && isMemOpHasNoClobberedMemOperand ( Load ) ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ;


<s> DAG . getNode ( <STR_LIT> , DL , MemVT , NewLD ) , NewLD . getValue ( <NUM_LIT> ) } ; return DAG . getMergeValues ( Ops , DL ) ; } if ( ! MemVT . isVector ( ) ) return SDValue ( ) ; assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ; unsigned AS = Load -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , MemVT , AS , Load -> getAlignment ( ) ) ) { SDValue Ops [ <NUM_LIT> ] ; std :: tie ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) = expandUnalignedLoad ( Load , DAG ) ; return DAG . getMergeValues ( Ops , DL ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT ) { if ( isMemOpUniform ( Load ) ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && isMemOpUniform ( Load ) && ! Load -> isVolatile ( ) && isMemOpHasNoClobberedMemOperand ( Load ) ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS_32BIT || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> :


<s> return ( MemVT . getSizeInBits ( ) <= <NUM_LIT> * <NUM_LIT> ) ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { unsigned MaxPrivateBits = <NUM_LIT> * getSubtarget ( ) -> getMaxPrivateElementSize ( ) ; return ( MemVT . getSizeInBits ( ) <= MaxPrivateBits ) ; } else if ( AS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS ) {


<s> SDLoc SL ( Op ) ; SDValue LHS = Op . getOperand ( <NUM_LIT> ) ; SDValue RHS = Op . getOperand ( <NUM_LIT> ) ; EVT VT = Op . getValueType ( ) ; bool Unsafe = DAG . getTarget ( ) . Options . UnsafeFPMath ; if ( ! Unsafe && VT == <STR_LIT> && Subtarget -> hasFP32Denormals ( ) ) return SDValue ( ) ; if ( const ConstantFPSDNode * CLHS = dyn_cast < ConstantFPSDNode > ( LHS ) ) { if ( Unsafe || VT == <STR_LIT> || VT == <STR_LIT> ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { if ( RHS . getOpcode ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , SL , VT , RHS . getOperand ( <NUM_LIT> ) ) ; return DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { SDValue FNegRHS = DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; return DAG . getNode ( <STR_LIT> , SL , VT , FNegRHS ) ; } }


<s> SDValue <ISA_LIT>TargetLowering :: LowerINTRIN<ISA_LIT>C_VOID ( SDValue Op , SelectionDAG & DAG ) const { SDLoc DL ( Op ) ; SDValue Chain = Op . getOperand ( <NUM_LIT> ) ; unsigned IntrinsicID = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; MachineFunction & MF = DAG . getMachineFunction ( ) ; switch ( IntrinsicID ) { case <STR_LIT> : { const ConstantSDNode * Tgt = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const ConstantSDNode * En = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const ConstantSDNode * Done = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const ConstantSDNode * VM = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const SDValue Ops [ ] = { Chain , DAG . getTargetConstant ( Tgt -> getZExtValue ( ) , DL , <STR_LIT> ) , DAG . getTargetConstant ( En -> getZExtValue ( ) , DL , <STR_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) , DAG . getTargetConstant ( VM -> getZExtValue ( ) , DL , <STR_LIT> ) } ; unsigned Opc = Done -> isNullValue ( ) ? <STR_LIT> : <STR_LIT> ; return DAG . getNode ( Opc , DL , Op -> getVTList ( ) , Ops ) ; } case <STR_LIT> : { const ConstantSDNode * Tgt = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const ConstantSDNode * En = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; SDValue Src0 = Op . getOperand ( <NUM_LIT> ) ; SDValue Src1 = Op . getOperand ( <NUM_LIT> ) ; const ConstantSDNode * Done = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; const ConstantSDNode * VM = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; SDValue Undef = DAG . getUNDEF ( <STR_LIT> ) ; const SDValue Ops [ ] = { Chain , DAG . getTargetConstant ( Tgt -> getZExtValue ( ) , DL , <STR_LIT> ) , DAG . getTargetConstant ( En -> getZExtValue ( ) , DL , <STR_LIT> ) , DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Src0 ) , DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Src1 ) , Undef , Undef , DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) , DAG . getTargetConstant ( VM -> getZExtValue ( ) , DL , <STR_LIT> ) } ; unsigned Opc = Done -> isNullValue ( ) ? <STR_LIT> : <STR_LIT> ; return DAG . getNode ( Opc , DL , Op -> getVTList ( ) , Ops ) ; } case <STR_LIT> : case <STR_LIT> : {

unsigned NodeOp = ( IntrinsicID == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ;


<s> return DAG . getMemIntrinsicNode ( Opc , DL , Op -> getVTList ( ) , Ops , IntVT , MMO ) ; } case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; EVT VT = Op . getOperand ( <NUM_LIT> ) . getValueType ( ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad , VT . getStoreSize ( ) , VT . getStoreSize ( ) ) ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Ops , VT , MMO ) ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : {

const ConstantSDNode * DMask = dyn_cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ;


<s> SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned Opc = ( IntrID == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; EVT VT = Op . getValueType ( ) ; EVT IntVT = VT . changeTypeToInteger ( ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( MFI -> getBufferPSV ( ) ) , MachineMemOperand :: MOLoad , VT . getStoreSize ( ) , VT . getStoreSize ( ) ) ; return DAG . getMemIntrinsicNode ( Opc , DL , Op -> getVTList ( ) , Ops , IntVT , MMO ) ; } case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; EVT VT = Op . getOperand ( <NUM_LIT> ) . getValueType ( ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOLoad , VT . getStoreSize ( ) , VT . getStoreSize ( ) ) ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Ops , VT , MMO ) ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> if ( ! MemVT . isVector ( ) ) return SDValue ( ) ; assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ; unsigned AS = Load -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , MemVT , AS , Load -> getAlignment ( ) ) ) { SDValue Ops [ <NUM_LIT> ] ; std :: tie ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) = expandUnalignedLoad ( Load , DAG ) ; return DAG . getMergeValues ( Ops , DL ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) { if ( isMemOpUniform ( Load ) ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && isMemOpUniform ( Load ) && ! Load -> isVolatile ( ) && isMemOpHasNoClobberedMemOperand ( Load ) ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> :


<s> NewOut . VT = VT ; unsigned NumElements = Out . ArgVT . getVectorNumElements ( ) ; for ( unsigned j = <NUM_LIT> ; j != NumElements ; ++ j ) { SDValue Elem = DAG . getNode ( <STR_LIT> , DL , VT , OutVals [ i ] , DAG . getConstant ( j , DL , <STR_LIT> ) ) ; SplitVals . push_back ( Elem ) ; Splits . push_back ( NewOut ) ; NewOut . PartOffset += NewOut . VT . getStoreSize ( ) ; } } else { SplitVals . push_back ( OutVals [ i ] ) ; Splits . push_back ( Out ) ; } } SmallVector < CCValAssign , <NUM_LIT> > RVLocs ; CCState CCInfo ( CallConv , isVarArg , DAG . getMachineFunction ( ) , RVLocs , * DAG . getContext ( ) ) ; CCInfo . AnalyzeReturn ( Splits , CCAssignFnForReturn ( CallConv , isVarArg ) ) ; SDValue Flag ; SmallVector < SDValue , <NUM_LIT> > RetOps ; RetOps . push_back ( Chain ) ; if ( ! Info -> isEntryFunction ( ) ) { const <ISA_LIT>RegisterInfo * TRI = getSubtarget ( ) -> getRegisterInfo ( ) ; SDValue ReturnAddrReg = CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getReturnAddressReg ( MF ) , <STR_LIT> ) ; SDValue PhysReturnAddrReg = DAG . getRegister ( TRI -> getReturnAddressReg ( MF ) , <STR_LIT> ) ; Chain = DAG . getCopyToReg ( Chain , DL , PhysReturnAddrReg , ReturnAddrReg , Flag ) ; Flag = Chain . getValue ( <NUM_LIT> ) ; RetOps . push_back ( PhysReturnAddrReg ) ; } for ( unsigned i = <NUM_LIT> , realRVLocIdx = <NUM_LIT> ; i != RVLocs . size ( ) ; ++ i , ++ realRVLocIdx ) {


<s> SDValue Args [ ] = { LHS , DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) , Cond } ; Opc = ( Opc == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; return DAG . getNode ( Opc , SL , VTList , Args ) ; } case <STR_LIT> : { auto C = dyn_cast < ConstantSDNode > ( RHS . getOperand ( <NUM_LIT> ) ) ; if ( ! C || C -> getZExtValue ( ) != <NUM_LIT> ) break ; SDValue Args [ ] = { LHS , RHS . getOperand ( <NUM_LIT> ) , RHS . getOperand ( <NUM_LIT> ) } ; return DAG . getNode ( <STR_LIT> , SDLoc ( N ) , RHS -> getVTList ( ) , Args ) ; } } return SDValue ( ) ;


<s> SDLoc SL ( N ) ; SDValue BFE = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LHS -> getOperand ( <NUM_LIT> ) , DAG . getConstant ( Offset , SL , <STR_LIT> ) , DAG . getConstant ( Bits , SL , <STR_LIT> ) ) ; EVT NarrowVT = EVT :: getIntegerVT ( * DAG . getContext ( ) , Bits ) ; SDValue Ext = DAG . getNode ( <STR_LIT> , SL , VT , BFE , DAG . getValueType ( NarrowVT ) ) ; SDValue Shl = DAG . getNode ( <STR_LIT> , SDLoc ( LHS ) , VT , Ext , DAG . getConstant ( NB , SDLoc ( CRHS ) , <STR_LIT> ) ) ; return Shl ; } } } } if ( LHS . getOpcode ( ) == <STR_LIT> && RHS . getOpcode ( ) == <STR_LIT> ) { <STR_LIT> LCC = cast < CondCodeSDNode > ( LHS . getOperand ( <NUM_LIT> ) ) -> get ( ) ; <STR_LIT> RCC = cast < CondCodeSDNode > ( RHS . getOperand ( <NUM_LIT> ) ) -> get ( ) ; SDValue X = LHS . getOperand ( <NUM_LIT> ) ; SDValue Y = RHS . getOperand ( <NUM_LIT> ) ; if ( Y . getOpcode ( ) != <STR_LIT> || Y . getOperand ( <NUM_LIT> ) != X ) return SDValue ( ) ; if ( LCC == <STR_LIT> ) { if ( X != LHS . getOperand ( <NUM_LIT> ) ) return SDValue ( ) ; if ( RCC == <STR_LIT> ) {

const ConstantFPSDNode * C1 = dyn_cast < ConstantFPSDNode > ( RHS . getOperand ( <NUM_LIT> ) ) ;


<s> case <STR_LIT> : { if ( DCI . getDAGCombineLevel ( ) >= AfterLegalizeDAG && getTargetMachine ( ) . getOptLevel ( ) > CodeGenOpt :: None ) return performMinMaxCombine ( N , DCI ) ; break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> : return performXorCombine ( N , DCI ) ; case <STR_LIT> : return performZeroExtendCombine ( N , DCI ) ; case <STR_LIT> : return performClassCombine ( N , DCI ) ; case <STR_LIT> : return performFCanonicalizeCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src = N -> getOperand ( <NUM_LIT> ) ; if ( Src . isUndef ( ) ) return Src ; break ; } case <STR_LIT> : case <STR_LIT> : return performUCharToFloatCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return performCvtF32UByteNCombine ( N , DCI ) ; case <STR_LIT> :

return performFMed3Combine ( N , DCI ) ;


<s> SDValue Vec = N -> getOperand ( <NUM_LIT> ) ; SelectionDAG & DAG = DCI . DAG ; if ( Vec . getOpcode ( ) == <STR_LIT> && allUsesHaveSourceMods ( N ) ) { SDLoc SL ( N ) ; EVT EltVT = N -> getValueType ( <NUM_LIT> ) ; SDValue Idx = N -> getOperand ( <NUM_LIT> ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ;

return DAG . getNode ( <STR_LIT> , SL , EltVT , Elt ) ;


<s> return DAG . getNode ( minMaxOpcToMin3Max3Opc ( Opc ) , DL , N -> getValueType ( <NUM_LIT> ) , Op0 . getOperand ( <NUM_LIT> ) , Op0 . getOperand ( <NUM_LIT> ) , Op1 ) ; } if ( Op1 . getOpcode ( ) == Opc && Op1 . hasOneUse ( ) ) { SDLoc DL ( N ) ; return DAG . getNode ( minMaxOpcToMin3Max3Opc ( Opc ) , DL , N -> getValueType ( <NUM_LIT> ) , Op0 , Op1 . getOperand ( <NUM_LIT> ) , Op1 . getOperand ( <NUM_LIT> ) ) ; } } if ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> && Op0 . hasOneUse ( ) ) { if ( SDValue Med3 = performIntMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 , true ) ) return Med3 ; } if ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> && Op0 . hasOneUse ( ) ) { if ( SDValue Med3 = performIntMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 , false ) ) return Med3 ; } if ( ( ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> ) || ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> ) ) && ( VT == <STR_LIT> || VT == <STR_LIT> || ( VT == <STR_LIT> && Subtarget -> has16BitInsts ( ) ) ) && Op0 . hasOneUse ( ) ) { if ( SDValue Res = performFPMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 ) ) return Res ; }


<s> SDValue <ISA_LIT>TargetLowering :: performMinMaxCombine ( SDNode * N , DAGCombinerInfo & DCI ) const { SelectionDAG & DAG = DCI . DAG ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; unsigned Opc = N -> getOpcode ( ) ; SDValue Op0 = N -> getOperand ( <NUM_LIT> ) ; SDValue Op1 = N -> getOperand ( <NUM_LIT> ) ; if ( Opc != <STR_LIT> && Opc != <STR_LIT> && VT != <STR_LIT> && ( ( VT != <STR_LIT> && VT != <STR_LIT> ) || Subtarget -> hasMin3Max3_16 ( ) ) ) { if ( Op0 . getOpcode ( ) == Opc && Op0 . hasOneUse ( ) ) { SDLoc DL ( N ) ; return DAG . getNode ( minMaxOpcToMin3Max3Opc ( Opc ) , DL , N -> getValueType ( <NUM_LIT> ) , Op0 . getOperand ( <NUM_LIT> ) , Op0 . getOperand ( <NUM_LIT> ) , Op1 ) ; } if ( Op1 . getOpcode ( ) == Opc && Op1 . hasOneUse ( ) ) { SDLoc DL ( N ) ; return DAG . getNode ( minMaxOpcToMin3Max3Opc ( Opc ) , DL , N -> getValueType ( <NUM_LIT> ) , Op0 , Op1 . getOperand ( <NUM_LIT> ) , Op1 . getOperand ( <NUM_LIT> ) ) ; } } if ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> && Op0 . hasOneUse ( ) ) { if ( SDValue Med3 = performIntMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 , true ) ) return Med3 ; } if ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> && Op0 . hasOneUse ( ) ) { if ( SDValue Med3 = performIntMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 , false ) ) return Med3 ; }

if ( ( ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> ) || ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> ) ) && ( VT == <STR_LIT> || VT == <STR_LIT> || ( VT == <STR_LIT> && Subtarget -> has16BitInsts ( ) ) ) && Op0 . hasOneUse ( ) ) {


<s> unsigned ReservedOffsetReg = TRI . reservedPrivateSegmentWaveByteOffsetReg ( MF ) ; Info . setScratchRSrcReg ( ReservedBufferReg ) ; Info . setScratchWaveOffsetReg ( ReservedOffsetReg ) ; } } else { unsigned ReservedBufferReg = TRI . reservedPrivateSegmentBufferReg ( MF ) ; Info . setScratchRSrcReg ( ReservedBufferReg ) ; if ( HasStackObjects ) { unsigned ScratchWaveOffsetReg = TRI . getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: PRIVATE_SEGMENT_WAVE_BYTE_OFFSET ) ; Info . setScratchWaveOffsetReg ( ScratchWaveOffsetReg ) ; } else { unsigned ReservedOffsetReg = TRI . reservedPrivateSegmentWaveByteOffsetReg ( MF ) ; Info . setScratchWaveOffsetReg ( ReservedOffsetReg ) ; } } if ( NeedSP ) { unsigned ReservedStackPtrOffsetReg = TRI . reservedStackPtrOffsetReg ( MF ) ; Info . setStackPtrOffsetReg ( ReservedStackPtrOffsetReg ) ; assert ( Info . getStackPtrOffsetReg ( ) != Info . getFrameOffsetReg ( ) ) ;

assert ( ! TRI . isSubRegister ( Info . getScratchRSrcReg ( ) , Info . getStackPtrOffsetReg ( ) ) ) ;


<s> if ( Subtarget -> hasUnalignedBufferAccess ( ) ) { if ( IsFast ) { * IsFast = ( AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS || AddrSpace == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) ? ( Align % <NUM_LIT> == <NUM_LIT> ) : true ; } return true ; } if ( VT . bitsLT ( <STR_LIT> ) ) return false ; if ( IsFast ) * IsFast = true ;

return VT . bitsGT ( <STR_LIT> ) && Align % <NUM_LIT> == <NUM_LIT> ;


<s> return AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ;


<s> Type * Ty = PtrVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; const DataLayout & DataLayout = DAG . getDataLayout ( ) ; unsigned Align = DataLayout . getABITypeAlignment ( PtrTy ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ;

return DAG . getLoad ( PtrVT , DL , DAG . getEntryNode ( ) , GOTAddr , PtrInfo , Align , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;


<s> if ( ( C1 = dyn_cast < ConstantSDNode > ( N0 . getOperand ( <NUM_LIT> ) ) ) ) N0 = N0 . getOperand ( <NUM_LIT> ) ; } else if ( ( C1 = dyn_cast < ConstantSDNode > ( N0 ) ) ) N0 = SDValue ( ) ; if ( C1 ) { unsigned ImmOffset = C1 -> getZExtValue ( ) ; unsigned Overflow = ImmOffset & ~ MaxImm ; ImmOffset -= Overflow ; C1 = cast < ConstantSDNode > ( DAG . getConstant ( ImmOffset , DL , <STR_LIT> ) ) ; if ( Overflow ) { auto OverflowVal = DAG . getConstant ( Overflow , DL , <STR_LIT> ) ; if ( ! N0 ) N0 = OverflowVal ; else { SDValue Ops [ ] = { N0 , OverflowVal } ; N0 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Ops ) ;


<s> } else if ( ( C1 = dyn_cast < ConstantSDNode > ( N0 ) ) ) N0 = SDValue ( ) ; if ( C1 ) { unsigned ImmOffset = C1 -> getZExtValue ( ) ; unsigned Overflow = ImmOffset & ~ MaxImm ; ImmOffset -= Overflow ; C1 = cast < ConstantSDNode > ( DAG . getConstant ( ImmOffset , DL , <STR_LIT> ) ) ; if ( Overflow ) { auto OverflowVal = DAG . getConstant ( Overflow , DL , <STR_LIT> ) ; if ( ! N0 ) N0 = OverflowVal ; else {

SDValue Ops [ ] = { N0 , OverflowVal } ;


<s> default : return ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; } unsigned NewOpcode = TII -> getMaskedMIMGOp ( MI -> getOpcode ( ) , BitsSet ) ; MI -> setDesc ( TII -> get ( NewOpcode ) ) ; MRI . setRegClass ( VReg , RC ) ; return ; } int NoRetAtomicOp = <STR_LIT> ( MI -> getOpcode ( ) ) ; if ( NoRetAtomicOp != - <NUM_LIT> ) { if ( ! Node -> hasAnyUseOfValue ( <NUM_LIT> ) ) { MI -> setDesc ( TII -> get ( NoRetAtomicOp ) ) ;


<s> const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( Subtarget -> getInstrInfo ( ) ) ; MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; if ( TII -> isVOP3 ( MI -> getOpcode ( ) ) ) { TII -> legalizeOperandsVOP3 ( MRI , MI ) ; return ; } if ( TII -> isMIMG ( * MI ) ) { unsigned VReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned DmaskIdx = MI -> getNumOperands ( ) == <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; unsigned Writemask = MI -> getOperand ( DmaskIdx ) . getImm ( ) ; unsigned BitsSet = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) BitsSet += Writemask & ( <NUM_LIT> << i ) ? <NUM_LIT> : <NUM_LIT> ; const TargetRegisterClass * RC ; switch ( BitsSet ) { default : return ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; } unsigned NewOpcode = TII -> getMaskedMIMGOp ( MI -> getOpcode ( ) , BitsSet ) ; MI -> setDesc ( TII -> get ( NewOpcode ) ) ; MRI . setRegClass ( VReg , RC ) ; return ;


<s> SDValue Target = BRCOND . getOperand ( <NUM_LIT> ) ; SDNode * BR = nullptr ; SDNode * SetCC = nullptr ; if ( Intr -> getOpcode ( ) == <STR_LIT> ) { SetCC = Intr ; Intr = SetCC -> getOperand ( <NUM_LIT> ) . getNode ( ) ; } else { BR = findUser ( BRCOND , <STR_LIT> ) ; Target = BR -> getOperand ( <NUM_LIT> ) ; } if ( Intr -> getOpcode ( ) != <STR_LIT> ) { return BRCOND ; } assert ( ! SetCC || ( SetCC -> getConstantOperandVal ( <NUM_LIT> ) == <NUM_LIT> && isCFIntrinsic ( Intr ) && cast < CondCodeSDNode > ( SetCC -> getOperand ( <NUM_LIT> ) . getNode ( ) ) -> get ( ) == <STR_LIT> ) ) ; ArrayRef < EVT > Res ( Intr -> value_begin ( ) + <NUM_LIT> , Intr -> value_end ( ) ) ; SmallVector < SDValue , <NUM_LIT> > Ops ; Ops . push_back ( BRCOND . getOperand ( <NUM_LIT> ) ) ; Ops . append ( Intr -> op_begin ( ) + <NUM_LIT> , Intr -> op_end ( ) ) ; Ops . push_back ( Target ) ; SDNode * Result = DAG . getNode ( Res . size ( ) > <NUM_LIT> ? <STR_LIT> : <STR_LIT> , DL , DAG . getVTList ( Res ) , Ops ) . getNode ( ) ; if ( BR ) {


<s> const SDValue One = DAG . getConstantFP ( <NUM_LIT> , SL , <STR_LIT> ) ; EVT SetCCVT = getSetCCResultType ( DAG . getDataLayout ( ) , * DAG . getContext ( ) , <STR_LIT> ) ; SDValue r2 = DAG . getSetCC ( SL , SetCCVT , r1 , K0 , <STR_LIT> ) ; SDValue r3 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , r2 , K1 , One ) ; r1 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , RHS , r3 ) ;

SDValue r0 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , r1 ) ;


<s> if ( Out . VT . isVector ( ) ) { MVT VT = Out . VT . getVectorElementType ( ) ; <STR_LIT> NewOut = Out ; NewOut . Flags . setSplit ( ) ; NewOut . VT = VT ; unsigned NumElements = Out . ArgVT . getVectorNumElements ( ) ; for ( unsigned j = <NUM_LIT> ; j != NumElements ; ++ j ) { SDValue Elem = DAG . getNode ( <STR_LIT> , DL , VT , OutVals [ i ] , DAG . getConstant ( j , DL , <STR_LIT> ) ) ; SplitVals . push_back ( Elem ) ; Splits . push_back ( NewOut ) ; NewOut . PartOffset += NewOut . VT . getStoreSize ( ) ; } } else { SplitVals . push_back ( OutVals [ i ] ) ; Splits . push_back ( Out ) ; } } SmallVector < CCValAssign , <NUM_LIT> > RVLocs ; CCState CCInfo ( CallConv , isVarArg , DAG . getMachineFunction ( ) , RVLocs , * DAG . getContext ( ) ) ; AnalyzeReturn ( CCInfo , Splits ) ; SDValue Flag ; SmallVector < SDValue , <NUM_LIT> > RetOps ; RetOps . push_back ( Chain ) ; for ( unsigned i = <NUM_LIT> , realRVLocIdx = <NUM_LIT> ; i != RVLocs . size ( ) ; ++ i , ++ realRVLocIdx ) { CCValAssign & VA = RVLocs [ i ] ; assert ( VA . isRegLoc ( ) && <STR_LIT> ) ; SDValue Arg = SplitVals [ realRVLocIdx ] ; switch ( VA . getLocInfo ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case CCValAssign :: Full : break ;


<s> switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return ScalarizeVectorStore ( Op , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable ( <STR_LIT> ) ; } } case <ISA_LIT>AS :: LOCAL_ADDRESS : return SplitVectorStore ( Op , DAG ) ; default : llvm_unreachable ( <STR_LIT> ) ; }


<s> if ( ! isKnownNeverSNan ( DAG , Var ) ) return SDValue ( ) ; return DAG . getNode ( <STR_LIT> , SL , K0 -> getValueType ( <NUM_LIT> ) , Var , SDValue ( K0 , <NUM_LIT> ) , SDValue ( K1 , <NUM_LIT> ) ) ;


<s> ConstantFPSDNode * K1 = dyn_cast < ConstantFPSDNode > ( Op1 ) ; if ( ! K1 ) return SDValue ( ) ; ConstantFPSDNode * K0 = dyn_cast < ConstantFPSDNode > ( Op0 . getOperand ( <NUM_LIT> ) ) ; if ( ! K0 ) return SDValue ( ) ; APFloat :: cmpResult Cmp = K0 -> getValueAPF ( ) . compare ( K1 -> getValueAPF ( ) ) ; if ( Cmp == APFloat :: cmpGreaterThan ) return SDValue ( ) ; SDValue Var = Op0 . getOperand ( <NUM_LIT> ) ; if ( ! isKnownNeverSNan ( DAG , Var ) ) return SDValue ( ) ;

return DAG . getNode ( <STR_LIT> , SL , K0 -> getValueType ( <NUM_LIT> ) , Var , SDValue ( K0 , <NUM_LIT> ) , SDValue ( K1 , <NUM_LIT> ) ) ;


<s> ConstantSDNode * K0 = dyn_cast < ConstantSDNode > ( Op0 . getOperand ( <NUM_LIT> ) ) ; if ( ! K0 ) return SDValue ( ) ; if ( Signed ) {

if ( K0 -> getAPIntValue ( ) . sge ( K1 -> getAPIntValue ( ) ) ) return SDValue ( ) ;


<s> void <ISA_LIT>TargetLowering :: AdjustInstrPostInstrSelection ( MachineInstr * MI , SDNode * Node ) const { const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( Subtarget -> getInstrInfo ( ) ) ; MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; TII -> legalizeOperands ( MI ) ; if ( TII -> isMIMG ( MI -> getOpcode ( ) ) ) { unsigned VReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Writemask = MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; unsigned BitsSet = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) BitsSet += Writemask & ( <NUM_LIT> << i ) ? <NUM_LIT> : <NUM_LIT> ; const TargetRegisterClass * RC ; switch ( BitsSet ) {


<s> case <NUM_LIT> : RC = & <STR_LIT> ; break ; } unsigned NewOpcode = TII -> getMaskedMIMGOp ( MI -> getOpcode ( ) , BitsSet ) ; MI -> setDesc ( TII -> get ( NewOpcode ) ) ; MRI . setRegClass ( VReg , RC ) ; return ; } int NoRetAtomicOp = <STR_LIT> ( MI -> getOpcode ( ) ) ; if ( NoRetAtomicOp != - <NUM_LIT> ) { if ( ! Node -> hasAnyUseOfValue ( <NUM_LIT> ) ) { MI -> setDesc ( TII -> get ( NoRetAtomicOp ) ) ; MI -> RemoveOperand ( <NUM_LIT> ) ;


<s> return buildRSRC ( DAG , DL , Ptr , <NUM_LIT> , TII -> getScratchRsrcWords23 ( ) ) ;


<s> MachineRegisterInfo & MRI = BB -> getParent ( ) -> getRegInfo ( ) ; unsigned Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; MachineInstrBuilder MIB = BuildMI ( * BB , I , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Reg ) ;

for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) MIB . addOperand ( MI -> getOperand ( i ) ) ;


<s> switch ( VT . SimpleTy ) { default : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : return std :: make_pair ( <NUM_LIT> , & <STR_LIT> ) ; case <STR_LIT> : return std :: make_pair ( <NUM_LIT> , & <STR_LIT> ) ; } } if ( Constraint . size ( ) > <NUM_LIT> ) { const TargetRegisterClass * RC = nullptr ; if ( Constraint [ <NUM_LIT> ] == 'v' ) { RC = & <STR_LIT> ;

} else if ( Constraint [ <NUM_LIT> ] == 's' ) {


<s> } else if ( Constraint [ <NUM_LIT> ] == 's' ) { RC = & <STR_LIT> ; } if ( RC ) { uint32_t Idx ; bool Failed = Constraint . substr ( <NUM_LIT> ) . getAsInteger ( <NUM_LIT> , Idx ) ; if ( ! Failed && Idx < RC -> getNumRegs ( ) ) return std :: make_pair ( RC -> getRegister ( Idx ) , RC ) ; }


<s> } Info -> P<ISA_LIT>nputAddr |= <NUM_LIT> << P<ISA_LIT>nputNum ++ ; } if ( Info -> getShaderType ( ) != ShaderType :: COMPUTE && Arg . VT . isVector ( ) ) { <STR_LIT> NewArg = Arg ; NewArg . Flags . setSplit ( ) ; NewArg . VT = Arg . VT . getVectorElementType ( ) ; Type * ParamType = FType -> getParamType ( Arg . getOrigArgIndex ( ) ) ; unsigned NumElements = ParamType -> getVectorNumElements ( ) ; for ( unsigned j = <NUM_LIT> ; j != NumElements ; ++ j ) { Splits . push_back ( NewArg ) ; NewArg . PartOffset += NewArg . VT . getStoreSize ( ) ; } } else if ( Info -> getShaderType ( ) != ShaderType :: COMPUTE ) { Splits . push_back ( Arg ) ; } } SmallVector < CCValAssign , <NUM_LIT> > ArgLocs ; CCState CCInfo ( CallConv , isVarArg , DAG . getMachineFunction ( ) , ArgLocs , * DAG . getContext ( ) ) ; if ( Info -> getShaderType ( ) == ShaderType :: PIXEL && ( Info -> P<ISA_LIT>nputAddr & <NUM_LIT> ) == <NUM_LIT> ) { Info -> P<ISA_LIT>nputAddr |= <NUM_LIT> ; CCInfo . AllocateReg ( <STR_LIT> ) ; CCInfo . AllocateReg ( <STR_LIT> ) ; } if ( Info -> getShaderType ( ) == ShaderType :: COMPUTE ) { if ( Subtarget -> isAmdHsaOS ( ) ) Info -> NumUserSGPRs = <NUM_LIT> ; else Info -> NumUserSGPRs = <NUM_LIT> ; unsigned InputPtrReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: INPUT_PTR ) ; unsigned InputPtrRegLo = TRI -> getPhysRegSubReg ( InputPtrReg , & <STR_LIT> , <NUM_LIT> ) ; unsigned InputPtrRegHi = TRI -> getPhysRegSubReg ( InputPtrReg , & <STR_LIT> , <NUM_LIT> ) ; unsigned ScratchPtrReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: SCRATCH_PTR ) ; unsigned ScratchPtrRegLo = TRI -> getPhysRegSubReg ( ScratchPtrReg , & <STR_LIT> , <NUM_LIT> ) ; unsigned ScratchPtrRegHi = TRI -> getPhysRegSubReg ( ScratchPtrReg , & <STR_LIT> , <NUM_LIT> ) ; CCInfo . AllocateReg ( InputPtrRegLo ) ; CCInfo . AllocateReg ( InputPtrRegHi ) ; CCInfo . AllocateReg ( ScratchPtrRegLo ) ; CCInfo . AllocateReg ( ScratchPtrRegHi ) ; MF . addLiveIn ( InputPtrReg , & <STR_LIT> ) ; MF . addLiveIn ( ScratchPtrReg , & <STR_LIT> ) ; } if ( Info -> getShaderType ( ) == ShaderType :: COMPUTE ) {

getOriginalFunctionArgs ( DAG , DAG . getMachineFunction ( ) . getFunction ( ) , Ins , Splits ) ;


<s> if ( GSD -> getAddressSpace ( ) != <ISA_LIT>AS :: CONSTANT_ADDRESS ) return <ISA_LIT>TargetLowering :: LowerGlobalAddress ( MFI , Op , DAG ) ; SDLoc DL ( GSD ) ; const GlobalValue * GV = GSD -> getGlobal ( ) ; MVT PtrVT = getPointerTy ( DAG . getDataLayout ( ) , GSD -> getAddressSpace ( ) ) ; SDValue Ptr = DAG . getNode ( <STR_LIT> , DL , PtrVT ) ; SDValue GA = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> ) ; SDValue PtrLo = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Ptr , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) ; SDValue PtrHi = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Ptr , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) ; SDValue Lo = DAG . getNode ( <STR_LIT> , DL , DAG . getVTList ( <STR_LIT> , <STR_LIT> ) , PtrLo , GA ) ; SDValue Hi = DAG . getNode ( <STR_LIT> , DL , DAG . getVTList ( <STR_LIT> , <STR_LIT> ) , PtrHi , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , SDValue ( Lo . getNode ( ) , <NUM_LIT> ) ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Lo , Hi ) ;


<s> SDValue Ptr = DAG . getNode ( <STR_LIT> , DL , PtrVT ) ; SDValue GA = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> ) ; SDValue PtrLo = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Ptr , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) ; SDValue PtrHi = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Ptr , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) ;

SDValue Lo = DAG . getNode ( <STR_LIT> , DL , DAG . getVTList ( <STR_LIT> , <STR_LIT> ) , PtrLo , GA ) ;


<s> assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ; unsigned NumElements = Op . getValueType ( ) . getVectorNumElements ( ) ; assert ( NumElements != <NUM_LIT> && <STR_LIT> ) ; switch ( Load -> getAddressSpace ( ) ) { default : break ; case <ISA_LIT>AS :: GLOBAL_ADDRESS : case <ISA_LIT>AS :: PRIVATE_ADDRESS : if ( NumElements <= <NUM_LIT> ) break ; case <ISA_LIT>AS :: LOCAL_ADDRESS : return ScalarizeVectorLoad ( Op , DAG ) ; } } return <ISA_LIT>TargetLowering :: LowerLOAD ( Op , DAG ) ;


<s> SDValue <ISA_LIT>TargetLowering :: LowerLOAD ( SDValue Op , SelectionDAG & DAG ) const { SDLoc DL ( Op ) ; LoadSDNode * Load = cast < LoadSDNode > ( Op ) ; if ( Op . getValueType ( ) . isVector ( ) ) {

assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ;


<s> PointerType * PtrTy = PointerType :: get ( Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; SDValue BasePtr = DAG . getCopyFromReg ( Chain , SL , MRI . getLiveInVirtReg ( InputPtrReg ) , PtrVT ) ; SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , PtrVT , BasePtr , DAG . getConstant ( Offset , SL , PtrVT ) ) ; SDValue PtrOffset = DAG . getUNDEF ( PtrVT ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned Align = DL . getABITypeAlignment ( Ty ) ; <STR_LIT> ExtTy = Signed ? <STR_LIT> : <STR_LIT> ; if ( MemVT . isFloatingPoint ( ) ) ExtTy = <STR_LIT> ; return DAG . getLoad ( <STR_LIT> , ExtTy , VT , SL , Chain , Ptr , PtrOffset , PtrInfo , MemVT , false , true , true , Align ) ;


<s> MachineRegisterInfo & MRI = DAG . getMachineFunction ( ) . getRegInfo ( ) ; MVT PtrVT = getPointerTy ( DL , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; PointerType * PtrTy = PointerType :: get ( Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; SDValue BasePtr = DAG . getCopyFromReg ( Chain , SL , MRI . getLiveInVirtReg ( InputPtrReg ) , PtrVT ) ; SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , PtrVT , BasePtr , DAG . getConstant ( Offset , SL , PtrVT ) ) ;

SDValue PtrOffset = DAG . getUNDEF ( PtrVT ) ;


<s> SDValue <ISA_LIT>TargetLowering :: LowerSTORE ( SDValue Op , SelectionDAG & DAG ) const { SDLoc DL ( Op ) ; StoreSDNode * Store = cast < StoreSDNode > ( Op ) ; EVT VT = Store -> getMemoryVT ( ) ; if ( Store -> getAddressSpace ( ) == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { if ( VT . isVector ( ) && VT . getVectorNumElements ( ) > <NUM_LIT> ) return ScalarizeVectorStore ( Op , DAG ) ; return SDValue ( ) ; } SDValue Ret = <ISA_LIT>TargetLowering :: LowerSTORE ( Op , DAG ) ; if ( Ret . getNode ( ) ) return Ret ; if ( VT . isVector ( ) && VT . getVectorNumElements ( ) >= <NUM_LIT> ) return ScalarizeVectorStore ( Op , DAG ) ; if ( VT == <STR_LIT> ) return DAG . getTruncStore ( Store -> getChain ( ) , DL , DAG . getSExtOrTrunc ( Store -> getValue ( ) , DL , <STR_LIT> ) , Store -> getBasePtr ( ) , <STR_LIT> , Store -> getMemOperand ( ) ) ; return SDValue ( ) ;


<s> for ( MachineBasicBlock * Succ : BB -> successors ( ) ) { for ( MachineInstr & MI : * Succ ) { if ( ! MI . isPHI ( ) ) break ; for ( unsigned I = <NUM_LIT> , E = MI . getNumOperands ( ) ; I != E ; I += <NUM_LIT> ) { unsigned IncomingReg = MI . getOperand ( I ) . getReg ( ) ; MachineOperand & FromBB = MI . getOperand ( I + <NUM_LIT> ) ; if ( BB == FromBB . getMBB ( ) ) { if ( SplitDefRegs . count ( IncomingReg ) ) FromBB . setMBB ( SplitBB ) ; break ; } } } } MF -> insert ( ++ MachineFunction :: iterator ( BB ) , SplitBB ) ; SplitBB -> splice ( SplitBB -> begin ( ) , BB , SplitPoint , BB -> end ( ) ) ; SplitBB -> transferSuccessors ( BB ) ;


<s> switch ( IntrinsicID ) { case <STR_LIT> : if ( ! Subtarget -> isAmdHsaOS ( ) ) { DiagnosticInfoUnsupported BadIntrin ( * MF . getFunction ( ) , <STR_LIT> ) ; DAG . getContext ( ) -> diagnose ( BadIntrin ) ; return DAG . getUNDEF ( VT ) ; } return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: DISPATCH_PTR ) , VT ) ; case <STR_LIT> : return LowerParameter ( DAG , VT , VT , DL , DAG . getEntryNode ( ) , <STR_LIT> :: NGROUPS_X , false ) ; case <STR_LIT> : return LowerParameter ( DAG , VT , VT , DL , DAG . getEntryNode ( ) , <STR_LIT> :: NGROUPS_Y , false ) ; case <STR_LIT> : return LowerParameter ( DAG , VT , VT , DL , DAG . getEntryNode ( ) , <STR_LIT> :: NGROUPS_Z , false ) ; case <STR_LIT> : return LowerParameter ( DAG , VT , VT , DL , DAG . getEntryNode ( ) , <STR_LIT> :: GLOBAL_<ISA_LIT>ZE_X , false ) ; case <STR_LIT> : return LowerParameter ( DAG , VT , VT , DL , DAG . getEntryNode ( ) , <STR_LIT> :: GLOBAL_<ISA_LIT>ZE_Y , false ) ; case <STR_LIT> : return LowerParameter ( DAG , VT , VT , DL , DAG . getEntryNode ( ) , <STR_LIT> :: GLOBAL_<ISA_LIT>ZE_Z , false ) ; case <STR_LIT> : return lowerImplicitZextParam ( DAG , Op , <STR_LIT> , <STR_LIT> :: LOCAL_<ISA_LIT>ZE_X ) ; case <STR_LIT> : return lowerImplicitZextParam ( DAG , Op , <STR_LIT> , <STR_LIT> :: LOCAL_<ISA_LIT>ZE_Y ) ; case <STR_LIT> : return lowerImplicitZextParam ( DAG , Op , <STR_LIT> , <STR_LIT> :: LOCAL_<ISA_LIT>ZE_Z ) ; case <STR_LIT> : return lowerImplicitZextParam ( DAG , Op , <STR_LIT> , getImplicitParameterOffset ( MFI , GRID_DIM ) ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: WORKGROUP_ID_X ) , VT ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: WORKGROUP_ID_Y ) , VT ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: WORKGROUP_ID_Z ) , VT ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: WORKITEM_ID_X ) , VT ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: WORKITEM_ID_Y ) , VT ) ; case <STR_LIT> : return CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: WORKITEM_ID_Z ) , VT ) ; case <STR_LIT> : { SDValue Ops [ ] = {

Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ;


<s> case <STR_LIT> : return LowerSampleIntrinsic ( <STR_LIT> , Op , DAG ) ; case <STR_LIT> : return LowerSampleIntrinsic ( <STR_LIT> , Op , DAG ) ; case <STR_LIT> : return LowerSampleIntrinsic ( <STR_LIT> , Op , DAG ) ; case <STR_LIT> : return DAG . getNode ( <STR_LIT> , DL , VT , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) ) ; case <STR_LIT> : case <STR_LIT> : return DAG . getNode ( <STR_LIT> , DL , VT , Op . getOperand ( <NUM_LIT> ) , DAG . getNode ( <STR_LIT> , DL , VT , Op . getOperand ( <NUM_LIT> ) ) ) ; case <STR_LIT> : { SDValue M0 = copyToM0 ( DAG , DAG . getEntryNode ( ) , DL , Op . getOperand ( <NUM_LIT> ) ) ; SDValue Glue = M0 . getValue ( <NUM_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Glue ) ; } case <STR_LIT> : if ( Op . getOperand ( <NUM_LIT> ) . isUndef ( ) && Op . getOperand ( <NUM_LIT> ) . isUndef ( ) ) return DAG . getUNDEF ( <STR_LIT> ) ; return Op ; case <STR_LIT> : { SDValue IJ = Op . getOperand ( <NUM_LIT> ) ; SDValue I = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , IJ , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) ; SDValue J = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , IJ , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ) ; SDValue M0 = copyToM0 ( DAG , DAG . getEntryNode ( ) , DL , Op . getOperand ( <NUM_LIT> ) ) ; SDValue Glue = M0 . getValue ( <NUM_LIT> ) ; SDValue P1 = DAG . getNode ( <STR_LIT> , DL , DAG . getVTList ( <STR_LIT> , <STR_LIT> ) , I , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Glue ) ; Glue = SDValue ( P1 . getNode ( ) , <NUM_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , P1 , J , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Glue ) ; } case <STR_LIT> : { SDValue M0 = copyToM0 ( DAG , DAG . getEntryNode ( ) , DL , Op . getOperand ( <NUM_LIT> ) ) ;