<s> Chain , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; EVT VT = Op . getOperand ( <NUM_LIT> ) . getValueType ( ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( ) , MachineMemOperand :: MOStore , VT . getStoreSize ( ) , <NUM_LIT> ) ; return DAG . getMemIntrinsicNode ( <STR_LIT> , DL , Op -> getVTList ( ) , Ops , VT , MMO ) ; } case <STR_LIT> : { SDValue Src = Op . getOperand ( <NUM_LIT> ) ; if ( const ConstantFPSDNode * K = dyn_cast < ConstantFPSDNode > ( Src ) ) { if ( ! K -> isNegative ( ) ) return Chain ; SDValue NegOne = DAG . getTargetConstant ( FloatToBits ( - <NUM_LIT> ) , DL , <STR_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> ,


<s> unsigned Opc = ( IntrID == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; SDValue Ops [ ] = { M -> getOperand ( <NUM_LIT> ) , M -> getOperand ( <NUM_LIT> ) , M -> getOperand ( <NUM_LIT> ) } ; return DAG . getMemIntrinsicNode ( Opc , SDLoc ( Op ) , M -> getVTList ( ) , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; } case <STR_LIT> : case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned Opc = ( IntrID == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; EVT VT = Op . getValueType ( ) ; EVT IntVT = VT . changeTypeToInteger ( ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( MFI -> getBufferPSV ( ) ) , MachineMemOperand :: MOLoad , VT . getStoreSize ( ) , VT . getStoreSize ( ) ) ; return DAG . getMemIntrinsicNode ( Opc , DL , Op -> getVTList ( ) , Ops , IntVT


<s> default : return <ISA_LIT>TargetLowering :: LowerOperation ( Op , DAG ) ; case <STR_LIT> : return LowerBRCOND ( Op , DAG ) ; case <STR_LIT> : { SDValue Result = LowerLOAD ( Op , DAG ) ; assert ( ( ! Result . getNode ( ) || Result . getNode ( ) -> getNumValues ( ) == <NUM_LIT> ) && <STR_LIT> ) ; return Result ; } case <STR_LIT> : case <STR_LIT> : return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op ,


<s> SDValue Load = DAG . getLoad ( MemVT , SL , Chain , Ptr , PtrInfo , Align , MachineMemOperand :: MONonTemporal | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ; SDValue Val = Load ; if ( Arg && ( Arg -> Flags . isSExt ( ) || Arg -> Flags . isZExt ( ) ) && VT . bitsLT ( MemVT ) ) { unsigned Opc = Arg -> Flags . isZExt ( ) ? <STR_LIT> : <STR_LIT> ; Val = DAG . getNode ( Opc , SL , MemVT , Val , DAG . getValueType ( VT ) ) ; } if ( MemVT . isFloatingPoint ( ) ) Val = getFPExtOrFPTrunc ( DAG , Val , SL , VT ) ; else if ( Signed ) Val = DAG . getSExtOrTrunc ( Val , SL , VT ) ; else Val = DAG . getZExtOrTrunc ( Val ,


<s> SDLoc DL ( Op ) ; StoreSDNode * Store = cast < StoreSDNode > ( Op ) ; EVT VT = Store -> getMemoryVT ( ) ; if ( VT == <STR_LIT> ) { return DAG . getTruncStore ( Store -> getChain ( ) , DL , DAG . getSExtOrTrunc ( Store -> getValue ( ) , DL , <STR_LIT> ) , Store -> getBasePtr ( ) , <STR_LIT> , Store -> getMemOperand ( ) ) ; } assert ( VT . isVector ( ) && Store -> getValue ( ) . getValueType ( ) . getScalarType ( ) == <STR_LIT> ) ; unsigned AS = Store -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , AS , Store -> getAlignment ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; switch ( AS ) { case <ISA_LIT>AS :: GLOBAL_ADDRESS : case <ISA_LIT>AS :: FLAT_ADDRESS : if ( NumElements > <NUM_LIT> ) return


<s> if ( DCI . isBeforeLegalize ( ) ) return SDValue ( ) ; SelectionDAG & DAG = DCI . DAG ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; SDValue LHS = N -> getOperand ( <NUM_LIT> ) ; SDValue RHS = N -> getOperand ( <NUM_LIT> ) ; if ( VT == <STR_LIT> ) { const ConstantSDNode * CRHS = dyn_cast < ConstantSDNode > ( RHS ) ; if ( CRHS ) { if ( SDValue Split = splitBinaryBitConstantOp ( DCI , SDLoc ( N ) , <STR_LIT> , LHS , CRHS ) ) return Split ; } } if ( LHS . getOpcode ( ) == <STR_LIT> && RHS . getOpcode ( ) == <STR_LIT> ) { <STR_LIT> LCC = cast < CondCodeSDNode > ( LHS . getOperand ( <NUM_LIT> ) ) -> get ( ) ; <STR_LIT> RCC = cast < CondCodeSDNode > ( RHS . getOperand ( <NUM_LIT> ) ) -> get ( ) ; SDValue X = LHS . getOperand ( <NUM_LIT> ) ; SDValue Y = RHS . getOperand ( <NUM_LIT> ) ; if ( Y . getOpcode ( ) != <STR_LIT> || Y . getOperand ( <NUM_LIT> ) != X ) return SDValue ( ) ; if ( LCC == <STR_LIT> ) { if ( X != LHS . getOperand ( <NUM_LIT> ) ) return SDValue ( ) ; if ( RCC == <STR_LIT> ) { const ConstantFPSDNode * C1 = dyn_cast < ConstantFPSDNode > ( RHS . getOperand ( <NUM_LIT> ) ) ; if ( ! C1 || ! C1 -> isInfinity ( ) || C1 -> isNegative ( ) ) return SDValue ( ) ; const uint32_t Mask = <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> ; static_assert ( ( ( ~ ( <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> ) ) & <NUM_LIT> ) == Mask , <STR_LIT> ) ; SDLoc DL ( N


<s> if ( X != LHS . getOperand ( <NUM_LIT> ) ) return SDValue ( ) ; if ( RCC == <STR_LIT> ) { const ConstantFPSDNode * C1 = dyn_cast < ConstantFPSDNode > ( RHS . getOperand ( <NUM_LIT> ) ) ; if ( ! C1 || ! C1 -> isInfinity ( ) || C1 -> isNegative ( ) ) return SDValue ( ) ; const uint32_t Mask = <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> ; static_assert ( ( ( ~ ( <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> ) ) & <NUM_LIT> ) == Mask , <STR_LIT> ) ; SDLoc DL ( N ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , X , DAG .

getConstant ( Mask , DL , <STR_LIT> ) ) ;


<s> SDValue Srl = N -> getOperand ( <NUM_LIT> ) ; if ( Srl . getOpcode ( ) == <STR_LIT> ) Srl = Srl . getOperand ( <NUM_LIT> ) ; if ( Srl . getOpcode ( ) == <STR_LIT> ) { if ( const ConstantSDNode * C = dyn_cast < ConstantSDNode > ( Srl . getOperand ( <NUM_LIT> ) ) ) { Srl = DAG . getZExtOrTrunc ( Srl . getOperand ( <NUM_LIT> ) , SDLoc ( Srl . getOperand ( <NUM_LIT> ) ) , EVT ( <STR_LIT> ) ) ; unsigned SrcOffset = C -> getZExtValue ( ) + <NUM_LIT> * Offset ; if ( SrcOffset < <NUM_LIT> && SrcOffset % <NUM_LIT> == <NUM_LIT> ) { return DAG . getNode ( <STR_LIT> + SrcOffset / <NUM_LIT> , SL , <STR_LIT> , Srl ) ; } } } APInt Demanded = APInt :: getBitsSet ( <NUM_LIT> , <NUM_LIT> * Offset , <NUM_LIT> * Offset + <NUM_LIT> ) ; APInt KnownZero , KnownOne ; TargetLowering :: TargetLoweringOpt TLO ( DAG , ! DCI . isBeforeLegalize ( ) , ! DCI . isBeforeLegalizeOps ( ) ) ; const TargetLowering & TLI = DAG


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; } case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> : return performXorCombine ( N , DCI ) ; case <STR_LIT> : return performClassCombine ( N , DCI ) ; case <STR_LIT> : return performFCanonicalizeCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src = N -> getOperand ( <NUM_LIT> ) ; if ( Src . isUndef ( ) ) return Src ; break ; } case <STR_LIT> : case <STR_LIT> : return performUCharToFloatCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT>


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; } case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> : return performXorCombine ( N , DCI ) ; case <STR_LIT> : return performClassCombine ( N , DCI ) ; case <STR_LIT> : return performFCanonicalizeCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src = N -> getOperand ( <NUM_LIT> ) ; if ( Src . isUndef ( ) )


<s> if ( DCI . getDAGCombineLevel ( ) < AfterLegalizeDAG ) return SDValue ( ) ; SelectionDAG & DAG = DCI . DAG ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; assert ( ! VT . isVector ( ) ) ; SDLoc SL ( N ) ; SDValue LHS = N -> getOperand ( <NUM_LIT> ) ; SDValue RHS = N -> getOperand ( <NUM_LIT> ) ; if ( LHS . getOpcode ( ) == <STR_LIT> ) { SDValue A = LHS . getOperand ( <NUM_LIT> ) ; if ( A == LHS . getOperand


<s> if ( VT == <STR_LIT> && ! Subtarget -> hasFP32Denormals ( ) ) return DAG . getConstantFP ( <NUM_LIT> , SDLoc ( N ) , VT ) ; if ( VT == <STR_LIT> && ! Subtarget -> hasFP64Denormals ( ) ) return DAG . getConstantFP ( <NUM_LIT> , SDLoc ( N ) , VT ) ; if ( VT == <STR_LIT> && ! Subtarget -> hasFP16Denormals ( ) ) return DAG . getConstantFP ( <NUM_LIT> , SDLoc ( N ) , VT ) ; } if ( C . isNaN ( ) ) { EVT VT = N -> getValueType ( <NUM_LIT> ) ; APFloat CanonicalQNaN = APFloat :: getQNaN ( C . getSemantics ( ) ) ; if ( C . isSignaling ( ) ) { return DAG . getConstantFP ( CanonicalQNaN , SDLoc ( N ) , VT


<s> const Triple & TT = getTargetMachine ( ) . getTargetTriple ( ) ; return GV -> getType ( ) -> getAddressSpace ( )

== <ISA_LIT>AS :: CONSTANT_ADDRESS && <STR_LIT> ( TT ) ;


<s> bool <ISA_LIT>TargetLowering :: shouldEmitFixup ( const GlobalValue * GV ) const { const Triple & TT = getTargetMachine


<s> MachineBasicBlock :: iterator SplitPoint ( & MI ) ; ++ SplitPoint ; if ( SplitPoint == BB -> end ( ) ) { MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; return BB ; } MachineFunction * MF = BB -> getParent (


<s> for ( unsigned i = <NUM_LIT> , Dmask = OldDmask ; i <= Lane ; i ++ ) { assert ( Dmask ) ; Comp = countTrailingZeros ( Dmask ) ; Dmask &= ~ ( <NUM_LIT> << Comp ) ; } if ( Users [ Lane ] ) return ; Users [ Lane ] = * I ; NewDmask |= <NUM_LIT> << Comp ; } if ( NewDmask == OldDmask ) return ; std :: vector < SDValue > Ops ; Ops . insert ( Ops . end ( ) , Node -> op_begin ( ) , Node -> op_begin ( ) + DmaskIdx ) ; Ops . push_back ( DAG . getTargetConstant ( NewDmask , SDLoc ( Node ) , <STR_LIT> ) ) ; Ops . insert ( Ops . end ( ) , Node -> op_begin ( ) + DmaskIdx + <NUM_LIT> , Node -> op_end ( ) ) ; Node = ( MachineSDNode * ) DAG . UpdateNodeOperands ( Node , Ops ) ; if ( NewDmask && ( NewDmask & ( NewDmask - <NUM_LIT> ) ) == <NUM_LIT> ) { SDValue RC = DAG . getTargetConstant ( <STR_LIT> , SDLoc ( ) , <STR_LIT> ) ; SDNode * Copy = DAG . getMachineNode ( TargetOpcode :: COPY_TO_REGCLASS , SDLoc ( ) , Users [ Lane ] -> getValueType ( <NUM_LIT> ) , SDValue ( Node , <NUM_LIT> ) , RC ) ; DAG . ReplaceAllUsesWith ( Users [ Lane ] , Copy ) ; return ; } for ( unsigned i = <NUM_LIT> , Idx = <STR_LIT> ; i < <NUM_LIT> ; ++ i ) { SDNode * User = Users [


<s> } if ( Info . hasDispatchPtr ( ) ) { unsigned DispatchPtrReg = Info . addDispatchPtr ( TRI ) ; MF . addLiveIn ( DispatchPtrReg , & <STR_LIT> ) ; CCInfo . AllocateReg ( DispatchPtrReg ) ; } if ( Info . hasQueuePtr ( ) ) { unsigned QueuePtrReg = Info . addQueuePtr ( TRI ) ; MF . addLiveIn ( QueuePtrReg , & <STR_LIT> ) ; CCInfo . AllocateReg ( QueuePtrReg ) ; } if ( Info . hasKernargSegmentPtr ( ) ) { unsigned InputPtrReg = Info . addKernargSegmentPtr ( TRI ) ; MF . addLiveIn ( InputPtrReg , & <STR_LIT> ) ; CCInfo . AllocateReg ( InputPtrReg ) ; } if ( Info .


<s> if ( Info . hasWorkItemIDY ( ) ) { unsigned Reg = TRI . getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: WORKITEM_ID_Y ) ; MF . addLiveIn ( Reg , & <STR_LIT> ) ; CCInfo . AllocateReg ( Reg ) ; } if ( Info . hasWorkItemIDZ ( ) ) { unsigned Reg = TRI . getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: WORKITEM_ID_Z ) ; MF . addLiveIn ( Reg , & <STR_LIT> ) ; CCInfo . AllocateReg ( Reg


<s> return false ; } if ( Subtarget -> hasUnalignedBufferAccess ( ) ) { if ( IsFast ) { * IsFast = ( AddrSpace == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ? ( Align % <NUM_LIT> == <NUM_LIT> ) : true ; } return true ; } if ( VT . bitsLT ( <STR_LIT> ) ) return false ; if ( IsFast ) *


<s> unsigned InitReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , InitReg ) ; if ( UseGPRIdxMode ) { MachineInstr * SetOn = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( VGPRIndexMode :: SRC0_ENABLE ) ; SetOn -> getOperand ( <NUM_LIT> ) . setIsUndef ( ) ; BuildMI ( MBB , std :: next ( I ) , DL , TII -> get ( <STR_LIT> ) ) ; } auto InsPt = loadM0FromVGPR ( TII , MBB , MI , InitReg , PhiReg , Offset , UseGPRIdxMode ) ; MachineBasicBlock * LoopBB = InsPt -> getParent ( ) ; if ( UseGPRIdxMode ) { BuildMI ( * LoopBB , InsPt , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , RegState :: Undef , SubReg ) . addReg (

SrcReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ;


<s> BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , RegState :: Undef , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ; } else { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , RegState :: Undef , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) ; } MI . eraseFromParent ( ) ; return & MBB ; } const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator I ( & MI ) ; unsigned PhiReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned InitReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , InitReg ) ; if ( UseGPRIdxMode ) { MachineInstr * SetOn = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( VGPRIndexMode :: SRC0_ENABLE ) ; SetOn -> getOperand ( <NUM_LIT> ) .


<s> return splitKillBlock ( MI , BB ) ; case <STR_LIT> : { MachineRegisterInfo & MRI = BB -> getParent ( ) -> getRegInfo ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src0 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src1 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; unsigned SrcCond = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned DstLo = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned DstHi = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) , DstLo ) . addReg ( Src0 , <NUM_LIT> , <STR_LIT> ) . addReg ( Src1 , <NUM_LIT> , <STR_LIT> ) . addReg ( SrcCond ) ; BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) , DstHi ) . addReg ( Src0 , <NUM_LIT> , <STR_LIT> ) . addReg ( Src1 , <NUM_LIT> , <STR_LIT> ) . addReg ( SrcCond ) ; BuildMI ( * BB , MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( DstLo ) . addImm ( <STR_LIT> ) . addReg ( DstHi ) . addImm ( <STR_LIT> ) ; MI . eraseFromParent ( ) ; return


<s> DiagnosticInfoUnsupported BadIntrin ( * DAG . getMachineFunction ( ) . getFunction ( ) , <STR_LIT> , DL . getDebugLoc ( ) ) ; DAG . getContext ( )


<s> static SDValue emitRemovedIntrinsicError ( SelectionDAG & DAG , const SDLoc & DL , EVT VT ) { DiagnosticInfoUnsupported BadIntrin ( * DAG . getMachineFunction ( ) . getFunction ( ) , <STR_LIT> , DL . getDebugLoc ( ) ) ; DAG . getContext ( ) -> diagnose ( BadIntrin ) ; return DAG . getUNDEF (


<s> DiagnosticInfoUnsupported BadIntrin ( * DAG . getMachineFunction ( ) . getFunction ( ) , <STR_LIT> , DL . getDebugLoc ( ) ) ; DAG . getContext ( ) -> diagnose ( BadIntrin ) ; return DAG . getUNDEF (


<s> case <STR_LIT> : case <STR_LIT> : { Value * Ptr = II -> getArgOperand ( <NUM_LIT> ) ; AccessTy = II -> getType ( ) ; Ops . push_back ( Ptr ) ; return true ; } default : return false


<s> case 's' : case 'r' : switch ( VT . getSizeInBits ( ) ) { default : return std :: make_pair ( <NUM_LIT> , nullptr ) ; case <NUM_LIT> : case <NUM_LIT> : return std :: make_pair ( <NUM_LIT> , & <STR_LIT> ) ; case <NUM_LIT> : return std :: make_pair ( <NUM_LIT> , & <STR_LIT> ) ; case <NUM_LIT> : return std :: make_pair ( <NUM_LIT> , & <STR_LIT> ) ; case <NUM_LIT> : return std :: make_pair ( <NUM_LIT> , & <STR_LIT> ) ; case <NUM_LIT> : return std :: make_pair ( <NUM_LIT> , & <STR_LIT> ) ; } case 'v' : switch ( VT . getSizeInBits ( ) ) { default : return std :: make_pair ( <NUM_LIT> , nullptr ) ; case <NUM_LIT> : case <NUM_LIT> : return std :: make_pair ( <NUM_LIT> , & <STR_LIT> ) ; case <NUM_LIT> : return std :: make_pair ( <NUM_LIT> , & <STR_LIT> ) ; case <NUM_LIT> : return std :: make_pair ( <NUM_LIT> , & <STR_LIT> ) ; case <NUM_LIT> : return std :: make_pair ( <NUM_LIT> , & <STR_LIT> ) ; case <NUM_LIT> : return std :: make_pair ( <NUM_LIT> , & <STR_LIT> ) ; case <NUM_LIT> : return std :: make_pair ( <NUM_LIT> , & <STR_LIT> ) ; } } } if ( Constraint . size ( ) > <NUM_LIT> ) { const TargetRegisterClass * RC = nullptr ; if ( Constraint [ <NUM_LIT> ] == 'v' ) { RC = & <STR_LIT> ; } else if ( Constraint [ <NUM_LIT> ] == 's' ) { RC = & <STR_LIT> ; } if ( RC ) { uint32_t Idx ; bool Failed = Constraint . substr ( <NUM_LIT> ) . getAsInteger ( <NUM_LIT> ,


<s> uint32_t StructOffset = ( AS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; SDValue Ptr = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , QueuePtr , DAG . getConstant ( StructOffset , DL , <STR_LIT> ) ) ; Value * V = UndefValue :: get ( PointerType :: get ( Type :: getInt8Ty ( * DAG . getContext ( ) ) , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ) ; MachinePointerInfo PtrInfo ( V , StructOffset ) ; return DAG . getLoad ( <STR_LIT> , DL , QueuePtr . getValue ( <NUM_LIT> ) , Ptr ,

PtrInfo , MinAlign ( <NUM_LIT> , StructOffset ) , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;


<s> Info . ptrVal = CI . getOperand ( <NUM_LIT> ) ; Info . align = <NUM_LIT> ; const ConstantInt * Vol = dyn_cast < ConstantInt > ( CI . getOperand ( <NUM_LIT> ) ) ; Info . vol = ! Vol || ! Vol -> isNullValue ( ) ; Info . readMem = true ; Info . writeMem = true ; return true ; } default : return false


<s> static bool isFlatGlobalAddrSpace ( unsigned AS ,


<s> static bool isFlatGlobalAddrSpace ( unsigned AS , <ISA_LIT>AS AMDGPUA<ISA_LIT> ) { return AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ||

AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ;


<s> bool <ISA_LIT>TargetLowering :: isFMAFasterThanFMulAndFAdd ( EVT VT ) const { VT = VT . getScalarType ( ) ; switch ( VT . getSimpleVT ( ) . SimpleTy ) { case <STR_LIT> : return Subtarget -> hasFP32Denormals ( ) && Subtarget -> hasFastFMAF32 ( ) ; case <STR_LIT> : return true ; case <STR_LIT>


<s> } else if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) { if ( AM . BaseOffs % <NUM_LIT> != <NUM_LIT> ) return isLegalMUBUFAddressingMode ( AM ) ; if ( DL . getTypeStoreSize ( Ty ) < <NUM_LIT> ) return isLegalMUBUFAddressingMode ( AM ) ; if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SEA_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; } else llvm_unreachable ( <STR_LIT> ) ; if ( AM . Scale == <NUM_LIT> ) return true ; if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ; return false ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { return isLegalMUBUFAddressingMode ( AM ) ; } else if ( AS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . REGION_ADDRESS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; if ( AM . Scale ==


<s> if ( DL . getTypeStoreSize ( Ty ) < <NUM_LIT> ) return isLegalMUBUFAddressingMode ( AM ) ; if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SEA_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; } else llvm_unreachable ( <STR_LIT> ) ; if ( AM . Scale == <NUM_LIT> ) return true ; if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ; return


<s> bool <ISA_LIT>TargetLowering :: isMemOpUniform ( const SDNode *


<s> bool <ISA_LIT>TargetLowering :: isMemOpUniform ( const SDNode * N ) const { const MemSDNode * MemNode = cast < MemSDNode > ( N ) ; return <STR_LIT> ( MemNode ->


<s> bool <ISA_LIT>TargetLowering :: isOffsetFoldingLegal ( const GlobalAddressSDNode * GA ) const { return ( GA -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || GA -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) && ! shouldEmitGOTReloc


<s> Result = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Result , DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ) ; } else { assert ( CIdx -> getZExtValue ( ) == <NUM_LIT> ) ; } if ( ResultVT . bitsLT ( <STR_LIT> ) ) Result = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Result ) ; return DAG . getNode ( <STR_LIT> , SL , ResultVT , Result ) ; } SDValue Sixteen = DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ; SDValue ScaledIdx = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Idx , Sixteen ) ; SDValue BC = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Vec ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL ,


<s> if ( ! Unsafe && VT == <STR_LIT> && Subtarget -> hasFP32Denormals ( ) ) return SDValue ( ) ; if ( const ConstantFPSDNode * CLHS = dyn_cast < ConstantFPSDNode > ( LHS ) ) { if ( Unsafe || VT == <STR_LIT> || VT == <STR_LIT> ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { if ( RHS . getOpcode ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , SL , VT , RHS . getOperand ( <NUM_LIT> ) ) ; return DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { SDValue FNegRHS = DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; return DAG . getNode ( <STR_LIT> , SL , VT , FNegRHS ) ; } } } const SDNodeFlags * Flags = Op -> getFlags ( ) ; if ( Unsafe || Flags -> hasAllowReciprocal ( ) ) { SDNodeFlags


<s> SDValue LHS = Op . getOperand ( <NUM_LIT> ) ; SDValue RHS = Op . getOperand ( <NUM_LIT> ) ; EVT VT = Op . getValueType ( ) ; bool Unsafe = DAG . getTarget ( ) . Options . UnsafeFPMath ; if ( ! Unsafe && VT == <STR_LIT> && Subtarget -> hasFP32Denormals ( ) ) return SDValue ( ) ; if ( const ConstantFPSDNode * CLHS = dyn_cast < ConstantFPSDNode > ( LHS ) ) { if ( Unsafe || VT == <STR_LIT> || VT == <STR_LIT> ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { if ( RHS . getOpcode ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , SL , VT , RHS . getOperand ( <NUM_LIT> ) ) ; return DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { SDValue FNegRHS = DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; return DAG . getNode ( <STR_LIT> , SL , VT , FNegRHS ) ; } } } const SDNodeFlags * Flags = Op ->


<s> assert ( ! IsKernel || ( Info -> hasWorkGroupIDX ( ) && Info -> hasWorkItemIDX ( ) ) ) ; } if ( IsEntryFunc ) { allocateSpecialInputVGPRs ( CCInfo , MF , * TRI , * Info ) ; allocateHSAUserSGPRs ( CCInfo , MF , * TRI , * Info ) ; } if ( IsKernel ) { analyzeFormalArgumentsCompute ( CCInfo , Ins ) ; } else { CCAssignFn * AssignFn = CCAssignFnForCall ( CallConv , isVarArg ) ; CCInfo . AnalyzeFormalArguments ( Splits , AssignFn ) ; } SmallVector < SDValue , <NUM_LIT> > Chains ; for ( unsigned i = <NUM_LIT> , e = Ins . size ( ) , ArgIdx = <NUM_LIT> ; i != e ; ++ i ) { const <STR_LIT> & Arg = Ins [ i ] ; if ( Skipped [ i ] ) { InVals . push_back ( DAG . getUNDEF ( Arg . VT ) ) ; continue ; } CCValAssign & VA = ArgLocs [ ArgIdx ++ ] ; MVT VT = VA . getLocVT ( ) ; if ( IsEntryFunc && VA . isMemLoc ( ) ) { VT = Ins [ i ] . VT ; EVT MemVT = VA . getLocVT ( ) ; const uint64_t Offset = Subtarget -> getExplicitKernelArgOffset ( MF ) + VA . getLocMemOffset ( ) ; Info -> setABIArgOffset ( Offset + MemVT . getStoreSize ( ) ) ; SDValue Arg = lowerKernargMemParameter ( DAG , VT , MemVT , DL , Chain , Offset , Ins [ i ] . Flags . isSExt ( ) , & Ins [ i ] ) ; Chains . push_back ( Arg . getValue ( <NUM_LIT> ) ) ; auto * ParamTy = dyn_cast < PointerType > ( FType -> getParamType ( Ins [ i ] . getOrigArgIndex ( ) ) ) ; if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS && ParamTy && ParamTy -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ) { Arg = DAG . getNode ( <STR_LIT> , DL , Arg . getValueType ( ) , Arg , DAG . getValueType ( <STR_LIT> ) ) ; } InVals . push_back ( Arg ) ; continue ; } if ( VA . isMemLoc ( ) ) report_fatal_error ( <STR_LIT> ) ; assert ( VA . isRegLoc ( ) && <STR_LIT>


<s> SDLoc DL ( GSD ) ; const GlobalValue * GV = GSD -> getGlobal ( ) ; EVT PtrVT = Op . getValueType ( ) ; if ( shouldEmitFixup ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT ) ; else if ( shouldEmitPCReloc ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL ,

GSD -> getOffset ( ) , PtrVT , <ISA_LIT>InstrInfo :: MO_REL32 ) ;


<s> SDLoc SL ( Op ) ; SDValue Param = lowerKernargMemParameter ( DAG , <STR_LIT> , <STR_LIT> , SL , DAG . getEntryNode ( ) , Offset , false ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Param , DAG . getValueType (


<s> SDValue Ops [ ] = { M -> getOperand ( <NUM_LIT> ) , M -> getOperand ( <NUM_LIT> ) , M -> getOperand ( <NUM_LIT> ) } ; return DAG . getMemIntrinsicNode ( Opc , SDLoc ( Op ) , M -> getVTList ( ) , Ops , M -> getMemoryVT ( ) , M -> getMemOperand ( ) ) ; } case <STR_LIT> : case <STR_LIT> : { SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) } ; MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned Opc = ( IntrID == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; EVT VT = Op . getValueType ( ) ; EVT IntVT = VT . changeTypeToInteger ( ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( MachinePointerInfo ( MFI -> getBufferPSV ( ) ) , MachineMemOperand :: MOLoad , VT . getStoreSize ( ) , VT . getStoreSize ( ) ) ; return DAG . getMemIntrinsicNode ( Opc , DL , Op -> getVTList ( ) , Ops , IntVT , MMO ) ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const ConstantSDNode * DMask = dyn_cast < ConstantSDNode > ( Op . getOperand


<s> const DataLayout & DL = DAG . getDataLayout ( ) ; Type * Ty = MemVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned Align = DL . getABITypeAlignment ( Ty ) ; SDValue Ptr = lowerKernArgParameterPtr ( DAG , SL , Chain , Offset ) ; SDValue Load = DAG . getLoad ( MemVT , SL , Chain , Ptr , PtrInfo , Align , MachineMemOperand :: MONonTemporal | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ; SDValue Val = convertArgType ( DAG , VT , MemVT , SL ,


<s> const DataLayout & DL = DAG . getDataLayout ( ) ; MachineFunction & MF = DAG . getMachineFunction ( ) ; const <ISA_LIT>RegisterInfo * TRI = getSubtarget ( ) -> getRegisterInfo ( ) ; unsigned InputPtrReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR ) ; MachineRegisterInfo & MRI = DAG . getMachineFunction ( ) . getRegInfo ( ) ; MVT PtrVT = getPointerTy ( DL , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; SDValue BasePtr = DAG . getCopyFromReg ( Chain , SL , MRI .

getLiveInVirtReg ( InputPtrReg ) , PtrVT ) ;


<s> MachineMemOperand * MMO = Load -> getMemOperand ( ) ; EVT RealMemVT = ( MemVT == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; SDValue NewLD = DAG . getExtLoad ( <STR_LIT> , DL , <STR_LIT> , Chain , BasePtr , RealMemVT , MMO ) ; SDValue Ops [ ] = { DAG . getNode ( <STR_LIT> , DL , MemVT , NewLD ) , NewLD . getValue ( <NUM_LIT> ) } ; return DAG . getMergeValues ( Ops , DL ) ; } if ( ! MemVT . isVector ( ) ) return SDValue ( ) ; assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ; unsigned AS = Load -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , MemVT , AS , Load -> getAlignment ( ) ) ) { SDValue Ops [ <NUM_LIT> ] ; std :: tie ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) = expandUnalignedLoad ( Load , DAG ) ; return DAG . getMergeValues ( Ops , DL ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) { if ( isMemOpUniform ( Load ) ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && isMemOpUniform ( Load ) && isMemOpHasNoClobberedMemOperand ( Load ) ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load


<s> return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op , DAG ) ; } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID ( Op , DAG ) ; case <STR_LIT> : return lowerADDRSPACECAST ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_VECTOR_ELT ( Op


<s> return Result ; } case <STR_LIT> : case <STR_LIT> : return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op , DAG ) ; } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID ( Op , DAG ) ; case <STR_LIT> : return lowerADDRSPACECAST ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_VECTOR_ELT ( Op , DAG


<s> if ( VT == <STR_LIT> ) { return DAG . getTruncStore ( Store -> getChain ( ) , DL , DAG . getSExtOrTrunc ( Store -> getValue ( ) , DL , <STR_LIT> ) , Store -> getBasePtr ( ) , <STR_LIT> , Store -> getMemOperand ( ) ) ; } assert ( VT . isVector ( ) && Store -> getValue ( ) . getValueType ( ) . getScalarType ( ) == <STR_LIT> ) ; unsigned AS = Store -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , AS , Store -> getAlignment ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable ( <STR_LIT>


<s> unsigned TrapID = Op . getOpcode ( ) == <STR_LIT> ? <ISA_LIT>Subtarget :: TrapIDLLVMDebugTrap : <ISA_LIT>Subtarget :: TrapIDLLVMTrap ; if ( Subtarget -> getTrapHandlerAbi ( ) == <ISA_LIT>Subtarget :: TrapHandlerAbiHsa && Subtarget -> isTrapHandlerEnabled ( ) ) { <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; SDValue SGPR01 = DAG . getRegister ( <STR_LIT> , <STR_LIT> ) ; SDValue ToReg = DAG . getCopyToReg ( Chain , SL , SGPR01 , QueuePtr , SDValue ( ) ) ; SDValue Ops [


<s> EVT SVT = VT . getScalarType ( ) ; if ( SVT == <STR_LIT> && ! Subtarget -> hasFP32Denormals ( ) ) return DAG . getConstantFP ( <NUM_LIT> , SDLoc ( N ) , VT ) ; if ( SVT == <STR_LIT> && ! Subtarget -> hasFP64Denormals ( ) ) return DAG . getConstantFP ( <NUM_LIT> , SDLoc ( N ) , VT ) ; if ( SVT == <STR_LIT> && ! Subtarget -> hasFP16Denormals ( ) ) return DAG . getConstantFP ( <NUM_LIT> , SDLoc ( N ) , VT ) ; } if ( C . isNaN ( ) ) { EVT VT = N -> getValueType ( <NUM_LIT> ) ; APFloat CanonicalQNaN = APFloat :: getQNaN ( C . getSemantics ( ) ) ; if ( C . isSignaling ( ) ) { return DAG . getConstantFP ( CanonicalQNaN , SDLoc ( N


<s> if ( Cmp == APFloat :: cmpGreaterThan ) return SDValue ( ) ; EVT VT = K0 -> getValueType ( <NUM_LIT> ) ; if ( Subtarget -> enableDX10Clamp ( ) ) { if ( K1 -> isExactlyValue ( <NUM_LIT> ) && K0 -> isExactlyValue ( <NUM_LIT> ) ) return DAG . getNode ( <STR_LIT> , SL , VT , Op0 . getOperand ( <NUM_LIT> ) ) ; } if ( VT == <STR_LIT> || ( VT == <STR_LIT> && ! Subtarget -> hasMed3_16 ( ) ) ) return SDValue


<s> SelectionDAG & DAG = DCI . DAG ; SDLoc SL ( N ) ; unsigned AS = N -> getAddressSpace ( ) ; if ( Ptr . getOpcode ( ) == <STR_LIT> && AS != AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { SDValue NewPtr = performSHLPtrCombine ( Ptr . getNode ( ) , AS , DCI ) ; if (


<s> if ( SDValue Med3 = performIntMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 , true ) ) return Med3 ; } if ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> && Op0 . hasOneUse ( ) ) { if ( SDValue Med3 = performIntMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 , false ) ) return Med3 ; } if ( ( ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> ) || ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> ) ) && ( VT == <STR_LIT> || VT == <STR_LIT> || ( VT == <STR_LIT> && Subtarget -> has16BitInsts ( ) ) ) && Op0 . hasOneUse ( ) ) { if ( SDValue Res = performFPMed3ImmCombine ( DAG , SDLoc (


<s> return GV -> getType ( ) -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS


<s> return GV -> getType ( ) -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS &&


<s> return ( GV -> getType ( ) -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || GV -> getType ( ) -> getAddressSpace ( ) == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) && ! shouldEmitFixup ( GV ) && ! getTargetMachine ( ) . shouldAssumeDSOLocal ( * GV -> getParent


<s> MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( SrcVec -> getReg ( ) , RegState :: Undef , SubReg ) . add ( * Val ) . addReg ( Dst , RegState :: ImplicitDefine ) . addReg ( SrcVec -> getReg ( ) , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ; } else { const MCInstrDesc & MovRelDesc = TII -> get ( getMOVRELDPseudo ( TRI , VecRC ) ) ; BuildMI ( MBB , I , DL , MovRelDesc ) . addReg ( Dst , RegState :: Define ) . addReg ( SrcVec -> getReg ( ) ) . add ( * Val ) . addImm ( SubReg - <STR_LIT> ) ; } MI . eraseFromParent ( ) ; return & MBB ; } if ( Val -> isReg ( ) ) MRI . clearKillFlags ( Val -> getReg ( ) ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { MachineBasicBlock :: iterator I ( & MI ) ; MachineInstr * SetOn = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( VGPRIndexMode :: DST_ENABLE ) ; SetOn -> getOperand ( <NUM_LIT> )


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; if ( Info -> isEntryFunction ( ) ) { reservePrivateMemoryRegs ( getTargetMachine ( ) , MF , * TRI , * Info ) ; } bool NeedSP = ! Info -> isEntryFunction ( ) || MFI . hasVarSizedObjects ( ) || MFI . hasCalls ( ) ; if ( NeedSP ) { unsigned ReservedStackPtrOffsetReg = TRI -> reservedStackPtrOffsetReg ( MF ) ; Info -> setStackPtrOffsetReg ( ReservedStackPtrOffsetReg ) ; assert ( Info -> getStackPtrOffsetReg ( ) != Info -> getFrameOffsetReg ( ) ) ; assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ; MRI . replaceRegWith ( <STR_LIT> , Info -> getStackPtrOffsetReg ( ) ) ; } MRI . replaceRegWith ( <STR_LIT> , Info -> getScratchRSrcReg (


<s> unsigned ReservedStackPtrOffsetReg = TRI -> reservedStackPtrOffsetReg ( MF ) ; Info -> setStackPtrOffsetReg ( ReservedStackPtrOffsetReg ) ; assert ( Info -> getStackPtrOffsetReg ( ) != Info -> getFrameOffsetReg ( ) ) ; assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ; MRI . replaceRegWith ( <STR_LIT> , Info -> getStackPtrOffsetReg ( ) ) ; } MRI . replaceRegWith ( <STR_LIT> , Info -> getScratchRSrcReg ( ) ) ; MRI . replaceRegWith ( <STR_LIT> , Info -> getFrameOffsetReg (


<s> if ( ( Options . AllowFPOpFusion == FPOpFusion :: Fast || Options . UnsafeFPMath || ( N0 -> getFlags ( ) . hasUnsafeAlgebra ( ) && N1 -> getFlags ( ) . hasUnsafeAlgebra ( ) ) ) &&


<s> SDValue <ISA_LIT>TargetLowering :: getImplicitArgPtr ( SelectionDAG & DAG , const SDLoc & SL ) const { auto MFI = DAG . getMachineFunction ( ) . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; uint64_t Offset = getImplicitParameterOffset ( MFI ,


<s> Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = MFI -> getImagePSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable ; Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = MFI -> getImagePSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable ; Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable ; return true ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . memVT = <STR_LIT> ( CI . getArgOperand ( <NUM_LIT> ) -> getType ( ) ) ; Info . flags = MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable ; return true ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore

| MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOVolatile ;


<s> if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) { if ( AM . BaseOffs % <NUM_LIT> != <NUM_LIT> ) return isLegalMUBUFAddressingMode ( AM ) ; if ( DL . getTypeStoreSize ( Ty ) < <NUM_LIT> ) return isLegalGlobalAddressingMode ( AM ) ; if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SEA_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; } else llvm_unreachable ( <STR_LIT> ) ; if ( AM . Scale == <NUM_LIT> ) return true ; if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg )


<s> BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , TmpExec ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SaveExec ) . addReg ( <STR_LIT> ) ; MachineBasicBlock * LoopBB = MF -> CreateMachineBasicBlock ( ) ; MachineBasicBlock * RemainderBB = MF -> CreateMachineBasicBlock ( ) ; MachineFunction :: iterator MBBI ( MBB ) ; ++ MBBI ; MF -> insert ( MBBI , LoopBB ) ; MF -> insert ( MBBI , RemainderBB ) ; LoopBB -> addSuccessor ( LoopBB ) ; LoopBB -> addSuccessor ( RemainderBB ) ; RemainderBB -> transferSuccessorsAndUpdatePHIs ( & MBB ) ; RemainderBB -> splice ( RemainderBB -> begin ( ) , & MBB , I , MBB . end ( ) ) ; MBB . addSuccessor ( LoopBB ) ; const MachineOperand * Idx = TII -> getNamedOperand ( MI , <STR_LIT> :: idx ) ; auto InsPt = emitLoadM0FromVGPRLoop ( TII , MRI , MBB , * LoopBB , DL , * Idx , InitResultReg , DstReg , PhiReg , TmpExec , Offset , UseGPRIdxMode ) ; MachineBasicBlock :: iterator First = RemainderBB -> begin ( ) ; BuildMI ( * RemainderBB , First , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( SaveExec ) ; return InsPt


<s> const SDNodeFlags Flags = Op -> getFlags ( ) ; bool Unsafe = DAG . getTarget ( ) . Options . UnsafeFPMath || Flags . hasUnsafeAlgebra ( ) || Flags . hasAllowReciprocal ( ) ; if ( ! Unsafe && VT == <STR_LIT> && Subtarget -> hasFP32Denormals ( ) ) return SDValue ( ) ; if ( const ConstantFPSDNode * CLHS = dyn_cast < ConstantFPSDNode > ( LHS ) ) { if ( Unsafe || VT == <STR_LIT> || VT == <STR_LIT> ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { if ( RHS . getOpcode ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , SL , VT , RHS . getOperand ( <NUM_LIT> ) ) ; return DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { SDValue FNegRHS = DAG . getNode ( <STR_LIT> , SL


<s> if ( ! MemVT . isVector ( ) ) return SDValue ( ) ; assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ; unsigned AS = Load -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , MemVT , AS , Load -> getAlignment ( ) ) ) { SDValue Ops [ <NUM_LIT> ] ; std :: tie ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) = expandUnalignedLoad ( Load , DAG ) ; return DAG . getMergeValues ( Ops , DL ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) { if ( isMemOpUniform ( Load ) ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && isMemOpUniform ( Load ) && ! Load -> isVolatile ( ) && isMemOpHasNoClobberedMemOperand ( Load ) ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op


<s> SDValue <ISA_LIT>TargetLowering :: lowerTRAP ( SDValue Op , SelectionDAG & DAG ) const { SDLoc SL ( Op ) ; MachineFunction & MF = DAG . getMachineFunction ( ) ; SDValue Chain = Op . getOperand ( <NUM_LIT> ) ; unsigned TrapID = Op . getOpcode ( ) == <STR_LIT> ? <ISA_LIT>Subtarget :: TrapIDLLVMDebugTrap : <ISA_LIT>Subtarget :: TrapIDLLVMTrap ; if ( Subtarget -> getTrapHandlerAbi ( ) == <ISA_LIT>Subtarget :: TrapHandlerAbiHsa && Subtarget -> isTrapHandlerEnabled ( ) ) { <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; SDValue SGPR01 = DAG . getRegister ( <STR_LIT> , <STR_LIT> ) ; SDValue ToReg = DAG . getCopyToReg ( Chain , SL , SGPR01 , QueuePtr , SDValue ( ) ) ; SDValue Ops [ ] = { ToReg , DAG . getTargetConstant ( TrapID , SL , <STR_LIT> ) , SGPR01 , ToReg . getValue ( <NUM_LIT> ) } ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Ops ) ; } switch ( TrapID ) { case <ISA_LIT>Subtarget :: TrapIDLLVMTrap : return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> ,


<s> void <ISA_LIT>TargetLowering :: passSpecialInputs ( CallLoweringInfo & CLI , const <ISA_LIT>MachineFunctionInfo & Info , SmallVectorImpl < std :: pair < unsigned , SDValue >> & RegsToPass , SmallVectorImpl < SDValue > & MemOpChains , SDValue Chain , SDValue StackPtr ) const { if ( ! CLI . CS ) return ; const Function * CalleeFunc = CLI . CS . getCalledFunction ( ) ; assert ( CalleeFunc ) ; SelectionDAG & DAG = CLI . DAG ; const SDLoc & DL = CLI . DL ; const <ISA_LIT>Subtarget * ST = getSubtarget ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST -> getRegisterInfo ( ) ; auto & ArgUsageInfo = DAG . getPass ( ) -> getAnalysis < <ISA_LIT>ArgumentUsageInfo > ( ) ; const <ISA_LIT>FunctionArgInfo & CalleeArgInfo = ArgUsageInfo . lookupFuncArgInfo ( * CalleeFunc ) ; const <ISA_LIT>FunctionArgInfo & CallerArgInfo = Info . getArgInfo ( ) ; <STR_LIT> InputRegs [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; for ( auto InputID : InputRegs ) { const ArgDescriptor * OutgoingArg ; const TargetRegisterClass * ArgRC ; std :: tie ( OutgoingArg , ArgRC ) = CalleeArgInfo . getPreloadedValue ( InputID ) ; if ( ! OutgoingArg ) continue ; const ArgDescriptor * IncomingArg ; const TargetRegisterClass * IncomingArgRC ; std :: tie ( IncomingArg , IncomingArgRC ) = CallerArgInfo . getPreloadedValue ( InputID ) ; assert ( IncomingArgRC == ArgRC ) ; EVT ArgVT = TRI -> getSpillSize ( * ArgRC ) == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; SDValue InputReg ; if ( IncomingArg ) { InputReg = loadInputValue ( DAG , ArgRC , ArgVT , DL , * IncomingArg ) ; } else { assert ( InputID == <STR_LIT> ) ; InputReg = getImplicitArgPtr ( DAG , DL ) ; } if ( OutgoingArg -> isRegister ( ) ) { RegsToPass . emplace_back ( OutgoingArg -> getRegister ( ) , InputReg ) ; } else { SDValue ArgStore = storeStackInputValue ( DAG , DL , Chain , StackPtr ,

InputReg , OutgoingArg -> getStackOffset ( ) ) ;


<s> if ( numBitsUnsigned ( MulLHS , DAG ) <= <NUM_LIT> && numBitsUnsigned ( MulRHS , DAG ) <= <NUM_LIT> ) { MulLHS = DAG . getZExtOrTrunc ( MulLHS , SL , <STR_LIT> ) ; MulRHS = DAG . getZExtOrTrunc ( MulRHS , SL , <STR_LIT> ) ; AddRHS = DAG . getZExtOrTrunc ( AddRHS , SL , <STR_LIT> ) ; return getMad64_32 ( DAG , SL , VT , MulLHS , MulRHS , AddRHS , false ) ; } if ( numBitsSigned ( MulLHS , DAG ) < <NUM_LIT> && numBitsSigned ( MulRHS , DAG ) < <NUM_LIT> ) { MulLHS = DAG . getSExtOrTrunc ( MulLHS , SL , <STR_LIT> ) ; MulRHS = DAG . getSExtOrTrunc ( MulRHS , SL , <STR_LIT> ) ; AddRHS = DAG . getSExtOrTrunc ( AddRHS , SL , <STR_LIT> ) ; return getMad64_32 ( DAG , SL , VT , MulLHS , MulRHS , AddRHS , true ) ; } return SDValue ( ) ; } if ( VT != <STR_LIT> ) return SDValue ( ) ; unsigned Opc = LHS . getOpcode ( ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) std :: swap ( RHS , LHS ) ; Opc = RHS . getOpcode ( ) ; switch ( Opc ) { default : break ; case <STR_LIT> : case <STR_LIT>


<s> if ( ( Offset & ( Bits - <NUM_LIT> ) ) == <NUM_LIT> ) { SDLoc SL ( N ) ; SDValue BFE = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LHS -> getOperand ( <NUM_LIT> ) , DAG . getConstant ( Offset , SL , <STR_LIT> ) , DAG . getConstant ( Bits , SL , <STR_LIT> ) ) ; EVT NarrowVT = EVT :: getIntegerVT ( * DAG . getContext ( ) , Bits ) ; SDValue Ext = DAG . getNode ( <STR_LIT> , SL , VT , BFE , DAG . getValueType ( NarrowVT ) ) ; SDValue Shl = DAG . getNode ( <STR_LIT> , SDLoc ( LHS ) , VT , Ext , DAG . getConstant ( NB , SDLoc ( CRHS ) , <STR_LIT> ) ) ; return Shl ; } } } } if ( LHS . getOpcode ( ) == <STR_LIT> && RHS . getOpcode ( ) == <STR_LIT> ) { <STR_LIT> LCC = cast < CondCodeSDNode > ( LHS . getOperand ( <NUM_LIT> ) ) -> get ( ) ; <STR_LIT> RCC = cast < CondCodeSDNode > ( RHS . getOperand ( <NUM_LIT> ) ) -> get ( ) ; SDValue X = LHS . getOperand ( <NUM_LIT> ) ; SDValue Y = RHS . getOperand ( <NUM_LIT> ) ; if ( Y . getOpcode ( ) != <STR_LIT> || Y . getOperand ( <NUM_LIT> ) != X ) return SDValue ( ) ; if ( LCC == <STR_LIT> ) { if ( X != LHS . getOperand ( <NUM_LIT> ) ) return SDValue ( ) ; if ( RCC == <STR_LIT> ) { const ConstantFPSDNode * C1 = dyn_cast < ConstantFPSDNode > ( RHS . getOperand ( <NUM_LIT> ) ) ; if ( ! C1 || ! C1 -> isInfinity ( ) || C1 -> isNegative ( ) ) return SDValue ( ) ; const uint32_t Mask = <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> ; static_assert ( ( ( ~ ( <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> ) ) & <NUM_LIT> ) == Mask , <STR_LIT> ) ; SDLoc DL ( N ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , X , DAG . getConstant ( Mask , DL , <STR_LIT> ) ) ; } } } if ( VT == <STR_LIT> && ( RHS . getOpcode ( ) == <STR_LIT> || LHS . getOpcode ( ) == <STR_LIT> ) ) { if ( RHS . getOpcode ( ) != <STR_LIT> ) std :: swap ( LHS , RHS ) ; if ( isBoolSGPR ( RHS . getOperand ( <NUM_LIT> ) ) ) return DAG . getSelect ( SDLoc ( N ) , <STR_LIT> , RHS . getOperand ( <NUM_LIT> ) , LHS , DAG . getConstant ( <NUM_LIT> , SDLoc


<s> SDValue Vec = N -> getOperand ( <NUM_LIT> ) ; SelectionDAG & DAG = DCI . DAG ; if ( Vec . getOpcode ( ) == <STR_LIT> && allUsesHaveSourceMods ( N ) ) { SDLoc SL ( N ) ; EVT EltVT = N -> getValueType


<s> SDValue Idx = N -> getOperand ( <NUM_LIT> ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; return DAG . getNode ( <STR_LIT> , SL , EltVT , Elt ) ; } return SDValue (


<s> SDValue Cvt = DAG . getNode ( Opcode , SL , <STR_LIT> , Src0 , Src1 ) ; Results . push_back ( DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Cvt ) ) ; return ; } } break ; } case <STR_LIT> : { SDLoc SL ( N ) ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VT ) ; SDValue LHS = DAG . getNode ( <STR_LIT> , SL , NewVT , N -> getOperand ( <NUM_LIT> ) ) ; SDValue RHS = DAG . getNode ( <STR_LIT> , SL , NewVT , N -> getOperand ( <NUM_LIT> ) ) ; EVT SelectVT = NewVT ; if ( NewVT . bitsLT ( <STR_LIT> ) ) { LHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LHS ) ; RHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , RHS ) ; SelectVT = <STR_LIT> ; } SDValue NewSelect = DAG . getNode ( <STR_LIT> , SL , SelectVT , N -> getOperand ( <NUM_LIT> ) , LHS , RHS ) ; if ( NewVT != SelectVT ) NewSelect = DAG . getNode ( <STR_LIT> , SL , NewVT , NewSelect ) ; Results . push_back ( DAG . getNode ( <STR_LIT> , SL


<s> Chains . push_back ( Arg . getValue ( <NUM_LIT> ) ) ; auto * ParamTy = dyn_cast < PointerType > ( FType -> getParamType ( Ins [ i ] . getOrigArgIndex ( ) ) ) ; if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS && ParamTy && ParamTy -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ) { Arg = DAG . getNode ( <STR_LIT> , DL , Arg . getValueType ( ) , Arg , DAG . getValueType ( <STR_LIT> ) ) ; } InVals . push_back ( Arg ) ; Info -> ABIArgOffset = Offset + MemVT . getStoreSize ( ) ; continue ; } assert ( VA . isRegLoc ( ) && <STR_LIT> ) ; unsigned Reg = VA . getLocReg ( ) ; if ( VT == <STR_LIT> ) { Reg = TRI -> getMatchingSuperReg ( Reg , <STR_LIT> , & <STR_LIT> ) ; Reg = MF . addLiveIn ( Reg , & <STR_LIT> ) ; SDValue Copy = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; InVals . push_back ( Copy ) ; continue ; } const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg , VT ) ; Reg = MF . addLiveIn ( Reg , RC ) ; SDValue Val = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; if ( Arg . VT . isVector ( ) ) { Type * ParamType = FType -> getParamType ( Arg . getOrigArgIndex ( ) ) ; unsigned NumElements = ParamType -> getVectorNumElements ( ) ; SmallVector < SDValue , <NUM_LIT> > Regs ; Regs . push_back ( Val ) ; for ( unsigned j = <NUM_LIT> ; j != NumElements ; ++ j ) { Reg = ArgLocs [ ArgIdx ++ ] . getLocReg ( ) ; Reg = MF . addLiveIn ( Reg , RC ) ; SDValue Copy = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; Regs . push_back ( Copy ) ; } NumElements = Arg . VT . getVectorNumElements ( ) - NumElements ; Regs . append ( NumElements , DAG . getUNDEF ( VT ) ) ; InVals . push_back ( DAG . getNode ( <STR_LIT> , DL , Arg . VT , Regs ) ) ; continue ; } InVals . push_back ( Val ) ; } if ( Info -> getShaderType ( ) != ShaderType :: COMPUTE ) { unsigned ScratchIdx = CCInfo . getFirstUnallocated ( ArrayRef < MCPhysReg > ( <STR_LIT> . begin ( ) , <STR_LIT> . getNumRegs ( ) ) ) ; Info -> ScratchOffsetReg = <STR_LIT> . getRegister ( ScratchIdx ) ; } if ( Chains . empty


<s> void <ISA_LIT>TargetLowering :: allocateSpecialInputSGPRs ( CCState & CCInfo , MachineFunction & MF , const <ISA_LIT>RegisterInfo & TRI , <ISA_LIT>MachineFunctionInfo & Info ) const { auto & ArgInfo = Info . getArgInfo ( ) ; if ( Info . hasDispatchPtr ( ) ) allocateSGPR64Input ( CCInfo , ArgInfo . DispatchPtr ) ; if ( Info . hasQueuePtr ( ) ) allocateSGPR64Input ( CCInfo , ArgInfo . QueuePtr ) ; if ( Info . hasImplicitArgPtr ( ) ) allocateSGPR64Input ( CCInfo , ArgInfo . ImplicitArgPtr ) ; if ( Info . hasDispatchID ( ) ) allocateSGPR64Input ( CCInfo , ArgInfo . DispatchID ) ; if ( Info . hasWorkGroupIDX ( ) ) allocateSGPR32Input ( CCInfo , ArgInfo . WorkGroupIDX ) ; if ( Info . hasWorkGroupIDY ( ) ) allocateSGPR32Input ( CCInfo , ArgInfo . WorkGroupIDY ) ; if ( Info . hasWorkGroupIDZ ( ) ) allocateSGPR32Input


<s> bool <ISA_LIT>TargetLowering :: getAddrModeArguments ( IntrinsicInst * II , SmallVectorImpl < Value * > & Ops , Type * & AccessTy ) const { switch ( II -> getIntrinsicID ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case


<s> return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , ApertureReg , ShiftAmount ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; if ( UserSGPR == <STR_LIT> ) { return DAG . getUNDEF ( <STR_LIT> ) ; } SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; SDValue Ptr = DAG . getObjectPtrOffset ( DL , QueuePtr , TypeSize :: Fixed ( StructOffset ) ) ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; return DAG . getLoad ( <STR_LIT> , DL , QueuePtr . getValue ( <NUM_LIT> ) , Ptr , PtrInfo , commonAlignment ( Align ( <NUM_LIT> ) , StructOffset

) , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;


<s> if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS && ParamTy && ( ParamTy -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS || ParamTy -> getAddressSpace ( ) == <ISA_LIT>AS :: REGION_ADDRESS ) ) { Arg = DAG . getNode ( <STR_LIT> , DL , Arg . getValueType ( ) , Arg , DAG . getValueType ( <STR_LIT> ) ) ; } InVals . push_back ( Arg ) ; continue ; } else if ( ! IsEntryFunc && VA . isMemLoc ( ) ) { SDValue Val = lowerStackParameter ( DAG , VA , DL , Chain , Arg ) ; InVals . push_back ( Val ) ; if ( ! Arg . Flags . isByVal ( ) ) Chains . push_back ( Val . getValue ( <NUM_LIT> ) ) ; continue ; } assert ( VA . isRegLoc ( ) && <STR_LIT> ) ; Register Reg = VA . getLocReg ( ) ; const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg , VT ) ; EVT ValVT = VA . getValVT ( ) ; Reg = MF . addLiveIn ( Reg , RC ) ; SDValue Val = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; if ( Arg . Flags . isSRet ( ) ) { unsigned NumBits = <NUM_LIT> - getSubtarget ( ) -> getKnownHighZeroBitsForFrameIndex ( ) ; Val = DAG . getNode ( <STR_LIT> , DL , VT , Val , DAG . getValueType ( EVT :: getIntegerVT ( * DAG . getContext ( ) , NumBits ) ) ) ; } switch ( VA . getLocInfo ( ) ) { case CCValAssign :: Full : break ; case CCValAssign :: BCvt : Val = DAG . getNode ( <STR_LIT> , DL , ValVT , Val ) ; break ; case CCValAssign :: SExt : Val = DAG . getNode ( <STR_LIT> , DL , VT , Val , DAG . getValueType ( ValVT ) ) ; Val = DAG . getNode ( <STR_LIT> , DL , ValVT , Val ) ; break ; case CCValAssign :: ZExt : Val = DAG . getNode ( <STR_LIT> , DL , VT , Val , DAG . getValueType ( ValVT ) ) ; Val = DAG . getNode ( <STR_LIT> , DL , ValVT , Val ) ; break ; case CCValAssign :: AExt : Val = DAG . getNode ( <STR_LIT> , DL , ValVT , Val ) ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } InVals . push_back ( Val ) ; } if ( IsEntryFunc ) { allocateSystemSGPRs ( CCInfo , MF , * Info , CallConv , IsGraphics ) ; } else { CCInfo . AllocateReg ( Info -> getScratchRSrcReg ( ) ) ; if ( ! IsGraphics ) allocateSpecialInputSGPRs ( CCInfo , MF , * TRI ,


<s> default : return <ISA_LIT>TargetLowering :: LowerOperation ( Op , DAG ) ; case <STR_LIT> : return LowerBRCOND ( Op , DAG ) ; case <STR_LIT> : return LowerRETURNADDR ( Op , DAG ) ; case <STR_LIT> : { SDValue Result = LowerLOAD ( Op , DAG ) ; assert ( ( ! Result . getNode ( ) || Result . getNode ( ) -> getNumValues ( ) == <NUM_LIT> ) && <STR_LIT> ) ; return Result ; } case <STR_LIT> : case <STR_LIT> : return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op , DAG ) ; } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID ( Op , DAG ) ; case <STR_LIT> : return lowerADDRSPACECAST ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_SUBVECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerEXTRACT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerVECTOR_SHUFFLE ( Op , DAG ) ; case <STR_LIT> : return lowerBUILD_VECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerFP_ROUND ( Op , DAG ) ; case <STR_LIT> : return lowerTRAP ( Op , DAG ) ; case <STR_LIT> : return lowerDEBUGTRAP ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return splitUnaryVectorOp ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : return lowerFMINNUM_FMAXNUM ( Op , DAG ) ; case <STR_LIT> : return splitTernaryVectorOp ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : return LowerFP_TO_INT ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT>


<s> default : return <ISA_LIT>TargetLowering :: LowerOperation ( Op , DAG ) ; case <STR_LIT> : return LowerBRCOND ( Op , DAG ) ; case <STR_LIT> : return LowerRETURNADDR ( Op , DAG ) ; case <STR_LIT> : { SDValue Result = LowerLOAD ( Op , DAG ) ; assert ( ( ! Result . getNode ( ) || Result . getNode ( ) -> getNumValues ( ) == <NUM_LIT> ) && <STR_LIT> ) ; return Result ; } case <STR_LIT> : case <STR_LIT> : return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op , DAG ) ; } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID ( Op , DAG ) ; case <STR_LIT> : return lowerADDRSPACECAST ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_SUBVECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerEXTRACT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerVECTOR_SHUFFLE ( Op , DAG ) ; case <STR_LIT> : return lowerBUILD_VECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerFP_ROUND ( Op , DAG ) ; case <STR_LIT> : return lowerTRAP ( Op , DAG ) ; case <STR_LIT> : return lowerDEBUGTRAP ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case


<s> return <ISA_LIT>TargetLowering :: LowerReturn ( Chain , CallConv , isVarArg , Outs , OutVals , DL , DAG ) ; } bool IsShader = <STR_LIT> ( CallConv ) ; Info -> setIfReturnsVoid ( Outs . empty ( ) ) ; bool IsWaveEnd = Info -> returnsVoid ( ) && IsShader ; SmallVector < CCValAssign , <NUM_LIT> > RVLocs ; SmallVector < <STR_LIT> , <NUM_LIT> > Splits ; CCState CCInfo ( CallConv , isVarArg , DAG . getMachineFunction ( ) , RVLocs , * DAG . getContext ( ) ) ; CCInfo . AnalyzeReturn ( Outs , CCAssignFnForReturn ( CallConv , isVarArg ) ) ; SDValue Flag ; SmallVector < SDValue , <NUM_LIT> > RetOps ; RetOps . push_back ( Chain ) ; if ( ! Info -> isEntryFunction ( ) ) { const <ISA_LIT>RegisterInfo * TRI = getSubtarget ( ) -> getRegisterInfo ( ) ; SDValue ReturnAddrReg = CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getReturnAddressReg ( MF ) , <STR_LIT> ) ; SDValue ReturnAddrVirtualReg = DAG . getRegister ( MF . getRegInfo ( ) . createVirtualRegister ( CallConv != CallingConv :: <ISA_LIT>_Gfx ? & <STR_LIT> : & <STR_LIT> ) , <STR_LIT> ) ; Chain = DAG . getCopyToReg ( Chain , DL , ReturnAddrVirtualReg , ReturnAddrReg , Flag ) ; Flag = Chain . getValue ( <NUM_LIT> ) ; RetOps . push_back ( ReturnAddrVirtualReg ) ; } for ( unsigned I = <NUM_LIT> , RealRVLocIdx = <NUM_LIT> , E = RVLocs . size ( ) ; I != E ; ++ I , ++ RealRVLocIdx ) { CCValAssign & VA = RVLocs [ I ] ; assert ( VA . isRegLoc ( ) && <STR_LIT> ) ; SDValue Arg = OutVals [ RealRVLocIdx ] ; switch ( VA . getLocInfo ( ) ) { case CCValAssign :: Full : break ; case CCValAssign :: BCvt : Arg = DAG . getNode ( <STR_LIT> , DL , VA . getLocVT ( ) , Arg ) ; break ; case CCValAssign :: SExt : Arg = DAG . getNode ( <STR_LIT> , DL , VA . getLocVT (


<s> SDValue ToReg = DAG . getCopyToReg ( Chain , SL , SGPR01 , QueuePtr , SDValue ( ) ) ; uint64_t TrapID = static_cast < uint64_t > ( GCNSubtarget :: TrapID :: LLVMAMDHSATrap ) ; SDValue Ops [ ] = { ToReg , DAG . getTargetConstant ( TrapID , SL , <STR_LIT> ) , SGPR01


<s> assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ; if ( Info -> getStackPtrOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getStackPtrOffsetReg ( ) ) ; if ( Info -> getScratchRSrcReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info


<s> if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return WidenVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return WidenVectorLoad ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable ( <STR_LIT> ) ; } } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) { if ( Subtarget -> useDS128 ( ) && Load -> getAlignment ( ) >= <NUM_LIT> && MemVT . getStoreSize ( ) == <NUM_LIT> ) return SDValue ( ) ; if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS && NumElements == <NUM_LIT> && MemVT . getStoreSize ( ) == <NUM_LIT> && Load -> getAlignment ( ) < <NUM_LIT> ) { return SplitVectorLoad ( Op , DAG ) ; } } return SDValue (


<s> MachineRegisterInfo & MRI = MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; if ( TII -> isVOP3 ( MI . getOpcode ( ) ) ) { TII -> legalizeOperandsVOP3 ( MRI , MI ) ; if ( const MCOperandInfo * OpInfo = MI . getDesc ( ) . OpInfo ) { unsigned Opc = MI . getOpcode ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget -> getRegisterInfo ( ) ; for ( auto I : { <STR_LIT> ( Opc , <STR_LIT> :: src0 ) , <STR_LIT> ( Opc , <STR_LIT> :: src1 ) } ) { if ( I == - <NUM_LIT> ) break ; MachineOperand & Op = MI . getOperand ( I ) ; if ( ( OpInfo [ I ] . RegClass != llvm :: <STR_LIT> && OpInfo [ I ] . RegClass != llvm :: <STR_LIT> ) || ! TargetRegisterInfo :: isVirtualRegister ( Op . getReg ( ) ) || ! TRI -> isAGPR ( MRI , Op . getReg ( ) ) ) continue ; auto * Src = MRI . getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Src || ! Src -> isCopy ( ) || ! TRI -> isSGPRReg ( MRI , Src -> getOperand ( <NUM_LIT> )


<s> unsigned Reg = Info . addWorkGroupIDX ( ) ; MF . addLiveIn ( Reg , & <STR_LIT> ) ; CCInfo . AllocateReg ( Reg ) ; } if ( Info . hasWorkGroupIDY ( ) ) { unsigned Reg = Info . addWorkGroupIDY ( ) ; MF . addLiveIn ( Reg , & <STR_LIT> ) ; CCInfo . AllocateReg ( Reg ) ; } if ( Info . hasWorkGroupIDZ ( ) ) { unsigned Reg = Info . addWorkGroupIDZ ( ) ; MF . addLiveIn ( Reg , & <STR_LIT> ) ; CCInfo . AllocateReg ( Reg ) ; } if ( Info . hasWorkGroupInfo ( ) ) { unsigned Reg = Info . addWorkGroupInfo ( ) ; MF . addLiveIn ( Reg , & <STR_LIT> ) ; CCInfo . AllocateReg ( Reg ) ; } if ( Info . hasPrivateSegmentWaveByteOffset ( ) ) { unsigned PrivateSegmentWaveByteOffsetReg ; if ( IsShader


<s> return ArgDescriptor :: createStack ( Offset , Mask ) ; } unsigned Reg = ArgVGPRs [ RegIdx ] ; Reg = CCInfo . AllocateReg ( Reg ) ; assert ( Reg != <STR_LIT> ) ; MachineFunction & MF = CCInfo . getMachineFunction ( ) ; MF . addLiveIn ( Reg , & <STR_LIT> ) ; return ArgDescriptor :: createRegister ( Reg ,


<s> unsigned LoFlags = GAFlags ; if ( LoFlags == <ISA_LIT>InstrInfo :: MO_NONE ) LoFlags = <ISA_LIT>InstrInfo :: MO_REL32 ; SDValue PtrLo = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> , Offset + <NUM_LIT> , LoFlags ) ; SDValue PtrHi ; if ( GAFlags == <ISA_LIT>InstrInfo :: MO_NONE ) { PtrHi = DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; } else { PtrHi = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT>

, Offset + <NUM_LIT> , GAFlags + <NUM_LIT> ) ;


<s> if ( LoFlags == <ISA_LIT>InstrInfo :: MO_NONE ) LoFlags = <ISA_LIT>InstrInfo :: MO_REL32 ; SDValue PtrLo = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> , Offset + <NUM_LIT> , LoFlags ) ; SDValue PtrHi ; if ( GAFlags == <ISA_LIT>InstrInfo :: MO_NONE ) { PtrHi = DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; } else


<s> const unsigned EncodedReg = <STR_LIT> :: encodeHwreg ( <STR_LIT> :: ID_TRAPSTS , <STR_LIT> :: OFFSET_MEM_VIOL , <NUM_LIT> ) ; BuildMI ( * LoopBB , LoopBB -> begin ( ) , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( EncodedReg ) ; if ( Src && TargetRegisterInfo :: isPhysicalRegister ( Src -> getReg ( ) ) ) { unsigned Data0 = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * BB , std :: next ( Prev ) , DL , TII -> get ( <STR_LIT> ) , Data0 ) . add ( * Src ) ; BuildMI ( * LoopBB , LoopBB -> begin ( ) , DL , TII -> get ( <STR_LIT> ) , Src -> getReg ( ) ) . addReg ( Data0 ) ; MRI . setSimpleHint ( Data0 , Src -> getReg ( ) ) ; } bundleInstWithWaitcnt ( MI ) ; unsigned Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) , Reg ) . addImm ( EncodedReg ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( LoopBB ) ; return RemainderBB


<s> } if ( ! Subtarget -> hasFlatScrRegister ( ) && Subtarget -> getRegisterInfo ( ) -> regsOverlap ( Reg , <STR_LIT> ) ) { report_fatal_error ( Twine ( <STR_LIT> + StringRef ( RegName ) + <STR_LIT> ) ) ; } switch ( Reg ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( VT . getSizeInBits ( ) == <NUM_LIT> ) return Reg ; break ; case <STR_LIT> : case <STR_LIT> : if ( VT . getSizeInBits ( ) == <NUM_LIT> ) return Reg ; break ; default : llvm_unreachable (


<s> const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator I ( & MI ) ; const auto * BoolXExecRC = TRI -> getRegClass ( <STR_LIT> ) ; unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned SaveExec = MRI . createVirtualRegister ( BoolXExecRC ) ; unsigned TmpExec = MRI . createVirtualRegister ( BoolXExecRC ) ; unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned MovExecOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , TmpExec ) ; BuildMI ( MBB , I , DL , TII -> get ( MovExecOpc ) , SaveExec ) . addReg ( Exec ) ; MachineBasicBlock * LoopBB ; MachineBasicBlock * RemainderBB


<s> return SplitVectorLoad ( Op , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { if ( ! Op -> isDivergent ( ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && ! Op -> isDivergent ( ) && ! Load -> isVolatile ( ) && isMemOpHasNoClobberedMemOperand ( Load ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return WidenVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return WidenVectorLoad (


<s> } assert ( VT . isVector ( ) && Store -> getValue ( ) . getValueType ( ) . getScalarType ( ) == <STR_LIT> ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , * Store -> getMemOperand ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } unsigned AS = Store -> getAddressSpace ( ) ; if ( Subtarget -> hasLDSMisalignedBug ( ) && AS == <ISA_LIT>AS :: FLAT_ADDRESS && Store -> getAlignment ( ) < VT . getStoreSize ( ) && VT . getSizeInBits ( ) > <NUM_LIT> ) { return SplitVectorStore ( Op , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> || NumElements == <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( )


<s> MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; bool HasStackObjects = MFI . hasStackObjects ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; if ( HasStackObjects ) Info . setHasNonSpillStackObjects ( true ) ; if ( TM . getOptLevel ( ) == CodeGenOpt :: None ) HasStackObjects = true ; bool RequiresStackAccess = HasStackObjects || MFI . hasCalls ( ) ; if ( RequiresStackAccess && ST . isAmdHsaOrMesa ( MF . getFunction ( ) ) ) { unsigned PrivateSegmentBufferReg = Info . getPreloadedReg ( <STR_LIT> ) ; Info . setScratchRSrcReg ( PrivateSegmentBufferReg ) ; } else { unsigned ReservedBufferReg = TRI . reservedPrivateSegmentBufferReg ( MF ) ; Info . setScratchRSrcReg ( ReservedBufferReg ) ; } if ( ST . getFrameLowering ( ) -> hasFP ( MF ) ) { MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; if ( ! MRI . isLiveIn ( <STR_LIT> ) ) { Info . setStackPtrOffsetReg ( <STR_LIT> ) ; } else


<s> return ; } int NoRetAtomicOp = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( NoRetAtomicOp != - <NUM_LIT> ) { if ( ! Node -> hasAnyUseOfValue ( <NUM_LIT> ) ) { MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; MI . RemoveOperand ( <NUM_LIT> ) ; return ; } if ( ( Node -> hasNUsesOfValue ( <NUM_LIT> , <NUM_LIT> ) && Node -> use_begin ( ) -> isMachineOpcode ( ) && Node -> use_begin ( ) -> getMachineOpcode ( ) == <STR_LIT> && ! Node -> use_begin ( ) -> hasAnyUseOfValue ( <NUM_LIT> ) ) ) { Register Def = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; MI . RemoveOperand ( <NUM_LIT> ) ; BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) ,


<s> if ( Info . hasDispatchPtr ( ) ) ArgInfo . DispatchPtr = allocateSGPR64Input ( CCInfo ) ; if ( Info . hasQueuePtr ( ) ) ArgInfo . QueuePtr = allocateSGPR64Input ( CCInfo ) ; if ( Info . hasImplicitArgPtr ( ) ) ArgInfo . ImplicitArgPtr = allocateSGPR64Input ( CCInfo ) ; if ( Info . hasDispatchID ( ) ) ArgInfo . DispatchID = allocateSGPR64Input ( CCInfo ) ; if ( Info . hasWorkGroupIDX ( ) ) ArgInfo . WorkGroupIDX = allocateSGPR32Input ( CCInfo ) ; if ( Info . hasWorkGroupIDY ( ) ) ArgInfo

. WorkGroupIDY = allocateSGPR32Input ( CCInfo ) ;


<s> int NumDataDwords = ( ! IsD16 || ( IsD16 && Unpacked ) ) ? ReqRetNumElts : ( ReqRetNumElts + <NUM_LIT> ) / <NUM_LIT> ; int MaskPopDwords = ( ! IsD16 || ( IsD16 && Unpacked ) ) ? DMaskPop : ( DMaskPop + <NUM_LIT> ) / <NUM_LIT> ; MVT DataDwordVT = NumDataDwords == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ( <STR_LIT> , NumDataDwords ) ; MVT MaskPopVT = MaskPopDwords == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ( <STR_LIT> , MaskPopDwords ) ; SDValue Data ( Result , <NUM_LIT> ) ; SDValue TexFail ; if ( IsTexFail ) { SDValue ZeroIdx = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; if ( MaskPopVT . isVector ( ) ) { Data = DAG . getNode ( <STR_LIT> , DL , MaskPopVT , SDValue ( Result , <NUM_LIT> ) , ZeroIdx ) ; } else { Data = DAG . getNode ( <STR_LIT> , DL , MaskPopVT , SDValue (


<s> unsigned SubReg ; std :: tie ( SubReg , Offset ) = computeIndirectRegAndOffset ( TRI , VecRC , SrcVec -> getReg ( ) , Offset ) ; const bool UseGPRIdxMode = ST . useVGPRIndexMode ( ) ; if ( Idx -> getReg ( ) == <STR_LIT> ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; assert ( Offset == <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: INSERT_SUBREG ) , Dst ) . add ( * SrcVec ) . add ( * Val ) . addImm ( SubReg ) ; MI . eraseFromParent ( ) ; return & MBB ; } const MCInstrDesc & MovRelDesc = TII -> getIndirectRegWritePseudo ( TRI . getRegSizeInBits ( * VecRC ) , <NUM_LIT> , false ) ; if ( setM0ToIndexFromSGPR ( TII , MRI , MI , Offset , UseGPRIdxMode , false ) ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; BuildMI ( MBB , I , DL , MovRelDesc , Dst ) . addReg ( SrcVec -> getReg ( ) ) . add ( * Val ) . addImm


<s> Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const MachineOperand * SrcVec = TII -> getNamedOperand ( MI , <STR_LIT> :: src ) ; const MachineOperand * Idx = TII -> getNamedOperand ( MI , <STR_LIT> :: idx ) ; const MachineOperand * Val = TII -> getNamedOperand ( MI , <STR_LIT> :: val ) ; int Offset = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> getImm ( ) ; const TargetRegisterClass * VecRC = MRI . getRegClass ( SrcVec -> getReg ( ) ) ; assert ( Val -> getReg ( ) ) ; unsigned SubReg ; std :: tie ( SubReg , Offset ) = computeIndirectRegAndOffset ( TRI , VecRC , SrcVec -> getReg ( ) , Offset ) ; const bool UseGPRIdxMode = ST . useVGPRIndexMode ( ) ; if ( Idx -> getReg ( ) == <STR_LIT> ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; assert ( Offset ==


<s> Register SrcReg = TII -> getNamedOperand ( MI , <STR_LIT> :: src ) -> getReg ( ) ; int Offset = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> getImm ( ) ; const TargetRegisterClass * VecRC = MRI . getRegClass ( SrcReg ) ; unsigned SubReg ; std :: tie ( SubReg , Offset ) = computeIndirectRegAndOffset ( TRI , VecRC , SrcReg , Offset ) ; const bool UseGPRIdxMode = ST . useVGPRIndexMode ( ) ; if ( setM0ToIndexFromSGPR ( TII , MRI , MI , Offset , UseGPRIdxMode , true ) ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , <NUM_LIT> , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ; } else { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , <NUM_LIT> , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) ; } MI . eraseFromParent ( ) ; return & MBB ; } const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator I ( & MI ) ; Register PhiReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register InitReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , InitReg ) ; auto InsPt = loadM0FromVGPR ( TII , MBB , MI , InitReg , PhiReg , Offset , UseGPRIdxMode , true ) ; MachineBasicBlock * LoopBB = InsPt -> getParent ( ) ; if ( UseGPRIdxMode ) { BuildMI ( * LoopBB , InsPt , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , <NUM_LIT> , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( * LoopBB , InsPt , DL , TII -> get ( <STR_LIT> ) ) ; } else { BuildMI ( * LoopBB , InsPt , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , <NUM_LIT> , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) ; } MI . eraseFromParent ( )


<s> MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register SrcReg = TII -> getNamedOperand ( MI , <STR_LIT> :: src ) -> getReg ( ) ; int Offset = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> getImm ( ) ; const TargetRegisterClass * VecRC = MRI . getRegClass ( SrcReg ) ; unsigned SubReg ; std :: tie ( SubReg , Offset ) = computeIndirectRegAndOffset ( TRI , VecRC , SrcReg , Offset ) ; const bool UseGPRIdxMode = ST . useVGPRIndexMode ( ) ; if ( setM0ToIndexFromSGPR ( TII , MRI , MI , Offset , UseGPRIdxMode , true ) ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , <NUM_LIT> , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ; } else


<s> if ( Info -> getScratchRSrcReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getScratchRSrcReg ( ) ) ; if ( Info -> getFrameOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getFrameOffsetReg ( ) ) ; Info -> limitOccupancy ( MF ) ; if ( ST . isWave32 ( ) && ! MF . empty ( ) ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; DebugLoc DL ; MachineBasicBlock & MBB = MF . front ( ) ; MachineBasicBlock :: iterator I = MBB . getFirstNonDebugInstr ( ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , <STR_LIT> ) ; for ( auto & MBB : MF ) { for ( auto & MI : MBB ) { TII -> fixImplicitOperands ( MI ) ; } } } TargetLoweringBase :: finalizeLowering ( MF ) ; if ( VGPRReserveforSGPRSpill && ! Info -> VGPRReservedForSGPRSpill && ! Info -> isEntryFunction ( ) && MF . getFrameInfo ( ) . hasStackObjects ( ) ) Info ->


<s> NumElts = <NUM_LIT> ; } else { assert ( Elts . size ( ) <= <NUM_LIT> ) ; Type = <STR_LIT> ; NumElts = <NUM_LIT> ; } SmallVector < SDValue , <NUM_LIT> > VecElts ( NumElts ) ; for ( unsigned i = <NUM_LIT> ; i < Elts . size ( ) ; ++ i ) { SDValue Elt = Elts [ i ] ; if ( Elt . getValueType ( ) != <STR_LIT> ) Elt = DAG . getBitcast ( <STR_LIT> , Elt ) ; VecElts [ i ] = Elt ; } for ( unsigned i = Elts . size ( ) ; i < NumElts ; ++ i ) VecElts [ i ] = DAG . getUNDEF ( <STR_LIT> ) ; if ( NumElts == <NUM_LIT> ) return VecElts [ <NUM_LIT>


<s> if ( ! RC ) return std :: make_pair ( <NUM_LIT> , nullptr ) ; break ; } break ; } if ( RC && ( isTypeLegal ( VT ) || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> ) ) return std :: make_pair ( <NUM_LIT> , RC ) ; } if ( Constraint . size ( ) > <NUM_LIT> ) { if ( Constraint [ <NUM_LIT> ] == 'v' ) { RC = & <STR_LIT> ; } else if ( Constraint [ <NUM_LIT> ] == 's' ) { RC = & <STR_LIT> ; } else if ( Constraint [ <NUM_LIT> ] == 'a' ) { RC = & <STR_LIT> ; } if ( RC ) { uint32_t Idx ; bool Failed = Constraint . substr ( <NUM_LIT> ) .


<s> EVT IntStoreVT = StoreVT . changeTypeToInteger ( ) ; SDValue IntVData = DAG . getNode ( <STR_LIT> , DL , IntStoreVT , VData ) ; EVT EquivStoreVT = EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , NumElements ) ; SDValue ZExt = DAG . getNode ( <STR_LIT> , DL , EquivStoreVT , IntVData ) ; return DAG . UnrollVectorOp ( ZExt . getNode ( ) ) ; } else if ( NumElements == <NUM_LIT> ) { EVT IntStoreVT = EVT :: getIntegerVT ( * DAG . getContext ( ) , StoreVT . getStoreSizeInBits ( ) ) ; SDValue IntVData = DAG . getNode ( <STR_LIT> , DL , IntStoreVT , VData ) ; EVT WidenedStoreVT = EVT :: getVectorVT ( * DAG . getContext ( ) , StoreVT . getVectorElementType ( ) , NumElements + <NUM_LIT> ) ; EVT WidenedIntVT = EVT :: getIntegerVT ( * DAG . getContext ( ) , WidenedStoreVT . getStoreSizeInBits ( ) ) ; SDValue ZExt = DAG . getNode ( <STR_LIT> ,


<s> <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op , DAG ) ; } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID ( Op , DAG ) ; case <STR_LIT> : return lowerADDRSPACECAST ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_SUBVECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerEXTRACT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerVECTOR_SHUFFLE ( Op , DAG ) ; case <STR_LIT> : return lowerBUILD_VECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerFP_ROUND ( Op , DAG ) ; case <STR_LIT> : return lowerTRAP ( Op , DAG ) ; case <STR_LIT> : return lowerDEBUGTRAP ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return splitUnaryVectorOp ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : return lowerFMINNUM_FMAXNUM ( Op , DAG ) ; case <STR_LIT> : return splitTernaryVectorOp ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case


<s> return expandUnalignedStore ( Store , DAG ) ; } unsigned AS = Store -> getAddressSpace ( ) ; if ( Subtarget -> hasLDSMisalignedBug ( ) && AS == <ISA_LIT>AS :: FLAT_ADDRESS && Store -> getAlignment ( ) < VT . getStoreSize ( ) && VT . getSizeInBits ( ) > <NUM_LIT> ) { return SplitVectorStore ( Op , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS && ! Subtarget -> hasMultiDwordFlatScratchAddressing ( ) ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> || NumElements == <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue (


<s> if ( ! allowsMemoryAccessForAlignment ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , * Store -> getMemOperand ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } unsigned AS = Store -> getAddressSpace ( ) ; if ( Subtarget -> hasLDSMisalignedBug ( ) && AS == <ISA_LIT>AS :: FLAT_ADDRESS && Store -> getAlignment ( ) < VT . getStoreSize ( ) && VT . getSizeInBits ( ) > <NUM_LIT> ) { return SplitVectorStore ( Op , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS && ! Subtarget -> hasMultiDwordFlatScratchAddressing ( ) ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return SplitVectorStore ( Op , DAG ) ; return SDValue


<s> auto Offsets = splitBufferOffsets ( Op . getOperand ( <NUM_LIT> ) , DAG ) ; SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , VData , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Offsets . first , Op . getOperand ( <NUM_LIT> ) , Offsets . second , Op .

getOperand ( <NUM_LIT> ) , DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) , } ;


<s> assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; SDValue SGPR01 = DAG . getRegister ( <STR_LIT> , <STR_LIT> ) ; SDValue ToReg = DAG . getCopyToReg ( Chain , SL , SGPR01 , QueuePtr , SDValue ( ) ) ; SDValue Ops [ ] = { ToReg , DAG . getTargetConstant ( GCNSubtarget :: TrapIDLLVMTrap , SL , <STR_LIT> )

, SGPR01 , ToReg . getValue ( <NUM_LIT> ) } ;


<s> if ( ( ( I + <NUM_LIT> ) >= EndIdx ) || ( ( NumGradients / <NUM_LIT> ) % <NUM_LIT> == <NUM_LIT> && ( I == DimIdx + ( NumGradients / <NUM_LIT> ) - <NUM_LIT> || I == DimIdx + NumGradients - <NUM_LIT> ) ) ) { if ( Addr . getValueType ( ) != <STR_LIT> ) Addr = DAG . getBitcast ( <STR_LIT> , Addr ) ; Addr = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Addr ) ; } else { Addr = DAG . getBuildVector ( PackVectorVT , DL , { Addr , Op . getOperand ( I + <NUM_LIT> ) } ) ; I ++ ; } Addr = DAG . getBitcast


<s> SDValue UndefReg = DAG . getRegister ( MRI . createVirtualRegister ( RC ) , VT ) ; SDValue ImpDef = DAG . getCopyToReg ( DAG . getEntryNode ( ) , SDLoc ( Node ) , UndefReg , Src0 , SDValue ( ) ) ; if ( Src0 . isMachineOpcode ( ) && Src0 . getMachineOpcode ( ) == <STR_LIT> ) { if ( Src1 . isMachineOpcode ( ) && Src1 . getMachineOpcode ( ) != <STR_LIT> ) Src0 = Src1 ; else if ( Src2 . isMachineOpcode ( ) && Src2 . getMachineOpcode ( ) != <STR_LIT> ) Src0 = Src2 ; else { assert ( Src1 . getMachineOpcode ( ) == <STR_LIT> ) ; Src0 = UndefReg ; Src1 = UndefReg ; } } else break ; SmallVector < SDValue , <NUM_LIT> > Ops = { Src0 , Src1 , Src2 } ; for ( unsigned I = <NUM_LIT> , N = Node -> getNumOperands ( ) ; I != N ; ++ I ) Ops . push_back (


<s> bool HasStackObjects = MFI . hasStackObjects ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; if ( HasStackObjects ) Info . setHasNonSpillStackObjects ( true ) ; if ( TM . getOptLevel ( ) == CodeGenOpt :: None ) HasStackObjects = true ; bool RequiresStackAccess = HasStackObjects || MFI . hasCalls ( ) ; if ( RequiresStackAccess && ST . isAmdHsaOrMesa ( MF . getFunction ( ) ) ) { Register PrivateSegmentBufferReg = Info . getPreloadedReg ( <STR_LIT> ) ; Info . setScratchRSrcReg (


<s> uint32_t Imm = C -> getZExtValue ( ) ; uint32_t SOffset , ImmOffset ; if ( <STR_LIT> ( Imm , SOffset , ImmOffset , Subtarget , Alignment ) ) { Offsets [ <NUM_LIT> ] = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; Offsets [ <NUM_LIT> ] = DAG . getConstant ( SOffset , DL , <STR_LIT> ) ; Offsets [ <NUM_LIT> ] = DAG . getTargetConstant ( ImmOffset , DL , <STR_LIT> ) ; return SOffset + ImmOffset ; } } if ( DAG . isBaseWithConstantOffset ( CombinedOffset ) ) { SDValue N0 = CombinedOffset . getOperand ( <NUM_LIT> ) ; SDValue N1 = CombinedOffset . getOperand ( <NUM_LIT> ) ; uint32_t SOffset , ImmOffset ; int Offset = cast < ConstantSDNode > ( N1 ) -> getSExtValue ( ) ; if ( Offset >= <NUM_LIT> && <STR_LIT> ( Offset , SOffset , ImmOffset , Subtarget , Alignment ) ) { Offsets [ <NUM_LIT> ] = N0 ; Offsets [ <NUM_LIT> ] = DAG . getConstant ( SOffset , DL , <STR_LIT> ) ; Offsets [ <NUM_LIT> ] = DAG . getTargetConstant ( ImmOffset , DL , <STR_LIT> ) ; return <NUM_LIT> ; } } Offsets [ <NUM_LIT> ] = CombinedOffset ; Offsets [ <NUM_LIT> ] = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; Offsets [ <NUM_LIT> ] = DAG . getTargetConstant ( <NUM_LIT> , DL ,


<s> LoopBB -> addSuccessor ( RemainderBB ) ; RemainderBB -> transferSuccessorsAndUpdatePHIs ( & MBB ) ; RemainderBB -> splice ( RemainderBB -> begin ( ) , & MBB , I , MBB . end ( ) ) ; MBB . addSuccessor ( LoopBB ) ; const MachineOperand * Idx = TII -> getNamedOperand ( MI , <STR_LIT> :: idx ) ; emitLoadM0FromVGPRLoop ( TII , MRI , MBB , * LoopBB , DL , MovRel , * Idx , InitResultReg , DstReg , PhiReg , TmpExec , Offset ) ; MachineBasicBlock :: iterator First = RemainderBB -> begin ( ) ; BuildMI ( * RemainderBB , First , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( SaveExec ) ; MI . eraseFromParent


<s> bool Unsafe = DAG . getTarget ( ) . Options . UnsafeFPMath ; if ( const ConstantFPSDNode * CLHS = dyn_cast < ConstantFPSDNode > ( LHS ) ) { if ( ( Unsafe || ( VT == <STR_LIT> && ! Subtarget -> hasFP32Denormals ( ) ) ) ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { if ( RHS . getOpcode ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , SL , VT ,


<s> } unsigned NumElements = MemVT . getVectorNumElements ( ) ; switch ( AS ) { case <ISA_LIT>AS :: CONSTANT_ADDRESS : if ( isMemOpUniform ( Load ) ) return SDValue ( ) ; LLVM_FALLTHROUGH ; case <ISA_LIT>AS :: GLOBAL_ADDRESS : case <ISA_LIT>AS :: FLAT_ADDRESS : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <ISA_LIT>AS :: PRIVATE_ADDRESS : { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable ( <STR_LIT> ) ; } } case <ISA_LIT>AS :: LOCAL_ADDRESS : { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> ) return SDValue ( ) ; return SplitVectorLoad ( Op , DAG ) ; } default : return SDValue


<s> SDValue BasePtr = Load -> getBasePtr ( ) ; MachineMemOperand * MMO = Load -> getMemOperand ( ) ; SDValue NewLD = DAG . getExtLoad ( <STR_LIT> , DL , <STR_LIT> , Chain , BasePtr , <STR_LIT> , MMO ) ; SDValue Ops [ ] = { DAG . getNode ( <STR_LIT> , DL , MemVT , NewLD ) , NewLD . getValue ( <NUM_LIT> ) } ; return DAG . getMergeValues ( Ops , DL ) ; } if ( ! MemVT . isVector ( ) ) return SDValue ( ) ; assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ; unsigned AS = Load -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , MemVT , AS , Load -> getAlignment ( ) ) ) { SDValue Ops [ <NUM_LIT> ] ; std :: tie ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) = expandUnalignedLoad ( Load , DAG ) ; return DAG . getMergeValues ( Ops , DL ) ; } unsigned NumElements = MemVT . getVectorNumElements ( ) ; switch ( AS ) { case <ISA_LIT>AS :: CONSTANT_ADDRESS : if ( isMemOpUniform ( Load ) ) return SDValue ( ) ; LLVM_FALLTHROUGH ; case <ISA_LIT>AS :: GLOBAL_ADDRESS : case <ISA_LIT>AS :: FLAT_ADDRESS : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <ISA_LIT>AS :: PRIVATE_ADDRESS : { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable ( <STR_LIT> ) ; } } case <ISA_LIT>AS :: LOCAL_ADDRESS : { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> ) return SDValue ( ) ; return SplitVectorLoad ( Op , DAG ) ; } default : return SDValue (