<s> const DataLayout & DL = DAG . getDataLayout ( ) ; Type * Ty = MemVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; unsigned Align = DL . getABITypeAlignment ( Ty ) ; SDValue Ptr = lowerKernArgParameterPtr ( DAG , SL , Chain , Offset ) ; SDValue Load = DAG . getLoad ( MemVT , SL , Chain , Ptr , PtrInfo , Align , MachineMemOperand :: MONonTemporal | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;

SDValue Val = convertArgType ( DAG , VT , MemVT , SL , Load , Signed , Arg ) ;


<s> SDValue Ptr = lowerKernArgParameterPtr ( DAG , SL , Chain , Offset ) ; SDValue Load = DAG . getLoad ( MemVT , SL , Chain , Ptr , PtrInfo , Align , MachineMemOperand :: MONonTemporal | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ; SDValue Val = convertArgType ( DAG , VT , MemVT , SL , Load , Signed , Arg ) ; return DAG . getMergeValues ( { Val , Load . getValue ( <NUM_LIT> ) } , SL ) ;


<s> const DataLayout & DL = DAG . getDataLayout ( ) ; MachineFunction & MF = DAG . getMachineFunction ( ) ; const <ISA_LIT>RegisterInfo * TRI = getSubtarget ( ) -> getRegisterInfo ( ) ; unsigned InputPtrReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR ) ; MachineRegisterInfo & MRI = DAG . getMachineFunction ( ) . getRegInfo ( ) ; MVT PtrVT = getPointerTy ( DL , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ;

SDValue BasePtr = DAG . getCopyFromReg ( Chain , SL , MRI . getLiveInVirtReg ( InputPtrReg ) , PtrVT ) ;


<s> MachineMemOperand * MMO = Load -> getMemOperand ( ) ; EVT RealMemVT = ( MemVT == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; SDValue NewLD = DAG . getExtLoad ( <STR_LIT> , DL , <STR_LIT> , Chain , BasePtr , RealMemVT , MMO ) ; SDValue Ops [ ] = { DAG . getNode ( <STR_LIT> , DL , MemVT , NewLD ) , NewLD . getValue ( <NUM_LIT> ) } ; return DAG . getMergeValues ( Ops , DL ) ; } if ( ! MemVT . isVector ( ) ) return SDValue ( ) ; assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ; unsigned AS = Load -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , MemVT , AS , Load -> getAlignment ( ) ) ) { SDValue Ops [ <NUM_LIT> ] ; std :: tie ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) = expandUnalignedLoad ( Load , DAG ) ; return DAG . getMergeValues ( Ops , DL ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) { if ( isMemOpUniform ( Load ) ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && isMemOpUniform ( Load ) && isMemOpHasNoClobberedMemOperand ( Load ) ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> :

return scalarizeVectorLoad ( Load , DAG ) ;


<s> return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op , DAG ) ; } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID ( Op , DAG ) ; case <STR_LIT> : return lowerADDRSPACECAST ( Op , DAG ) ; case <STR_LIT> :

return lowerINSERT_VECTOR_ELT ( Op , DAG ) ;


<s> return Result ; } case <STR_LIT> : case <STR_LIT> : return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op , DAG ) ; } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID ( Op , DAG ) ; case <STR_LIT> : return lowerADDRSPACECAST ( Op , DAG ) ; case <STR_LIT> :


<s> const <STR_LIT> & Out = Outs [ i ] ; if ( Out . VT . isVector ( ) ) { MVT VT = Out . VT . getVectorElementType ( ) ; <STR_LIT> NewOut = Out ; NewOut . Flags . setSplit ( ) ; NewOut . VT = VT ; unsigned NumElements = Out . ArgVT . getVectorNumElements ( ) ; for ( unsigned j = <NUM_LIT> ; j != NumElements ; ++ j ) { SDValue Elem = DAG . getNode ( <STR_LIT> , DL , VT , OutVals [ i ] , DAG . getConstant ( j , DL , <STR_LIT> ) ) ; SplitVals . push_back ( Elem ) ; Splits . push_back ( NewOut ) ; NewOut . PartOffset += NewOut . VT . getStoreSize ( ) ; } } else { SplitVals . push_back ( OutVals [ i ] ) ; Splits . push_back ( Out ) ; } } SmallVector < CCValAssign , <NUM_LIT> > RVLocs ; CCState CCInfo ( CallConv , isVarArg , DAG . getMachineFunction ( ) , RVLocs , * DAG . getContext ( ) ) ; AnalyzeReturn ( CCInfo , Splits ) ; SDValue Flag ; SmallVector < SDValue , <NUM_LIT> > RetOps ; RetOps . push_back ( Chain ) ; for ( unsigned i = <NUM_LIT> , realRVLocIdx = <NUM_LIT> ; i != RVLocs . size ( ) ; ++ i , ++ realRVLocIdx ) { CCValAssign & VA = RVLocs [ i ] ; assert ( VA . isRegLoc ( ) && <STR_LIT> ) ; SDValue Arg = SplitVals [ realRVLocIdx ] ; switch ( VA . getLocInfo ( ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case CCValAssign :: Full : break ; case CCValAssign :: BCvt : Arg = DAG . getNode ( <STR_LIT> , DL , VA . getLocVT ( ) , Arg ) ;


<s> if ( VT == <STR_LIT> ) { return DAG . getTruncStore ( Store -> getChain ( ) , DL , DAG . getSExtOrTrunc ( Store -> getValue ( ) , DL , <STR_LIT> ) , Store -> getBasePtr ( ) , <STR_LIT> , Store -> getMemOperand ( ) ) ; } assert ( VT . isVector ( ) && Store -> getValue ( ) . getValueType ( ) . getScalarType ( ) == <STR_LIT> ) ; unsigned AS = Store -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , AS , Store -> getAlignment ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; default :


<s> unsigned TrapID = Op . getOpcode ( ) == <STR_LIT> ? <ISA_LIT>Subtarget :: TrapIDLLVMDebugTrap : <ISA_LIT>Subtarget :: TrapIDLLVMTrap ; if ( Subtarget -> getTrapHandlerAbi ( ) == <ISA_LIT>Subtarget :: TrapHandlerAbiHsa && Subtarget -> isTrapHandlerEnabled ( ) ) { <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; SDValue SGPR01 = DAG . getRegister ( <STR_LIT> , <STR_LIT> ) ; SDValue ToReg = DAG . getCopyToReg ( Chain , SL , SGPR01 , QueuePtr , SDValue ( ) ) ;


<s> APInt Demanded = APInt :: getBitsSet ( <NUM_LIT> , <NUM_LIT> * Offset , <NUM_LIT> * Offset + <NUM_LIT> ) ; APInt KnownZero , KnownOne ; TargetLowering :: TargetLoweringOpt TLO ( DAG , ! DCI . isBeforeLegalize ( ) , ! DCI . isBeforeLegalizeOps ( ) ) ; const TargetLowering & TLI = DAG . getTargetLoweringInfo ( ) ; if ( TLI . ShrinkDemandedConstant ( Src , Demanded , TLO ) || TLI . SimplifyDemandedBits ( Src , Demanded , KnownZero , KnownOne , TLO ) ) { DCI . CommitTargetLoweringOpt ( TLO ) ; } return SDValue ( ) ;


<s> return <ISA_LIT>TargetLowering :: PerformDAGCombine ( N , DCI ) ; case <STR_LIT> : return performFAddCombine ( N , DCI ) ; case <STR_LIT> : return performFSubCombine ( N , DCI ) ; case <STR_LIT> : return performSetCCCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( DCI . getDAGCombineLevel ( ) >= AfterLegalizeDAG && getTargetMachine ( ) . getOptLevel ( ) > CodeGenOpt :: None ) return performMinMaxCombine ( N , DCI ) ; break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> : return performXorCombine ( N , DCI ) ; case <STR_LIT> : return performZeroExtendCombine ( N , DCI ) ; case <STR_LIT> : return performClassCombine ( N , DCI ) ; case <STR_LIT> : return performFCanonicalizeCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src = N -> getOperand ( <NUM_LIT> ) ; if ( Src . isUndef ( ) ) return Src ; break ; } case <STR_LIT> : case <STR_LIT> : return performUCharToFloatCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return performCvtF32UByteNCombine ( N , DCI ) ; case <STR_LIT> : return performFMed3Combine ( N , DCI ) ; case <STR_LIT> : return performCvtPkRTZCombine ( N , DCI ) ; case <STR_LIT> : { SelectionDAG & DAG = DCI . DAG ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; if ( VT == <STR_LIT> || VT == <STR_LIT> ) { SDLoc SL ( N ) ; SDValue Src = N -> getOperand ( <NUM_LIT> ) ; EVT EltVT = Src . getValueType ( ) ; if ( EltVT == <STR_LIT> ) Src = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; SDValue Ext = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; return DAG . getNode ( <STR_LIT> , SL , VT , Ext ) ; } break ; } } return <ISA_LIT>TargetLowering :: PerformDAGCombine ( N , DCI ) ;


<s> EVT SVT = VT . getScalarType ( ) ; if ( SVT == <STR_LIT> && ! Subtarget -> hasFP32Denormals ( ) ) return DAG . getConstantFP ( <NUM_LIT> , SDLoc ( N ) , VT ) ; if ( SVT == <STR_LIT> && ! Subtarget -> hasFP64Denormals ( ) ) return DAG . getConstantFP ( <NUM_LIT> , SDLoc ( N ) , VT ) ; if ( SVT == <STR_LIT> && ! Subtarget -> hasFP16Denormals ( ) ) return DAG . getConstantFP ( <NUM_LIT> , SDLoc ( N ) , VT ) ; } if ( C . isNaN ( ) ) { EVT VT = N -> getValueType ( <NUM_LIT> ) ; APFloat CanonicalQNaN = APFloat :: getQNaN ( C . getSemantics ( ) ) ; if ( C . isSignaling ( ) ) {

return DAG . getConstantFP ( CanonicalQNaN , SDLoc ( N ) , VT ) ;


<s> if ( Cmp == APFloat :: cmpGreaterThan ) return SDValue ( ) ; EVT VT = K0 -> getValueType ( <NUM_LIT> ) ; if ( Subtarget -> enableDX10Clamp ( ) ) { if ( K1 -> isExactlyValue ( <NUM_LIT> ) && K0 -> isExactlyValue ( <NUM_LIT> ) ) return DAG . getNode ( <STR_LIT> , SL , VT , Op0 . getOperand ( <NUM_LIT> ) ) ; }

if ( VT == <STR_LIT> || ( VT == <STR_LIT> && ! Subtarget -> hasMed3_16 ( ) ) ) return SDValue ( ) ;


<s> if ( Ptr . getOpcode ( ) == <STR_LIT> && AS != AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { SDValue NewPtr = performSHLPtrCombine ( Ptr . getNode ( ) , AS , DCI ) ; if ( NewPtr ) { SmallVector < SDValue , <NUM_LIT> > NewOps ( N -> op_begin ( ) , N -> op_end ( ) ) ; NewOps [ N -> getOpcode ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ] = NewPtr ; return SDValue ( DAG . UpdateNodeOperands ( N , NewOps ) , <NUM_LIT> ) ;


<s> SelectionDAG & DAG = DCI . DAG ; SDLoc SL ( N ) ; unsigned AS = N -> getAddressSpace ( ) ; if ( Ptr . getOpcode ( ) == <STR_LIT> && AS != AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { SDValue NewPtr = performSHLPtrCombine ( Ptr . getNode ( ) , AS , DCI ) ;


<s> if ( SDValue Med3 = performIntMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 , true ) ) return Med3 ; } if ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> && Op0 . hasOneUse ( ) ) { if ( SDValue Med3 = performIntMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 , false ) ) return Med3 ; } if ( ( ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> ) || ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> ) ) && ( VT == <STR_LIT> || VT == <STR_LIT> || ( VT == <STR_LIT> && Subtarget -> has16BitInsts ( ) ) ) && Op0 . hasOneUse ( ) ) {

if ( SDValue Res = performFPMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 ) ) return Res ;


<s> SDLoc SL ( N ) ; SDValue Cvt = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src0 , Src1 ) ; Results . push_back ( DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Cvt ) ) ; return ; } default : break ; } } case <STR_LIT> : { SDLoc SL ( N ) ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VT ) ; SDValue LHS = DAG . getNode ( <STR_LIT> , SL , NewVT , N -> getOperand ( <NUM_LIT> ) ) ; SDValue RHS = DAG . getNode ( <STR_LIT> , SL , NewVT , N -> getOperand ( <NUM_LIT> ) ) ; EVT SelectVT = NewVT ; if ( NewVT . bitsLT ( <STR_LIT> ) ) { LHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LHS ) ; RHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , RHS ) ; SelectVT = <STR_LIT> ; } SDValue NewSelect = DAG . getNode ( <STR_LIT> , SL , SelectVT , N -> getOperand ( <NUM_LIT> ) , LHS , RHS ) ; if ( NewVT != SelectVT ) NewSelect = DAG . getNode ( <STR_LIT> , SL , NewVT , NewSelect ) ; Results . push_back ( DAG . getNode ( <STR_LIT> , SL , VT , NewSelect ) ) ;


<s> unsigned ReservedOffsetReg = TRI . reservedPrivateSegmentWaveByteOffsetReg ( MF ) ; Info . setScratchRSrcReg ( ReservedBufferReg ) ; Info . setScratchWaveOffsetReg ( ReservedOffsetReg ) ; } } else { unsigned ReservedBufferReg = TRI . reservedPrivateSegmentBufferReg ( MF ) ; Info . setScratchRSrcReg ( ReservedBufferReg ) ; if ( HasStackObjects ) { unsigned ScratchWaveOffsetReg = TRI . getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: PRIVATE_SEGMENT_WAVE_BYTE_OFFSET ) ; Info . setScratchWaveOffsetReg ( ScratchWaveOffsetReg ) ; } else { unsigned ReservedOffsetReg = TRI . reservedPrivateSegmentWaveByteOffsetReg ( MF ) ; Info . setScratchWaveOffsetReg ( ReservedOffsetReg ) ; }


<s> MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( SrcVec -> getReg ( ) , RegState :: Undef , SubReg ) . add ( * Val ) . addReg ( Dst , RegState :: ImplicitDefine ) . addReg ( SrcVec -> getReg ( ) , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ; } else { const MCInstrDesc & MovRelDesc = TII -> get ( getMOVRELDPseudo ( TRI , VecRC ) ) ; BuildMI ( MBB , I , DL , MovRelDesc ) . addReg ( Dst , RegState :: Define ) . addReg ( SrcVec -> getReg ( ) ) . add ( * Val ) . addImm ( SubReg - <STR_LIT> ) ; } MI . eraseFromParent ( ) ; return & MBB ; } if ( Val -> isReg ( ) ) MRI . clearKillFlags ( Val -> getReg ( ) ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { MachineBasicBlock :: iterator I ( & MI ) ; MachineInstr * SetOn = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( VGPRIndexMode :: DST_ENABLE ) ;

SetOn -> getOperand ( <NUM_LIT> ) . setIsUndef ( ) ;


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; if ( Info -> isEntryFunction ( ) ) { reservePrivateMemoryRegs ( getTargetMachine ( ) , MF , * TRI , * Info ) ; } bool NeedSP = ! Info -> isEntryFunction ( ) || MFI . hasVarSizedObjects ( ) || MFI . hasCalls ( ) ; if ( NeedSP ) { unsigned ReservedStackPtrOffsetReg = TRI -> reservedStackPtrOffsetReg ( MF ) ; Info -> setStackPtrOffsetReg ( ReservedStackPtrOffsetReg ) ; assert ( Info -> getStackPtrOffsetReg ( ) != Info -> getFrameOffsetReg ( ) ) ; assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ; MRI . replaceRegWith ( <STR_LIT> , Info -> getStackPtrOffsetReg ( ) ) ; }

MRI . replaceRegWith ( <STR_LIT> , Info -> getScratchRSrcReg ( ) ) ;


<s> unsigned ReservedStackPtrOffsetReg = TRI -> reservedStackPtrOffsetReg ( MF ) ; Info -> setStackPtrOffsetReg ( ReservedStackPtrOffsetReg ) ; assert ( Info -> getStackPtrOffsetReg ( ) != Info -> getFrameOffsetReg ( ) ) ; assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ; MRI . replaceRegWith ( <STR_LIT> , Info -> getStackPtrOffsetReg ( ) ) ; } MRI . replaceRegWith ( <STR_LIT> , Info -> getScratchRSrcReg ( ) ) ;

MRI . replaceRegWith ( <STR_LIT> , Info -> getFrameOffsetReg ( ) ) ;


<s> const TargetOptions & Options = DAG . getTarget ( ) . Options ; if ( ( Options . AllowFPOpFusion == FPOpFusion :: Fast || Options . UnsafeFPMath || ( N0 -> getFlags ( ) . hasUnsafeAlgebra ( ) && N1 -> getFlags ( ) . hasUnsafeAlgebra ( ) ) ) && isFMAFasterThanFMulAndFAdd ( VT ) ) { return <STR_LIT> ;


<s> SDValue <ISA_LIT>TargetLowering :: getImplicitArgPtr ( SelectionDAG & DAG , const SDLoc & SL ) const { auto MFI = DAG . getMachineFunction ( ) . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

uint64_t Offset = getImplicitParameterOffset ( MFI , FIRST_IMPLICIT ) ;


<s> Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = MFI -> getImagePSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable ; Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . ptrVal = MFI -> getImagePSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable ; Info . flags |= MachineMemOperand :: MOVolatile ; return true ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ; Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MODereferenceable ; return true ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . memVT = <STR_LIT> ( CI . getArgOperand ( <NUM_LIT> ) -> getType ( ) ) ; Info . flags = MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable ; return true ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Info . opc = <STR_LIT> ; Info . ptrVal = MFI -> getBufferPSV ( * MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) , CI . getArgOperand ( <NUM_LIT> ) ) ; Info . memVT = <STR_LIT> ( CI . getType ( ) ) ;

Info . flags = MachineMemOperand :: MOLoad | MachineMemOperand :: MOStore | MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOVolatile ;


<s> if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) { if ( AM . BaseOffs % <NUM_LIT> != <NUM_LIT> ) return isLegalMUBUFAddressingMode ( AM ) ; if ( DL . getTypeStoreSize ( Ty ) < <NUM_LIT> ) return isLegalGlobalAddressingMode ( AM ) ; if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SEA_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs / <NUM_LIT> ) ) return false ; } else if ( Subtarget -> getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( ! isUInt < <NUM_LIT> > ( AM . BaseOffs ) ) return false ; } else llvm_unreachable ( <STR_LIT> ) ; if ( AM . Scale == <NUM_LIT> ) return true ;

if ( AM . Scale == <NUM_LIT> && AM . HasBaseReg ) return true ;


<s> BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , TmpExec ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SaveExec ) . addReg ( <STR_LIT> ) ; MachineBasicBlock * LoopBB = MF -> CreateMachineBasicBlock ( ) ; MachineBasicBlock * RemainderBB = MF -> CreateMachineBasicBlock ( ) ; MachineFunction :: iterator MBBI ( MBB ) ; ++ MBBI ; MF -> insert ( MBBI , LoopBB ) ; MF -> insert ( MBBI , RemainderBB ) ; LoopBB -> addSuccessor ( LoopBB ) ; LoopBB -> addSuccessor ( RemainderBB ) ; RemainderBB -> transferSuccessorsAndUpdatePHIs ( & MBB ) ; RemainderBB -> splice ( RemainderBB -> begin ( ) , & MBB , I , MBB . end ( ) ) ; MBB . addSuccessor ( LoopBB ) ; const MachineOperand * Idx = TII -> getNamedOperand ( MI , <STR_LIT> :: idx ) ; auto InsPt = emitLoadM0FromVGPRLoop ( TII , MRI , MBB , * LoopBB , DL , * Idx , InitResultReg , DstReg , PhiReg , TmpExec , Offset , UseGPRIdxMode ) ; MachineBasicBlock :: iterator First = RemainderBB -> begin ( ) ; BuildMI ( * RemainderBB , First , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( SaveExec ) ;


<s> const SDNodeFlags Flags = Op -> getFlags ( ) ; bool Unsafe = DAG . getTarget ( ) . Options . UnsafeFPMath || Flags . hasUnsafeAlgebra ( ) || Flags . hasAllowReciprocal ( ) ; if ( ! Unsafe && VT == <STR_LIT> && Subtarget -> hasFP32Denormals ( ) ) return SDValue ( ) ; if ( const ConstantFPSDNode * CLHS = dyn_cast < ConstantFPSDNode > ( LHS ) ) { if ( Unsafe || VT == <STR_LIT> || VT == <STR_LIT> ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { if ( RHS . getOpcode ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , SL , VT , RHS . getOperand ( <NUM_LIT> ) ) ; return DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) {

SDValue FNegRHS = DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ;


<s> SDLoc SL ( Op ) ; SDValue LHS = Op . getOperand ( <NUM_LIT> ) ; SDValue RHS = Op . getOperand ( <NUM_LIT> ) ; EVT VT = Op . getValueType ( ) ; const SDNodeFlags Flags = Op -> getFlags ( ) ; bool Unsafe = DAG . getTarget ( ) . Options . UnsafeFPMath || Flags . hasUnsafeAlgebra ( ) || Flags . hasAllowReciprocal ( ) ; if ( ! Unsafe && VT == <STR_LIT> && Subtarget -> hasFP32Denormals ( ) ) return SDValue ( ) ; if ( const ConstantFPSDNode * CLHS = dyn_cast < ConstantFPSDNode > ( LHS ) ) { if ( Unsafe || VT == <STR_LIT> || VT == <STR_LIT> ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { if ( RHS . getOpcode ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , SL , VT , RHS . getOperand ( <NUM_LIT> ) ) ; return DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { SDValue FNegRHS = DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; return DAG . getNode ( <STR_LIT> , SL , VT , FNegRHS ) ;


<s> SDLoc DL ( GSD ) ; EVT PtrVT = Op . getValueType ( ) ; if ( shouldEmitFixup ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT ) ; else if ( shouldEmitPCReloc ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT , <ISA_LIT>InstrInfo :: MO_REL32 ) ; SDValue GOTAddr = buildPCRelGlobalAddress ( DAG , GV , DL , <NUM_LIT> , PtrVT , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; Type * Ty = PtrVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; const DataLayout & DataLayout = DAG . getDataLayout ( ) ; unsigned Align = DataLayout . getABITypeAlignment ( PtrTy ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; return DAG . getLoad ( PtrVT , DL , DAG . getEntryNode ( ) , GOTAddr , PtrInfo , Align , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;


<s> Type * Ty = PtrVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; const DataLayout & DataLayout = DAG . getDataLayout ( ) ; unsigned Align = DataLayout . getABITypeAlignment ( PtrTy ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; return DAG . getLoad ( PtrVT , DL , DAG . getEntryNode ( ) , GOTAddr , PtrInfo , Align , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;


<s> SDValue BasePtr = DAG . getCopyFromReg ( Chain , SL , MRI . getLiveInVirtReg ( InputPtrReg -> getRegister ( ) ) , PtrVT ) ; return DAG . getNode ( <STR_LIT> , SL , PtrVT , BasePtr , DAG . getConstant ( Offset , SL , PtrVT ) ) ;


<s> if ( ! MemVT . isVector ( ) ) return SDValue ( ) ; assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ; unsigned AS = Load -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , MemVT , AS , Load -> getAlignment ( ) ) ) { SDValue Ops [ <NUM_LIT> ] ; std :: tie ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) = expandUnalignedLoad ( Load , DAG ) ; return DAG . getMergeValues ( Ops , DL ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS : AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) { if ( isMemOpUniform ( Load ) ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && isMemOpUniform ( Load ) && ! Load -> isVolatile ( ) && isMemOpHasNoClobberedMemOperand ( Load ) ) return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS || AS == AMDGPUA<ISA_LIT> . GLOBAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> :


<s> NewOut . VT = VT ; unsigned NumElements = Out . ArgVT . getVectorNumElements ( ) ; for ( unsigned j = <NUM_LIT> ; j != NumElements ; ++ j ) { SDValue Elem = DAG . getNode ( <STR_LIT> , DL , VT , OutVals [ i ] , DAG . getConstant ( j , DL , <STR_LIT> ) ) ; SplitVals . push_back ( Elem ) ; Splits . push_back ( NewOut ) ; NewOut . PartOffset += NewOut . VT . getStoreSize ( ) ; } } else { SplitVals . push_back ( OutVals [ i ] ) ; Splits . push_back ( Out ) ; } } SmallVector < CCValAssign , <NUM_LIT> > RVLocs ; CCState CCInfo ( CallConv , isVarArg , DAG . getMachineFunction ( ) , RVLocs , * DAG . getContext ( ) ) ; CCInfo . AnalyzeReturn ( Splits , CCAssignFnForReturn ( CallConv , isVarArg ) ) ; SDValue Flag ; SmallVector < SDValue , <NUM_LIT> > RetOps ; RetOps . push_back ( Chain ) ; if ( ! Info -> isEntryFunction ( ) ) { const <ISA_LIT>RegisterInfo * TRI = getSubtarget ( ) -> getRegisterInfo ( ) ; SDValue ReturnAddrReg = CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getReturnAddressReg ( MF ) , <STR_LIT> ) ; SDValue PhysReturnAddrReg = DAG . getRegister ( TRI -> getReturnAddressReg ( MF ) , <STR_LIT> ) ; Chain = DAG . getCopyToReg ( Chain , DL , PhysReturnAddrReg , ReturnAddrReg , Flag ) ; Flag = Chain . getValue ( <NUM_LIT> ) ; RetOps . push_back ( PhysReturnAddrReg ) ; } for ( unsigned i = <NUM_LIT> , realRVLocIdx = <NUM_LIT> ; i != RVLocs . size ( ) ; ++ i , ++ realRVLocIdx ) { CCValAssign & VA = RVLocs [ i ] ; assert ( VA . isRegLoc ( ) && <STR_LIT> ) ; SDValue Arg = SplitVals [ realRVLocIdx ] ; switch ( VA . getLocInfo ( ) ) { case CCValAssign :: Full : break ; case CCValAssign :: BCvt : Arg = DAG . getNode ( <STR_LIT> , DL , VA . getLocVT ( ) , Arg ) ; break ; case CCValAssign :: SExt : Arg = DAG . getNode ( <STR_LIT> , DL , VA . getLocVT ( ) , Arg ) ; break ; case CCValAssign :: ZExt : Arg = DAG . getNode ( <STR_LIT> , DL , VA . getLocVT ( ) , Arg ) ;


<s> SDValue <ISA_LIT>TargetLowering :: lowerTRAP ( SDValue Op , SelectionDAG & DAG ) const { SDLoc SL ( Op ) ; MachineFunction & MF = DAG . getMachineFunction ( ) ; SDValue Chain = Op . getOperand ( <NUM_LIT> ) ; unsigned TrapID = Op . getOpcode ( ) == <STR_LIT> ? <ISA_LIT>Subtarget :: TrapIDLLVMDebugTrap : <ISA_LIT>Subtarget :: TrapIDLLVMTrap ; if ( Subtarget -> getTrapHandlerAbi ( ) == <ISA_LIT>Subtarget :: TrapHandlerAbiHsa && Subtarget -> isTrapHandlerEnabled ( ) ) { <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; SDValue SGPR01 = DAG . getRegister ( <STR_LIT> , <STR_LIT> ) ; SDValue ToReg = DAG . getCopyToReg ( Chain , SL , SGPR01 , QueuePtr , SDValue ( ) ) ; SDValue Ops [ ] = { ToReg , DAG . getTargetConstant ( TrapID , SL , <STR_LIT> ) , SGPR01 , ToReg . getValue ( <NUM_LIT> ) } ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Ops ) ; } switch ( TrapID ) { case <ISA_LIT>Subtarget :: TrapIDLLVMTrap :

return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Chain ) ;


<s> void <ISA_LIT>TargetLowering :: passSpecialInputs ( CallLoweringInfo & CLI , const <ISA_LIT>MachineFunctionInfo & Info , SmallVectorImpl < std :: pair < unsigned , SDValue >> & RegsToPass , SmallVectorImpl < SDValue > & MemOpChains , SDValue Chain , SDValue StackPtr ) const { if ( ! CLI . CS ) return ; const Function * CalleeFunc = CLI . CS . getCalledFunction ( ) ; assert ( CalleeFunc ) ; SelectionDAG & DAG = CLI . DAG ; const SDLoc & DL = CLI . DL ; const <ISA_LIT>Subtarget * ST = getSubtarget ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST -> getRegisterInfo ( ) ; auto & ArgUsageInfo = DAG . getPass ( ) -> getAnalysis < <ISA_LIT>ArgumentUsageInfo > ( ) ; const <ISA_LIT>FunctionArgInfo & CalleeArgInfo = ArgUsageInfo . lookupFuncArgInfo ( * CalleeFunc ) ; const <ISA_LIT>FunctionArgInfo & CallerArgInfo = Info . getArgInfo ( ) ; <STR_LIT> InputRegs [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; for ( auto InputID : InputRegs ) { const ArgDescriptor * OutgoingArg ; const TargetRegisterClass * ArgRC ; std :: tie ( OutgoingArg , ArgRC ) = CalleeArgInfo . getPreloadedValue ( InputID ) ; if ( ! OutgoingArg ) continue ; const ArgDescriptor * IncomingArg ; const TargetRegisterClass * IncomingArgRC ; std :: tie ( IncomingArg , IncomingArgRC ) = CallerArgInfo . getPreloadedValue ( InputID ) ; assert ( IncomingArgRC == ArgRC ) ; EVT ArgVT = TRI -> getSpillSize ( * ArgRC ) == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; SDValue InputReg ; if ( IncomingArg ) { InputReg = loadInputValue ( DAG , ArgRC , ArgVT , DL , * IncomingArg ) ; } else { assert ( InputID == <STR_LIT> ) ; InputReg = getImplicitArgPtr ( DAG , DL ) ; } if ( OutgoingArg -> isRegister ( ) ) { RegsToPass . emplace_back ( OutgoingArg -> getRegister ( ) , InputReg ) ; } else {

SDValue ArgStore = storeStackInputValue ( DAG , DL , Chain , StackPtr , InputReg , OutgoingArg -> getStackOffset ( ) ) ;


<s> if ( numBitsUnsigned ( MulLHS , DAG ) <= <NUM_LIT> && numBitsUnsigned ( MulRHS , DAG ) <= <NUM_LIT> ) { MulLHS = DAG . getZExtOrTrunc ( MulLHS , SL , <STR_LIT> ) ; MulRHS = DAG . getZExtOrTrunc ( MulRHS , SL , <STR_LIT> ) ; AddRHS = DAG . getZExtOrTrunc ( AddRHS , SL , <STR_LIT> ) ; return getMad64_32 ( DAG , SL , VT , MulLHS , MulRHS , AddRHS , false ) ; } if ( numBitsSigned ( MulLHS , DAG ) < <NUM_LIT> && numBitsSigned ( MulRHS , DAG ) < <NUM_LIT> ) { MulLHS = DAG . getSExtOrTrunc ( MulLHS , SL , <STR_LIT> ) ; MulRHS = DAG . getSExtOrTrunc ( MulRHS , SL , <STR_LIT> ) ; AddRHS = DAG . getSExtOrTrunc ( AddRHS , SL , <STR_LIT> ) ; return getMad64_32 ( DAG , SL , VT , MulLHS , MulRHS , AddRHS , true ) ; } return SDValue ( ) ; } if ( VT != <STR_LIT> ) return SDValue ( ) ; unsigned Opc = LHS . getOpcode ( ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) std :: swap ( RHS , LHS ) ; Opc = RHS . getOpcode ( ) ; switch ( Opc ) { default : break ; case <STR_LIT> :


<s> if ( ( Offset & ( Bits - <NUM_LIT> ) ) == <NUM_LIT> ) { SDLoc SL ( N ) ; SDValue BFE = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LHS -> getOperand ( <NUM_LIT> ) , DAG . getConstant ( Offset , SL , <STR_LIT> ) , DAG . getConstant ( Bits , SL , <STR_LIT> ) ) ; EVT NarrowVT = EVT :: getIntegerVT ( * DAG . getContext ( ) , Bits ) ; SDValue Ext = DAG . getNode ( <STR_LIT> , SL , VT , BFE , DAG . getValueType ( NarrowVT ) ) ; SDValue Shl = DAG . getNode ( <STR_LIT> , SDLoc ( LHS ) , VT , Ext , DAG . getConstant ( NB , SDLoc ( CRHS ) , <STR_LIT> ) ) ; return Shl ; } } } } if ( LHS . getOpcode ( ) == <STR_LIT> && RHS . getOpcode ( ) == <STR_LIT> ) { <STR_LIT> LCC = cast < CondCodeSDNode > ( LHS . getOperand ( <NUM_LIT> ) ) -> get ( ) ; <STR_LIT> RCC = cast < CondCodeSDNode > ( RHS . getOperand ( <NUM_LIT> ) ) -> get ( ) ; SDValue X = LHS . getOperand ( <NUM_LIT> ) ; SDValue Y = RHS . getOperand ( <NUM_LIT> ) ; if ( Y . getOpcode ( ) != <STR_LIT> || Y . getOperand ( <NUM_LIT> ) != X ) return SDValue ( ) ; if ( LCC == <STR_LIT> ) { if ( X != LHS . getOperand ( <NUM_LIT> ) ) return SDValue ( ) ; if ( RCC == <STR_LIT> ) { const ConstantFPSDNode * C1 = dyn_cast < ConstantFPSDNode > ( RHS . getOperand ( <NUM_LIT> ) ) ; if ( ! C1 || ! C1 -> isInfinity ( ) || C1 -> isNegative ( ) ) return SDValue ( ) ; const uint32_t Mask = <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> ; static_assert ( ( ( ~ ( <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> ) ) & <NUM_LIT> ) == Mask , <STR_LIT> ) ; SDLoc DL ( N ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , X , DAG . getConstant ( Mask , DL , <STR_LIT> ) ) ; } } } if ( VT == <STR_LIT> && ( RHS . getOpcode ( ) == <STR_LIT> || LHS . getOpcode ( ) == <STR_LIT> ) ) { if ( RHS . getOpcode ( ) != <STR_LIT> ) std :: swap ( LHS , RHS ) ;

if ( isBoolSGPR ( RHS . getOperand ( <NUM_LIT> ) ) ) return DAG . getSelect ( SDLoc ( N ) , <STR_LIT> , RHS . getOperand ( <NUM_LIT> ) , LHS , DAG . getConstant ( <NUM_LIT> , SDLoc ( N ) , <STR_LIT> ) ) ;


<s> SDValue Vec = N -> getOperand ( <NUM_LIT> ) ; SelectionDAG & DAG = DCI . DAG ; if ( Vec . getOpcode ( ) == <STR_LIT> && allUsesHaveSourceMods ( N ) ) { SDLoc SL ( N ) ;

EVT EltVT = N -> getValueType ( <NUM_LIT> ) ;


<s> SDValue Idx = N -> getOperand ( <NUM_LIT> ) ; SDValue Elt = DAG . getNode ( <STR_LIT> , SL , EltVT , Vec . getOperand ( <NUM_LIT> ) , Idx ) ; return DAG . getNode ( <STR_LIT> , SL , EltVT , Elt ) ; }


<s> SDLoc DL ( N ) ; return DAG . getNode ( minMaxOpcToMin3Max3Opc ( Opc ) , DL , N -> getValueType ( <NUM_LIT> ) , Op0 . getOperand ( <NUM_LIT> ) , Op0 . getOperand ( <NUM_LIT> ) , Op1 ) ; } if ( Op1 . getOpcode ( ) == Opc && Op1 . hasOneUse ( ) ) { SDLoc DL ( N ) ; return DAG . getNode ( minMaxOpcToMin3Max3Opc ( Opc ) , DL , N -> getValueType ( <NUM_LIT> ) , Op0 , Op1 . getOperand ( <NUM_LIT> ) , Op1 . getOperand ( <NUM_LIT> ) ) ; } } if ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> && Op0 . hasOneUse ( ) ) { if ( SDValue Med3 = performIntMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 , true ) ) return Med3 ; } if ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> && Op0 . hasOneUse ( ) ) { if ( SDValue Med3 = performIntMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 , false ) ) return Med3 ; } if ( ( ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> ) || ( Opc == <STR_LIT> && Op0 . getOpcode ( ) == <STR_LIT> ) ) && ( VT == <STR_LIT> || VT == <STR_LIT> || ( VT == <STR_LIT> && Subtarget -> has16BitInsts ( ) ) || ( VT == <STR_LIT> && Subtarget -> hasVOP3PInsts ( ) ) ) && Op0 . hasOneUse ( ) ) { if ( SDValue Res = performFPMed3ImmCombine ( DAG , SDLoc ( N ) , Op0 , Op1 ) ) return Res ; } return SDValue ( ) ;


<s> } if ( CRHS && VT == <STR_LIT> && LHS . getOpcode ( ) == <STR_LIT> && isBoolSGPR ( LHS . getOperand ( <NUM_LIT> ) ) ) { if ( ( CRHS -> isAllOnesValue ( ) && ( CC == <STR_LIT> || CC == <STR_LIT> || CC == <STR_LIT> ) ) || ( CRHS -> isNullValue ( ) && ( CC == <STR_LIT> || CC == <STR_LIT> || CC == <STR_LIT> ) ) ) return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LHS . getOperand ( <NUM_LIT> ) , DAG . getConstant ( - <NUM_LIT> , SL , <STR_LIT> ) ) ; if ( ( CRHS -> isAllOnesValue ( ) && ( CC == <STR_LIT> || CC == <STR_LIT> || CC == <STR_LIT> ) ) || ( CRHS -> isNullValue ( ) && ( CC == <STR_LIT> || CC == <STR_LIT> || CC == <STR_LIT> ) ) ) return LHS . getOperand ( <NUM_LIT> ) ; } if ( VT != <STR_LIT> && VT != <STR_LIT> && ( Subtarget -> has16BitInsts ( ) && VT != <STR_LIT> ) ) return SDValue ( ) ; if ( CC == <STR_LIT> && LHS . getOpcode ( ) == <STR_LIT> ) { const ConstantFPSDNode * CRHS = dyn_cast < ConstantFPSDNode > ( RHS ) ; if ( ! CRHS ) return SDValue ( ) ; const APFloat & APF = CRHS -> getValueAPF ( ) ; if ( APF . isInfinity ( ) && ! APF . isNegative ( ) ) { unsigned Mask = <STR_LIT> | <STR_LIT> ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LHS . getOperand ( <NUM_LIT> ) , DAG . getConstant ( Mask , SL , <STR_LIT> ) ) ; } } return SDValue ( ) ;


<s> SDValue Cvt = DAG . getNode ( Opcode , SL , <STR_LIT> , Src0 , Src1 ) ; Results . push_back ( DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Cvt ) ) ; return ; } } break ; } case <STR_LIT> : { SDLoc SL ( N ) ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; EVT NewVT = getEquivalentMemType ( * DAG . getContext ( ) , VT ) ; SDValue LHS = DAG . getNode ( <STR_LIT> , SL , NewVT , N -> getOperand ( <NUM_LIT> ) ) ; SDValue RHS = DAG . getNode ( <STR_LIT> , SL , NewVT , N -> getOperand ( <NUM_LIT> ) ) ; EVT SelectVT = NewVT ; if ( NewVT . bitsLT ( <STR_LIT> ) ) { LHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LHS ) ; RHS = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , RHS ) ; SelectVT = <STR_LIT> ; } SDValue NewSelect = DAG . getNode ( <STR_LIT> , SL , SelectVT , N -> getOperand ( <NUM_LIT> ) , LHS , RHS ) ; if ( NewVT != SelectVT ) NewSelect = DAG . getNode ( <STR_LIT> , SL , NewVT , NewSelect ) ;

Results . push_back ( DAG . getNode ( <STR_LIT> , SL , VT , NewSelect ) ) ;


<s> Chains . push_back ( Arg . getValue ( <NUM_LIT> ) ) ; auto * ParamTy = dyn_cast < PointerType > ( FType -> getParamType ( Ins [ i ] . getOrigArgIndex ( ) ) ) ; if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS && ParamTy && ParamTy -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS ) { Arg = DAG . getNode ( <STR_LIT> , DL , Arg . getValueType ( ) , Arg , DAG . getValueType ( <STR_LIT> ) ) ; } InVals . push_back ( Arg ) ; Info -> ABIArgOffset = Offset + MemVT . getStoreSize ( ) ; continue ; } assert ( VA . isRegLoc ( ) && <STR_LIT> ) ; unsigned Reg = VA . getLocReg ( ) ; if ( VT == <STR_LIT> ) { Reg = TRI -> getMatchingSuperReg ( Reg , <STR_LIT> , & <STR_LIT> ) ; Reg = MF . addLiveIn ( Reg , & <STR_LIT> ) ; SDValue Copy = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; InVals . push_back ( Copy ) ; continue ; } const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg , VT ) ; Reg = MF . addLiveIn ( Reg , RC ) ; SDValue Val = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; if ( Arg . VT . isVector ( ) ) { Type * ParamType = FType -> getParamType ( Arg . getOrigArgIndex ( ) ) ; unsigned NumElements = ParamType -> getVectorNumElements ( ) ; SmallVector < SDValue , <NUM_LIT> > Regs ; Regs . push_back ( Val ) ; for ( unsigned j = <NUM_LIT> ; j != NumElements ; ++ j ) { Reg = ArgLocs [ ArgIdx ++ ] . getLocReg ( ) ; Reg = MF . addLiveIn ( Reg , RC ) ; SDValue Copy = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; Regs . push_back ( Copy ) ; } NumElements = Arg . VT . getVectorNumElements ( ) - NumElements ; Regs . append ( NumElements , DAG . getUNDEF ( VT ) ) ; InVals . push_back ( DAG . getNode ( <STR_LIT> , DL , Arg . VT , Regs ) ) ; continue ; } InVals . push_back ( Val ) ; } if ( Info -> getShaderType ( ) != ShaderType :: COMPUTE ) { unsigned ScratchIdx = CCInfo . getFirstUnallocated ( ArrayRef < MCPhysReg > ( <STR_LIT> . begin ( ) , <STR_LIT> . getNumRegs ( ) ) ) ; Info -> ScratchOffsetReg = <STR_LIT> . getRegister ( ScratchIdx ) ; }

if ( Chains . empty ( ) ) return Chain ;


<s> MachineOperand & CPol = MI . getOperand ( CPolIdx ) ; CPol . setImm ( CPol . getImm ( ) & ~ <STR_LIT> :: GLC ) ; } MI . RemoveOperand ( <NUM_LIT> ) ; MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; return ; } if ( ( Node -> hasNUsesOfValue ( <NUM_LIT> , <NUM_LIT> ) && Node -> use_begin ( ) -> isMachineOpcode ( ) && Node -> use_begin ( ) -> getMachineOpcode ( ) == <STR_LIT> && ! Node -> use_begin ( ) -> hasAnyUseOfValue ( <NUM_LIT> ) ) ) { Register Def = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; MI . RemoveOperand ( <NUM_LIT> ) ; BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Def ) ; } return ; } if ( TII -> isMIMG ( MI ) && ! MI . mayStore ( ) ) AddIMGInit ( MI ) ;


<s> void <ISA_LIT>TargetLowering :: allocateSpecialInputSGPRs ( CCState & CCInfo , MachineFunction & MF , const <ISA_LIT>RegisterInfo & TRI , <ISA_LIT>MachineFunctionInfo & Info ) const { auto & ArgInfo = Info . getArgInfo ( ) ; if ( Info . hasDispatchPtr ( ) ) allocateSGPR64Input ( CCInfo , ArgInfo . DispatchPtr ) ; if ( Info . hasQueuePtr ( ) ) allocateSGPR64Input ( CCInfo , ArgInfo . QueuePtr ) ; if ( Info . hasImplicitArgPtr ( ) ) allocateSGPR64Input ( CCInfo , ArgInfo . ImplicitArgPtr ) ; if ( Info . hasDispatchID ( ) ) allocateSGPR64Input ( CCInfo , ArgInfo . DispatchID ) ; if ( Info . hasWorkGroupIDX ( ) ) allocateSGPR32Input ( CCInfo , ArgInfo . WorkGroupIDX ) ; if ( Info . hasWorkGroupIDY ( ) ) allocateSGPR32Input ( CCInfo , ArgInfo . WorkGroupIDY ) ;

if ( Info . hasWorkGroupIDZ ( ) ) allocateSGPR32Input ( CCInfo , ArgInfo . WorkGroupIDZ ) ;


<s> bool <ISA_LIT>TargetLowering :: getAddrModeArguments ( IntrinsicInst * II , SmallVectorImpl < Value * > & Ops , Type * & AccessTy ) const { switch ( II -> getIntrinsicID ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , ApertureReg , ShiftAmount ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; if ( UserSGPR == <STR_LIT> ) { return DAG . getUNDEF ( <STR_LIT> ) ; } SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; uint32_t StructOffset = ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) ? <NUM_LIT> : <NUM_LIT> ; SDValue Ptr = DAG . getObjectPtrOffset ( DL , QueuePtr , TypeSize :: Fixed ( StructOffset ) ) ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ;

return DAG . getLoad ( <STR_LIT> , DL , QueuePtr . getValue ( <NUM_LIT> ) , Ptr , PtrInfo , commonAlignment ( Align ( <NUM_LIT> ) , StructOffset ) , MachineMemOperand :: MODereferenceable | MachineMemOperand :: MOInvariant ) ;


<s> if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS && ParamTy && ( ParamTy -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS || ParamTy -> getAddressSpace ( ) == <ISA_LIT>AS :: REGION_ADDRESS ) ) { Arg = DAG . getNode ( <STR_LIT> , DL , Arg . getValueType ( ) , Arg , DAG . getValueType ( <STR_LIT> ) ) ; } InVals . push_back ( Arg ) ; continue ; } else if ( ! IsEntryFunc && VA . isMemLoc ( ) ) { SDValue Val = lowerStackParameter ( DAG , VA , DL , Chain , Arg ) ; InVals . push_back ( Val ) ; if ( ! Arg . Flags . isByVal ( ) ) Chains . push_back ( Val . getValue ( <NUM_LIT> ) ) ; continue ; } assert ( VA . isRegLoc ( ) && <STR_LIT> ) ; Register Reg = VA . getLocReg ( ) ; const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg , VT ) ; EVT ValVT = VA . getValVT ( ) ; Reg = MF . addLiveIn ( Reg , RC ) ; SDValue Val = DAG . getCopyFromReg ( Chain , DL , Reg , VT ) ; if ( Arg . Flags . isSRet ( ) ) { unsigned NumBits = <NUM_LIT> - getSubtarget ( ) -> getKnownHighZeroBitsForFrameIndex ( ) ; Val = DAG . getNode ( <STR_LIT> , DL , VT , Val , DAG . getValueType ( EVT :: getIntegerVT ( * DAG . getContext ( ) , NumBits ) ) ) ; } switch ( VA . getLocInfo ( ) ) { case CCValAssign :: Full : break ; case CCValAssign :: BCvt : Val = DAG . getNode ( <STR_LIT> , DL , ValVT , Val ) ; break ; case CCValAssign :: SExt : Val = DAG . getNode ( <STR_LIT> , DL , VT , Val , DAG . getValueType ( ValVT ) ) ; Val = DAG . getNode ( <STR_LIT> , DL , ValVT , Val ) ; break ; case CCValAssign :: ZExt : Val = DAG . getNode ( <STR_LIT> , DL , VT , Val , DAG . getValueType ( ValVT ) ) ; Val = DAG . getNode ( <STR_LIT> , DL , ValVT , Val ) ; break ; case CCValAssign :: AExt : Val = DAG . getNode ( <STR_LIT> , DL , ValVT , Val ) ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } InVals . push_back ( Val ) ; } if ( IsEntryFunc ) { allocateSystemSGPRs ( CCInfo , MF , * Info , CallConv , IsGraphics ) ; } else { CCInfo . AllocateReg ( Info -> getScratchRSrcReg ( ) ) ;

if ( ! IsGraphics ) allocateSpecialInputSGPRs ( CCInfo , MF , * TRI , * Info ) ;


<s> default : return <ISA_LIT>TargetLowering :: LowerOperation ( Op , DAG ) ; case <STR_LIT> : return LowerBRCOND ( Op , DAG ) ; case <STR_LIT> : return LowerRETURNADDR ( Op , DAG ) ; case <STR_LIT> : { SDValue Result = LowerLOAD ( Op , DAG ) ; assert ( ( ! Result . getNode ( ) || Result . getNode ( ) -> getNumValues ( ) == <NUM_LIT> ) && <STR_LIT> ) ; return Result ; } case <STR_LIT> : case <STR_LIT> : return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op , DAG ) ; } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID ( Op , DAG ) ; case <STR_LIT> : return lowerADDRSPACECAST ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_SUBVECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerEXTRACT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerVECTOR_SHUFFLE ( Op , DAG ) ; case <STR_LIT> : return lowerBUILD_VECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerFP_ROUND ( Op , DAG ) ; case <STR_LIT> : return lowerTRAP ( Op , DAG ) ; case <STR_LIT> : return lowerDEBUGTRAP ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return splitUnaryVectorOp ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : return lowerFMINNUM_FMAXNUM ( Op , DAG ) ; case <STR_LIT> : return splitTernaryVectorOp ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : return LowerFP_TO_INT ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> default : return <ISA_LIT>TargetLowering :: LowerOperation ( Op , DAG ) ; case <STR_LIT> : return LowerBRCOND ( Op , DAG ) ; case <STR_LIT> : return LowerRETURNADDR ( Op , DAG ) ; case <STR_LIT> : { SDValue Result = LowerLOAD ( Op , DAG ) ; assert ( ( ! Result . getNode ( ) || Result . getNode ( ) -> getNumValues ( ) == <NUM_LIT> ) && <STR_LIT> ) ; return Result ; } case <STR_LIT> : case <STR_LIT> : return LowerTrig ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT ( Op , DAG ) ; case <STR_LIT> : return LowerFDIV ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_CMP_SWAP ( Op , DAG ) ; case <STR_LIT> : return LowerSTORE ( Op , DAG ) ; case <STR_LIT> : { MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op , DAG ) ; } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID ( Op , DAG ) ; case <STR_LIT> : return lowerADDRSPACECAST ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_SUBVECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerEXTRACT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerVECTOR_SHUFFLE ( Op , DAG ) ; case <STR_LIT> : return lowerBUILD_VECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerFP_ROUND ( Op , DAG ) ; case <STR_LIT> : return lowerTRAP ( Op , DAG ) ; case <STR_LIT> : return lowerDEBUGTRAP ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> :


<s> return <ISA_LIT>TargetLowering :: LowerReturn ( Chain , CallConv , isVarArg , Outs , OutVals , DL , DAG ) ; } bool IsShader = <STR_LIT> ( CallConv ) ; Info -> setIfReturnsVoid ( Outs . empty ( ) ) ; bool IsWaveEnd = Info -> returnsVoid ( ) && IsShader ; SmallVector < CCValAssign , <NUM_LIT> > RVLocs ; SmallVector < <STR_LIT> , <NUM_LIT> > Splits ; CCState CCInfo ( CallConv , isVarArg , DAG . getMachineFunction ( ) , RVLocs , * DAG . getContext ( ) ) ; CCInfo . AnalyzeReturn ( Outs , CCAssignFnForReturn ( CallConv , isVarArg ) ) ; SDValue Flag ; SmallVector < SDValue , <NUM_LIT> > RetOps ; RetOps . push_back ( Chain ) ; if ( ! Info -> isEntryFunction ( ) ) { const <ISA_LIT>RegisterInfo * TRI = getSubtarget ( ) -> getRegisterInfo ( ) ; SDValue ReturnAddrReg = CreateLiveInRegister ( DAG , & <STR_LIT> , TRI -> getReturnAddressReg ( MF ) , <STR_LIT> ) ; SDValue ReturnAddrVirtualReg = DAG . getRegister ( MF . getRegInfo ( ) . createVirtualRegister ( CallConv != CallingConv :: <ISA_LIT>_Gfx ? & <STR_LIT> : & <STR_LIT> ) , <STR_LIT> ) ; Chain = DAG . getCopyToReg ( Chain , DL , ReturnAddrVirtualReg , ReturnAddrReg , Flag ) ; Flag = Chain . getValue ( <NUM_LIT> ) ; RetOps . push_back ( ReturnAddrVirtualReg ) ; } for ( unsigned I = <NUM_LIT> , RealRVLocIdx = <NUM_LIT> , E = RVLocs . size ( ) ; I != E ; ++ I , ++ RealRVLocIdx ) { CCValAssign & VA = RVLocs [ I ] ; assert ( VA . isRegLoc ( ) && <STR_LIT> ) ; SDValue Arg = OutVals [ RealRVLocIdx ] ; switch ( VA . getLocInfo ( ) ) { case CCValAssign :: Full : break ; case CCValAssign :: BCvt : Arg = DAG . getNode ( <STR_LIT> , DL , VA . getLocVT ( ) , Arg ) ; break ; case CCValAssign :: SExt :


<s> SDValue ToReg = DAG . getCopyToReg ( Chain , SL , SGPR01 , QueuePtr , SDValue ( ) ) ; uint64_t TrapID = static_cast < uint64_t > ( GCNSubtarget :: TrapID :: LLVMAMDHSATrap ) ; SDValue Ops [ ] = {

ToReg , DAG . getTargetConstant ( TrapID , SL , <STR_LIT> ) , SGPR01 , ToReg . getValue ( <NUM_LIT> ) } ;


<s> if ( ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) && Subtarget -> hasAtomicFaddInsts ( ) ) { if ( RMW -> getFunction ( ) -> getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) != <STR_LIT> ) return AtomicExpansionKind :: CmpXChg ; if ( Subtarget -> hasGFX90AInsts ( ) ) { if ( Ty -> isFloatTy ( ) && AS == <ISA_LIT>AS :: FLAT_ADDRESS ) return AtomicExpansionKind :: CmpXChg ; auto S<ISA_LIT>D = RMW -> getSyncScopeID ( ) ; if ( S<ISA_LIT>D == SyncScope :: System || S<ISA_LIT>D == RMW -> getContext ( ) . getOrInsertSyncScopeID ( <STR_LIT> ) ) return AtomicExpansionKind :: CmpXChg ; return ReportUnsafeHWInst ( AtomicExpansionKind :: None ) ; } if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) return AtomicExpansionKind :: CmpXChg ; return RMW -> use_empty ( ) ? ReportUnsafeHWInst ( AtomicExpansionKind :: None ) : AtomicExpansionKind :: CmpXChg ; } if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS && Subtarget -> hasLDSFPAtomicAdd ( ) ) { if ( ! Ty -> isDoubleTy ( ) ) return AtomicExpansionKind :: None ; if ( fpModeMatchesGlobalFPAtomicMode ( RMW ) ) return AtomicExpansionKind :: None ; return RMW -> getFunction ( ) -> getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) == <STR_LIT> ? ReportUnsafeHWInst ( AtomicExpansionKind :: None ) : AtomicExpansionKind :: CmpXChg ; } return AtomicExpansionKind :: CmpXChg ;


<s> assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ; if ( Info -> getStackPtrOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getStackPtrOffsetReg ( ) ) ; if ( Info -> getScratchRSrcReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getScratchRSrcReg ( ) ) ; if ( Info -> getFrameOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getFrameOffsetReg ( ) ) ; if ( Info -> getScratchWaveOffsetReg ( ) != <STR_LIT> ) { MRI . replaceRegWith ( <STR_LIT> , Info -> getScratchWaveOffsetReg ( ) ) ;


<s> assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ; if ( Info -> getStackPtrOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getStackPtrOffsetReg ( ) ) ;

if ( Info -> getScratchRSrcReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getScratchRSrcReg ( ) ) ;


<s> if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return WidenVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return WidenVectorLoad ( Op , DAG ) ; return SDValue ( ) ; default : llvm_unreachable ( <STR_LIT> ) ; } } else if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS ) { if ( Subtarget -> useDS128 ( ) && Load -> getAlignment ( ) >= <NUM_LIT> && MemVT . getStoreSize ( ) == <NUM_LIT> ) return SDValue ( ) ; if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS && NumElements == <NUM_LIT> && MemVT . getStoreSize ( ) == <NUM_LIT> && Load -> getAlignment ( ) < <NUM_LIT> ) { return SplitVectorLoad ( Op , DAG ) ; } }


<s> if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , MemVT , * Load -> getMemOperand ( ) ) ) { SDValue Ops [ <NUM_LIT> ] ; std :: tie ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) = expandUnalignedLoad ( Load , DAG ) ; return DAG . getMergeValues ( Ops , DL ) ; } unsigned Alignment = Load -> getAlignment ( ) ; unsigned AS = Load -> getAddressSpace ( ) ; if ( Subtarget -> hasLDSMisalignedBug ( ) && AS == <ISA_LIT>AS :: FLAT_ADDRESS && Alignment < MemVT . getStoreSize ( ) && MemVT . getSizeInBits ( ) > <NUM_LIT> ) { return SplitVectorLoad ( Op , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { if ( ! Op -> isDivergent ( ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && ! Op -> isDivergent ( ) && ! Load -> isVolatile ( ) && isMemOpHasNoClobberedMemOperand ( Load ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ;


<s> unsigned Opc = MI . getOpcode ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget -> getRegisterInfo ( ) ; for ( auto I : { <STR_LIT> ( Opc , <STR_LIT> :: src0 ) , <STR_LIT> ( Opc , <STR_LIT> :: src1 ) } ) { if ( I == - <NUM_LIT> ) break ; MachineOperand & Op = MI . getOperand ( I ) ; if ( ( OpInfo [ I ] . RegClass != llvm :: <STR_LIT> && OpInfo [ I ] . RegClass != llvm :: <STR_LIT> ) || ! TargetRegisterInfo :: isVirtualRegister ( Op . getReg ( ) ) || ! TRI -> isAGPR ( MRI , Op . getReg ( ) ) ) continue ; auto * Src = MRI . getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Src || ! Src -> isCopy ( ) || ! TRI -> isSGPRReg ( MRI , Src -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; auto * RC = TRI -> getRegClassForReg ( MRI , Op . getReg ( ) ) ; auto * NewRC = TRI -> getEquivalentVGPRClass ( RC ) ; MRI . setRegClass ( Op . getReg ( ) , NewRC ) ; } } return ; } int NoRetAtomicOp = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( NoRetAtomicOp != - <NUM_LIT> ) { if ( ! Node -> hasAnyUseOfValue ( <NUM_LIT> ) ) { MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; MI . RemoveOperand ( <NUM_LIT> ) ;


<s> MachineRegisterInfo & MRI = MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; if ( TII -> isVOP3 ( MI . getOpcode ( ) ) ) { TII -> legalizeOperandsVOP3 ( MRI , MI ) ; if ( const MCOperandInfo * OpInfo = MI . getDesc ( ) . OpInfo ) { unsigned Opc = MI . getOpcode ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget -> getRegisterInfo ( ) ; for ( auto I : { <STR_LIT> ( Opc , <STR_LIT> :: src0 ) , <STR_LIT> ( Opc , <STR_LIT> :: src1 ) } ) { if ( I == - <NUM_LIT> ) break ; MachineOperand & Op = MI . getOperand ( I ) ; if ( ( OpInfo [ I ] . RegClass != llvm :: <STR_LIT> && OpInfo [ I ] . RegClass != llvm :: <STR_LIT> ) || ! TargetRegisterInfo :: isVirtualRegister ( Op . getReg ( ) ) || ! TRI -> isAGPR ( MRI , Op . getReg ( ) ) ) continue ; auto * Src = MRI . getUniqueVRegDef ( Op . getReg ( ) ) ;

if ( ! Src || ! Src -> isCopy ( ) || ! TRI -> isSGPRReg ( MRI , Src -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ;


if ( PrivateSegmentWaveByteOffsetReg == <STR_LIT> ) {


<s> unsigned Reg = Info . addWorkGroupIDX ( ) ; MF . addLiveIn ( Reg , & <STR_LIT> ) ; CCInfo . AllocateReg ( Reg ) ; } if ( Info . hasWorkGroupIDY ( ) ) { unsigned Reg = Info . addWorkGroupIDY ( ) ; MF . addLiveIn ( Reg , & <STR_LIT> ) ; CCInfo . AllocateReg ( Reg ) ; } if ( Info . hasWorkGroupIDZ ( ) ) { unsigned Reg = Info . addWorkGroupIDZ ( ) ; MF . addLiveIn ( Reg , & <STR_LIT> ) ; CCInfo . AllocateReg ( Reg ) ; } if ( Info . hasWorkGroupInfo ( ) ) { unsigned Reg = Info . addWorkGroupInfo ( ) ; MF . addLiveIn ( Reg , & <STR_LIT> ) ; CCInfo . AllocateReg ( Reg ) ; } if ( Info . hasPrivateSegmentWaveByteOffset ( ) ) { unsigned PrivateSegmentWaveByteOffsetReg ;


<s> return ArgDescriptor :: createStack ( Offset , Mask ) ; } unsigned Reg = ArgVGPRs [ RegIdx ] ; Reg = CCInfo . AllocateReg ( Reg ) ; assert ( Reg != <STR_LIT> ) ; MachineFunction & MF = CCInfo . getMachineFunction ( ) ; MF . addLiveIn ( Reg , & <STR_LIT> ) ;

return ArgDescriptor :: createRegister ( Reg , Mask ) ;


<s> unsigned LoFlags = GAFlags ; if ( LoFlags == <ISA_LIT>InstrInfo :: MO_NONE ) LoFlags = <ISA_LIT>InstrInfo :: MO_REL32 ; SDValue PtrLo = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> , Offset + <NUM_LIT> , LoFlags ) ; SDValue PtrHi ; if ( GAFlags == <ISA_LIT>InstrInfo :: MO_NONE ) { PtrHi = DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; } else {

PtrHi = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> , Offset + <NUM_LIT> , GAFlags + <NUM_LIT> ) ;


<s> if ( LoFlags == <ISA_LIT>InstrInfo :: MO_NONE ) LoFlags = <ISA_LIT>InstrInfo :: MO_REL32 ; SDValue PtrLo = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> , Offset + <NUM_LIT> , LoFlags ) ; SDValue PtrHi ; if ( GAFlags == <ISA_LIT>InstrInfo :: MO_NONE ) { PtrHi = DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) ;


<s> const unsigned EncodedReg = <STR_LIT> :: encodeHwreg ( <STR_LIT> :: ID_TRAPSTS , <STR_LIT> :: OFFSET_MEM_VIOL , <NUM_LIT> ) ; BuildMI ( * LoopBB , LoopBB -> begin ( ) , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( EncodedReg ) ; if ( Src && TargetRegisterInfo :: isPhysicalRegister ( Src -> getReg ( ) ) ) { unsigned Data0 = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * BB , std :: next ( Prev ) , DL , TII -> get ( <STR_LIT> ) , Data0 ) . add ( * Src ) ; BuildMI ( * LoopBB , LoopBB -> begin ( ) , DL , TII -> get ( <STR_LIT> ) , Src -> getReg ( ) ) . addReg ( Data0 ) ; MRI . setSimpleHint ( Data0 , Src -> getReg ( ) ) ; } bundleInstWithWaitcnt ( MI ) ; unsigned Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) , Reg ) . addImm ( EncodedReg ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; BuildMI ( * LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( LoopBB ) ;


<s> const TargetRegisterClass * BoolRC = TRI -> getBoolRC ( ) ; unsigned PhiExec = MRI . createVirtualRegister ( BoolRC ) ; unsigned NewExec = MRI . createVirtualRegister ( BoolRC ) ; unsigned CurrentIdxReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned CondReg = MRI . createVirtualRegister ( BoolRC ) ; BuildMI ( LoopBB , I , DL , TII -> get ( TargetOpcode :: PHI ) , PhiReg ) . addReg ( InitReg ) . addMBB ( & OrigBB ) . addReg ( ResultReg ) . addMBB ( & LoopBB ) ; BuildMI ( LoopBB , I , DL , TII -> get ( TargetOpcode :: PHI ) , PhiExec ) . addReg ( InitSaveExecReg ) . addMBB ( & OrigBB ) . addReg ( NewExec ) . addMBB ( & LoopBB ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , CurrentIdxReg ) . addReg ( IdxReg . getReg ( ) , getUndefRegState ( IdxReg . isUndef ( ) ) ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , CondReg ) . addReg ( CurrentIdxReg ) . addReg ( IdxReg . getReg ( ) , <NUM_LIT> , IdxReg . getSubReg ( ) ) ; BuildMI ( LoopBB , I , DL , TII -> get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , NewExec ) . addReg ( CondReg , RegState :: Kill ) ; MRI . setSimpleHint ( NewExec , CondReg ) ; if ( UseGPRIdxMode ) { unsigned IdxReg ; if ( Offset == <NUM_LIT> ) { IdxReg = CurrentIdxReg ; } else { IdxReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , IdxReg ) . addReg ( CurrentIdxReg , RegState :: Kill ) . addImm ( Offset ) ;


<s> } if ( ! Subtarget -> hasFlatScrRegister ( ) && Subtarget -> getRegisterInfo ( ) -> regsOverlap ( Reg , <STR_LIT> ) ) { report_fatal_error ( Twine ( <STR_LIT> + StringRef ( RegName ) + <STR_LIT> ) ) ; } switch ( Reg ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( VT . getSizeInBits ( ) == <NUM_LIT> ) return Reg ; break ; case <STR_LIT> : case <STR_LIT> : if ( VT . getSizeInBits ( ) == <NUM_LIT> ) return Reg ; break ; default :


<s> const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator I ( & MI ) ; const auto * BoolXExecRC = TRI -> getRegClass ( <STR_LIT> ) ; unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned SaveExec = MRI . createVirtualRegister ( BoolXExecRC ) ; unsigned TmpExec = MRI . createVirtualRegister ( BoolXExecRC ) ; unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned MovExecOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , TmpExec ) ; BuildMI ( MBB , I , DL , TII -> get ( MovExecOpc ) , SaveExec ) . addReg ( Exec ) ; MachineBasicBlock * LoopBB ;


<s> return SplitVectorLoad ( Op , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { if ( ! Op -> isDivergent ( ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && ! Op -> isDivergent ( ) && ! Load -> isVolatile ( ) && isMemOpHasNoClobberedMemOperand ( Load ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) { if ( MemVT . isPow2VectorType ( ) ) return SDValue ( ) ; if ( NumElements == <NUM_LIT> ) return WidenVectorLoad ( Op , DAG ) ; return SplitVectorLoad ( Op , DAG ) ; } } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return WidenVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorLoad ( Load , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ;


<s> DAG . getEntryNode ( ) , Rsrc , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , { } , { } , { } , DAG . getConstant ( CachePolicy , DL , <STR_LIT> ) , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , } ; setBufferOffsets ( Offset , DAG , & Ops [ <NUM_LIT> ] , NumLoads > <NUM_LIT> ? <NUM_LIT> * NumLoads : <NUM_LIT> ) ; uint64_t InstOffset = cast < ConstantSDNode > ( Ops [ <NUM_LIT> ] ) -> getZExtValue ( ) ; for ( unsigned i = <NUM_LIT> ; i < NumLoads ; ++ i ) { Ops [ <NUM_LIT> ] = DAG . getConstant ( InstOffset + <NUM_LIT> * i , DL , <STR_LIT> ) ; Loads . push_back ( DAG . getMemIntrinsicNode ( <STR_LIT> , DL , VTList , Ops , LoadVT , MMO ) ) ;


<s> } assert ( VT . isVector ( ) && Store -> getValue ( ) . getValueType ( ) . getScalarType ( ) == <STR_LIT> ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , * Store -> getMemOperand ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } unsigned AS = Store -> getAddressSpace ( ) ; if ( Subtarget -> hasLDSMisalignedBug ( ) && AS == <ISA_LIT>AS :: FLAT_ADDRESS && Store -> getAlignment ( ) < VT . getStoreSize ( ) && VT . getSizeInBits ( ) > <NUM_LIT> ) { return SplitVectorStore ( Op , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> || NumElements == <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ;


<s> MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; bool HasStackObjects = MFI . hasStackObjects ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; if ( HasStackObjects ) Info . setHasNonSpillStackObjects ( true ) ; if ( TM . getOptLevel ( ) == CodeGenOpt :: None ) HasStackObjects = true ; bool RequiresStackAccess = HasStackObjects || MFI . hasCalls ( ) ; if ( RequiresStackAccess && ST . isAmdHsaOrMesa ( MF . getFunction ( ) ) ) { unsigned PrivateSegmentBufferReg = Info . getPreloadedReg ( <STR_LIT> ) ; Info . setScratchRSrcReg ( PrivateSegmentBufferReg ) ; } else { unsigned ReservedBufferReg = TRI . reservedPrivateSegmentBufferReg ( MF ) ; Info . setScratchRSrcReg ( ReservedBufferReg ) ; } if ( ST . getFrameLowering ( ) -> hasFP ( MF ) ) { MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; if ( ! MRI . isLiveIn ( <STR_LIT> ) ) { Info . setStackPtrOffsetReg ( <STR_LIT> ) ;


<s> return ; } int NoRetAtomicOp = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( NoRetAtomicOp != - <NUM_LIT> ) { if ( ! Node -> hasAnyUseOfValue ( <NUM_LIT> ) ) { MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; MI . RemoveOperand ( <NUM_LIT> ) ; return ; } if ( ( Node -> hasNUsesOfValue ( <NUM_LIT> , <NUM_LIT> ) && Node -> use_begin ( ) -> isMachineOpcode ( ) && Node -> use_begin ( ) -> getMachineOpcode ( ) == <STR_LIT> && ! Node -> use_begin ( ) -> hasAnyUseOfValue ( <NUM_LIT> ) ) ) { Register Def = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; MI . RemoveOperand ( <NUM_LIT> ) ;

BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Def ) ;


<s> if ( Info . hasDispatchPtr ( ) ) ArgInfo . DispatchPtr = allocateSGPR64Input ( CCInfo ) ; if ( Info . hasQueuePtr ( ) ) ArgInfo . QueuePtr = allocateSGPR64Input ( CCInfo ) ; if ( Info . hasImplicitArgPtr ( ) ) ArgInfo . ImplicitArgPtr = allocateSGPR64Input ( CCInfo ) ; if ( Info . hasDispatchID ( ) ) ArgInfo . DispatchID = allocateSGPR64Input ( CCInfo ) ; if ( Info . hasWorkGroupIDX ( ) ) ArgInfo . WorkGroupIDX = allocateSGPR32Input ( CCInfo ) ;

if ( Info . hasWorkGroupIDY ( ) ) ArgInfo . WorkGroupIDY = allocateSGPR32Input ( CCInfo ) ;


<s> if ( IsFast ) * IsFast = false ; if ( VT == <STR_LIT> || ( VT != <STR_LIT> && VT . getSizeInBits ( ) > <NUM_LIT> && VT . getStoreSize ( ) > <NUM_LIT> ) ) { return false ;


<s> } if ( Size == <NUM_LIT> ) { bool AlignedBy4 = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ; } if ( Size == <NUM_LIT> ) { bool Aligned = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = Aligned ; return Aligned ; } if ( Size == <NUM_LIT> ) { bool Aligned = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = Aligned ; return Aligned ; } } if ( ! Subtarget -> hasUnalignedScratchAccess ( ) && ( AddrSpace == <ISA_LIT>AS :: PRIVATE_ADDRESS || AddrSpace == <ISA_LIT>AS :: FLAT_ADDRESS ) ) { bool AlignedBy4 = Alignment >= Align ( <NUM_LIT> ) ; if ( IsFast ) * IsFast = AlignedBy4 ; return AlignedBy4 ;


<s> break ; case <STR_LIT> : knownBitsForWorkitemID ( * getSubtarget ( ) , KB , Known , <NUM_LIT> ) ; break ; case <STR_LIT> : knownBitsForWorkitemID ( * getSubtarget ( ) , KB , Known , <NUM_LIT> ) ; break ; case <STR_LIT> : case <STR_LIT> : { unsigned Size = MRI . getType ( R ) . getSizeInBits ( ) ; Known . Zero . setHighBits ( Size - getSubtarget ( ) -> getWavefrontSizeLog2 ( ) ) ; break ; } case <STR_LIT> : { Known . Zero . setHighBits ( countLeadingZeros ( getSubtarget ( ) -> getLocalMemorySize ( ) ) ) ; break ;


<s> int NumDataDwords = ( ! IsD16 || ( IsD16 && Unpacked ) ) ? ReqRetNumElts : ( ReqRetNumElts + <NUM_LIT> ) / <NUM_LIT> ; int MaskPopDwords = ( ! IsD16 || ( IsD16 && Unpacked ) ) ? DMaskPop : ( DMaskPop + <NUM_LIT> ) / <NUM_LIT> ; MVT DataDwordVT = NumDataDwords == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ( <STR_LIT> , NumDataDwords ) ; MVT MaskPopVT = MaskPopDwords == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ( <STR_LIT> , MaskPopDwords ) ; SDValue Data ( Result , <NUM_LIT> ) ; SDValue TexFail ; if ( IsTexFail ) { SDValue ZeroIdx = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; if ( MaskPopVT . isVector ( ) ) { Data = DAG . getNode ( <STR_LIT> , DL , MaskPopVT , SDValue ( Result , <NUM_LIT> ) , ZeroIdx ) ; } else {


<s> unsigned SubReg ; std :: tie ( SubReg , Offset ) = computeIndirectRegAndOffset ( TRI , VecRC , SrcVec -> getReg ( ) , Offset ) ; const bool UseGPRIdxMode = ST . useVGPRIndexMode ( ) ; if ( Idx -> getReg ( ) == <STR_LIT> ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; assert ( Offset == <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: INSERT_SUBREG ) , Dst ) . add ( * SrcVec ) . add ( * Val ) . addImm ( SubReg ) ; MI . eraseFromParent ( ) ; return & MBB ; } const MCInstrDesc & MovRelDesc = TII -> getIndirectRegWritePseudo ( TRI . getRegSizeInBits ( * VecRC ) , <NUM_LIT> , false ) ; if ( setM0ToIndexFromSGPR ( TII , MRI , MI , Offset , UseGPRIdxMode , false ) ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ;

BuildMI ( MBB , I , DL , MovRelDesc , Dst ) . addReg ( SrcVec -> getReg ( ) ) . add ( * Val ) . addImm ( SubReg ) ;


<s> Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const MachineOperand * SrcVec = TII -> getNamedOperand ( MI , <STR_LIT> :: src ) ; const MachineOperand * Idx = TII -> getNamedOperand ( MI , <STR_LIT> :: idx ) ; const MachineOperand * Val = TII -> getNamedOperand ( MI , <STR_LIT> :: val ) ; int Offset = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> getImm ( ) ; const TargetRegisterClass * VecRC = MRI . getRegClass ( SrcVec -> getReg ( ) ) ; assert ( Val -> getReg ( ) ) ; unsigned SubReg ; std :: tie ( SubReg , Offset ) = computeIndirectRegAndOffset ( TRI , VecRC , SrcVec -> getReg ( ) , Offset ) ; const bool UseGPRIdxMode = ST . useVGPRIndexMode ( ) ; if ( Idx -> getReg ( ) == <STR_LIT> ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ;


<s> Register SrcReg = TII -> getNamedOperand ( MI , <STR_LIT> :: src ) -> getReg ( ) ; int Offset = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> getImm ( ) ; const TargetRegisterClass * VecRC = MRI . getRegClass ( SrcReg ) ; unsigned SubReg ; std :: tie ( SubReg , Offset ) = computeIndirectRegAndOffset ( TRI , VecRC , SrcReg , Offset ) ; const bool UseGPRIdxMode = ST . useVGPRIndexMode ( ) ; if ( setM0ToIndexFromSGPR ( TII , MRI , MI , Offset , UseGPRIdxMode , true ) ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , <NUM_LIT> , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ; } else { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , <NUM_LIT> , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) ; } MI . eraseFromParent ( ) ; return & MBB ; } const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator I ( & MI ) ; Register PhiReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register InitReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , InitReg ) ; auto InsPt = loadM0FromVGPR ( TII , MBB , MI , InitReg , PhiReg , Offset , UseGPRIdxMode , true ) ; MachineBasicBlock * LoopBB = InsPt -> getParent ( ) ; if ( UseGPRIdxMode ) { BuildMI ( * LoopBB , InsPt , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , <NUM_LIT> , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( * LoopBB , InsPt , DL , TII -> get ( <STR_LIT> ) ) ; } else { BuildMI ( * LoopBB , InsPt , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , <NUM_LIT> , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) ; }


<s> MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register SrcReg = TII -> getNamedOperand ( MI , <STR_LIT> :: src ) -> getReg ( ) ; int Offset = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> getImm ( ) ; const TargetRegisterClass * VecRC = MRI . getRegClass ( SrcReg ) ; unsigned SubReg ; std :: tie ( SubReg , Offset ) = computeIndirectRegAndOffset ( TRI , VecRC , SrcReg , Offset ) ; const bool UseGPRIdxMode = ST . useVGPRIndexMode ( ) ; if ( setM0ToIndexFromSGPR ( TII , MRI , MI , Offset , UseGPRIdxMode , true ) ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , <NUM_LIT> , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ;


<s> if ( Info -> getScratchRSrcReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getScratchRSrcReg ( ) ) ; if ( Info -> getFrameOffsetReg ( ) != <STR_LIT> ) MRI . replaceRegWith ( <STR_LIT> , Info -> getFrameOffsetReg ( ) ) ; Info -> limitOccupancy ( MF ) ; if ( ST . isWave32 ( ) && ! MF . empty ( ) ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; DebugLoc DL ; MachineBasicBlock & MBB = MF . front ( ) ; MachineBasicBlock :: iterator I = MBB . getFirstNonDebugInstr ( ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , <STR_LIT> ) ; for ( auto & MBB : MF ) { for ( auto & MI : MBB ) { TII -> fixImplicitOperands ( MI ) ; } } } TargetLoweringBase :: finalizeLowering ( MF ) ;

if ( VGPRReserveforSGPRSpill && ! Info -> VGPRReservedForSGPRSpill && ! Info -> isEntryFunction ( ) && MF . getFrameInfo ( ) . hasStackObjects ( ) ) Info -> reserveVGPRforSGPRSpills ( MF ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Value * Ptr = II -> getArgOperand ( <NUM_LIT> ) ; AccessTy = II -> getType ( ) ; Ops . push_back ( Ptr ) ; return true ; }


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Value * Ptr = II -> getArgOperand ( <NUM_LIT> ) ; AccessTy = II -> getType ( ) ; Ops . push_back ( Ptr ) ; return true ; }


<s> NumElts = <NUM_LIT> ; } else { assert ( Elts . size ( ) <= <NUM_LIT> ) ; Type = <STR_LIT> ; NumElts = <NUM_LIT> ; } SmallVector < SDValue , <NUM_LIT> > VecElts ( NumElts ) ; for ( unsigned i = <NUM_LIT> ; i < Elts . size ( ) ; ++ i ) { SDValue Elt = Elts [ i ] ; if ( Elt . getValueType ( ) != <STR_LIT> ) Elt = DAG . getBitcast ( <STR_LIT> , Elt ) ; VecElts [ i ] = Elt ; } for ( unsigned i = Elts . size ( ) ; i < NumElts ; ++ i ) VecElts [ i ] = DAG . getUNDEF ( <STR_LIT> ) ;

if ( NumElts == <NUM_LIT> ) return VecElts [ <NUM_LIT> ] ;


<s> if ( ! RC ) return std :: make_pair ( <NUM_LIT> , nullptr ) ; break ; } break ; } if ( RC && ( isTypeLegal ( VT ) || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> ) ) return std :: make_pair ( <NUM_LIT> , RC ) ; } if ( Constraint . size ( ) > <NUM_LIT> ) { if ( Constraint [ <NUM_LIT> ] == 'v' ) { RC = & <STR_LIT> ; } else if ( Constraint [ <NUM_LIT> ] == 's' ) { RC = & <STR_LIT> ; } else if ( Constraint [ <NUM_LIT> ] == 'a' ) { RC = & <STR_LIT> ; } if ( RC ) { uint32_t Idx ;

bool Failed = Constraint . substr ( <NUM_LIT> ) . getAsInteger ( <NUM_LIT> , Idx ) ;


<s> auto Size = DL . getTypeSizeInBits ( Ty ) ; if ( Size <= <NUM_LIT> ) return Cost ; Cost . first = ( Size + <NUM_LIT> ) / <NUM_LIT> ; return Cost ;


<s> EVT IntStoreVT = StoreVT . changeTypeToInteger ( ) ; SDValue IntVData = DAG . getNode ( <STR_LIT> , DL , IntStoreVT , VData ) ; EVT EquivStoreVT = EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , NumElements ) ; SDValue ZExt = DAG . getNode ( <STR_LIT> , DL , EquivStoreVT , IntVData ) ; return DAG . UnrollVectorOp ( ZExt . getNode ( ) ) ; } else if ( NumElements == <NUM_LIT> ) { EVT IntStoreVT = EVT :: getIntegerVT ( * DAG . getContext ( ) , StoreVT . getStoreSizeInBits ( ) ) ; SDValue IntVData = DAG . getNode ( <STR_LIT> , DL , IntStoreVT , VData ) ; EVT WidenedStoreVT = EVT :: getVectorVT ( * DAG . getContext ( ) , StoreVT . getVectorElementType ( ) , NumElements + <NUM_LIT> ) ; EVT WidenedIntVT = EVT :: getIntegerVT ( * DAG . getContext ( ) , WidenedStoreVT . getStoreSizeInBits ( ) ) ;

SDValue ZExt = DAG . getNode ( <STR_LIT> , DL , WidenedIntVT , IntVData ) ;


<s> case <STR_LIT> : { const RegisterSDNode * R = cast < RegisterSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; const MachineRegisterInfo & MRI = FLI -> MF -> getRegInfo ( ) ; const <ISA_LIT>RegisterInfo * TRI = Subtarget -> getRegisterInfo ( ) ; Register Reg = R -> getReg ( ) ; if ( Reg . isPhysical ( ) || MRI . isLiveIn ( Reg ) ) return ! TRI -> isSGPRReg ( MRI , Reg ) ; if ( const Value * V = FLI -> getValueFromVirtualReg ( R -> getReg ( ) ) ) return KDA -> isDivergent ( V ) ; assert ( Reg == FLI -> DemoteRegister || isCopyFromRegOfInlineAsm ( N ) ) ; return ! TRI -> isSGPRReg ( MRI , Reg ) ; } case <STR_LIT> : { const LoadSDNode * L = cast < LoadSDNode > ( N ) ; unsigned AS = L -> getAddressSpace ( ) ; return AS == <ISA_LIT>AS :: PRIVATE_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ;


<s> const SDNodeFlags Flags = Op -> getFlags ( ) ; bool AllowInaccurateRcp = DAG . getTarget ( ) . Options . UnsafeFPMath || Flags . hasApproximateFuncs ( ) ; if ( ! AllowInaccurateRcp ) return SDValue ( ) ; if ( const ConstantFPSDNode * CLHS = dyn_cast < ConstantFPSDNode > ( LHS ) ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { if ( RHS . getOpcode ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , SL , VT , RHS . getOperand ( <NUM_LIT> ) ) ; return DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { SDValue FNegRHS = DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; return DAG . getNode ( <STR_LIT> , SL , VT , FNegRHS ) ; } } SDValue Recip = DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; return DAG . getNode ( <STR_LIT> , SL , VT , LHS , Recip , Flags ) ;


<s> <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; return LowerGlobalAddress ( MFI , Op , DAG ) ; } case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_W_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINTRIN<ISA_LIT>C_VOID ( Op , DAG ) ; case <STR_LIT> : return lowerADDRSPACECAST ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_SUBVECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerINSERT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerEXTRACT_VECTOR_ELT ( Op , DAG ) ; case <STR_LIT> : return lowerVECTOR_SHUFFLE ( Op , DAG ) ; case <STR_LIT> : return lowerBUILD_VECTOR ( Op , DAG ) ; case <STR_LIT> : return lowerFP_ROUND ( Op , DAG ) ; case <STR_LIT> : return lowerTRAP ( Op , DAG ) ; case <STR_LIT> : return lowerDEBUGTRAP ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return splitUnaryVectorOp ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : return lowerFMINNUM_FMAXNUM ( Op , DAG ) ; case <STR_LIT> : return splitTernaryVectorOp ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> M -> getMemOperand ( ) -> setOffset ( getBufferOffsetForMMO ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) ) ; EVT MemVT = VData . getValueType ( ) ; return DAG . getMemIntrinsicNode ( NewOpcode , DL , Op -> getVTList ( ) , Ops , MemVT , M -> getMemOperand ( ) ) ;


<s> SDValue <ISA_LIT>TargetLowering :: lowerRawBufferAtomicIntrin ( SDValue Op , SelectionDAG & DAG , unsigned NewOpcode ) const { SDLoc DL ( Op ) ; SDValue VData = Op . getOperand ( <NUM_LIT> ) ; auto Offsets = splitBufferOffsets ( Op . getOperand ( <NUM_LIT> ) , DAG ) ; SDValue Ops [ ] = { Op . getOperand ( <NUM_LIT> ) , VData , Op . getOperand ( <NUM_LIT> ) , DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) , Offsets . first , Op . getOperand ( <NUM_LIT> ) , Offsets . second , Op . getOperand ( <NUM_LIT> ) , DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) , } ; auto * M = cast < MemSDNode > ( Op ) ; M -> getMemOperand ( ) -> setOffset ( getBufferOffsetForMMO ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) ) ; EVT MemVT = VData . getValueType ( ) ; return DAG . getMemIntrinsicNode ( NewOpcode , DL , Op -> getVTList ( ) , Ops , MemVT , M -> getMemOperand ( ) ) ;


<s> return expandUnalignedStore ( Store , DAG ) ; } unsigned AS = Store -> getAddressSpace ( ) ; if ( Subtarget -> hasLDSMisalignedBug ( ) && AS == <ISA_LIT>AS :: FLAT_ADDRESS && Store -> getAlignment ( ) < VT . getStoreSize ( ) && VT . getSizeInBits ( ) > <NUM_LIT> ) { return SplitVectorStore ( Op , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS && ! Subtarget -> hasMultiDwordFlatScratchAddressing ( ) ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { switch ( Subtarget -> getMaxPrivateElementSize ( ) ) { case <NUM_LIT> : return scalarizeVectorStore ( Store , DAG ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; case <NUM_LIT> : if ( NumElements > <NUM_LIT> || NumElements == <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ;


<s> if ( ! allowsMemoryAccessForAlignment ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , * Store -> getMemOperand ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } unsigned AS = Store -> getAddressSpace ( ) ; if ( Subtarget -> hasLDSMisalignedBug ( ) && AS == <ISA_LIT>AS :: FLAT_ADDRESS && Store -> getAlignment ( ) < VT . getStoreSize ( ) && VT . getSizeInBits ( ) > <NUM_LIT> ) { return SplitVectorStore ( Op , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS && ! Subtarget -> hasMultiDwordFlatScratchAddressing ( ) ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; if ( NumElements == <NUM_LIT> && ! Subtarget -> hasDwordx3LoadStores ( ) ) return SplitVectorStore ( Op , DAG ) ;


<s> Op . getOperand ( <NUM_LIT> ) , VData , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Offsets . first , Op . getOperand ( <NUM_LIT> ) , Offsets . second , Op . getOperand ( <NUM_LIT> ) , DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) , } ; auto * M = cast < MemSDNode > ( Op ) ; M -> getMemOperand ( ) -> setOffset ( getBufferOffsetForMMO ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) ) ; EVT MemVT = VData . getValueType ( ) ; return DAG . getMemIntrinsicNode ( NewOpcode , DL , Op -> getVTList ( ) , Ops , MemVT , M -> getMemOperand ( ) ) ;


<s> auto Offsets = splitBufferOffsets ( Op . getOperand ( <NUM_LIT> ) , DAG ) ; SDValue Ops [ ] = {

Op . getOperand ( <NUM_LIT> ) , VData , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) , Offsets . first , Op . getOperand ( <NUM_LIT> ) , Offsets . second , Op . getOperand ( <NUM_LIT> ) , DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) , } ;


<s> assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; SDValue SGPR01 = DAG . getRegister ( <STR_LIT> , <STR_LIT> ) ; SDValue ToReg = DAG . getCopyToReg ( Chain , SL , SGPR01 , QueuePtr , SDValue ( ) ) ; SDValue Ops [ ] = {

ToReg , DAG . getTargetConstant ( GCNSubtarget :: TrapIDLLVMTrap , SL , <STR_LIT> ) , SGPR01 , ToReg . getValue ( <NUM_LIT> ) } ;


<s> if ( ( ( I + <NUM_LIT> ) >= EndIdx ) || ( ( NumGradients / <NUM_LIT> ) % <NUM_LIT> == <NUM_LIT> && ( I == DimIdx + ( NumGradients / <NUM_LIT> ) - <NUM_LIT> || I == DimIdx + NumGradients - <NUM_LIT> ) ) ) { if ( Addr . getValueType ( ) != <STR_LIT> ) Addr = DAG . getBitcast ( <STR_LIT> , Addr ) ; Addr = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Addr ) ; } else { Addr = DAG . getBuildVector ( PackVectorVT , DL , { Addr , Op . getOperand ( I + <NUM_LIT> ) } ) ; I ++ ; }


<s> for ( unsigned I = DimIdx ; I < EndIdx ; I ++ ) { SDValue Addr = Op . getOperand ( I ) ; if ( ( ( I + <NUM_LIT> ) >= EndIdx ) || ( ( NumGradients / <NUM_LIT> ) % <NUM_LIT> == <NUM_LIT> && ( I == DimIdx + ( NumGradients / <NUM_LIT> ) - <NUM_LIT> || I == DimIdx + NumGradients - <NUM_LIT> ) ) ) { if ( Addr . getValueType ( ) != <STR_LIT> ) Addr = DAG . getBitcast ( <STR_LIT> , Addr ) ; Addr = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Addr ) ; } else { Addr = DAG . getBuildVector ( PackVectorVT , DL , { Addr , Op . getOperand ( I + <NUM_LIT> ) } ) ; I ++ ; } Addr = DAG . getBitcast ( <STR_LIT> , Addr ) ; PackedAddrs . push_back ( Addr ) ; }


<s> return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , LHS . getOperand ( <NUM_LIT> ) , LHS . getOperand ( <NUM_LIT> ) , DAG . getConstant ( Sel , DL , <STR_LIT> ) ) ; } const <ISA_LIT>InstrInfo * TII = getSubtarget ( ) -> getInstrInfo ( ) ; if ( VT == <STR_LIT> && LHS . hasOneUse ( ) && RHS . hasOneUse ( ) && N -> isDivergent ( ) && TII -> pseudoToMCOpcode ( <STR_LIT> ) != - <NUM_LIT> ) { uint32_t LHSMask = getPermuteMask ( DAG , LHS ) ; uint32_t RHSMask = getPermuteMask ( DAG , RHS ) ; if ( LHSMask != ~ <NUM_LIT> && RHSMask != ~ <NUM_LIT> ) { if ( LHSMask > RHSMask ) { std :: swap ( LHSMask , RHSMask ) ; std :: swap ( LHS , RHS ) ; } uint32_t LHSUsedLanes = ~ ( LHSMask & <NUM_LIT> ) & <NUM_LIT> ; uint32_t RHSUsedLanes = ~ ( RHSMask & <NUM_LIT> ) & <NUM_LIT> ; if ( ! ( LHSUsedLanes & RHSUsedLanes ) && ! ( LHSUsedLanes == <NUM_LIT> && RHSUsedLanes == <NUM_LIT> ) ) { LHSMask &= ~ RHSUsedLanes ; RHSMask &= ~ LHSUsedLanes ; LHSMask |= LHSUsedLanes & <NUM_LIT> ; uint32_t Sel = LHSMask | RHSMask ; SDLoc DL ( N ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , LHS . getOperand ( <NUM_LIT> ) , RHS . getOperand ( <NUM_LIT> ) , DAG . getConstant ( Sel , DL , <STR_LIT> ) ) ; } } } if ( VT != <STR_LIT> || DCI . isBeforeLegalizeOps ( ) ) return SDValue ( ) ; if ( LHS . getOpcode ( ) == <STR_LIT> && RHS . getOpcode ( ) != <STR_LIT> ) std :: swap ( LHS , RHS ) ; if ( RHS . getOpcode ( ) == <STR_LIT> ) { SDValue ExtSrc = RHS . getOperand ( <NUM_LIT> ) ; EVT SrcVT = ExtSrc . getValueType ( ) ; if ( SrcVT == <STR_LIT> ) { SDLoc SL ( N ) ; SDValue LowLHS , HiBits ; std :: tie ( LowLHS , HiBits ) = split64BitValue ( LHS , DAG ) ; SDValue LowOr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LowLHS , ExtSrc ) ; DCI . AddToWorklist ( LowOr . getNode ( ) ) ; DCI . AddToWorklist ( HiBits . getNode ( ) ) ; SDValue Vec = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LowOr , HiBits ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Vec ) ; } } const ConstantSDNode * CRHS = dyn_cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) ; if ( CRHS ) { if ( SDValue Split = splitBinaryBitConstantOp ( DCI , SDLoc ( N ) , <STR_LIT> , LHS , CRHS ) ) return Split ;


<s> SDValue UndefReg = DAG . getRegister ( MRI . createVirtualRegister ( RC ) , VT ) ; SDValue ImpDef = DAG . getCopyToReg ( DAG . getEntryNode ( ) , SDLoc ( Node ) , UndefReg , Src0 , SDValue ( ) ) ; if ( Src0 . isMachineOpcode ( ) && Src0 . getMachineOpcode ( ) == <STR_LIT> ) { if ( Src1 . isMachineOpcode ( ) && Src1 . getMachineOpcode ( ) != <STR_LIT> ) Src0 = Src1 ; else if ( Src2 . isMachineOpcode ( ) && Src2 . getMachineOpcode ( ) != <STR_LIT> ) Src0 = Src2 ; else { assert ( Src1 . getMachineOpcode ( ) == <STR_LIT> ) ; Src0 = UndefReg ; Src1 = UndefReg ; } } else break ; SmallVector < SDValue , <NUM_LIT> > Ops = { Src0 , Src1 , Src2 } ;

for ( unsigned I = <NUM_LIT> , N = Node -> getNumOperands ( ) ; I != N ; ++ I ) Ops . push_back ( Node -> getOperand ( I ) ) ;


<s> legalizeTargetIndependentNode ( Node , DAG ) ; return Node ; } switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : { SDValue Src0 = Node -> getOperand ( <NUM_LIT> ) ; SDValue Src1 = Node -> getOperand ( <NUM_LIT> ) ; SDValue Src2 = Node -> getOperand ( <NUM_LIT> ) ; if ( ( Src0 . isMachineOpcode ( ) && Src0 . getMachineOpcode ( ) != <STR_LIT> ) && ( Src0 == Src1 || Src0 == Src2 ) ) break ; MVT VT = Src0 . getValueType ( ) . getSimpleVT ( ) ; const TargetRegisterClass * RC = getRegClassFor ( VT , Src0 . getNode ( ) -> isDivergent ( ) ) ; MachineRegisterInfo & MRI = DAG . getMachineFunction ( ) . getRegInfo ( ) ; SDValue UndefReg = DAG . getRegister ( MRI . createVirtualRegister ( RC ) , VT ) ; SDValue ImpDef = DAG . getCopyToReg ( DAG . getEntryNode ( ) , SDLoc ( Node ) , UndefReg , Src0 , SDValue ( ) ) ; if ( Src0 . isMachineOpcode ( ) && Src0 . getMachineOpcode ( ) == <STR_LIT> ) { if ( Src1 . isMachineOpcode ( ) && Src1 . getMachineOpcode ( ) != <STR_LIT> ) Src0 = Src1 ; else if ( Src2 . isMachineOpcode ( ) && Src2 . getMachineOpcode ( ) != <STR_LIT> ) Src0 = Src2 ; else { assert ( Src1 . getMachineOpcode ( ) == <STR_LIT> ) ; Src0 = UndefReg ; Src1 = UndefReg ; } } else break ; SmallVector < SDValue , <NUM_LIT> > Ops = { Src0 , Src1 , Src2 } ; for ( unsigned I = <NUM_LIT> , N = Node -> getNumOperands ( ) ; I != N ; ++ I ) Ops . push_back ( Node -> getOperand ( I ) ) ; Ops . push_back ( ImpDef . getValue ( <NUM_LIT> ) ) ; return DAG . getMachineNode ( Opcode , SDLoc ( Node ) , Node -> getVTList ( ) , Ops ) ; } default : break ;


<s> bool HasStackObjects = MFI . hasStackObjects ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; if ( HasStackObjects ) Info . setHasNonSpillStackObjects ( true ) ; if ( TM . getOptLevel ( ) == CodeGenOpt :: None ) HasStackObjects = true ; bool RequiresStackAccess = HasStackObjects || MFI . hasCalls ( ) ; if ( RequiresStackAccess && ST . isAmdHsaOrMesa ( MF . getFunction ( ) ) ) { Register PrivateSegmentBufferReg = Info . getPreloadedReg ( <STR_LIT> ) ;

Info . setScratchRSrcReg ( PrivateSegmentBufferReg ) ;


<s> uint32_t Imm = C -> getZExtValue ( ) ; uint32_t SOffset , ImmOffset ; if ( <STR_LIT> ( Imm , SOffset , ImmOffset , Subtarget , Alignment ) ) { Offsets [ <NUM_LIT> ] = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ; Offsets [ <NUM_LIT> ] = DAG . getConstant ( SOffset , DL , <STR_LIT> ) ; Offsets [ <NUM_LIT> ] = DAG . getTargetConstant ( ImmOffset , DL , <STR_LIT> ) ; return SOffset + ImmOffset ; } } if ( DAG . isBaseWithConstantOffset ( CombinedOffset ) ) { SDValue N0 = CombinedOffset . getOperand ( <NUM_LIT> ) ; SDValue N1 = CombinedOffset . getOperand ( <NUM_LIT> ) ; uint32_t SOffset , ImmOffset ; int Offset = cast < ConstantSDNode > ( N1 ) -> getSExtValue ( ) ; if ( Offset >= <NUM_LIT> && <STR_LIT> ( Offset , SOffset , ImmOffset , Subtarget , Alignment ) ) { Offsets [ <NUM_LIT> ] = N0 ; Offsets [ <NUM_LIT> ] = DAG . getConstant ( SOffset , DL , <STR_LIT> ) ; Offsets [ <NUM_LIT> ] = DAG . getTargetConstant ( ImmOffset , DL , <STR_LIT> ) ; return <NUM_LIT> ; } } Offsets [ <NUM_LIT> ] = CombinedOffset ; Offsets [ <NUM_LIT> ] = DAG . getConstant ( <NUM_LIT> , DL , <STR_LIT> ) ;

Offsets [ <NUM_LIT> ] = DAG . getTargetConstant ( <NUM_LIT> , DL , <STR_LIT> ) ;


<s> switch ( RMW -> getOperation ( ) ) { case AtomicRMWInst :: FAdd : { Type * Ty = RMW -> getType ( ) ; if ( Ty -> isHalfTy ( ) ) return AtomicExpansionKind :: None ; if ( ! Ty -> isFloatTy ( ) ) return AtomicExpansionKind :: CmpXChg ; unsigned AS = RMW -> getPointerAddressSpace ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS && Subtarget -> hasAtomicFaddInsts ( ) ) { if ( ! fpModeMatchesGlobalFPAtomicMode ( RMW ) ) return AtomicExpansionKind :: CmpXChg ; return RMW -> use_empty ( ) ? AtomicExpansionKind :: None : AtomicExpansionKind :: CmpXChg ; } return ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS && Subtarget -> hasLDSFPAtomics ( ) ) ? AtomicExpansionKind :: None : AtomicExpansionKind :: CmpXChg ; } default :


<s> LoopBB -> addSuccessor ( RemainderBB ) ; RemainderBB -> transferSuccessorsAndUpdatePHIs ( & MBB ) ; RemainderBB -> splice ( RemainderBB -> begin ( ) , & MBB , I , MBB . end ( ) ) ; MBB . addSuccessor ( LoopBB ) ; const MachineOperand * Idx = TII -> getNamedOperand ( MI , <STR_LIT> :: idx ) ; emitLoadM0FromVGPRLoop ( TII , MRI , MBB , * LoopBB , DL , MovRel , * Idx , InitResultReg , DstReg , PhiReg , TmpExec , Offset ) ; MachineBasicBlock :: iterator First = RemainderBB -> begin ( ) ; BuildMI ( * RemainderBB , First , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( SaveExec ) ;