<s> SmallDenseSet < unsigned > SGPRsUsed ; Register SGPRReg = findUsedSGPR ( MI , VOP3Idx ) ; if ( SGPRReg != <STR_LIT> ) { SGPRsUsed . insert ( SGPRReg ) ; -- ConstantBusLimit ; } for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { int Idx = VOP3Idx [ i ] ; if ( Idx == - <NUM_LIT> ) break ; MachineOperand & MO = MI . getOperand ( Idx ) ; if ( ! MO . isReg ( ) ) { if ( ! isLiteralConstantLike ( MO , get ( Opc ) . OpInfo [ Idx ] ) ) continue ; if ( LiteralLimit > <NUM_LIT> && ConstantBusLimit > <NUM_LIT> ) { -- LiteralLimit ; -- ConstantBusLimit ; continue ; } -- LiteralLimit ; -- ConstantBusLimit ; legalizeOpWithMove ( MI , Idx ) ; continue ; } if ( RI . hasAGPRs ( MRI . getRegClass ( MO . getReg ( ) ) ) && ! isOperandLegal ( MI , Idx , & MO ) ) { legalizeOpWithMove ( MI , Idx ) ; continue ; } if ( ! RI . isSGPRClass ( MRI . getRegClass ( MO . getReg ( ) ) ) ) continue ; if ( SGPRsUsed . count ( MO . getReg ( ) ) ) continue ; if ( ConstantBusLimit >


<s> void <ISA_LIT>InstrInfo :: legalizeOperandsVOP3 ( MachineRegisterInfo & MRI , MachineInstr & MI ) const { unsigned Opc = MI . getOpcode ( ) ; int VOP3Idx [ <NUM_LIT> ] = { <STR_LIT> ( Opc , <STR_LIT> :: src0 ) , <STR_LIT> ( Opc , <STR_LIT> :: src1 ) , <STR_LIT> ( Opc , <STR_LIT> :: src2 ) } ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { MachineOperand & Src1 = MI . getOperand ( VOP3Idx [ <NUM_LIT> ] ) ; MachineOperand & Src2 = MI . getOperand ( VOP3Idx [ <NUM_LIT> ] ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( Src1 . isReg ( ) && ! RI . isSGPRClass ( MRI . getRegClass ( Src1 . getReg ( ) ) ) ) { Register Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MI . getParent ( ) , MI , DL , get ( <STR_LIT> ) , Reg ) . add ( Src1 ) ; Src1 . ChangeToRegister ( Reg , false ) ; } if ( Src2 . isReg ( ) && ! RI . isSGPRClass ( MRI . getRegClass ( Src2 . getReg ( ) ) ) ) { Register Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MI . getParent ( ) , MI , DL , get ( <STR_LIT> ) , Reg ) . add ( Src2 ) ; Src2 . ChangeToRegister ( Reg , false ) ; } } int ConstantBusLimit = ST . getConstantBusLimit ( Opc ) ; int LiteralLimit = ST . hasVOP3Literal ( ) ? <NUM_LIT> : <NUM_LIT> ; SmallDenseSet < unsigned > SGPRsUsed ; Register SGPRReg = findUsedSGPR ( MI , VOP3Idx ) ; if ( SGPRReg != <STR_LIT>


<s> case <STR_LIT> : if ( isLiteralConstant ( MI . getOperand ( i ) , RI . getRegClass ( RegClass ) -> getSize ( ) ) ) { ErrInfo = <STR_LIT> ; return false ; } break ; case MCOI :: OPERAND_IMMEDIATE : case <STR_LIT> : if ( ! MI . getOperand ( i ) . isImm ( ) && ! MI . getOperand ( i ) . isFI ( ) ) { ErrInfo = <STR_LIT> ; return false ; } default : continue ; } if ( ! MI . getOperand ( i ) . isReg ( ) ) continue ; if ( RegClass != - <NUM_LIT> ) { unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; if ( Reg == <STR_LIT> || TargetRegisterInfo :: isVirtualRegister ( Reg ) ) continue ; const TargetRegisterClass * RC = RI . getRegClass ( RegClass ) ; if ( ! RC -> contains ( Reg ) ) { ErrInfo = <STR_LIT> ; return false ; } } } if ( isVOP1 ( MI ) || isVOP2 ( MI ) || isVOP3 ( MI ) || isVOPC ( MI ) ) { const int OpIndices [ ] = { Src0Idx , Src1Idx , Src2Idx } ; unsigned ConstantBusCount = <NUM_LIT> ; if ( <STR_LIT> ( Opcode , <STR_LIT> :: imm ) != - <NUM_LIT> ) ++ ConstantBusCount ; unsigned SGPRUsed = findImplicitSGPRRead ( MI ) ; if ( SGPRUsed != <STR_LIT> ) ++ ConstantBusCount ; for ( int OpIdx : OpIndices ) { if ( OpIdx == - <NUM_LIT> ) break ; const MachineOperand & MO = MI . getOperand ( OpIdx ) ; if ( usesConstantBus ( MRI , MO , getOpSize ( Opcode , OpIdx ) ) ) { if ( MO . isReg ( ) ) { if ( MO . getReg ( ) != SGPRUsed ) ++ ConstantBusCount ; SGPRUsed = MO . getReg ( ) ; } else { ++ ConstantBusCount ; } } } if ( ConstantBusCount > <NUM_LIT> ) { ErrInfo =


<s> case PseudoSourceValue :: ExternalSymbolCallEntry : case PseudoSourceValue :: TargetCustom : return ST . get<ISA_LIT>AS ( ) . CONSTANT_ADDRESS ; } return ST . get<ISA_LIT>AS


<s> switch ( Kind ) { case PseudoSourceValue :: Stack : case PseudoSourceValue :: FixedStack : return ST . get<ISA_LIT>AS ( ) . PRIVATE_ADDRESS ; case PseudoSourceValue :: ConstantPool : case PseudoSourceValue :: GOT : case PseudoSourceValue :: JumpTable : case PseudoSourceValue :: GlobalValueCallEntry : case PseudoSourceValue :: ExternalSymbolCallEntry : case PseudoSourceValue :: TargetCustom : return ST . get<ISA_LIT>AS ( ) . CONSTANT_ADDRESS ; } return ST . get<ISA_LIT>AS ( )


<s> const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( Register :: isVirtualRegister ( SrcReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; if ( RI . spillSGPRToVGPR ( ) ) FrameInfo .

setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ;


<s> case PseudoSourceValue :: JumpTable : case PseudoSourceValue :: GlobalValueCallEntry : case PseudoSourceValue :: ExternalSymbolCallEntry : case PseudoSourceValue :: TargetCustom : return <ISA_LIT>AS :: CONSTANT_ADDRESS ; case PseudoSourceValue ::


<s> case PseudoSourceValue :: ExternalSymbolCallEntry : case PseudoSourceValue :: TargetCustom : return <ISA_LIT>AS :: CONSTANT_ADDRESS ; case PseudoSourceValue :: CapTable : return <ISA_LIT>AS ::


<s> if ( Register :: isVirtualRegister ( DestReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; return ; } unsigned Opcode = RI . hasAGPRs ( RC ) ? getAGPRSpillRestoreOpcode ( SpillSize ) : getVGPRSpillRestoreOpcode ( SpillSize ) ; auto MIB = BuildMI ( MBB , MI , DL , get


<s> MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; assert ( DestReg != <STR_LIT> && <STR_LIT> ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillRestoreOpcode ( SpillSize ) ) ; if ( Register :: isVirtualRegister ( DestReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex ,


<s> if ( Src -> isImm ( ) ) Offset = Src -> getImm ( ) ; else if ( Src -> isCImm ( ) ) Offset = Src -> getCImm ( ) -> getZExtValue ( ) ; } if ( Offset && isLegalMUBUFImmOffset ( Offset ) ) VAddr = getNamedOperand ( * Add , <STR_LIT> :: src1 ) ; else Offset = <NUM_LIT> ; } BuildMI ( * MBB , Inst , Inst . getDebugLoc ( ) , get ( <STR_LIT> ) , VDst ) . add ( * VAddr ) . add ( * getNamedOperand ( Inst , <STR_LIT> :: sbase ) ) . addImm ( <NUM_LIT> ) . addImm ( Offset ) . addImm ( getNamedOperand ( Inst , <STR_LIT> :: glc ) -> getImm ( ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . setMemRefs ( Inst . memoperands_begin ( ) , Inst . memoperands_end ( ) ) ; MRI . replaceRegWith ( getNamedOperand ( Inst , <STR_LIT> :: sdst ) -> getReg ( ) , VDst ) ; addUsersToMoveToVALUWorklist ( VDst , MRI , Worklist ) ; Inst . eraseFromParent ( ) ; continue ; } } if ( NewOpcode == <STR_LIT> ) { legalizeOperands ( Inst ) ; continue ; } const MCInstrDesc & NewDesc = get ( NewOpcode ) ; Inst . setDesc ( NewDesc ) ; for ( unsigned i = Inst . getNumOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; -- i ) { MachineOperand & Op = Inst . getOperand ( i ) ; if ( Op . isReg ( ) && Op . getReg ( ) == <STR_LIT> ) { Inst . RemoveOperand ( i ) ; addSCCDefUsersToVALUWorklist ( Inst , Worklist ) ; } } if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { unsigned Size = ( Opcode == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; Inst . addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; Inst . addOperand ( MachineOperand :: CreateImm ( Size ) ) ; } else if ( Opcode == <STR_LIT> ) { Inst . addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; } Inst . addImplicitDefUseOperands ( * Inst . getParent ( ) -> getParent ( ) ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { const MachineOperand & OffsetWidthOp = Inst . getOperand ( <NUM_LIT> ) ; assert ( OffsetWidthOp . isImm ( ) && <STR_LIT> ) ; uint32_t Imm = OffsetWidthOp . getImm ( ) ; uint32_t Offset = Imm & <NUM_LIT> ; uint32_t BitWidth = ( Imm & <NUM_LIT> ) >> <NUM_LIT> ; Inst . RemoveOperand ( <NUM_LIT> ) ; Inst . addOperand ( MachineOperand :: CreateImm ( Offset ) ) ; Inst . addOperand ( MachineOperand :: CreateImm ( BitWidth ) ) ; } bool HasDst = Inst . getOperand ( <NUM_LIT> ) . isReg ( ) && Inst . getOperand ( <NUM_LIT>


<s> return ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_3_64 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_7_64 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_15_64 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; return ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_1 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_2 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_3 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_7 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_15 ; } else { llvm_unreachable ( <STR_LIT> ) ; } if ( RI . getHWRegIndex ( DestReg ) <= RI . getHWRegIndex ( SrcReg ) ) Forward = true ; else Forward = false ; for ( unsigned Idx = <NUM_LIT> ; Idx < SubIndices . size ( ) ; ++ Idx ) { unsigned SubIdx ; if ( Forward ) SubIdx = SubIndices [ Idx ] ; else SubIdx = SubIndices [ SubIndices . size ( ) - Idx - <NUM_LIT>


<s> <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineFrameInfo * FrameInfo = MF -> getFrameInfo ( ) ; DebugLoc DL = MBB . findDebugLoc ( MI ) ; unsigned Size = FrameInfo -> getObjectSize ( FrameIndex ) ; unsigned Align = FrameInfo -> getObjectAlignment ( FrameIndex ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SrcReg ) && RC -> getSize ( ) == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } unsigned Opcode = getSGPRSpillSaveOpcode ( RC -> getSize ( ) ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) ) . addReg ( SrcReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) ; return ; } if ( ! ST . isVGPRSpillingEnabled ( * MF -> getFunction ( ) ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg ) ; return ; } assert ( RI . hasVGPRs ( RC ) && <STR_LIT> ) ; unsigned Opcode = getVGPRSpillSaveOpcode ( RC -> getSize ( ) ) ; MFI -> setHasSpilledVGPRs (


<s> if ( ST . hasAddNoCarry ( ) ) return BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DestReg ) ; Register UnusedCarry = RS . scavengeRegister ( RI . getBoolRC ( ) , I , <NUM_LIT> ,


<s> if ( ! UnusedCarry . isValid ( ) ) return MachineInstrBuilder ( ) ; return BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( UnusedCarry , RegState


<s> return ; } if ( ! ST . isVGPRSpillingEnabled ( MFI ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg ) ; return ; } assert ( RI . hasVGPRs ( RC ) && <STR_LIT> ) ; unsigned Opcode = getVGPRSpillSaveOpcode ( RC -> getSize ( ) ) ; MFI -> setHasSpilledVGPRs ( ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) ) . addReg ( SrcReg ) . addFrameIndex ( FrameIndex ) . addReg ( <STR_LIT> , RegState

:: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addMemOperand ( MMO ) ;


<s> if ( MI . mayStore ( ) && isSMRD ( MI ) ) return true ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return true ; if ( MI . isCall ( ) || MI . isInlineAsm ( ) ) return true ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return true ; return false


<s> unsigned OpSubReg = Op . getSubReg ( ) ; const TargetRegisterClass * OpRC = RI . getSubClassWithSubReg ( RI . getRegClassForReg ( MRI , OpReg ) , OpSubReg ) ; if ( DstRC == OpRC ) return ; unsigned DstReg = MRI . createVirtualRegister ( DstRC ) ; MachineInstr * Copy = BuildMI ( InsertMBB , I , DL , get ( <STR_LIT> ) , DstReg ) . add ( Op ) ; Op . setReg ( DstReg ) ; Op . setSubReg ( <NUM_LIT> ) ; MachineInstr * Def = MRI . getVRegDef (


<s> unsigned OpSubReg = Op . getSubReg ( ) ; const TargetRegisterClass * OpRC = RI . getSubClassWithSubReg ( RI . getRegClassForReg ( MRI , OpReg ) , OpSubReg ) ; if ( DstRC == OpRC ) return ; unsigned DstReg = MRI . createVirtualRegister ( DstRC ) ; MachineInstr * Copy = BuildMI ( InsertMBB , I , DL , get ( <STR_LIT> ) , DstReg ) . add ( Op ) ; Op . setReg ( DstReg ) ; Op . setSubReg ( <NUM_LIT> ) ; MachineInstr * Def = MRI . getVRegDef ( OpReg ) ; if ( ! Def ) return ; if ( Def -> isMoveImmediate ( ) && DstRC != & <STR_LIT> ) FoldImmediate ( *


<s> unsigned TIDIGXReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned TIDIGYReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned TIDIGZReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned InputPtrReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; for ( unsigned Reg : { TIDIGXReg , TIDIGYReg , TIDIGZReg } ) { if ( ! Entry . isLiveIn ( Reg ) ) Entry . addLiveIn ( Reg ) ; } RS -> enterBasicBlock ( Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; getAddNoCarry ( Entry , Insert , DL , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) ; } else


<s> unsigned TIDIGZReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned InputPtrReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; for ( unsigned Reg : { TIDIGXReg , TIDIGYReg , TIDIGZReg } ) { if ( ! Entry . isLiveIn ( Reg ) ) Entry . addLiveIn ( Reg ) ; } RS -> enterBasicBlock ( Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg

( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ;


<s> if ( ! BaseOp ) { return false ; } BaseOps . push_back ( BaseOp ) ; Offset = OffsetOp -> getImm ( ) ; } else { const MachineOperand * Offset0Op = getNamedOperand ( LdSt , <STR_LIT> :: offset0 ) ; const MachineOperand * Offset1Op = getNamedOperand ( LdSt , <STR_LIT> :: offset1 ) ; unsigned Offset0 = Offset0Op -> getImm ( ) ; unsigned Offset1 = Offset1Op -> getImm ( ) ; if ( Offset0 + <NUM_LIT> != Offset1 ) return false ; unsigned EltSize ; if ( LdSt . mayLoad ( ) ) EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , <NUM_LIT> ) ) / <NUM_LIT> ; else { assert ( LdSt . mayStore ( ) ) ; int Data0Idx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , Data0Idx ) ) / <NUM_LIT> ; } if ( isStride64 ( Opc ) ) EltSize *= <NUM_LIT> ; BaseOps . push_back ( BaseOp ) ; Offset = EltSize * Offset0 ; } return true ; } if ( isMUBUF ( LdSt ) || isMTBUF ( LdSt ) ) { const MachineOperand * SOffset = getNamedOperand ( LdSt , <STR_LIT> :: soffset ) ; if ( SOffset && SOffset -> isReg ( ) ) { const MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( AddrReg && ! AddrReg -> isFI ( ) ) return false ; const MachineOperand * RSrc = getNamedOperand ( LdSt , <STR_LIT> :: srsrc ) ; const <ISA_LIT>MachineFunctionInfo * MFI = LdSt . getParent ( ) -> getParent ( ) -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( RSrc -> getReg ( ) != MFI -> getScratchRSrcReg ( ) ) return false ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; BaseOps . push_back ( RSrc ) ; BaseOps . push_back ( SOffset ) ; Offset = OffsetImm -> getImm ( ) ; return true ; } BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: srsrc ) ; if ( ! BaseOp ) return false ; BaseOps . push_back


<s> bool <ISA_LIT>InstrInfo :: isReallyTriviallyReMaterializable ( const MachineInstr & MI , AAResults * AA ) const { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return MI . getNumOperands ( ) == MI . getDesc


<s> BaseOps . push_back ( BaseOp ) ; Offset = OffsetOp -> getImm ( ) ; DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdst ) ; if ( DataOpIdx == - <NUM_LIT> ) DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; Width = getOpSize ( LdSt , DataOpIdx ) ; } else { const MachineOperand * Offset0Op = getNamedOperand ( LdSt , <STR_LIT> :: offset0 ) ; const MachineOperand * Offset1Op = getNamedOperand ( LdSt , <STR_LIT> :: offset1 ) ; unsigned Offset0 = Offset0Op -> getImm ( ) ; unsigned Offset1 = Offset1Op -> getImm ( ) ; if ( Offset0 + <NUM_LIT> != Offset1 ) return false ; unsigned EltSize ; if ( LdSt . mayLoad ( ) ) EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , <NUM_LIT> ) ) / <NUM_LIT> ; else { assert ( LdSt . mayStore ( ) ) ; int Data0Idx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , Data0Idx ) ) / <NUM_LIT> ; } if ( isStride64 ( Opc ) ) EltSize *= <NUM_LIT> ; BaseOps . push_back ( BaseOp ) ; Offset = EltSize * Offset0 ; DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdst ) ; if ( DataOpIdx == - <NUM_LIT> ) { DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; Width = getOpSize ( LdSt , DataOpIdx ) ; DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: data1 ) ; Width += getOpSize ( LdSt , DataOpIdx ) ; } else { Width = getOpSize ( LdSt , DataOpIdx ) ; } } return true ; } if ( isMUBUF ( LdSt ) || isMTBUF ( LdSt ) ) { const MachineOperand * SOffset = getNamedOperand ( LdSt , <STR_LIT> :: soffset ) ; if ( SOffset && SOffset -> isReg ( ) ) { const MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( AddrReg && ! AddrReg -> isFI ( ) ) return false ; const MachineOperand * RSrc = getNamedOperand ( LdSt , <STR_LIT> :: srsrc ) ; const <ISA_LIT>MachineFunctionInfo * MFI = LdSt . getParent ( ) -> getParent ( ) -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( RSrc -> getReg ( ) != MFI -> getScratchRSrcReg


<s> BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( BaseOp ) BaseOps . push_back ( BaseOp ) ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; Offset = OffsetImm -> getImm ( ) ; if ( SOffset ) Offset += SOffset -> getImm ( ) ; } DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdst ) ; if ( DataOpIdx == - <NUM_LIT> ) DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdata ) ; Width = getOpSize ( LdSt , DataOpIdx ) ; return true ; } if ( isMIMG ( LdSt ) ) { int SRsrcIdx = <STR_LIT> ( Opc , <STR_LIT> :: srsrc ) ; BaseOps . push_back ( & LdSt . getOperand ( SRsrcIdx ) ) ; int VAddr0Idx = <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) ; if ( VAddr0Idx >= <NUM_LIT> ) { for ( int I = VAddr0Idx ; I < SRsrcIdx ; ++ I ) BaseOps . push_back ( & LdSt . getOperand ( I ) ) ; } else { BaseOps . push_back ( getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ) ; } Offset = <NUM_LIT> ; return true ; } if ( isSMRD ( LdSt ) ) { BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: sbase ) ; if ( ! BaseOp ) return false ; BaseOps . push_back ( BaseOp ) ; OffsetOp = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; Offset = OffsetOp ? OffsetOp -> getImm ( ) : <NUM_LIT> ; DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: sdst ) ; Width = getOpSize ( LdSt , DataOpIdx ) ; return true ; } if ( isFLAT ( LdSt ) ) { BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( BaseOp ) BaseOps . push_back ( BaseOp ) ; BaseOp = getNamedOperand ( LdSt , <STR_LIT> :: saddr ) ; if ( BaseOp ) BaseOps . push_back ( BaseOp ) ; Offset = getNamedOperand ( LdSt , <STR_LIT> :: offset ) -> getImm ( ) ; DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdst ) ; if ( DataOpIdx == - <NUM_LIT> ) DataOpIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdata ) ; Width = getOpSize ( LdSt , DataOpIdx ) ; return


<s> SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdata ) ; if ( ! SecondDst ) SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdst ) ; } else if ( isSMRD ( FirstLdSt ) && isSMRD ( SecondLdSt ) ) { FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: sdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: sdst ) ; } else if ( isDS ( FirstLdSt ) && isDS ( SecondLdSt ) ) { FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: vdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdst ) ; } if ( ! FirstDst || ! SecondDst ) return false ; unsigned LoadClusterThreshold = <NUM_LIT> ; const MachineRegisterInfo & MRI = FirstLdSt . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; const Register Reg = FirstDst -> getReg ( ) ; const TargetRegisterClass * DstRC = Register :: isVirtualRegister ( Reg ) ? MRI .

getRegClass ( Reg ) : RI . getPhysRegClass ( Reg ) ;


<s> if ( isMUBUF ( Opc1 ) || isMTBUF ( Opc1 ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return ! isFLAT ( Opc1 ) && ! isSMRD ( Opc1 ) ; } if ( isSMRD ( Opc0 ) ) { if ( isSMRD ( Opc1 ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return ! isFLAT ( Opc1 ) && ! isMUBUF ( Opc0 ) && ! isMTBUF ( Opc0 ) ; } if ( isFLAT ( Opc0 ) ) { if ( isFLAT ( Opc1 ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return false ; } return


<s> assert ( MIb && ( MIb -> mayLoad ( ) || MIb -> mayStore ( ) ) && <STR_LIT> ) ; if ( MIa -> hasUnmodeledSideEffects ( ) || MIb -> hasUnmodeledSideEffects ( ) ) return false ; if ( MIa -> hasOrderedMemoryRef ( ) || MIb -> hasOrderedMemoryRef ( ) ) return false ; if ( isDS ( Opc0 ) ) { if ( isDS ( Opc1 ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return ! isFLAT ( Opc1 ) ; } if ( isMUBUF ( Opc0 ) || isMTBUF ( Opc0 ) ) { if ( isMUBUF ( Opc1 ) || isMTBUF ( Opc1 ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return ! isFLAT ( Opc1 ) &&


<s> unsigned TIDIGYReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: TIDIG_Y ) ; unsigned TIDIGZReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: TIDIG_Z ) ; unsigned InputPtrReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: INPUT_PTR ) ; for ( unsigned Reg : { TIDIGXReg , TIDIGYReg , TIDIGZReg } ) { if ( ! Entry . isLiveIn ( Reg ) ) Entry . addLiveIn ( Reg ) ; } RS -> enterBasicBlock ( & Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) ; } else { BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> )


<s> if ( ! MFI -> hasCalculatedTID ( ) ) { MachineBasicBlock & Entry = MBB . getParent ( ) -> front ( ) ; MachineBasicBlock :: iterator Insert = Entry . front ( ) ; DebugLoc DL = Insert -> getDebugLoc ( ) ; TIDReg = RI . findUnusedRegister ( MF -> getRegInfo ( ) , & <STR_LIT> ) ; if ( TIDReg == <STR_LIT> ) return TIDReg ; if ( MFI -> getShaderType ( ) == ShaderType :: COMPUTE && WorkGroupSize > WavefrontSize ) { unsigned TIDIGXReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: TIDIG_X ) ; unsigned TIDIGYReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: TIDIG_Y ) ; unsigned TIDIGZReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: TIDIG_Z ) ; unsigned InputPtrReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: INPUT_PTR ) ; for ( unsigned Reg : { TIDIGXReg , TIDIGYReg , TIDIGZReg } ) { if ( ! Entry . isLiveIn ( Reg ) ) Entry . addLiveIn ( Reg ) ; } RS -> enterBasicBlock ( & Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) ; } else { BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addReg ( TIDReg ) ; } BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( <NUM_LIT> ) . addReg ( TIDReg ) ; MFI -> setTIDReg ( TIDReg ) ; } unsigned LDSOffset = MFI -> LDSSize + ( FrameOffset * WorkGroupSize ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , TmpReg ) . addImm ( LDSOffset ) . addReg ( TIDReg ) ; return


<s> if ( CommutedOpcode == - <NUM_LIT> ) return nullptr ; int Src0Idx = <STR_LIT> ( MI -> getOpcode ( ) , <STR_LIT> :: src0 ) ; MachineOperand & Src0 = MI -> getOperand ( Src0Idx ) ; if ( ! Src0 . isReg ( ) ) return nullptr ; int Src1Idx = <STR_LIT> ( MI -> getOpcode ( ) , <STR_LIT> :: src1 ) ; if ( ( OpIdx0 != static_cast < unsigned > ( Src0Idx ) || OpIdx1 != static_cast < unsigned > ( Src1Idx ) ) && ( OpIdx0 != static_cast < unsigned > ( Src1Idx ) || OpIdx1 != static_cast < unsigned > ( Src0Idx ) ) ) return nullptr ; MachineOperand & Src1 = MI -> getOperand ( Src1Idx ) ; if ( isVOP2 ( MI -> getOpcode ( ) ) && ( ! isOperandLegal ( MI , Src0Idx , & Src1 ) || ! isOperandLegal ( MI , Src1Idx , & Src0 ) ) ) { return nullptr ; } if ( ! Src1 . isReg ( ) ) { if ( NewMI || ! Src1 . isImm ( ) || ( ! isVOP2 ( MI -> getOpcode ( ) ) && ! isVOP3 ( MI


<s> return commuteOpcode ( MI . getOpcode ( )


<s> return commuteOpcode ( MI . getOpcode


<s> assert ( <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; } return ; } assert ( <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; return ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_3 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_7 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_15 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; return ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_1 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_2 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_3 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_7 ; } else if ( <STR_LIT> . contains ( DestReg ) ) { assert ( <STR_LIT> . contains ( SrcReg ) || <STR_LIT> . contains ( SrcReg ) ) ; Opcode = <STR_LIT> ; SubIndices = Sub0_15 ; } else { llvm_unreachable ( <STR_LIT> ) ; } while ( unsigned SubIdx = * SubIndices ++ ) { MachineInstrBuilder Builder = BuildMI ( MBB , MI , DL , get ( Opcode ) , RI . getSubReg


<s> const MachineOperand & MO = MI -> getOperand ( Idx ) ; if ( RI . isSGPRClassID ( Desc . OpInfo [ Idx ] . RegClass ) ) SGPRReg = MO . getReg ( ) ; if ( MO . isReg ( ) && RI . isSGPRClass ( MRI . getRegClass ( MO . getReg ( ) ) ) ) UsedSGPRs [ i ] = MO . getReg ( ) ; } if ( SGPRReg != <STR_LIT> ) return SGPRReg ; if ( UsedSGPRs [ <NUM_LIT> ] != <STR_LIT> ) { if ( UsedSGPRs [ <NUM_LIT> ] == UsedSGPRs [ <NUM_LIT> ] || UsedSGPRs [ <NUM_LIT>

] == UsedSGPRs [ <NUM_LIT> ] ) SGPRReg = UsedSGPRs [ <NUM_LIT> ] ;


<s> uint64_t getClampMask ( const MachineInstr & MI ) const { const uint64_t ClampFlags = <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> ; return MI . getDesc ( ) .


<s> const uint64_t ClampFlags = <STR_LIT> | <STR_LIT> | <STR_LIT> | <STR_LIT> ; return MI . getDesc ( )


<s> else { assert ( LdSt -> mayStore ( ) ) ; int Data0Idx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; EltSize = getOpRegClass ( * LdSt , Data0Idx ) -> getSize ( ) ; } if ( isStride64 ( Opc ) ) EltSize *= <NUM_LIT> ; const MachineOperand * AddrReg = getNamedOperand ( * LdSt , <STR_LIT> :: addr ) ; BaseReg = AddrReg -> getReg ( ) ; Offset = EltSize * Offset0 ; return true ; } return false ; } if ( isMUBUF ( Opc ) || isMTBUF ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: soffset ) != - <NUM_LIT> ) return false ; const MachineOperand * AddrReg = getNamedOperand ( * LdSt , <STR_LIT> :: vaddr ) ; if ( ! AddrReg ) return false ; const MachineOperand * OffsetImm = getNamedOperand ( * LdSt , <STR_LIT> :: offset ) ; BaseReg = AddrReg -> getReg ( ) ; Offset = OffsetImm -> getImm ( ) ; return true ; } if ( isSMRD ( Opc ) ) { const MachineOperand * OffsetImm = getNamedOperand (


<s> unsigned getOpSize ( const MachineInstr & MI , unsigned OpNo


<s> const <ISA_LIT>RegisterInfo & getRegisterInfo ( ) const { return RI


<s> static bool hasIntClamp ( const MachineInstr & MI ) { return MI . getDesc ( )


<s> Count -= <NUM_LIT> ; BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc

( ) , get ( <STR_LIT> ) ) . addImm ( Arg ) ;


<s> bool isDPP ( uint16_t Opcode ) const { return get ( Opcode ) . TSFlags &


<s> bool isGather4 ( uint16_t Opcode ) const { return get ( Opcode ) . TSFlags &


<s> const MachineOperand & MO = MI . getOperand ( OpIdx ) ; return MO . isImm ( ) &&


<s> const MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; const MCInstrDesc & InstDesc = get ( MI -> getOpcode ( ) ) ; const MCOperandInfo & OpInfo = InstDesc . OpInfo [ OpIdx ] ; const TargetRegisterClass * DefinedRC = OpInfo . RegClass != - <NUM_LIT> ? RI . getRegClass ( OpInfo . RegClass ) : nullptr ; if ( ! MO ) MO = & MI -> getOperand ( OpIdx ) ; if ( isVALU ( InstDesc . Opcode ) && usesConstantBus ( MRI , * MO , DefinedRC -> getSize ( ) ) ) { unsigned SGPRUsed = MO -> isReg ( ) ? MO -> getReg ( ) : ( unsigned ) <STR_LIT> ; for ( unsigned i = <NUM_LIT> , e = MI -> getNumOperands ( ) ; i != e ; ++ i ) { if ( i == OpIdx ) continue ; const MachineOperand & Op = MI -> getOperand ( i ) ; if ( Op . isReg ( ) && Op . getReg ( ) != SGPRUsed && usesConstantBus ( MRI , Op , getOpSize ( * MI , i ) ) ) { return false ; } } } if ( MO -> isReg ( ) ) { assert ( DefinedRC ) ; const TargetRegisterClass * RC = TargetRegisterInfo :: isVirtualRegister ( MO -> getReg ( ) ) ? MRI . getRegClass ( MO -> getReg ( ) ) : RI . getPhysRegClass ( MO -> getReg ( ) ) ; return RI . getCommonSubClass ( RC , RI . getRegClass ( OpInfo . RegClass ) ) == RC ; } assert ( MO -> isImm ( ) || MO -> isTargetIndex ( ) || MO -> isFI ( ) ) ; if ( ! DefinedRC ) { return true ; } return isImmOperandLegal ( MI , OpIdx , *


<s> auto Flags = MI . getDesc ( ) . TSFlags ; return ( Flags & <STR_LIT> ) && ! ( Flags &


<s> bool isVGPRCopy ( const MachineInstr & MI ) const { assert ( MI . isCopy ( ) ) ; unsigned Dest = MI . getOperand ( <NUM_LIT>


<s> bool isVGPRCopy ( const MachineInstr & MI ) const { assert ( MI . isCopy ( ) ) ; unsigned Dest = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const MachineFunction & MF = * MI . getParent


<s> return isMUBUF ( Opcode ) || isMTBUF ( Opcode


<s> bool isVMEM ( uint16_t Opcode ) const { return isMUBUF ( Opcode ) || isMTBUF (


<s> } else if ( RI . hasVGPRs ( RC ) && ST . isVGPRSpillingEnabled ( MFI ) ) { switch ( RC -> getSize ( ) * <NUM_LIT> ) { case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; } } if ( Opcode != - <NUM_LIT> ) { unsigned Align = <NUM_LIT> ; FrameInfo -> setObjectAlignment ( FrameIndex , Align ) ; unsigned Size = FrameInfo -> getObjectSize ( FrameIndex ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) . addFrameIndex ( FrameIndex ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addMemOperand ( MMO ) ; } else { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ,


<s> if ( NewOpcode == <STR_LIT> ) { legalizeOperands ( Inst ) ; continue ; } const MCInstrDesc & NewDesc = get ( NewOpcode ) ; Inst -> setDesc ( NewDesc ) ; for ( unsigned i = Inst -> getNumOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; -- i ) { MachineOperand & Op = Inst -> getOperand ( i ) ; if ( Op . isReg ( ) && Op . getReg ( ) == <STR_LIT> ) Inst -> RemoveOperand ( i ) ; } if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { unsigned Size = ( Opcode == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; Inst -> addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; Inst -> addOperand ( MachineOperand :: CreateImm ( Size ) ) ; } else if ( Opcode == <STR_LIT> ) { Inst -> addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; } Inst -> addImplicitDefUseOperands ( * Inst -> getParent ( ) -> getParent ( ) ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { const MachineOperand & OffsetWidthOp = Inst -> getOperand ( <NUM_LIT> ) ; assert ( OffsetWidthOp . isImm ( ) && <STR_LIT> ) ; uint32_t Imm = OffsetWidthOp . getImm ( ) ; uint32_t Offset = Imm & <NUM_LIT> ; uint32_t BitWidth = ( Imm & <NUM_LIT> ) >> <NUM_LIT> ; Inst -> RemoveOperand ( <NUM_LIT> ) ; Inst -> addOperand ( MachineOperand :: CreateImm


<s> if ( isDS ( Opc0 ) && isDS ( Opc1 ) ) return true ; if ( isSMRD ( Opc0 ) && isSMRD ( Opc1 ) ) return true ; if ( ( isMUBUF ( Opc0 ) || isMTBUF ( Opc0 ) ) &&

( isMUBUF ( Opc1 ) || isMTBUF ( Opc1 ) ) ) return true ;


<s> bool sopkIsZext ( uint16_t Opcode ) const { return get ( Opcode ) . TSFlags


<s> case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; case <NUM_LIT> : Opcode = <STR_LIT> ; break ; } } if ( Opcode != - <NUM_LIT> ) { MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; unsigned Size = FrameInfo -> getObjectSize ( FrameIndex ) ; unsigned Align = FrameInfo -> getObjectAlignment ( FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , Size , Align ) ; FrameInfo -> setObjectAlignment ( FrameIndex , <NUM_LIT> ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) ) . addReg ( SrcReg ) . addFrameIndex ( FrameIndex ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addMemOperand ( MMO ) ; } else { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) )


<s> static bool usesLGKM_CNT ( const MachineInstr & MI ) { return MI . getDesc ( ) . TSFlags


<s> static bool usesLGKM_CNT ( const MachineInstr & MI ) { return MI . getDesc (


<s> static bool usesVM_CNT ( const MachineInstr &


<s> if ( IsFast ) * IsFast = false ; return allowsMisalignedMemoryAccessesImpl ( Ty . getSizeInBits ( ) , AddrSpace , Alignment ,


<s> return allowsMisalignedMemoryAccessesImpl ( Ty . getSizeInBits ( ) , AddrSpace , Alignment ,


<s> if ( ! Node -> hasAnyUseOfValue ( <NUM_LIT> ) ) { MI . setDesc ( TII -> get ( NoRetAtomicOp ) ) ; MI . RemoveOperand ( <NUM_LIT> ) ; return ; } if ( ( Node -> hasNUsesOfValue ( <NUM_LIT> , <NUM_LIT> ) && Node -> use_begin ( ) -> isMachineOpcode ( ) && Node -> use_begin ( ) ->

getMachineOpcode ( ) == <STR_LIT> && ! Node -> use_begin ( ) -> hasAnyUseOfValue ( <NUM_LIT> ) ) ) {


<s> EVT IntLoadVT = LoadVT . changeTypeToInteger ( ) ; SmallVector < SDValue , <NUM_LIT> > Elts ; DAG . ExtractVectorElements ( Result , Elts ) ; for ( SDValue & Elt : Elts ) Elt = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Elt ) ; Result = DAG . getBuildVector ( IntLoadVT , DL , Elts ) ; return DAG . getNode ( <STR_LIT> , DL , LoadVT


<s> static SDValue adjustLoadValueTypeImpl ( SDValue Result , EVT LoadVT , const SDLoc & DL , SelectionDAG & DAG , bool Unpacked ) { if ( ! LoadVT . isVector ( ) ) return Result ; if ( Unpacked ) { EVT IntLoadVT = LoadVT .


<s> return allocateSGPR32InputImpl ( CCInfo , &


<s> static ArgDescriptor allocateSGPR32InputImpl ( CCState & CCInfo , const TargetRegisterClass * RC , unsigned NumArgRegs ) { ArrayRef < MCPhysReg > ArgSGPRs = makeArrayRef ( RC -> begin ( ) , <NUM_LIT> ) ; unsigned RegIdx = CCInfo . getFirstUnallocated ( ArgSGPRs ) ; if ( RegIdx == ArgSGPRs . size ( ) ) report_fatal_error ( <STR_LIT> ) ; unsigned Reg = ArgSGPRs [ RegIdx ] ; Reg = CCInfo . AllocateReg ( Reg ) ; assert ( Reg != <STR_LIT> ) ; MachineFunction & MF = CCInfo . getMachineFunction ( ) ; MF . addLiveIn ( Reg , RC ) ; return ArgDescriptor :: createRegister


<s> static ArgDescriptor allocateSGPR64Input ( CCState


<s> if ( Info . hasDispatchID ( ) ) ArgInfo . DispatchID = allocateSGPR64Input ( CCInfo ) ; if ( Info . hasWorkGroupIDX ( ) ) ArgInfo . WorkGroupIDX = allocateSGPR32Input ( CCInfo ) ; if ( Info . hasWorkGroupIDY ( ) ) ArgInfo . WorkGroupIDY = allocateSGPR32Input ( CCInfo ) ; if ( Info . hasWorkGroupIDZ ( ) ) ArgInfo . WorkGroupIDZ = allocateSGPR32Input ( CCInfo ) ; if ( Info . hasImplicitArgPtr ( ) ) ArgInfo . ImplicitArgPtr


<s> if ( Info . hasWorkItemIDX ( ) ) Info . setWorkItemIDX ( allocateVGPR32Input ( CCInfo ) ) ; if ( Info . hasWorkItemIDY ( ) ) Info . setWorkItemIDY ( allocateVGPR32Input ( CCInfo ) ) ; if ( Info . hasWorkItemIDZ ( ) ) Info . setWorkItemIDZ ( allocateVGPR32Input


<s> static void allocateSpecialInputVGPRs ( CCState & CCInfo , MachineFunction & MF , const <ISA_LIT>RegisterInfo & TRI , <ISA_LIT>MachineFunctionInfo & Info ) { if ( Info . hasWorkItemIDX ( ) ) Info . setWorkItemIDX ( allocateVGPR32Input ( CCInfo ) ) ; if ( Info . hasWorkItemIDY ( ) ) Info . setWorkItemIDY (


<s> if ( IsFast ) * IsFast = false ; if ( VT == <STR_LIT> || ( VT != <STR_LIT> && VT . getSizeInBits ( ) > <NUM_LIT> && VT . getStoreSize ( ) > <NUM_LIT> ) ) { return false ; } if ( AddrSpace == <ISA_LIT>AS :: LOCAL_ADDRESS || AddrSpace == <ISA_LIT>AS :: REGION_ADDRESS ) { bool AlignedBy4 = ( Align % <NUM_LIT> == <NUM_LIT> ) ; if ( IsFast )


<s> static bool bitOpWithConstantIsReducible ( unsigned Opc , uint32_t Val ) { return ( Opc == <STR_LIT> && ( Val == <NUM_LIT> || Val == <NUM_LIT> ) ) || ( Opc == <STR_LIT> && ( Val == <NUM_LIT> || Val == <NUM_LIT> ) ) || ( Opc == <STR_LIT>


<s> static SDValue buildPCRelGlobalAddress ( SelectionDAG & DAG , const GlobalValue * GV , const SDLoc &

DL , unsigned Offset , EVT PtrVT , unsigned GAFlags = <ISA_LIT>InstrInfo :: MO_NONE ) {


<s> static SDValue buildPCRelGlobalAddress ( SelectionDAG & DAG , const GlobalValue * GV , const SDLoc & DL , unsigned Offset , EVT PtrVT , unsigned GAFlags = <ISA_LIT>InstrInfo :: MO_NONE ) { SDValue PtrLo = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> , Offset + <NUM_LIT> , GAFlags ) ; SDValue PtrHi = DAG . getTargetGlobalAddress ( GV , DL , <STR_LIT> , Offset + <NUM_LIT> , GAFlags == <ISA_LIT>InstrInfo

:: MO_NONE ? GAFlags : GAFlags + <NUM_LIT> ) ;


<s> MachineSDNode * <ISA_LIT>TargetLowering :: buildRSRC ( SelectionDAG & DAG , const SDLoc & DL , SDValue Ptr , uint32_t RsrcDword1 , uint64_t RsrcDword2And3 ) const { SDValue PtrLo = DAG . getTargetExtractSubreg ( <STR_LIT> , DL , <STR_LIT> , Ptr ) ; SDValue PtrHi = DAG . getTargetExtractSubreg ( <STR_LIT> , DL , <STR_LIT> , Ptr ) ; if ( RsrcDword1 ) { PtrHi = SDValue ( DAG . getMachineNode ( <STR_LIT> , DL , <STR_LIT> , PtrHi , DAG . getConstant ( RsrcDword1 , DL , <STR_LIT> ) ) , <NUM_LIT> ) ; } SDValue DataLo = buildSMovImm32 ( DAG , DL , RsrcDword2And3 & UINT64_C ( <NUM_LIT> ) ) ; SDValue DataHi = buildSMovImm32 ( DAG , DL , RsrcDword2And3 >> <NUM_LIT> ) ; const SDValue Ops [ ] = { DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) , PtrLo , DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) , PtrHi , DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) , DataLo , DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) , DataHi , DAG . getTargetConstant ( <STR_LIT> , DL , <STR_LIT> ) } ; return DAG . getMachineNode ( <STR_LIT> ,


<s> return SDValue ( DAG . getMachineNode ( <STR_LIT> , DL , <STR_LIT> ,


<s> bool <ISA_LIT>TargetLowering :: CanLowerReturn ( CallingConv :: ID CallConv , MachineFunction & MF , bool IsVarArg , const SmallVectorImpl < <STR_LIT> > & Outs , LLVMContext & Context ) const { if ( <STR_LIT> ( CallConv ) ) return true ; SmallVector < CCValAssign , <NUM_LIT> > RVLocs ; CCState CCInfo ( CallConv , IsVarArg , MF ,


<s> bool <ISA_LIT>TargetLowering :: CanLowerReturn ( CallingConv :: ID CallConv , MachineFunction & MF , bool IsVarArg , const SmallVectorImpl < <STR_LIT> > & Outs , LLVMContext & Context ) const { if ( <STR_LIT> ( CallConv ) ) return true ; SmallVector < CCValAssign , <NUM_LIT> > RVLocs ; CCState CCInfo ( CallConv , IsVarArg , MF , RVLocs , Context ) ; return CCInfo . CheckReturn ( Outs , CCAssignFnForReturn ( CallConv ,


<s> static std :: pair < unsigned , int > computeIndirectRegAndOffset ( const <ISA_LIT>RegisterInfo & TRI , const TargetRegisterClass *

SuperRC , unsigned VecReg , int Offset ) {


<s> void <ISA_LIT>TargetLowering :: computeKnownBitsForFrameIndex ( const SDValue Op , KnownBits & Known , const APInt & DemandedElts , const SelectionDAG & DAG , unsigned Depth ) const { TargetLowering :: computeKnownBitsForFrameIndex ( Op , Known , DemandedElts , DAG , Depth ) ; if ( getSubtarget ( ) -> enableHugePrivateBuffer ( ) ) return ; Known . Zero . setHighBits ( AssumeFrameIndexHighZeroBits


<s> } if ( MemVT . isFloatingPoint ( ) ) Val = getFPExtOrFPTrunc ( DAG , Val , SL , VT ) ; else if ( Signed ) Val = DAG . getSExtOrTrunc ( Val , SL , VT ) ; else Val = DAG . getZExtOrTrunc ( Val


<s> SDValue <ISA_LIT>TargetLowering :: copyToM0 ( SelectionDAG & DAG , SDValue Chain , const


<s> SDNode * M0 = DAG . getMachineNode ( <STR_LIT> , DL , <STR_LIT> , <STR_LIT> ,


<s> bool <ISA_LIT>TargetLowering :: denormalsEnabledForType ( EVT VT ) const { switch ( VT . getScalarType ( ) . getSimpleVT ( ) . SimpleTy ) { case <STR_LIT> : return Subtarget -> hasFP32Denormals ( ) ; case <STR_LIT>


<s> MI . eraseFromParent ( ) ; return & MBB ; } if ( setM0ToIndexFromSGPR ( TII , MRI , MI , Offset , UseGPRIdxMode , false ) ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( SrcVec -> getReg ( ) , RegState :: Undef , SubReg ) . add ( * Val ) . addReg ( Dst , RegState :: ImplicitDefine ) . addReg ( SrcVec -> getReg ( ) , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ; } else { const MCInstrDesc & MovRelDesc = TII -> get ( getMOVRELDPseudo ( TRI , VecRC ) ) ; BuildMI ( MBB , I , DL , MovRelDesc ) . addReg ( Dst , RegState :: Define ) . addReg ( SrcVec -> getReg ( ) ) . add ( * Val ) . addImm ( SubReg - <STR_LIT> ) ; } MI . eraseFromParent ( ) ; return & MBB ; } if ( Val -> isReg ( ) ) MRI . clearKillFlags ( Val -> getReg ( ) ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; unsigned PhiReg = MRI . createVirtualRegister ( VecRC ) ; auto InsPt = loadM0FromVGPR ( TII , MBB , MI , SrcVec -> getReg ( ) , PhiReg , Offset , UseGPRIdxMode , false ) ; MachineBasicBlock * LoopBB = InsPt -> getParent ( ) ; if ( UseGPRIdxMode ) { BuildMI ( * LoopBB , InsPt , DL , TII -> get ( <STR_LIT> ) ) . addReg ( PhiReg , RegState :: Undef , SubReg ) . add ( * Val ) . addReg ( Dst , RegState :: ImplicitDefine ) . addReg ( PhiReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( * LoopBB , InsPt , DL , TII ->


<s> MachineFunction * MF = MBB . getParent ( ) ; MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned SrcReg = TII -> getNamedOperand ( MI , <STR_LIT> :: src ) -> getReg ( ) ; int Offset = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> getImm ( ) ; const TargetRegisterClass * VecRC = MRI . getRegClass ( SrcReg ) ; unsigned SubReg ; std :: tie ( SubReg , Offset ) = computeIndirectRegAndOffset ( TRI , VecRC , SrcReg , Offset ) ; bool UseGPRIdxMode = ST . useVGPRIndexMode ( EnableVGPRIndexMode ) ; if ( setM0ToIndexFromSGPR ( TII , MRI , MI , Offset , UseGPRIdxMode , true ) ) { MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( UseGPRIdxMode ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , RegState :: Undef , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) . addReg ( <STR_LIT> , RegState :: Implicit ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) ; } else { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( SrcReg , RegState :: Undef , SubReg ) . addReg ( SrcReg , RegState :: Implicit ) ; } MI . eraseFromParent ( ) ; return & MBB


<s> unsigned CondReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII -> get ( TargetOpcode :: PHI ) , PhiReg ) . addReg ( InitReg ) . addMBB ( & OrigBB ) . addReg ( ResultReg ) . addMBB ( & LoopBB ) ; BuildMI ( LoopBB , I , DL , TII -> get ( TargetOpcode :: PHI ) , PhiExec ) . addReg ( InitSaveExecReg ) . addMBB ( & OrigBB ) . addReg ( NewExec ) . addMBB ( & LoopBB ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , CurrentIdxReg ) . addReg ( IdxReg . getReg ( ) , getUndefRegState ( IdxReg . isUndef ( ) ) ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , CondReg ) . addReg ( CurrentIdxReg ) . addReg ( IdxReg . getReg ( ) , <NUM_LIT> , IdxReg . getSubReg ( ) ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , NewExec ) . addReg ( CondReg , RegState :: Kill ) ; MRI . setSimpleHint ( NewExec , CondReg ) ; if ( UseGPRIdxMode ) { unsigned IdxReg ; if ( Offset == <NUM_LIT> ) { IdxReg = CurrentIdxReg ; } else { IdxReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , IdxReg ) . addReg ( CurrentIdxReg , RegState :: Kill ) . addImm ( Offset ) ; } unsigned IdxMode = IsIndirectSrc ? VGPRIndexMode :: SRC0_ENABLE : VGPRIndexMode :: DST_ENABLE ; MachineInstr * SetOn = BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( IdxReg , RegState :: Kill ) . addImm ( IdxMode ) ; SetOn -> getOperand ( <NUM_LIT> ) . setIsUndef ( ) ; } else { if ( Offset == <NUM_LIT> ) { BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( CurrentIdxReg , RegState :: Kill ) ; } else { BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( CurrentIdxReg , RegState :: Kill ) . addImm ( Offset ) ; } } MachineInstr * InsertPt = BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( NewExec ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( & LoopBB ) ; return InsertPt -> getIterator


<s> BuildMI ( LoopBB , I , DL , TII -> get ( TargetOpcode :: PHI ) , PhiReg ) . addReg ( InitReg ) . addMBB ( & OrigBB ) . addReg ( ResultReg ) . addMBB ( & LoopBB ) ; BuildMI ( LoopBB , I , DL , TII -> get ( TargetOpcode :: PHI ) , PhiExec ) . addReg ( InitSaveExecReg ) . addMBB ( & OrigBB ) . addReg ( NewExec ) . addMBB ( & LoopBB ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , CurrentIdxReg ) . addReg ( IdxReg . getReg ( ) , getUndefRegState ( IdxReg . isUndef ( ) ) ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , CondReg ) . addReg ( CurrentIdxReg ) . addReg ( IdxReg . getReg ( ) , <NUM_LIT> , IdxReg . getSubReg ( ) ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , NewExec ) . addReg ( CondReg , RegState :: Kill ) ; MRI . setSimpleHint ( NewExec , CondReg ) ; if ( UseGPRIdxMode ) { unsigned IdxReg ; if ( Offset == <NUM_LIT> ) { IdxReg = CurrentIdxReg ; } else { IdxReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , IdxReg ) . addReg ( CurrentIdxReg , RegState :: Kill ) . addImm ( Offset ) ; } unsigned IdxMode = IsIndirectSrc ? VGPRIndexMode :: SRC0_ENABLE : VGPRIndexMode :: DST_ENABLE ; MachineInstr * SetOn = BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( IdxReg , RegState :: Kill ) . addImm ( IdxMode ) ; SetOn -> getOperand ( <NUM_LIT> ) . setIsUndef ( ) ; } else { if ( Offset ==


<s> DiagnosticInfoUnsupported BadIntrin ( DAG . getMachineFunction ( ) . getFunction ( ) , <STR_LIT> , DL . getDebugLoc ( ) ) ; DAG . getContext ( ) -> diagnose ( BadIntrin ) ; return DAG . getUNDEF ( VT


<s> if ( Info -> isEntryFunction ( ) ) { reservePrivateMemoryRegs ( getTargetMachine ( ) , MF , * TRI , * Info ) ; } bool NeedSP = ! Info -> isEntryFunction ( ) || MFI . hasVarSizedObjects ( ) || MFI . hasCalls ( ) ; if ( NeedSP ) { unsigned ReservedStackPtrOffsetReg = TRI -> reservedStackPtrOffsetReg ( MF ) ; Info -> setStackPtrOffsetReg ( ReservedStackPtrOffsetReg ) ; assert ( Info -> getStackPtrOffsetReg ( ) != Info -> getFrameOffsetReg ( ) ) ; assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ; MRI . replaceRegWith ( <STR_LIT> , Info


<s> reservePrivateMemoryRegs ( getTargetMachine ( ) , MF , * TRI , * Info ) ; } bool NeedSP = ! Info -> isEntryFunction ( ) || MFI . hasVarSizedObjects ( ) || MFI . hasCalls ( ) ; if ( NeedSP ) { unsigned ReservedStackPtrOffsetReg = TRI -> reservedStackPtrOffsetReg ( MF ) ; Info -> setStackPtrOffsetReg ( ReservedStackPtrOffsetReg ) ; assert ( Info -> getStackPtrOffsetReg ( ) != Info -> getFrameOffsetReg ( ) ) ; assert ( ! TRI -> isSubRegister ( Info -> getScratchRSrcReg ( ) , Info -> getStackPtrOffsetReg ( ) ) ) ; MRI . replaceRegWith ( <STR_LIT> ,


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case


<s> if ( GlueChain -> getNumValues ( ) <= <NUM_LIT> ) { return DAG . getNode ( Opcode , SL , VT , A , B ) ; } assert ( GlueChain -> getNumValues ( ) == <NUM_LIT> ) ; SDVTList VTList = DAG . getVTList ( VT , <STR_LIT> , <STR_LIT> ) ; switch ( Opcode ) { default : llvm_unreachable ( <STR_LIT>


<s> static SDValue getFPTernOp ( SelectionDAG & DAG , unsigned Opcode , const SDLoc & SL , EVT VT , SDValue A , SDValue B , SDValue C , SDValue GlueChain ) { if ( GlueChain -> getNumValues ( ) <= <NUM_LIT> ) { return DAG . getNode ( Opcode , SL , VT , A ,


<s> EVT VT = N0 -> getValueType ( <NUM_LIT> ) ; if ( ( VT == <STR_LIT> && ! Subtarget -> hasFP32Denormals ( ) ) || ( VT == <STR_LIT> && ! Subtarget -> hasFP16Denormals ( ) ) ) return <STR_LIT> ; const TargetOptions & Options = DAG . getTarget ( ) . Options ; if ( ( Options . AllowFPOpFusion == FPOpFusion :: Fast || Options . UnsafeFPMath || ( N0 -> getFlags ( ) . hasAllowContract ( ) && N1 -> getFlags ( ) . hasAllowContract ( ) ) ) && isFMAFasterThanFMulAndFAdd ( VT ) ) { return <STR_LIT> ; } return


<s> EVT VT = N0 -> getValueType ( <NUM_LIT> ) ; if ( ( VT == <STR_LIT> && ! Subtarget -> hasFP32Denormals ( ) ) || ( VT == <STR_LIT> && ! Subtarget -> hasFP16Denormals ( ) ) ) return <STR_LIT> ; const TargetOptions & Options = DAG . getTarget ( ) . Options ; if ( ( Options . AllowFPOpFusion == FPOpFusion :: Fast || Options . UnsafeFPMath || ( N0 -> getFlags ( ) . hasAllowContract ( ) && N1 -> getFlags ( ) . hasAllowContract ( ) ) ) && isFMAFasterThanFMulAndFAdd ( VT ) ) { return


<s> return lowerKernArgParameterPtr ( DAG , SL , DAG . getEntryNode (


<s> static SDValue getLoadExtOrTrunc ( SelectionDAG & DAG , <STR_LIT> ExtType , SDValue Op , const SDLoc & SL , EVT VT ) { if ( VT . bitsLT ( Op . getValueType ( ) ) ) return DAG . getNode ( <STR_LIT> , SL , VT , Op ) ; switch ( ExtType ) { case <STR_LIT>


<s> if ( VT . bitsLT ( Op . getValueType ( ) ) ) return DAG . getNode ( <STR_LIT> , SL , VT , Op ) ; switch ( ExtType ) { case <STR_LIT> : return DAG . getNode ( <STR_LIT> , SL , VT , Op ) ; case <STR_LIT> : return DAG . getNode ( <STR_LIT> , SL , VT , Op ) ; case


<s> SDVTList VTs = DAG . getVTList ( <STR_LIT> , <STR_LIT> ) ; SDValue Mad = DAG . getNode ( MadOpc , SL , VTs , N0 , N1 , N2 ) ; return DAG . getNode ( <STR_LIT> , SL , VT ,


<s> if ( Size == <NUM_LIT> ) return NumElts ; if ( Size == <NUM_LIT> ) return <NUM_LIT> * NumElts ; if ( Size == <NUM_LIT> && Subtarget -> has16BitInsts ( ) ) return ( VT . getVectorNumElements ( )


<s> if ( VT . getVectorNumElements ( ) != <NUM_LIT> && VT . getScalarType ( ) . bitsLE

( <STR_LIT> ) ) return TypeSplitVector ;


<s> SDValue <ISA_LIT>TargetLowering :: getPreloadedValue ( SelectionDAG & DAG , const <ISA_LIT>MachineFunctionInfo & MFI , EVT VT , <STR_LIT> PVID ) const { const ArgDescriptor * Reg ; const TargetRegisterClass * RC ; std :: tie ( Reg , RC ) = MFI . getPreloadedValue ( PVID ) ; return CreateLiveInRegister ( DAG , RC , Reg -> getRegister ( )


<s> const TargetRegisterClass * RC = nullptr ; if ( Constraint . size ( ) == <NUM_LIT> ) { switch ( Constraint [ <NUM_LIT> ] ) { default : return TargetLowering :: getRegForInlineAsmConstraint ( TRI , Constraint , VT ) ; case 's' : case 'r' : switch ( VT . getSizeInBits ( ) ) { default : return std :: make_pair ( <NUM_LIT> , nullptr ) ; case <NUM_LIT> : case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT> : RC = & <STR_LIT> ; break ; } break ; case 'v' : switch ( VT . getSizeInBits ( ) ) { default : return std :: make_pair ( <NUM_LIT> , nullptr ) ; case <NUM_LIT> : case <NUM_LIT> : RC = & <STR_LIT> ; break ; case <NUM_LIT>


<s> break ; } if ( RC && ( isTypeLegal ( VT ) || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> || VT . SimpleTy == <STR_LIT> ) ) return std :: make_pair ( <NUM_LIT> , RC ) ; } if ( Constraint . size ( ) > <NUM_LIT> ) { if ( Constraint [ <NUM_LIT> ] == 'v' ) { RC = & <STR_LIT> ; } else if ( Constraint [ <NUM_LIT> ] == 's' ) { RC = & <STR_LIT> ; } if ( RC ) { uint32_t Idx ; bool Failed = Constraint . substr ( <NUM_LIT> ) . getAsInteger ( <NUM_LIT> , Idx ) ; if ( ! Failed && Idx < RC -> getNumRegs ( ) ) return std

:: make_pair ( RC -> getRegister ( Idx ) , RC ) ;


<s> if ( Reg == <STR_LIT> ) { report_fatal_error ( Twine ( <STR_LIT> + StringRef ( RegName ) + <STR_LIT> ) ) ; } if ( Subtarget -> getGeneration ( ) == <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS && Subtarget -> getRegisterInfo ( ) -> regsOverlap ( Reg , <STR_LIT> ) ) { report_fatal_error ( Twine ( <STR_LIT> + StringRef ( RegName ) + <STR_LIT> ) ) ; } switch ( Reg ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT>


<s> unsigned Size = ScalarVT . getSizeInBits ( ) ; if ( Size == <NUM_LIT> ) return ScalarVT . getSimpleVT ( ) ; if ( Size == <NUM_LIT> ) return <STR_LIT> ; if ( Size == <NUM_LIT> && Subtarget -> has16BitInsts ( ) ) return VT . isInteger


<s> return ( VT == <STR_LIT> ) ? <STR_LIT>


<s> if ( Subtarget -> hasApertureRegs ( ) ) { unsigned Offset = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: OFFSET_SRC_SHARED_BASE : <STR_LIT> :: OFFSET_SRC_PRIVATE_BASE ; unsigned WidthM1 = AS == <ISA_LIT>AS :: LOCAL_ADDRESS ? <STR_LIT> :: WIDTH_M1_SRC_SHARED_BASE : <STR_LIT> :: WIDTH_M1_SRC_PRIVATE_BASE ; unsigned Encoding = <STR_LIT> :: ID_MEM_BASES << <STR_LIT> :: ID_SHIFT_ | Offset << <STR_LIT> :: OFFSET_SHIFT_ | WidthM1 << <STR_LIT> :: WIDTH_M1_SHIFT_ ; SDValue EncodingImm = DAG . getTargetConstant ( Encoding , DL , <STR_LIT> ) ; SDValue ApertureReg = SDValue ( DAG . getMachineNode ( <STR_LIT> , DL , <STR_LIT> , EncodingImm ) , <NUM_LIT> ) ; SDValue ShiftAmount = DAG . getTargetConstant ( WidthM1 + <NUM_LIT> , DL , <STR_LIT> ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , ApertureReg , ShiftAmount ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned UserSGPR = Info -> getQueuePtrUserSGPR


<s> if ( ConstantFPSDNode * C = dyn_cast < ConstantFPSDNode > ( Op ) ) return C ; if ( BuildVectorSDNode * BV = dyn_cast < BuildVectorSDNode > ( Op ) ) { if ( ConstantFPSDNode * C = BV -> getConstantFPSplatNode ( ) ) return C ; } return nullptr


<s> static ConstantFPSDNode * getSplatConstantFP ( SDValue Op ) { if ( ConstantFPSDNode * C = dyn_cast < ConstantFPSDNode


<s> const GCNSubtarget * <ISA_LIT>TargetLowering :: getSubtarget ( ) const { return Subtarget


<s> SDValue IntVData = DAG . getNode ( <STR_LIT> , DL , IntStoreVT , VData ) ; EVT EquivStoreVT = EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , StoreVT . getVectorNumElements ( ) ) ; SDValue ZExt = DAG . getNode ( <STR_LIT> , DL , EquivStoreVT , IntVData ) ; return DAG . UnrollVectorOp ( ZExt . getNode ( ) ) ; } assert ( isTypeLegal ( StoreVT )


<s> if ( Subtarget -> hasUnpackedD16VMem ( ) ) { EVT IntStoreVT = StoreVT . changeTypeToInteger ( ) ; SDValue IntVData = DAG . getNode ( <STR_LIT> , DL , IntStoreVT , VData ) ; EVT EquivStoreVT = EVT :: getVectorVT ( * DAG . getContext ( ) , <STR_LIT> , StoreVT . getVectorNumElements ( ) ) ; SDValue ZExt = DAG . getNode ( <STR_LIT> , DL , EquivStoreVT , IntVData ) ; return DAG . UnrollVectorOp ( ZExt


<s> bool <ISA_LIT>TargetLowering :: hasBitPreservingFPLogic ( EVT VT ) const { return isTypeLegal ( VT . getScalarType


<s> void <ISA_LIT>TargetLowering :: insertCopiesSplitCSR ( MachineBasicBlock * Entry , const SmallVectorImpl < MachineBasicBlock * > & Exits ) const { const <ISA_LIT>RegisterInfo * TRI = getSubtarget ( ) -> getRegisterInfo ( ) ; const MCPhysReg * IStart = TRI -> getCalleeSavedRegsViaCopy ( Entry -> getParent ( ) ) ; if ( ! IStart ) return ; const TargetInstrInfo * TII = Subtarget -> getInstrInfo ( ) ; MachineRegisterInfo * MRI = & Entry -> getParent ( ) -> getRegInfo ( ) ; MachineBasicBlock :: iterator MBBI = Entry


<s> switch ( V . getOpcode ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; } return


<s> unsigned Opcode = Op . getOpcode ( ) ; if ( Opcode == <STR_LIT> ) return true ; if ( auto * CFP = dyn_cast < ConstantFPSDNode > ( Op ) ) { auto F = CFP -> getValueAPF ( ) ; if ( F . isNaN ( ) && F . isSignaling ( ) ) return false ; return ! F . isDenormal ( ) || denormalsEnabledForType ( Op . getValueType ( ) ) ; } if ( MaxDepth == <NUM_LIT> ) return false ; switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return isCanonicalized ( DAG , Op . getOperand ( <NUM_LIT> ) , MaxDepth - <NUM_LIT> ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return Op . getValueType ( ) . getScalarType ( ) != <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT>


<s> if ( Intr -> getOpcode ( ) == <STR_LIT> ) { switch ( cast < ConstantSDNode > ( Intr -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) { case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : llvm_unreachable ( <STR_LIT> ) ; default : return <NUM_LIT> ; } } return <NUM_LIT>


<s> bool <ISA_LIT>TargetLowering :: isCheapAddrSpaceCast ( unsigned SrcAS , unsigned DestAS ) const { if ( SrcAS == <ISA_LIT>AS :: FLAT_ADDRESS )


<s> return ( CA -> isExactlyValue ( <NUM_LIT> ) && CB -> isExactlyValue ( <NUM_LIT> ) ) || ( CA

-> isExactlyValue ( <NUM_LIT> ) && CB -> isExactlyValue ( <NUM_LIT> ) ) ;


<s> LLVM_ATTRIBUTE_UNUSED static bool isCopyFromRegOfInlineAsm ( const SDNode * N ) { assert ( N -> getOpcode ( ) == <STR_LIT> ) ; do { N = N -> getOperand ( <NUM_LIT> ) . getNode ( ) ; if ( N -> getOpcode ( ) == <STR_LIT> ) return


<s> if ( ! mayTailCallThisCC ( CalleeCC ) ) return false ; MachineFunction & MF = DAG . getMachineFunction ( ) ; const Function & CallerF = MF . getFunction ( ) ; CallingConv :: ID CallerCC = CallerF . getCallingConv ( ) ; const <ISA_LIT>RegisterInfo * TRI = getSubtarget ( ) -> getRegisterInfo ( ) ; const uint32_t * CallerPreserved = TRI -> getCallPreservedMask ( MF , CallerCC ) ; if ( ! CallerPreserved ) return false ; bool CCMatch = CallerCC == CalleeCC ; if ( DAG . getTarget ( ) . Options . GuaranteedTailCallOpt ) { if ( canGuaranteeTCO ( CalleeCC ) && CCMatch ) return true ; return false ; } if ( IsVarArg ) return false ; for ( const Argument & Arg : CallerF . args ( ) ) { if ( Arg . hasByValAttr ( ) ) return false ; } LLVMContext & Ctx = * DAG


<s> switch ( VT . getSimpleVT ( ) . SimpleTy ) { case <STR_LIT> : { if ( Subtarget -> hasFP32Denormals ( ) ) return Subtarget -> hasFastFMAF32 ( ) || Subtarget -> hasDLInsts ( ) ; return Subtarget -> hasFastFMAF32 ( ) && Subtarget -> hasDLInsts ( ) ; } case <STR_LIT> : return


<s> return ( ( Opcode == <STR_LIT> && Subtarget -> hasMadMixInsts ( ) ) || ( Opcode == <STR_LIT> && Subtarget -> hasFmaMixInsts ( ) ) ) && DestVT . getScalarType ( ) == <STR_LIT>

&& ! Subtarget -> hasFP32Denormals ( ) && SrcVT . getScalarType ( ) == <STR_LIT> ;


<s> bool <ISA_LIT>TargetLowering :: isLegalGlobalAddressingMode ( const AddrMode & AM ) const { if ( Subtarget -> hasFlatGlobalInsts ( ) ) return isInt < <NUM_LIT> > ( AM . BaseOffs )


<s> const MemSDNode * MemNode = cast < MemSDNode > ( N ) ; const Value * Ptr = MemNode -> getMemOperand ( ) -> getValue


<s> bool <ISA_LIT>TargetLowering :: isMemOpHasNoClobberedMemOperand ( const SDNode * N ) const { const MemSDNode * MemNode = cast < MemSDNode > ( N ) ; const Value * Ptr = MemNode -> getMemOperand ( ) -> getValue ( ) ; const Instruction * I = dyn_cast_or_null < Instruction > ( Ptr ) ; return I && I -> getMetadata (


<s> const MemSDNode * MemNode = cast < MemSDNode >


<s> bool <ISA_LIT>TargetLowering :: isNoopAddrSpaceCast ( unsigned


<s> const <ISA_LIT>RegisterInfo & TRI = ST . getInstrInfo ( ) -> getRegisterInfo ( ) ; unsigned Reg = R -> getReg ( ) ; if ( TRI . isPhysicalRegister ( Reg ) ) return ! TRI . isSGPRReg ( MRI , Reg ) ; if ( MRI . isLiveIn ( Reg ) ) { if ( ! TRI . isSGPRReg ( MRI , Reg ) ) return true ; else if ( ! <STR_LIT> ( FLI -> Fn -> getCallingConv ( ) ) ) return true ; return false ; } const Value * V = FLI -> getValueFromVirtualReg ( Reg ) ; if ( V ) return KDA -> isDivergent ( V ) ; assert ( Reg == FLI -> DemoteRegister || isCopyFromRegOfInlineAsm ( N ) ) ; return ! TRI . isSGPRReg ( MRI , Reg ) ; } break ; case <STR_LIT> : { const LoadSDNode * L = cast < LoadSDNode > ( N ) ; unsigned AS = L -> getAddressSpace ( ) ; return AS == <ISA_LIT>AS :: PRIVATE_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ; } break ; case <STR_LIT> : return true ; break ; case <STR_LIT> : { } return <STR_LIT> ( cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; case <STR_LIT> : return <STR_LIT> ( cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; case <STR_LIT> : case <STR_LIT>


<s> const Value * V = FLI -> getValueFromVirtualReg ( Reg ) ; if ( V ) return KDA -> isDivergent ( V ) ; assert ( Reg == FLI -> DemoteRegister || isCopyFromRegOfInlineAsm ( N ) ) ; return ! TRI . isSGPRReg ( MRI , Reg ) ; } break ; case <STR_LIT> : { const LoadSDNode * L = cast < LoadSDNode > ( N ) ; unsigned AS = L -> getAddressSpace ( ) ; return AS == <ISA_LIT>AS :: PRIVATE_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ; } break ; case <STR_LIT> : return true ; break ; case <STR_LIT> : { } return <STR_LIT> ( cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; case <STR_LIT> : return <STR_LIT> ( cast < ConstantSDNode > ( N -> getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ) ; case <STR_LIT> : case <STR_LIT> : case


<s> bool <ISA_LIT>TargetLowering :: isShuffleMaskLegal ( ArrayRef < int > , EVT ) const { return


<s> return true ; default : return false ; } } if ( VT == <STR_LIT> && Op == <STR_LIT> ) return false ; return TargetLowering :: isTypeDesirableForOp (


<s> SDNode * Glued = Node -> getGluedNode ( ) ; SDValue ToVReg = DAG . getCopyToReg ( Node -> getOperand ( <NUM_LIT> ) , SL , VReg , SrcVal , SDValue ( Glued , Glued ? Glued -> getNumValues ( ) - <NUM_LIT> : <NUM_LIT> ) ) ; SDValue ToResultReg = DAG . getCopyToReg ( ToVReg , SL , SDValue ( DestReg , <NUM_LIT> ) , VReg , ToVReg . getValue ( <NUM_LIT> ) ) ; DAG . ReplaceAllUsesWith ( Node , ToResultReg . getNode ( ) ) ; DAG . RemoveDeadNode ( Node ) ; return ToResultReg . getNode ( ) ; } } SmallVector < SDValue , <NUM_LIT> > Ops ; for ( unsigned i = <NUM_LIT> ; i < Node -> getNumOperands ( ) ; ++ i ) { if ( ! isFrameIndexOp ( Node -> getOperand ( i ) ) ) { Ops . push_back ( Node -> getOperand ( i


<s> unsigned SaveExec = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned TmpExec = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , TmpExec ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SaveExec ) . addReg ( <STR_LIT> ) ; MachineBasicBlock * LoopBB = MF -> CreateMachineBasicBlock ( ) ; MachineBasicBlock * RemainderBB = MF -> CreateMachineBasicBlock ( ) ; MachineFunction :: iterator MBBI ( MBB ) ; ++ MBBI ; MF -> insert ( MBBI , LoopBB ) ; MF -> insert ( MBBI , RemainderBB ) ; LoopBB -> addSuccessor ( LoopBB ) ; LoopBB -> addSuccessor ( RemainderBB ) ; RemainderBB -> transferSuccessorsAndUpdatePHIs ( & MBB ) ; RemainderBB -> splice ( RemainderBB -> begin ( ) , & MBB , I , MBB . end ( ) ) ; MBB . addSuccessor ( LoopBB ) ; const MachineOperand * Idx = TII -> getNamedOperand ( MI , <STR_LIT> :: idx ) ; auto InsPt = emitLoadM0FromVGPRLoop ( TII , MRI , MBB , * LoopBB , DL , * Idx , InitResultReg , DstReg , PhiReg , TmpExec , Offset , UseGPRIdxMode , IsIndirectSrc ) ; MachineBasicBlock :: iterator First = RemainderBB -> begin ( ) ; BuildMI ( * RemainderBB , First , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( SaveExec ) ; return InsPt


<s> const <ISA_LIT>TargetMachine & TM = static_cast < const <ISA_LIT>TargetMachine & > ( getTargetMachine ( ) ) ; if ( ASC -> getSrcAddressSpace ( ) == <ISA_LIT>AS :: FLAT_ADDRESS ) { unsigned DestAS = ASC -> getDestAddressSpace ( ) ; if ( DestAS == <ISA_LIT>AS :: LOCAL_ADDRESS || DestAS == <ISA_LIT>AS :: PRIVATE_ADDRESS ) { unsigned NullVal = TM . getNullPointerValue ( DestAS ) ; SDValue SegmentNullPtr = DAG . getConstant ( NullVal , SL , <STR_LIT> ) ; SDValue NonNull = DAG . getSetCC ( SL , <STR_LIT> , Src , FlatNullPtr , <STR_LIT> ) ; SDValue Ptr = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , NonNull , Ptr , SegmentNullPtr ) ; } } if ( ASC -> getDestAddressSpace ( ) ==


<s> SDValue Addr = Op . getOperand ( <NUM_LIT> ) ; SDValue Old = Op . getOperand ( <NUM_LIT> ) ; SDValue New = Op . getOperand ( <NUM_LIT> ) ; EVT VT = Op . getValueType ( ) ; MVT SimpleVT = VT . getSimpleVT ( ) ; MVT VecType = <STR_LIT> (


<s> SetCC = Intr ; Intr = SetCC -> getOperand ( <NUM_LIT> ) . getNode ( ) ; } else { BR = findUser ( BRCOND , <STR_LIT> ) ; Target = BR -> getOperand ( <NUM_LIT> ) ; } unsigned CFNode = isCFIntrinsic ( Intr ) ; if ( CFNode == <NUM_LIT> ) { return BRCOND ; } bool HaveChain = Intr -> getOpcode ( ) == <STR_LIT> || Intr -> getOpcode ( ) == <STR_LIT> ; assert ( ! SetCC || ( SetCC -> getConstantOperandVal ( <NUM_LIT> ) == <NUM_LIT> && cast < CondCodeSDNode > ( SetCC -> getOperand ( <NUM_LIT> ) . getNode ( ) ) -> get ( ) == <STR_LIT> ) ) ; SmallVector < SDValue , <NUM_LIT> > Ops ; if ( HaveChain ) Ops . push_back ( BRCOND . getOperand ( <NUM_LIT> ) ) ; Ops . append ( Intr -> op_begin ( ) + ( HaveChain ? <NUM_LIT> : <NUM_LIT> ) , Intr -> op_end ( ) ) ; Ops . push_back ( Target ) ; ArrayRef < EVT > Res ( Intr -> value_begin ( ) + <NUM_LIT> , Intr -> value_end ( ) ) ; SDNode * Result = DAG . getNode ( CFNode , DL , DAG . getVTList ( Res ) , Ops ) . getNode ( ) ; if ( ! HaveChain ) { SDValue Ops [ ] = { SDValue ( Result , <NUM_LIT> ) , BRCOND . getOperand ( <NUM_LIT> ) } ; Result = DAG . getMergeValues ( Ops , DL ) . getNode ( ) ; } if ( BR


<s> assert ( VT == <STR_LIT> || VT == <STR_LIT> ) ; assert ( ! Subtarget -> hasVOP3PInsts ( ) && <STR_LIT> ) ; SDValue Lo = Op . getOperand ( <NUM_LIT> ) ; SDValue Hi = Op . getOperand ( <NUM_LIT> ) ; if ( Hi . isUndef ( ) ) { Lo = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Lo ) ; SDValue ExtLo = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Lo ) ; return DAG . getNode ( <STR_LIT> , SL , VT , ExtLo ) ; } Hi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Hi ) ; Hi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Hi ) ; SDValue ShlHi = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Hi , DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ) ; if ( Lo . isUndef ( ) ) return DAG . getNode ( <STR_LIT> , SL , VT , ShlHi ) ; Lo = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Lo ) ; Lo = DAG . getNode ( <STR_LIT> , SL ,


<s> } else if ( VA . isMemLoc ( ) ) { report_fatal_error ( <STR_LIT> ) ; } else llvm_unreachable ( <STR_LIT> ) ; switch ( VA . getLocInfo ( ) ) { case CCValAssign :: Full : break ; case CCValAssign :: BCvt : Val = DAG . getNode ( <STR_LIT> , DL , VA . getValVT ( ) , Val ) ; break ; case CCValAssign :: ZExt : Val = DAG . getNode ( <STR_LIT> , DL , VA . getLocVT ( ) , Val , DAG . getValueType ( VA . getValVT ( ) ) ) ; Val = DAG . getNode ( <STR_LIT> , DL , VA . getValVT ( ) , Val ) ; break ; case CCValAssign :: SExt : Val = DAG . getNode ( <STR_LIT> , DL , VA . getLocVT ( ) , Val , DAG . getValueType ( VA . getValVT ( ) ) ) ; Val = DAG . getNode ( <STR_LIT> , DL , VA . getValVT ( ) ,


<s> if ( Subtarget -> getTrapHandlerAbi ( ) != GCNSubtarget :: TrapHandlerAbiHsa || ! Subtarget -> isTrapHandlerEnabled ( ) ) { DiagnosticInfoUnsupported NoTrap ( MF . getFunction ( ) , <STR_LIT> , Op . getDebugLoc ( ) , DS_Warning ) ; LLVMContext & Ctx = MF . getFunction ( ) . getContext ( ) ; Ctx . diagnose ( NoTrap ) ; return Chain ; } SDValue Ops [


<s> SDValue <ISA_LIT>TargetLowering :: lowerFastUnsafeFDIV ( SDValue Op , SelectionDAG & DAG ) const { SDLoc SL ( Op ) ; SDValue LHS = Op . getOperand ( <NUM_LIT> ) ; SDValue RHS = Op . getOperand ( <NUM_LIT> ) ; EVT VT = Op . getValueType ( ) ; const SDNodeFlags Flags = Op -> getFlags ( ) ; bool Unsafe = DAG . getTarget ( ) . Options . UnsafeFPMath || Flags . hasAllowReciprocal ( ) ; if ( ! Unsafe && VT == <STR_LIT> && Subtarget -> hasFP32Denormals ( ) ) return SDValue ( ) ; if ( const ConstantFPSDNode * CLHS = dyn_cast < ConstantFPSDNode > ( LHS ) ) { if ( Unsafe || VT == <STR_LIT> || VT == <STR_LIT> ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { if ( RHS . getOpcode ( ) == <STR_LIT> ) return DAG . getNode

( <STR_LIT> , SL , VT , RHS . getOperand ( <NUM_LIT> ) ) ;


<s> if ( Unsafe || VT == <STR_LIT> || VT == <STR_LIT> ) { if ( CLHS -> isExactlyValue ( <NUM_LIT> ) ) { if ( RHS . getOpcode ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , SL , VT , RHS . getOperand ( <NUM_LIT> ) ) ; return DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; } if ( CLHS -> isExactlyValue ( - <NUM_LIT> ) ) { SDValue FNegRHS = DAG . getNode ( <STR_LIT> , SL , VT , RHS ) ; return DAG . getNode ( <STR_LIT> , SL , VT , FNegRHS ) ; } } } if ( Unsafe


<s> if ( VT == <STR_LIT> ) return LowerFDIV64 ( Op , DAG ) ; if ( VT == <STR_LIT> ) return LowerFDIV16 ( Op ,


<s> SDValue Src1 = Op . getOperand ( <NUM_LIT> ) ; SDValue CvtSrc0 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src0 ) ; SDValue CvtSrc1 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Src1 ) ; SDValue RcpSrc1 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , CvtSrc1 ) ; SDValue Quot = DAG . getNode ( <STR_LIT> , SL


<s> SDLoc SL ( Op ) ; SDValue X = Op . getOperand ( <NUM_LIT> ) ; SDValue Y = Op . getOperand ( <NUM_LIT> ) ; const SDValue One = DAG . getConstantFP ( <NUM_LIT> , SL , <STR_LIT> ) ; SDVTList ScaleVT = DAG . getVTList ( <STR_LIT> , <STR_LIT> ) ; SDValue DivScale0 = DAG . getNode ( <STR_LIT> , SL , ScaleVT , Y , Y , X ) ; SDValue NegDivScale0 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , DivScale0 ) ; SDValue Rcp = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , DivScale0 ) ; SDValue Fma0 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , NegDivScale0 , Rcp , One ) ; SDValue Fma1 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Rcp , Fma0 , Rcp ) ; SDValue Fma2 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , NegDivScale0 , Fma1 , One ) ; SDValue DivScale1 = DAG . getNode ( <STR_LIT> , SL , ScaleVT , X , Y , X ) ; SDValue Fma3 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Fma1


<s> SDLoc SL ( Op ) ; SDValue LHS = Op . getOperand ( <NUM_LIT> ) ; SDValue RHS = Op . getOperand ( <NUM_LIT> ) ; SDValue r1 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , RHS ) ; const APFloat K0Val ( BitsToFloat ( <NUM_LIT> ) ) ; const SDValue K0 = DAG . getConstantFP ( K0Val , SL , <STR_LIT> ) ; const APFloat K1Val ( BitsToFloat ( <NUM_LIT> ) ) ; const SDValue K1 = DAG . getConstantFP ( K1Val , SL , <STR_LIT> ) ; const SDValue One = DAG . getConstantFP ( <NUM_LIT> , SL , <STR_LIT> ) ; EVT SetCCVT = getSetCCResultType ( DAG . getDataLayout ( ) , * DAG . getContext ( ) , <STR_LIT> ) ; SDValue r2 = DAG . getSetCC ( SL , SetCCVT , r1 , K0 , <STR_LIT> ) ; SDValue r3 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , r2 , K1 , One ) ; r1 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , RHS , r3 ) ; SDValue r0 = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , r1 ) ; SDValue Mul = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LHS , r0 ) ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT>


<s> if ( IsIEEEMode ) return expandFMINNUM_FMAXNUM ( Op . getNode ( ) , DAG ) ; if ( VT == <STR_LIT> ) return splitBinaryVectorOp ( Op , DAG ) ; return Op


<s> SDLoc DL ( Op ) ; SDValue FpToFp16 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Src ) ; SDValue Trunc = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , FpToFp16 ) ; return DAG . getNode ( <STR_LIT> , DL , <STR_LIT>


<s> if ( GSD -> getAddressSpace ( ) == <ISA_LIT>AS :: LOCAL_ADDRESS || GSD -> getAddressSpace ( ) == <ISA_LIT>AS :: REGION_ADDRESS || GSD -> getAddressSpace ( ) == <ISA_LIT>AS :: PRIVATE_ADDRESS ) return <ISA_LIT>TargetLowering :: LowerGlobalAddress ( MFI , Op , DAG ) ; SDLoc DL ( GSD ) ; EVT PtrVT = Op . getValueType ( ) ; if ( shouldEmitFixup ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT ) ; else if ( shouldEmitPCReloc ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT , <ISA_LIT>InstrInfo :: MO_REL32 ) ; SDValue GOTAddr = buildPCRelGlobalAddress ( DAG , GV , DL , <NUM_LIT> , PtrVT , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; Type * Ty = PtrVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; const DataLayout & DataLayout = DAG .


<s> else if ( shouldEmitPCReloc ( GV ) ) return buildPCRelGlobalAddress ( DAG , GV , DL , GSD -> getOffset ( ) , PtrVT , <ISA_LIT>InstrInfo :: MO_REL32 ) ; SDValue GOTAddr = buildPCRelGlobalAddress ( DAG , GV , DL , <NUM_LIT> , PtrVT , <ISA_LIT>InstrInfo :: MO_GOTPCREL32 ) ; Type * Ty = PtrVT . getTypeForEVT ( * DAG . getContext ( ) ) ; PointerType * PtrTy = PointerType :: get ( Ty , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; const DataLayout & DataLayout = DAG . getDataLayout ( ) ; unsigned Align = DataLayout . getABITypeAlignment (


<s> if ( CondCode < ICmpInst :: Predicate :: FIRST_ICMP_PREDICATE || CondCode > ICmpInst :: Predicate :: LAST_ICMP_PREDICATE ) return DAG . getUNDEF ( VT ) ; ICmpInst :: Predicate IcInput = static_cast < ICmpInst :: Predicate > ( CondCode ) ; SDValue LHS = N -> getOperand ( <NUM_LIT> ) ; SDValue RHS = N -> getOperand ( <NUM_LIT> ) ; SDLoc DL ( N ) ; EVT CmpVT = LHS . getValueType ( ) ; if ( CmpVT == <STR_LIT> && ! TLI . isTypeLegal ( <STR_LIT> ) ) { unsigned PromoteOp = ICmpInst :: isSigned ( IcInput ) ? <STR_LIT> : <STR_LIT> ; LHS = DAG . getNode ( PromoteOp , DL , <STR_LIT> , LHS ) ; RHS = DAG . getNode ( PromoteOp , DL , <STR_LIT> , RHS ) ; } <STR_LIT> CCOpcode = getICmpCondCode (


<s> SDValue Param = lowerKernargMemParameter ( DAG , <STR_LIT> , <STR_LIT> , SL , DAG . getEntryNode ( ) ,


<s> SDValue Idx = Op . getOperand ( <NUM_LIT> ) ; EVT VecVT = Vec . getValueType ( ) ; EVT EltVT = VecVT . getVectorElementType ( ) ; unsigned VecSize = VecVT . getSizeInBits ( ) ; unsigned EltSize = EltVT . getSizeInBits ( ) ; assert ( VecSize <= <NUM_LIT> ) ; unsigned NumElts = VecVT . getVectorNumElements ( ) ; SDLoc SL ( Op ) ; auto KIdx = dyn_cast < ConstantSDNode > ( Idx ) ; if ( NumElts == <NUM_LIT> && EltSize == <NUM_LIT> && KIdx ) { SDValue BCVec = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Vec ) ; SDValue LoHalf = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , BCVec , DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ) ; SDValue HiHalf = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , BCVec , DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) ) ; SDValue LoVec = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , LoHalf ) ; SDValue HiVec = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , HiHalf ) ; unsigned Idx = KIdx -> getZExtValue ( ) ; bool InsertLo = Idx < <NUM_LIT> ; SDValue InsHalf = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , InsertLo ? LoVec : HiVec , DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , InsVal ) , DAG . getConstant ( InsertLo ? Idx : ( Idx - <NUM_LIT> ) , SL , <STR_LIT> ) ) ; InsHalf = DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , InsHalf ) ; SDValue Concat = InsertLo ? DAG . getBuildVector ( <STR_LIT> , SL , { InsHalf , HiHalf } ) : DAG . getBuildVector ( <STR_LIT> , SL , { LoHalf , InsHalf } ) ; return DAG . getNode ( <STR_LIT> , SL , VecVT , Concat ) ; } if ( isa < ConstantSDNode > ( Idx ) ) return SDValue (


<s> const DataLayout & DL = DAG . getDataLayout ( ) ; MachineFunction & MF = DAG . getMachineFunction ( ) ; const <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const ArgDescriptor * InputPtrReg ; const TargetRegisterClass * RC ; std :: tie ( InputPtrReg , RC ) = Info -> getPreloadedValue ( <STR_LIT> ) ; MachineRegisterInfo & MRI = DAG . getMachineFunction ( ) . getRegInfo ( ) ; MVT PtrVT = getPointerTy ( DL , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; SDValue BasePtr = DAG . getCopyFromReg ( Chain , SL , MRI . getLiveInVirtReg ( InputPtrReg -> getRegister ( ) ) , PtrVT ) ; return DAG . getObjectPtrOffset ( SL , BasePtr


<s> LoadSDNode * Load = cast < LoadSDNode > ( Op ) ; <STR_LIT> ExtType = Load -> getExtensionType ( ) ; EVT MemVT = Load -> getMemoryVT ( ) ; if ( ExtType == <STR_LIT> && MemVT . getSizeInBits ( ) < <NUM_LIT> ) { if ( MemVT == <STR_LIT> && isTypeLegal ( <STR_LIT> ) ) return SDValue ( ) ; SDValue Chain = Load -> getChain ( ) ; SDValue BasePtr = Load -> getBasePtr ( ) ; MachineMemOperand * MMO = Load -> getMemOperand ( ) ; EVT RealMemVT = ( MemVT == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; SDValue NewLD = DAG . getExtLoad ( <STR_LIT> , DL , <STR_LIT> , Chain , BasePtr , RealMemVT , MMO ) ; SDValue Ops [ ] = { DAG . getNode ( <STR_LIT> , DL , MemVT , NewLD ) , NewLD . getValue ( <NUM_LIT> ) } ; return DAG . getMergeValues ( Ops , DL ) ; } if ( ! MemVT . isVector ( ) ) return SDValue ( ) ; assert ( Op . getValueType ( ) . getVectorElementType ( ) == <STR_LIT> && <STR_LIT> ) ; unsigned Alignment = Load -> getAlignment ( ) ; unsigned AS = Load -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , MemVT , AS , Alignment ) ) { SDValue Ops [ <NUM_LIT> ] ; std :: tie ( Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) = expandUnalignedLoad ( Load , DAG ) ; return DAG . getMergeValues ( Ops , DL ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = MemVT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT ) { if ( ! Op -> isDivergent ( ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) return SDValue ( ) ; } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS ) { if ( Subtarget -> getScalarizeGlobalBehavior ( ) && ! Op -> isDivergent ( ) && ! Load -> isVolatile ( ) && isMemOpHasNoClobberedMemOperand ( Load ) && Alignment >= <NUM_LIT> && NumElements < <NUM_LIT> ) return SDValue ( ) ; } if ( AS == <ISA_LIT>AS :: CONSTANT_ADDRESS || AS == <ISA_LIT>AS :: CONSTANT_ADDRESS_32BIT || AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorLoad ( Op , DAG ) ; return SDValue ( ) ; } if ( AS ==


<s> Arg = DAG . getNode ( <STR_LIT> , DL , VA . getLocVT ( ) , Arg ) ; break ; case CCValAssign :: AExt : Arg = DAG . getNode ( <STR_LIT> , DL , VA . getLocVT ( ) , Arg ) ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } Chain = DAG . getCopyToReg ( Chain , DL , VA . getLocReg ( ) , Arg , Flag ) ; Flag = Chain . getValue ( <NUM_LIT> ) ; RetOps . push_back ( DAG . getRegister ( VA . getLocReg ( ) , VA . getLocVT ( ) ) ) ; } if ( ! Info -> isEntryFunction ( ) ) { const <ISA_LIT>RegisterInfo * TRI = Subtarget -> getRegisterInfo ( ) ; const MCPhysReg * I = TRI -> getCalleeSavedRegsViaCopy ( & DAG . getMachineFunction ( ) ) ; if ( I ) { for ( ; * I ; ++ I ) { if ( <STR_LIT> . contains ( * I ) ) RetOps . push_back ( DAG . getRegister ( * I , <STR_LIT> ) ) ; else if ( <STR_LIT> . contains ( * I ) ) RetOps . push_back ( DAG . getRegister ( * I , <STR_LIT> ) ) ; else llvm_unreachable ( <STR_LIT> ) ; } } } RetOps [ <NUM_LIT> ] = Chain ; if ( Flag . getNode ( ) ) RetOps . push_back ( Flag ) ; unsigned Opc = <STR_LIT>


<s> SDValue Lo = DAG . getSelect ( DL , <STR_LIT> , Cond , Lo0 , Lo1 ) ; SDValue Hi0 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , LHS , One ) ; SDValue Hi1 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , RHS , One ) ; SDValue Hi = DAG . getSelect ( DL , <STR_LIT> , Cond , Hi0 , Hi1 ) ; SDValue Res = DAG . getBuildVector ( <STR_LIT> , DL , { Lo , Hi


<s> unsigned AS = Store -> getAddressSpace ( ) ; if ( ! allowsMemoryAccess ( * DAG . getContext ( ) , DAG . getDataLayout ( ) , VT , AS , Store -> getAlignment ( ) ) ) { return expandUnalignedStore ( Store , DAG ) ; } MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( AS == <ISA_LIT>AS :: FLAT_ADDRESS ) AS = MFI -> hasFlatScratchInit ( ) ? <ISA_LIT>AS :: PRIVATE_ADDRESS : <ISA_LIT>AS :: GLOBAL_ADDRESS ; unsigned NumElements = VT . getVectorNumElements ( ) ; if ( AS == <ISA_LIT>AS :: GLOBAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) { if ( NumElements > <NUM_LIT> ) return SplitVectorStore ( Op , DAG ) ; return SDValue ( ) ; } else if ( AS == <ISA_LIT>AS


<s> unsigned UserSGPR = Info -> getQueuePtrUserSGPR ( ) ; assert ( UserSGPR != <STR_LIT> ) ; SDValue QueuePtr = CreateLiveInRegister ( DAG , & <STR_LIT> , UserSGPR , <STR_LIT> ) ; SDValue SGPR01 = DAG . getRegister ( <STR_LIT> , <STR_LIT> ) ; SDValue ToReg = DAG . getCopyToReg ( Chain , SL , SGPR01 , QueuePtr , SDValue ( ) ) ; SDValue Ops [ ] = { ToReg , DAG . getTargetConstant ( GCNSubtarget :: TrapIDLLVMTrap , SL , <STR_LIT> ) , SGPR01 , ToReg . getValue ( <NUM_LIT> ) } ; return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> ,


<s> if ( Subtarget -> getTrapHandlerAbi ( ) != GCNSubtarget :: TrapHandlerAbiHsa || ! Subtarget -> isTrapHandlerEnabled ( ) ) return DAG . getNode ( <STR_LIT> , SL , <STR_LIT> , Chain ) ; MachineFunction & MF = DAG . getMachineFunction ( ) ; <ISA_LIT>MachineFunctionInfo * Info = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned UserSGPR = Info -> getQueuePtrUserSGPR (


<s> if ( <STR_LIT> ( ParentFn -> getCallingConv ( ) ) ) return false ; auto Attr = ParentFn -> getFnAttribute


<s> bool <ISA_LIT>TargetLowering :: mayBeEmittedAsTailCall ( const CallInst * CI ) const { if ( ! CI -> isTailCall ( ) ) return false ; const Function * ParentFn = CI -> getParent ( ) -> getParent ( ) ; if ( <STR_LIT> ( ParentFn -> getCallingConv ( ) ) ) return false ; auto Attr = ParentFn -> getFnAttribute ( <STR_LIT> ) ; return ( Attr . getValueAsString (


<s> static bool mayTailCallThisCC ( CallingConv :: ID CC ) { switch ( CC )


<s> SDLoc DL ( CachePolicy ) ; if ( GLC ) { * GLC = DAG . getTargetConstant ( ( Value & <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> , DL , <STR_LIT> ) ; Value &= ~ ( uint64_t ) <NUM_LIT> ; } if ( SLC ) { * SLC = DAG . getTargetConstant ( ( Value & <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> , DL , <STR_LIT> ) ; Value &= ~ ( uint64_t


<s> const <ISA_LIT>RegisterInfo * TRI = Subtarget -> getRegisterInfo ( ) ; auto & ArgUsageInfo = DAG . getPass ( ) -> getAnalysis < <ISA_LIT>ArgumentUsageInfo > ( ) ; const <ISA_LIT>FunctionArgInfo & CalleeArgInfo = ArgUsageInfo . lookupFuncArgInfo ( * CalleeFunc ) ; const <ISA_LIT>FunctionArgInfo & CallerArgInfo = Info . getArgInfo ( ) ; <STR_LIT> InputRegs [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; for ( auto InputID : InputRegs ) { const ArgDescriptor * OutgoingArg ; const TargetRegisterClass * ArgRC ; std :: tie ( OutgoingArg , ArgRC ) = CalleeArgInfo . getPreloadedValue ( InputID ) ; if ( ! OutgoingArg ) continue ; const ArgDescriptor * IncomingArg ; const TargetRegisterClass * IncomingArgRC ; std :: tie ( IncomingArg , IncomingArgRC ) = CallerArgInfo . getPreloadedValue ( InputID ) ; assert ( IncomingArgRC == ArgRC ) ; EVT ArgVT = TRI -> getSpillSize ( * ArgRC ) == <NUM_LIT> ? <STR_LIT> : <STR_LIT> ; SDValue InputReg ; if ( IncomingArg ) { InputReg = loadInputValue ( DAG , ArgRC , ArgVT , DL , * IncomingArg ) ; } else { assert ( InputID == <STR_LIT> ) ; InputReg = getImplicitArgPtr ( DAG , DL ) ; } if ( OutgoingArg -> isRegister ( ) ) { RegsToPass . emplace_back ( OutgoingArg -> getRegister ( ) , InputReg ) ; } else { unsigned SpecialArgOffset = CCInfo . AllocateStack ( ArgVT . getStoreSize ( ) , <NUM_LIT> ) ; SDValue ArgStore = storeStackInputValue ( DAG , DL , Chain ,


<s> return getMad64_32 ( DAG , SL , VT , MulLHS , MulRHS , AddRHS , true ) ; } return SDValue ( ) ; } if ( VT != <STR_LIT> || ! DCI . isAfterLegalizeDAG ( ) ) return SDValue ( ) ; unsigned Opc = LHS . getOpcode ( ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) std :: swap ( RHS , LHS ) ; Opc = RHS . getOpcode ( ) ; switch ( Opc ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { auto Cond = RHS . getOperand ( <NUM_LIT> ) ; if ( ! isBoolSGPR ( Cond ) ) break ; SDVTList VTList = DAG . getVTList ( <STR_LIT> , <STR_LIT> ) ; SDValue Args [ ] = { LHS , DAG . getConstant ( <NUM_LIT> , SL , <STR_LIT> ) , Cond } ; Opc = ( Opc == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; return DAG . getNode ( Opc , SL , VTList , Args ) ; } case <STR_LIT> : { auto C = dyn_cast < ConstantSDNode > (


<s> APFloat Zero = APFloat :: getZero ( F . getSemantics ( ) ) ; APFloat :: cmpResult Cmp0 = F . compare ( Zero ) ; if ( Cmp0 == APFloat :: cmpLessThan || ( Cmp0 == APFloat :: cmpUnordered && Subtarget -> enableDX10Clamp ( ) ) ) { return DCI . DAG . getConstantFP ( Zero , SDLoc ( N ) , N -> getValueType ( <NUM_LIT> ) ) ; } APFloat One ( F . getSemantics ( ) , <STR_LIT> ) ; APFloat :: cmpResult Cmp1 = F . compare ( One ) ; if ( Cmp1 == APFloat :: cmpGreaterThan ) return DCI . DAG . getConstantFP ( One , SDLoc ( N


<s> APFloat Zero = APFloat :: getZero ( F . getSemantics ( ) ) ; APFloat :: cmpResult Cmp0 = F . compare ( Zero ) ; if ( Cmp0 == APFloat :: cmpLessThan || ( Cmp0 == APFloat :: cmpUnordered && Subtarget -> enableDX10Clamp ( ) ) ) { return DCI . DAG . getConstantFP ( Zero , SDLoc ( N ) , N -> getValueType ( <NUM_LIT> ) ) ; } APFloat One ( F . getSemantics (


<s> SelectionDAG & DAG = DCI . DAG ; SDValue Mask = N -> getOperand ( <NUM_LIT> ) ; if ( const ConstantSDNode * CMask = dyn_cast < ConstantSDNode


<s> if ( Srl . getOpcode ( ) == <STR_LIT> ) { if ( const ConstantSDNode * C = dyn_cast < ConstantSDNode > ( Srl . getOperand ( <NUM_LIT> ) ) ) { Srl = DAG . getZExtOrTrunc ( Srl . getOperand ( <NUM_LIT> ) , SDLoc ( Srl . getOperand ( <NUM_LIT> ) ) , EVT ( <STR_LIT> ) ) ; unsigned SrcOffset = C -> getZExtValue ( ) + <NUM_LIT> * Offset ; if ( SrcOffset < <NUM_LIT> && SrcOffset % <NUM_LIT> == <NUM_LIT> ) { return DAG . getNode ( <STR_LIT> + SrcOffset / <NUM_LIT> , SL , <STR_LIT> , Srl ) ; } } } APInt Demanded = APInt :: getBitsSet ( <NUM_LIT> , <NUM_LIT> * Offset , <NUM_LIT> * Offset + <NUM_LIT> ) ; KnownBits Known ; TargetLowering :: TargetLoweringOpt TLO ( DAG , ! DCI . isBeforeLegalize ( ) , ! DCI . isBeforeLegalizeOps ( ) ) ; const TargetLowering & TLI = DAG . getTargetLoweringInfo ( ) ; if ( TLI . ShrinkDemandedConstant ( Src , Demanded , TLO ) || TLI . SimplifyDemandedBits ( Src ,


<s> SDValue Src1 = N -> getOperand ( <NUM_LIT> ) ; if ( Src0 . isUndef ( ) && Src1 . isUndef ( ) ) return DCI . DAG . getUNDEF ( N -> getValueType ( <NUM_LIT> ) ) ; return SDValue ( )


<s> return performSetCCCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return performMinMaxCombine ( N , DCI ) ; case <STR_LIT> : return performFMACombine ( N , DCI ) ; case <STR_LIT> : { if ( SDValue Widended = widenLoad ( cast < LoadSDNode > ( N ) , DCI ) ) return Widended ; LLVM_FALLTHROUGH ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( DCI . isBeforeLegalize ( ) ) break ; return performMemSDNodeCombine ( cast < MemSDNode > ( N ) , DCI ) ; case <STR_LIT> : return performAndCombine ( N , DCI ) ; case <STR_LIT> : return performOrCombine ( N , DCI ) ; case <STR_LIT> : return performXorCombine ( N , DCI ) ; case <STR_LIT> : return performZeroExtendCombine ( N , DCI ) ; case <STR_LIT> : return performClassCombine ( N , DCI ) ; case <STR_LIT> : return performFCanonicalizeCombine ( N , DCI ) ; case <STR_LIT> : return performRcpCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { SDValue Src = N -> getOperand ( <NUM_LIT> ) ; if ( Src . isUndef ( ) ) return Src ; break ; } case <STR_LIT> : case <STR_LIT> : return performUCharToFloatCombine ( N , DCI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return performCvtF32UByteNCombine ( N , DCI


<s> SDValue N0 = N -> getOperand ( <NUM_LIT> ) ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; if ( N0 . isUndef ( ) ) { APFloat QNaN = APFloat :: getQNaN ( SelectionDAG :: EVTToAPFloatSemantics ( VT ) ) ; return DAG . getConstantFP ( QNaN , SDLoc ( N ) , VT ) ; } if ( ConstantFPSDNode * CFP = isConstOrConstSplatFP ( N0 ) ) { EVT VT = N -> getValueType ( <NUM_LIT> ) ; return getCanonicalConstantFP ( DAG , SDLoc ( N ) , VT , CFP -> getValueAPF ( ) ) ; } if ( N0 . getOpcode ( ) == <STR_LIT> && VT == <STR_LIT> && isTypeLegal ( <STR_LIT> ) ) { SDLoc SL ( N ) ; SDValue NewElts [ <NUM_LIT> ] ; SDValue Lo = N0 . getOperand ( <NUM_LIT> ) ; SDValue Hi = N0 . getOperand ( <NUM_LIT> ) ; EVT EltVT = Lo . getValueType ( ) ; if ( vectorEltWillFoldAway ( Lo ) || vectorEltWillFoldAway ( Hi ) ) { for ( unsigned I = <NUM_LIT> ; I != <NUM_LIT> ; ++ I ) { SDValue Op = N0 . getOperand ( I ) ; if ( ConstantFPSDNode * CFP = dyn_cast < ConstantFPSDNode > (


<s> SDValue <ISA_LIT>TargetLowering :: performFMACombine ( SDNode * N , DAGCombinerInfo & DCI ) const { SelectionDAG & DAG = DCI . DAG ; EVT VT = N -> getValueType ( <NUM_LIT> ) ; SDLoc SL ( N ) ; if ( ! Subtarget -> hasDLInsts ( ) || VT != <STR_LIT> ) return SDValue ( ) ; SDValue Op1 = N -> getOperand ( <NUM_LIT> ) ; SDValue Op2 = N -> getOperand ( <NUM_LIT> ) ; SDValue FMA = N -> getOperand ( <NUM_LIT> ) ; if ( FMA . getOpcode ( ) != <STR_LIT> || Op1 . getOpcode ( ) != <STR_LIT> || Op2 . getOpcode ( ) != <STR_LIT> ) return SDValue ( ) ; const TargetOptions & Options = DAG . getTarget ( ) . Options ; if ( Options . AllowFPOpFusion == FPOpFusion :: Fast || Options . UnsafeFPMath || ( N -> getFlags ( ) . hasAllowContract ( ) && FMA -> getFlags ( ) . hasAllowContract ( ) ) ) { Op1 = Op1 . getOperand ( <NUM_LIT> ) ; Op2 = Op2 . getOperand


<s> SDValue Src1 = N -> getOperand ( <NUM_LIT> ) ; SDValue Src2 = N -> getOperand ( <NUM_LIT> ) ; if ( isClampZeroToOne ( Src0 , Src1 ) ) { return DAG . getNode ( <STR_LIT> , SL , VT , Src2 ) ; } if ( Subtarget -> enableDX10Clamp ( ) ) { if ( isa < ConstantFPSDNode > ( Src0 ) && ! isa < ConstantFPSDNode > ( Src1 ) ) std :: swap ( Src0


<s> APFloat :: cmpResult Cmp = K0 -> getValueAPF ( ) . compare ( K1 -> getValueAPF ( ) ) ; if ( Cmp == APFloat :: cmpGreaterThan ) return SDValue ( ) ; EVT VT = Op0 . getValueType ( ) ; if ( Subtarget -> enableDX10Clamp ( ) ) { if ( K1 -> isExactlyValue ( <NUM_LIT> ) && K0 -> isExactlyValue ( <NUM_LIT> ) ) return DAG . getNode ( <STR_LIT> , SL , VT , Op0 . getOperand ( <NUM_LIT> ) ) ; } if ( VT == <STR_LIT> || ( VT == <STR_LIT>


<s> if ( ! K0 ) return SDValue ( ) ; APFloat :: cmpResult Cmp = K0 -> getValueAPF ( ) . compare ( K1 -> getValueAPF ( ) ) ; if ( Cmp == APFloat :: cmpGreaterThan ) return SDValue ( ) ; EVT VT = Op0 . getValueType ( ) ; if ( Subtarget -> enableDX10Clamp ( ) ) { if ( K1 -> isExactlyValue ( <NUM_LIT> ) && K0 -> isExactlyValue ( <NUM_LIT> ) ) return DAG . getNode ( <STR_LIT> , SL , VT , Op0 . getOperand ( <NUM_LIT> ) ) ; } if ( VT == <STR_LIT> || ( VT == <STR_LIT> && Subtarget -> hasMed3_16 ( ) ) ) { SDValue Var = Op0 . getOperand ( <NUM_LIT> ) ; if ( ! DAG . isKnownNeverSNaN ( Var ) ) return SDValue ( ) ; const <ISA_LIT>InstrInfo * TII = getSubtarget ( ) -> getInstrInfo ( ) ; if ( ( ! K0 -> hasOneUse ( ) || TII -> isInlineConstant ( K0 -> getValueAPF ( ) . bitcastToAPInt ( ) ) ) && ( ! K1 -> hasOneUse ( ) || TII -> isInlineConstant ( K1 -> getValueAPF ( ) . bitcastToAPInt ( ) ) ) ) { return DAG . getNode ( <STR_LIT> , SL , K0 -> getValueType ( <NUM_LIT> ) , Var

, SDValue ( K0 , <NUM_LIT> ) , SDValue ( K1 , <NUM_LIT> ) ) ;