<s> unsigned NumOps = MI . getNumOperands ( ) ; for ( unsigned I = <NUM_LIT> ; I < NumOps ; ++ I ) { MachineOperand & Op = MI . getOperand ( I ) ; if ( Op . isReg ( ) && Op . isKill ( ) ) LV -> replaceKillInstruction ( Op . getReg ( ) , MI , NewMI ) ;


<s> static void updateLiveVariables ( LiveVariables * LV , MachineInstr & MI , MachineInstr & NewMI ) { if ( LV ) { unsigned NumOps = MI . getNumOperands ( ) ; for ( unsigned I = <NUM_LIT> ; I < NumOps ; ++ I ) {

MachineOperand & Op = MI . getOperand ( I ) ;


<s> bool <ISA_LIT>InstrInfo :: usesConstantBus ( const MachineRegisterInfo & MRI , const MachineOperand & MO , const MCOperandInfo & OpInfo ) const { if ( MO . isImm ( ) ) return ! isInlineConstant ( MO , OpInfo ) ; if ( ! MO . isReg ( ) ) return true ; if ( ! MO . isUse ( ) ) return false ; if ( MO . getReg ( ) . isVirtual ( ) ) return RI . isSGPRClass ( MRI . getRegClass ( MO . getReg ( ) ) ) ; if ( MO . getReg ( ) == <STR_LIT> ) return false ; if ( MO . isImplicit ( ) ) {

return MO . getReg ( ) == <STR_LIT> || MO . getReg ( ) == <STR_LIT> || MO . getReg ( ) == <STR_LIT> ;


<s> if ( ! MO . isReg ( ) ) return true ; if ( ! MO . isUse ( ) ) return false ; if ( MO . getReg ( ) . isVirtual ( ) ) return RI . isSGPRClass ( MRI . getRegClass ( MO . getReg ( ) ) ) ;

if ( MO . getReg ( ) == <STR_LIT> ) return false ;


<s> static unsigned adjustAllocatableRegClass ( const GCNSubtarget & ST , const MachineRegisterInfo & MRI , const MCInstrDesc & TID , unsigned RCID , bool IsAllocatable ) { if ( ( IsAllocatable || ! ST . hasGFX90AInsts ( ) || ! MRI . reservedRegsFrozen ( ) ) && ( TID . mayLoad ( ) || TID . mayStore ( ) || ( TID . TSFlags & ( <STR_LIT> | <STR_LIT> ) ) ) ) { switch ( RCID ) { case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; default : break ; } } return RCID ;


<s> if ( Src2 ) { switch ( MI . getOpcode ( ) ) { default : return false ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( ! Src1 -> isReg ( ) || ! RI . isVGPR ( MRI , Src1 -> getReg ( ) ) ) return false ; return true ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( ! Src2 -> isReg ( ) || ! RI . isVGPR ( MRI , Src2 -> getReg ( ) ) || hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ; break ; case <STR_LIT> : break ; } } const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 && ( ! Src1 -> isReg ( ) || ! RI . isVGPR ( MRI , Src1 -> getReg ( ) ) || hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) ) ) return false ;

if ( hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) ) return false ;


<s> const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mods = getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; const MachineOperand * Src2 = getNamedOperand ( MI , <STR_LIT> :: src2 ) ; const MachineOperand * Clamp = getNamedOperand ( MI , <STR_LIT> :: clamp ) ; const MachineOperand * Omod = getNamedOperand ( MI , <STR_LIT> :: omod ) ; MachineInstrBuilder MIB ; MachineBasicBlock & MBB = * MI . getParent ( ) ; if ( ! Src0Mods && ! Src1Mods && ! Clamp && ! Omod && ! IsF64 && ( ST . getConstantBusLimit ( Opc ) > <NUM_LIT> || ! Src0 -> isReg ( ) || ! RI . isSGPRReg ( MBB . getParent ( ) -> getRegInfo ( ) , Src0 -> getReg ( ) ) ) ) { int64_t Imm ; if ( getFoldableImm ( Src2 , Imm ) ) { unsigned NewOpc = IsFMA ? ( IsF16 ? <STR_LIT> : <STR_LIT> ) : ( IsF16 ? <STR_LIT> : <STR_LIT> ) ; if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> ) { MIB = BuildMI ( MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . add ( * Src0 ) . add ( * Src1 ) . addImm ( Imm ) ; updateLiveVariables ( LV , MI , * MIB ) ; return MIB ; } } unsigned NewOpc = IsFMA ? ( IsF16 ? <STR_LIT> : <STR_LIT> ) : ( IsF16 ? <STR_LIT> : <STR_LIT> ) ; if ( getFoldableImm ( Src1 , Imm ) ) { if ( pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> ) { MIB = BuildMI ( MBB , MI , MI . getDebugLoc ( ) , get ( NewOpc ) ) . add ( * Dst ) . add ( * Src0 ) . addImm ( Imm ) . add ( * Src2 ) ; updateLiveVariables ( LV , MI , * MIB ) ; return MIB ; }


<s> unsigned SaveExecOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned XorTermOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned AndOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; const auto * BoolXExecRC = TRI -> getRegClass ( <STR_LIT> ) ; MachineBasicBlock :: iterator I = LoopBB . begin ( ) ; SmallVector < Register , <NUM_LIT> > ReadlanePieces ; Register CondReg = <STR_LIT> ; Register VRsrc = Rsrc . getReg ( ) ; unsigned VRsrcUndef = getUndefRegState ( Rsrc . isUndef ( ) ) ; unsigned RegSize = TRI -> getRegSizeInBits ( Rsrc . getReg ( ) , MRI ) ; unsigned NumSubRegs = RegSize / <NUM_LIT> ; assert ( NumSubRegs % <NUM_LIT> == <NUM_LIT> && NumSubRegs <= <NUM_LIT> && <STR_LIT> ) ; for ( unsigned Idx = <NUM_LIT> ; Idx < NumSubRegs ; Idx += <NUM_LIT> ) { Register CurRegLo = MRI . createVirtualRegister ( & <STR_LIT> ) ; Register CurRegHi = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CurRegLo ) . addReg ( VRsrc , VRsrcUndef , TRI -> getSubRegFromChannel ( Idx ) ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CurRegHi ) . addReg ( VRsrc , VRsrcUndef , TRI -> getSubRegFromChannel ( Idx + <NUM_LIT> ) ) ; ReadlanePieces . push_back ( CurRegLo ) ; ReadlanePieces . push_back ( CurRegHi ) ; Register CurReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CurReg ) . addReg ( CurRegLo ) . addImm ( <STR_LIT> ) . addReg ( CurRegHi ) . addImm ( <STR_LIT> ) ; Register NewCondReg = MRI . createVirtualRegister ( BoolXExecRC ) ; auto Cmp = BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , NewCondReg ) . addReg ( CurReg ) ; if ( NumSubRegs <= <NUM_LIT> ) Cmp . addReg ( VRsrc ) ; else Cmp . addReg ( VRsrc , VRsrcUndef , TRI -> getSubRegFromChannel ( Idx , <NUM_LIT> ) ) ; if ( CondReg == <STR_LIT> ) CondReg = NewCondReg ; else { Register AndReg = MRI . createVirtualRegister ( BoolXExecRC ) ; BuildMI ( LoopBB , I , DL , TII . get ( AndOpc ) , AndReg ) . addReg ( CondReg ) . addReg ( NewCondReg ) ; CondReg = AndReg ; } } auto SRsrcRC = TRI -> getEquivalentSGPRClass ( MRI . getRegClass ( VRsrc ) ) ; Register SRsrc = MRI . createVirtualRegister ( SRsrcRC ) ; auto Merge = BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrc ) ; unsigned Channel = <NUM_LIT> ; for ( Register Piece : ReadlanePieces ) { Merge . addReg ( Piece ) . addImm ( TRI -> getSubRegFromChannel ( Channel ++ ) ) ;


<s> static bool getFoldableImm ( const MachineOperand * MO , int64_t & Imm ) { if ( ! MO -> isReg ( ) ) return false ;

const MachineFunction * MF = MO -> getParent ( ) -> getParent ( ) -> getParent ( ) ;


<s> if ( ! MO -> isReg ( ) ) return false ; const MachineFunction * MF = MO -> getParent ( ) -> getParent ( ) -> getParent ( ) ; const MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ;

return getFoldableImm ( MO -> getReg ( ) , MRI , Imm ) ;


<s> } if ( isVALU ( MI ) || isSALU ( MI ) ) { int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; if ( Src0Idx == - <NUM_LIT> ) return DescSize ; if ( isLiteralConstantLike ( MI . getOperand ( Src0Idx ) , Desc . OpInfo [ Src0Idx ] ) ) return isVOP3 ( MI ) ? <NUM_LIT> : ( DescSize + <NUM_LIT> ) ; int Src1Idx = <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ; if ( Src1Idx == - <NUM_LIT> ) return DescSize ; if ( isLiteralConstantLike ( MI . getOperand ( Src1Idx ) , Desc . OpInfo [ Src1Idx ] ) ) return isVOP3 ( MI ) ? <NUM_LIT> : ( DescSize + <NUM_LIT> ) ; int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( Src2Idx == - <NUM_LIT> ) return DescSize ; if ( isLiteralConstantLike ( MI . getOperand ( Src2Idx ) , Desc . OpInfo [ Src2Idx ] ) ) return isVOP3 ( MI ) ? <NUM_LIT> : ( DescSize + <NUM_LIT> ) ; return DescSize ; } if ( isMIMG ( MI ) ) { int VAddr0Idx = <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) ; if ( VAddr0Idx < <NUM_LIT> ) return <NUM_LIT> ; int RSrcIdx = <STR_LIT> ( Opc , <STR_LIT> :: srsrc ) ; return <NUM_LIT> + <NUM_LIT> * ( ( RSrcIdx - VAddr0Idx + <NUM_LIT> ) / <NUM_LIT> ) ; } switch ( Opc ) { case TargetOpcode :: BUNDLE : return getInstBundleSize ( MI ) ; case TargetOpcode :: INLINEASM : case TargetOpcode :: INLINEASM_BR : { const MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; const char * AsmStr = MI . getOperand ( <NUM_LIT> ) . getSymbolName ( ) ; return getInlineAsmLength ( AsmStr , * MF -> getTarget ( ) . getMCAsmInfo ( ) , & ST ) ;


<s> if ( isFixedSize ( MI ) ) { unsigned Size = DescSize ; if ( MI . isBranch ( ) && ST . hasOffset3fBug ( ) ) Size += <NUM_LIT> ; return Size ; } if ( isVALU ( MI ) || isSALU ( MI ) ) { int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; if ( Src0Idx == - <NUM_LIT> ) return DescSize ; if ( isLiteralConstantLike ( MI . getOperand ( Src0Idx ) , Desc . OpInfo [ Src0Idx ] ) ) return isVOP3 ( MI ) ? <NUM_LIT> : ( DescSize + <NUM_LIT> ) ; int Src1Idx = <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ; if ( Src1Idx == - <NUM_LIT> ) return DescSize ; if ( isLiteralConstantLike ( MI . getOperand ( Src1Idx ) , Desc . OpInfo [ Src1Idx ] ) ) return isVOP3 ( MI ) ? <NUM_LIT> : ( DescSize + <NUM_LIT> ) ; int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( Src2Idx == - <NUM_LIT> ) return DescSize ; if ( isLiteralConstantLike ( MI . getOperand ( Src2Idx ) , Desc . OpInfo [ Src2Idx ] ) ) return isVOP3 ( MI ) ? <NUM_LIT> : ( DescSize + <NUM_LIT> ) ; return DescSize ; } if ( isMIMG ( MI ) ) { int VAddr0Idx = <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) ; if ( VAddr0Idx < <NUM_LIT> ) return <NUM_LIT> ; int RSrcIdx = <STR_LIT> ( Opc , <STR_LIT> :: srsrc ) ; return <NUM_LIT> + <NUM_LIT> * ( ( RSrcIdx - VAddr0Idx + <NUM_LIT> ) / <NUM_LIT> ) ; } switch ( Opc ) { case TargetOpcode :: BUNDLE : return getInstBundleSize ( MI ) ; case TargetOpcode :: INLINEASM : case TargetOpcode :: INLINEASM_BR : { const MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; const char * AsmStr = MI . getOperand ( <NUM_LIT> ) . getSymbolName ( ) ; return getInlineAsmLength ( AsmStr , * MF -> getTarget ( ) . getMCAsmInfo ( ) , & ST ) ; } default : if ( MI . isMetaInstruction ( ) ) return <NUM_LIT> ; return DescSize ; }


<s> const TargetRegisterClass * <ISA_LIT>InstrInfo :: getOpRegClass ( const MachineInstr & MI , unsigned OpNo ) const { const MachineRegisterInfo & MRI = MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; const MCInstrDesc & Desc = get ( MI . getOpcode ( ) ) ; if ( MI . isVariadic ( ) || OpNo >= Desc . getNumOperands ( ) || Desc . OpInfo [ OpNo ] . RegClass == - <NUM_LIT> ) { Register Reg = MI . getOperand ( OpNo ) . getReg ( ) ; if ( Reg . isVirtual ( ) ) return MRI . getRegClass ( Reg ) ; return RI . getPhysRegClass ( Reg ) ; } unsigned RCID = Desc . OpInfo [ OpNo ] . RegClass ;

RCID = adjustAllocatableRegClass ( ST , MRI , Desc , RCID , true ) ;


<s> const MCInstrDesc & Desc = get ( MI . getOpcode ( ) ) ; if ( MI . isVariadic ( ) || OpNo >= Desc . getNumOperands ( ) || Desc . OpInfo [ OpNo ] . RegClass == - <NUM_LIT> ) { Register Reg = MI . getOperand ( OpNo ) . getReg ( ) ; if ( Reg . isVirtual ( ) ) return MRI . getRegClass ( Reg ) ; return RI . getPhysRegClass ( Reg ) ; } unsigned RCID = Desc . OpInfo [ OpNo ] . RegClass ; RCID = adjustAllocatableRegClass ( ST , MRI , Desc , RCID , true ) ; return RI . getRegClass ( RCID ) ;


<s> const TargetRegisterClass * <ISA_LIT>InstrInfo :: getRegClass ( const MCInstrDesc & TID , unsigned OpNum , const TargetRegisterInfo * TRI , const MachineFunction & MF ) const { if ( OpNum >= TID . getNumOperands ( ) ) return nullptr ; auto RegClass = TID . OpInfo [ OpNum ] . RegClass ; bool IsAllocatable = false ;

if ( TID . TSFlags & ( <STR_LIT> | <STR_LIT> ) ) {


<s> Register Tmp = RS . scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; if ( ! Tmp ) report_fatal_error ( <STR_LIT> ) ; RS . setRegUsed ( Tmp ) ; if ( ! TII . getSubtarget ( ) . hasGFX90AInsts ( ) ) { while ( RegNo -- && RS . FindUnusedReg ( & <STR_LIT> ) ) { Register Tmp2 = RS . scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; if ( ! Tmp2 || RI . getHWRegIndex ( Tmp2 ) >= MaxVGPRs ) break ; Tmp = Tmp2 ; RS . setRegUsed ( Tmp ) ; } } unsigned TmpCopyOp = <STR_LIT> ; if ( <STR_LIT> . contains ( SrcReg ) ) { TmpCopyOp = <STR_LIT> ; } else { assert ( <STR_LIT> . contains ( SrcReg ) ) ; } MachineInstrBuilder UseBuilder = BuildMI ( MBB , MI , DL , TII . get ( TmpCopyOp ) , Tmp ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; if ( ImpUseSuperReg ) { UseBuilder . addReg ( ImpUseSuperReg , getKillRegState ( KillSrc ) | RegState :: Implicit ) ;


<s> Register PCReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; auto I = MBB . end ( ) ; MachineInstr * GetPC = BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , PCReg ) ; auto & MCCtx = MF -> getContext ( ) ; MCSymbol * PostGetPCLabel = MCCtx . createTempSymbol ( <STR_LIT> , true ) ; GetPC -> setPostInstrSymbol ( * MF , PostGetPCLabel ) ; MCSymbol * OffsetLo = MCCtx . createTempSymbol ( <STR_LIT> , true ) ; MCSymbol * OffsetHi = MCCtx . createTempSymbol ( <STR_LIT> , true ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) ) . addReg ( PCReg , RegState :: Define , <STR_LIT> ) . addReg ( PCReg , <NUM_LIT> , <STR_LIT> ) . addSym ( OffsetLo , MO_FAR_BRANCH_OFFSET ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) ) . addReg ( PCReg , RegState :: Define , <STR_LIT> ) . addReg ( PCReg , <NUM_LIT> , <STR_LIT> ) . addSym ( OffsetHi , MO_FAR_BRANCH_OFFSET ) ; BuildMI ( & MBB , DL , get ( <STR_LIT> ) ) . addReg ( PCReg ) ; auto ComputeBlockSize = [ ] ( const TargetInstrInfo * TII , const MachineBasicBlock & MBB ) { unsigned Size = <NUM_LIT> ; for ( const MachineInstr & MI : MBB ) Size += TII -> getInstSizeInBytes ( MI ) ; return Size ; } ; RS -> enterBasicBlockEnd ( MBB ) ; Register Scav = RS -> scavengeRegisterBackwards ( <STR_LIT> , MachineBasicBlock :: iterator ( GetPC ) , false , <NUM_LIT> ) ; MRI . replaceRegWith ( PCReg , Scav ) ; MRI . clearVirtRegs ( ) ;


<s> assert ( MBB . empty ( ) && <STR_LIT> ) ; assert ( MBB . pred_size ( ) == <NUM_LIT> ) ; MachineFunction * MF = MBB . getParent ( ) ; MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; Register PCReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; auto I = MBB . end ( ) ; MachineInstr * GetPC = BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , PCReg ) ; auto & MCCtx = MF -> getContext ( ) ; MCSymbol * PostGetPCLabel = MCCtx . createTempSymbol ( <STR_LIT> , true ) ; GetPC -> setPostInstrSymbol ( * MF , PostGetPCLabel ) ; MCSymbol * OffsetLo = MCCtx . createTempSymbol ( <STR_LIT> , true ) ; MCSymbol * OffsetHi = MCCtx . createTempSymbol ( <STR_LIT> , true ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) ) . addReg ( PCReg , RegState :: Define , <STR_LIT> ) . addReg ( PCReg , <NUM_LIT> , <STR_LIT> ) . addSym ( OffsetLo , MO_FAR_BRANCH_OFFSET ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) ) . addReg ( PCReg , RegState :: Define , <STR_LIT> ) . addReg ( PCReg , <NUM_LIT> , <STR_LIT> ) . addSym ( OffsetHi , MO_FAR_BRANCH_OFFSET ) ;

BuildMI ( & MBB , DL , get ( <STR_LIT> ) ) . addReg ( PCReg ) ;


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { unsigned NumOps = MI . getDesc ( ) . getNumOperands ( ) + MI . getDesc ( ) . getNumImplicitUses ( ) ; return MI . getNumOperands ( ) == NumOps ; }


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { unsigned NumOps = MI . getDesc ( ) . getNumOperands ( ) + MI . getDesc ( ) . getNumImplicitUses ( ) ; return MI . getNumOperands ( ) == NumOps ; }


<s> bool <ISA_LIT>InstrInfo :: isIgnorableUse ( const MachineOperand & MO ) const {

return MO . getReg ( ) == <STR_LIT> && MO . isImplicit ( ) && isVALU ( * MO . getParent ( ) ) ;


<s> bool <ISA_LIT>InstrInfo :: isIgnorableUse ( const MachineOperand & MO ) const { return MO . getReg ( ) == <STR_LIT> && MO . isImplicit ( ) && isVALU ( * MO . getParent ( ) ) ;


<s> } } } if ( MO -> isReg ( ) ) { assert ( DefinedRC ) ; if ( ! isLegalRegOperand ( MRI , OpInfo , * MO ) ) return false ; bool IsAGPR = RI . isAGPR ( MRI , MO -> getReg ( ) ) ; if ( IsAGPR && ! ST . hasMAIInsts ( ) ) return false ; unsigned Opc = MI . getOpcode ( ) ; if ( IsAGPR && ( ! ST . hasGFX90AInsts ( ) || ! MRI . reservedRegsFrozen ( ) ) && ( MI . mayLoad ( ) || MI . mayStore ( ) || isDS ( Opc ) || isMIMG ( Opc ) ) ) return false ; const int VDstIdx = <STR_LIT> ( Opc , <STR_LIT> :: vdst ) ; const int DataIdx = <STR_LIT> ( Opc , isDS ( Opc ) ? <STR_LIT> :: data0 : <STR_LIT> :: vdata ) ; if ( ( int ) OpIdx == VDstIdx && DataIdx != - <NUM_LIT> && MI . getOperand ( DataIdx ) . isReg ( ) && RI . isAGPR ( MRI , MI . getOperand ( DataIdx ) . getReg ( ) ) != IsAGPR ) return false ; if ( ( int ) OpIdx == DataIdx ) { if ( VDstIdx != - <NUM_LIT> && RI . isAGPR ( MRI , MI . getOperand ( VDstIdx ) . getReg ( ) ) != IsAGPR ) return false ; const int Data1Idx = <STR_LIT> ( Opc , <STR_LIT> :: data1 ) ; if ( Data1Idx != - <NUM_LIT> && MI . getOperand ( Data1Idx ) . isReg ( ) && RI . isAGPR ( MRI , MI . getOperand ( Data1Idx ) . getReg ( ) ) != IsAGPR ) return false ; } if ( Opc == <STR_LIT> && ( int ) OpIdx == <STR_LIT> ( Opc , <STR_LIT> :: src0 ) && RI . isSGPRReg ( MRI , MO -> getReg ( ) ) ) return false ; return true ; } assert ( MO -> isImm ( ) || MO -> isTargetIndex ( ) || MO -> isFI ( ) || MO -> isGlobal ( ) ) ; if ( ! DefinedRC ) { return true ; }

return isImmOperandLegal ( MI , OpIdx , * MO ) ;


<s> bool <ISA_LIT>InstrInfo :: isReallyTriviallyReMaterializable ( const MachineInstr & MI , AAResults * AA ) const { if ( isVOP1 ( MI ) || isVOP2 ( MI ) || isVOP3 ( MI ) || isSDWA ( MI ) ) {

return ! MI . hasImplicitDef ( ) && MI . getNumImplicitOperands ( ) == MI . getDesc ( ) . getNumImplicitUses ( ) && ! MI . mayRaiseFPException ( ) ;


<s> BuildMI ( * MI . getParent ( ) , MI , DL , get ( <STR_LIT> ) , Reg ) . add ( Src1 ) ; Src1 . ChangeToRegister ( Reg , false ) ; } if ( Src2 . isReg ( ) && ! RI . isSGPRClass ( MRI . getRegClass ( Src2 . getReg ( ) ) ) ) { Register Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MI . getParent ( ) , MI , DL , get ( <STR_LIT> ) , Reg ) . add ( Src2 ) ; Src2 . ChangeToRegister ( Reg , false ) ; } } int ConstantBusLimit = ST . getConstantBusLimit ( Opc ) ; int LiteralLimit = ST . hasVOP3Literal ( ) ? <NUM_LIT> : <NUM_LIT> ; SmallDenseSet < unsigned > SGPRsUsed ; Register SGPRReg = findUsedSGPR ( MI , VOP3Idx ) ; if ( SGPRReg != <STR_LIT> ) { SGPRsUsed . insert ( SGPRReg ) ; -- ConstantBusLimit ; } for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { int Idx = VOP3Idx [ i ] ; if ( Idx == - <NUM_LIT> ) break ; MachineOperand & MO = MI . getOperand ( Idx ) ; if ( ! MO . isReg ( ) ) { if ( ! isLiteralConstantLike ( MO , get ( Opc ) . OpInfo [ Idx ] ) ) continue ; if ( LiteralLimit > <NUM_LIT> && ConstantBusLimit > <NUM_LIT> ) { -- LiteralLimit ; -- ConstantBusLimit ;


<s> MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; return ; } unsigned Opcode = RI . hasAGPRs ( RC ) ? getAGPRSpillRestoreOpcode ( SpillSize ) : getVGPRSpillRestoreOpcode ( SpillSize ) ;

BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) . addFrameIndex ( FrameIndex ) . addReg ( MFI -> getStackPtrOffsetReg ( ) ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , FrameInfo . getObjectSize ( FrameIndex ) , FrameInfo . getObjectAlign ( FrameIndex ) ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; assert ( DestReg != <STR_LIT> && <STR_LIT> ) ; assert ( DestReg != <STR_LIT> && DestReg != <STR_LIT> && DestReg != <STR_LIT> && <STR_LIT> ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillRestoreOpcode ( SpillSize ) ) ; if ( DestReg . isVirtual ( ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; return ; }

unsigned Opcode = RI . hasAGPRs ( RC ) ? getAGPRSpillRestoreOpcode ( SpillSize ) : getVGPRSpillRestoreOpcode ( SpillSize ) ;


<s> const DebugLoc & DL = MI . getDebugLoc ( ) ; unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned MovExecOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; const auto * BoolXExecRC = TRI -> getRegClass ( <STR_LIT> ) ; Register SaveExec = MRI . createVirtualRegister ( BoolXExecRC ) ; BuildMI ( MBB , Begin , DL , TII . get ( MovExecOpc ) , SaveExec ) . addReg ( Exec ) ; MachineBasicBlock :: iterator AfterMI = MI ; ++ AfterMI ; for ( auto I = Begin ; I != AfterMI ; I ++ ) { for ( auto & MO : I -> uses ( ) ) { if ( MO . isReg ( ) && MO . isUse ( ) ) { MRI . clearKillFlags ( MO . getReg ( ) ) ; } } } MachineBasicBlock * LoopBB = MF . CreateMachineBasicBlock ( ) ; MachineBasicBlock * RemainderBB = MF . CreateMachineBasicBlock ( ) ; MachineFunction :: iterator MBBI ( MBB ) ; ++ MBBI ; MF . insert ( MBBI , LoopBB ) ; MF . insert ( MBBI , RemainderBB ) ; LoopBB -> addSuccessor ( LoopBB ) ; LoopBB -> addSuccessor ( RemainderBB ) ; RemainderBB -> transferSuccessorsAndUpdatePHIs ( & MBB ) ; RemainderBB -> splice ( RemainderBB -> begin ( ) , & MBB , End , MBB . end ( ) ) ; LoopBB -> splice ( LoopBB -> begin ( ) , & MBB , Begin , MBB . end ( ) ) ;


<s> MachineOperand & Src0 = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Cond = Inst . getOperand ( <NUM_LIT> ) ; Register SCCSource = Cond . getReg ( ) ; bool IsSCC = ( SCCSource == <STR_LIT> ) ; if ( ! IsSCC && Src0 . isImm ( ) && ( Src0 . getImm ( ) == - <NUM_LIT> ) && Src1 . isImm ( ) && ( Src1 . getImm ( ) == <NUM_LIT> ) ) { MRI . replaceRegWith ( Dest . getReg ( ) , SCCSource ) ; return ; } const TargetRegisterClass * TC = RI . getRegClass ( <STR_LIT> ) ; Register CopySCC = MRI . createVirtualRegister ( TC ) ; if ( IsSCC ) { bool CopyFound = false ; for ( MachineInstr & CandI : make_range ( std :: next ( MachineBasicBlock :: reverse_iterator ( Inst ) ) , Inst . getParent ( ) -> rend ( ) ) ) { if ( CandI . findRegisterDefOperandIdx ( <STR_LIT> , false , false , & RI ) != - <NUM_LIT> ) { if ( CandI . isCopy ( ) && CandI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) { BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , CopySCC ) . addReg ( CandI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; CopyFound = true ; } break ; } } if ( ! CopyFound ) { unsigned Opcode = ( ST . getWavefrontSize ( ) == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; auto NewSelect = BuildMI ( MBB , MII , DL , get ( Opcode ) , CopySCC ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ;

NewSelect -> getOperand ( <NUM_LIT> ) . setIsUndef ( Cond . isUndef ( ) ) ;


<s> MachineRegisterInfo & MRI = Inst . getMF ( ) -> getRegInfo ( ) ; MachineOperand & SAddr = Inst . getOperand ( OldSAddrIdx ) ; if ( RI . isSGPRReg ( MRI , SAddr . getReg ( ) ) ) return false ; int NewVAddrIdx = <STR_LIT> ( NewOpc , <STR_LIT> :: vaddr ) ; if ( NewVAddrIdx < <NUM_LIT> ) return false ; int OldVAddrIdx = <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) ; MachineInstr * VAddrDef = nullptr ; if ( OldVAddrIdx >= <NUM_LIT> ) { MachineOperand & VAddr = Inst . getOperand ( OldVAddrIdx ) ; VAddrDef = MRI . getUniqueVRegDef ( VAddr . getReg ( ) ) ; if ( ! VAddrDef || VAddrDef -> getOpcode ( ) != <STR_LIT> || ! VAddrDef -> getOperand ( <NUM_LIT> ) . isImm ( ) || VAddrDef -> getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) return false ; } const MCInstrDesc & NewDesc = get ( NewOpc ) ; Inst . setDesc ( NewDesc ) ; if ( OldVAddrIdx == NewVAddrIdx ) { MachineOperand & NewVAddr = Inst . getOperand ( NewVAddrIdx ) ; MRI . removeRegOperandFromUseList ( & NewVAddr ) ; MRI . moveOperands ( & NewVAddr , & SAddr , <NUM_LIT> ) ; Inst . RemoveOperand ( OldSAddrIdx ) ; MRI . removeRegOperandFromUseList ( & NewVAddr ) ; MRI . addRegOperandToUseList ( & NewVAddr ) ;


<s> MachineBasicBlock & MBB = * Inst . getParent ( ) ; MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; Register OldDstReg = Inst . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register ResultReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned Opc = Inst . getOpcode ( ) ; assert ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ; unsigned NewOpc = Opc == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; assert ( Inst . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) ; Inst . RemoveOperand ( <NUM_LIT> ) ; Inst . setDesc ( get ( NewOpc ) ) ; Inst . addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; Inst . addImplicitDefUseOperands ( * MBB . getParent ( ) ) ;

MRI . replaceRegWith ( OldDstReg , ResultReg ) ;


<s> Mask &= maxUIntN ( SrcSize ) ; return isPowerOf2_64 ( Mask ) ; } ; MachineOperand * SrcOp = & Def -> getOperand ( <NUM_LIT> ) ; if ( isMask ( SrcOp ) ) SrcOp = & Def -> getOperand ( <NUM_LIT> ) ; else if ( isMask ( & Def -> getOperand ( <NUM_LIT> ) ) ) SrcOp = & Def -> getOperand ( <NUM_LIT> ) ; else return false ; unsigned BitNo = countTrailingZeros ( ( uint64_t ) Mask ) ; if ( IsSigned && BitNo == SrcSize - <NUM_LIT> ) return false ; ExpectedValue <<= BitNo ; bool IsReversedCC = false ; if ( CmpValue != ExpectedValue ) { if ( ! IsReversable ) return false ; IsReversedCC = CmpValue == ( ExpectedValue ^ Mask ) ; if ( ! IsReversedCC ) return false ; } Register DefReg = Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( IsReversedCC && ! MRI -> hasOneNonDBGUse ( DefReg ) ) return false ; for ( auto I = std :: next ( Def -> getIterator ( ) ) , E = CmpInstr . getIterator ( ) ; I != E ; ++ I ) { if ( I -> modifiesRegister ( <STR_LIT> , & RI ) || I -> killsRegister ( <STR_LIT> , & RI ) ) return false ; } MachineOperand * SccDef = Def -> findRegisterDefOperand ( <STR_LIT> ) ; SccDef -> setIsDead ( false ) ; CmpInstr . eraseFromParent ( ) ; if ( ! MRI -> use_nodbg_empty ( DefReg ) ) { assert ( ! IsReversedCC ) ; return true ; } MachineBasicBlock * MBB = Def -> getParent ( ) ; unsigned NewOpc = ( SrcSize == <NUM_LIT> ) ? IsReversedCC ? <STR_LIT> : <STR_LIT> : IsReversedCC ? <STR_LIT> : <STR_LIT> ; BuildMI ( * MBB , Def , Def -> getDebugLoc ( ) , get ( NewOpc ) ) . add ( * SrcOp ) . addImm ( BitNo ) ; Def -> eraseFromParent ( ) ; return true ; } ; switch ( CmpInstr . getOpcode ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return optimizeCmpAnd ( <NUM_LIT> , <NUM_LIT> , true , false ) ; case <STR_LIT> : case <STR_LIT> : return optimizeCmpAnd ( <NUM_LIT> , <NUM_LIT> , false , false ) ; case <STR_LIT> : case <STR_LIT> : return optimizeCmpAnd ( <NUM_LIT> , <NUM_LIT> , false , true ) ; case <STR_LIT> : return optimizeCmpAnd ( <NUM_LIT> , <NUM_LIT> , true , false ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> } } int MCOp = <STR_LIT> ( Opcode , Gen ) ; if ( MCOp == - <NUM_LIT> ) return Opcode ; if ( ST . hasGFX90AInsts ( ) ) { uint16_t NMCOp = ( uint16_t ) - <NUM_LIT> ; NMCOp = <STR_LIT> ( Opcode , <STR_LIT> ) ; if ( NMCOp == ( uint16_t ) - <NUM_LIT> ) NMCOp = <STR_LIT> ( Opcode , <STR_LIT> ) ; if ( NMCOp != ( uint16_t ) - <NUM_LIT> ) MCOp = NMCOp ; } if ( MCOp == ( uint16_t ) - <NUM_LIT> ) return - <NUM_LIT> ;

if ( isAsmOnlyOpcode ( MCOp ) ) return - <NUM_LIT> ;


<s> unsigned <ISA_LIT>InstrInfo :: removeBranch ( MachineBasicBlock & MBB , int * BytesRemoved ) const { MachineBasicBlock :: iterator I = MBB . getFirstTerminator ( ) ; unsigned Count = <NUM_LIT> ; unsigned RemovedSize = <NUM_LIT> ; while ( I != MBB . end ( ) ) { MachineBasicBlock :: iterator Next = std :: next ( I ) ; RemovedSize += getInstSizeInBytes ( * I ) ; I -> eraseFromParent ( ) ; ++ Count ; I = Next ; }

if ( BytesRemoved ) * BytesRemoved = RemovedSize ;


<s> Register DestSub1 = MRI . createVirtualRegister ( & <STR_LIT> ) ; const TargetRegisterClass * Src0RC = nullptr ; const TargetRegisterClass * Src0SubRC = nullptr ; if ( Src0 . isReg ( ) ) { Src0RC = MRI . getRegClass ( Src0 . getReg ( ) ) ; Src0SubRC = RI . getSubRegClass ( Src0RC , <STR_LIT> ) ; } const TargetRegisterClass * Src1RC = nullptr ; const TargetRegisterClass * Src1SubRC = nullptr ; if ( Src1 . isReg ( ) ) { Src1RC = MRI . getRegClass ( Src1 . getReg ( ) ) ; Src1SubRC = RI . getSubRegClass ( Src1RC , <STR_LIT> ) ; } MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm ( MII , MRI , Src1 , Src1RC , <STR_LIT> , Src1SubRC ) ; MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src1 , Src1RC , <STR_LIT> , Src1SubRC ) ; MachineInstr * LoHalf = BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , DestSub0 ) . add ( SrcReg0Sub0 ) . add ( SrcReg1Sub0 ) ; LoHalf -> getOperand ( <NUM_LIT> ) . setReg ( SCCSource ) ; Worklist . insert ( LoHalf ) ; MachineInstr * HiHalf = BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , DestSub1 ) . add ( SrcReg0Sub1 ) . add ( SrcReg1Sub1 ) ; HiHalf -> getOperand ( <NUM_LIT> ) . setReg ( SCCSource ) ; Worklist . insert ( HiHalf ) ;

BuildMI ( MBB , MII , DL , get ( TargetOpcode :: REG_SEQUENCE ) , FullDestReg ) . addReg ( DestSub0 ) . addImm ( <STR_LIT> ) . addReg ( DestSub1 ) . addImm ( <STR_LIT> ) ;


<s> if ( Src0 . isReg ( ) ) { Src0RC = MRI . getRegClass ( Src0 . getReg ( ) ) ; Src0SubRC = RI . getSubRegClass ( Src0RC , <STR_LIT> ) ; } const TargetRegisterClass * Src1RC = nullptr ; const TargetRegisterClass * Src1SubRC = nullptr ; if ( Src1 . isReg ( ) ) { Src1RC = MRI . getRegClass ( Src1 . getReg ( ) ) ; Src1SubRC = RI . getSubRegClass ( Src1RC , <STR_LIT> ) ; } MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm ( MII , MRI , Src1 , Src1RC , <STR_LIT> , Src1SubRC ) ; MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src1 , Src1RC , <STR_LIT> , Src1SubRC ) ; MachineInstr * LoHalf = BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , DestSub0 ) . add ( SrcReg0Sub0 ) . add ( SrcReg1Sub0 ) ; LoHalf -> getOperand ( <NUM_LIT> ) . setReg ( SCCSource ) ; Worklist . insert ( LoHalf ) ; MachineInstr * HiHalf = BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , DestSub1 ) . add ( SrcReg0Sub1 ) . add ( SrcReg1Sub1 ) ; HiHalf -> getOperand ( <NUM_LIT> ) . setReg ( SCCSource ) ; Worklist . insert ( HiHalf ) ; BuildMI ( MBB , MII , DL , get ( TargetOpcode :: REG_SEQUENCE ) , FullDestReg ) . addReg ( DestSub0 ) . addImm ( <STR_LIT> ) . addReg ( DestSub1 ) . addImm ( <STR_LIT> ) ; MRI . replaceRegWith ( Dest . getReg ( ) , FullDestReg ) ; legalizeOperands ( * LoHalf , MDT ) ;


<s> MFI -> setHasSpilledSGPRs ( ) ; assert ( SrcReg != <STR_LIT> && <STR_LIT> ) ; assert ( SrcReg != <STR_LIT> && SrcReg != <STR_LIT> && SrcReg != <STR_LIT> && <STR_LIT> ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( SrcReg . isVirtual ( ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ;

if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ;


<s> return <STR_LIT> ( Imm ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( isInt < <NUM_LIT> > ( Imm ) || isUInt < <NUM_LIT> > ( Imm ) ) { int16_t Trunc = static_cast < int16_t > ( Imm ) ; return ST . has16BitInsts ( ) && <STR_LIT> ( Trunc , ST . hasInv2PiInlineImm ( ) ) ; } return false ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { uint32_t Trunc = static_cast < uint32_t > ( Imm ) ; return <STR_LIT> ( Trunc , ST . hasInv2PiInlineImm ( ) ) ; } default : llvm_unreachable ( <STR_LIT> ) ; }


<s> case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ( Imm ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( isInt < <NUM_LIT> > ( Imm ) || isUInt < <NUM_LIT> > ( Imm ) ) { int16_t Trunc = static_cast < int16_t > ( Imm ) ; return ST . has16BitInsts ( ) && <STR_LIT> ( Trunc , ST . hasInv2PiInlineImm ( ) ) ; } return false ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { uint32_t Trunc = static_cast < uint32_t > ( Imm ) ; return <STR_LIT> ( Trunc , ST . hasInv2PiInlineImm ( ) ) ; } default :


<s> if ( I -> isBranch ( ) || I -> isReturn ( ) ) { RemovedSize += getInstSizeInBytes ( * I ) ; I -> eraseFromParent ( ) ; ++ Count ; } I = Next ;


<s> llvm :: make_range ( MachineBasicBlock :: iterator ( SCCDefInst ) , SCCDefInst . getParent ( ) -> end ( ) ) ) { if ( MI . findRegisterDefOperandIdx ( <STR_LIT> ) != - <NUM_LIT> ) return ;

if ( MI . findRegisterUseOperandIdx ( <STR_LIT> ) != - <NUM_LIT> ) Worklist . insert ( & MI ) ;


<s> for ( MachineInstr & MI : llvm :: make_range ( MachineBasicBlock :: iterator ( SCCDefInst ) , SCCDefInst . getParent ( ) -> end ( ) ) ) { if ( MI . findRegisterDefOperandIdx ( <STR_LIT> ) != - <NUM_LIT> ) return ; if ( MI . findRegisterUseOperandIdx ( <STR_LIT> ) != - <NUM_LIT> ) Worklist . insert ( & MI ) ;


<s> const MachineOperand * Src0Mods = getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ; const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mods = getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; const MachineOperand * Src2 = getNamedOperand ( MI , <STR_LIT> :: src2 ) ; const MachineOperand * Clamp = getNamedOperand ( MI , <STR_LIT> :: clamp ) ; const MachineOperand * Omod = getNamedOperand ( MI , <STR_LIT> :: omod ) ;

return BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( IsF16 ? <STR_LIT> : <STR_LIT> ) ) . add ( * Dst ) . addImm ( Src0Mods ? Src0Mods -> getImm ( ) : <NUM_LIT> ) . add ( * Src0 ) . addImm ( Src1Mods ? Src1Mods -> getImm ( ) : <NUM_LIT> ) . add ( * Src1 ) . addImm ( <NUM_LIT> ) . add ( * Src2 ) . addImm ( Clamp ? Clamp -> getImm ( ) : <NUM_LIT> ) . addImm ( Omod ? Omod -> getImm ( ) : <NUM_LIT> ) ;


<s> break ; case <STR_LIT> : IsF16 = true ; LLVM_FALLTHROUGH ; case <STR_LIT> : { int Src0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: src0 ) ; const MachineOperand * Src0 = & MI . getOperand ( Src0Idx ) ; if ( Src0 -> isImm ( ) && ! isInlineConstant ( MI , Src0Idx , * Src0 ) ) return nullptr ; break ; } } const MachineOperand * Dst = getNamedOperand ( MI , <STR_LIT> :: vdst ) ; const MachineOperand * Src0 = getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src0Mods = getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ;

const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ;


<s> return false ; } if ( isMUBUF ( LdSt ) || isMTBUF ( LdSt ) ) { const MachineOperand * SOffset = getNamedOperand ( LdSt , <STR_LIT> :: soffset ) ; if ( SOffset && SOffset -> isReg ( ) ) return false ; const MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( ! AddrReg ) return false ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; BaseReg = AddrReg -> getReg ( ) ; Offset = OffsetImm -> getImm ( ) ; if ( SOffset ) Offset += SOffset -> getImm ( ) ; return true ; } if ( isSMRD ( LdSt ) ) { const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; if ( ! OffsetImm ) return false ; const MachineOperand * SBaseReg = getNamedOperand ( LdSt , <STR_LIT> :: sbase ) ; BaseReg = SBaseReg -> getReg ( ) ;


<s> break ; } case <ISA_LIT>InstrInfo :: EXECNZ : { unsigned SReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SReg2 = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg2 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addReg ( FalseReg ) . addReg ( TrueReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: EXECZ : { unsigned SReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SReg2 = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg2 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addReg ( FalseReg ) . addReg ( TrueReg ) . addReg ( SReg ) ; llvm_unreachable ( <STR_LIT> ) ; break ; } default : llvm_unreachable ( <STR_LIT> ) ; } } else { llvm_unreachable ( <STR_LIT> ) ;


<s> break ; } case <ISA_LIT>InstrInfo :: EXECNZ : { unsigned SReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SReg2 = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg2 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addReg ( FalseReg ) . addReg ( TrueReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: EXECZ : { unsigned SReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SReg2 = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg2 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addReg ( FalseReg ) . addReg ( TrueReg ) . addReg ( SReg ) ; llvm_unreachable ( <STR_LIT> ) ; break ; } default : llvm_unreachable ( <STR_LIT> ) ; } } else {


<s> case <STR_LIT> : case <STR_LIT> : { int32_t Trunc = static_cast < int32_t > ( Imm ) ; return Trunc == Imm && <STR_LIT> ( Trunc , ST . hasInv2PiInlineImm ( ) ) ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { return <STR_LIT> ( MO . getImm ( ) , ST . hasInv2PiInlineImm ( ) ) ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( isInt < <NUM_LIT> > ( Imm ) || isUInt < <NUM_LIT> > ( Imm ) ) { int16_t Trunc = static_cast < int16_t > ( Imm ) ; return ST . has16BitInsts ( ) && <STR_LIT> ( Trunc , ST . hasInv2PiInlineImm ( ) ) ; } return false ; } case <STR_LIT> : case <STR_LIT> : {

uint32_t Trunc = static_cast < uint32_t > ( Imm ) ;


<s> } if ( ! ST . isVGPRSpillingEnabled ( * MF -> getFunction ( ) ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) ; return ; } assert ( RI . hasVGPRs ( RC ) && <STR_LIT> ) ; unsigned Opcode = getVGPRSpillRestoreOpcode ( SpillSize ) ;

BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) . addFrameIndex ( FrameIndex ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( MFI -> getFrameOffsetReg ( ) ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } FrameInfo . setStackID ( FrameIndex , <NUM_LIT> ) ; MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getFrameOffsetReg ( ) , RegState :: Implicit ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ; } return ; } if ( ! ST . isVGPRSpillingEnabled ( * MF -> getFunction ( ) ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) ; return ; } assert ( RI . hasVGPRs ( RC ) && <STR_LIT> ) ; unsigned Opcode = getVGPRSpillRestoreOpcode ( SpillSize ) ; BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) . addFrameIndex ( FrameIndex ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( MFI -> getFrameOffsetReg ( ) ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> MachineBasicBlock & MBB = * Inst . getParent ( ) ; MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; MachineBasicBlock :: iterator MII = Inst ; DebugLoc DL = Inst . getDebugLoc ( ) ; MachineOperand & Dest = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src = Inst . getOperand ( <NUM_LIT> ) ; unsigned TmpReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned ResultReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , TmpReg ) . addImm ( <NUM_LIT> ) . addReg ( Src . getReg ( ) ) ; BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , ResultReg ) . addReg ( Src . getReg ( ) ) . addReg ( TmpReg ) ;

MRI . replaceRegWith ( Dest . getReg ( ) , ResultReg ) ;


<s> MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; MachineBasicBlock :: iterator MII = Inst ; DebugLoc DL = Inst . getDebugLoc ( ) ; MachineOperand & Dest = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src = Inst . getOperand ( <NUM_LIT> ) ; unsigned TmpReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned ResultReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , TmpReg ) . addImm ( <NUM_LIT> ) . addReg ( Src . getReg ( ) ) ; BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , ResultReg ) . addReg ( Src . getReg ( ) ) . addReg ( TmpReg ) ; MRI . replaceRegWith ( Dest . getReg ( ) , ResultReg ) ; addUsersToMoveToVALUWorklist ( ResultReg , MRI , Worklist ) ;


<s> case <STR_LIT> : BuildMI ( * MBB , Inst , Inst . getDebugLoc ( ) , get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ; break ; case <STR_LIT> : case <STR_LIT> : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { movePackToVALU ( Worklist , MRI , Inst ) ; Inst . eraseFromParent ( ) ; continue ; } } if ( NewOpcode == <STR_LIT> ) { legalizeOperands ( Inst ) ; continue ; } const MCInstrDesc & NewDesc = get ( NewOpcode ) ; Inst . setDesc ( NewDesc ) ; for ( unsigned i = Inst . getNumOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; -- i ) { MachineOperand & Op = Inst . getOperand ( i ) ; if ( Op . isReg ( ) && Op . getReg ( ) == <STR_LIT> ) { Inst . RemoveOperand ( i ) ; addSCCDefUsersToVALUWorklist ( Inst , Worklist ) ; } } if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { unsigned Size = ( Opcode == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; Inst . addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; Inst . addOperand ( MachineOperand :: CreateImm ( Size ) ) ; } else if ( Opcode == <STR_LIT> ) { Inst . addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; } Inst . addImplicitDefUseOperands ( * Inst . getParent ( ) -> getParent ( ) ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { const MachineOperand & OffsetWidthOp = Inst . getOperand ( <NUM_LIT> ) ; assert ( OffsetWidthOp . isImm ( ) && <STR_LIT> ) ; uint32_t Imm = OffsetWidthOp . getImm ( ) ; uint32_t Offset = Imm & <NUM_LIT> ; uint32_t BitWidth = ( Imm & <NUM_LIT> ) >> <NUM_LIT> ; Inst . RemoveOperand ( <NUM_LIT> ) ; Inst . addOperand ( MachineOperand :: CreateImm ( Offset ) ) ; Inst . addOperand ( MachineOperand :: CreateImm ( BitWidth ) ) ; } bool HasDst = Inst . getOperand ( <NUM_LIT> ) . isReg ( ) && Inst . getOperand ( <NUM_LIT> ) . isDef ( ) ; unsigned NewDstReg = <STR_LIT> ; if ( HasDst ) {

unsigned DstReg = Inst . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdata ) ; } else if ( isSMRD ( FirstLdSt ) && isSMRD ( SecondLdSt ) ) { FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: sdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: sdst ) ; } else if ( isDS ( FirstLdSt ) && isDS ( SecondLdSt ) ) { FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: vdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdst ) ; } if ( ! FirstDst || ! SecondDst ) return false ; unsigned LoadClusterThreshold = <NUM_LIT> ; const MachineRegisterInfo & MRI = FirstLdSt . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; const TargetRegisterClass * DstRC = MRI . getRegClass ( FirstDst -> getReg ( ) ) ;

return ( NumLoads * ( RI . getRegSizeInBits ( * DstRC ) / <NUM_LIT> ) ) <= LoadClusterThreshold ;


<s> SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdata ) ; } else if ( isSMRD ( FirstLdSt ) && isSMRD ( SecondLdSt ) ) { FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: sdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: sdst ) ; } else if ( isDS ( FirstLdSt ) && isDS ( SecondLdSt ) ) { FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: vdst ) ;

SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdst ) ;


<s> MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getFrameOffsetReg ( ) , RegState :: Implicit ) ; FrameInfo . setStackID ( FrameIndex , <NUM_LIT> ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ; } return ; } if ( ! ST . isVGPRSpillingEnabled ( * MF -> getFunction ( ) ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg ) ; return ;


<s> MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getFrameOffsetReg ( ) , RegState :: Implicit ) ; FrameInfo . setStackID ( FrameIndex , <NUM_LIT> ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ; } return ; } if ( ! ST . isVGPRSpillingEnabled ( * MF -> getFunction ( ) ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg ) ;


<s> bool <ISA_LIT>InstrInfo :: areMemAccessesTriviallyDisjoint ( MachineInstr & MIa , MachineInstr & MIb , AliasAnalysis * AA ) const { assert ( ( MIa . mayLoad ( ) || MIa . mayStore ( ) ) && <STR_LIT> ) ; assert ( ( MIb . mayLoad ( ) || MIb . mayStore ( ) ) && <STR_LIT> ) ; if ( MIa . hasUnmodeledSideEffects ( ) || MIb . hasUnmodeledSideEffects ( ) ) return false ; if ( MIa . hasOrderedMemoryRef ( ) || MIb . hasOrderedMemoryRef ( ) ) return false ; if ( isDS ( MIa ) ) { if ( isDS ( MIb ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return ! isFLAT ( MIb ) || isSegmentSpecificFLAT ( MIb ) ; } if ( isMUBUF ( MIa ) || isMTBUF ( MIa ) ) {

if ( isMUBUF ( MIb ) || isMTBUF ( MIb ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ;


<s> case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return ST . hasDLInsts ( ) ? <STR_LIT> : <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ;


<s> bool llvm :: isEXECMaskConstantBetweenDefAndUses ( unsigned VReg , MachineRegisterInfo & MRI ) { assert ( MRI . isSSA ( ) && <STR_LIT> ) ; auto * TRI = MRI . getTargetRegisterInfo ( ) ; auto * DefI = MRI . getVRegDef ( VReg ) ; auto * BB = DefI -> getParent ( ) ; DenseSet < MachineInstr * > Uses ; for ( auto & Use : MRI . use_nodbg_operands ( VReg ) ) {


<s> bool <ISA_LIT>InstrInfo :: mayReadEXEC ( const MachineRegisterInfo & MRI , const MachineInstr & MI ) const { if ( MI . isMetaInstruction ( ) ) return false ; if ( MI . isCopyLike ( ) ) {

if ( ! RI . isSGPRReg ( MRI , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) return true ;


<s> if ( MI . isMetaInstruction ( ) ) return false ; if ( MI . isCopyLike ( ) ) { if ( ! RI . isSGPRReg ( MRI , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) return true ; return MI . readsRegister ( <STR_LIT> , & RI ) ; } if ( ! isTargetSpecificOpcode ( MI . getOpcode ( ) ) ) return true ; return ! isSALU ( MI ) || MI . readsRegister ( <STR_LIT> , & RI ) ;


<s> if ( TargetRegisterInfo :: isVirtualRegister ( DestReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getFrameOffsetReg ( ) , RegState :: Implicit ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ; } return ; } if ( ! ST . isVGPRSpillingEnabled ( * MF -> getFunction ( ) ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) ; return ; } assert ( RI . hasVGPRs ( RC ) && <STR_LIT> ) ; unsigned Opcode = getVGPRSpillRestoreOpcode ( SpillSize ) ;


<s> MachineFunction * MF = MBB . getParent ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; DebugLoc DL = MBB . findDebugLoc ( MI ) ; unsigned Align = FrameInfo . getObjectAlignment ( FrameIndex ) ; unsigned Size = FrameInfo . getObjectSize ( FrameIndex ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { const MCInstrDesc & OpDesc = get ( getSGPRSpillRestoreOpcode ( SpillSize ) ) ; if ( TargetRegisterInfo :: isVirtualRegister ( DestReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getFrameOffsetReg ( ) , RegState :: Implicit ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ; } return ; } if ( ! ST . isVGPRSpillingEnabled ( * MF -> getFunction ( ) ) ) { LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) ; return ;


<s> MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , Size , Align ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; if ( RI . isSGPRClass ( RC ) ) { MFI -> setHasSpilledSGPRs ( ) ; const MCInstrDesc & OpDesc = get ( getSGPRSpillSaveOpcode ( SpillSize ) ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SrcReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( SrcReg , & <STR_LIT> ) ; } MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getFrameOffsetReg ( ) , RegState :: Implicit ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ; } return ;


<s> BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; return ; } assert ( <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; return ; } if ( RC == & <STR_LIT> ) { if ( DestReg == <STR_LIT> ) { if ( <STR_LIT> . contains ( SrcReg ) ) { BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; } else { assert ( <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; } return ; } assert ( <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; return ;


<s> splitScalar64BitBCNT ( Worklist , Inst ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT> : { splitScalar64BitBFE ( Worklist , Inst ) ; Inst . eraseFromParent ( ) ; continue ; } case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : lowerScalarAbs ( Worklist , Inst ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT> : case <STR_LIT> : BuildMI ( * MBB , Inst , Inst . getDebugLoc ( ) , get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ; break ; case <STR_LIT> : case <STR_LIT> : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { movePackToVALU ( Worklist , MRI , Inst ) ; Inst . eraseFromParent ( ) ; continue ; } } if ( NewOpcode == <STR_LIT> ) { legalizeOperands ( Inst ) ; continue ; } const MCInstrDesc & NewDesc = get ( NewOpcode ) ;


<s> case <STR_LIT> : if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { NewOpcode = <STR_LIT> ; swapOperands ( Inst ) ; } break ; case <STR_LIT> : lowerScalarAbs ( Worklist , Inst ) ; Inst . eraseFromParent ( ) ; continue ; case <STR_LIT> : case <STR_LIT> : BuildMI ( * MBB , Inst , Inst . getDebugLoc ( ) , get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ; break ; case <STR_LIT> : case <STR_LIT> : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { movePackToVALU ( Worklist , MRI , Inst ) ; Inst . eraseFromParent ( ) ; continue ; } } if ( NewOpcode == <STR_LIT> ) { legalizeOperands ( Inst ) ; continue ; } const MCInstrDesc & NewDesc = get ( NewOpcode ) ; Inst . setDesc ( NewDesc ) ; for ( unsigned i = Inst . getNumOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; -- i ) { MachineOperand & Op = Inst . getOperand ( i ) ; if ( Op . isReg ( ) && Op . getReg ( ) == <STR_LIT> ) { Inst . RemoveOperand ( i ) ; addSCCDefUsersToVALUWorklist ( Inst , Worklist ) ; } } if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { unsigned Size = ( Opcode == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; Inst . addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; Inst . addOperand ( MachineOperand :: CreateImm ( Size ) ) ; } else if ( Opcode == <STR_LIT> ) { Inst . addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; } Inst . addImplicitDefUseOperands ( * Inst . getParent ( ) -> getParent ( ) ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { const MachineOperand & OffsetWidthOp = Inst . getOperand ( <NUM_LIT> ) ; assert ( OffsetWidthOp . isImm ( ) && <STR_LIT> ) ; uint32_t Imm = OffsetWidthOp . getImm ( ) ; uint32_t Offset = Imm & <NUM_LIT> ; uint32_t BitWidth = ( Imm & <NUM_LIT> ) >> <NUM_LIT> ; Inst . RemoveOperand ( <NUM_LIT> ) ; Inst . addOperand ( MachineOperand :: CreateImm ( Offset ) ) ; Inst . addOperand ( MachineOperand :: CreateImm ( BitWidth ) ) ; } bool HasDst = Inst . getOperand ( <NUM_LIT> ) . isReg ( ) && Inst . getOperand ( <NUM_LIT> ) . isDef ( ) ; unsigned NewDstReg = <STR_LIT> ;


<s> if ( I -> findRegisterDefOperandIdx ( <STR_LIT> ) != - <NUM_LIT> ) return ; if ( I -> findRegisterUseOperandIdx ( <STR_LIT> ) != - <NUM_LIT> ) Worklist . push_back ( I ) ;


<s> MachineBasicBlock :: iterator Insert = Entry . front ( ) ; DebugLoc DL = Insert -> getDebugLoc ( ) ; TIDReg = RI . findUnusedRegister ( MF -> getRegInfo ( ) , & <STR_LIT> ) ; if ( TIDReg == <STR_LIT> ) return TIDReg ; if ( ! <STR_LIT> ( MF -> getFunction ( ) -> getCallingConv ( ) ) && WorkGroupSize > WavefrontSize ) { unsigned TIDIGXReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: WORKGROUP_ID_X ) ; unsigned TIDIGYReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: WORKGROUP_ID_Y ) ; unsigned TIDIGZReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: WORKGROUP_ID_Z ) ; unsigned InputPtrReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR ) ; for ( unsigned Reg : { TIDIGXReg , TIDIGYReg , TIDIGZReg } ) { if ( ! Entry . isLiveIn ( Reg ) ) Entry . addLiveIn ( Reg ) ; } RS -> enterBasicBlock ( Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ;

BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ;


<s> DebugLoc DL = Insert -> getDebugLoc ( ) ; TIDReg = RI . findUnusedRegister ( MF -> getRegInfo ( ) , & <STR_LIT> ) ; if ( TIDReg == <STR_LIT> ) return TIDReg ; if ( ! <STR_LIT> ( MF -> getFunction ( ) -> getCallingConv ( ) ) && WorkGroupSize > WavefrontSize ) { unsigned TIDIGXReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: WORKGROUP_ID_X ) ; unsigned TIDIGYReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: WORKGROUP_ID_Y ) ; unsigned TIDIGZReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: WORKGROUP_ID_Z ) ; unsigned InputPtrReg = TRI -> getPreloadedValue ( * MF , <ISA_LIT>RegisterInfo :: KERNARG_SEGMENT_PTR ) ; for ( unsigned Reg : { TIDIGXReg , TIDIGYReg , TIDIGZReg } ) { if ( ! Entry . isLiveIn ( Reg ) ) Entry . addLiveIn ( Reg ) ; } RS -> enterBasicBlock ( Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp0 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Z ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( InputPtrReg ) . addImm ( <STR_LIT> :: NGROUPS_Y ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , STmp1 ) . addReg ( STmp1 ) . addReg ( STmp0 ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp1 ) . addReg ( TIDIGXReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( STmp0 ) . addReg ( TIDIGYReg ) . addReg ( TIDReg ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addReg ( TIDReg ) . addReg ( TIDIGZReg ) ; } else { BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( - <NUM_LIT> ) . addReg ( TIDReg ) ; }

BuildMI ( Entry , Insert , DL , get ( <STR_LIT> ) , TIDReg ) . addImm ( <NUM_LIT> ) . addReg ( TIDReg ) ;


<s> bool <ISA_LIT>InstrInfo :: checkInstOffsetsDoNotOverlap ( MachineInstr * MIa , MachineInstr * MIb ) const { unsigned BaseReg0 , BaseReg1 ; int64_t Offset0 , Offset1 ; if ( getMemOpBaseRegImmOfs ( MIa , BaseReg0 , Offset0 , & RI ) && getMemOpBaseRegImmOfs ( MIb , BaseReg1 , Offset1 , & RI ) ) { if ( ! MIa -> hasOneMemOperand ( ) || ! MIb -> hasOneMemOperand ( ) ) { return false ; }

unsigned Width0 = ( * MIa -> memoperands_begin ( ) ) -> getSize ( ) ;


<s> int Src0Idx = <STR_LIT> ( MI -> getOpcode ( ) , <STR_LIT> :: src0 ) ; MachineOperand & Src0 = MI -> getOperand ( Src0Idx ) ; if ( ! Src0 . isReg ( ) ) return nullptr ; int Src1Idx = <STR_LIT> ( MI -> getOpcode ( ) , <STR_LIT> :: src1 ) ; if ( ( OpIdx0 != static_cast < unsigned > ( Src0Idx ) || OpIdx1 != static_cast < unsigned > ( Src1Idx ) ) && ( OpIdx0 != static_cast < unsigned > ( Src1Idx ) || OpIdx1 != static_cast < unsigned > ( Src0Idx ) ) ) return nullptr ; MachineOperand & Src1 = MI -> getOperand ( Src1Idx ) ; if ( isVOP2 ( * MI ) || isVOPC ( * MI ) ) { const MCInstrDesc & InstrDesc = MI -> getDesc ( ) ; const MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; if ( ! isLegalRegOperand ( MRI , InstrDesc . OpInfo [ Src1Idx ] , Src0 ) ) return nullptr ; } if ( ! Src1 . isReg ( ) ) { if ( NewMI || ! Src1 . isImm ( ) || ( ! isVOP2 ( * MI ) && ! isVOP3 ( * MI ) ) ) { return nullptr ; } if ( MachineOperand * Src0Mods = getNamedOperand ( * MI , <STR_LIT> :: src0_modifiers ) ) { MachineOperand * Src1Mods = getNamedOperand ( * MI , <STR_LIT> :: src1_modifiers ) ; int Src0ModsVal = Src0Mods -> getImm ( ) ; if ( ! Src1Mods && Src0ModsVal != <NUM_LIT> ) return nullptr ; int Src1ModsVal = Src1Mods -> getImm ( ) ; assert ( ( Src1ModsVal == <NUM_LIT> ) && <STR_LIT> ) ; Src1Mods -> setImm ( Src0ModsVal ) ; Src0Mods -> setImm ( Src1ModsVal ) ; }


<s> const MachineOperand * Src1 = getNamedOperand ( * MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src2 = getNamedOperand ( * MI , <STR_LIT> :: src2 ) ;

return BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , get ( <STR_LIT> ) ) . addOperand ( * Dst ) . addImm ( <NUM_LIT> ) . addOperand ( * Src0 ) . addImm ( <NUM_LIT> ) . addOperand ( * Src1 ) . addImm ( <NUM_LIT> ) . addOperand ( * Src2 ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ;


<s> BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DstHi ) . addReg ( RI . getSubReg ( SrcOp . getReg ( ) , <STR_LIT> ) ) . addReg ( Dst , RegState :: Implicit | RegState :: Define ) ; } MI -> eraseFromParent ( ) ; break ; } case <STR_LIT> : { unsigned Dst = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned DstLo = RI . getSubReg ( Dst , <STR_LIT> ) ; unsigned DstHi = RI . getSubReg ( Dst , <STR_LIT> ) ; unsigned Src0 = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src1 = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const MachineOperand & SrcCond = MI -> getOperand ( <NUM_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DstLo ) . addReg ( RI . getSubReg ( Src0 , <STR_LIT> ) ) . addReg ( RI . getSubReg ( Src1 , <STR_LIT> ) ) . addReg ( SrcCond . getReg ( ) ) . addReg ( Dst , RegState :: Implicit | RegState :: Define ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DstHi ) . addReg ( RI . getSubReg ( Src0 , <STR_LIT> ) ) . addReg ( RI . getSubReg ( Src1 , <STR_LIT> ) ) . addReg ( SrcCond . getReg ( ) , getKillRegState ( SrcCond . isKill ( ) ) ) . addReg ( Dst , RegState :: Implicit | RegState :: Define ) ; MI -> eraseFromParent ( ) ; break ;


<s> UseMI -> RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: omod ) ) ; UseMI -> RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: clamp ) ) ; unsigned Src1Reg = Src1 -> getReg ( ) ; unsigned Src1SubReg = Src1 -> getSubReg ( ) ; Src0 -> setReg ( Src1Reg ) ; Src0 -> setSubReg ( Src1SubReg ) ; Src0 -> setIsKill ( Src1 -> isKill ( ) ) ; if ( Opc == <STR_LIT> ) { UseMI -> untieRegOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ) ; } Src1 -> ChangeToImmediate ( Imm ) ; removeModOperands ( * UseMI ) ; UseMI -> setDesc ( get ( <STR_LIT> ) ) ; bool DeleteDef = MRI -> hasOneNonDBGUse ( Reg ) ; if ( DeleteDef ) DefMI -> eraseFromParent ( ) ; return true ; } if ( Src2 -> isReg ( ) && Src2 -> getReg ( ) == Reg ) { if ( ! Src0 -> isImm ( ) && ( Src0 -> isReg ( ) && RI . isSGPRClass ( MRI -> getRegClass ( Src0 -> getReg ( ) ) ) ) ) return false ; if ( ! Src1 -> isReg ( ) || RI . isSGPRClass ( MRI -> getRegClass ( Src1 -> getReg ( ) ) ) ) return false ; const int64_t Imm = DefMI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; UseMI -> RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: omod ) ) ; UseMI -> RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: clamp ) ) ; if ( Opc == <STR_LIT> ) { UseMI -> untieRegOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ) ; } Src2 -> ChangeToImmediate ( Imm ) ;


<s> if ( isStride64 ( Opc ) ) EltSize *= <NUM_LIT> ; const MachineOperand * AddrReg = getNamedOperand ( * LdSt , <STR_LIT> :: addr ) ; BaseReg = AddrReg -> getReg ( ) ; Offset = EltSize * Offset0 ; return true ; } return false ; } if ( isMUBUF ( * LdSt ) || isMTBUF ( * LdSt ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: soffset ) != - <NUM_LIT> ) return false ; const MachineOperand * AddrReg = getNamedOperand ( * LdSt , <STR_LIT> :: vaddr ) ; if ( ! AddrReg ) return false ; const MachineOperand * OffsetImm = getNamedOperand ( * LdSt , <STR_LIT> :: offset ) ; BaseReg = AddrReg -> getReg ( ) ; Offset = OffsetImm -> getImm ( ) ; return true ; } if ( isSMRD ( * LdSt ) ) { const MachineOperand * OffsetImm = getNamedOperand ( * LdSt , <STR_LIT> :: offset ) ; if ( ! OffsetImm ) return false ; const MachineOperand * SBaseReg = getNamedOperand ( * LdSt , <STR_LIT> :: sbase ) ; BaseReg = SBaseReg -> getReg ( ) ; Offset = OffsetImm -> getImm ( ) ; return true ; } if ( isFLAT ( * LdSt ) ) { const MachineOperand * AddrReg = getNamedOperand ( * LdSt , <STR_LIT> :: addr ) ;


<s> Offset = OffsetImm -> getImm ( ) ; return true ; } const MachineOperand * Offset0Imm = getNamedOperand ( * LdSt , <STR_LIT> :: offset0 ) ; const MachineOperand * Offset1Imm = getNamedOperand ( * LdSt , <STR_LIT> :: offset1 ) ; uint8_t Offset0 = Offset0Imm -> getImm ( ) ; uint8_t Offset1 = Offset1Imm -> getImm ( ) ; if ( Offset1 > Offset0 && Offset1 - Offset0 == <NUM_LIT> ) { unsigned EltSize ; if ( LdSt -> mayLoad ( ) ) EltSize = getOpRegClass ( * LdSt , <NUM_LIT> ) -> getSize ( ) / <NUM_LIT> ; else { assert ( LdSt -> mayStore ( ) ) ; int Data0Idx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; EltSize = getOpRegClass ( * LdSt , Data0Idx ) -> getSize ( ) ; } if ( isStride64 ( Opc ) ) EltSize *= <NUM_LIT> ; const MachineOperand * AddrReg = getNamedOperand ( * LdSt , <STR_LIT> :: addr ) ; BaseReg = AddrReg -> getReg ( ) ; Offset = EltSize * Offset0 ; return true ; } return false ; } if ( isMUBUF ( * LdSt ) || isMTBUF ( * LdSt ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: soffset ) != - <NUM_LIT> ) return false ; const MachineOperand * AddrReg = getNamedOperand ( * LdSt , <STR_LIT> :: vaddr ) ; if ( ! AddrReg ) return false ; const MachineOperand * OffsetImm = getNamedOperand ( * LdSt , <STR_LIT> :: offset ) ; BaseReg = AddrReg -> getReg ( ) ; Offset = OffsetImm -> getImm ( ) ; return true ; } if ( isSMRD ( * LdSt ) ) { const MachineOperand * OffsetImm = getNamedOperand ( * LdSt , <STR_LIT> :: offset ) ; if ( ! OffsetImm ) return false ; const MachineOperand * SBaseReg = getNamedOperand ( * LdSt , <STR_LIT> :: sbase ) ;


<s> uint64_t Rsrc23 = getDefaultRsrcDataFormat ( ) | <STR_LIT> | <NUM_LIT> ; uint64_t EltSizeValue = Log2_32 ( ST . getMaxPrivateElementSize ( ) ) - <NUM_LIT> ; Rsrc23 |= ( EltSizeValue << <STR_LIT> ) | ( UINT64_C ( <NUM_LIT> ) << <STR_LIT> ) ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) Rsrc23 &= ~ <STR_LIT> ; return Rsrc23 ;


<s> uint64_t EltSizeValue = Log2_32 ( ST . getMaxPrivateElementSize ( ) ) - <NUM_LIT> ; Rsrc23 |= ( EltSizeValue << <STR_LIT> ) | ( UINT64_C ( <NUM_LIT> ) << <STR_LIT> ) ;

if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) Rsrc23 &= ~ <STR_LIT> ;


<s> unsigned Opc = MI -> getOpcode ( ) ; return isMUBUF ( Opc ) || isMTBUF ( Opc ) || isMIMG ( Opc ) ;


<s> bool <ISA_LIT>InstrInfo :: isLowLatencyInstruction ( const MachineInstr * MI ) const { unsigned Opc = MI -> getOpcode ( ) ;


<s> const TargetRegisterInfo * TRI = MF . getSubtarget ( ) . getRegisterInfo ( ) ;

if ( MI -> modifiesRegister ( <STR_LIT> , TRI ) ) return true ;


<s> MachineOperand * SBase = getNamedOperand ( * MI , <STR_LIT> :: sbase ) ; if ( SBase && ! RI . isSGPRClass ( MRI . getRegClass ( SBase -> getReg ( ) ) ) ) { unsigned SGPR = readlaneVGPRToSGPR ( SBase -> getReg ( ) , MI , MRI ) ; SBase -> setReg ( SGPR ) ;


<s> if ( SBase && ! RI . isSGPRClass ( MRI . getRegClass ( SBase -> getReg ( ) ) ) ) { unsigned SGPR = readlaneVGPRToSGPR ( SBase -> getReg ( ) , MI , MRI ) ; SBase -> setReg ( SGPR ) ; }


<s> } break ; case <STR_LIT> : lowerScalarAbs ( Worklist , Inst ) ; Inst -> eraseFromParent ( ) ; continue ; case <STR_LIT> : case <STR_LIT> : BuildMI ( * MBB , Inst , Inst -> getDebugLoc ( ) , get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ; break ; case <STR_LIT> : case <STR_LIT> : llvm_unreachable ( <STR_LIT> ) ; } if ( NewOpcode == <STR_LIT> ) { legalizeOperands ( Inst ) ; continue ; } const MCInstrDesc & NewDesc = get ( NewOpcode ) ; Inst -> setDesc ( NewDesc ) ; for ( unsigned i = Inst -> getNumOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; -- i ) { MachineOperand & Op = Inst -> getOperand ( i ) ; if ( Op . isReg ( ) && Op . getReg ( ) == <STR_LIT> ) { Inst -> RemoveOperand ( i ) ; addSCCDefUsersToVALUWorklist ( Inst , Worklist ) ; } } if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { unsigned Size = ( Opcode == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; Inst -> addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; Inst -> addOperand ( MachineOperand :: CreateImm ( Size ) ) ; } else if ( Opcode == <STR_LIT> ) { Inst -> addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; } Inst -> addImplicitDefUseOperands ( * Inst -> getParent ( ) -> getParent ( ) ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { const MachineOperand & OffsetWidthOp = Inst -> getOperand ( <NUM_LIT> ) ; assert ( OffsetWidthOp . isImm ( ) && <STR_LIT> ) ; uint32_t Imm = OffsetWidthOp . getImm ( ) ; uint32_t Offset = Imm & <NUM_LIT> ; uint32_t BitWidth = ( Imm & <NUM_LIT> ) >> <NUM_LIT> ; Inst -> RemoveOperand ( <NUM_LIT> ) ; Inst -> addOperand ( MachineOperand :: CreateImm ( Offset ) ) ; Inst -> addOperand ( MachineOperand :: CreateImm ( BitWidth ) ) ; } bool HasDst = Inst -> getOperand ( <NUM_LIT> ) . isReg ( ) && Inst -> getOperand ( <NUM_LIT> ) . isDef ( ) ; unsigned NewDstReg = <STR_LIT> ; if ( HasDst ) { const TargetRegisterClass * NewDstRC = getDestEquivalentVGPRClass ( * Inst ) ; if ( ! NewDstRC ) continue ; unsigned DstReg = Inst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; NewDstReg = MRI . createVirtualRegister ( NewDstRC ) ; MRI . replaceRegWith ( DstReg , NewDstReg ) ; } legalizeOperands ( Inst ) ;

if ( HasDst ) addUsersToMoveToVALUWorklist ( NewDstReg , MRI , Worklist ) ;


<s> break ; case <STR_LIT> : lowerScalarAbs ( Worklist , Inst ) ; Inst -> eraseFromParent ( ) ; continue ; case <STR_LIT> : case <STR_LIT> : BuildMI ( * MBB , Inst , Inst -> getDebugLoc ( ) , get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ; break ; case <STR_LIT> : case <STR_LIT> : llvm_unreachable ( <STR_LIT> ) ; } if ( NewOpcode == <STR_LIT> ) { legalizeOperands ( Inst ) ; continue ; } const MCInstrDesc & NewDesc = get ( NewOpcode ) ; Inst -> setDesc ( NewDesc ) ; for ( unsigned i = Inst -> getNumOperands ( ) - <NUM_LIT> ; i > <NUM_LIT> ; -- i ) { MachineOperand & Op = Inst -> getOperand ( i ) ; if ( Op . isReg ( ) && Op . getReg ( ) == <STR_LIT> ) { Inst -> RemoveOperand ( i ) ; addSCCDefUsersToVALUWorklist ( Inst , Worklist ) ; } } if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { unsigned Size = ( Opcode == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; Inst -> addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; Inst -> addOperand ( MachineOperand :: CreateImm ( Size ) ) ; } else if ( Opcode == <STR_LIT> ) { Inst -> addOperand ( MachineOperand :: CreateImm ( <NUM_LIT> ) ) ; } Inst -> addImplicitDefUseOperands ( * Inst -> getParent ( ) -> getParent ( ) ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { const MachineOperand & OffsetWidthOp = Inst -> getOperand ( <NUM_LIT> ) ; assert ( OffsetWidthOp . isImm ( ) && <STR_LIT> ) ; uint32_t Imm = OffsetWidthOp . getImm ( ) ; uint32_t Offset = Imm & <NUM_LIT> ; uint32_t BitWidth = ( Imm & <NUM_LIT> ) >> <NUM_LIT> ; Inst -> RemoveOperand ( <NUM_LIT> ) ; Inst -> addOperand ( MachineOperand :: CreateImm ( Offset ) ) ; Inst -> addOperand ( MachineOperand :: CreateImm ( BitWidth ) ) ; } bool HasDst = Inst -> getOperand ( <NUM_LIT> ) . isReg ( ) && Inst -> getOperand ( <NUM_LIT> ) . isDef ( ) ; unsigned NewDstReg = <STR_LIT> ; if ( HasDst ) { const TargetRegisterClass * NewDstRC = getDestEquivalentVGPRClass ( * Inst ) ; if ( ! NewDstRC ) continue ; unsigned DstReg = Inst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; NewDstReg = MRI . createVirtualRegister ( NewDstRC ) ;

MRI . replaceRegWith ( DstReg , NewDstReg ) ;


<s> unsigned DstReg = MRI . createVirtualRegister ( SRC ) ; unsigned SubRegs = VRC -> getSize ( ) / <NUM_LIT> ; SmallVector < unsigned , <NUM_LIT> > SRegs ; for ( unsigned i = <NUM_LIT> ; i < SubRegs ; ++ i ) { unsigned SGPR = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * UseMI -> getParent ( ) , UseMI , UseMI -> getDebugLoc ( ) , get ( <STR_LIT> ) , SGPR ) . addReg ( SrcReg , <NUM_LIT> , RI . getSubRegFromChannel ( i ) ) ; SRegs . push_back ( SGPR ) ; } MachineInstrBuilder MIB = BuildMI ( * UseMI -> getParent ( ) , UseMI , UseMI -> getDebugLoc ( ) , get ( <STR_LIT> ) , DstReg ) ; for ( unsigned i = <NUM_LIT> ; i < SubRegs ; ++ i ) { MIB . addReg ( SRegs [ i ] ) ; MIB . addImm ( RI . getSubRegFromChannel ( i ) ) ; } return DstReg ;


<s> MachineInstrBuilder MIB = BuildMI ( * UseMI -> getParent ( ) , UseMI , UseMI -> getDebugLoc ( ) , get ( <STR_LIT> ) , DstReg ) ; for ( unsigned i = <NUM_LIT> ; i < SubRegs ; ++ i ) { MIB . addReg ( SRegs [ i ] ) ; MIB . addImm ( RI . getSubRegFromChannel ( i ) ) ; }


<s> if ( isSMRD ( * FirstLdSt ) && isSMRD ( * SecondLdSt ) ) { FirstDst = getNamedOperand ( * FirstLdSt , <STR_LIT> :: sdst ) ; SecondDst = getNamedOperand ( * SecondLdSt , <STR_LIT> :: sdst ) ; } if ( ( isMUBUF ( * FirstLdSt ) && isMUBUF ( * SecondLdSt ) ) || ( isMTBUF ( * FirstLdSt ) && isMTBUF ( * SecondLdSt ) ) ) { FirstDst = getNamedOperand ( * FirstLdSt , <STR_LIT> :: vdata ) ; SecondDst = getNamedOperand ( * SecondLdSt , <STR_LIT> :: vdata ) ; } if ( ! FirstDst || ! SecondDst ) return false ; unsigned LoadClusterThreshold = <NUM_LIT> ; const MachineRegisterInfo & MRI = FirstLdSt -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; const TargetRegisterClass * DstRC = MRI . getRegClass ( FirstDst -> getReg ( ) ) ; return ( NumLoads * DstRC -> getSize ( ) ) <= LoadClusterThreshold ;


<s> MachineInstr & UseMI = * I -> getParent ( ) ; unsigned OpNo = <NUM_LIT> ; switch ( UseMI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : break ; default : OpNo = I . getOperandNo ( ) ;


<s> const TargetRegisterClass * <ISA_LIT>InstrInfo :: getDestEquivalentVGPRClass ( const MachineInstr & Inst ) const { const TargetRegisterClass * NewDstRC = getOpRegClass ( Inst , <NUM_LIT> ) ; switch ( Inst . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const TargetRegisterClass * SrcRC = getOpRegClass ( Inst , <NUM_LIT> ) ; if ( RI . hasAGPRs ( SrcRC ) ) { if ( RI . hasAGPRs ( NewDstRC ) ) return nullptr ; NewDstRC = RI . getEquivalentAGPRClass ( NewDstRC ) ; if ( ! NewDstRC ) return nullptr ; } else { if ( RI . hasVGPRs ( NewDstRC ) ) return nullptr ; NewDstRC = RI . getEquivalentVGPRClass ( NewDstRC ) ; if ( ! NewDstRC ) return nullptr ; } return NewDstRC ; } default :


<s> switch ( Inst . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const TargetRegisterClass * SrcRC = getOpRegClass ( Inst , <NUM_LIT> ) ; if ( RI . hasAGPRs ( SrcRC ) ) { if ( RI . hasAGPRs ( NewDstRC ) ) return nullptr ; NewDstRC = RI . getEquivalentAGPRClass ( NewDstRC ) ;


<s> case <STR_LIT> : break ; default : OpNo = I . getOperandNo ( ) ; break ; } if ( ! RI . hasVectorRegisters ( getOpRegClass ( UseMI , OpNo ) ) ) { Worklist . insert ( & UseMI ) ; do { ++ I ; } while ( I != E && I -> getParent ( ) == & UseMI ) ;


<s> MachineBasicBlock * MBB = MI . getParent ( ) ; ; MachineInstrBuilder Inst32 = BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( Op32 ) ) ; int Op32DstIdx = <STR_LIT> ( Op32 , <STR_LIT> :: vdst ) ; if ( Op32DstIdx != - <NUM_LIT> ) { Inst32 . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else { assert ( ( ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) || ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) ) && <STR_LIT> ) ; } Inst32 . add ( * getNamedOperand ( MI , <STR_LIT> :: src0 ) ) ; const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) Inst32 . add ( * Src1 ) ; const MachineOperand * Src2 = getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( Src2 ) { int Op32Src2Idx = <STR_LIT> ( Op32 , <STR_LIT> :: src2 ) ; if ( Op32Src2Idx != - <NUM_LIT> ) { Inst32 . add ( * Src2 ) ; } else { copyFlagsToImplicitVCC ( * Inst32 , * Src2 ) ; } } return Inst32 ;


<s> int Op32DstIdx = <STR_LIT> ( Op32 , <STR_LIT> :: vdst ) ; if ( Op32DstIdx != - <NUM_LIT> ) { Inst32 . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else { assert ( ( ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) || ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) ) && <STR_LIT> ) ; } Inst32 . add ( * getNamedOperand ( MI , <STR_LIT> :: src0 ) ) ; const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) Inst32 . add ( * Src1 ) ; const MachineOperand * Src2 = getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( Src2 ) { int Op32Src2Idx = <STR_LIT> ( Op32 , <STR_LIT> :: src2 ) ; if ( Op32Src2Idx != - <NUM_LIT> ) { Inst32 . add ( * Src2 ) ; } else { copyFlagsToImplicitVCC ( * Inst32 , * Src2 ) ; } }


<s> const MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; const TargetRegisterClass * RC = MRI . getRegClass ( TrueReg ) ; assert ( MRI . getRegClass ( FalseReg ) == RC ) ; int NumInsts = <STR_LIT> ( RC -> getID ( ) ) / <NUM_LIT> ; if ( NumInsts % <NUM_LIT> == <NUM_LIT> ) NumInsts /= <NUM_LIT> ; CondCycles = TrueCycles = FalseCycles = NumInsts ; return RI . isSGPRClass ( RC ) ;


<s> case SCC_FALSE : { const MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; const TargetRegisterClass * RC = MRI . getRegClass ( TrueReg ) ; assert ( MRI . getRegClass ( FalseReg ) == RC ) ; int NumInsts = <STR_LIT> ( RC -> getID ( ) ) / <NUM_LIT> ; if ( NumInsts % <NUM_LIT> == <NUM_LIT> ) NumInsts /= <NUM_LIT> ; CondCycles = TrueCycles = FalseCycles = NumInsts ; return RI . isSGPRClass ( RC ) ; } default : return false ;


<s> unsigned DstReg = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ; MachineInstr * <ISA_LIT>IF = BuildMI ( * MF , Branch -> getDebugLoc ( ) , get ( <STR_LIT> ) , DstReg ) . add ( Branch -> getOperand ( <NUM_LIT> ) ) . add ( Branch -> getOperand ( <NUM_LIT> ) ) ; MachineInstr * <ISA_LIT>END = BuildMI ( * MF , Branch -> getDebugLoc ( ) , get ( <STR_LIT> ) ) . addReg ( DstReg ) ; IfEntry -> erase ( TI ) ; IfEntry -> insert ( IfEntry -> end ( ) , <ISA_LIT>IF ) ;

IfEnd -> insert ( IfEnd -> getFirstNonPHI ( ) , <ISA_LIT>END ) ;


<s> MachineRegisterInfo & MRI = IfEntry -> getParent ( ) -> getRegInfo ( ) ; if ( Branch -> getOpcode ( ) == <STR_LIT> ) { unsigned DstReg = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ; MachineInstr * <ISA_LIT>IF = BuildMI ( * MF , Branch -> getDebugLoc ( ) , get ( <STR_LIT> ) , DstReg ) . add ( Branch -> getOperand ( <NUM_LIT> ) ) . add ( Branch -> getOperand ( <NUM_LIT> ) ) ; MachineInstr * <ISA_LIT>END = BuildMI ( * MF , Branch -> getDebugLoc ( ) , get ( <STR_LIT> ) ) . addReg ( DstReg ) ;


<s> } MachineInstr * HeaderPhi = HeaderPHIBuilder ; MachineInstr * <ISA_LIT>IFBREAK = BuildMI ( * ( MF ) , Branch -> getDebugLoc ( ) , get ( <STR_LIT> ) , BackEdgeReg ) . addReg ( DstReg ) . add ( Branch -> getOperand ( <NUM_LIT> ) ) ; MachineInstr * <ISA_LIT>LOOP = BuildMI ( * ( MF ) , Branch -> getDebugLoc ( ) , get ( <STR_LIT> ) ) . addReg ( BackEdgeReg ) . addMBB ( LoopEntry ) ; LoopEntry -> insert ( LoopEntry -> begin ( ) , HeaderPhi ) ; LoopEnd -> erase ( TI ) ; LoopEnd -> insert ( LoopEnd -> end ( ) , <ISA_LIT>IFBREAK ) ; LoopEnd -> insert ( LoopEnd -> end ( ) , <ISA_LIT>LOOP ) ; }


<s> MachineBasicBlock :: iterator I = LoopBB . begin ( ) ; unsigned VRsrc = Rsrc . getReg ( ) ; unsigned VRsrcUndef = getUndefRegState ( Rsrc . isUndef ( ) ) ; unsigned SaveExec = MRI . createVirtualRegister ( BoolXExecRC ) ; unsigned CondReg0 = MRI . createVirtualRegister ( BoolXExecRC ) ; unsigned CondReg1 = MRI . createVirtualRegister ( BoolXExecRC ) ; unsigned AndCond = MRI . createVirtualRegister ( BoolXExecRC ) ; unsigned SRsrcSub0 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcSub1 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcSub2 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcSub3 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrc = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub0 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub1 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub2 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub3 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrc ) . addReg ( SRsrcSub0 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcSub1 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcSub2 ) . addImm ( <STR_LIT> ) . addReg ( SRsrcSub3 ) . addImm ( <STR_LIT> ) ; Rsrc . setReg ( SRsrc ) ; Rsrc . setIsKill ( true ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CondReg0 ) . addReg ( SRsrc , <NUM_LIT> , <STR_LIT> ) . addReg ( VRsrc , <NUM_LIT> , <STR_LIT> ) ;

BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , CondReg1 ) . addReg ( SRsrc , <NUM_LIT> , <STR_LIT> ) . addReg ( VRsrc , <NUM_LIT> , <STR_LIT> ) ;


<s> static void emitLoadSRsrcFromVGPRLoop ( const <ISA_LIT>InstrInfo & TII , MachineRegisterInfo & MRI , MachineBasicBlock & OrigBB , MachineBasicBlock & LoopBB , const DebugLoc & DL , MachineOperand & Rsrc ) { MachineFunction & MF = * OrigBB . getParent ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned SaveExecOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned XorTermOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned AndOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; const auto * BoolXExecRC = TRI -> getRegClass ( <STR_LIT> ) ; MachineBasicBlock :: iterator I = LoopBB . begin ( ) ; unsigned VRsrc = Rsrc . getReg ( ) ; unsigned VRsrcUndef = getUndefRegState ( Rsrc . isUndef ( ) ) ; unsigned SaveExec = MRI . createVirtualRegister ( BoolXExecRC ) ; unsigned CondReg0 = MRI . createVirtualRegister ( BoolXExecRC ) ; unsigned CondReg1 = MRI . createVirtualRegister ( BoolXExecRC ) ; unsigned AndCond = MRI . createVirtualRegister ( BoolXExecRC ) ; unsigned SRsrcSub0 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcSub1 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcSub2 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrcSub3 = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SRsrc = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub0 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ;

BuildMI ( LoopBB , I , DL , TII . get ( <STR_LIT> ) , SRsrcSub1 ) . addReg ( VRsrc , VRsrcUndef , <STR_LIT> ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return MO . getReg ( ) ; default : break ; } } return <STR_LIT> ;


<s> unsigned UnusedCarry = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ; MRI . setRegAllocationHint ( UnusedCarry , <NUM_LIT> , RI . getVCC ( ) ) ;

return BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( UnusedCarry , RegState :: Define | RegState :: Dead ) ;


<s> if ( ST . hasAddNoCarry ( ) ) return BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DestReg ) ; MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; unsigned UnusedCarry = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ; MRI . setRegAllocationHint ( UnusedCarry , <NUM_LIT> , RI . getVCC ( ) ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const TargetRegisterClass * SrcRC = getOpRegClass ( Inst , <NUM_LIT> ) ; if ( RI . hasAGPRs ( SrcRC ) ) { if ( RI . hasAGPRs ( NewDstRC ) ) return nullptr ; NewDstRC = RI . getEquivalentAGPRClass ( NewDstRC ) ; if ( ! NewDstRC ) return nullptr ; } else { if ( RI . hasVGPRs ( NewDstRC ) ) return nullptr ; NewDstRC = RI . getEquivalentVGPRClass ( NewDstRC ) ; if ( ! NewDstRC ) return nullptr ; }


<s> case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; }


<s> return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ;


<s> return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> :


<s> case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ;


<s> if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return true ; if ( MI . isCall ( ) || MI . isInlineAsm ( ) ) return true ;

if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return true ;


<s> MachineRegisterInfo & MRI = MBB -> getParent ( ) -> getRegInfo ( ) ; unsigned Reg = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ;

BuildMI ( * MBB , I , DL , get ( <STR_LIT> ) , Reg ) . addImm ( Value ) . addReg ( SrcReg ) ;


<s> unsigned <ISA_LIT>InstrInfo :: insertNE ( MachineBasicBlock * MBB , MachineBasicBlock :: iterator I , const DebugLoc & DL , unsigned SrcReg , int Value ) const { MachineRegisterInfo & MRI = MBB -> getParent ( ) -> getRegInfo ( ) ;

unsigned Reg = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ;


<s> return ; } if ( DstSize == <NUM_LIT> && Pred == SCC_TRUE ) { MachineInstr * Select = BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addReg ( FalseReg ) . addReg ( TrueReg ) ; preserveCondRegFlags ( Select -> getOperand ( <NUM_LIT> ) , Cond [ <NUM_LIT> ] ) ; return ; } static const int16_t Sub0_15 [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , } ; static const int16_t Sub0_15_64 [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , } ; unsigned SelOp = <STR_LIT> ; const TargetRegisterClass * EltRC = & <STR_LIT> ; const int16_t * SubIndices = Sub0_15 ; int NElts = DstSize / <NUM_LIT> ; if ( Pred == SCC_TRUE ) { if ( NElts % <NUM_LIT> ) { SelOp = <STR_LIT> ; EltRC = & <STR_LIT> ; } else { SelOp = <STR_LIT> ; EltRC = & <STR_LIT> ; SubIndices = Sub0_15_64 ; NElts /= <NUM_LIT> ; } }

MachineInstrBuilder MIB = BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) ;


<s> BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: EXECNZ : { unsigned SReg = MRI . createVirtualRegister ( BoolXExecRC ) ; unsigned SReg2 = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg2 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: EXECZ : { unsigned SReg = MRI . createVirtualRegister ( BoolXExecRC ) ; unsigned SReg2 = MRI . createVirtualRegister ( RI . getBoolRC ( ) ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg2 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , SReg ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addReg ( FalseReg ) . addImm ( <NUM_LIT> ) . addReg ( TrueReg ) . addReg ( SReg ) ; llvm_unreachable ( <STR_LIT> ) ; break ;


<s> const MCInstrDesc & InstDesc = MI . getDesc ( ) ; const MCOperandInfo & OpInfo = InstDesc . OpInfo [ OpNo ] ; assert ( MO . isImm ( ) || MO . isTargetIndex ( ) || MO . isFI ( ) || MO . isGlobal ( ) ) ;

if ( OpInfo . OperandType == MCOI :: OPERAND_IMMEDIATE ) return true ;


<s> bool HasImplicitSGPR = findImplicitSGPRRead ( MI ) != <STR_LIT> ; if ( HasImplicitSGPR && ST . getConstantBusLimit ( Opc ) <= <NUM_LIT> && Src0 . isReg ( ) && ( RI . isSGPRReg ( MRI , Src0 . getReg ( ) ) || isLiteralConstantLike ( Src0 , InstrDesc . OpInfo [ Src0Idx ] ) ) ) legalizeOpWithMove ( MI , Src0Idx ) ; if ( Opc == <STR_LIT> ) { const DebugLoc & DL = MI . getDebugLoc ( ) ; if ( Src0 . isReg ( ) && RI . isVGPR ( MRI , Src0 . getReg ( ) ) ) { unsigned Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MI . getParent ( ) , MI , DL , get ( <STR_LIT> ) , Reg ) . add ( Src0 ) ; Src0 . ChangeToRegister ( Reg , false ) ; } if ( Src1 . isReg ( ) && RI . isVGPR ( MRI , Src1 . getReg ( ) ) ) { unsigned Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; BuildMI ( * MI . getParent ( ) , MI , DL , get ( <STR_LIT> ) , Reg ) . add ( Src1 ) ;

Src1 . ChangeToRegister ( Reg , false ) ;


<s> MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } if ( RI . spillSGPRToVGPR ( ) ) FrameInfo . setStackID ( FrameIndex , TargetStackID :: SGPRSpill ) ; MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getStackPtrOffsetReg ( ) , RegState :: Implicit ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ; } return ; } unsigned Opcode = RI . hasAGPRs ( RC ) ? getAGPRSpillRestoreOpcode ( SpillSize ) : getVGPRSpillRestoreOpcode ( SpillSize ) ; auto MIB = BuildMI ( MBB , MI , DL , get ( Opcode ) , DestReg ) ;


<s> MachineFunction & MF = * MBB . getParent ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MachineBasicBlock :: iterator I ( & MI ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned MovExecOpc = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; const auto * BoolXExecRC = TRI -> getRegClass ( <STR_LIT> ) ; unsigned SaveExec = MRI . createVirtualRegister ( BoolXExecRC ) ; BuildMI ( MBB , I , DL , TII . get ( MovExecOpc ) , SaveExec ) . addReg ( Exec ) ; for ( auto & MO : MI . uses ( ) ) { if ( MO . isReg ( ) && MO . isUse ( ) ) { MRI . clearKillFlags ( MO . getReg ( ) ) ; } } MachineBasicBlock * LoopBB = MF . CreateMachineBasicBlock ( ) ; MachineBasicBlock * RemainderBB = MF . CreateMachineBasicBlock ( ) ; MachineFunction :: iterator MBBI ( MBB ) ; ++ MBBI ; MF . insert ( MBBI , LoopBB ) ; MF . insert ( MBBI , RemainderBB ) ; LoopBB -> addSuccessor ( LoopBB ) ; LoopBB -> addSuccessor ( RemainderBB ) ; MachineBasicBlock :: iterator J = I ++ ; RemainderBB -> transferSuccessorsAndUpdatePHIs ( & MBB ) ; RemainderBB -> splice ( RemainderBB -> begin ( ) , & MBB , I , MBB . end ( ) ) ; LoopBB -> splice ( LoopBB -> begin ( ) , & MBB , J ) ; MBB . addSuccessor ( LoopBB ) ; if ( MDT ) { MDT -> addNewBlock ( LoopBB , & MBB ) ; MDT -> addNewBlock ( RemainderBB , LoopBB ) ; for ( auto & Succ : RemainderBB -> successors ( ) ) { if ( MDT -> dominates ( & MBB , Succ ) ) { MDT -> changeImmediateDominator ( Succ , RemainderBB ) ; } } } emitLoadSRsrcFromVGPRLoop ( TII , MRI , MBB , * LoopBB , DL , Rsrc ) ; MachineBasicBlock :: iterator First = RemainderBB -> begin ( ) ;

BuildMI ( * RemainderBB , First , DL , TII . get ( MovExecOpc ) , Exec ) . addReg ( SaveExec ) ;


<s> void <ISA_LIT>InstrInfo :: lowerScalarXnor ( SetVectorType & Worklist , MachineInstr & Inst ) const { MachineBasicBlock & MBB = * Inst . getParent ( ) ; MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; MachineBasicBlock :: iterator MII = Inst ; const DebugLoc & DL = Inst . getDebugLoc ( ) ; MachineOperand & Dest = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src0 = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = Inst . getOperand ( <NUM_LIT> ) ; if ( ST . hasDLInsts ( ) ) { unsigned NewDest = MRI . createVirtualRegister ( & <STR_LIT> ) ; legalizeGenericOperand ( MBB , MII , & <STR_LIT> , Src0 , MRI , DL ) ; legalizeGenericOperand ( MBB , MII , & <STR_LIT> , Src1 , MRI , DL ) ; BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , NewDest ) . add ( Src0 ) . add ( Src1 ) ; MRI . replaceRegWith ( Dest . getReg ( ) , NewDest ) ; addUsersToMoveToVALUWorklist ( NewDest , MRI , Worklist ) ; } else { bool Src0IsSGPR = Src0 . isReg ( ) && RI . isSGPRClass ( MRI . getRegClass ( Src0 . getReg ( ) ) ) ; bool Src1IsSGPR = Src1 . isReg ( ) && RI . isSGPRClass ( MRI . getRegClass ( Src1 . getReg ( ) ) ) ; MachineInstr * Xor ; unsigned Temp = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned NewDest = MRI . createVirtualRegister ( & <STR_LIT> ) ; if ( Src0IsSGPR ) {

BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , Temp ) . add ( Src0 ) ;


<s> unsigned NewSrcReg = MRI . createVirtualRegister ( VRC ) ; BuildMI ( * UseMI . getParent ( ) , UseMI , UseMI . getDebugLoc ( ) , get ( TargetOpcode :: COPY ) , NewSrcReg ) . addReg ( SrcReg ) ; SrcReg = NewSrcReg ; } if ( SubRegs == <NUM_LIT> ) { BuildMI ( * UseMI . getParent ( ) , UseMI , UseMI . getDebugLoc ( ) , get ( <STR_LIT> ) , DstReg ) . addReg ( SrcReg ) ; return DstReg ; } SmallVector < unsigned , <NUM_LIT> > SRegs ; for ( unsigned i = <NUM_LIT> ; i < SubRegs ; ++ i ) { unsigned SGPR = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * UseMI . getParent ( ) , UseMI , UseMI . getDebugLoc ( ) , get ( <STR_LIT> ) , SGPR ) . addReg ( SrcReg , <NUM_LIT> , RI . getSubRegFromChannel ( i ) ) ; SRegs . push_back ( SGPR ) ; }

MachineInstrBuilder MIB = BuildMI ( * UseMI . getParent ( ) , UseMI , UseMI . getDebugLoc ( ) , get ( <STR_LIT> ) , DstReg ) ;


<s> MachineBasicBlock :: iterator MII = Inst ; const TargetRegisterClass * Src0RC = MRI . getRegClass ( Src0 . getReg ( ) ) ; const TargetRegisterClass * Src1RC = MRI . getRegClass ( Src1 . getReg ( ) ) ; const TargetRegisterClass * Src0SubRC = RI . getSubRegClass ( Src0RC , <STR_LIT> ) ; const TargetRegisterClass * Src1SubRC = RI . getSubRegClass ( Src1RC , <STR_LIT> ) ; MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm ( MII , MRI , Src1 , Src1RC , <STR_LIT> , Src1SubRC ) ; MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src1 , Src1RC , <STR_LIT> , Src1SubRC ) ; unsigned LoOpc = IsAdd ? <STR_LIT> : <STR_LIT> ; MachineInstr * LoHalf = BuildMI ( MBB , MII , DL , get ( LoOpc ) , DestSub0 ) . addReg ( CarryReg , RegState :: Define ) . add ( SrcReg0Sub0 ) . add ( SrcReg1Sub0 ) . addImm ( <NUM_LIT> ) ; unsigned HiOpc = IsAdd ? <STR_LIT> : <STR_LIT> ; MachineInstr * HiHalf = BuildMI ( MBB , MII , DL , get ( HiOpc ) , DestSub1 ) . addReg ( DeadCarryReg , RegState :: Define | RegState :: Dead ) . add ( SrcReg0Sub1 ) . add ( SrcReg1Sub1 ) . addReg ( CarryReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , MII , DL , get ( TargetOpcode :: REG_SEQUENCE ) , FullDestReg ) . addReg ( DestSub0 ) . addImm ( <STR_LIT> ) . addReg ( DestSub1 ) . addImm ( <STR_LIT> ) ; MRI . replaceRegWith ( Dest . getReg ( ) , FullDestReg ) ; legalizeOperands ( * LoHalf , MDT ) ; legalizeOperands ( * HiHalf , MDT ) ;

addUsersToMoveToVALUWorklist ( FullDestReg , MRI , Worklist ) ;