<s> const TargetRegisterClass * RC = TII -> getOpRegClass ( MI , i ) ; RegInterval Interval = getRegInterval ( RC , Op ) ; for ( unsigned j = Interval . first ; j < Interval . second ; ++ j ) { if ( Op . isDef ( ) ) { increaseCounters ( Result , UsedRegs [ j ] ) ; increaseCounters ( Result , DefinedRegs [ j ] ) ; } if ( Op . isUse ( ) ) increaseCounters ( Result , DefinedRegs [ j ] ) ; } }


<s> if ( ST -> getGeneration ( ) < <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return ; if ( LastInstWritesM0 && ( I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; LastInstWritesM0 = false ; return ; } LastInstWritesM0 = false ; unsigned NumOperands = I -> getNumOperands ( ) ; for ( unsigned i = <NUM_LIT> ; i < NumOperands ; i ++ ) { const MachineOperand & Op = I -> getOperand ( i ) ;

if ( Op . isReg ( ) && Op . isDef ( ) && Op . getReg ( ) == <STR_LIT> ) LastInstWritesM0 = true ;


<s> bool <ISA_LIT>InsertWaits :: hasOutstandingLGKM ( ) const { return WaitedOn . Named . LGKM != LastIssued . Named . LGKM ;


<s> static bool hasTrivialSuccessor ( const MachineBasicBlock & MBB ) { if ( MBB . succ_size ( ) != <NUM_LIT> ) return false ;

const MachineBasicBlock * Succ = * MBB . succ_begin ( ) ;


<s> if ( MBB . succ_size ( ) != <NUM_LIT> ) return false ; const MachineBasicBlock * Succ = * MBB . succ_begin ( ) ;

return ( Succ -> pred_size ( ) == <NUM_LIT> ) && MBB . isLayoutSuccessor ( Succ ) ;


<s> MachineOperand * Data = TII -> getNamedOperand ( MI , <STR_LIT> :: vdata ) ; if ( Data && Op . isIdenticalTo ( * Data ) ) return true ; } for ( MachineInstr :: mop_iterator I = MI . operands_begin ( ) , E = MI . operands_end ( ) ; I != E ; ++ I ) { if ( I -> isReg ( ) && I -> isUse ( ) ) return Op . isIdenticalTo ( * I ) ; }


<s> LastIssued . Array [ i ] += Increment . Array [ i ] ; if ( Increment . Array [ i ] ) Limit . Array [ i ] = LastIssued . Array [ i ] ; Sum += Increment . Array [ i ] ; } if ( Sum == <NUM_LIT> ) { LastOpcodeType = OTHER ; return ; } if ( ST -> getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( LastOpcodeType == VMEM && Increment . Named . VM ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; LastInstWritesM0 = false ; } if ( TII -> isSMRD ( * I ) ) LastOpcodeType = SMEM ; else if ( Increment . Named . VM ) LastOpcodeType = VMEM ; } if ( Increment . Named . EXP ) { ExpInstrTypesSeen |= TII -> isEXP ( * I ) ? <NUM_LIT> : <NUM_LIT> ; } for ( unsigned i = <NUM_LIT> , e = I -> getNumOperands ( ) ; i != e ; ++ i ) { MachineOperand & Op = I -> getOperand ( i ) ; if ( ! isOpRelevant ( Op ) ) continue ; const TargetRegisterClass * RC = TII -> getOpRegClass ( * I , i ) ; RegInterval Interval = getRegInterval ( RC , Op ) ; for ( unsigned j = Interval . first ; j < Interval . second ; ++ j ) {

if ( Op . isDef ( ) ) DefinedRegs [ j ] = Limit ;


<s> <ISA_LIT>InsertWaits ( ) : MachineFunctionPass ( ID ) {


<s> assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) && <STR_LIT> ) ; const TargetRegisterClass * RC = TII -> getOpRegClass ( MI , <NUM_LIT> ) ; unsigned Size = RC -> getSize ( ) ; Result . Named . LGKM = Size > <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; } else { Result . Named . LGKM = <NUM_LIT> ; } } else { Result . Named . LGKM = <NUM_LIT> ; } } else { Result . Named . LGKM = <NUM_LIT> ; } return Result ;


<s> if ( TSFlags & <STR_LIT> ) { if ( TII -> isSMRD ( MI ) ) { if ( MI . getNumOperands ( ) != <NUM_LIT> ) { assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) && <STR_LIT> ) ; const TargetRegisterClass * RC = TII -> getOpRegClass ( MI , <NUM_LIT> ) ; unsigned Size = RC -> getSize ( ) ; Result . Named . LGKM = Size > <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; } else { Result . Named . LGKM = <NUM_LIT> ; } } else { Result . Named . LGKM = <NUM_LIT> ; } } else { Result . Named . LGKM = <NUM_LIT> ; } return Result ;


<s> Counters Counts , WaitOn ; Counts . Named . VM = decodeVmcnt ( IV , Imm ) ; Counts . Named . EXP = decodeExpcnt ( IV , Imm ) ; Counts . Named . LGKM = decodeLgkmcnt ( IV , Imm ) ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { if ( Counts . Array [ i ] <= LastIssued . Array [ i ] ) WaitOn . Array [ i ] = LastIssued . Array [ i ] - Counts . Array [ i ] ; else WaitOn . Array [ i ] = <NUM_LIT> ; } increaseCounters ( DelayedWaitOn , WaitOn ) ;


<s> if ( Required . Array [ i ] <= WaitedOn . Array [ i ] ) continue ; NeedWait = true ; if ( Ordered [ i ] ) { unsigned Value = LastIssued . Array [ i ] - Required . Array [ i ] ; Counts . Array [ i ] = std :: min ( Value , HardwareLimits . Array [ i ] ) ; } else Counts . Array [ i ] = <NUM_LIT> ; WaitedOn . Array [ i ] = LastIssued . Array [ i ] - Counts . Array [ i ] ; } if ( ! NeedWait ) return false ; if ( Counts . Named . EXP == <NUM_LIT> ) ExpInstrTypesSeen = <NUM_LIT> ;

BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( encodeWaitcnt ( IV , Counts . Named . VM , Counts . Named . EXP , Counts . Named . LGKM ) ) ;


<s> NeedWait = true ; if ( Ordered [ i ] ) { unsigned Value = LastIssued . Array [ i ] - Required . Array [ i ] ; Counts . Array [ i ] = std :: min ( Value , HardwareLimits . Array [ i ] ) ; } else Counts . Array [ i ] = <NUM_LIT> ; WaitedOn . Array [ i ] = LastIssued . Array [ i ] - Counts . Array [ i ] ; } if ( ! NeedWait ) return false ; if ( Counts . Named . EXP == <NUM_LIT> ) ExpInstrTypesSeen = <NUM_LIT> ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( encodeWaitcnt ( IV , Counts . Named . VM , Counts . Named . EXP , Counts . Named . LGKM ) ) ; LastOpcodeType = OTHER ;


<s> HardwareLimits . Named . EXP = <STR_LIT> ( ISA ) ; HardwareLimits . Named . LGKM = <STR_LIT> ( ISA ) ; WaitedOn = ZeroCounts ; DelayedWaitOn = ZeroCounts ; LastIssued = ZeroCounts ; LastOpcodeType = OTHER ; LastInstWritesM0 = false ; IsFlatOutstanding = false ; ReturnsVoid = MFI -> returnsVoid ( ) ; memset ( & UsedRegs , <NUM_LIT> , sizeof ( UsedRegs ) ) ; memset ( & DefinedRegs , <NUM_LIT> , sizeof ( DefinedRegs ) ) ; SmallVector < MachineInstr * , <NUM_LIT> > RemoveMI ; SmallVector < MachineBasicBlock * , <NUM_LIT> > EndPgmBlocks ; bool HaveScalarStores = false ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { if ( ! HaveScalarStores && TII -> isScalarStore ( * I ) ) HaveScalarStores = true ; if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: SEA_ISLANDS ) { if ( TII -> isSMRD ( I -> getOpcode ( ) ) ) { VCCZCorrupt = true ; } else if ( ! hasOutstandingLGKM ( ) && I -> modifiesRegister ( <STR_LIT> , TRI ) ) { VCCZCorrupt = false ; } if ( VCCZCorrupt && readsVCCZ ( * I ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * I << '\n' ) ; insertWait ( MBB , I , LastIssued ) ; BuildMI ( MBB , I , I -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; } } if ( I -> getOpcode ( ) == <STR_LIT> ) { handleExistingWait ( * I ) ; RemoveMI . push_back ( & * I ) ; continue ; } Counters Required ; if ( ( I -> getOpcode ( ) == <STR_LIT> && ! ST -> hasAutoWaitcntBeforeBarrier ( ) ) || I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) Required = LastIssued ; else Required = handleOperands ( * I ) ; Counters Increment = getHwCounts ( * I ) ; if ( countersNonZero ( Required ) || countersNonZero ( Increment ) ) increaseCounters ( Required , DelayedWaitOn ) ; Changes |= insertWait ( MBB , I , Required ) ; pushInstruction ( MBB , I , Increment ) ; handleSendMsg ( MBB , I ) ; if ( I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) EndPgmBlocks . push_back ( & MBB ) ;


<s> Ordered [ <NUM_LIT> ] = ExpInstrTypesSeen == <NUM_LIT> ; Ordered [ <NUM_LIT> ] = false ; Counters Counts = WaitCounts ; bool NeedWait = false ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { if ( Required . Array [ i ] <= WaitedOn . Array [ i ] ) continue ; NeedWait = true ; if ( Ordered [ i ] ) { unsigned Value = LastIssued . Array [ i ] - Required . Array [ i ] ; Counts . Array [ i ] = std :: min ( Value , WaitCounts . Array [ i ] ) ; } else Counts . Array [ i ] = <NUM_LIT> ; WaitedOn . Array [ i ] = LastIssued . Array [ i ] - Counts . Array [ i ] ; } if ( ! NeedWait ) return false ; if ( Counts . Named . EXP == <NUM_LIT> ) ExpInstrTypesSeen = <NUM_LIT> ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( ( Counts . Named . VM & <NUM_LIT> ) | ( ( Counts . Named . EXP & <NUM_LIT> ) << <NUM_LIT> ) | ( ( Counts . Named . LGKM & <NUM_LIT> ) << <NUM_LIT> ) ) ;


<s> memset ( & DefinedRegs , <NUM_LIT> , sizeof ( DefinedRegs ) ) ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { if ( I -> getOpcode ( ) == <STR_LIT> ) Changes |= insertWait ( MBB , I , LastIssued ) ; else Changes |= insertWait ( MBB , I , handleOperands ( * I ) ) ; pushInstruction ( MBB , I ) ; handleSendMsg ( MBB , I ) ; } Changes |= insertWait ( MBB , MBB . getFirstTerminator ( ) , LastIssued ) ; if ( ! ReturnsVoid ) { MachineBasicBlock :: iterator I = MBB . getFirstTerminator ( ) ; if ( I != MBB . end ( ) && I -> getOpcode ( ) == <STR_LIT> ) I -> eraseFromParent ( ) ; } }


<s> LastOpcodeType = OTHER ; LastInstWritesM0 = false ; ReturnsVoid = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ; memset ( & UsedRegs , <NUM_LIT> , sizeof ( UsedRegs ) ) ; memset ( & DefinedRegs , <NUM_LIT> , sizeof ( DefinedRegs ) ) ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ;

for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ;


<s> Counters Counts = WaitCounts ; bool NeedWait = false ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { if ( Required . Array [ i ] <= WaitedOn . Array [ i ] ) continue ; NeedWait = true ; if ( Ordered [ i ] ) { unsigned Value = LastIssued . Array [ i ] - Required . Array [ i ] ; Counts . Array [ i ] = std :: min ( Value , WaitCounts . Array [ i ] ) ; } else Counts . Array [ i ] = <NUM_LIT> ;

WaitedOn . Array [ i ] = LastIssued . Array [ i ] - Counts . Array [ i ] ;


<s> if ( ( LastOpcodeType == SMEM && TII -> isSMRD ( I -> getOpcode ( ) ) ) || ( LastOpcodeType == VMEM && Increment . Named . VM ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; LastInstWritesM0 = false ; } if ( TII -> isSMRD ( I -> getOpcode ( ) ) ) LastOpcodeType = SMEM ; else if ( Increment . Named . VM ) LastOpcodeType = VMEM ; } if ( Increment . Named . EXP ) { ExpInstrTypesSeen |= I -> getOpcode ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; } for ( unsigned i = <NUM_LIT> , e = I -> getNumOperands ( ) ; i != e ; ++ i ) { MachineOperand & Op = I -> getOperand ( i ) ; if ( ! isOpRelevant ( Op ) ) continue ; RegInterval Interval = getRegInterval ( Op ) ; for ( unsigned j = Interval . first ; j < Interval . second ; ++ j ) { if ( Op . isDef ( ) ) DefinedRegs [ j ] = Limit ; if ( Op . isUse ( ) ) UsedRegs [ j ] = Limit ;


<s> if ( TSFlags & <STR_LIT> ) { if ( TII -> isSMRD ( MI . getOpcode ( ) ) ) { MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; assert ( Op . isReg ( ) && <STR_LIT> ) ; unsigned Reg = Op . getReg ( ) ; unsigned Size = TRI -> getMinimalPhysRegClass ( Reg ) -> getSize ( ) ;

Result . Named . LGKM = Size > <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ;


<s> Result . Named . VM = ! ! ( TSFlags & <STR_LIT> ) ; Result . Named . EXP = ! ! ( TSFlags & <STR_LIT> && ( MI . getOpcode ( ) == <STR_LIT> || MI . getDesc ( ) . mayStore ( ) ) ) ; if ( TSFlags & <STR_LIT> ) {

if ( TII -> isSMRD ( MI . getOpcode ( ) ) ) {


<s> const char * getPassName ( ) const override { return <STR_LIT> ;


<s> RegInterval <ISA_LIT>InsertWaits :: getRegInterval ( MachineOperand & Op ) { if ( ! Op . isReg ( ) || ! TRI -> isInAllocatableClass ( Op . getReg ( ) ) ) return std :: make_pair ( <NUM_LIT> , <NUM_LIT> ) ; unsigned Reg = Op . getReg ( ) ; unsigned Size = TRI -> getMinimalPhysRegClass ( Reg ) -> getSize ( ) ; assert ( Size >= <NUM_LIT> ) ; RegInterval Result ; Result . first = TRI -> getEncodingValue ( Reg ) ;

Result . second = Result . first + Size / <NUM_LIT> ;


<s> if ( ! Op . isReg ( ) ) return false ; if ( Op . isDef ( ) ) return true ; MachineInstr & MI = * Op . getParent ( ) ; if ( MI . getOpcode ( ) == <STR_LIT> ) return true ; if ( ! MI . getDesc ( ) . mayStore ( ) ) return false ; if ( TII -> isDS ( MI . getOpcode ( ) ) ) { MachineOperand * Data = TII -> getNamedOperand ( MI , <STR_LIT> :: data ) ; if ( Data && Op . isIdenticalTo ( * Data ) ) return true ; MachineOperand * Data0 = TII -> getNamedOperand ( MI , <STR_LIT> :: data0 ) ; if ( Data0 && Op . isIdenticalTo ( * Data0 ) ) return true ; MachineOperand * Data1 = TII -> getNamedOperand ( MI , <STR_LIT> :: data1 ) ; if ( Data1 && Op . isIdenticalTo ( * Data1 ) ) return true ;


<s> MachineInstr & MI = * Op . getParent ( ) ; if ( MI . getOpcode ( ) == <STR_LIT> ) return true ; if ( ! MI . getDesc ( ) . mayStore ( ) ) return false ; if ( TII -> isDS ( MI . getOpcode ( ) ) ) { MachineOperand * Data = TII -> getNamedOperand ( MI , <STR_LIT> :: data ) ;

if ( Data && Op . isIdenticalTo ( * Data ) ) return true ;


<s> } if ( Sum == <NUM_LIT> ) { LastOpcodeType = OTHER ; return ; } if ( MBB . getParent ( ) -> getSubtarget < <ISA_LIT>Subtarget > ( ) . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( ( LastOpcodeType == SMEM && TII -> isSMRD ( I -> getOpcode ( ) ) ) || ( LastOpcodeType == VMEM && Increment . Named . VM ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; LastInstWritesM0 = false ; } if ( TII -> isSMRD ( I -> getOpcode ( ) ) ) LastOpcodeType = SMEM ; else if ( Increment . Named . VM ) LastOpcodeType = VMEM ; } if ( Increment . Named . EXP ) { ExpInstrTypesSeen |= I -> getOpcode ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; } for ( unsigned i = <NUM_LIT> , e = I -> getNumOperands ( ) ; i != e ; ++ i ) {

MachineOperand & Op = I -> getOperand ( i ) ;


<s> } if ( VCCZCorrupt && readsVCCZ ( * I ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * I << '\n' ) ; insertWait ( MBB , I , LastIssued ) ; BuildMI ( MBB , I , I -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; } } if ( I -> getOpcode ( ) == <STR_LIT> ) { handleExistingWait ( * I ) ; RemoveMI . push_back ( & * I ) ; continue ; } Counters Required ; if ( ( I -> getOpcode ( ) == <STR_LIT> && ST -> needWaitcntBeforeBarrier ( ) ) || I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) Required = LastIssued ; else Required = handleOperands ( * I ) ; Counters Increment = getHwCounts ( * I ) ; if ( countersNonZero ( Required ) || countersNonZero ( Increment ) ) increaseCounters ( Required , DelayedWaitOn ) ; Changes |= insertWait ( MBB , I , Required ) ; pushInstruction ( MBB , I , Increment ) ; handleSendMsg ( MBB , I ) ; if ( I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) EndPgmBlocks . push_back ( & MBB ) ; } if ( ! hasTrivialSuccessor ( MBB ) ) Changes |= insertWait ( MBB , MBB . getFirstTerminator ( ) , LastIssued ) ; } if ( HaveScalarStores ) { for ( MachineBasicBlock * MBB : EndPgmBlocks ) { bool SeenDCacheWB = false ; for ( MachineBasicBlock :: iterator I = MBB -> begin ( ) , E = MBB -> end ( ) ;


<s> Counts . Named . LGKM = ( Imm >> <NUM_LIT> ) & <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { if ( Counts . Array [ i ] <= LastIssued . Array [ i ] ) WaitOn . Array [ i ] = LastIssued . Array [ i ] - Counts . Array [ i ] ; else WaitOn . Array [ i ] = <NUM_LIT> ; }

increaseCounters ( DelayedWaitOn , WaitOn ) ;


<s> if ( Counts . Array [ i ] <= LastIssued . Array [ i ] ) WaitOn . Array [ i ] = LastIssued . Array [ i ] - Counts . Array [ i ] ;


<s> for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { LastIssued . Array [ i ] += Increment . Array [ i ] ; if ( Increment . Array [ i ] ) Limit . Array [ i ] = LastIssued . Array [ i ] ; Sum += Increment . Array [ i ] ; } if ( Sum == <NUM_LIT> ) { LastOpcodeType = OTHER ; return ; } if ( MBB . getParent ( ) -> getSubtarget < <ISA_LIT>Subtarget > ( ) . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( LastOpcodeType == VMEM && Increment . Named . VM ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; LastInstWritesM0 = false ; } if ( TII -> isSMRD ( * I ) ) LastOpcodeType = SMEM ; else if ( Increment . Named . VM ) LastOpcodeType = VMEM ; } if ( Increment . Named . EXP ) { ExpInstrTypesSeen |= I -> getOpcode ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; } for ( unsigned i = <NUM_LIT> , e = I -> getNumOperands ( ) ; i != e ; ++ i ) { MachineOperand & Op = I -> getOperand ( i ) ;


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; MRI = & MF . getRegInfo ( ) ; WaitedOn = ZeroCounts ; DelayedWaitOn = ZeroCounts ; LastIssued = ZeroCounts ; LastOpcodeType = OTHER ; LastInstWritesM0 = false ; ReturnsVoid = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ; memset ( & UsedRegs , <NUM_LIT> , sizeof ( UsedRegs ) ) ; memset ( & DefinedRegs , <NUM_LIT> , sizeof ( DefinedRegs ) ) ; SmallVector < MachineInstr * , <NUM_LIT> > RemoveMI ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { if ( ST . getGeneration ( ) <= <ISA_LIT>Subtarget :: SEA_ISLANDS ) { if ( TII -> isSMRD ( I -> getOpcode ( ) ) ) { VCCZCorrupt = true ; } else if ( ! hasOutstandingLGKM ( ) && I -> modifiesRegister ( <STR_LIT> , TRI ) ) { VCCZCorrupt = false ; } if ( readsVCCZ ( I -> getOpcode ( ) ) && VCCZCorrupt ) { DEBUG ( dbgs ( ) << <STR_LIT> << * I << '\n' ) ; insertWait ( MBB , I , LastIssued ) ; BuildMI ( MBB , I , I -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; }


<s> MachineFunctionPass ( ID ) , TII ( nullptr ) , TRI ( nullptr ) , ExpInstrTypesSeen ( <NUM_LIT> ) , VCCZCorrupt ( false ) {


<s> IsFlatOutstanding = false ; ReturnsVoid = MFI -> returnsVoid ( ) ; memset ( & UsedRegs , <NUM_LIT> , sizeof ( UsedRegs ) ) ; memset ( & DefinedRegs , <NUM_LIT> , sizeof ( DefinedRegs ) ) ; SmallVector < MachineInstr * , <NUM_LIT> > RemoveMI ; SmallVector < MachineBasicBlock * , <NUM_LIT> > EndPgmBlocks ; bool HaveScalarStores = false ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { if ( ! HaveScalarStores && TII -> isScalarStore ( * I ) ) HaveScalarStores = true ; if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: SEA_ISLANDS ) { if ( TII -> isSMRD ( I -> getOpcode ( ) ) ) { VCCZCorrupt = true ; } else if ( ! hasOutstandingLGKM ( ) && I -> modifiesRegister ( <STR_LIT> , TRI ) ) { VCCZCorrupt = false ; } if ( VCCZCorrupt && readsVCCZ ( * I ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * I << '\n' ) ; insertWait ( MBB , I , LastIssued ) ; BuildMI ( MBB , I , I -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; } } if ( I -> getOpcode ( ) == <STR_LIT> ) { handleExistingWait ( * I ) ; RemoveMI . push_back ( & * I ) ; continue ; } Counters Required ;

if ( ( I -> getOpcode ( ) == <STR_LIT> && ST -> needWaitcntBeforeBarrier ( ) ) || I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) Required = LastIssued ;


<s> void <ISA_LIT>InsertWaits :: handleSendMsg ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator I ) { if ( ST -> getGeneration ( ) < <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return ; if ( LastInstWritesM0 && I -> getOpcode ( ) == <STR_LIT> ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ;


<s> if ( LastInstWritesM0 && I -> getOpcode ( ) == <STR_LIT> ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; LastInstWritesM0 = false ; return ; }


<s> unsigned Sum = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { LastIssued . Array [ i ] += Increment . Array [ i ] ; if ( Increment . Array [ i ] ) Limit . Array [ i ] = LastIssued . Array [ i ] ; Sum += Increment . Array [ i ] ; } if ( Sum == <NUM_LIT> ) { LastOpcodeType = OTHER ; return ; } if ( ST -> getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( LastOpcodeType == VMEM && Increment . Named . VM ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; LastInstWritesM0 = false ; } if ( TII -> isSMRD ( * I ) ) LastOpcodeType = SMEM ; else if ( Increment . Named . VM ) LastOpcodeType = VMEM ; } if ( Increment . Named . EXP ) { ExpInstrTypesSeen |= I -> getOpcode ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; } for ( unsigned i = <NUM_LIT> , e = I -> getNumOperands ( ) ; i != e ; ++ i ) { MachineOperand & Op = I -> getOperand ( i ) ; if ( ! isOpRelevant ( Op ) ) continue ; const TargetRegisterClass * RC = TII -> getOpRegClass ( * I , i ) ; RegInterval Interval = getRegInterval ( RC , Op ) ;

for ( unsigned j = Interval . first ; j < Interval . second ; ++ j ) {


<s> insertWait ( MBB , I , LastIssued ) ; BuildMI ( MBB , I , I -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; } } if ( I -> getOpcode ( ) == <STR_LIT> ) { handleExistingWait ( * I ) ; RemoveMI . push_back ( & * I ) ; continue ; } Counters Required ; if ( I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) Required = LastIssued ; else Required = handleOperands ( * I ) ; Counters Increment = getHwCounts ( * I ) ; if ( countersNonZero ( Required ) || countersNonZero ( Increment ) ) increaseCounters ( Required , DelayedWaitOn ) ; Changes |= insertWait ( MBB , I , Required ) ; pushInstruction ( MBB , I , Increment ) ; handleSendMsg ( MBB , I ) ; } Changes |= insertWait ( MBB , MBB . getFirstTerminator ( ) , LastIssued ) ; }

for ( MachineInstr * I : RemoveMI ) I -> eraseFromParent ( ) ;


<s> DelayedWaitOn = ZeroCounts ; LastIssued = ZeroCounts ; LastOpcodeType = OTHER ; LastInstWritesM0 = false ; ReturnsVoid = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ; memset ( & UsedRegs , <NUM_LIT> , sizeof ( UsedRegs ) ) ; memset ( & DefinedRegs , <NUM_LIT> , sizeof ( DefinedRegs ) ) ; SmallVector < MachineInstr * , <NUM_LIT> > RemoveMI ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { if ( ST -> getGeneration ( ) <= <ISA_LIT>Subtarget :: SEA_ISLANDS ) { if ( TII -> isSMRD ( I -> getOpcode ( ) ) ) { VCCZCorrupt = true ;

} else if ( ! hasOutstandingLGKM ( ) && I -> modifiesRegister ( <STR_LIT> , TRI ) ) {


<s> LastInstWritesM0 = false ; ReturnsVoid = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ; memset ( & UsedRegs , <NUM_LIT> , sizeof ( UsedRegs ) ) ; memset ( & DefinedRegs , <NUM_LIT> , sizeof ( DefinedRegs ) ) ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ;


<s> MachineFunctionPass ( ID ) , TII ( nullptr ) , TRI ( nullptr ) , ExpInstrTypesSeen ( <NUM_LIT> ) {


MachineFunctionPass ( ID ) , TII ( nullptr ) , TRI ( nullptr ) , ExpInstrTypesSeen ( <NUM_LIT> ) {


<s> return new <ISA_LIT>InsertWaits ( tm ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>InsertWaits ( TargetMachine & tm ) { return new <ISA_LIT>InsertWaits ( tm ) ;


<s> if ( TII -> isSMRD ( MI . getOpcode ( ) ) ) { if ( MI . getNumOperands ( ) != <NUM_LIT> ) { assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) && <STR_LIT> ) ; const TargetRegisterClass * RC = TII -> getOpRegClass ( MI , <NUM_LIT> ) ; unsigned Size = RC -> getSize ( ) ; Result . Named . LGKM = Size > <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; } else { Result . Named . LGKM = <NUM_LIT> ; } } else { Result . Named . LGKM = <NUM_LIT> ; } } else {


<s> if ( TSFlags & <STR_LIT> ) { if ( TII -> isSMRD ( MI . getOpcode ( ) ) ) { if ( MI . getNumOperands ( ) != <NUM_LIT> ) { assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) && <STR_LIT> ) ; const TargetRegisterClass * RC = TII -> getOpRegClass ( MI , <NUM_LIT> ) ; unsigned Size = RC -> getSize ( ) ; Result . Named . LGKM = Size > <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; } else { Result . Named . LGKM = <NUM_LIT> ; } } else { Result . Named . LGKM = <NUM_LIT> ; } } else { Result . Named . LGKM = <NUM_LIT> ;


<s> RegInterval <ISA_LIT>InsertWaits :: getRegInterval ( const TargetRegisterClass * RC , const MachineOperand & Reg ) const { unsigned Size = RC -> getSize ( ) ; assert ( Size >= <NUM_LIT> ) ; RegInterval Result ;

Result . first = TRI -> getEncodingValue ( Reg . getReg ( ) ) ;


<s> if ( MI . getOpcode ( ) == <STR_LIT> ) return LastIssued ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { MachineOperand & Op = MI . getOperand ( i ) ; if ( ! Op . isReg ( ) || ! TRI -> isInAllocatableClass ( Op . getReg ( ) ) ) continue ; const TargetRegisterClass * RC = TII -> getOpRegClass ( MI , i ) ; RegInterval Interval = getRegInterval ( RC , Op ) ; for ( unsigned j = Interval . first ; j < Interval . second ; ++ j ) { if ( Op . isDef ( ) ) {

increaseCounters ( Result , UsedRegs [ j ] ) ;


<s> for ( unsigned i = <NUM_LIT> ; i < NumOperands ; i ++ ) { const MachineOperand & Op = I -> getOperand ( i ) ; if ( Op . isReg ( ) && Op . isDef ( ) && Op . getReg ( ) == <STR_LIT> ) LastInstWritesM0 = true ;


<s> Ordered [ <NUM_LIT> ] = ExpInstrTypesSeen == <NUM_LIT> ; Ordered [ <NUM_LIT> ] = false ; Counters Counts = WaitCounts ; bool NeedWait = false ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { if ( Required . Array [ i ] <= WaitedOn . Array [ i ] ) continue ; NeedWait = true ; if ( Ordered [ i ] ) { unsigned Value = LastIssued . Array [ i ] - Required . Array [ i ] ; Counts . Array [ i ] = std :: min ( Value , WaitCounts . Array [ i ] ) ; } else Counts . Array [ i ] = <NUM_LIT> ; WaitedOn . Array [ i ] = LastIssued . Array [ i ] - Counts . Array [ i ] ; } if ( ! NeedWait ) return false ; if ( Counts . Named . EXP == <NUM_LIT> ) ExpInstrTypesSeen = <NUM_LIT> ;

BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( ( Counts . Named . VM & <NUM_LIT> ) | ( ( Counts . Named . EXP & <NUM_LIT> ) << <NUM_LIT> ) | ( ( Counts . Named . LGKM & <NUM_LIT> ) << <NUM_LIT> ) ) ;


<s> bool NeedWait = false ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { if ( Required . Array [ i ] <= WaitedOn . Array [ i ] ) continue ; NeedWait = true ; if ( Ordered [ i ] ) { unsigned Value = LastIssued . Array [ i ] - Required . Array [ i ] ; Counts . Array [ i ] = std :: min ( Value , WaitCounts . Array [ i ] ) ; } else Counts . Array [ i ] = <NUM_LIT> ; WaitedOn . Array [ i ] = LastIssued . Array [ i ] - Counts . Array [ i ] ; } if ( ! NeedWait ) return false ; if ( Counts . Named . EXP == <NUM_LIT> ) ExpInstrTypesSeen = <NUM_LIT> ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( ( Counts . Named . VM & <NUM_LIT> ) | ( ( Counts . Named . EXP & <NUM_LIT> ) << <NUM_LIT> ) | ( ( Counts . Named . LGKM & <NUM_LIT> ) << <NUM_LIT> ) ) ; LastOpcodeType = OTHER ;


<s> if ( Data && Op . isIdenticalTo ( * Data ) ) return true ; MachineOperand * Data0 = TII -> getNamedOperand ( MI , <STR_LIT> :: data0 ) ; if ( Data0 && Op . isIdenticalTo ( * Data0 ) ) return true ; MachineOperand * Data1 = TII -> getNamedOperand ( MI , <STR_LIT> :: data1 ) ; if ( Data1 && Op . isIdenticalTo ( * Data1 ) ) return true ; return false ; } for ( MachineInstr :: mop_iterator I = MI . operands_begin ( ) , E = MI . operands_end ( ) ; I != E ; ++ I ) { if ( I -> isReg ( ) && I -> isUse ( ) ) return Op . isIdenticalTo ( * I ) ; }


<s> MachineOperand * Data0 = TII -> getNamedOperand ( MI , <STR_LIT> :: data0 ) ; if ( Data0 && Op . isIdenticalTo ( * Data0 ) ) return true ; MachineOperand * Data1 = TII -> getNamedOperand ( MI , <STR_LIT> :: data1 ) ; if ( Data1 && Op . isIdenticalTo ( * Data1 ) ) return true ; return false ; } for ( MachineInstr :: mop_iterator I = MI . operands_begin ( ) , E = MI . operands_end ( ) ; I != E ; ++ I ) { if ( I -> isReg ( ) && I -> isUse ( ) ) return Op . isIdenticalTo ( * I ) ; } return false ;


<s> if ( TII -> isSMRD ( I -> getOpcode ( ) ) ) LastOpcodeType = SMEM ; else if ( Increment . Named . VM ) LastOpcodeType = VMEM ; } if ( Increment . Named . EXP ) { ExpInstrTypesSeen |= I -> getOpcode ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; } for ( unsigned i = <NUM_LIT> , e = I -> getNumOperands ( ) ; i != e ; ++ i ) { MachineOperand & Op = I -> getOperand ( i ) ; if ( ! isOpRelevant ( Op ) ) continue ; const TargetRegisterClass * RC = TII -> getOpRegClass ( * I , i ) ; RegInterval Interval = getRegInterval ( RC , Op ) ; for ( unsigned j = Interval . first ; j < Interval . second ; ++ j ) { if ( Op . isDef ( ) ) DefinedRegs [ j ] = Limit ; if ( Op . isUse ( ) ) UsedRegs [ j ] = Limit ;


<s> Counters Limit = ZeroCounts ; unsigned Sum = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { LastIssued . Array [ i ] += Increment . Array [ i ] ; if ( Increment . Array [ i ] ) Limit . Array [ i ] = LastIssued . Array [ i ] ; Sum += Increment . Array [ i ] ; } if ( Sum == <NUM_LIT> ) { LastOpcodeType = OTHER ; return ; } if ( MBB . getParent ( ) -> getSubtarget < <ISA_LIT>Subtarget > ( ) . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( ( LastOpcodeType == SMEM && TII -> isSMRD ( I -> getOpcode ( ) ) ) || ( LastOpcodeType == VMEM && Increment . Named . VM ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; LastInstWritesM0 = false ; } if ( TII -> isSMRD ( I -> getOpcode ( ) ) ) LastOpcodeType = SMEM ; else if ( Increment . Named . VM ) LastOpcodeType = VMEM ; } if ( Increment . Named . EXP ) { ExpInstrTypesSeen |= I -> getOpcode ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; } for ( unsigned i = <NUM_LIT> , e = I -> getNumOperands ( ) ; i != e ; ++ i ) { MachineOperand & Op = I -> getOperand ( i ) ; if ( ! isOpRelevant ( Op ) ) continue ; const TargetRegisterClass * RC = TII -> getOpRegClass ( * I , i ) ; RegInterval Interval = getRegInterval ( RC , Op ) ; for ( unsigned j = Interval . first ; j < Interval . second ; ++ j ) { if ( Op . isDef ( ) ) DefinedRegs [ j ] = Limit ; if ( Op . isUse ( ) ) UsedRegs [ j ] = Limit ;


<s> BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { if ( I -> getOpcode ( ) == <STR_LIT> ) Changes |= insertWait ( MBB , I , LastIssued ) ; else Changes |= insertWait ( MBB , I , handleOperands ( * I ) ) ; pushInstruction ( MBB , I ) ; handleSendMsg ( MBB , I ) ; } Changes |= insertWait ( MBB , MBB . getFirstTerminator ( ) , LastIssued ) ; } return Changes ;


<s> for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { if ( I -> getOpcode ( ) == <STR_LIT> ) Changes |= insertWait ( MBB , I , LastIssued ) ;

else Changes |= insertWait ( MBB , I , handleOperands ( * I ) ) ;


<s> void <ISA_LIT>InstrInfo :: addSCCDefUsersToVALUWorklist ( MachineInstr & SCCDefInst , SetVectorType & Worklist ) const { for ( MachineInstr & MI : make_range ( MachineBasicBlock :: iterator ( SCCDefInst ) , SCCDefInst . getParent ( ) -> end ( ) ) ) { if ( MI . findRegisterDefOperandIdx ( <STR_LIT> ) != - <NUM_LIT> ) return ;

if ( MI . findRegisterUseOperandIdx ( <STR_LIT> ) != - <NUM_LIT> ) Worklist . insert ( & MI ) ;


<s> void <ISA_LIT>InstrInfo :: addUsersToMoveToVALUWorklist ( unsigned DstReg , MachineRegisterInfo & MRI , SetVectorType & Worklist ) const { for ( MachineRegisterInfo :: use_iterator I = MRI . use_begin ( DstReg ) , E = MRI . use_end ( ) ; I != E ; ) { MachineInstr & UseMI = * I -> getParent ( ) ; if ( ! canReadVGPR ( UseMI , I . getOperandNo ( ) ) ) { Worklist . insert ( & UseMI ) ; do { ++ I ; } while ( I != E && I -> getParent ( ) == & UseMI ) ; } else { ++ I ; } }


<s> for ( MachineRegisterInfo :: use_iterator I = MRI . use_begin ( DstReg ) , E = MRI . use_end ( ) ; I != E ; ) { MachineInstr & UseMI = * I -> getParent ( ) ; if ( ! canReadVGPR ( UseMI , I . getOperandNo ( ) ) ) { Worklist . insert ( & UseMI ) ;


<s> if ( I == MBB . end ( ) ) return true ; if ( analyzeBranchImpl ( MBB , I , TBB , FBB , Cond , AllowModify ) ) return true ; MachineBasicBlock * MaskBrDest = I -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( TBB != MaskBrDest || Cond . empty ( ) ) return true ; auto Pred = Cond [ <NUM_LIT> ] . getImm ( ) ; return ( Pred != EXECZ && Pred != EXECNZ ) ;


<s> return false ; } MachineBasicBlock * CondBB = nullptr ; if ( I -> getOpcode ( ) == <STR_LIT> ) { CondBB = I -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; Cond . push_back ( I -> getOperand ( <NUM_LIT> ) ) ; } else { BranchPredicate Pred = getBranchPredicate ( I -> getOpcode ( ) ) ; if ( Pred == INVALID_BR ) return true ; CondBB = I -> getOperand ( <NUM_LIT> ) . getMBB ( ) ; Cond . push_back ( MachineOperand :: CreateImm ( Pred ) ) ; Cond . push_back ( I -> getOperand ( <NUM_LIT> ) ) ; } ++ I ; if ( I == MBB . end ( ) ) { TBB = CondBB ; return false ; } if ( I -> getOpcode ( ) == <STR_LIT> ) { TBB = CondBB ;

FBB = I -> getOperand ( <NUM_LIT> ) . getMBB ( ) ;


<s> assert ( ( MIa . mayLoad ( ) || MIa . mayStore ( ) ) && <STR_LIT> ) ; assert ( ( MIb . mayLoad ( ) || MIb . mayStore ( ) ) && <STR_LIT> ) ; if ( MIa . hasUnmodeledSideEffects ( ) || MIb . hasUnmodeledSideEffects ( ) ) return false ; if ( MIa . hasOrderedMemoryRef ( ) || MIb . hasOrderedMemoryRef ( ) ) return false ; if ( AA && MIa . hasOneMemOperand ( ) && MIb . hasOneMemOperand ( ) ) { const MachineMemOperand * MMOa = * MIa . memoperands_begin ( ) ; const MachineMemOperand * MMOb = * MIb . memoperands_begin ( ) ;

if ( MMOa -> getValue ( ) && MMOb -> getValue ( ) ) {


<s> if ( MIa . hasUnmodeledSideEffects ( ) || MIb . hasUnmodeledSideEffects ( ) ) return false ; if ( MIa . hasOrderedMemoryRef ( ) || MIb . hasOrderedMemoryRef ( ) ) return false ; if ( AA && MIa . hasOneMemOperand ( ) && MIb . hasOneMemOperand ( ) ) { const MachineMemOperand * MMOa = * MIa . memoperands_begin ( ) ; const MachineMemOperand * MMOb = * MIb . memoperands_begin ( ) ; if ( MMOa -> getValue ( ) && MMOb -> getValue ( ) ) { MemoryLocation LocA ( MMOa -> getValue ( ) , MMOa -> getSize ( ) , MMOa -> getAAInfo ( ) ) ; MemoryLocation LocB ( MMOb -> getValue ( ) , MMOb -> getSize ( ) , MMOb -> getAAInfo ( ) ) ; if ( ! AA -> alias ( LocA , LocB ) ) return true ; } } if ( isDS ( MIa ) ) { if ( isDS ( MIb ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return ! isFLAT ( MIb ) || isSegmentSpecificFLAT ( MIb ) ; } if ( isMUBUF ( MIa ) || isMTBUF ( MIa ) ) { if ( isMUBUF ( MIb ) || isMTBUF ( MIb ) ) return checkInstOffsetsDoNotOverlap ( MIa , MIb ) ; return ! isFLAT ( MIb ) && ! isSMRD ( MIb ) ;


<s> BuildMI ( * MBB , MI , DL , get ( TargetOpcode :: COPY ) , NewSuperReg ) . addReg ( SuperReg . getReg ( ) , <NUM_LIT> , SuperReg . getSubReg ( ) ) ;

BuildMI ( * MBB , MI , DL , get ( TargetOpcode :: COPY ) , SubReg ) . addReg ( NewSuperReg , <NUM_LIT> , SubIdx ) ;


<s> DebugLoc DL = MI -> getDebugLoc ( ) ; unsigned SubReg = MRI . createVirtualRegister ( SubRC ) ; if ( SuperReg . getSubReg ( ) == <STR_LIT> ) { BuildMI ( * MBB , MI , DL , get ( TargetOpcode :: COPY ) , SubReg ) . addReg ( SuperReg . getReg ( ) , <NUM_LIT> , SubIdx ) ; return SubReg ; } unsigned NewSuperReg = MRI . createVirtualRegister ( SuperRC ) ; BuildMI ( * MBB , MI , DL , get ( TargetOpcode :: COPY ) , NewSuperReg ) . addReg ( SuperReg . getReg ( ) , <NUM_LIT> , SuperReg . getSubReg ( ) ) ;


<s> MachineOperand <ISA_LIT>InstrInfo :: buildExtractSubRegOrImm ( MachineBasicBlock :: iterator MII , MachineRegisterInfo & MRI , MachineOperand & Op , const TargetRegisterClass * SuperRC , unsigned SubIdx , const TargetRegisterClass * SubRC ) const { if ( Op . isImm ( ) ) { if ( SubIdx == <STR_LIT> ) return MachineOperand :: CreateImm ( static_cast < int32_t > ( Op . getImm ( ) ) ) ;

if ( SubIdx == <STR_LIT> ) return MachineOperand :: CreateImm ( static_cast < int32_t > ( Op . getImm ( ) >> <NUM_LIT> ) ) ;


<s> if ( Op . isImm ( ) ) { if ( SubIdx == <STR_LIT> ) return MachineOperand :: CreateImm ( static_cast < int32_t > ( Op . getImm ( ) ) ) ; if ( SubIdx == <STR_LIT> ) return MachineOperand :: CreateImm ( static_cast < int32_t > ( Op . getImm ( ) >> <NUM_LIT> ) ) ; llvm_unreachable ( <STR_LIT> ) ; }

unsigned SubReg = buildExtractSubReg ( MII , MRI , Op , SuperRC , SubIdx , SubRC ) ;


<s> const <ISA_LIT>Subtarget & ST = MF -> getSubtarget < <ISA_LIT>Subtarget > ( ) ; DebugLoc DL = MBB . findDebugLoc ( MI ) ; unsigned WorkGroupSize = MFI -> getMaxFlatWorkGroupSize ( ) ; unsigned WavefrontSize = ST . getWavefrontSize ( ) ; unsigned TIDReg = MFI -> getTIDReg ( ) ; if ( ! MFI -> hasCalculatedTID ( ) ) { MachineBasicBlock & Entry = MBB . getParent ( ) -> front ( ) ; MachineBasicBlock :: iterator Insert = Entry . front ( ) ; DebugLoc DL = Insert -> getDebugLoc ( ) ; TIDReg = RI . findUnusedRegister ( MF -> getRegInfo ( ) , & <STR_LIT> , * MF ) ; if ( TIDReg == <STR_LIT> ) return TIDReg ; if ( ! <STR_LIT> ( MF -> getFunction ( ) . getCallingConv ( ) ) && WorkGroupSize > WavefrontSize ) { unsigned TIDIGXReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned TIDIGYReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned TIDIGZReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; unsigned InputPtrReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; for ( unsigned Reg : { TIDIGXReg , TIDIGYReg , TIDIGZReg } ) { if ( ! Entry . isLiveIn ( Reg ) ) Entry . addLiveIn ( Reg ) ; } RS -> enterBasicBlock ( Entry ) ; unsigned STmp0 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ;

unsigned STmp1 = RS -> scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :

return RI . hasVGPRs ( getOpRegClass ( MI , <NUM_LIT> ) ) ;


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> if ( getMemOpBaseRegImmOfs ( MIa , BaseReg0 , Offset0 , & RI ) && getMemOpBaseRegImmOfs ( MIb , BaseReg1 , Offset1 , & RI ) ) { if ( ! MIa . hasOneMemOperand ( ) || ! MIb . hasOneMemOperand ( ) ) { return false ; } unsigned Width0 = ( * MIa . memoperands_begin ( ) ) -> getSize ( ) ; unsigned Width1 = ( * MIb . memoperands_begin ( ) ) -> getSize ( ) ; if ( BaseReg0 == BaseReg1 && offsetsDoNotOverlap ( Width0 , Offset0 , Width1 , Offset1 ) ) {


<s> unsigned Opc = MI . getOpcode ( ) ; int CommutedOpcode = commuteOpcode ( Opc ) ; if ( CommutedOpcode == - <NUM_LIT> ) return nullptr ; assert ( <STR_LIT> ( Opc , <STR_LIT> :: src0 ) == static_cast < int > ( Src0Idx ) && <STR_LIT> ( Opc , <STR_LIT> :: src1 ) == static_cast < int > ( Src1Idx ) && <STR_LIT> ) ; MachineOperand & Src0 = MI . getOperand ( Src0Idx ) ; MachineOperand & Src1 = MI . getOperand ( Src1Idx ) ; MachineInstr * CommutedMI = nullptr ; if ( Src0 . isReg ( ) && Src1 . isReg ( ) ) { if ( isOperandLegal ( MI , Src1Idx , & Src0 ) ) { CommutedMI = TargetInstrInfo :: commuteInstructionImpl ( MI , NewMI , Src0Idx , Src1Idx ) ; } } else if ( Src0 . isReg ( ) && ! Src1 . isReg ( ) ) { CommutedMI = swapRegAndNonRegOperand ( MI , Src0 , Src1 ) ; } else if ( ! Src0 . isReg ( ) && Src1 . isReg ( ) ) {

if ( isOperandLegal ( MI , Src1Idx , & Src0 ) ) CommutedMI = swapRegAndNonRegOperand ( MI , Src1 , Src0 ) ;


<s> int <ISA_LIT>InstrInfo :: commuteOpcode ( unsigned Opcode ) const { int NewOpc ; NewOpc = <STR_LIT> ( Opcode ) ; if ( NewOpc != - <NUM_LIT> ) return pseudoToMCOpcode ( NewOpc ) != - <NUM_LIT> ? NewOpc : - <NUM_LIT> ;


<s> MachineInstr * Branch = & ( * TI ) ; MachineFunction * MF = IfEntry -> getParent ( ) ; MachineRegisterInfo & MRI = IfEntry -> getParent ( ) -> getRegInfo ( ) ; if ( Branch -> getOpcode ( ) == <STR_LIT> ) { unsigned DstReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * <ISA_LIT>IF = BuildMI ( * MF , Branch -> getDebugLoc ( ) , get ( <STR_LIT> ) , DstReg ) . add ( Branch -> getOperand ( <NUM_LIT> ) ) . add ( Branch -> getOperand ( <NUM_LIT> ) ) ; MachineInstr * <ISA_LIT>END = BuildMI ( * MF , Branch -> getDebugLoc ( ) , get ( <STR_LIT> ) ) . addReg ( DstReg ) ;


<s> assert ( TI != LoopEnd -> end ( ) ) ; MachineInstr * Branch = & ( * TI ) ; MachineFunction * MF = LoopEnd -> getParent ( ) ; MachineRegisterInfo & MRI = LoopEnd -> getParent ( ) -> getRegInfo ( ) ; if ( Branch -> getOpcode ( ) == <STR_LIT> ) { unsigned DstReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned BackEdgeReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; MachineInstrBuilder HeaderPHIBuilder = BuildMI ( * ( MF ) , Branch -> getDebugLoc ( ) , get ( TargetOpcode :: PHI ) , DstReg ) ; for ( MachineBasicBlock :: pred_iterator PI = LoopEntry -> pred_begin ( ) , E = LoopEntry -> pred_end ( ) ; PI != E ; ++ PI ) { if ( * PI == LoopEnd ) {

HeaderPHIBuilder . addReg ( BackEdgeReg ) ;


<s> } const MachineOperand * Dst = getNamedOperand ( MI , <STR_LIT> :: vdst ) ; const MachineOperand * Src0 = getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src0Mods = getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ; const MachineOperand * Src1 = getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mods = getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; const MachineOperand * Src2 = getNamedOperand ( MI , <STR_LIT> :: src2 ) ; const MachineOperand * Clamp = getNamedOperand ( MI , <STR_LIT> :: clamp ) ; const MachineOperand * Omod = getNamedOperand ( MI , <STR_LIT> :: omod ) ; if ( ! Src0Mods && ! Src1Mods && ! Clamp && ! Omod && ( ! Src0 -> isReg ( ) || ! RI . isSGPRReg ( MBB -> getParent ( ) -> getRegInfo ( ) , Src0 -> getReg ( ) ) ) ) { if ( auto Imm = getFoldableImm ( Src2 ) ) { return BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( IsF16 ? <STR_LIT> : <STR_LIT> ) ) . add ( * Dst ) . add ( * Src0 ) . add ( * Src1 ) . addImm ( Imm ) ; } if ( auto Imm = getFoldableImm ( Src1 ) ) { return BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( IsF16 ? <STR_LIT> : <STR_LIT> ) ) . add ( * Dst ) . add ( * Src0 ) . addImm ( Imm ) . add ( * Src2 ) ; } if ( auto Imm = getFoldableImm ( Src0 ) ) {

if ( isOperandLegal ( MI , <STR_LIT> ( <STR_LIT> , <STR_LIT> :: src0 ) , Src1 ) ) return BuildMI ( * MBB , MI , MI . getDebugLoc ( ) , get ( IsF16 ? <STR_LIT> : <STR_LIT> ) ) . add ( * Dst ) . add ( * Src1 ) . addImm ( Imm ) . add ( * Src2 ) ;


<s> if ( SrcReg == <STR_LIT> ) { BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; return ; } if ( ! <STR_LIT> . contains ( SrcReg ) ) { reportIllegalCopy ( this , MBB , MI , DL , DestReg , SrcReg , KillSrc ) ; return ; } BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; return ; } if ( RC == & <STR_LIT> ) { if ( DestReg == <STR_LIT> ) { if ( <STR_LIT> . contains ( SrcReg ) ) { BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; } else { assert ( <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; } return ; } if ( ! <STR_LIT> . contains ( SrcReg ) ) { reportIllegalCopy ( this , MBB , MI , DL , DestReg , SrcReg , KillSrc ) ; return ; } BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ; return ; } if ( DestReg == <STR_LIT> ) { assert ( <STR_LIT> . contains ( SrcReg ) ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) . addImm ( <NUM_LIT> ) ; return ; } unsigned EltSize = <NUM_LIT> ;


<s> return new GCNHazardRecognizer ( MF ) ;


<s> return std :: make_pair ( TF & MO_MASK , TF & ~ MO_MASK ) ;


<s> std :: pair < unsigned , unsigned > <ISA_LIT>InstrInfo :: decomposeMachineOperandsTargetFlags ( unsigned TF ) const { return std :: make_pair ( TF & MO_MASK , TF & ~ MO_MASK ) ;


<s> static SDValue findChainOperand ( SDNode * Load ) {

SDValue LastOp = Load -> getOperand ( getNumOperandsNoGlue ( Load ) - <NUM_LIT> ) ;


<s> if ( ! MI . isCommutable ( ) ) return false ; unsigned Opc = MI . getOpcode ( ) ; int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; if ( Src0Idx == - <NUM_LIT> ) return false ; int Src1Idx = <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ; if ( Src1Idx == - <NUM_LIT> ) return false ; return fixCommutedOpIndices ( SrcOpIdx0 , SrcOpIdx1 , Src0Idx , Src1Idx ) ;


<s> static unsigned findImplicitSGPRRead ( const MachineInstr & MI ) { for ( const MachineOperand & MO : MI . implicit_operands ( ) ) { if ( MO . isDef ( ) ) continue ; switch ( MO . getReg ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return MO . getReg ( ) ; default :


<s> if ( RI . isSGPRClass ( RegRC ) ) UsedSGPRs [ i ] = Reg ; } if ( UsedSGPRs [ <NUM_LIT> ] != <STR_LIT> ) { if ( UsedSGPRs [ <NUM_LIT> ] == UsedSGPRs [ <NUM_LIT> ] || UsedSGPRs [ <NUM_LIT> ] == UsedSGPRs [ <NUM_LIT> ] ) SGPRReg = UsedSGPRs [ <NUM_LIT> ] ; } if ( SGPRReg == <STR_LIT> && UsedSGPRs [ <NUM_LIT> ] != <STR_LIT> ) {

if ( UsedSGPRs [ <NUM_LIT> ] == UsedSGPRs [ <NUM_LIT> ] ) SGPRReg = UsedSGPRs [ <NUM_LIT> ] ;


<s> bool IsRequiredSGPR = RI . isSGPRClass ( OpRC ) ; if ( IsRequiredSGPR ) return MO . getReg ( ) ; unsigned Reg = MO . getReg ( ) ; const TargetRegisterClass * RegRC = MRI . getRegClass ( Reg ) ; if ( RI . isSGPRClass ( RegRC ) ) UsedSGPRs [ i ] = Reg ; } if ( UsedSGPRs [ <NUM_LIT> ] != <STR_LIT> ) {

if ( UsedSGPRs [ <NUM_LIT> ] == UsedSGPRs [ <NUM_LIT> ] || UsedSGPRs [ <NUM_LIT> ] == UsedSGPRs [ <NUM_LIT> ] ) SGPRReg = UsedSGPRs [ <NUM_LIT> ] ;


<s> unsigned NewOpc = isVGPRCopy ? <STR_LIT> : <STR_LIT> ; UseMI . setDesc ( get ( NewOpc ) ) ; UseMI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( ImmOp -> getImm ( ) ) ; UseMI . addImplicitDefUseOperands ( * UseMI . getParent ( ) -> getParent ( ) ) ; return true ; } if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) { if ( hasAnyModifiersSet ( UseMI ) ) return false ; MachineOperand * Src0 = getNamedOperand ( UseMI , <STR_LIT> :: src0 ) ; if ( isInlineConstant ( UseMI , * Src0 , * ImmOp ) ) return false ; bool IsF32 = Opc == <STR_LIT> || Opc == <STR_LIT> ; MachineOperand * Src1 = getNamedOperand ( UseMI , <STR_LIT> :: src1 ) ; MachineOperand * Src2 = getNamedOperand ( UseMI , <STR_LIT> :: src2 ) ; if ( Src0 -> isReg ( ) && Src0 -> getReg ( ) == Reg ) { if ( ! Src1 -> isReg ( ) || RI . isSGPRClass ( MRI -> getRegClass ( Src1 -> getReg ( ) ) ) ) return false ; if ( ! Src2 -> isReg ( ) || RI . isSGPRClass ( MRI -> getRegClass ( Src2 -> getReg ( ) ) ) ) return false ; const int64_t Imm = ImmOp -> getImm ( ) ; UseMI . RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: omod ) ) ; UseMI . RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: clamp ) ) ; unsigned Src1Reg = Src1 -> getReg ( ) ; unsigned Src1SubReg = Src1 -> getSubReg ( ) ; Src0 -> setReg ( Src1Reg ) ; Src0 -> setSubReg ( Src1SubReg ) ; Src0 -> setIsKill ( Src1 -> isKill ( ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) UseMI . untieRegOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ) ; Src1 -> ChangeToImmediate ( Imm ) ;


<s> MachineInstrBuilder <ISA_LIT>InstrInfo :: getAddNoCarry ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator I , const DebugLoc & DL , unsigned DestReg ) const { if ( ST . hasAddNoCarry ( ) ) return BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DestReg ) ; MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; unsigned UnusedCarry = MRI . createVirtualRegister ( & <STR_LIT> ) ; MRI . setRegAllocationHint ( UnusedCarry , <NUM_LIT> , <STR_LIT> ) ; return BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DestReg ) . addReg ( UnusedCarry , RegState :: Define | RegState :: Dead ) ;


<s> MachineInstrBuilder <ISA_LIT>InstrInfo :: getAddNoCarry ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator I , const DebugLoc & DL , unsigned DestReg ) const {

if ( ST . hasAddNoCarry ( ) ) return BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DestReg ) ;


<s> return AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ; case PseudoSourceValue :: ConstantPool : case PseudoSourceValue :: GOT : case PseudoSourceValue :: JumpTable : case PseudoSourceValue :: GlobalValueCallEntry : case PseudoSourceValue :: ExternalSymbolCallEntry : case PseudoSourceValue :: TargetCustom : return AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ;


<s> } return MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ;


return MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ;


<s> case <STR_LIT> : return SCC_FALSE ; case <STR_LIT> : return SCC_TRUE ; case <STR_LIT> : return VCCNZ ; case <STR_LIT> : return VCCZ ; case <STR_LIT> : return EXECNZ ;


<s> if ( ST . getGeneration ( ) <= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) RsrcDataFormat |= ( <NUM_LIT> << <NUM_LIT> ) ; if ( ST . getGeneration ( ) == <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) RsrcDataFormat |= ( <NUM_LIT> << <NUM_LIT> ) ;


<s> if ( ST . getGeneration ( ) <= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) RsrcDataFormat |= ( <NUM_LIT> << <NUM_LIT> ) ;

if ( ST . getGeneration ( ) == <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) RsrcDataFormat |= ( <NUM_LIT> << <NUM_LIT> ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( RI . hasVGPRs ( NewDstRC ) ) return nullptr ;

NewDstRC = RI . getEquivalentVGPRClass ( NewDstRC ) ;


<s> unsigned Size = <NUM_LIT> ; MachineBasicBlock :: const_instr_iterator I = MI . getIterator ( ) ; MachineBasicBlock :: const_instr_iterator E = MI . getParent ( ) -> instr_end ( ) ;

while ( ++ I != E && I -> isInsideBundle ( ) ) {


<s> unsigned <ISA_LIT>InstrInfo :: getInstSizeInBytes ( const MachineInstr & MI ) const { unsigned Opc = MI . getOpcode ( ) ; const MCInstrDesc & Desc = getMCOpcodeFromPseudo ( Opc ) ; unsigned DescSize = Desc . getSize ( ) ; if ( DescSize != <NUM_LIT> && DescSize != <NUM_LIT> ) return DescSize ; if ( isVALU ( MI ) || isSALU ( MI ) ) { if ( isFixedSize ( MI ) ) return DescSize ; int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; if ( Src0Idx == - <NUM_LIT> ) return <NUM_LIT> ; if ( isLiteralConstantLike ( MI . getOperand ( Src0Idx ) , Desc . OpInfo [ Src0Idx ] ) ) return <NUM_LIT> ; int Src1Idx = <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ; if ( Src1Idx == - <NUM_LIT> ) return <NUM_LIT> ; if ( isLiteralConstantLike ( MI . getOperand ( Src1Idx ) , Desc . OpInfo [ Src1Idx ] ) ) return <NUM_LIT> ; return <NUM_LIT> ; } if ( DescSize == <NUM_LIT> ) return <NUM_LIT> ; switch ( Opc ) { case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: DBG_VALUE : case TargetOpcode :: EH_LABEL :


<s> switch ( Opcode ) { case <STR_LIT> : return get ( <STR_LIT> ) ;


<s> switch ( Opcode ) { case <STR_LIT> : return get ( <STR_LIT> ) ; case <STR_LIT> : return get ( <STR_LIT> ) ;


<s> unsigned getMachineCSELookAheadLimit ( ) const override {


<s> if ( OffsetImm ) { const MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: addr ) ; BaseReg = AddrReg -> getReg ( ) ; Offset = OffsetImm -> getImm ( ) ; return true ; } const MachineOperand * Offset0Imm = getNamedOperand ( LdSt , <STR_LIT> :: offset0 ) ; const MachineOperand * Offset1Imm = getNamedOperand ( LdSt , <STR_LIT> :: offset1 ) ; uint8_t Offset0 = Offset0Imm -> getImm ( ) ; uint8_t Offset1 = Offset1Imm -> getImm ( ) ; if ( Offset1 > Offset0 && Offset1 - Offset0 == <NUM_LIT> ) { unsigned EltSize ; if ( LdSt . mayLoad ( ) ) EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , <NUM_LIT> ) ) / <NUM_LIT> ; else { assert ( LdSt . mayStore ( ) ) ; int Data0Idx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , Data0Idx ) ) / <NUM_LIT> ; } if ( isStride64 ( Opc ) ) EltSize *= <NUM_LIT> ; const MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: addr ) ; BaseReg = AddrReg -> getReg ( ) ; Offset = EltSize * Offset0 ; return true ; } return false ; } if ( isMUBUF ( LdSt ) || isMTBUF ( LdSt ) ) { const MachineOperand * SOffset = getNamedOperand ( LdSt , <STR_LIT> :: soffset ) ; if ( SOffset && SOffset -> isReg ( ) ) return false ; const MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( ! AddrReg ) return false ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; BaseReg = AddrReg -> getReg ( ) ; Offset = OffsetImm -> getImm ( ) ; if ( SOffset ) Offset += SOffset -> getImm ( ) ; return true ; }


<s> uint8_t Offset0 = Offset0Imm -> getImm ( ) ; uint8_t Offset1 = Offset1Imm -> getImm ( ) ; if ( Offset1 > Offset0 && Offset1 - Offset0 == <NUM_LIT> ) { unsigned EltSize ; if ( LdSt . mayLoad ( ) ) EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , <NUM_LIT> ) ) / <NUM_LIT> ; else { assert ( LdSt . mayStore ( ) ) ; int Data0Idx = <STR_LIT> ( Opc , <STR_LIT> :: data0 ) ; EltSize = TRI -> getRegSizeInBits ( * getOpRegClass ( LdSt , Data0Idx ) ) / <NUM_LIT> ; } if ( isStride64 ( Opc ) ) EltSize *= <NUM_LIT> ; const MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: addr ) ; BaseReg = AddrReg -> getReg ( ) ; Offset = EltSize * Offset0 ; return true ; } return false ; } if ( isMUBUF ( LdSt ) || isMTBUF ( LdSt ) ) { const MachineOperand * SOffset = getNamedOperand ( LdSt , <STR_LIT> :: soffset ) ; if ( SOffset && SOffset -> isReg ( ) ) return false ; const MachineOperand * AddrReg = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ; if ( ! AddrReg ) return false ; const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; BaseReg = AddrReg -> getReg ( ) ; Offset = OffsetImm -> getImm ( ) ; if ( SOffset ) Offset += SOffset -> getImm ( ) ; return true ; } if ( isSMRD ( LdSt ) ) { const MachineOperand * OffsetImm = getNamedOperand ( LdSt , <STR_LIT> :: offset ) ; if ( ! OffsetImm ) return false ; const MachineOperand * SBaseReg = getNamedOperand ( LdSt , <STR_LIT> :: sbase ) ; BaseReg = SBaseReg -> getReg ( ) ; Offset = OffsetImm -> getImm ( ) ; return true ; } if ( isFLAT ( LdSt ) ) {

const MachineOperand * VAddr = getNamedOperand ( LdSt , <STR_LIT> :: vaddr ) ;


<s> return RI . isSGPRClass ( DstRC ) ? <STR_LIT> : <STR_LIT> ; } else if ( RI . getRegSizeInBits ( * DstRC ) == <NUM_LIT> && RI . isSGPRClass ( DstRC ) ) { return <STR_LIT> ; } else if ( RI . getRegSizeInBits ( * DstRC ) == <NUM_LIT> && ! RI . isSGPRClass ( DstRC ) ) { return <STR_LIT> ; }


<s> return MI . getOperand ( Idx ) . getImm ( ) ;


<s> return getNamedOperand ( const_cast < MachineInstr & > ( MI ) , OpName ) ;


<s> return MI . getOperand ( <NUM_LIT> ) . getImm ( ) + <NUM_LIT> ;


<s> unsigned getOpSize ( const MachineInstr & MI , unsigned OpNo ) const { return getOpRegClass ( MI , OpNo ) -> getSize ( ) ;


<s> const TargetRegisterClass * <ISA_LIT>InstrInfo :: getPreferredSelectRegClass ( unsigned Size ) const { return & <STR_LIT> ;


<s> const <ISA_LIT>RegisterInfo & getRegisterInfo ( ) const override { return RI ;


<s> const <ISA_LIT>RegisterInfo & getRegisterInfo ( ) const override {


<s> uint64_t <ISA_LIT>InstrInfo :: getScratchRsrcWords23 ( ) const { uint64_t Rsrc23 = getDefaultRsrcDataFormat ( ) | <STR_LIT> | <NUM_LIT> ; if ( ST . getGeneration ( ) <= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { uint64_t EltSizeValue = Log2_32 ( ST . getMaxPrivateElementSize ( ) ) - <NUM_LIT> ; Rsrc23 |= EltSizeValue << <STR_LIT> ; } Rsrc23 |= UINT64_C ( <NUM_LIT> ) << <STR_LIT> ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) Rsrc23 &= ~ <STR_LIT> ; return Rsrc23 ;


<s> uint64_t Rsrc23 = getDefaultRsrcDataFormat ( ) | <STR_LIT> | <NUM_LIT> ; if ( ST . getGeneration ( ) <= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { uint64_t EltSizeValue = Log2_32 ( ST . getMaxPrivateElementSize ( ) ) - <NUM_LIT> ;


<s> { MO_GOTPCREL , <STR_LIT> } , { MO_GOTPCREL32_LO , <STR_LIT> } , { MO_GOTPCREL32_HI , <STR_LIT> } , { MO_REL32_LO , <STR_LIT> } , { MO_REL32_HI , <STR_LIT> } } ;


<s> static const std :: pair < unsigned , const char * > TargetFlags [ ] = { { MO_GOTPCREL , <STR_LIT> } , { MO_GOTPCREL32_LO , <STR_LIT> } , { MO_GOTPCREL32_HI , <STR_LIT> } , { MO_REL32_LO , <STR_LIT> } , { MO_REL32_HI , <STR_LIT>


<s> { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> } } ;


<s> ArrayRef < std :: pair < int , const char * >> <ISA_LIT>InstrInfo :: getSerializableTargetIndices ( ) const {

static const std :: pair < int , const char * > TargetIndices [ ] = {


<s> switch ( Size ) { case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> :


<s> return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; default :


<s> case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ;


<s> return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; }


<s> return hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) || hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) || hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) || hasModifiersSet ( MI , <STR_LIT> :: clamp ) || hasModifiersSet ( MI , <STR_LIT> :: omod ) ;


<s> bool <ISA_LIT>InstrInfo :: hasModifiers ( unsigned Opcode ) const { return <STR_LIT> ( Opcode , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ;


<s> bool <ISA_LIT>InstrInfo :: hasModifiers ( unsigned Opcode ) const {

return <STR_LIT> ( Opcode , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ;


<s> bool <ISA_LIT>InstrInfo :: hasModifiersSet ( const MachineInstr & MI , unsigned OpName ) const {

const MachineOperand * Mods = getNamedOperand ( MI , OpName ) ;


<s> bool <ISA_LIT>InstrInfo :: hasVALU32BitEncoding ( unsigned Opcode ) const { int Op32 = <STR_LIT> ( Opcode ) ;

if ( Op32 == - <NUM_LIT> ) return false ;


<s> if ( Op32 == - <NUM_LIT> ) return false ; return pseudoToMCOpcode ( Op32 ) != - <NUM_LIT> ;


<s> unsigned <ISA_LIT>InstrInfo :: insertBranch ( MachineBasicBlock & MBB , MachineBasicBlock * TBB , MachineBasicBlock * FBB , ArrayRef < MachineOperand > Cond , const DebugLoc & DL , int * BytesAdded ) const { if ( ! FBB && Cond . empty ( ) ) { BuildMI ( & MBB , DL , get ( <STR_LIT> ) ) . addMBB ( TBB ) ; if ( BytesAdded ) * BytesAdded = <NUM_LIT> ; return <NUM_LIT> ; } if ( Cond . size ( ) == <NUM_LIT> && Cond [ <NUM_LIT> ] . isReg ( ) ) { BuildMI ( & MBB , DL , get ( <STR_LIT> ) ) . add ( Cond [ <NUM_LIT> ] ) . addMBB ( TBB ) ; return <NUM_LIT> ; } assert ( TBB && Cond [ <NUM_LIT> ] . isImm ( ) ) ; unsigned Opcode = getBranchOpcode ( static_cast < BranchPredicate > ( Cond [ <NUM_LIT> ] . getImm ( ) ) ) ; if ( ! FBB ) {


<s> unsigned <ISA_LIT>InstrInfo :: insertEQ ( MachineBasicBlock * MBB , MachineBasicBlock :: iterator I , const DebugLoc & DL , unsigned SrcReg , int Value ) const { MachineRegisterInfo & MRI = MBB -> getParent ( ) -> getRegInfo ( ) ; unsigned Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , I , DL , get ( <STR_LIT> ) , Reg ) . addImm ( Value ) . addReg ( SrcReg ) ; return Reg ;


<s> MachineRegisterInfo & MRI = MBB -> getParent ( ) -> getRegInfo ( ) ; unsigned Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , I , DL , get ( <STR_LIT> ) , Reg ) . addImm ( Value ) . addReg ( SrcReg ) ; return Reg ;


<s> BuildMI ( MBB , I , DL , get ( <STR_LIT> ) ) . addReg ( PCReg , RegState :: Define , <STR_LIT> ) . addReg ( PCReg , <NUM_LIT> , <STR_LIT> ) . addImm ( <NUM_LIT> ) ; } else { BuildMI ( MBB , I , DL , get ( <STR_LIT> ) ) . addReg ( PCReg , RegState :: Define , <STR_LIT> ) . addReg ( PCReg , <NUM_LIT> , <STR_LIT> ) . addMBB ( & DestBB , <STR_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) ) . addReg ( PCReg , RegState :: Define , <STR_LIT> ) . addReg ( PCReg , <NUM_LIT> , <STR_LIT> ) . addImm ( <NUM_LIT> ) ; } BuildMI ( & MBB , DL , get ( <STR_LIT> ) ) . addReg ( PCReg ) ;


<s> MachineInstr * GetPC = BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , PCReg ) ; if ( BrOffset >= <NUM_LIT> ) { BuildMI ( MBB , I , DL , get ( <STR_LIT> ) ) . addReg ( PCReg , RegState :: Define , <STR_LIT> ) . addReg ( PCReg , <NUM_LIT> , <STR_LIT> ) . addMBB ( & DestBB , <STR_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) ) . addReg ( PCReg , RegState :: Define , <STR_LIT> ) . addReg ( PCReg , <NUM_LIT> , <STR_LIT> ) . addImm ( <NUM_LIT> ) ; } else { BuildMI ( MBB , I , DL , get ( <STR_LIT> ) ) . addReg ( PCReg , RegState :: Define , <STR_LIT> ) . addReg ( PCReg , <NUM_LIT> , <STR_LIT> ) . addMBB ( & DestBB , <STR_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) ) . addReg ( PCReg , RegState :: Define , <STR_LIT> ) . addReg ( PCReg , <NUM_LIT> , <STR_LIT> ) . addImm ( <NUM_LIT> ) ; } BuildMI ( & MBB , DL , get ( <STR_LIT> ) ) . addReg ( PCReg ) ;


<s> MachineRegisterInfo & MRI = MBB -> getParent ( ) -> getRegInfo ( ) ; unsigned Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , I , DL , get ( <STR_LIT> ) , Reg ) . addImm ( Value ) . addReg ( SrcReg ) ;


<s> insertWaitStates ( MBB , MI , <NUM_LIT> ) ;


<s> if ( HasNoTerminator ) BuildMI ( MBB , MBB . end ( ) , DebugLoc ( ) , get ( Info -> returnsVoid ( ) ? <STR_LIT> : <STR_LIT> ) ) ;


<s> void <ISA_LIT>InstrInfo :: insertReturn ( MachineBasicBlock & MBB ) const { auto MF = MBB . getParent ( ) ;

<ISA_LIT>MachineFunctionInfo * Info = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;


<s> MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; assert ( MRI . getRegClass ( DstReg ) == & <STR_LIT> && <STR_LIT> ) ; if ( Cond . size ( ) == <NUM_LIT> ) { unsigned SReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . add ( Cond [ <NUM_LIT> ] ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addReg ( FalseReg ) . addReg ( TrueReg ) . addReg ( SReg ) ; } else if ( Cond . size ( ) == <NUM_LIT> ) { assert ( Cond [ <NUM_LIT> ] . isImm ( ) && <STR_LIT> ) ; switch ( Cond [ <NUM_LIT> ] . getImm ( ) ) { case <ISA_LIT>InstrInfo :: SCC_TRUE : { unsigned SReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addReg ( FalseReg ) . addReg ( TrueReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: SCC_FALSE : { unsigned SReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addReg ( FalseReg ) . addReg ( TrueReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: VCCNZ : { MachineOperand RegOp = Cond [ <NUM_LIT> ] ; RegOp . setImplicit ( false ) ; unsigned SReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . add ( RegOp ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addReg ( FalseReg ) . addReg ( TrueReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: VCCZ : { MachineOperand RegOp = Cond [ <NUM_LIT> ] ; RegOp . setImplicit ( false ) ; unsigned SReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . add ( RegOp ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addReg ( TrueReg ) . addReg ( FalseReg ) . addReg ( SReg ) ; break ; }


<s> case <ISA_LIT>InstrInfo :: VCCZ : { MachineOperand RegOp = Cond [ <NUM_LIT> ] ; RegOp . setImplicit ( false ) ; unsigned SReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . add ( RegOp ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addReg ( TrueReg ) . addReg ( FalseReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: EXECNZ : { unsigned SReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SReg2 = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg2 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . addImm ( - <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addReg ( FalseReg ) . addReg ( TrueReg ) . addReg ( SReg ) ; break ; } case <ISA_LIT>InstrInfo :: EXECZ : { unsigned SReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SReg2 = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg2 ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , SReg ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) ; BuildMI ( MBB , I , DL , get ( <STR_LIT> ) , DstReg ) . addReg ( FalseReg ) . addReg ( TrueReg ) . addReg ( SReg ) ; llvm_unreachable ( <STR_LIT> ) ;


<s> if ( Count >= <NUM_LIT> ) Arg = <NUM_LIT> ; else Arg = Count - <NUM_LIT> ; Count -= <NUM_LIT> ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addImm ( Arg ) ; }


<s> while ( Count > <NUM_LIT> ) { int Arg ; if ( Count >= <NUM_LIT> ) Arg = <NUM_LIT> ; else Arg = Count - <NUM_LIT> ; Count -= <NUM_LIT> ;

BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addImm ( Arg ) ;


<s> return ! MI . isTerminator ( ) && MI . getOpcode ( ) != <STR_LIT> && MI . modifiesRegister ( <STR_LIT> , & RI ) ;


<s> bool <ISA_LIT>InstrInfo :: isBasicBlockPrologue ( const MachineInstr & MI ) const {

return ! MI . isTerminator ( ) && MI . getOpcode ( ) != <STR_LIT> && MI . modifiesRegister ( <STR_LIT> , & RI ) ;


<s> bool <ISA_LIT>InstrInfo :: isBranchOffsetInRange ( unsigned BranchOp , int64_t BrOffset ) const { assert ( BranchOp != <STR_LIT> ) ; BrOffset /= <NUM_LIT> ; BrOffset -= <NUM_LIT> ;

return isIntN ( BranchOffsetBits , BrOffset ) ;


<s> assert ( BranchOp != <STR_LIT> ) ; BrOffset /= <NUM_LIT> ;


<s> return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> bool isFLAT ( uint16_t Opcode ) const {

return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> bool <ISA_LIT>InstrInfo :: isFoldableCopy ( const MachineInstr & MI ) const { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> :


<s> bool <ISA_LIT>InstrInfo :: isFoldableCopy ( const MachineInstr & MI ) const { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { unsigned NumOps = MI . getDesc ( ) . getNumOperands ( ) + MI . getDesc ( ) . getNumImplicitUses ( ) ; return MI . getNumOperands ( ) == NumOps ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> bool <ISA_LIT>InstrInfo :: isHighLatencyInstruction ( const MachineInstr & MI ) const { unsigned Opc = MI . getOpcode ( ) ; return isMUBUF ( Opc ) || isMTBUF ( Opc ) || isMIMG ( Opc ) ;


<s> bool <ISA_LIT>InstrInfo :: isHighLatencyInstruction ( const MachineInstr & MI ) const {


<s> if ( OpInfo . OperandType == MCOI :: OPERAND_IMMEDIATE ) return true ; if ( OpInfo . RegClass < <NUM_LIT> ) return false ; if ( MO . isImm ( ) && isInlineConstant ( MO , OpInfo ) ) return RI . opCanUseInlineConstant ( OpInfo . OperandType ) ;

return RI . opCanUseLiteralConstant ( OpInfo . OperandType ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ( MO . getImm ( ) , ST . hasInv2PiInlineImm ( ) ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( isInt < <NUM_LIT> > ( Imm ) || isUInt < <NUM_LIT> > ( Imm ) ) { int16_t Trunc = static_cast < int16_t > ( Imm ) ; return ST . has16BitInsts ( ) && <STR_LIT> ( Trunc , ST . hasInv2PiInlineImm ( ) ) ; } return false ; } case <STR_LIT> : case <STR_LIT> : { uint32_t Trunc = static_cast < uint32_t > ( Imm ) ; return <STR_LIT> ( Trunc , ST . hasInv2PiInlineImm ( ) ) ; }


<s> case <STR_LIT> : return true ; default : return false ; }


<s> unsigned Reg = MO . getReg ( ) ; const TargetRegisterClass * RC = TargetRegisterInfo :: isVirtualRegister ( Reg ) ? MRI . getRegClass ( Reg ) : RI . getPhysRegClass ( Reg ) ;

const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ;


<s> if ( ! MO . isReg ( ) ) return false ; unsigned Reg = MO . getReg ( ) ; const TargetRegisterClass * RC = TargetRegisterInfo :: isVirtualRegister ( Reg ) ? MRI . getRegClass ( Reg ) : RI . getPhysRegClass ( Reg ) ; const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; RC = TRI -> getSubRegClass ( RC , MO . getSubReg ( ) ) ;

return RI . getCommonSubClass ( RC , RI . getRegClass ( OpInfo . RegClass ) ) == RC ;


<s> assert ( MO . isImm ( ) || MO . isTargetIndex ( ) || MO . isFI ( ) ) ; return true ;


<s> bool <ISA_LIT>InstrInfo :: isLegalVSrcOperand ( const MachineRegisterInfo & MRI , const MCOperandInfo & OpInfo , const MachineOperand & MO ) const { if ( MO . isReg ( ) ) return isLegalRegOperand ( MRI , OpInfo , MO ) ; assert ( MO . isImm ( ) || MO . isTargetIndex ( ) || MO . isFI ( ) ) ; return true ;


<s> bool <ISA_LIT>InstrInfo :: isLiteralConstantLike ( const MachineOperand & MO , const MCOperandInfo & OpInfo ) const { switch ( MO . getType ( ) ) { case MachineOperand :: MO_Register : return false ; case MachineOperand :: MO_Immediate : return ! isInlineConstant ( MO , OpInfo ) ; case MachineOperand :: MO_FrameIndex :

case MachineOperand :: MO_MachineBasicBlock :


<s> if ( isMUBUF ( MI ) || isVGPRSpill ( MI ) ) return isStackAccess ( MI , FrameIndex ) ; if ( isSGPRSpill ( MI ) ) return isSGPRStackAccess ( MI , FrameIndex ) ; return <STR_LIT> ;


<s> if ( ! MI . mayLoad ( ) ) return <STR_LIT> ; if ( isMUBUF ( MI ) || isVGPRSpill ( MI ) ) return isStackAccess ( MI , FrameIndex ) ; if ( isSGPRSpill ( MI ) ) return isSGPRStackAccess ( MI , FrameIndex ) ; return <STR_LIT> ;


<s> bool <ISA_LIT>InstrInfo :: isLowLatencyInstruction ( const MachineInstr & MI ) const { unsigned Opc = MI . getOpcode ( ) ;


<s> bool isMTBUF ( uint16_t Opcode ) const {


<s> bool isMUBUF ( uint16_t Opcode ) const {


<s> bool <ISA_LIT>InstrInfo :: isNonUniformBranchInstr ( MachineInstr & Branch ) const { return Branch . getOpcode ( ) == <STR_LIT> ;


<s> for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( i == OpIdx ) continue ; const MachineOperand & Op = MI . getOperand ( i ) ; if ( Op . isReg ( ) ) { if ( ( Op . getReg ( ) != SGPRUsed . Reg || Op . getSubReg ( ) != SGPRUsed . SubReg ) && usesConstantBus ( MRI , Op , InstDesc . OpInfo [ i ] ) ) { return false ; } } else if ( InstDesc . OpInfo [ i ] . OperandType == <STR_LIT> ) { return false ; } } } if ( MO -> isReg ( ) ) { assert ( DefinedRC ) ; return isLegalRegOperand ( MRI , OpInfo , * MO ) ; } assert ( MO -> isImm ( ) || MO -> isTargetIndex ( ) || MO -> isFI ( ) ) ; if ( ! DefinedRC ) {


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : return false ;


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : return false ;


<s> bool isSALU ( uint16_t Opcode ) const { return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> return TargetInstrInfo :: isSchedulingBoundary ( MI , MBB , MF ) || MI . modifiesRegister ( <STR_LIT> , & RI ) || MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> || changesVGPRIndexingMode ( MI ) ;


<s> assert ( Addr && Addr -> isFI ( ) ) ; FrameIndex = Addr -> getIndex ( ) ;

return getNamedOperand ( MI , <STR_LIT> :: data ) -> getReg ( ) ;


<s> bool isSMRD ( uint16_t Opcode ) const {


<s> bool isSOP2 ( uint16_t Opcode ) const {


<s> bool isSOPC ( uint16_t Opcode ) const { return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> bool isSOPK ( uint16_t Opcode ) const {


<s> bool isSOPK ( uint16_t Opcode ) const { return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> assert ( ! MI . memoperands_empty ( ) && ( * MI . memoperands_begin ( ) ) -> getAddrSpace ( ) == AMDGPUA<ISA_LIT> . PRIVATE_ADDRESS ) ; FrameIndex = Addr -> getIndex ( ) ;

return getNamedOperand ( MI , <STR_LIT> :: vdata ) -> getReg ( ) ;


<s> if ( isMUBUF ( MI ) || isVGPRSpill ( MI ) ) return isStackAccess ( MI , FrameIndex ) ; if ( isSGPRSpill ( MI ) ) return isSGPRStackAccess ( MI , FrameIndex ) ; return <STR_LIT> ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default :


<s> case <STR_LIT> : case <STR_LIT> : return true ; default :


<s> static bool isSubRegOf ( const <ISA_LIT>RegisterInfo & TRI , const MachineOperand & SuperVec , const MachineOperand & SubReg ) { if ( TargetRegisterInfo :: isPhysicalRegister ( SubReg . getReg ( ) ) ) return TRI . isSubRegister ( SuperVec . getReg ( ) , SubReg . getReg ( ) ) ; return SubReg . getSubReg ( ) != <STR_LIT> && SubReg . getReg ( ) == SuperVec . getReg ( ) ;


<s> bool isVALU ( uint16_t Opcode ) const {


<s> bool isVOP2 ( uint16_t Opcode ) const {


<s> bool isVOP2 ( uint16_t Opcode ) const { return get ( Opcode ) . TSFlags & <STR_LIT> ;


<s> if ( DstRC == OpRC ) return ; unsigned DstReg = MRI . createVirtualRegister ( DstRC ) ; MachineInstr * Copy = BuildMI ( InsertMBB , I , DL , get ( <STR_LIT> ) , DstReg ) . add ( Op ) ; Op . setReg ( DstReg ) ; Op . setSubReg ( <NUM_LIT> ) ; MachineInstr * Def = MRI . getVRegDef ( OpReg ) ;


<s> if ( Src0 . isReg ( ) && RI . isSGPRReg ( MRI , Src0 . getReg ( ) ) ) legalizeOpWithMove ( MI , Src0Idx ) ; } if ( isLegalRegOperand ( MRI , InstrDesc . OpInfo [ Src1Idx ] , Src1 ) ) return ; if ( Opc == <STR_LIT> && Src1 . isReg ( ) && RI . isVGPR ( MRI , Src1 . getReg ( ) ) ) { unsigned Reg = MRI . createVirtualRegister ( & <STR_LIT> ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; BuildMI ( * MI . getParent ( ) , MI , DL , get ( <STR_LIT> ) , Reg ) . add ( Src1 ) ; Src1 . ChangeToRegister ( Reg , false ) ; return ; } if ( HasImplicitSGPR || ! MI . isCommutable ( ) ) { legalizeOpWithMove ( MI , Src1Idx ) ; return ; } int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; MachineOperand & Src0 = MI . getOperand ( Src0Idx ) ;

if ( ( ! Src1 . isImm ( ) && ! Src1 . isReg ( ) ) || ! isLegalRegOperand ( MRI , InstrDesc . OpInfo [ Src1Idx ] , Src0 ) ) {


<s> int VOP3Idx [ <NUM_LIT> ] = { <STR_LIT> ( Opc , <STR_LIT> :: src0 ) , <STR_LIT> ( Opc , <STR_LIT> :: src1 ) , <STR_LIT> ( Opc , <STR_LIT> :: src2 ) } ; unsigned SGPRReg = findUsedSGPR ( MI , VOP3Idx ) ;

for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) {


<s> const TargetRegisterClass * RC = RI . getRegClass ( RCID ) ; unsigned Opcode = <STR_LIT> ; if ( MO . isReg ( ) ) Opcode = <STR_LIT> ; else if ( RI . isSGPRClass ( RC ) ) Opcode = <STR_LIT> ; const TargetRegisterClass * VRC = RI . getEquivalentVGPRClass ( RC ) ; if ( RI . getCommonSubClass ( & <STR_LIT> , VRC ) ) VRC = & <STR_LIT> ; else VRC = & <STR_LIT> ; unsigned Reg = MRI . createVirtualRegister ( VRC ) ; DebugLoc DL = MBB -> findDebugLoc ( I ) ;

BuildMI ( * MI . getParent ( ) , I , DL , get ( Opcode ) , Reg ) . add ( MO ) ;


<s> if ( MO . isReg ( ) ) Opcode = <STR_LIT> ; else if ( RI . isSGPRClass ( RC ) ) Opcode = <STR_LIT> ; const TargetRegisterClass * VRC = RI . getEquivalentVGPRClass ( RC ) ; if ( RI . getCommonSubClass ( & <STR_LIT> , VRC ) ) VRC = & <STR_LIT> ; else VRC = & <STR_LIT> ; unsigned Reg = MRI . createVirtualRegister ( VRC ) ; DebugLoc DL = MBB -> findDebugLoc ( I ) ;


<s> DebugLoc DL = MBB . findDebugLoc ( MI ) ; unsigned Align = FrameInfo . getObjectAlignment ( FrameIndex ) ; unsigned Size = FrameInfo . getObjectSize ( FrameIndex ) ; unsigned SpillSize = TRI -> getSpillSize ( * RC ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , FrameIndex ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; if ( RI . isSGPRClass ( RC ) ) { const MCInstrDesc & OpDesc = get ( getSGPRSpillRestoreOpcode ( SpillSize ) ) ; if ( TargetRegisterInfo :: isVirtualRegister ( DestReg ) && SpillSize == <NUM_LIT> ) { MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; MRI . constrainRegClass ( DestReg , & <STR_LIT> ) ; } FrameInfo . setStackID ( FrameIndex , <NUM_LIT> ) ; MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc , DestReg ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getFrameOffsetReg ( ) , RegState :: Implicit ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ;


<s> void <ISA_LIT>InstrInfo :: lowerScalarAbs ( SetVectorType & Worklist , MachineInstr & Inst ) const { MachineBasicBlock & MBB = * Inst . getParent ( ) ; MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; MachineBasicBlock :: iterator MII = Inst ; DebugLoc DL = Inst . getDebugLoc ( ) ; MachineOperand & Dest = Inst . getOperand ( <NUM_LIT> ) ;

MachineOperand & Src = Inst . getOperand ( <NUM_LIT> ) ;


<s> MachineBasicBlock & MBB = * Inst . getParent ( ) ; MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; MachineBasicBlock :: iterator MII = Inst ; DebugLoc DL = Inst . getDebugLoc ( ) ; MachineOperand & Dest = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src = Inst . getOperand ( <NUM_LIT> ) ; unsigned TmpReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned ResultReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned SubOp = ST . hasAddNoCarry ( ) ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , MII , DL , get ( SubOp ) , TmpReg ) . addImm ( <NUM_LIT> ) . addReg ( Src . getReg ( ) ) ; BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , ResultReg ) . addReg ( Src . getReg ( ) ) . addReg ( TmpReg ) ; MRI . replaceRegWith ( Dest . getReg ( ) , ResultReg ) ; addUsersToMoveToVALUWorklist ( ResultReg , MRI , Worklist ) ;


<s> MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; const TargetRegisterClass * RegClass = MRI . getRegClass ( DestReg ) ; if ( RegClass == & <STR_LIT> || RegClass == & <STR_LIT> || RegClass == & <STR_LIT> || RegClass == & <STR_LIT> ) { BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addImm ( Value ) ; return ; } if ( RegClass == & <STR_LIT> || RegClass == & <STR_LIT> || RegClass == & <STR_LIT> ) { BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addImm ( Value ) ; return ; } if ( RegClass == & <STR_LIT> ) { BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) , DestReg ) . addImm ( Value ) ; return ; }


<s> if ( ! isFLAT ( MI ) ) return false ;

if ( MI . memoperands_empty ( ) ) return true ;


<s> for ( const MachineMemOperand * MMO : MI . memoperands ( ) ) { if ( MMO -> getAddrSpace ( ) == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) return true ; } return false ;


<s> auto MO1 = * MI1 . memoperands_begin ( ) ; auto MO2 = * MI2 . memoperands_begin ( ) ; if ( MO1 -> getAddrSpace ( ) != MO2 -> getAddrSpace ( ) ) return false ; auto Base1 = MO1 -> getValue ( ) ; auto Base2 = MO2 -> getValue ( ) ; if ( ! Base1 || ! Base2 ) return false ; const MachineFunction & MF = * MI1 . getParent ( ) -> getParent ( ) ; const DataLayout & DL = MF . getFunction ( ) . getParent ( ) -> getDataLayout ( ) ; Base1 = GetUnderlyingObject ( Base1 , DL ) ; Base2 = GetUnderlyingObject ( Base1 , DL ) ;

if ( isa < UndefValue > ( Base1 ) || isa < UndefValue > ( Base2 ) ) return false ;


<s> const MachineFunction & MF = * MI1 . getParent ( ) -> getParent ( ) ; const DataLayout & DL = MF . getFunction ( ) . getParent ( ) -> getDataLayout ( ) ; Base1 = GetUnderlyingObject ( Base1 , DL ) ; Base2 = GetUnderlyingObject ( Base1 , DL ) ; if ( isa < UndefValue > ( Base1 ) || isa < UndefValue > ( Base2 ) ) return false ; return Base1 == Base2 ;


<s> MachineBasicBlock * MBB = Inst . getParent ( ) ; MachineOperand & Src0 = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = Inst . getOperand ( <NUM_LIT> ) ; const DebugLoc & DL = Inst . getDebugLoc ( ) ; switch ( Inst . getOpcode ( ) ) { case <STR_LIT> : { unsigned ImmReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned TmpReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , Inst , DL , get ( <STR_LIT> ) , ImmReg ) . addImm ( <NUM_LIT> ) ; BuildMI ( * MBB , Inst , DL , get ( <STR_LIT> ) , TmpReg ) . addReg ( ImmReg , RegState :: Kill ) . add ( Src0 ) ; BuildMI ( * MBB , Inst , DL , get ( <STR_LIT> ) , ResultReg ) . add ( Src1 ) . addImm ( <NUM_LIT> ) . addReg ( TmpReg , RegState :: Kill ) ; break ; } case <STR_LIT> : { unsigned ImmReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , Inst , DL , get ( <STR_LIT> ) , ImmReg ) . addImm ( <NUM_LIT> ) ; BuildMI ( * MBB , Inst , DL , get ( <STR_LIT> ) , ResultReg ) . addReg ( ImmReg , RegState :: Kill ) . add ( Src0 ) . add ( Src1 ) ; break ; } case <STR_LIT> : { unsigned ImmReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned TmpReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , Inst , DL , get ( <STR_LIT> ) , TmpReg ) . addImm ( <NUM_LIT> ) . add ( Src0 ) ; BuildMI ( * MBB , Inst , DL , get ( <STR_LIT> ) , ImmReg ) . addImm ( <NUM_LIT> ) ; BuildMI ( * MBB , Inst , DL , get ( <STR_LIT> ) , ResultReg ) . add ( Src1 ) . addReg ( ImmReg , RegState :: Kill ) . addReg ( TmpReg , RegState :: Kill ) ; break ; } default : llvm_unreachable ( <STR_LIT> ) ; } MachineOperand & Dest = Inst . getOperand ( <NUM_LIT> ) ; MRI . replaceRegWith ( Dest . getReg ( ) , ResultReg ) ;

addUsersToMoveToVALUWorklist ( ResultReg , MRI , Worklist ) ;


<s> unsigned NewOpc = Opc == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; assert ( Inst . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) ; Inst . RemoveOperand ( <NUM_LIT> ) ; Inst . setDesc ( get ( NewOpc ) ) ; Inst . addImplicitDefUseOperands ( * MBB . getParent ( ) ) ; MRI . replaceRegWith ( OldDstReg , ResultReg ) ; legalizeOperands ( Inst ) ; addUsersToMoveToVALUWorklist ( ResultReg , MRI , Worklist ) ; return true ; }


<s> assert ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ; unsigned NewOpc = Opc == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; assert ( Inst . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) ; Inst . RemoveOperand ( <NUM_LIT> ) ; Inst . setDesc ( get ( NewOpc ) ) ; Inst . addImplicitDefUseOperands ( * MBB . getParent ( ) ) ; MRI . replaceRegWith ( OldDstReg , ResultReg ) ; legalizeOperands ( Inst ) ;


<s> unsigned Opc0 = N0 -> getMachineOpcode ( ) ; unsigned Opc1 = N1 -> getMachineOpcode ( ) ; int Op0Idx = <STR_LIT> ( Opc0 , OpName ) ; int Op1Idx = <STR_LIT> ( Opc1 , OpName ) ; if ( Op0Idx == - <NUM_LIT> && Op1Idx == - <NUM_LIT> ) return true ;

if ( ( Op0Idx == - <NUM_LIT> && Op1Idx != - <NUM_LIT> ) || ( Op1Idx == - <NUM_LIT> && Op0Idx != - <NUM_LIT> ) ) return false ;


<s> int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB ; int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA ; int LowWidth = ( LowOffset == OffsetA ) ? WidthA : WidthB ;

return LowOffset + LowWidth <= HighOffset ;


<s> int LowWidth = ( LowOffset == OffsetA ) ? WidthA : WidthB ; return LowOffset + LowWidth <= HighOffset ;


<s> CondReg . setIsUndef ( OrigCond . isUndef ( ) ) ; CondReg . setIsKill ( OrigCond . isKill ( ) ) ;


<s> unsigned SGPR = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * UseMI . getParent ( ) , UseMI , UseMI . getDebugLoc ( ) , get ( <STR_LIT> ) , SGPR ) . addReg ( SrcReg , <NUM_LIT> , RI . getSubRegFromChannel ( i ) ) ; SRegs . push_back ( SGPR ) ; } MachineInstrBuilder MIB = BuildMI ( * UseMI . getParent ( ) , UseMI , UseMI . getDebugLoc ( ) , get ( <STR_LIT> ) , DstReg ) ; for ( unsigned i = <NUM_LIT> ; i < SubRegs ; ++ i ) { MIB . addReg ( SRegs [ i ] ) ; MIB . addImm ( RI . getSubRegFromChannel ( i ) ) ;


<s> for ( unsigned i = <NUM_LIT> ; i < SubRegs ; ++ i ) { unsigned SGPR = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * UseMI . getParent ( ) , UseMI , UseMI . getDebugLoc ( ) , get ( <STR_LIT> ) , SGPR ) . addReg ( SrcReg , <NUM_LIT> , RI . getSubRegFromChannel ( i ) ) ; SRegs . push_back ( SGPR ) ; }

MachineInstrBuilder MIB = BuildMI ( * UseMI . getParent ( ) , UseMI , UseMI . getDebugLoc ( ) , get ( <STR_LIT> ) , DstReg ) ;


<s> unsigned RemovedSize = <NUM_LIT> ; while ( I != MBB . end ( ) ) { MachineBasicBlock :: iterator Next = std :: next ( I ) ;

if ( I -> getOpcode ( ) == <STR_LIT> ) {


<s> MachineBasicBlock :: iterator I = MBB . getFirstTerminator ( ) ; unsigned Count = <NUM_LIT> ; unsigned RemovedSize = <NUM_LIT> ; while ( I != MBB . end ( ) ) { MachineBasicBlock :: iterator Next = std :: next ( I ) ;


<s> int Src1ModIdx = <STR_LIT> ( Opc , <STR_LIT> :: src1_modifiers ) ; int Src2ModIdx = <STR_LIT> ( Opc , <STR_LIT> :: src2_modifiers ) ; MI . RemoveOperand ( Src2ModIdx ) ; MI . RemoveOperand ( Src1ModIdx ) ;


<s> LLVMContext & C = MF -> getFunction ( ) . getContext ( ) ; C . diagnose ( IllegalCopy ) ;

BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , DestReg ) . addReg ( SrcReg , getKillRegState ( KillSrc ) ) ;


<s> bool <ISA_LIT>InstrInfo :: reverseBranchCondition ( SmallVectorImpl < MachineOperand > & Cond ) const { if ( Cond . size ( ) != <NUM_LIT> ) { return true ; } if ( Cond [ <NUM_LIT> ] . isImm ( ) ) { Cond [ <NUM_LIT> ] . setImm ( - Cond [ <NUM_LIT> ] . getImm ( ) ) ; return false ;


<s> bool <ISA_LIT>InstrInfo :: reverseBranchCondition ( SmallVectorImpl < MachineOperand > & Cond ) const { if ( Cond . size ( ) != <NUM_LIT> ) { return true ; } if ( Cond [ <NUM_LIT> ] . isImm ( ) ) {

Cond [ <NUM_LIT> ] . setImm ( - Cond [ <NUM_LIT> ] . getImm ( ) ) ;


<s> bool <ISA_LIT>InstrInfo :: shouldClusterMemOps ( MachineInstr & FirstLdSt , unsigned BaseReg1 , MachineInstr & SecondLdSt , unsigned BaseReg2 , unsigned NumLoads ) const { if ( ! memOpsHaveSameBasePtr ( FirstLdSt , BaseReg1 , SecondLdSt , BaseReg2 ) ) return false ; const MachineOperand * FirstDst = nullptr ; const MachineOperand * SecondDst = nullptr ; if ( ( isMUBUF ( FirstLdSt ) && isMUBUF ( SecondLdSt ) ) || ( isMTBUF ( FirstLdSt ) && isMTBUF ( SecondLdSt ) ) || ( isFLAT ( FirstLdSt ) && isFLAT ( SecondLdSt ) ) ) { const unsigned MaxGlobalLoadCluster = <NUM_LIT> ; if ( NumLoads > MaxGlobalLoadCluster ) return false ; FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: vdata ) ; if ( ! FirstDst ) FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: vdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdata ) ; if ( ! SecondDst ) SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: vdst ) ; } else if ( isSMRD ( FirstLdSt ) && isSMRD ( SecondLdSt ) ) { FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: sdst ) ; SecondDst = getNamedOperand ( SecondLdSt , <STR_LIT> :: sdst ) ; } else if ( isDS ( FirstLdSt ) && isDS ( SecondLdSt ) ) { FirstDst = getNamedOperand ( FirstLdSt , <STR_LIT> :: vdst ) ;


<s> static bool shouldReadExec ( const MachineInstr & MI ) { if ( <ISA_LIT>InstrInfo :: isVALU ( MI ) ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


if ( <ISA_LIT>InstrInfo :: isGenericOpcode ( MI . getOpcode ( ) ) || <ISA_LIT>InstrInfo :: isSALU ( MI ) || <ISA_LIT>InstrInfo :: isSMRD ( MI ) ) return false ;


<s> <ISA_LIT>InstrInfo :: <ISA_LIT>InstrInfo ( const <ISA_LIT>Subtarget & ST ) : <ISA_LIT>InstrInfo ( ST ) , RI ( ST ) , ST ( ST ) {


<s> MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm ( MII , MRI , Src1 , Src1RC , <STR_LIT> , Src1SubRC ) ; MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src1 , Src1RC , <STR_LIT> , Src1SubRC ) ; unsigned LoOpc = IsAdd ? <STR_LIT> : <STR_LIT> ; MachineInstr * LoHalf = BuildMI ( MBB , MII , DL , get ( LoOpc ) , DestSub0 ) . addReg ( CarryReg , RegState :: Define ) . add ( SrcReg0Sub0 ) . add ( SrcReg1Sub0 ) ; unsigned HiOpc = IsAdd ? <STR_LIT> : <STR_LIT> ; MachineInstr * HiHalf = BuildMI ( MBB , MII , DL , get ( HiOpc ) , DestSub1 ) . addReg ( DeadCarryReg , RegState :: Define | RegState :: Dead ) . add ( SrcReg0Sub1 ) . add ( SrcReg1Sub1 ) . addReg ( CarryReg , RegState :: Kill ) ; BuildMI ( MBB , MII , DL , get ( TargetOpcode :: REG_SEQUENCE ) , FullDestReg ) . addReg ( DestSub0 ) . addImm ( <STR_LIT> ) . addReg ( DestSub1 ) . addImm ( <STR_LIT> ) ; MRI . replaceRegWith ( Dest . getReg ( ) , FullDestReg ) ; legalizeOperands ( * LoHalf ) ;


<s> MachineBasicBlock & MBB = * Inst . getParent ( ) ; MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; MachineBasicBlock :: iterator MII = Inst ; DebugLoc DL = Inst . getDebugLoc ( ) ; MachineOperand & Dest = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src = Inst . getOperand ( <NUM_LIT> ) ; const MCInstrDesc & InstDesc = get ( <STR_LIT> ) ; const TargetRegisterClass * SrcRC = Src . isReg ( ) ? MRI . getRegClass ( Src . getReg ( ) ) : & <STR_LIT> ; unsigned MidReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned ResultReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; const TargetRegisterClass * SrcSubRC = RI . getSubRegClass ( SrcRC , <STR_LIT> ) ; MachineOperand SrcRegSub0 = buildExtractSubRegOrImm ( MII , MRI , Src , SrcRC , <STR_LIT> , SrcSubRC ) ; MachineOperand SrcRegSub1 = buildExtractSubRegOrImm ( MII , MRI , Src , SrcRC , <STR_LIT> , SrcSubRC ) ;

BuildMI ( MBB , MII , DL , InstDesc , MidReg ) . add ( SrcRegSub0 ) . addImm ( <NUM_LIT> ) ;


<s> uint32_t BitWidth = ( Imm & <NUM_LIT> ) >> <NUM_LIT> ; ( void ) Offset ; assert ( Inst . getOpcode ( ) == <STR_LIT> && BitWidth <= <NUM_LIT> && Offset == <NUM_LIT> && <STR_LIT> ) ; if ( BitWidth < <NUM_LIT> ) { unsigned MidRegLo = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned MidRegHi = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned ResultReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , MidRegLo ) . addReg ( Inst . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , <STR_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( BitWidth ) ; BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , MidRegHi ) . addImm ( <NUM_LIT> ) . addReg ( MidRegLo ) ; BuildMI ( MBB , MII , DL , get ( TargetOpcode :: REG_SEQUENCE ) , ResultReg ) . addReg ( MidRegLo ) . addImm ( <STR_LIT> ) . addReg ( MidRegHi ) . addImm ( <STR_LIT> ) ; MRI . replaceRegWith ( Dest . getReg ( ) , ResultReg ) ; addUsersToMoveToVALUWorklist ( ResultReg , MRI , Worklist ) ; return ; } MachineOperand & Src = Inst . getOperand ( <NUM_LIT> ) ;

unsigned TmpReg = MRI . createVirtualRegister ( & <STR_LIT> ) ;


<s> BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , MidRegHi ) . addImm ( <NUM_LIT> ) . addReg ( MidRegLo ) ; BuildMI ( MBB , MII , DL , get ( TargetOpcode :: REG_SEQUENCE ) , ResultReg ) . addReg ( MidRegLo ) . addImm ( <STR_LIT> ) . addReg ( MidRegHi ) . addImm ( <STR_LIT> ) ; MRI . replaceRegWith ( Dest . getReg ( ) , ResultReg ) ; addUsersToMoveToVALUWorklist ( ResultReg , MRI , Worklist ) ; return ; } MachineOperand & Src = Inst . getOperand ( <NUM_LIT> ) ; unsigned TmpReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; unsigned ResultReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , MII , DL , get ( <STR_LIT> ) , TmpReg ) . addImm ( <NUM_LIT> ) . addReg ( Src . getReg ( ) , <NUM_LIT> , <STR_LIT> ) ;

BuildMI ( MBB , MII , DL , get ( TargetOpcode :: REG_SEQUENCE ) , ResultReg ) . addReg ( Src . getReg ( ) , <NUM_LIT> , <STR_LIT> ) . addImm ( <STR_LIT> ) . addReg ( TmpReg ) . addImm ( <STR_LIT> ) ;


<s> const TargetRegisterClass * NewDestSubRC = RI . getSubRegClass ( NewDestRC , <STR_LIT> ) ; unsigned DestSub0 = MRI . createVirtualRegister ( NewDestSubRC ) ; MachineInstr & LoHalf = * BuildMI ( MBB , MII , DL , InstDesc , DestSub0 ) . add ( SrcReg0Sub0 ) . add ( SrcReg1Sub0 ) ; MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src1 , Src1RC , <STR_LIT> , Src1SubRC ) ; unsigned DestSub1 = MRI . createVirtualRegister ( NewDestSubRC ) ; MachineInstr & HiHalf = * BuildMI ( MBB , MII , DL , InstDesc , DestSub1 ) . add ( SrcReg0Sub1 ) . add ( SrcReg1Sub1 ) ; unsigned FullDestReg = MRI . createVirtualRegister ( NewDestRC ) ;

BuildMI ( MBB , MII , DL , get ( TargetOpcode :: REG_SEQUENCE ) , FullDestReg ) . addReg ( DestSub0 ) . addImm ( <STR_LIT> ) . addReg ( DestSub1 ) . addImm ( <STR_LIT> ) ;


<s> MachineOperand & Dest = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src0 = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = Inst . getOperand ( <NUM_LIT> ) ; DebugLoc DL = Inst . getDebugLoc ( ) ; MachineBasicBlock :: iterator MII = Inst ; const MCInstrDesc & InstDesc = get ( Opcode ) ; const TargetRegisterClass * Src0RC = Src0 . isReg ( ) ? MRI . getRegClass ( Src0 . getReg ( ) ) : & <STR_LIT> ; const TargetRegisterClass * Src0SubRC = RI . getSubRegClass ( Src0RC , <STR_LIT> ) ; const TargetRegisterClass * Src1RC = Src1 . isReg ( ) ? MRI . getRegClass ( Src1 . getReg ( ) ) : & <STR_LIT> ; const TargetRegisterClass * Src1SubRC = RI . getSubRegClass ( Src1RC , <STR_LIT> ) ; MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm ( MII , MRI , Src1 , Src1RC , <STR_LIT> , Src1SubRC ) ;

const TargetRegisterClass * DestRC = MRI . getRegClass ( Dest . getReg ( ) ) ;


<s> void <ISA_LIT>InstrInfo :: splitScalar64BitUnaryOp ( SetVectorType & Worklist , MachineInstr & Inst , unsigned Opcode ) const { MachineBasicBlock & MBB = * Inst . getParent ( ) ; MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; MachineOperand & Dest = Inst . getOperand ( <NUM_LIT> ) ; MachineOperand & Src0 = Inst . getOperand ( <NUM_LIT> ) ; DebugLoc DL = Inst . getDebugLoc ( ) ; MachineBasicBlock :: iterator MII = Inst ; const MCInstrDesc & InstDesc = get ( Opcode ) ; const TargetRegisterClass * Src0RC = Src0 . isReg ( ) ? MRI . getRegClass ( Src0 . getReg ( ) ) : & <STR_LIT> ; const TargetRegisterClass * Src0SubRC = RI . getSubRegClass ( Src0RC , <STR_LIT> ) ; MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ;


<s> const TargetRegisterClass * Src0SubRC = RI . getSubRegClass ( Src0RC , <STR_LIT> ) ; MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; const TargetRegisterClass * DestRC = MRI . getRegClass ( Dest . getReg ( ) ) ; const TargetRegisterClass * NewDestRC = RI . getEquivalentVGPRClass ( DestRC ) ; const TargetRegisterClass * NewDestSubRC = RI . getSubRegClass ( NewDestRC , <STR_LIT> ) ; unsigned DestSub0 = MRI . createVirtualRegister ( NewDestSubRC ) ; BuildMI ( MBB , MII , DL , InstDesc , DestSub0 ) . add ( SrcReg0Sub0 ) ; MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm ( MII , MRI , Src0 , Src0RC , <STR_LIT> , Src0SubRC ) ; unsigned DestSub1 = MRI . createVirtualRegister ( NewDestSubRC ) ;

BuildMI ( MBB , MII , DL , InstDesc , DestSub1 ) . add ( SrcReg0Sub1 ) ;


<s> MachineInstrBuilder Spill = BuildMI ( MBB , MI , DL , OpDesc ) . addReg ( SrcReg , getKillRegState ( isKill ) ) . addFrameIndex ( FrameIndex ) . addMemOperand ( MMO ) . addReg ( MFI -> getScratchRSrcReg ( ) , RegState :: Implicit ) . addReg ( MFI -> getFrameOffsetReg ( ) , RegState :: Implicit ) ; FrameInfo . setStackID ( FrameIndex , <NUM_LIT> ) ; if ( ST . hasScalarStores ( ) ) { Spill . addReg ( <STR_LIT> , RegState :: ImplicitDefine | RegState :: Dead ) ; } return ; } if ( ! ST . isVGPRSpillingEnabled ( MF -> getFunction ( ) ) ) { LLVMContext & Ctx = MF -> getFunction ( ) . getContext ( ) ; Ctx . emitError ( <STR_LIT> <STR_LIT> ) ; BuildMI ( MBB , MI , DL , get ( <STR_LIT> ) ) . addReg ( SrcReg ) ;


<s> void <ISA_LIT>InstrInfo :: swapOperands ( MachineInstr & Inst ) const { assert ( Inst . getNumExplicitOperands ( ) == <NUM_LIT> ) ; MachineOperand Op1 = Inst . getOperand ( <NUM_LIT> ) ; Inst . RemoveOperand ( <NUM_LIT> ) ; Inst . addOperand ( Op1 ) ;


<s> bool IsUndef = RegOp . isUndef ( ) ; bool IsDebug = RegOp . isDebug ( ) ; if ( NonRegOp . isImm ( ) ) RegOp . ChangeToImmediate ( NonRegOp . getImm ( ) ) ; else if ( NonRegOp . isFI ( ) ) RegOp . ChangeToFrameIndex ( NonRegOp . getIndex ( ) ) ; else return nullptr ; NonRegOp . ChangeToRegister ( Reg , false , false , IsKill , IsDead , IsUndef , IsDebug ) ;


<s> bool <ISA_LIT>InstrInfo :: swapSourceModifiers ( MachineInstr & MI , MachineOperand & Src0 , unsigned Src0OpName , MachineOperand & Src1 , unsigned Src1OpName ) const { MachineOperand * Src0Mods = getNamedOperand ( MI , Src0OpName ) ; if ( ! Src0Mods ) return false ; MachineOperand * Src1Mods = getNamedOperand ( MI , Src1OpName ) ; assert ( Src1Mods && <STR_LIT> ) ; int Src0ModsVal = Src0Mods -> getImm ( ) ;

int Src1ModsVal = Src1Mods -> getImm ( ) ;


<s> for ( MachineRegisterInfo :: use_iterator I = MRI . use_begin ( DstReg ) , E = MRI . use_end ( ) ; I != E ; ++ I ) { MachineInstr & UseMI = * I -> getParent ( ) ; if ( ! canReadVGPR ( UseMI , I . getOperandNo ( ) ) ) { Worklist . push_back ( & UseMI ) ; }


<s> for ( MachineRegisterInfo :: use_iterator I = MRI . use_begin ( DstReg ) , E = MRI . use_end ( ) ; I != E ; ++ I ) { MachineInstr & UseMI = * I -> getParent ( ) ;

if ( ! canReadVGPR ( UseMI , I . getOperandNo ( ) ) ) {