<s> auto IsHazardFn = [ TII ] ( const MachineInstr & MI ) { return TII -> isVOPC ( MI ) ; } ; auto IsExpiredFn = [ ] ( const MachineInstr & MI , int ) { unsigned Opc = MI . getOpcode ( ) ; return <ISA_LIT>InstrInfo :: isVALU ( MI ) && Opc != <STR_LIT> && Opc != <STR_LIT> && Opc != <STR_LIT> ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ;

auto * Src0 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) ;


<s> bool GCNHazardRecognizer :: fixVMEMtoScalarWriteHazards ( MachineInstr * MI ) { if ( ! ST . hasVMEMtoScalarWriteHazard ( ) ) return false ; if ( ! <ISA_LIT>InstrInfo :: isSALU ( * MI ) && ! <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) return false ; if ( MI -> getNumDefs ( ) == <NUM_LIT> ) return false ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; auto IsHazardFn = [ TRI , MI ] ( const MachineInstr & I ) { if ( ! <ISA_LIT>InstrInfo :: isVMEM ( I ) && ! <ISA_LIT>InstrInfo :: isDS ( I ) && ! <ISA_LIT>InstrInfo :: isFLAT ( I ) ) return false ; for ( const MachineOperand & Def : MI -> defs ( ) ) { const MachineOperand * Op = I . findRegisterUseOperand ( Def . getReg ( ) , false , TRI ) ; if ( ! Op ) continue ; return true ; } return false ; } ; auto IsExpiredFn = [ ] ( const MachineInstr & MI , int ) { return <ISA_LIT>InstrInfo :: isVALU ( MI ) || ( MI . getOpcode ( ) == <STR_LIT> && ! MI . getOperand ( <NUM_LIT> ) . getImm ( ) ) || ( MI . getOpcode ( ) == <STR_LIT> && MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ;

BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ;


<s> IsHazardRecognizerMode ( false ) , CurrCycleInstr ( nullptr ) , MF ( MF ) , ST ( MF . getSubtarget < GCNSubtarget > ( ) ) , TII ( * ST . getInstrInfo ( ) ) , TRI ( TII . getRegisterInfo ( ) ) , ClauseUses ( TRI . getNumRegUnits ( ) ) , ClauseDefs ( TRI . getNumRegUnits ( ) ) { MaxLookAhead = MF . getRegInfo ( ) . isPhysRegUsed ( <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; TSchedModel . init ( & ST ) ; RunLdsBranchVmemWARHazardFixup = shouldRunLdsBranchVmemWARHazardFixup ( MF , ST ) ;


<s> if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) || <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) || <ISA_LIT>InstrInfo :: isDS ( * MI ) || <ISA_LIT>InstrInfo :: isEXP ( * MI ) ) && checkMAIVALUHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ( ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) ) || ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) ) || ( ST . hasReadM0LdsDmaHazard ( ) && isLdsDma ( * MI ) ) || ( ST . hasReadM0LdsDirectHazard ( ) && MI -> readsRegister ( <STR_LIT> ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) || <ISA_LIT>InstrInfo :: isDS ( * MI ) ) && checkMAILdStHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return HazardType ; return NoHazard ;


<s> static unsigned getHWReg ( const <ISA_LIT>InstrInfo * TII , const MachineInstr & RegInstr ) { const MachineOperand * RegOp = TII -> getNamedOperand ( RegInstr , <STR_LIT> :: simm16 ) ; return RegOp -> getImm ( ) & <STR_LIT> :: ID_MASK_ ;


<s> const MachineOperand * RegOp = TII -> getNamedOperand ( RegInstr , <STR_LIT> :: simm16 ) ;

return RegOp -> getImm ( ) & <STR_LIT> :: ID_MASK_ ;


<s> const MCSchedClassDesc * SC = TSchedModel . resolveSchedClass ( & MI ) ; assert ( TSchedModel . getWriteProcResBegin ( SC ) != TSchedModel . getWriteProcResEnd ( SC ) ) ;

return TSchedModel . getWriteProcResBegin ( SC ) -> Cycles ;


<s> unsigned GCNHazardRecognizer :: getMFMAPipelineWaitStates ( const MachineInstr & MI ) const { const MCSchedClassDesc * SC = TSchedModel . resolveSchedClass ( & MI ) ;

assert ( TSchedModel . getWriteProcResBegin ( SC ) != TSchedModel . getWriteProcResEnd ( SC ) ) ;


<s> return IsHazardDef ( MI ) && MI . modifiesRegister ( Reg , TRI ) ; } ; return getWaitStatesSince ( IsHazardFn , Limit ) ;


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ;

auto IsHazardFn = [ IsHazardDef , TRI , Reg ] ( const MachineInstr & MI ) {


<s> int GCNHazardRecognizer :: getWaitStatesSinceSetReg ( IsHazardFn IsHazard , int Limit ) { auto IsHazardFn = [ IsHazard ] ( const MachineInstr & MI ) {

return isSSetReg ( MI . getOpcode ( ) ) && IsHazard ( MI ) ;


<s> static void insertNoopsInBundle ( MachineInstr * MI , const <ISA_LIT>InstrInfo & TII , unsigned Quantity ) { while ( Quantity > <NUM_LIT> ) {

unsigned Arg = std :: min ( Quantity , <NUM_LIT> ) ;


<s> BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc ( ) , TII . get ( <STR_LIT> ) ) . addImm ( Arg - <NUM_LIT> ) ;


<s> static bool isDivFMas ( unsigned Opcode ) { return Opcode == <STR_LIT> || Opcode == <STR_LIT> ;


<s> return Opcode == <STR_LIT> || Opcode == <STR_LIT> ;


<s> return <ISA_LIT>InstrInfo :: isVALU ( MI ) && ( <ISA_LIT>InstrInfo :: isMUBUF ( MI ) || <ISA_LIT>InstrInfo :: isFLAT ( MI ) ) ;


<s> unsigned Opcode = MI . getOpcode ( ) ; return Opcode == <STR_LIT> || Opcode == <STR_LIT> ;


<s> static bool isRFE ( unsigned Opcode ) {


<s> if ( TII . isAlwaysGDS ( MI . getOpcode ( ) ) ) return true ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return false ;


<s> return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return false ; default : if ( TII . isDS ( MI . getOpcode ( ) ) ) { int GDS = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: gds ) ;

if ( MI . getOperand ( GDS ) . getImm ( ) ) return true ;


<s> static bool isSGetReg ( unsigned Opcode ) { return Opcode == <STR_LIT> ;


<s> static bool isSGetReg ( unsigned Opcode ) {


<s> case <STR_LIT> : return true ; default : return false ; }


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default :


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> static bool isSSetReg ( unsigned Opcode ) { switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> MFMAPaddingRatioParser ( cl :: Option & O ) : cl :: parser < unsigned > ( O ) {


<s> if ( Arg . getAsInteger ( <NUM_LIT> , Value ) ) return O . error ( <STR_LIT> + Arg + <STR_LIT> ) ; if ( Value > <NUM_LIT> ) return O . error ( <STR_LIT> + Arg + <STR_LIT> ) ;


<s> bool parse ( cl :: Option & O , StringRef ArgName , StringRef Arg , unsigned & Value ) { if ( Arg . getAsInteger ( <NUM_LIT> , Value ) ) return O . error ( <STR_LIT> + Arg + <STR_LIT> ) ;

if ( Value > <NUM_LIT> ) return O . error ( <STR_LIT> + Arg + <STR_LIT> ) ;


<s> IsHazardRecognizerMode = true ; CurrCycleInstr = MI ; unsigned W = PreEmitNoopsCommon ( MI ) ; fixHazards ( MI ) ; CurrCycleInstr = nullptr ; return W ;


<s> unsigned GCNHazardRecognizer :: PreEmitNoops ( MachineInstr * MI ) { IsHazardRecognizerMode = true ; CurrCycleInstr = MI ;

unsigned W = PreEmitNoopsCommon ( MI ) ;


<s> if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) || <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) || <ISA_LIT>InstrInfo :: isDS ( * MI ) || <ISA_LIT>InstrInfo :: isEXP ( * MI ) ) && checkMAIVALUHazards ( MI ) > <NUM_LIT> ) WaitStates = std :: max ( WaitStates , checkMAIVALUHazards ( MI ) ) ; if ( MI -> isInlineAsm ( ) ) return std :: max ( WaitStates , checkInlineAsmHazards ( MI ) ) ; if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ; if ( ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) ) || ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) ) || ( ST . hasReadM0LdsDmaHazard ( ) && isLdsDma ( * MI ) ) || ( ST . hasReadM0LdsDirectHazard ( ) && MI -> readsRegister ( <STR_LIT> ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) ) return std :: max ( WaitStates , checkMAIHazards ( MI ) ) ;

if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) || <ISA_LIT>InstrInfo :: isDS ( * MI ) ) return std :: max ( WaitStates , checkMAILdStHazards ( MI ) ) ;


<s> int WaitStates = <NUM_LIT> ; if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) return std :: max ( WaitStates , checkSMRDHazards ( MI ) ) ; if ( ST . hasNSAtoVMEMBug ( ) ) WaitStates = std :: max ( WaitStates , checkNSAtoVMEMHazard ( MI ) ) ; WaitStates = std :: max ( WaitStates , checkFPAtomicToDenormModeHazard ( MI ) ) ; if ( ST . hasNoDataDepHazard ( ) ) return WaitStates ; if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVMEMHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ;

if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ;


<s> CurrCycleInstr = & * MI ; unsigned WaitStates = PreEmitNoopsCommon ( CurrCycleInstr ) ; if ( IsHazardRecognizerMode ) { fixHazards ( CurrCycleInstr ) ; insertNoopsInBundle ( CurrCycleInstr , TII , WaitStates ) ; } for ( unsigned i = <NUM_LIT> , e = std :: min ( WaitStates , MaxLookAhead - <NUM_LIT> ) ; i < e ; ++ i ) EmittedInstrs . push_front ( nullptr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; EmittedInstrs . resize ( MaxLookAhead ) ; }


<s> MachineBasicBlock :: instr_iterator E = CurrCycleInstr -> getParent ( ) -> instr_end ( ) ; for ( ; MI != E && MI -> isInsideBundle ( ) ; ++ MI ) { CurrCycleInstr = & * MI ; unsigned WaitStates = PreEmitNoopsCommon ( CurrCycleInstr ) ; if ( IsHazardRecognizerMode ) { fixHazards ( CurrCycleInstr ) ; insertNoopsInBundle ( CurrCycleInstr , TII , WaitStates ) ; } for ( unsigned i = <NUM_LIT> , e = std :: min ( WaitStates , MaxLookAhead - <NUM_LIT> ) ; i < e ; ++ i ) EmittedInstrs . push_front ( nullptr ) ;

EmittedInstrs . push_front ( CurrCycleInstr ) ;


<s> void GCNHazardRecognizer :: RecedeCycle ( ) {


<s> void GCNHazardRecognizer :: Reset ( ) {


<s> void resetClause ( ) { ClauseUses . reset ( ) ;


<s> for ( auto & MI : MBB ) { HasLds |= <ISA_LIT>InstrInfo :: isDS ( MI ) ; HasVmem |= <ISA_LIT>InstrInfo :: isVMEM ( MI ) || <ISA_LIT>InstrInfo :: isSegmentSpecificFLAT ( MI ) ; if ( HasLds && HasVmem ) return true ; }


<s> for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; } EmittedInstrs . resize ( getMaxLookAhead ( ) ) ;


<s> auto IsHazardFn = [ ] ( MachineInstr * MI ) { return MI -> getOpcode ( ) == <STR_LIT> ; } ; int WaitStatesNeededForUse = MovFedWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardFn ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const int DivFMasWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ; int WaitStatesNeeded = getWaitStatesSinceDef ( <STR_LIT> , IsHazardDefFn ) ;

return DivFMasWaitStates - WaitStatesNeeded ;


<s> int GCNHazardRecognizer :: checkDivFMasHazards ( MachineInstr * DivFMas ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ;

const int DivFMasWaitStates = <NUM_LIT> ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; int DppVgprWaitStates = <NUM_LIT> ; int DppExecWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ;

for ( const MachineOperand & Use : DPP -> uses ( ) ) {


<s> int GCNHazardRecognizer :: checkGetRegHazards ( MachineInstr * GetRegInstr ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ;

unsigned GetRegHWReg = getHWReg ( TII , * GetRegInstr ) ;


<s> unsigned GetRegHWReg = getHWReg ( TII , * GetRegInstr ) ; const int GetRegWaitStates = <NUM_LIT> ; auto IsHazardFn = [ TII , GetRegHWReg ] ( MachineInstr * MI ) { return GetRegHWReg == getHWReg ( TII , * MI ) ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const int SMovRelWaitStates = <NUM_LIT> ; auto IsHazardFn = [ TII ] ( MachineInstr * MI ) { return TII -> isSALU ( * MI ) ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const int SMovRelWaitStates = <NUM_LIT> ; auto IsHazardFn = [ TII ] ( MachineInstr * MI ) {


<s> const int RFEWaitStates = <NUM_LIT> ; auto IsHazardFn = [ TII ] ( MachineInstr * MI ) { return getHWReg ( TII , * MI ) == <STR_LIT> :: ID_TRAPSTS ; } ; int WaitStatesNeeded = getWaitStatesSinceSetReg ( IsHazardFn ) ; return RFEWaitStates - WaitStatesNeeded ;


<s> int GCNHazardRecognizer :: checkRFEHazards ( MachineInstr * RFE ) { if ( ST . getGeneration ( ) < <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return <NUM_LIT> ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ;


<s> const MachineOperand * LaneSelectOp = TII -> getNamedOperand ( * RWLane , <STR_LIT> :: src1 ) ; if ( ! LaneSelectOp -> isReg ( ) || ! TRI -> isSGPRReg ( MRI , LaneSelectOp -> getReg ( ) ) ) return <NUM_LIT> ; unsigned LaneSelectReg = LaneSelectOp -> getReg ( ) ; auto IsHazardFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ;

const int RWLaneWaitStates = <NUM_LIT> ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; unsigned HWReg = getHWReg ( TII , * SetRegInstr ) ; const int SetRegWaitStates = ST . getGeneration ( ) <= <ISA_LIT>Subtarget :: SEA_ISLANDS ? <NUM_LIT> : <NUM_LIT> ; auto IsHazardFn = [ TII , HWReg ] ( MachineInstr * MI ) { return HWReg == getHWReg ( TII , * MI ) ; } ; int WaitStatesNeeded = getWaitStatesSinceSetReg ( IsHazardFn ) ; return SetRegWaitStates - WaitStatesNeeded ;


<s> unsigned HWReg = getHWReg ( TII , * SetRegInstr ) ; const int SetRegWaitStates = ST . getGeneration ( ) <= <ISA_LIT>Subtarget :: SEA_ISLANDS ? <NUM_LIT> : <NUM_LIT> ;

auto IsHazardFn = [ TII , HWReg ] ( MachineInstr * MI ) {


<s> int GCNHazardRecognizer :: checkSMRDHazards ( MachineInstr * SMRD ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; int WaitStatesNeeded = <NUM_LIT> ; WaitStatesNeeded = checkSoftClauseHazards ( SMRD ) ; if ( ST . getGeneration ( ) != <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return WaitStatesNeeded ; int SmrdSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isVALU ( * MI ) ; } ; auto IsBufferHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isSALU ( * MI ) ; } ; bool IsBufferSMRD = TII . isBufferSMRD ( * SMRD ) ;

for ( const MachineOperand & Use : SMRD -> uses ( ) ) {


<s> int WaitStatesNeeded = <NUM_LIT> ; WaitStatesNeeded = checkSoftClauseHazards ( SMRD ) ; if ( ST . getGeneration ( ) != <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return WaitStatesNeeded ; int SmrdSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isVALU ( * MI ) ; } ; auto IsBufferHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isSALU ( * MI ) ; } ; bool IsBufferSMRD = TII . isBufferSMRD ( * SMRD ) ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) { if ( ! Use . isReg ( ) ) continue ;

int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn ) ;


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const int VALUWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; if ( ! TRI -> isVGPR ( MRI , Def . getReg ( ) ) ) return WaitStatesNeeded ; unsigned Reg = Def . getReg ( ) ; auto IsHazardFn = [ this , Reg , TRI ] ( MachineInstr * MI ) { int DataIdx = createsVALUHazard ( * MI ) ;

return DataIdx >= <NUM_LIT> && TRI -> regsOverlap ( MI -> getOperand ( DataIdx ) . getReg ( ) , Reg ) ;


<s> if ( ! Use . isReg ( ) || TRI . isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; int WaitStatesNeededForUse = VmemSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; } return WaitStatesNeeded ;


<s> GCNHazardRecognizer :: GCNHazardRecognizer ( const MachineFunction & MF ) : CurrCycleInstr ( nullptr ) , MF ( MF ) , ST ( MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ) , TII ( * ST . getInstrInfo ( ) ) , TRI ( TII . getRegisterInfo ( ) ) , ClauseUses ( TRI . getNumRegUnits ( ) ) , ClauseDefs ( TRI . getNumRegUnits ( ) ) { MaxLookAhead = <NUM_LIT> ;


<s> if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;

if ( checkAnyInstHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;

if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> if ( IsHazard ( MI ) ) return WaitStates ; unsigned Opcode = MI -> getOpcode ( ) ;

if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) continue ;


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; auto IsHazardFn = [ IsHazardDef , TRI , Reg ] ( MachineInstr * MI ) {

return IsHazardDef ( MI ) && MI -> modifiesRegister ( Reg , TRI ) ;


<s> auto IsHazardFn = [ IsHazard ] ( MachineInstr * MI ) { return isSSetReg ( MI -> getOpcode ( ) ) && IsHazard ( MI ) ; } ; return getWaitStatesSince ( IsHazardFn ) ;


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : return false ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : return false ;


<s> if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( MI -> isInlineAsm ( ) ) return std :: max ( WaitStates , checkInlineAsmHazards ( MI ) ) ; if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ;

if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ;


<s> if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVMEMHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( MI -> isInlineAsm ( ) ) return std :: max ( WaitStates , checkInlineAsmHazards ( MI ) ) ; if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( * MI ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ; return WaitStates ;


<s> for ( const MachineOperand & Use : MI -> uses ( ) ) { if ( ! Use . isReg ( ) || TRI -> isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; auto IsHazardFn = [ ] ( MachineInstr * MI ) {

return MI -> getOpcode ( ) == <STR_LIT> ;


<s> int GCNHazardRecognizer :: checkAnyInstHazards ( MachineInstr * MI ) { if ( MI -> isDebugValue ( ) ) return <NUM_LIT> ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; if ( ! ST . hasSMovFedHazard ( ) ) return <NUM_LIT> ; int MovFedWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; for ( const MachineOperand & Use : MI -> uses ( ) ) { if ( ! Use . isReg ( ) || TRI -> isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; auto IsHazardFn = [ ] ( MachineInstr * MI ) { return MI -> getOpcode ( ) == <STR_LIT> ;


<s> if ( ! CurrCycleInstr ) return ; if ( CurrCycleInstr -> isImplicitDef ( ) || CurrCycleInstr -> isDebugInstr ( ) || CurrCycleInstr -> isKill ( ) ) return ; if ( CurrCycleInstr -> isBundle ( ) ) { processBundle ( ) ;


<s> auto IsHazardFn = [ ] ( MachineInstr * I ) { if ( ! <ISA_LIT>InstrInfo :: isVMEM ( * I ) && ! <ISA_LIT>InstrInfo :: isFLAT ( * I ) ) return false ; return <ISA_LIT>InstrInfo :: isFPAtomic ( * I ) ; } ; auto IsExpiredFn = [ ] ( MachineInstr * MI , int WaitStates ) { if ( WaitStates >= <NUM_LIT> || <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) return true ; switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : break ; }


<s> if ( ! ST . hasMAIInsts ( ) ) return <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; auto IsAccVgprReadFn = [ ] ( MachineInstr * MI ) { return MI -> getOpcode ( ) == <STR_LIT> ; } ; for ( const MachineOperand & Op : MI -> explicit_uses ( ) ) { if ( ! Op . isReg ( ) || ! TRI . isVGPR ( MF . getRegInfo ( ) , Op . getReg ( ) ) ) continue ; Register Reg = Op . getReg ( ) ; const int AccVgprReadLdStWaitStates = <NUM_LIT> ;


<s> int NSAtoVMEMWaitStates = <NUM_LIT> ; if ( ! ST . hasNSAtoVMEMBug ( ) ) return <NUM_LIT> ; if ( ! <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) && ! <ISA_LIT>InstrInfo :: isMTBUF ( * MI ) ) return <NUM_LIT> ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const auto * Offset = TII -> getNamedOperand ( * MI , <STR_LIT> :: offset ) ; if ( ! Offset || ( Offset -> getImm ( ) & <NUM_LIT> ) == <NUM_LIT> ) return <NUM_LIT> ; auto IsHazardFn = [ TII ] ( MachineInstr * I ) { if ( ! <ISA_LIT>InstrInfo :: isMIMG ( * I ) ) return false ;

const <STR_LIT> * Info = <STR_LIT> ( I -> getOpcode ( ) ) ;


<s> if ( ! Offset || ( Offset -> getImm ( ) & <NUM_LIT> ) == <NUM_LIT> ) return <NUM_LIT> ; auto IsHazardFn = [ TII ] ( MachineInstr * I ) { if ( ! <ISA_LIT>InstrInfo :: isMIMG ( * I ) ) return false ; const <STR_LIT> * Info = <STR_LIT> ( I -> getOpcode ( ) ) ; return Info -> MIMGEncoding == <STR_LIT> && TII -> getInstSizeInBytes ( * I ) >= <NUM_LIT> ;


<s> auto IsHazardFn = [ TII ] ( MachineInstr * MI ) { return getHWReg ( TII , * MI ) == <STR_LIT> :: ID_TRAPSTS ; } ; int WaitStatesNeeded = getWaitStatesSinceSetReg ( IsHazardFn , RFEWaitStates ) ;

return RFEWaitStates - WaitStatesNeeded ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const int RFEWaitStates = <NUM_LIT> ; auto IsHazardFn = [ TII ] ( MachineInstr * MI ) { return getHWReg ( TII , * MI ) == <STR_LIT> :: ID_TRAPSTS ; } ; int WaitStatesNeeded = getWaitStatesSinceSetReg ( IsHazardFn , RFEWaitStates ) ; return RFEWaitStates - WaitStatesNeeded ;


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const MachineOperand * LaneSelectOp = TII -> getNamedOperand ( * RWLane , <STR_LIT> :: src1 ) ; if ( ! LaneSelectOp -> isReg ( ) || ! TRI -> isSGPRReg ( MRI , LaneSelectOp -> getReg ( ) ) ) return <NUM_LIT> ; Register LaneSelectReg = LaneSelectOp -> getReg ( ) ; auto IsHazardFn = [ TII ] ( MachineInstr * MI ) {


<s> int GCNHazardRecognizer :: checkSetRegHazards ( MachineInstr * SetRegInstr ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; unsigned HWReg = getHWReg ( TII , * SetRegInstr ) ; const int SetRegWaitStates = ST . getSetRegWaitStates ( ) ; auto IsHazardFn = [ TII , HWReg ] ( MachineInstr * MI ) { return HWReg == getHWReg ( TII , * MI ) ; } ;

int WaitStatesNeeded = getWaitStatesSinceSetReg ( IsHazardFn , SetRegWaitStates ) ;


<s> auto IsBufferHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isSALU ( * MI ) ; } ; bool IsBufferSMRD = TII . isBufferSMRD ( * SMRD ) ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) { if ( ! Use . isReg ( ) ) continue ; int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn , SmrdSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ;


<s> int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn , SmrdSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; if ( IsBufferSMRD ) { int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsBufferHazardDefFn , SmrdSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; }


<s> Register Reg = Def . getReg ( ) ; auto IsHazardFn = [ this , Reg , TRI ] ( MachineInstr * MI ) { int DataIdx = createsVALUHazard ( * MI ) ; return DataIdx >= <NUM_LIT> && TRI -> regsOverlap ( MI -> getOperand ( DataIdx ) . getReg ( ) , Reg ) ; } ; int WaitStatesNeededForDef = VALUWaitStates - getWaitStatesSince ( IsHazardFn , VALUWaitStates ) ;

WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForDef ) ;


<s> if ( ! ST . hasVMEMReadSGPRVALUDefHazard ( ) ) return <NUM_LIT> ; int WaitStatesNeeded = checkSoftClauseHazards ( VMEM ) ; const int VmemSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isVALU ( * MI ) ; } ; for ( const MachineOperand & Use : VMEM -> uses ( ) ) {

if ( ! Use . isReg ( ) || TRI . isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ;


<s> for ( const MachineOperand & Use : VMEM -> uses ( ) ) { if ( ! Use . isReg ( ) || TRI . isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; int WaitStatesNeededForUse = VmemSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn , VmemSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; }


<s> } ; auto IsHazardFn = [ InstType , & IsHazardInst ] ( MachineInstr * I ) { if ( ! I -> isBranch ( ) ) return false ; auto IsHazardFn = [ InstType , IsHazardInst ] ( MachineInstr * I ) { auto InstType2 = IsHazardInst ( I ) ; return InstType2 && InstType != InstType2 ; } ; auto IsExpiredFn = [ InstType , & IsHazardInst ] ( MachineInstr * I , int ) { if ( ! I ) return false ; auto InstType2 = IsHazardInst ( I ) ; if ( InstType == InstType2 ) return true ; return I -> getOpcode ( ) == <STR_LIT> && I -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && ! I -> getOperand ( <NUM_LIT> ) . getImm ( ) ;


<s> break ; } } } if ( ! SDST ) return false ; const Register SDSTReg = SDST -> getReg ( ) ; auto IsHazardFn = [ SDSTReg , TRI ] ( MachineInstr * I ) { return <ISA_LIT>InstrInfo :: isSMRD ( * I ) && I -> readsRegister ( SDSTReg , TRI ) ; } ; auto IsExpiredFn = [ TII , IV ] ( MachineInstr * MI , int ) { if ( MI ) { if ( TII -> isSALU ( * MI ) ) { switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return false ; case <STR_LIT> : return ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) && ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) ; case <STR_LIT> : { const int64_t Imm = MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; <STR_LIT> Decoded = <STR_LIT> ( IV , Imm ) ; return ( Decoded . LgkmCnt == <NUM_LIT> ) ; } default : if ( TII -> isSOPP ( * MI ) ) return false ; return true ; } }


<s> if ( ! MI -> modifiesRegister ( <STR_LIT> , TRI ) ) return false ; auto IsHazardFn = [ TRI ] ( MachineInstr * I ) { if ( <ISA_LIT>InstrInfo :: isVALU ( * I ) ) return false ; return I -> readsRegister ( <STR_LIT> , TRI ) ; } ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; auto IsExpiredFn = [ TII , TRI ] ( MachineInstr * MI , int ) { if ( ! MI ) return false ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) { if ( TII -> getNamedOperand ( * MI , <STR_LIT> :: sdst ) ) return true ; for ( auto MO : MI -> implicit_operands ( ) ) if ( MO . isDef ( ) && TRI -> isSGPRClass ( TRI -> getPhysRegClass ( MO . getReg ( ) ) ) ) return true ; } if ( MI -> getOpcode ( ) == <STR_LIT> && ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> ) == <NUM_LIT> ) return true ;


<s> return <ISA_LIT>InstrInfo :: isVALU ( * MI ) && Opc != <STR_LIT> && Opc != <STR_LIT> && Opc != <STR_LIT> ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ; auto * Src0 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) ; Register Reg = Src0 -> getReg ( ) ; bool IsUndef = Src0 -> isUndef ( ) ; BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Define | ( IsUndef ? RegState :: Dead : <NUM_LIT> ) ) . addReg ( Reg , IsUndef ? RegState :: Undef : RegState :: Kill ) ;


<s> auto IsExpiredFn = [ ] ( MachineInstr * MI , int ) { if ( ! MI ) return false ; unsigned Opc = MI -> getOpcode ( ) ; return <ISA_LIT>InstrInfo :: isVALU ( * MI ) && Opc != <STR_LIT> && Opc != <STR_LIT> && Opc != <STR_LIT> ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ; auto * Src0 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) ; Register Reg = Src0 -> getReg ( ) ;


<s> auto IsExpiredFn = [ ] ( MachineInstr * MI , int ) { return MI && ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) || ( MI -> getOpcode ( ) == <STR_LIT> && ! MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ) ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ;


<s> IsHazardRecognizerMode ( false ) , CurrCycleInstr ( nullptr ) , MF ( MF ) , ST ( MF . getSubtarget < GCNSubtarget > ( ) ) , TII ( * ST . getInstrInfo ( ) ) , TRI ( TII . getRegisterInfo ( ) ) , ClauseUses ( TRI . getNumRegUnits ( ) ) , ClauseDefs ( TRI . getNumRegUnits ( ) ) {

MaxLookAhead = MF . getRegInfo ( ) . isPhysRegUsed ( <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ;


<s> if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( MI -> mayLoad ( ) || MI -> mayStore ( ) ) && checkMAILdStHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkAnyInstHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( MI -> mayLoad ( ) || MI -> mayStore ( ) ) && checkMAILdStHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> static int getWaitStatesSince ( GCNHazardRecognizer :: IsHazardFn IsHazard , MachineBasicBlock * MBB , MachineBasicBlock :: reverse_instr_iterator I , int WaitStates , IsExpiredFn IsExpired , DenseSet < const MachineBasicBlock * > & Visited ) { for ( auto E = MBB -> instr_rend ( ) ; I != E ; ++ I ) { if ( I -> isBundle ( ) ) continue ; if ( IsHazard ( & * I ) ) return WaitStates ; if ( I -> isInlineAsm ( ) || I -> isImplicitDef ( ) || I -> isDebugInstr ( ) ) continue ; WaitStates += <ISA_LIT>InstrInfo :: getNumWaitStates ( * I ) ; if ( IsExpired ( & * I , WaitStates ) ) return std :: numeric_limits < int > :: max ( ) ; } int MinWaitStates = WaitStates ; bool Found = false ; for ( MachineBasicBlock * Pred : MBB -> predecessors ( ) ) { if ( ! Visited . insert ( Pred ) . second ) continue ;

int W = getWaitStatesSince ( IsHazard , Pred , Pred -> instr_rbegin ( ) , WaitStates , IsExpired , Visited ) ;


<s> BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc ( ) , TII . get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ;


<s> unsigned Opcode = MI . getOpcode ( ) ; return Opcode == <STR_LIT> || Opcode == <STR_LIT> ;


<s> unsigned Opcode = MI . getOpcode ( ) ;

return Opcode == <STR_LIT> || Opcode == <STR_LIT> ;


<s> void GCNHazardRecognizer :: processBundle ( ) { MachineBasicBlock :: instr_iterator MI = std :: next ( CurrCycleInstr -> getIterator ( ) ) ; MachineBasicBlock :: instr_iterator E = CurrCycleInstr -> getParent ( ) -> instr_end ( ) ; for ( ; MI != E && MI -> isInsideBundle ( ) ; ++ MI ) { CurrCycleInstr = & * MI ;

unsigned WaitStates = PreEmitNoopsCommon ( CurrCycleInstr ) ;


<s> void GCNHazardRecognizer :: processBundle ( ) { MachineBasicBlock :: instr_iterator MI = std :: next ( CurrCycleInstr -> getIterator ( ) ) ; MachineBasicBlock :: instr_iterator E = CurrCycleInstr -> getParent ( ) -> instr_end ( ) ; for ( ; MI != E && MI -> isInsideBundle ( ) ; ++ MI ) { CurrCycleInstr = & * MI ; unsigned WaitStates = PreEmitNoopsCommon ( CurrCycleInstr ) ; if ( IsHazardRecognizerMode ) fixHazards ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> ; i < WaitStates ; ++ i ) insertNoopInBundle ( CurrCycleInstr , TII ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( WaitStates , MaxLookAhead - <NUM_LIT> ) ; i < e ; ++ i ) EmittedInstrs . push_front ( nullptr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ;

EmittedInstrs . resize ( MaxLookAhead ) ;


<s> else if ( CurrCycleInstr -> isDebugInstr ( ) ) return ; unsigned NumWaitStates = TII . getNumWaitStates ( * CurrCycleInstr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; }

EmittedInstrs . resize ( getMaxLookAhead ( ) ) ;


<s> int GCNHazardRecognizer :: checkSMRDHazards ( MachineInstr * SMRD ) { const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; int WaitStatesNeeded = <NUM_LIT> ; WaitStatesNeeded = checkSoftClauseHazards ( SMRD ) ; if ( ST . getGeneration ( ) != <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return WaitStatesNeeded ; int SmrdSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isVALU ( * MI ) ; } ; auto IsBufferHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isSALU ( * MI ) ; } ; bool IsBufferSMRD = TII . isBufferSMRD ( * SMRD ) ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) {


<s> auto IsBufferHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isSALU ( * MI ) ; } ; bool IsBufferSMRD = TII . isBufferSMRD ( * SMRD ) ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) { if ( ! Use . isReg ( ) ) continue ; int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; if ( IsBufferSMRD ) { int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsBufferHazardDefFn ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; }


<s> for ( const MachineOperand & Use : VMEM -> uses ( ) ) { if ( ! Use . isReg ( ) || TRI . isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; int WaitStatesNeededForUse = VmemSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ;


<s> GCNHazardRecognizer :: GCNHazardRecognizer ( const MachineFunction & MF ) :

CurrCycleInstr ( nullptr ) , MF ( MF ) , ST ( MF . getSubtarget < GCNSubtarget > ( ) ) , TII ( * ST . getInstrInfo ( ) ) , TRI ( TII . getRegisterInfo ( ) ) , ClauseUses ( TRI . getNumRegUnits ( ) ) , ClauseDefs ( TRI . getNumRegUnits ( ) ) {


<s> if ( Opcode == <STR_LIT> ) continue ; } ++ WaitStates ; } return std :: numeric_limits < int > :: max ( ) ;


<s> return ; } if ( CurrCycleInstr -> isImplicitDef ( ) || CurrCycleInstr -> isDebugInstr ( ) || CurrCycleInstr -> isKill ( ) ) { CurrCycleInstr = nullptr ; return ; } if ( CurrCycleInstr -> isBundle ( ) ) { processBundle ( ) ; return ; } unsigned NumWaitStates = TII . getNumWaitStates ( * CurrCycleInstr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; } EmittedInstrs . resize ( getMaxLookAhead ( ) ) ;


<s> for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; } EmittedInstrs . resize ( getMaxLookAhead ( ) ) ;


<s> int FPAtomicToDenormModeWaitStates = <NUM_LIT> ; if ( MI -> getOpcode ( ) != <STR_LIT> ) return <NUM_LIT> ; auto IsHazardFn = [ ] ( MachineInstr * I ) { if ( ! <ISA_LIT>InstrInfo :: isVMEM ( * I ) && ! <ISA_LIT>InstrInfo :: isFLAT ( * I ) ) return false ; return <ISA_LIT>InstrInfo :: isFPAtomic ( * I ) ; } ; auto IsExpiredFn = [ ] ( MachineInstr * MI , int WaitStates ) { if ( WaitStates >= <NUM_LIT> || <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) return true ; switch ( MI -> getOpcode ( ) ) { case <STR_LIT> :


<s> if ( ! Op ) continue ; return true ; } return false ; } ; auto IsExpiredFn = [ ] ( MachineInstr * MI , int ) { return MI && ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) || ( MI -> getOpcode ( ) == <STR_LIT> && ! MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) || ( MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) ) ; } ;

if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ;


<s> } ; auto IsExpiredFn = [ ] ( MachineInstr * MI , int ) { return MI && ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) || ( MI -> getOpcode ( ) == <STR_LIT> && ! MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) || ( MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) ) ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ;

const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ;


<s> if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) || <ISA_LIT>InstrInfo :: isDS ( * MI ) ) && checkMAILdStHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return HazardType ;


<s> auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard ; if ( MI -> isBundle ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) && checkSMRDHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ST . hasNSAtoVMEMBug ( ) && checkNSAtoVMEMHazard ( MI ) > <NUM_LIT> ) return HazardType ; if ( checkFPAtomicToDenormModeHazard ( MI ) > <NUM_LIT> ) return HazardType ; if ( ST . hasNoDataDepHazard ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return HazardType ;

if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return HazardType ;


<s> if ( IsExpired ( & * I , WaitStates ) ) return std :: numeric_limits < int > :: max ( ) ; } int MinWaitStates = WaitStates ; bool Found = false ; for ( MachineBasicBlock * Pred : MBB -> predecessors ( ) ) { if ( ! Visited . insert ( Pred ) . second ) continue ; int W = getWaitStatesSince ( IsHazard , Pred , Pred -> instr_rbegin ( ) , WaitStates , IsExpired , Visited ) ; if ( W == std :: numeric_limits < int > :: max ( ) ) continue ; MinWaitStates = Found ? std :: min ( MinWaitStates , W ) : W ;

if ( IsExpired ( nullptr , MinWaitStates ) ) return MinWaitStates ;


<s> if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( MI -> isInlineAsm ( ) ) return std :: max ( WaitStates , checkInlineAsmHazards ( MI ) ) ; if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ;

if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ;


<s> return MAI != nullptr ; } ; MachineInstr * MI = SU -> getInstr ( ) ; if ( IsMFMAFn ( MI ) ) { int W = getWaitStatesSince ( IsMFMAFn , <NUM_LIT> ) ; if ( MAI ) return W < ( int ) TSchedModel . computeInstrLatency ( MAI ) ;


<s> MachineInstr * MI = SU -> getInstr ( ) ; if ( IsMFMAFn ( MI ) ) { int W = getWaitStatesSince ( IsMFMAFn , <NUM_LIT> ) ; if ( MAI ) return W < ( int ) TSchedModel . computeInstrLatency ( MAI ) ;


<s> void GCNHazardRecognizer :: AdvanceCycle ( ) { if ( ! CurrCycleInstr ) { EmittedInstrs . push_front ( nullptr ) ; return ; } if ( CurrCycleInstr -> isMetaInstruction ( ) ) { CurrCycleInstr = nullptr ; return ; } if ( CurrCycleInstr -> isBundle ( ) ) { processBundle ( ) ; return ; } unsigned NumWaitStates = TII . getNumWaitStates ( * CurrCycleInstr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ;


<s> processBundle ( ) ; return ; } unsigned NumWaitStates = TII . getNumWaitStates ( * CurrCycleInstr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; } EmittedInstrs . resize ( getMaxLookAhead ( ) ) ; CurrCycleInstr = nullptr ;


<s> } ; auto IsExpiredFn = [ InstType , & IsHazardInst ] ( const MachineInstr & I , int ) { auto InstType2 = IsHazardInst ( I ) ; if ( InstType == InstType2 ) return true ; return I . getOpcode ( ) == <STR_LIT> && I . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && ! I . getOperand ( <NUM_LIT> ) . getImm ( ) ; } ; return :: getWaitStatesSince ( IsHazardFn , & I , IsExpiredFn ) != std :: numeric_limits < int > :: max ( ) ; } ;


<s> IsHazardRecognizerMode ( false ) , CurrCycleInstr ( nullptr ) , MF ( MF ) , ST ( MF . getSubtarget < GCNSubtarget > ( ) ) , TII ( * ST . getInstrInfo ( ) ) , TRI ( TII . getRegisterInfo ( ) ) , ClauseUses ( TRI . getNumRegUnits ( ) ) , ClauseDefs ( TRI . getNumRegUnits ( ) ) { MaxLookAhead = MF . getRegInfo ( ) . isPhysRegUsed ( <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; TSchedModel . init ( & ST ) ;


<s> GCNHazardRecognizer :: GCNHazardRecognizer ( const MachineFunction & MF ) : IsHazardRecognizerMode ( false ) , CurrCycleInstr ( nullptr ) , MF ( MF ) , ST ( MF . getSubtarget < GCNSubtarget > ( ) ) , TII ( * ST . getInstrInfo ( ) ) , TRI ( TII . getRegisterInfo ( ) ) , ClauseUses ( TRI . getNumRegUnits ( ) ) , ClauseDefs ( TRI . getNumRegUnits ( ) ) { MaxLookAhead = MF . getRegInfo ( ) . isPhysRegUsed ( <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; TSchedModel . init ( & ST ) ;


<s> for ( MachineBasicBlock * Pred : MBB -> predecessors ( ) ) { if ( ! Visited . insert ( Pred ) . second ) continue ; int W = getWaitStatesSince ( IsHazard , Pred , Pred -> instr_rbegin ( ) , WaitStates , IsExpired , Visited ) ; MinWaitStates = std :: min ( MinWaitStates , W ) ; } return MinWaitStates ;


<s> case <STR_LIT> : case <STR_LIT> : SDSTName = <STR_LIT> :: vdst ; break ; default : SDSTName = <STR_LIT> :: sdst ; break ; } const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <STR_LIT> IV = <STR_LIT> ( ST . getCPU ( ) ) ; const MachineOperand * SDST = TII -> getNamedOperand ( * MI , SDSTName ) ; if ( ! SDST ) { for ( const auto & MO : MI -> implicit_operands ( ) ) { if ( MO . isDef ( ) && TRI -> isSGPRClass ( TRI -> getPhysRegClass ( MO . getReg ( ) ) ) ) { SDST = & MO ; break ; } } } if ( ! SDST ) return false ; const unsigned SDSTReg = SDST -> getReg ( ) ; auto IsHazardFn = [ SDSTReg , TRI ] ( MachineInstr * I ) { return <ISA_LIT>InstrInfo :: isSMRD ( * I ) && I -> readsRegister ( SDSTReg , TRI ) ; } ; auto IsExpiredFn = [ TII , IV ] ( MachineInstr * MI , int ) { if ( MI ) { if ( TII -> isSALU ( * MI ) ) { switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return false ; case <STR_LIT> :

return ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) && ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) ;


<s> unsigned Opc = MI -> getOpcode ( ) ; return <ISA_LIT>InstrInfo :: isVALU ( * MI ) && Opc != <STR_LIT> && Opc != <STR_LIT> && Opc != <STR_LIT> ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ; auto * Src0 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) ; unsigned Reg = Src0 -> getReg ( ) ; bool IsUndef = Src0 -> isUndef ( ) ; BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Define | ( IsUndef ? RegState :: Dead : <NUM_LIT> ) ) . addReg ( Reg , IsUndef ? RegState :: Undef : RegState :: Kill ) ;


<s> if ( ! <ISA_LIT>InstrInfo :: isSALU ( * MI ) && ! <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) return false ; if ( MI -> getNumDefs ( ) == <NUM_LIT> ) return false ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; auto IsHazardFn = [ TRI , MI ] ( MachineInstr * I ) { if ( ! <ISA_LIT>InstrInfo :: isVMEM ( * I ) && ! <ISA_LIT>InstrInfo :: isDS ( * I ) && ! <ISA_LIT>InstrInfo :: isFLAT ( * I ) ) return false ; for ( const MachineOperand & Def : MI -> defs ( ) ) { MachineOperand * Op = I -> findRegisterUseOperand ( Def . getReg ( ) , false , TRI ) ; if ( ! Op || ( Op -> isImplicit ( ) && Op -> getReg ( ) == <STR_LIT> ) ) continue ;


<s> if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;

if ( checkAnyInstHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> if ( checkFPAtomicToDenormModeHazard ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasNoDataDepHazard ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> WaitStates = std :: max ( WaitStates , checkFPAtomicToDenormModeHazard ( MI ) ) ; if ( ST . hasNoDataDepHazard ( ) ) return WaitStates ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( MI -> isInlineAsm ( ) ) return std :: max ( WaitStates , checkInlineAsmHazards ( MI ) ) ;

if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ;


<s> int GCNHazardRecognizer :: checkReadM0Hazards ( MachineInstr * MI ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const int SMovRelWaitStates = <NUM_LIT> ; auto IsHazardFn = [ TII ] ( const MachineInstr & MI ) { return TII -> isSALU ( MI ) ; } ; return SMovRelWaitStates - getWaitStatesSinceDef ( <STR_LIT> , IsHazardFn , SMovRelWaitStates ) ;


<s> int GCNHazardRecognizer :: checkReadM0Hazards ( MachineInstr * MI ) {


<s> int DataIdx = createsVALUHazard ( MI ) ; return DataIdx >= <NUM_LIT> && TRI -> regsOverlap ( MI . getOperand ( DataIdx ) . getReg ( ) , Reg ) ; } ; int WaitStatesNeededForDef = VALUWaitStates - getWaitStatesSince ( IsHazardFn , VALUWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForDef ) ; return WaitStatesNeeded ;


<s> if ( ST . hasNoDataDepHazard ( ) ) return NoHazard ; if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) || <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) || <ISA_LIT>InstrInfo :: isDS ( * MI ) || <ISA_LIT>InstrInfo :: isEXP ( * MI ) ) && checkMAIVALUHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return HazardType ;

if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return HazardType ;


<s> static bool isDGEMM ( unsigned Opcode ) { return Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ;


<s> static bool isXDL ( const GCNSubtarget & ST , const MachineInstr & MI ) { unsigned Opcode = MI . getOpcode ( ) ;

if ( ! <ISA_LIT>InstrInfo :: isMAI ( MI ) || isDGEMM ( Opcode ) || Opcode == <STR_LIT> || Opcode == <STR_LIT> ) return false ;


<s> if ( MI -> isBundle ( ) ) return <NUM_LIT> ; int WaitStates = <NUM_LIT> ; if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) return std :: max ( WaitStates , checkSMRDHazards ( MI ) ) ; if ( ST . hasNSAtoVMEMBug ( ) ) WaitStates = std :: max ( WaitStates , checkNSAtoVMEMHazard ( MI ) ) ; WaitStates = std :: max ( WaitStates , checkFPAtomicToDenormModeHazard ( MI ) ) ; if ( ST . hasNoDataDepHazard ( ) ) return WaitStates ; if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVMEMHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) || <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) || <ISA_LIT>InstrInfo :: isDS ( * MI ) || <ISA_LIT>InstrInfo :: isEXP ( * MI ) ) && checkMAIVALUHazards ( MI ) > <NUM_LIT> ) WaitStates = std :: max ( WaitStates , checkMAIVALUHazards ( MI ) ) ; if ( MI -> isInlineAsm ( ) ) return std :: max ( WaitStates , checkInlineAsmHazards ( MI ) ) ; if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ;

if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ;


<s> if ( Op . isReg ( ) ) Set . insert ( Op . getReg ( ) ) ; }


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; unsigned NumWaitStates = TII -> getNumWaitStates ( * CurrCycleInstr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; }


<s> int GCNHazardRecognizer :: checkDPPHazards ( MachineInstr * DPP ) { const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; int DppVgprWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; for ( const MachineOperand & Use : DPP -> uses ( ) ) { if ( ! Use . isReg ( ) || ! TRI -> isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ;

int WaitStatesNeededForUse = DppVgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) ) ;


<s> if ( ! Use . isReg ( ) || ! TRI -> isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; int WaitStatesNeededForUse = DppVgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; }


<s> if ( ! LaneSelectOp -> isReg ( ) || ! TRI -> isSGPRReg ( MRI , LaneSelectOp -> getReg ( ) ) ) return <NUM_LIT> ; unsigned LaneSelectReg = LaneSelectOp -> getReg ( ) ;

auto IsHazardFn = [ TII ] ( MachineInstr * MI ) {


<s> if ( ST . getGeneration ( ) < <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return <NUM_LIT> ; std :: set < unsigned > ClauseDefs ; std :: set < unsigned > ClauseUses ; for ( MachineInstr * MI : EmittedInstrs ) { if ( ! MI || ! <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) break ; addRegsToSet ( MI -> defs ( ) , ClauseDefs ) ; addRegsToSet ( MI -> uses ( ) , ClauseUses ) ; } if ( ClauseDefs . empty ( ) ) return <NUM_LIT> ; if ( SMEM -> mayStore ( ) ) return <NUM_LIT> ; addRegsToSet ( SMEM -> defs ( ) , ClauseDefs ) ; addRegsToSet ( SMEM -> uses ( ) , ClauseUses ) ;

std :: vector < unsigned > Result ( std :: max ( ClauseDefs . size ( ) , ClauseUses . size ( ) ) ) ;


<s> for ( MachineInstr * MI : EmittedInstrs ) { if ( ! MI || ! <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) break ; addRegsToSet ( MI -> defs ( ) , ClauseDefs ) ; addRegsToSet ( MI -> uses ( ) , ClauseUses ) ; } if ( ClauseDefs . empty ( ) ) return <NUM_LIT> ; if ( SMEM -> mayStore ( ) ) return <NUM_LIT> ; addRegsToSet ( SMEM -> defs ( ) , ClauseDefs ) ; addRegsToSet ( SMEM -> uses ( ) , ClauseUses ) ; std :: vector < unsigned > Result ( std :: max ( ClauseDefs . size ( ) , ClauseUses . size ( ) ) ) ; std :: vector < unsigned > :: iterator End ; End = std :: set_intersection ( ClauseDefs . begin ( ) , ClauseDefs . end ( ) , ClauseUses . begin ( ) , ClauseUses . end ( ) , Result . begin ( ) ) ; if ( End != Result . begin ( ) ) return <NUM_LIT> ;


<s> if ( ST . getGeneration ( ) != <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return WaitStatesNeeded ; int SmrdSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ;

for ( const MachineOperand & Use : SMRD -> uses ( ) ) {


<s> int WaitStatesNeeded = <NUM_LIT> ; WaitStatesNeeded = checkSMEMSoftClauseHazards ( SMRD ) ; if ( ST . getGeneration ( ) != <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return WaitStatesNeeded ; int SmrdSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) {


<s> auto IsHazardFn = [ this , Reg , TRI ] ( MachineInstr * MI ) { int DataIdx = createsVALUHazard ( * MI ) ; return DataIdx >= <NUM_LIT> && TRI -> regsOverlap ( MI -> getOperand ( DataIdx ) . getReg ( ) , Reg ) ; } ;

int WaitStatesNeededForDef = VALUWaitStates - getWaitStatesSince ( IsHazardFn ) ;


<s> int GCNHazardRecognizer :: checkVALUHazards ( MachineInstr * VALU ) { if ( ! ST . has12DWordStoreHazard ( ) ) return <NUM_LIT> ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const MachineRegisterInfo & MRI = VALU -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; const int VALUWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; for ( const MachineOperand & Def : VALU -> defs ( ) ) {

if ( ! TRI -> isVGPR ( MRI , Def . getReg ( ) ) ) continue ;


<s> int WaitStatesNeeded = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ; for ( const MachineOperand & Use : VMEM -> uses ( ) ) { if ( ! Use . isReg ( ) || TRI . isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; int WaitStatesNeededForUse = VmemSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; } return WaitStatesNeeded ;


<s> CurrCycleInstr ( nullptr ) , MF ( MF ) , ST ( MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ) {


<s> GCNHazardRecognizer :: GCNHazardRecognizer ( const MachineFunction & MF ) : CurrCycleInstr ( nullptr ) , MF ( MF ) , ST ( MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ) { MaxLookAhead = <NUM_LIT> ;


<s> if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( isVInterp ( MI -> getOpcode ( ) ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkAnyInstHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> int WaitStates = - <NUM_LIT> ; for ( MachineInstr * MI : EmittedInstrs ) { ++ WaitStates ; if ( ! MI || ! IsHazard ( MI ) ) continue ; return WaitStates ;


<s> ++ WaitStates ; if ( ! MI || ! IsHazard ( MI ) ) continue ; return WaitStates ; } return std :: numeric_limits < int > :: max ( ) ;


<s> static bool isVInterp ( unsigned Opcode ) {

return Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ;


<s> static bool isVInterp ( unsigned Opcode ) { return Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ;


<s> } if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ; if ( isSMovRel ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ;


<s> if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) return std :: max ( WaitStates , checkSMRDHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) { WaitStates = std :: max ( WaitStates , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVMEMHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( isVInterp ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ; return WaitStates ;


<s> case <STR_LIT> : case <STR_LIT> : SDSTName = <STR_LIT> :: vdst ; break ; default : SDSTName = <STR_LIT> :: sdst ; break ; } const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <STR_LIT> IV = <STR_LIT> ( ST . getCPU ( ) ) ; const MachineOperand * SDST = TII -> getNamedOperand ( * MI , SDSTName ) ; if ( ! SDST ) { for ( const auto & MO : MI -> implicit_operands ( ) ) { if ( MO . isDef ( ) && TRI -> isSGPRClass ( TRI -> getPhysRegClass ( MO . getReg ( ) ) ) ) { SDST = & MO ; break ; } } } if ( ! SDST ) return false ; const Register SDSTReg = SDST -> getReg ( ) ; auto IsHazardFn = [ SDSTReg , TRI ] ( MachineInstr * I ) { return <ISA_LIT>InstrInfo :: isSMRD ( * I ) && I -> readsRegister ( SDSTReg , TRI ) ;


<s> if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasNSAtoVMEMBug ( ) && checkNSAtoVMEMHazard ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkFPAtomicToDenormModeHazard ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasNoDataDepHazard ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> mayLoadOrStore ( ) && checkMAILdStHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> mayLoadOrStore ( ) && checkMAILdStHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) ) return std :: max ( WaitStates , checkMAIHazards ( MI ) ) ;

if ( MI -> mayLoadOrStore ( ) ) return std :: max ( WaitStates , checkMAILdStHazards ( MI ) ) ;


<s> if ( ! SU -> isInstr ( ) ) return false ; MachineInstr * MAI = nullptr ; auto IsMFMAFn = [ & MAI ] ( MachineInstr * MI ) { MAI = nullptr ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && MI -> getOpcode ( ) != <STR_LIT> && MI -> getOpcode ( ) != <STR_LIT> ) MAI = MI ; return MAI != nullptr ; } ;

MachineInstr * MI = SU -> getInstr ( ) ;


<s> if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;

if ( MI -> mayLoadOrStore ( ) && checkMAILdStHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> ScheduleHazardRecognizer :: HazardType GCNHazardRecognizer :: getHazardType ( SUnit * SU , int Stalls ) { MachineInstr * MI = SU -> getInstr ( ) ; if ( MI -> isBundle ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) && checkSMRDHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasNSAtoVMEMBug ( ) && checkNSAtoVMEMHazard ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkFPAtomicToDenormModeHazard ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasNoDataDepHazard ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;

if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> if ( ST . hasNSAtoVMEMBug ( ) ) WaitStates = std :: max ( WaitStates , checkNSAtoVMEMHazard ( MI ) ) ; WaitStates = std :: max ( WaitStates , checkFPAtomicToDenormModeHazard ( MI ) ) ; if ( ST . hasNoDataDepHazard ( ) ) return WaitStates ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( MI -> isInlineAsm ( ) ) return std :: max ( WaitStates , checkInlineAsmHazards ( MI ) ) ; if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ;

if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ;


<s> if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkAnyInstHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; return NoHazard ;


<s> if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;

if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVMEMHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ;

if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( * MI ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ;


<s> int GCNHazardRecognizer :: checkSMRDHazards ( MachineInstr * SMRD ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; int WaitStatesNeeded = <NUM_LIT> ; WaitStatesNeeded = checkSMEMSoftClauseHazards ( SMRD ) ;

if ( ST . getGeneration ( ) != <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return WaitStatesNeeded ;


<s> CurrCycleInstr ( nullptr ) , MF ( MF ) , ST ( MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ) , TII ( * ST . getInstrInfo ( ) ) { MaxLookAhead = <NUM_LIT> ;


<s> if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkAnyInstHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ;

if ( checkAnyInstHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> unsigned GCNHazardRecognizer :: PreEmitNoops ( MachineInstr * MI ) { int WaitStates = std :: max ( <NUM_LIT> , checkAnyInstHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) return std :: max ( WaitStates , checkSMRDHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) { WaitStates = std :: max ( WaitStates , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVMEMHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( TII . isVINTRP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ; return WaitStates ; } if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ;

if ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ;


<s> const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ; unsigned NumWaitStates = TII -> getNumWaitStates ( * CurrCycleInstr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; } EmittedInstrs . resize ( getMaxLookAhead ( ) ) ;


<s> if ( ! CurrCycleInstr ) return ; const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ; unsigned NumWaitStates = TII -> getNumWaitStates ( * CurrCycleInstr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ;


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( ST . getRegisterInfo ( ) ) ; int DppVgprWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ;

for ( const MachineOperand & Use : DPP -> uses ( ) ) {


<s> int GCNHazardRecognizer :: checkSMEMSoftClauseHazards ( MachineInstr * SMEM ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ST . getGeneration ( ) < <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return <NUM_LIT> ; std :: set < unsigned > ClauseDefs ; std :: set < unsigned > ClauseUses ; for ( MachineInstr * MI : EmittedInstrs ) { if ( ! MI || ! <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) break ; addRegsToSet ( MI -> defs ( ) , ClauseDefs ) ; addRegsToSet ( MI -> uses ( ) , ClauseUses ) ;


<s> for ( const MachineOperand & Use : SMRD -> uses ( ) ) { if ( ! Use . isReg ( ) ) continue ; int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ;


<s> if ( ST . getGeneration ( ) != <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return WaitStatesNeeded ; int SmrdSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) { if ( ! Use . isReg ( ) ) continue ; int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ;


<s> int GCNHazardRecognizer :: checkVMEMHazards ( MachineInstr * VMEM ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( ST . getInstrInfo ( ) ) ;

if ( ST . getGeneration ( ) < <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return <NUM_LIT> ;


<s> CurrCycleInstr ( nullptr ) , MF ( MF ) {


<s> if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) && checkSMRDHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;

if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> int WaitStates = - <NUM_LIT> ; for ( MachineInstr * MI : EmittedInstrs ) { ++ WaitStates ; if ( ! MI || ! IsHazardDef ( MI ) ) continue ; if ( MI -> modifiesRegister ( Reg , TRI ) ) return WaitStates ;


<s> if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) return std :: max ( <NUM_LIT> , checkSMRDHazards ( MI ) ) ;

if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) ) return std :: max ( <NUM_LIT> , checkVMEMHazards ( MI ) ) ;


<s> auto IsHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isVALU ( * MI ) ; } ; auto IsBufferHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isSALU ( * MI ) ; } ; bool IsBufferSMRD = TII . isBufferSMRD ( * SMRD ) ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) { if ( ! Use . isReg ( ) ) continue ; int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn , SmrdSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; if ( IsBufferSMRD ) { int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsBufferHazardDefFn , SmrdSgprWaitStates ) ;


<s> fixSMEMtoVectorWriteHazards ( MI ) ; fixVcmpxExecWARHazard ( MI ) ; fixLdsBranchVmemWARHazard ( MI ) ;


<s> if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ;

if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> ++ WaitStates ; if ( ! MI || ! IsHazardDef ( MI ) ) continue ; if ( MI -> modifiesRegister ( Reg , TRI ) ) return WaitStates ; }

return std :: numeric_limits < int > :: max ( ) ;


<s> int WaitStates = - <NUM_LIT> ; for ( MachineInstr * MI : EmittedInstrs ) { ++ WaitStates ; if ( ! MI || ! IsHazardDef ( MI ) ) continue ; if ( MI -> modifiesRegister ( Reg , TRI ) ) return WaitStates ; } return std :: numeric_limits < int > :: max ( ) ;


<s> if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;

if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVMEMHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ;

if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ;


<s> if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) { int WaitStates = std :: max ( <NUM_LIT> , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVMEMHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ;

if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ;


<s> static bool isDGEMM ( unsigned Opcode ) { return Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> ;


<s> if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;

if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) || <ISA_LIT>InstrInfo :: isDS ( * MI ) ) && checkMAILdStHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> MachineInstr * MI = SU -> getInstr ( ) ; if ( MI -> isBundle ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) && checkSMRDHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasNSAtoVMEMBug ( ) && checkNSAtoVMEMHazard ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkFPAtomicToDenormModeHazard ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasNoDataDepHazard ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> unsigned NumWaitStates = TII . getNumWaitStates ( * CurrCycleInstr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; } EmittedInstrs . resize ( getMaxLookAhead ( ) ) ; CurrCycleInstr = nullptr ;


int WaitStatesNeededForUse = AccVgprReadLdStWaitStates - getWaitStatesSinceDef ( Reg , IsAccVgprReadFn , MaxWaitStates ) ;


<s> unsigned Opcode = MI -> getOpcode ( ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) continue ; } ++ WaitStates ; }


<s> ++ WaitStates ; if ( ! MI || ! IsHazardDef ( MI ) ) continue ; if ( MI -> modifiesRegister ( Reg , TRI ) ) return WaitStates ; } return std :: numeric_limits < int > :: max ( ) ;


<s> const TargetRegisterInfo * TRI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ; int WaitStates = - <NUM_LIT> ; for ( MachineInstr * MI : EmittedInstrs ) {


<s> } const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <STR_LIT> IV = <STR_LIT> ( ST . getCPU ( ) ) ; const MachineOperand * SDST = TII -> getNamedOperand ( * MI , SDSTName ) ; if ( ! SDST ) { for ( const auto & MO : MI -> implicit_operands ( ) ) { if ( MO . isDef ( ) && TRI -> isSGPRClass ( TRI -> getPhysRegClass ( MO . getReg ( ) ) ) ) { SDST = & MO ; break ; } } } if ( ! SDST ) return false ; const unsigned SDSTReg = SDST -> getReg ( ) ; auto IsHazardFn = [ SDSTReg , TRI ] ( MachineInstr * I ) { return <ISA_LIT>InstrInfo :: isSMRD ( * I ) && I -> readsRegister ( SDSTReg , TRI ) ; } ; auto IsExpiredFn = [ TII , IV ] ( MachineInstr * MI , int ) { if ( MI ) { if ( TII -> isSALU ( * MI ) ) { switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return false ; case <STR_LIT> : return ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) && ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) ; case <STR_LIT> : { const int64_t Imm = MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; <STR_LIT> Decoded = <STR_LIT> ( IV , Imm ) ; return ( Decoded . LgkmCnt == <NUM_LIT> ) ; } default : if ( TII -> isSOPP ( * MI ) ) return false ; return true ; }


<s> if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ;

if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ;


<s> static void addRegsToSet ( const <ISA_LIT>RegisterInfo & TRI , iterator_range < MachineInstr :: const_mop_iterator > Ops , BitVector & Set ) { for ( const MachineOperand & Op : Ops ) { if ( Op . isReg ( ) ) addRegUnits ( TRI , Set , Op . getReg ( ) ) ; }


<s> for ( MCRegUnitIterator RUI ( Reg , & TRI ) ; RUI . isValid ( ) ; ++ RUI ) BV . set ( * RUI ) ;


<s> else if ( CurrCycleInstr -> isDebugInstr ( ) ) return ; unsigned NumWaitStates = TII . getNumWaitStates ( * CurrCycleInstr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) {

EmittedInstrs . push_front ( nullptr ) ;


<s> if ( MI -> isDebugInstr ( ) ) return <NUM_LIT> ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; if ( ! ST . hasSMovFedHazard ( ) ) return <NUM_LIT> ; int MovFedWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; for ( const MachineOperand & Use : MI -> uses ( ) ) { if ( ! Use . isReg ( ) || TRI -> isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; auto IsHazardFn = [ ] ( MachineInstr * MI ) { return MI -> getOpcode ( ) == <STR_LIT> ; } ;

int WaitStatesNeededForUse = MovFedWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardFn , MovFedWaitStates ) ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const int DivFMasWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ; int WaitStatesNeeded = getWaitStatesSinceDef ( <STR_LIT> , IsHazardDefFn , DivFMasWaitStates ) ;

return DivFMasWaitStates - WaitStatesNeeded ;


<s> int DppExecWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ; for ( const MachineOperand & Use : DPP -> uses ( ) ) { if ( ! Use . isReg ( ) || ! TRI -> isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; int WaitStatesNeededForUse = DppVgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , [ ] ( MachineInstr * ) { return true ; } , DppVgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; } WaitStatesNeeded = std :: max ( WaitStatesNeeded , DppExecWaitStates - getWaitStatesSinceDef ( <STR_LIT> , IsHazardDefFn , DppExecWaitStates ) ) ;


<s> const int GetRegWaitStates = <NUM_LIT> ; auto IsHazardFn = [ TII , GetRegHWReg ] ( MachineInstr * MI ) { return GetRegHWReg == getHWReg ( TII , * MI ) ; } ; int WaitStatesNeeded = getWaitStatesSinceSetReg ( IsHazardFn , GetRegWaitStates ) ; return GetRegWaitStates - WaitStatesNeeded ;


<s> unsigned GetRegHWReg = getHWReg ( TII , * GetRegInstr ) ; const int GetRegWaitStates = <NUM_LIT> ; auto IsHazardFn = [ TII , GetRegHWReg ] ( MachineInstr * MI ) { return GetRegHWReg == getHWReg ( TII , * MI ) ; } ; int WaitStatesNeeded = getWaitStatesSinceSetReg ( IsHazardFn , GetRegWaitStates ) ; return GetRegWaitStates - WaitStatesNeeded ;


<s> auto IsHazardFn = [ TII ] ( MachineInstr * MI ) { return TII -> isSALU ( * MI ) ; } ; return SMovRelWaitStates - getWaitStatesSinceDef ( <STR_LIT> , IsHazardFn , SMovRelWaitStates ) ;


<s> const int RFEWaitStates = <NUM_LIT> ; auto IsHazardFn = [ TII ] ( MachineInstr * MI ) { return getHWReg ( TII , * MI ) == <STR_LIT> :: ID_TRAPSTS ;


<s> unsigned LaneSelectReg = LaneSelectOp -> getReg ( ) ; auto IsHazardFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ; const int RWLaneWaitStates = <NUM_LIT> ; int WaitStatesSince = getWaitStatesSinceDef ( LaneSelectReg , IsHazardFn , RWLaneWaitStates ) ;

return RWLaneWaitStates - WaitStatesSince ;


<s> int GCNHazardRecognizer :: checkRWLaneHazards ( MachineInstr * RWLane ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ;

const MachineOperand * LaneSelectOp = TII -> getNamedOperand ( * RWLane , <STR_LIT> :: src1 ) ;


<s> auto IsHazardFn = [ TII , HWReg ] ( MachineInstr * MI ) { return HWReg == getHWReg ( TII , * MI ) ; } ; int WaitStatesNeeded = getWaitStatesSinceSetReg ( IsHazardFn , SetRegWaitStates ) ; return SetRegWaitStates - WaitStatesNeeded ;


<s> bool IsBufferSMRD = TII . isBufferSMRD ( * SMRD ) ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) { if ( ! Use . isReg ( ) ) continue ; int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn , SmrdSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; if ( IsBufferSMRD ) { int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsBufferHazardDefFn , SmrdSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; } }


<s> int WaitStatesNeeded = <NUM_LIT> ; WaitStatesNeeded = checkSoftClauseHazards ( SMRD ) ; if ( ST . getGeneration ( ) != <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return WaitStatesNeeded ; int SmrdSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isVALU ( * MI ) ; } ; auto IsBufferHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isSALU ( * MI ) ; } ; bool IsBufferSMRD = TII . isBufferSMRD ( * SMRD ) ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) { if ( ! Use . isReg ( ) ) continue ; int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn , SmrdSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; if ( IsBufferSMRD ) { int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsBufferHazardDefFn , SmrdSgprWaitStates ) ;


<s> if ( IsSMRD != <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) break ; addClauseInst ( * MI ) ; } if ( ClauseDefs . none ( ) ) return <NUM_LIT> ;

if ( MEM -> mayStore ( ) ) return <NUM_LIT> ;


<s> const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; int WaitStatesNeeded = <NUM_LIT> ; for ( const MachineOperand & Def : VALU -> defs ( ) ) { WaitStatesNeeded = std :: max ( WaitStatesNeeded , checkVALUHazardsHelper ( Def , MRI ) ) ; } return WaitStatesNeeded ;


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const int VALUWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; if ( ! TRI -> isVGPR ( MRI , Def . getReg ( ) ) ) return WaitStatesNeeded ; unsigned Reg = Def . getReg ( ) ; auto IsHazardFn = [ this , Reg , TRI ] ( MachineInstr * MI ) {

int DataIdx = createsVALUHazard ( * MI ) ;


<s> return DataIdx >= <NUM_LIT> && TRI -> regsOverlap ( MI -> getOperand ( DataIdx ) . getReg ( ) , Reg ) ; } ; int WaitStatesNeededForDef = VALUWaitStates - getWaitStatesSince ( IsHazardFn , VALUWaitStates ) ;

WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForDef ) ;


<s> for ( const MachineOperand & Use : VMEM -> uses ( ) ) { if ( ! Use . isReg ( ) || TRI . isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; int WaitStatesNeededForUse = VmemSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn , VmemSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ;


<s> const int VmemSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isVALU ( * MI ) ; } ; for ( const MachineOperand & Use : VMEM -> uses ( ) ) { if ( ! Use . isReg ( ) || TRI . isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; int WaitStatesNeededForUse = VmemSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn , VmemSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ;


<s> IsHazardRecognizerMode ( false ) , CurrCycleInstr ( nullptr ) , MF ( MF ) , ST ( MF . getSubtarget < GCNSubtarget > ( ) ) , TII ( * ST . getInstrInfo ( ) ) , TRI ( TII . getRegisterInfo ( ) ) , ClauseUses ( TRI . getNumRegUnits ( ) ) , ClauseDefs ( TRI . getNumRegUnits ( ) ) { MaxLookAhead = <NUM_LIT> ;