<s> MachineFunction * MF = MBB . getParent ( ) ; DebugLoc DL ; if ( ! EarlyExitBlock ) { EarlyExitBlock = MF -> CreateMachineBasicBlock ( ) ; MF -> insert ( MF -> end ( ) , EarlyExitBlock ) ; generatePsEndPgm ( * EarlyExitBlock , EarlyExitBlock -> end ( ) , DL , TII ) ; EarlyExitClearsExec = false ; } if ( ClearExec && ! EarlyExitClearsExec ) { const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; unsigned Mov = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ;

Register Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ;


<s> BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <STR_LIT> :: ET_NULL ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ;

BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ;


<s> AU . addRequired < MachineDominatorTree > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ;

MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } const GCNSubtarget & ST = MBB . getParent ( ) -> getSubtarget < GCNSubtarget > ( ) ; if ( ST . hasNoSdstCMPX ( ) ) Opcode = <STR_LIT> ( Opcode ) ; assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) ; if ( TRI -> isVGPR ( MBB . getParent ( ) -> getRegInfo ( ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Opcode = <STR_LIT> ( Opcode ) ; BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else { auto I = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) ; if ( ! ST . hasNoSdstCMPX ( ) ) I . addReg ( <STR_LIT> , RegState :: Define ) ; I . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; I . addImm ( <NUM_LIT> ) ; } return true ; } case <STR_LIT> : { const MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; int64_t KillVal = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; assert ( KillVal == <NUM_LIT> || KillVal == - <NUM_LIT> ) ; if ( Op . isImm ( ) ) { int64_t Imm = Op . getImm ( ) ; assert ( Imm == <NUM_LIT> || Imm == - <NUM_LIT> ) ; if ( Imm == KillVal ) { BuildMI ( MBB , & MI , DL , TII -> get ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) , Exec ) . addImm ( <NUM_LIT> ) ; return true ; }


<s> break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } const GCNSubtarget & ST = MBB . getParent ( ) -> getSubtarget < GCNSubtarget > ( ) ; if ( ST . hasNoSdstCMPX ( ) ) Opcode = <STR_LIT> ( Opcode ) ; assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) ; if ( TRI -> isVGPR ( MBB . getParent ( ) -> getRegInfo ( ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Opcode = <STR_LIT> ( Opcode ) ; BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else { auto I = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) ; if ( ! ST . hasNoSdstCMPX ( ) ) I . addReg ( <STR_LIT> , RegState :: Define ) ; I . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; I . addImm ( <NUM_LIT> ) ; } return true ; } case <STR_LIT> : { const MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; int64_t KillVal = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ;

assert ( KillVal == <NUM_LIT> || KillVal == - <NUM_LIT> ) ;


<s> if ( MI . isMetaInstruction ( ) ) return true ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> :


<s> bool <ISA_LIT>InsertSkips :: runOnMachineFunction ( MachineFunction & MF ) { const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; SkipThreshold = SkipThresholdFlag ; SmallVector < MachineInstr * , <NUM_LIT> > KillInstrs ; bool MadeChange = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> :

if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) {


<s> if ( TII -> hasUnwantedEffectsWhenEXECEmpty ( * I ) ) return true ; if ( TII -> isSMRD ( * I ) || TII -> isVMEM ( * I ) || TII -> isFLAT ( * I ) || I -> getOpcode ( ) == <STR_LIT> ) return true ; ++ NumInstr ; if ( NumInstr >= SkipThreshold ) return true ; } }


<s> <ISA_LIT>InsertSkips ( ) : MachineFunctionPass ( ID ) {


<s> assert ( MF -> getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ) ; auto NextBBI = std :: next ( MBB . getIterator ( ) ) ; bool NoSuccessor = I == MBB . end ( ) && ! llvm :: is_contained ( MBB . successors ( ) , & * NextBBI ) ; if ( NoSuccessor ) { generatePsEndPgm ( MBB , I , DL , TII ) ; } else { ensureEarlyExitBlock ( MBB , false ) ;

MachineInstr * BranchMI = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( EarlyExitBlock ) ;


<s> ensureEarlyExitBlock ( MBB , false ) ; MachineInstr * BranchMI = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( EarlyExitBlock ) ; auto Next = std :: next ( BranchMI -> getIterator ( ) ) ; if ( Next != MBB . end ( ) && ! Next -> isTerminator ( ) ) splitBlock ( MBB , * BranchMI , MDT ) ; MBB . addSuccessor ( EarlyExitBlock ) ; MDT -> getBase ( ) . insertEdge ( & MBB , EarlyExitBlock ) ; }


<s> bool <ISA_LIT>InsertSkips :: skipMaskBranch ( MachineInstr & MI , MachineBasicBlock & SrcMBB ) { MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! shouldSkip ( * * SrcMBB . succ_begin ( ) , * DestBB ) ) return false ;

const DebugLoc & DL = MI . getDebugLoc ( ) ;


<s> MachineBasicBlock :: iterator InsPt = std :: next ( MI . getIterator ( ) ) ; BuildMI ( SrcMBB , InsPt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; return true ;


<s> DTUpdates . push_back ( { DomTreeT :: Insert , SplitBB , Succ } ) ; DTUpdates . push_back ( { DomTreeT :: Delete , & MBB , Succ } ) ; } DTUpdates . push_back ( { DomTreeT :: Insert , & MBB , SplitBB } ) ; MDT -> getBase ( ) . applyUpdates ( DTUpdates ) ;


<s> void addKillWaitBracket ( BlockWaitcntBrackets * Bracket ) {

KillWaitBrackets . push_back ( llvm :: make_unique < BlockWaitcntBrackets > ( * Bracket ) ) ;


<s> for ( enum InstCounterType T = VM_CNT ; T < NUM_INST_CNTS ; T = ( enum InstCounterType ) ( T + <NUM_LIT> ) ) { memset ( VgprScores [ T ] , <NUM_LIT> , sizeof ( VgprScores [ T ] ) ) ;


<s> case LGKM_CNT : { if ( EventUBs [ SMEM_ACCESS ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ SMEM_ACCESS ] <= ScoreUBs [ LGKM_CNT ] ) { return true ; } int NumEventTypes = <NUM_LIT> ; if ( EventUBs [ LDS_ACCESS ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ LDS_ACCESS ] <= ScoreUBs [ LGKM_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ GDS_ACCESS ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ GDS_ACCESS ] <= ScoreUBs [ LGKM_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ SQ_MESSAGE ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ SQ_MESSAGE ] <= ScoreUBs [ LGKM_CNT ] ) { NumEventTypes ++ ; }


<s> } int NumEventTypes = <NUM_LIT> ; if ( EventUBs [ LDS_ACCESS ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ LDS_ACCESS ] <= ScoreUBs [ LGKM_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ GDS_ACCESS ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ GDS_ACCESS ] <= ScoreUBs [ LGKM_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ SQ_MESSAGE ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ SQ_MESSAGE ] <= ScoreUBs [ LGKM_CNT ] ) { NumEventTypes ++ ; } if ( NumEventTypes <= <NUM_LIT> ) { return false ; } break ; } case EXP_CNT : { if ( MixedExpTypes ) return true ; int NumEventTypes = <NUM_LIT> ; if ( EventUBs [ EXP_GPR_LOCK ] > ScoreLBs [ EXP_CNT ] && EventUBs [ EXP_GPR_LOCK ] <= ScoreUBs [ EXP_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ GDS_GPR_LOCK ] > ScoreLBs [ EXP_CNT ] && EventUBs [ GDS_GPR_LOCK ] <= ScoreUBs [ EXP_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ VMW_GPR_LOCK ] > ScoreLBs [ EXP_CNT ] && EventUBs [ VMW_GPR_LOCK ] <= ScoreUBs [ EXP_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ EXP_PARAM_ACCESS ] > ScoreLBs [ EXP_CNT ] && EventUBs [ EXP_PARAM_ACCESS ] <= ScoreUBs [ EXP_CNT ] ) { NumEventTypes ++ ; } if ( EventUBs [ EXP_POS_ACCESS ] > ScoreLBs [ EXP_CNT ] && EventUBs [ EXP_POS_ACCESS ] <= ScoreUBs [ EXP_CNT ] ) {


<s> return VM_CNT ; case LDS_ACCESS : case GDS_ACCESS : case SQ_MESSAGE : case SMEM_ACCESS : return LGKM_CNT ; case EXP_GPR_LOCK :


<s> if ( PredEventUB > KillWaitBrackets [ I ] -> getScoreLB ( T ) ) { int NewEventUB = MaxPending [ T ] + PredEventUB - KillWaitBrackets [ I ] -> getScoreUB ( T ) ;


<s> AU . addRequired < MachineLoopInfo > ( ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override {


<s> int32_t getIterCnt ( ) { return IterCnt ;


<s> int32_t getPostOrder ( ) const { return PostOrder ;


<s> if ( TRI -> isVGPR ( MRIA , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( MRIA , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . SGPR0 && Reg < SQ_MAX_PGM_SGPRS ) ; Result . first = Reg - RegisterEncoding . SGPR0 + NUM_ALL_VGPRS ; assert ( Result . first >= NUM_ALL_VGPRS && Result . first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS ) ;


<s> return VgprScores [ T ] [ GprNo ] ; } return SgprScores [ GprNo - NUM_ALL_VGPRS ] ;


<s> assert ( T < NUM_INST_CNTS ) ; if ( T >= NUM_INST_CNTS ) return <NUM_LIT> ;


<s> assert ( T < NUM_INST_CNTS ) ; if ( T >= NUM_INST_CNTS ) return <NUM_LIT> ; return ScoreLBs [ T ] ;


<s> if ( T >= NUM_INST_CNTS ) return <NUM_LIT> ; return ScoreUBs [ T ] ;


<s> int32_t getScoreUB ( InstCounterType T ) { assert ( T < NUM_INST_CNTS ) ; if ( T >= NUM_INST_CNTS ) return <NUM_LIT> ; return ScoreUBs [ T ] ;


<s> return HardwareLimits . VmcntMax ; case LGKM_CNT : return HardwareLimits . LgkmcntMax ; case EXP_CNT : return HardwareLimits . ExpcntMax ; default :


return ( EventUBs [ SMEM_ACCESS ] > ScoreLBs [ LGKM_CNT ] && EventUBs [ SMEM_ACCESS ] <= ScoreUBs [ LGKM_CNT ] ) ;


<s> } MachineBasicBlock :: iterator It = MBB . end ( ) ; MachineInstr * MI = & * ( -- It ) ; if ( MI -> isBranch ( ) ) { MBB . insert ( It , Waitcnt ) ; } else { MBB . push_back ( Waitcnt ) ; }


<s> MBB . push_back ( Waitcnt ) ; return ; } MachineBasicBlock :: iterator It = MBB . end ( ) ; MachineInstr * MI = & * ( -- It ) ; if ( MI -> isBranch ( ) ) { MBB . insert ( It , Waitcnt ) ; } else { MBB . push_back ( Waitcnt ) ;


<s> BlockWaitcntBrackets * ScoreBrackets = BlockWaitcntBracketsMap [ & Block ] . get ( ) ; DEBUG ( { dbgs ( ) << <STR_LIT> << Block . getNumber ( ) ;


BlockWaitcntBrackets * ScoreBrackets = BlockWaitcntBracketsMap [ & Block ] . get ( ) ;


<s> MachineBasicBlock * <ISA_LIT>InsertWaitcnts :: loopBottom ( const MachineLoop * Loop ) {

MachineBasicBlock * Bottom = Loop -> getHeader ( ) ;


<s> for ( MachineBasicBlock * MBB : Loop -> blocks ( ) ) if ( MBB -> getNumber ( ) > Bottom -> getNumber ( ) ) Bottom = MBB ; return Bottom ;


<s> if ( MI . memoperands_empty ( ) ) return true ; for ( const MachineMemOperand * Memop : MI . memoperands ( ) ) {

unsigned AS = Memop -> getAddrSpace ( ) ;


<s> OS << RelScore << <STR_LIT> << J << <STR_LIT> ; } else { OS << RelScore << <STR_LIT> ; } } if ( T == LGKM_CNT ) { for ( int J = <NUM_LIT> ; J <= getMaxSGPR ( ) ; J ++ ) { int RegScore = getRegScore ( J + NUM_ALL_VGPRS , LGKM_CNT ) ; if ( RegScore <= LB ) continue ; int RelScore = RegScore - LB - <NUM_LIT> ; OS << RelScore << <STR_LIT> << J << <STR_LIT> ; } } } OS << '\n' ; }


<s> assert ( TRI -> isVGPR ( * MRI , Opnd . getReg ( ) ) ) ; } ) ; for ( signed RegNo = Interval . first ; RegNo < Interval . second ; ++ RegNo ) { setRegScore ( RegNo , EXP_CNT , Val ) ; }


<s> DEBUG ( { const MachineOperand & Opnd = MI -> getOperand ( OpNo ) ; assert ( TRI -> isVGPR ( * MRI , Opnd . getReg ( ) ) ) ; } ) ; for ( signed RegNo = Interval . first ; RegNo < Interval . second ; ++ RegNo ) { setRegScore ( RegNo , EXP_CNT , Val ) ; }


<s> void setMixedExpTypes ( bool MixedExpTypesIn ) {


<s> void setRegScore ( int GprNo , InstCounterType T , int32_t Val ) { if ( GprNo < NUM_ALL_VGPRS ) { if ( GprNo > VgprUB ) { VgprUB = GprNo ; } VgprScores [ T ] [ GprNo ] = Val ; } else { assert ( T == LGKM_CNT ) ; if ( GprNo - NUM_ALL_VGPRS > SgprUB ) {


<s> void setRevisitLoop ( bool RevisitLoopIn ) {


<s> assert ( T < NUM_INST_CNTS ) ; if ( T >= NUM_INST_CNTS ) return ;


<s> void setScoreUB ( InstCounterType T , int32_t Val ) { assert ( T < NUM_INST_CNTS ) ; if ( T >= NUM_INST_CNTS ) return ;


<s> void setWaitcnt ( MachineInstr * WaitcntIn ) { LfWaitcnt = WaitcntIn ;


<s> <ISA_LIT>InsertWaitcnts ( ) : MachineFunctionPass ( ID ) {


<s> MachineOperand & DefMO = Inst . getOperand ( I ) ; if ( DefMO . isReg ( ) && DefMO . isDef ( ) && TRI -> isVGPR ( MRIA , DefMO . getReg ( ) ) ) { setRegScore ( TRI -> getEncodingValue ( DefMO . getReg ( ) ) , EXP_CNT , CurrScore ) ; } } } for ( unsigned I = <NUM_LIT> , E = Inst . getNumOperands ( ) ; I != E ; ++ I ) { MachineOperand & MO = Inst . getOperand ( I ) ; if ( MO . isReg ( ) && ! MO . isDef ( ) && TRI -> isVGPR ( MRIA , MO . getReg ( ) ) ) { setExpScore ( & Inst , TII , TRI , MRI , I , CurrScore ) ; } } } } else if ( Inst . getOpcode ( ) == <STR_LIT> || Inst . getOpcode ( ) == <STR_LIT> || Inst . getOpcode ( ) == <STR_LIT> ) { MachineOperand * MO = TII -> getNamedOperand ( Inst , <STR_LIT> :: data ) ; unsigned OpNo ; RegInterval Interval = getRegInterval ( & Inst , TII , MRI , TRI , OpNo , false ) ; for ( signed RegNo = Interval . first ; RegNo < Interval . second ; ++ RegNo ) { setRegScore ( RegNo + NUM_ALL_VGPRS , t , CurrScore ) ; } } else { for ( unsigned I = <NUM_LIT> , E = Inst . getNumOperands ( ) ; I != E ; ++ I ) { RegInterval Interval = getRegInterval ( & Inst , TII , MRI , TRI , I , true ) ; if ( T == VM_CNT && Interval . first >= NUM_ALL_VGPRS ) continue ; for ( signed RegNo = Interval . first ; RegNo < Interval . second ; ++ RegNo ) { setRegScore ( RegNo , T , CurrScore ) ; } } if ( TII -> isDS ( Inst ) && Inst . mayStore ( ) ) { setRegScore ( SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS , T , CurrScore ) ; }


<s> unsigned int NeedWait = <NUM_LIT> ; if ( ScoreToWait == - <NUM_LIT> ) { NeedWait = CNT_MASK ( T ) ; setScoreLB ( T , getScoreUB ( T ) ) ; return NeedWait ; } const int32_t LB = getScoreLB ( T ) ; const int32_t UB = getScoreUB ( T ) ; if ( ( UB >= ScoreToWait ) && ( ScoreToWait > LB ) ) { if ( T == VM_CNT && hasPendingFlat ( ) ) { NeedWait = CNT_MASK ( T ) ; setScoreLB ( T , getScoreUB ( T ) ) ; } else if ( counterOutOfOrder ( T ) ) { NeedWait = CNT_MASK ( T ) ; setScoreLB ( T , getScoreUB ( T ) ) ;


<s> ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; } } else if ( TII -> isFLAT ( Inst ) ) { assert ( Inst . mayLoad ( ) || Inst . mayStore ( ) ) ; if ( TII -> usesVM_CNT ( Inst ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; if ( TII -> usesLGKM_CNT ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; if ( mayAccessLDSThroughFlat ( Inst ) ) ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; if ( ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; } else { switch ( Inst . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SQ_MESSAGE , Inst ) ; break ; case <STR_LIT> :


<s> void addKillWaitBracket ( BlockWaitcntBrackets * Bracket ) {

KillWaitBrackets . push_back ( make_unique < BlockWaitcntBrackets > ( * Bracket ) ) ;


<s> BlockWaitcntBrackets ( ) : WaitAtBeginning ( false ) , ValidLoop ( false ) , MixedExpTypes ( false ) , LoopRegion ( NULL ) , PostOrder ( <NUM_LIT> ) , Waitcnt ( NULL ) , VgprUB ( <NUM_LIT> ) , SgprUB ( <NUM_LIT> ) { for ( enum InstCounterType T = VM_CNT ; T < NUM_INST_CNTS ;

T = ( enum InstCounterType ) ( T + <NUM_LIT> ) ) {


<s> void clearWaitcnt ( ) { Waitcnt = NULL ;


<s> MBB . push_back ( Waitcnt ) ; return ; } MachineBasicBlock :: iterator It = MBB . end ( ) ; MachineInstr * MI = & * ( -- It ) ; if ( MI -> isBranch ( ) ) { MBB . insert ( It , Waitcnt ) ;


<s> LoopWaitcntData ( ) : LfWaitcnt ( NULL ) , IterCnt ( <NUM_LIT> ) {


<s> for ( enum InstCounterType T = VM_CNT ; T < NUM_INST_CNTS ; T = ( enum InstCounterType ) ( T + <NUM_LIT> ) ) { int LB = getScoreLB ( T ) ; int UB = getScoreUB ( T ) ; switch ( T ) { case VM_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; case LGKM_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; case EXP_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; default : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; } if ( LB < UB ) { for ( int J = <NUM_LIT> ; J <= getMaxVGPR ( ) ; J ++ ) { int RegScore = getRegScore ( J , T ) ; if ( RegScore <= LB ) continue ; int RelScore = RegScore - LB - <NUM_LIT> ; if ( J < SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS ) { OS << RelScore << <STR_LIT> << J << <STR_LIT> ; } else { OS << RelScore << <STR_LIT> ; } } if ( T == LGKM_CNT ) { for ( int J = <NUM_LIT> ; J <= getMaxSGPR ( ) ; J ++ ) { int RegScore = getRegScore ( J + NUM_ALL_VGPRS , LGKM_CNT ) ; if ( RegScore <= LB ) continue ; int RelScore = RegScore - LB - <NUM_LIT> ; OS << RelScore << <STR_LIT> << J << <STR_LIT> ; } } } OS << '\n' ;


<s> case VM_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; case LGKM_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; case EXP_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; default : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; } if ( LB < UB ) { for ( int J = <NUM_LIT> ; J <= getMaxVGPR ( ) ; J ++ ) { int RegScore = getRegScore ( J , T ) ; if ( RegScore <= LB ) continue ; int RelScore = RegScore - LB - <NUM_LIT> ; if ( J < SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS ) { OS << RelScore << <STR_LIT> << J << <STR_LIT> ; } else { OS << RelScore << <STR_LIT> ; } } if ( T == LGKM_CNT ) { for ( int J = <NUM_LIT> ; J <= getMaxSGPR ( ) ; J ++ ) { int RegScore = getRegScore ( J + NUM_ALL_VGPRS , LGKM_CNT ) ; if ( RegScore <= LB ) continue ; int RelScore = RegScore - LB - <NUM_LIT> ; OS << RelScore << <STR_LIT> << J << <STR_LIT> ;


<s> <ISA_LIT>InsertWaitcnts ( ) : MachineFunctionPass ( ID ) , ST ( nullptr ) , TII ( nullptr ) , TRI ( nullptr ) , MRI ( nullptr ) , MLI ( nullptr ) {


<s> if ( ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; } else { switch ( Inst . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SQ_MESSAGE , Inst ) ; break ; case <STR_LIT> : case <STR_LIT> : { int Imm = TII -> getNamedOperand ( Inst , <STR_LIT> :: tgt ) -> getImm ( ) ; if ( Imm >= <NUM_LIT> && Imm <= <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_PARAM_ACCESS , Inst ) ; else if ( Imm >= <NUM_LIT> && Imm <= <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_POS_ACCESS , Inst ) ; else ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_GPR_LOCK , Inst ) ; break ; } case <STR_LIT> : case <STR_LIT> : ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; break ; default : break ; } }


<s> if ( TII -> getNamedOperand ( Inst , <STR_LIT> :: gds ) -> getImm ( ) != <NUM_LIT> ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_ACCESS , Inst ) ; ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_GPR_LOCK , Inst ) ; } else { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; } } else if ( TII -> isFLAT ( Inst ) ) { assert ( Inst . mayLoad ( ) || Inst . mayStore ( ) ) ; ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; bool FoundLDSMem = false ; for ( const MachineMemOperand * Memop : Inst . memoperands ( ) ) { unsigned AS = Memop -> getAddrSpace ( ) ; if ( AS == AMDGPUA<ISA_LIT> . LOCAL_ADDRESS || AS == AMDGPUA<ISA_LIT> . FLAT_ADDRESS ) FoundLDSMem = true ; } if ( FoundLDSMem ) { ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; if ( ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; } else { switch ( Inst . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SQ_MESSAGE , Inst ) ; break ; case <STR_LIT> : case <STR_LIT> : { int Imm = TII -> getNamedOperand ( Inst , <STR_LIT> :: tgt ) -> getImm ( ) ; if ( Imm >= <NUM_LIT> && Imm <= <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_PARAM_ACCESS , Inst ) ; else if ( Imm >= <NUM_LIT> && Imm <= <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_POS_ACCESS , Inst ) ; else ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_GPR_LOCK , Inst ) ; break ; } case <STR_LIT> : case <STR_LIT> : ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; break ; default : break ; } }


<s> unsigned VmCnt = std :: min ( <STR_LIT> ( IV , LHS ) , <STR_LIT> ( IV , RHS ) ) ; unsigned LgkmCnt = std :: min ( <STR_LIT> ( IV , LHS ) , <STR_LIT> ( IV , RHS ) ) ; unsigned ExpCnt = std :: min ( <STR_LIT> ( IV , LHS ) , <STR_LIT> ( IV , RHS ) ) ; return <STR_LIT> ( IV , VmCnt , ExpCnt , LgkmCnt ) ;


<s> unsigned <ISA_LIT>InsertWaitcnts :: countNumBottomBlocks ( const MachineLoop * Loop ) { unsigned Count = <NUM_LIT> ; for ( MachineBasicBlock * MBB : Loop -> blocks ( ) ) { if ( MBB -> isSuccessor ( Loop -> getHeader ( ) ) ) { Count ++ ; }


<s> LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> << Block . getNumber ( ) << <STR_LIT> ;


<s> bool isForceEmitWaitcnt ( ) const { for ( enum InstCounterType T = VM_CNT ; T < NUM_INST_CNTS ;

T = ( enum InstCounterType ) ( T + <NUM_LIT> ) ) if ( ForceEmitWaitcnt [ T ] ) return true ;


<s> bool <ISA_LIT>InsertWaitcnts :: isWaitcntStronger ( unsigned LHS , unsigned RHS ) { if ( <STR_LIT> ( IV , LHS ) > <STR_LIT> ( IV , RHS ) ) return false ; if ( <STR_LIT> ( IV , LHS ) > <STR_LIT> ( IV , RHS ) ) return false ; if ( <STR_LIT> ( IV , LHS ) > <STR_LIT> ( IV , RHS ) ) return false ;


LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << IterCnt << '\n' ; ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << IterCnt << '\n' ; ) ;


<s> if ( ( std :: count ( BlockWaitcntProcessedSet . begin ( ) , BlockWaitcntProcessedSet . end ( ) , & MBB ) < Count ) ) { BlockWaitcntBracketsMap [ & MBB ] -> setRevisitLoop ( true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ContainingLoop -> getHeader ( ) -> getNumber ( ) << '\n' ; ) ; } } insertWaitcntInBlock ( MF , MBB ) ; BlockWaitcntProcessedSet . push_back ( & MBB ) ; if ( ContainingLoop && isLoopBottom ( ContainingLoop , & MBB ) && std :: count ( BlockWaitcntProcessedSet . begin ( ) , BlockWaitcntProcessedSet . end ( ) , & MBB ) == <NUM_LIT> ) { MachineBasicBlock * EntryBB = ContainingLoop -> getHeader ( ) ; BlockWaitcntBrackets * EntrySB = BlockWaitcntBracketsMap [ EntryBB ] . get ( ) ; if ( EntrySB && EntrySB -> getRevisitLoop ( ) ) { EntrySB -> setRevisitLoop ( false ) ; J = I ; int32_t PostOrder = EntrySB -> getPostOrder ( ) ; for ( ReversePostOrderTraversal < MachineFunction * > :: rpo_iterator X = RPOT . begin ( ) , Y = RPOT . end ( ) ; X != Y ; ++ X ) { MachineBasicBlock & MBBX = * * X ; if ( MBBX . getNumber ( ) == PostOrder ) { I = X ; break ; } } LoopWaitcntData * WaitcntData = LoopWaitcntDataMap [ ContainingLoop ] . get ( ) ; WaitcntData -> incIterCnt ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << EntryBB -> getNumber ( ) << '\n' ; ) ; continue ; } else { LoopWaitcntData * WaitcntData = LoopWaitcntDataMap [ ContainingLoop ] . get ( ) ; WaitcntData -> resetIterCnt ( ) ; } } J = I ; ++ I ; } SmallVector < MachineBasicBlock * , <NUM_LIT> > EndPgmBlocks ; bool HaveScalarStores = false ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { if ( ! HaveScalarStores && TII -> isScalarStore ( * I ) ) HaveScalarStores = true ; if ( I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) EndPgmBlocks . push_back ( & MBB ) ;


<s> RegInterval Interval = getRegInterval ( MI , TII , MRI , TRI , OpNo , false ) ; LLVM_DEBUG ( { const MachineOperand & Opnd = MI -> getOperand ( OpNo ) ;

assert ( TRI -> isVGPR ( * MRI , Opnd . getReg ( ) ) ) ;


<s> void BlockWaitcntBrackets :: setExpScore ( const MachineInstr * MI , const <ISA_LIT>InstrInfo * TII , const <ISA_LIT>RegisterInfo * TRI , const MachineRegisterInfo * MRI , unsigned OpNo , int32_t Val ) { RegInterval Interval = getRegInterval ( MI , TII , MRI , TRI , OpNo , false ) ;


<s> if ( TII -> hasModifiersSet ( Inst , <STR_LIT> :: gds ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_ACCESS , Inst ) ; ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_GPR_LOCK , Inst ) ; } else { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; } } else if ( TII -> isFLAT ( Inst ) ) { assert ( Inst . mayLoad ( ) || Inst . mayStore ( ) ) ; if ( TII -> usesVM_CNT ( Inst ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; if ( TII -> usesLGKM_CNT ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; if ( mayAccessLDSThroughFlat ( Inst ) ) ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; } else { switch ( Inst . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SQ_MESSAGE , Inst ) ; break ; case <STR_LIT> : case <STR_LIT> : { int Imm = TII -> getNamedOperand ( Inst , <STR_LIT> :: tgt ) -> getImm ( ) ; if ( Imm >= <NUM_LIT> && Imm <= <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_PARAM_ACCESS , Inst ) ; else if ( Imm >= <NUM_LIT> && Imm <= <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_POS_ACCESS , Inst ) ; else ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_GPR_LOCK , Inst ) ; break ; } case <STR_LIT> : case <STR_LIT> : ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; break ; default : break ;


<s> assert ( Inst . mayLoad ( ) || Inst . mayStore ( ) ) ; if ( TII -> usesVM_CNT ( Inst ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; if ( TII -> usesLGKM_CNT ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; if ( mayAccessLDSThroughFlat ( Inst ) ) ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; } else { switch ( Inst . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SQ_MESSAGE , Inst ) ; break ; case <STR_LIT> : case <STR_LIT> : { int Imm = TII -> getNamedOperand ( Inst , <STR_LIT> :: tgt ) -> getImm ( ) ; if ( Imm >= <NUM_LIT> && Imm <= <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_PARAM_ACCESS , Inst ) ; else if ( Imm >= <NUM_LIT> && Imm <= <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_POS_ACCESS , Inst ) ;

else ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_GPR_LOCK , Inst ) ;


<s> unsigned getIterCnt ( ) { return IterCnt ;


<s> BlockWaitcntBrackets * ScoreBrackets = BlockWaitcntBracketsMap [ & MBB ] . get ( ) ; if ( ! ScoreBrackets ) { BlockWaitcntBracketsMap [ & MBB ] = llvm :: make_unique < BlockWaitcntBrackets > ( ST ) ; ScoreBrackets = BlockWaitcntBracketsMap [ & MBB ] . get ( ) ; } ScoreBrackets -> setPostOrder ( MBB . getNumber ( ) ) ; MachineLoop * ContainingLoop = MLI -> getLoopFor ( & MBB ) ; if ( ContainingLoop && LoopWaitcntDataMap [ ContainingLoop ] == nullptr ) LoopWaitcntDataMap [ ContainingLoop ] = llvm :: make_unique < LoopWaitcntData > ( ) ; if ( ContainingLoop && ContainingLoop -> getHeader ( ) == & MBB ) { unsigned Count = countNumBottomBlocks ( ContainingLoop ) ; if ( ( std :: count ( BlockWaitcntProcessedSet . begin ( ) , BlockWaitcntProcessedSet . end ( ) , & MBB ) < ( int ) Count ) ) { BlockWaitcntBracketsMap [ & MBB ] -> setRevisitLoop ( true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ContainingLoop -> getHeader ( ) -> getNumber ( ) << '\n' ; ) ; } } insertWaitcntInBlock ( MF , MBB ) ; BlockWaitcntProcessedSet . push_back ( & MBB ) ; if ( ContainingLoop && isLoopBottom ( ContainingLoop , & MBB ) && std :: count ( BlockWaitcntProcessedSet . begin ( ) , BlockWaitcntProcessedSet . end ( ) , & MBB ) == <NUM_LIT> ) { MachineBasicBlock * EntryBB = ContainingLoop -> getHeader ( ) ; BlockWaitcntBrackets * EntrySB = BlockWaitcntBracketsMap [ EntryBB ] . get ( ) ; if ( EntrySB && EntrySB -> getRevisitLoop ( ) ) { EntrySB -> setRevisitLoop ( false ) ; J = I ; int32_t PostOrder = EntrySB -> getPostOrder ( ) ; for ( ReversePostOrderTraversal < MachineFunction * > :: rpo_iterator X = RPOT . begin ( ) , Y = RPOT . end ( ) ; X != Y ; ++ X ) { MachineBasicBlock & MBBX = * * X ; if ( MBBX . getNumber ( ) == PostOrder ) { I = X ; break ; } } LoopWaitcntData * WaitcntData = LoopWaitcntDataMap [ ContainingLoop ] . get ( ) ; WaitcntData -> incIterCnt ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << EntryBB -> getNumber ( ) << '\n' ; ) ; continue ; } else { LoopWaitcntData * WaitcntData = LoopWaitcntDataMap [ ContainingLoop ] . get ( ) ; WaitcntData -> resetIterCnt ( ) ; } } J = I ; ++ I ; } SmallVector < MachineBasicBlock * , <NUM_LIT> > EndPgmBlocks ; bool HaveScalarStores = false ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) {


<s> NeedWait = CNT_MASK ( T ) ; setScoreLB ( T , getScoreUB ( T ) ) ; return NeedWait ; } const int32_t LB = getScoreLB ( T ) ; const int32_t UB = getScoreUB ( T ) ;

if ( ( UB >= ScoreToWait ) && ( ScoreToWait > LB ) ) {


<s> assert ( Reg >= RegisterEncoding . SGPR0 && Reg < SQ_MAX_PGM_SGPRS ) ; Result . first = Reg - RegisterEncoding . SGPR0 + NUM_ALL_VGPRS ; assert ( Result . first >= NUM_ALL_VGPRS && Result . first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS ) ; } else return { - <NUM_LIT> , - <NUM_LIT> } ; const TargetRegisterClass * RC = TII -> getOpRegClass ( * MI , OpNo ) ; unsigned Size = TRI -> getRegSizeInBits ( * RC ) ; Result . second = Result . first + ( ( Size + <NUM_LIT> ) / <NUM_LIT> ) ; return Result ;


<s> } if ( TII -> usesLGKM_CNT ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; if ( mayAccessLDSThroughFlat ( Inst ) ) ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) || ( TII -> isMIMG ( Inst ) && ! Inst . mayLoad ( ) && ! Inst . mayStore ( ) ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else if ( Inst . mayStore ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) {

ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ;


<s> void <ISA_LIT>InsertWaitcnts :: updateEventWaitcntAfter ( MachineInstr & Inst , WaitcntBrackets * ScoreBrackets ) { if ( TII -> isDS ( Inst ) && TII -> usesLGKM_CNT ( Inst ) ) { if ( TII -> isAlwaysGDS ( Inst . getOpcode ( ) ) || TII -> hasModifiersSet ( Inst , <STR_LIT> :: gds ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_ACCESS , Inst ) ; ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_GPR_LOCK , Inst ) ; } else { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; } } else if ( TII -> isFLAT ( Inst ) ) { assert ( Inst . mayLoadOrStore ( ) ) ; if ( TII -> usesVM_CNT ( Inst ) ) { if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; } if ( TII -> usesLGKM_CNT ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; if ( mayAccessLDSThroughFlat ( Inst ) ) ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) || ( TII -> isMIMG ( Inst ) && ! Inst . mayLoad ( ) && ! Inst . mayStore ( ) ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else if ( Inst . mayStore ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ;


<s> assert ( II -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) ; if ( ! TrackedWaitcntSet . count ( & * II ) ) { unsigned OldVSCnt = TII -> getNamedOperand ( * II , <STR_LIT> :: simm16 ) -> getImm ( ) ; Wait . VsCnt = std :: min ( Wait . VsCnt , OldVSCnt ) ; } if ( ! WaitcntVsCntInstr ) { WaitcntVsCntInstr = & * II ; } else { II -> eraseFromParent ( ) ; Modified = true ; } } } if ( WaitcntInstr ) { if ( Wait . hasWaitExceptVsCnt ( ) ) { unsigned NewEnc = <STR_LIT> ( IV , Wait ) ; unsigned OldEnc = WaitcntInstr -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( OldEnc != NewEnc ) { WaitcntInstr -> getOperand ( <NUM_LIT> ) . setImm ( NewEnc ) ; Modified = true ; } ScoreBrackets . applyWaitcnt ( Wait ) ; Wait . VmCnt = ~ <NUM_LIT> ; Wait . LgkmCnt = ~ <NUM_LIT> ; Wait . ExpCnt = ~ <NUM_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << * MI << <STR_LIT> << * WaitcntInstr << '\n' ) ; } else { WaitcntInstr -> eraseFromParent ( ) ; Modified = true ; } } if ( WaitcntVsCntInstr ) { if ( Wait . hasWaitVsCnt ( ) ) { assert ( ST -> hasVscnt ( ) ) ; unsigned OldVSCnt = TII -> getNamedOperand ( * WaitcntVsCntInstr , <STR_LIT> :: simm16 ) -> getImm ( ) ; if ( Wait . VsCnt != OldVSCnt ) { TII -> getNamedOperand ( * WaitcntVsCntInstr , <STR_LIT> :: simm16 ) -> setImm ( Wait . VsCnt ) ; Modified = true ; } ScoreBrackets . applyWaitcnt ( Wait ) ; Wait . VsCnt = ~ <NUM_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << * MI << <STR_LIT> << * WaitcntVsCntInstr << '\n' ) ; } else { WaitcntVsCntInstr -> eraseFromParent ( ) ; Modified = true ; } } return Modified ;


<s> } if ( WaitcntInstr ) { if ( Wait . hasWaitExceptVsCnt ( ) ) { unsigned NewEnc = <STR_LIT> ( IV , Wait ) ; unsigned OldEnc = WaitcntInstr -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( OldEnc != NewEnc ) { WaitcntInstr -> getOperand ( <NUM_LIT> ) . setImm ( NewEnc ) ; Modified = true ; } ScoreBrackets . applyWaitcnt ( Wait ) ; Wait . VmCnt = ~ <NUM_LIT> ; Wait . LgkmCnt = ~ <NUM_LIT> ; Wait . ExpCnt = ~ <NUM_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << * MI << <STR_LIT> << * WaitcntInstr << '\n' ) ; } else { WaitcntInstr -> eraseFromParent ( ) ; Modified = true ; } } if ( WaitcntVsCntInstr ) { if ( Wait . hasWaitVsCnt ( ) ) { assert ( ST -> hasVscnt ( ) ) ; unsigned OldVSCnt = TII -> getNamedOperand ( * WaitcntVsCntInstr , <STR_LIT> :: simm16 ) -> getImm ( ) ; if ( Wait . VsCnt != OldVSCnt ) { TII -> getNamedOperand ( * WaitcntVsCntInstr , <STR_LIT> :: simm16 ) -> setImm ( Wait . VsCnt ) ; Modified = true ; } ScoreBrackets . applyWaitcnt ( Wait ) ; Wait . VsCnt = ~ <NUM_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << * MI << <STR_LIT> << * WaitcntVsCntInstr << '\n' ) ; } else { WaitcntVsCntInstr -> eraseFromParent ( ) ; Modified = true ; } } return Modified ;


<s> RegInterval WaitcntBrackets :: getRegInterval ( const MachineInstr * MI , const <ISA_LIT>InstrInfo * TII , const MachineRegisterInfo * MRI , const <ISA_LIT>RegisterInfo * TRI , unsigned OpNo ) const { const MachineOperand & Op = MI -> getOperand ( OpNo ) ; if ( ! TRI -> isInAllocatableClass ( Op . getReg ( ) ) ) return { - <NUM_LIT> , - <NUM_LIT> } ; assert ( ! Op . getSubReg ( ) || ! Op . isUndef ( ) ) ; RegInterval Result ; unsigned Reg = TRI -> getEncodingValue ( <STR_LIT> ( Op . getReg ( ) , * ST ) ) ; if ( TRI -> isVectorRegister ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= Encoding . VGPR0 && Reg <= Encoding . VGPRL ) ; Result . first = Reg - Encoding . VGPR0 ; if ( TRI -> isAGPR ( * MRI , Op . getReg ( ) ) ) Result . first += AGPR_OFFSET ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= Encoding . SGPR0 && Reg < SQ_MAX_PGM_SGPRS ) ; Result . first = Reg - Encoding . SGPR0 + NUM_ALL_VGPRS ; assert ( Result . first >= NUM_ALL_VGPRS && Result . first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS ) ; } else return { - <NUM_LIT> , - <NUM_LIT> } ; const TargetRegisterClass * RC = TII -> getOpRegClass ( * MI , OpNo ) ;

unsigned Size = TRI -> getRegSizeInBits ( * RC ) ;


<s> assert ( ! Op . getSubReg ( ) || ! Op . isUndef ( ) ) ; RegInterval Result ; unsigned Reg = TRI -> getEncodingValue ( <STR_LIT> ( Op . getReg ( ) , * ST ) ) ; if ( TRI -> isVectorRegister ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= Encoding . VGPR0 && Reg <= Encoding . VGPRL ) ; Result . first = Reg - Encoding . VGPR0 ; if ( TRI -> isAGPR ( * MRI , Op . getReg ( ) ) ) Result . first += AGPR_OFFSET ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ;

} else if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) {


<s> const <STR_LIT> * Info = <STR_LIT> ( Inst . getOpcode ( ) ) ;

const <STR_LIT> * BaseInfo = <STR_LIT> ( Info -> BaseOpcode ) ;


<s> if ( ! <ISA_LIT>InstrInfo :: isMIMG ( Inst ) ) return VMEM_NOSAMPLER ; const <STR_LIT> * Info = <STR_LIT> ( Inst . getOpcode ( ) ) ; const <STR_LIT> * BaseInfo = <STR_LIT> ( Info -> BaseOpcode ) ; return BaseInfo -> BVH ? VMEM_BVH : BaseInfo -> Sampler ? VMEM_SAMPLER : VMEM_NOSAMPLER ;


<s> case VM_CNT : return Limits . VmcntMax ; case LGKM_CNT : return Limits . LgkmcntMax ; case EXP_CNT : return Limits . ExpcntMax ; case VS_CNT : return Limits . VscntMax ; default : break ; } return <NUM_LIT> ;


<s> if ( ST -> isTgSplitEnabled ( ) ) return false ; if ( MI . memoperands_empty ( ) ) return true ;

for ( const MachineMemOperand * Memop : MI . memoperands ( ) ) {


<s> if ( ! TII -> usesLGKM_CNT ( MI ) ) return false ; if ( ST -> isTgSplitEnabled ( ) ) return false ; if ( MI . memoperands_empty ( ) ) return true ; for ( const MachineMemOperand * Memop : MI . memoperands ( ) ) { unsigned AS = Memop -> getAddrSpace ( ) ; if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) return true ; } return false ;


<s> RegStrictDom |= mergeScore ( M , VgprScores [ T ] [ J ] , Other . VgprScores [ T ] [ J ] ) ; } if ( T == VM_CNT ) { for ( int J = <NUM_LIT> ; J <= VgprUB ; J ++ ) { unsigned char NewVmemTypes = VgprVmemTypes [ J ] | Other . VgprVmemTypes [ J ] ; RegStrictDom |= NewVmemTypes != VgprVmemTypes [ J ] ; VgprVmemTypes [ J ] = NewVmemTypes ; } } if ( T == LGKM_CNT ) { for ( int J = <NUM_LIT> ; J <= SgprUB ; J ++ ) { RegStrictDom |= mergeScore ( M , SgprScores [ J ] , Other . SgprScores [ J ] ) ;


<s> if ( NewUB < ScoreLBs [ T ] ) report_fatal_error ( <STR_LIT> ) ; MergeInfo M ; M . OldLB = ScoreLBs [ T ] ; M . OtherLB = Other . ScoreLBs [ T ] ; M . MyShift = NewUB - ScoreUBs [ T ] ; M . OtherShift = NewUB - Other . ScoreUBs [ T ] ; ScoreUBs [ T ] = NewUB ; StrictDom |= mergeScore ( M , LastFlat [ T ] , Other . LastFlat [ T ] ) ; bool RegStrictDom = false ; for ( int J = <NUM_LIT> ; J <= VgprUB ; J ++ ) { RegStrictDom |= mergeScore ( M , VgprScores [ T ] [ J ] , Other . VgprScores [ T ] [ J ] ) ; } if ( T == VM_CNT ) { for ( int J = <NUM_LIT> ; J <= VgprUB ; J ++ ) { unsigned char NewVmemTypes = VgprVmemTypes [ J ] | Other . VgprVmemTypes [ J ] ; RegStrictDom |= NewVmemTypes != VgprVmemTypes [ J ] ; VgprVmemTypes [ J ] = NewVmemTypes ; } } if ( T == LGKM_CNT ) { for ( int J = <NUM_LIT> ; J <= SgprUB ; J ++ ) { RegStrictDom |= mergeScore ( M , SgprScores [ J ] , Other . SgprScores [ J ] ) ; } } if ( RegStrictDom ) StrictDom = true ; }


<s> Limits . VscntMax = ST -> hasVscnt ( ) ? <NUM_LIT> : <NUM_LIT> ; unsigned NumVGPRsMax = ST -> getAddressableNumVGPRs ( ) ; unsigned NumSGPRsMax = ST -> getAddressableNumSGPRs ( ) ; assert ( NumVGPRsMax <= SQ_MAX_PGM_VGPRS ) ; assert ( NumSGPRsMax <= SQ_MAX_PGM_SGPRS ) ; RegisterEncoding Encoding = { } ; Encoding . VGPR0 = TRI -> getEncodingValue ( <STR_LIT> ) ; Encoding . VGPRL = Encoding . VGPR0 + NumVGPRsMax - <NUM_LIT> ; Encoding . SGPR0 = TRI -> getEncodingValue ( <STR_LIT> ) ; Encoding . SGPRL = Encoding . SGPR0 + NumSGPRsMax - <NUM_LIT> ; TrackedWaitcntSet . clear ( ) ; BlockInfos . clear ( ) ; bool Modified = false ; if ( ! MFI -> isEntryFunction ( ) ) { MachineBasicBlock & EntryBB = MF . front ( ) ; MachineBasicBlock :: iterator I = EntryBB . begin ( ) ; for ( MachineBasicBlock :: iterator E = EntryBB . end ( ) ; I != E && ( I -> isPHI ( ) || I -> isMetaInstruction ( ) ) ; ++ I ) ; BuildMI ( EntryBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; if ( ST -> hasVscnt ( ) ) BuildMI ( EntryBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) ; Modified = true ; } for ( auto * MBB : ReversePostOrderTraversal < MachineFunction * > ( & MF ) ) BlockInfos . insert ( { MBB , BlockInfo ( MBB ) } ) ; std :: unique_ptr < WaitcntBrackets > Brackets ; bool Repeat ; do { Repeat = false ; for ( auto BII = BlockInfos . begin ( ) , BIE = BlockInfos . end ( ) ; BII != BIE ; ++ BII ) { BlockInfo & BI = BII -> second ; if ( ! BI . Dirty ) continue ; if ( BI . Incoming ) { if ( ! Brackets ) Brackets = std :: make_unique < WaitcntBrackets > ( * BI . Incoming ) ; else * Brackets = * BI . Incoming ; } else { if ( ! Brackets ) Brackets = std :: make_unique < WaitcntBrackets > ( ST , Limits , Encoding ) ; else * Brackets = WaitcntBrackets ( ST , Limits , Encoding ) ; } Modified |= insertWaitcntInBlock ( MF , * BI . MBB , * Brackets ) ; BI . Dirty = false ; if ( Brackets -> hasPending ( ) ) { BlockInfo * MoveBracketsToSucc = nullptr ; for ( MachineBasicBlock * Succ : BI . MBB -> successors ( ) ) { auto SuccBII = BlockInfos . find ( Succ ) ;

BlockInfo & SuccBI = SuccBII -> second ;


<s> void WaitcntBrackets :: setExpScore ( const MachineInstr * MI , const <ISA_LIT>InstrInfo * TII , const <ISA_LIT>RegisterInfo * TRI , const MachineRegisterInfo * MRI , unsigned OpNo , unsigned Val ) {

RegInterval Interval = getRegInterval ( MI , TII , MRI , TRI , OpNo ) ;


<s> const unsigned UB = getScoreUB ( T ) ; if ( Count >= UB - LB ) Count = ~ <NUM_LIT> ;


<s> if ( TII -> isDS ( Inst ) && TII -> usesLGKM_CNT ( Inst ) ) { if ( TII -> isAlwaysGDS ( Inst . getOpcode ( ) ) || TII -> hasModifiersSet ( Inst , <STR_LIT> :: gds ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_ACCESS , Inst ) ; ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_GPR_LOCK , Inst ) ; } else { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; } } else if ( TII -> isFLAT ( Inst ) ) { assert ( Inst . mayLoadOrStore ( ) ) ; int FlatASCount = <NUM_LIT> ; if ( mayAccessVMEMThroughFlat ( Inst ) ) { ++ FlatASCount ; if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( Inst . mayLoad ( ) && ! <ISA_LIT>InstrInfo :: isAtomicNoRet ( Inst ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; } if ( mayAccessLDSThroughFlat ( Inst ) ) { ++ FlatASCount ; ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; } assert ( FlatASCount ) ;

if ( FlatASCount > <NUM_LIT> ) ScoreBrackets -> setPendingFlat ( ) ;


<s> WaitcntBrackets ( const GCNSubtarget * SubTarget , HardwareLimits Limits , RegisterEncoding Encoding ) : ST ( SubTarget ) , Limits ( Limits ) , Encoding ( Encoding ) {


<s> if ( FlatASCount > <NUM_LIT> ) ScoreBrackets -> setPendingFlat ( ) ; } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) || ( TII -> isMIMG ( Inst ) && ! Inst . mayLoad ( ) && ! Inst . mayStore ( ) ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else if ( Inst . mayStore ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; } else if ( Inst . isCall ( ) ) { if ( callWaitsOnFunctionReturn ( Inst ) ) { ScoreBrackets -> applyWaitcnt ( <STR_LIT> :: allZero ( ST -> hasVscnt ( ) ) ) ; } else { ScoreBrackets -> applyWaitcnt ( <STR_LIT> ( ) ) ; }

} else if ( <ISA_LIT>InstrInfo :: isEXP ( Inst ) ) {


<s> return HardwareLimits . ExpcntMax ; case VS_CNT : return HardwareLimits . VscntMax ; default :


<s> ScoreUBs [ T ] = NewUB ; ScoreLBs [ T ] = std :: min ( M . OldLB + M . MyShift , M . OtherLB + M . OtherShift ) ; StrictDom |= mergeScore ( M , LastFlat [ T ] , Other . LastFlat [ T ] ) ; bool RegStrictDom = false ; for ( int J = <NUM_LIT> , E = std :: max ( getMaxVGPR ( ) , Other . getMaxVGPR ( ) ) + <NUM_LIT> ; J != E ; J ++ ) { RegStrictDom |= mergeScore ( M , VgprScores [ T ] [ J ] , Other . VgprScores [ T ] [ J ] ) ; } if ( T == LGKM_CNT ) { for ( int J = <NUM_LIT> , E = std :: max ( getMaxSGPR ( ) , Other . getMaxSGPR ( ) ) + <NUM_LIT> ; J != E ; J ++ ) { RegStrictDom |= mergeScore ( M , SgprScores [ J ] , Other . SgprScores [ J ] ) ; } } if ( RegStrictDom && ! OldOutOfOrder ) StrictDom = true ;


<s> if ( NewUB < ScoreUBs [ T ] ) report_fatal_error ( <STR_LIT> ) ; ScoreUBs [ T ] = NewUB ; ScoreLBs [ T ] = std :: min ( M . OldLB + M . MyShift , M . OtherLB + M . OtherShift ) ; StrictDom |= mergeScore ( M , LastFlat [ T ] , Other . LastFlat [ T ] ) ; bool RegStrictDom = false ; for ( int J = <NUM_LIT> , E = std :: max ( getMaxVGPR ( ) , Other . getMaxVGPR ( ) ) + <NUM_LIT> ; J != E ; J ++ ) { RegStrictDom |= mergeScore ( M , VgprScores [ T ] [ J ] , Other . VgprScores [ T ] [ J ] ) ; } if ( T == LGKM_CNT ) { for ( int J = <NUM_LIT> , E = std :: max ( getMaxSGPR ( ) , Other . getMaxSGPR ( ) ) + <NUM_LIT> ; J != E ; J ++ ) { RegStrictDom |= mergeScore ( M , SgprScores [ J ] , Other . SgprScores [ J ] ) ; }


<s> uint32_t LB = getScoreLB ( T ) ; uint32_t UB = getScoreUB ( T ) ; switch ( T ) { case VM_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; case LGKM_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; case EXP_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; case VS_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; default : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; } if ( LB < UB ) { for ( int J = <NUM_LIT> ; J <= getMaxVGPR ( ) ; J ++ ) { uint32_t RegScore = getRegScore ( J , T ) ; if ( RegScore <= LB ) continue ; uint32_t RelScore = RegScore - LB - <NUM_LIT> ; if ( J < SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS ) { OS << RelScore << <STR_LIT> << J << <STR_LIT> ; } else { OS << RelScore << <STR_LIT> ; } } if ( T == LGKM_CNT ) { for ( int J = <NUM_LIT> ; J <= getMaxSGPR ( ) ; J ++ ) { uint32_t RegScore = getRegScore ( J + NUM_ALL_VGPRS , LGKM_CNT ) ; if ( RegScore <= LB ) continue ; uint32_t RelScore = RegScore - LB - <NUM_LIT> ; OS << RelScore << <STR_LIT> << J << <STR_LIT> ; } } } OS << '\n' ; } OS << '\n' ;


<s> Wait . LgkmCnt = std :: min ( Wait . LgkmCnt , Count ) ; break ; case VS_CNT : Wait . VsCnt = std :: min ( Wait . VsCnt , Count ) ; break ; default : llvm_unreachable ( <STR_LIT> ) ; }


<s> if ( Count >= UB ) return ; if ( Count != <NUM_LIT> ) { if ( counterOutOfOrder ( T ) ) return ; setScoreLB ( T , std :: max ( getScoreLB ( T ) , UB - Count ) ) ; } else {


<s> static bool callWaitsOnFunctionReturn ( const MachineInstr & MI ) { return true ;


<s> assert ( GprNo < NUM_ALL_VGPRS ) ; VgprVmemTypes [ GprNo ] = <NUM_LIT> ;


<s> if ( T == LGKM_CNT && hasPendingEvent ( SMEM_ACCESS ) ) return true ;


<s> const unsigned UB = getScoreUB ( T ) ; if ( ( UB >= ScoreToWait ) && ( ScoreToWait > LB ) ) { if ( ( T == VM_CNT || T == LGKM_CNT ) && hasPendingFlat ( ) && ! ST -> hasFlatLgkmVMemCountInOrder ( ) ) {


<s> if ( WaitEventMaskForInst [ VS_CNT ] & ( <NUM_LIT> << E ) ) return VS_CNT ; assert ( WaitEventMaskForInst [ EXP_CNT ] & ( <NUM_LIT> << E ) ) ;


<s> if ( WaitEventMaskForInst [ LGKM_CNT ] & ( <NUM_LIT> << E ) ) return LGKM_CNT ; if ( WaitEventMaskForInst [ VS_CNT ] & ( <NUM_LIT> << E ) ) return VS_CNT ;

assert ( WaitEventMaskForInst [ EXP_CNT ] & ( <NUM_LIT> << E ) ) ;


<s> AU . addRequired < MachinePostDominatorTree > ( ) ;

MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> if ( TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . SGPR0 && Reg < SQ_MAX_PGM_SGPRS ) ;

Result . first = Reg - RegisterEncoding . SGPR0 + NUM_ALL_VGPRS ;


<s> assert ( ! Op . getSubReg ( ) || ! Op . isUndef ( ) ) ; RegInterval Result ; unsigned Reg = TRI -> getEncodingValue ( <STR_LIT> ( Op . getReg ( ) , * ST ) ) ; if ( TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . SGPR0 && Reg < SQ_MAX_PGM_SGPRS ) ;


<s> return VgprScores [ T ] [ GprNo ] ; } assert ( T == LGKM_CNT ) ;

return SgprScores [ GprNo - NUM_ALL_VGPRS ] ;


<s> unsigned getScoreUB ( InstCounterType T ) const { assert ( T < NUM_INST_CNTS ) ; return ScoreUBs [ T ] ;


<s> const <STR_LIT> * Info = <STR_LIT> ( Inst . getOpcode ( ) ) ;

return <STR_LIT> ( Info -> BaseOpcode ) -> Sampler ? VMEM_SAMPLER : VMEM_NOSAMPLER ;


<s> VmemType getVmemType ( const MachineInstr & Inst ) { assert ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) ) ; if ( ! <ISA_LIT>InstrInfo :: isMIMG ( Inst ) ) return VMEM_NOSAMPLER ; const <STR_LIT> * Info = <STR_LIT> ( Inst . getOpcode ( ) ) ;


<s> return HardwareLimits . VmcntMax ; case LGKM_CNT : return HardwareLimits . LgkmcntMax ; case EXP_CNT : return HardwareLimits . ExpcntMax ; case VS_CNT : return HardwareLimits . VscntMax ; default : break ; } return <NUM_LIT> ;


<s> return HardwareLimits . ExpcntMax ; case VS_CNT : return HardwareLimits . VscntMax ; default : break ;


<s> return Events & ( Events - <NUM_LIT> ) ;


<s> bool <ISA_LIT>InsertWaitcnts :: mayAccessLDSThroughFlat ( const MachineInstr & MI ) const { assert ( TII -> isFLAT ( MI ) ) ;

if ( ! TII -> usesLGKM_CNT ( MI ) ) return false ;


<s> if ( MI . memoperands_empty ( ) ) return true ; for ( const MachineMemOperand * Memop : MI . memoperands ( ) ) { unsigned AS = Memop -> getAddrSpace ( ) ; if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) return true ;


<s> bool <ISA_LIT>InsertWaitcnts :: mayAccessVMEMThroughFlat ( const MachineInstr & MI ) const { assert ( TII -> isFLAT ( MI ) ) ; assert ( TII -> usesVM_CNT ( MI ) ) ; if ( MI . memoperands_empty ( ) ) return true ; for ( const MachineMemOperand * Memop : MI . memoperands ( ) ) {

unsigned AS = Memop -> getAddrSpace ( ) ;


<s> for ( const MachineMemOperand * Memop : MI . memoperands ( ) ) { unsigned AS = Memop -> getAddrSpace ( ) ; assert ( AS != <ISA_LIT>AS :: REGION_ADDRESS ) ; if ( AS != <ISA_LIT>AS :: LOCAL_ADDRESS ) return true ; }


<s> M . OldLB = ScoreLBs [ T ] ; M . OtherLB = Other . ScoreLBs [ T ] ; M . MyShift = NewUB - ScoreUBs [ T ] ; M . OtherShift = NewUB - Other . ScoreUBs [ T ] ; ScoreUBs [ T ] = NewUB ; StrictDom |= mergeScore ( M , LastFlat [ T ] , Other . LastFlat [ T ] ) ; bool RegStrictDom = false ; for ( int J = <NUM_LIT> ; J <= VgprUB ; J ++ ) { RegStrictDom |= mergeScore ( M , VgprScores [ T ] [ J ] , Other . VgprScores [ T ] [ J ] ) ; } if ( T == VM_CNT ) { for ( int J = <NUM_LIT> ; J <= VgprUB ; J ++ ) { unsigned char NewVmemTypes = VgprVmemTypes [ J ] | Other . VgprVmemTypes [ J ] ; RegStrictDom |= NewVmemTypes != VgprVmemTypes [ J ] ; VgprVmemTypes [ J ] = NewVmemTypes ; }


<s> PendingEvents |= OtherEvents ; const unsigned MyPending = ScoreUBs [ T ] - ScoreLBs [ T ] ; const unsigned OtherPending = Other . ScoreUBs [ T ] - Other . ScoreLBs [ T ] ; const unsigned NewUB = ScoreLBs [ T ] + std :: max ( MyPending , OtherPending ) ; if ( NewUB < ScoreLBs [ T ] ) report_fatal_error ( <STR_LIT> ) ; MergeInfo M ; M . OldLB = ScoreLBs [ T ] ; M . OtherLB = Other . ScoreLBs [ T ] ; M . MyShift = NewUB - ScoreUBs [ T ] ; M . OtherShift = NewUB - Other . ScoreUBs [ T ] ; ScoreUBs [ T ] = NewUB ; StrictDom |= mergeScore ( M , LastFlat [ T ] , Other . LastFlat [ T ] ) ;


<s> unsigned OtherShifted = OtherScore <= M . OtherLB ? <NUM_LIT> : OtherScore + M . OtherShift ; Score = std :: max ( MyShifted , OtherShifted ) ; return OtherShifted > MyShifted ;


<s> for ( auto T : inst_counter_types ( ) ) { unsigned LB = getScoreLB ( T ) ; unsigned UB = getScoreUB ( T ) ; switch ( T ) { case VM_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; case LGKM_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; case EXP_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; case VS_CNT : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; default : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; } if ( LB < UB ) { for ( int J = <NUM_LIT> ; J <= VgprUB ; J ++ ) { unsigned RegScore = getRegScore ( J , T ) ; if ( RegScore <= LB ) continue ; unsigned RelScore = RegScore - LB - <NUM_LIT> ; if ( J < SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS ) { OS << RelScore << <STR_LIT> << J << <STR_LIT> ;


<s> TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; IV = <STR_LIT> ( ST -> getCPU ( ) ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; PDT = & getAnalysis < MachinePostDominatorTree > ( ) ; ForceEmitZeroWaitcnts = ForceEmitZeroFlag ; for ( auto T : inst_counter_types ( ) ) ForceEmitWaitcnt [ T ] = false ; HardwareLimits . VmcntMax = <STR_LIT> ( IV ) ; HardwareLimits . ExpcntMax = <STR_LIT> ( IV ) ; HardwareLimits . LgkmcntMax = <STR_LIT> ( IV ) ; HardwareLimits . VscntMax = ST -> hasVscnt ( ) ? <NUM_LIT> : <NUM_LIT> ; unsigned NumVGPRsMax = ST -> getAddressableNumVGPRs ( ) ; unsigned NumSGPRsMax = ST -> getAddressableNumSGPRs ( ) ; assert ( NumVGPRsMax <= SQ_MAX_PGM_VGPRS ) ; assert ( NumSGPRsMax <= SQ_MAX_PGM_SGPRS ) ; RegisterEncoding . VGPR0 = TRI -> getEncodingValue ( <STR_LIT> ) ; RegisterEncoding . VGPRL = RegisterEncoding . VGPR0 + NumVGPRsMax - <NUM_LIT> ; RegisterEncoding . SGPR0 = TRI -> getEncodingValue ( <STR_LIT> ) ; RegisterEncoding . SGPRL = RegisterEncoding . SGPR0 + NumSGPRsMax - <NUM_LIT> ; TrackedWaitcntSet . clear ( ) ; BlockInfos . clear ( ) ; for ( auto * MBB : ReversePostOrderTraversal < MachineFunction * > ( & MF ) ) BlockInfos . insert ( { MBB , BlockInfo ( MBB ) } ) ; std :: unique_ptr < WaitcntBrackets > Brackets ; bool Modified = false ; bool Repeat ; do { Repeat = false ; for ( auto BII = BlockInfos . begin ( ) , BIE = BlockInfos . end ( ) ; BII != BIE ; ++ BII ) { BlockInfo & BI = BII -> second ; if ( ! BI . Dirty ) continue ; if ( BI . Incoming ) { if ( ! Brackets ) Brackets = std :: make_unique < WaitcntBrackets > ( * BI . Incoming ) ; else * Brackets = * BI . Incoming ; } else { if ( ! Brackets ) Brackets = std :: make_unique < WaitcntBrackets > ( ST ) ; else * Brackets = WaitcntBrackets ( ST ) ; } Modified |= insertWaitcntInBlock ( MF , * BI . MBB , * Brackets ) ; BI . Dirty = false ; if ( Brackets -> hasPending ( ) ) { BlockInfo * MoveBracketsToSucc = nullptr ; for ( MachineBasicBlock * Succ : BI . MBB -> successors ( ) ) { auto SuccBII = BlockInfos . find ( Succ ) ; BlockInfo & SuccBI = SuccBII -> second ; if ( ! SuccBI . Incoming ) { SuccBI . Dirty = true ; if ( SuccBII <= BII ) Repeat = true ; if ( ! MoveBracketsToSucc ) { MoveBracketsToSucc = & SuccBI ; } else { SuccBI . Incoming = std :: make_unique < WaitcntBrackets > ( * Brackets ) ; } } else if ( SuccBI . Incoming -> merge ( * Brackets ) ) {


<s> RegInterval Interval = getRegInterval ( MI , TII , MRI , TRI , OpNo ) ; assert ( TRI -> isVGPR ( * MRI , MI -> getOperand ( OpNo ) . getReg ( ) ) ) ; for ( int RegNo = Interval . first ; RegNo < Interval . second ; ++ RegNo ) { setRegScore ( RegNo , EXP_CNT , Val ) ;


<s> SgprUB = std :: max ( SgprUB , GprNo - NUM_ALL_VGPRS ) ; SgprScores [ GprNo - NUM_ALL_VGPRS ] = Val ; }


<s> void setScoreLB ( InstCounterType T , unsigned Val ) {


<s> if ( T == EXP_CNT ) { unsigned UB = ScoreUBs [ T ] - getWaitCountMax ( EXP_CNT ) ; if ( ScoreLBs [ T ] < UB && UB < ScoreUBs [ T ] ) ScoreLBs [ T ] = UB ;


<s> unsigned UB = ScoreUBs [ T ] - getWaitCountMax ( EXP_CNT ) ;

if ( ScoreLBs [ T ] < UB && UB < ScoreUBs [ T ] ) ScoreLBs [ T ] = UB ;


<s> if ( Count < UB && UB - Count > LB ) return true ; Count = ~ <NUM_LIT> ; return false ;


<s> if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) || ( TII -> isMIMG ( Inst ) && ! Inst . mayLoad ( ) && ! Inst . mayStore ( ) ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else if ( Inst . mayStore ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; } else if ( Inst . isCall ( ) ) { if ( callWaitsOnFunctionReturn ( Inst ) ) { ScoreBrackets -> applyWaitcnt ( <STR_LIT> :: allZero ( ST -> hasVscnt ( ) ) ) ; } else { ScoreBrackets -> applyWaitcnt ( <STR_LIT> ( ) ) ; } } else { switch ( Inst . getOpcode ( ) ) { case <STR_LIT> :


<s> WaitcntBrackets ( const GCNSubtarget * SubTarget ) : ST ( SubTarget ) {


<s> BlockWaitcntBrackets ( ) : WaitAtBeginning ( false ) , RevisitLoop ( false ) , ValidLoop ( false ) , MixedExpTypes ( false ) , LoopRegion ( NULL ) , PostOrder ( <NUM_LIT> ) , Waitcnt ( NULL ) , VgprUB ( <NUM_LIT> ) , SgprUB ( <NUM_LIT> ) {

for ( enum InstCounterType T = VM_CNT ; T < NUM_INST_CNTS ;


<s> for ( enum InstCounterType T = VM_CNT ; T < NUM_INST_CNTS ; T = ( enum InstCounterType ) ( T + <NUM_LIT> ) ) {

memset ( VgprScores [ T ] , <NUM_LIT> , sizeof ( VgprScores [ T ] ) ) ;


<s> static bool mayWriteLDSThroughDMA ( const MachineInstr & MI ) {

return <ISA_LIT>InstrInfo :: isVALU ( MI ) && ( <ISA_LIT>InstrInfo :: isMUBUF ( MI ) || <ISA_LIT>InstrInfo :: isFLAT ( MI ) ) && MI . getOpcode ( ) != <STR_LIT> ;


<s> return <ISA_LIT>InstrInfo :: isVALU ( MI ) && ( <ISA_LIT>InstrInfo :: isMUBUF ( MI ) || <ISA_LIT>InstrInfo :: isFLAT ( MI ) ) && MI . getOpcode ( ) != <STR_LIT> ;


<s> setExpScore ( & Inst , TII , TRI , MRI , AddrOpIdx , CurrScore ) ; } if ( Inst . mayStore ( ) ) { if ( <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data0 ) != - <NUM_LIT> ) { setExpScore ( & Inst , TII , TRI , MRI , <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data0 ) , CurrScore ) ; } if ( <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data1 ) != - <NUM_LIT> ) { setExpScore ( & Inst , TII , TRI , MRI , <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data1 ) , CurrScore ) ; } } else if ( <ISA_LIT>InstrInfo :: isAtomicRet ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { for ( unsigned I = <NUM_LIT> , E = Inst . getNumOperands ( ) ; I != E ; ++ I ) { const MachineOperand & Op = Inst . getOperand ( I ) ; if ( Op . isReg ( ) && ! Op . isDef ( ) && TRI -> isVectorRegister ( * MRI , Op . getReg ( ) ) ) { setExpScore ( & Inst , TII , TRI , MRI , I , CurrScore ) ; } } } } else if ( TII -> isFLAT ( Inst ) ) { if ( Inst . mayStore ( ) ) { setExpScore ( & Inst , TII , TRI , MRI , <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data ) , CurrScore ) ; } else if ( <ISA_LIT>InstrInfo :: isAtomicRet ( Inst ) ) { setExpScore ( & Inst , TII , TRI , MRI , <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data ) , CurrScore ) ; } } else if ( TII -> isMIMG ( Inst ) ) { if ( Inst . mayStore ( ) ) { setExpScore ( & Inst , TII , TRI , MRI , <NUM_LIT> , CurrScore ) ; } else if ( <ISA_LIT>InstrInfo :: isAtomicRet ( Inst ) ) { setExpScore ( & Inst , TII , TRI , MRI , <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data ) , CurrScore ) ; } } else if ( TII -> isMTBUF ( Inst ) ) { if ( Inst . mayStore ( ) ) { setExpScore ( & Inst , TII , TRI , MRI , <NUM_LIT> , CurrScore ) ; } } else if ( TII -> isMUBUF ( Inst ) ) {


<s> if ( ( T == VM_CNT || T == LGKM_CNT ) && hasPendingFlat ( ) && ! ST -> hasFlatLgkmVMemCountInOrder ( ) ) { addWait ( Wait , T , <NUM_LIT> ) ; } else if ( counterOutOfOrder ( T ) ) {


<s> const uint32_t UB = getScoreUB ( T ) ; if ( ( UB >= ScoreToWait ) && ( ScoreToWait > LB ) ) { if ( ( T == VM_CNT || T == LGKM_CNT ) && hasPendingFlat ( ) && ! ST -> hasFlatLgkmVMemCountInOrder ( ) ) { addWait ( Wait , T , <NUM_LIT> ) ; } else if ( counterOutOfOrder ( T ) ) { addWait ( Wait , T , <NUM_LIT> ) ;


<s> const MachineRegisterInfo & MRIA = * MRI ; unsigned Reg = TRI -> getEncodingValue ( Op . getReg ( ) ) ; if ( TRI -> isVGPR ( MRIA , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( MRIA , Op . getReg ( ) ) ) {

assert ( Reg >= RegisterEncoding . SGPR0 && Reg < SQ_MAX_PGM_SGPRS ) ;


<s> BlockInfos . clear ( ) ; for ( MachineBasicBlock * MBB : ReversePostOrderTraversal < MachineFunction * > ( & MF ) ) { RpotIdxMap [ MBB ] = BlockInfos . size ( ) ; BlockInfos . emplace_back ( MBB ) ; } std :: unique_ptr < WaitcntBrackets > Brackets ; bool Modified = false ; bool Repeat ; do { Repeat = false ; for ( BlockInfo & BI : BlockInfos ) { if ( ! BI . Dirty ) continue ; unsigned Idx = std :: distance ( & * BlockInfos . begin ( ) , & BI ) ; if ( BI . Incoming ) { if ( ! Brackets ) Brackets = std :: make_unique < WaitcntBrackets > ( * BI . Incoming ) ; else * Brackets = * BI . Incoming ; } else { if ( ! Brackets ) Brackets = std :: make_unique < WaitcntBrackets > ( ST ) ; else Brackets -> clear ( ) ; } Modified |= insertWaitcntInBlock ( MF , * BI . MBB , * Brackets ) ; BI . Dirty = false ; if ( Brackets -> hasPending ( ) ) { BlockInfo * MoveBracketsToSucc = nullptr ; for ( MachineBasicBlock * Succ : BI . MBB -> successors ( ) ) { unsigned SuccIdx = RpotIdxMap [ Succ ] ; BlockInfo & SuccBI = BlockInfos [ SuccIdx ] ; if ( ! SuccBI . Incoming ) { SuccBI . Dirty = true ; if ( SuccIdx <= Idx ) Repeat = true ; if ( ! MoveBracketsToSucc ) { MoveBracketsToSucc = & SuccBI ; } else { SuccBI . Incoming = std :: make_unique < WaitcntBrackets > ( * Brackets ) ; } } else if ( SuccBI . Incoming -> merge ( * Brackets ) ) { SuccBI . Dirty = true ; if ( SuccIdx <= Idx ) Repeat = true ; }


<s> else if ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; } if ( TII -> usesLGKM_CNT ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; if ( mayAccessLDSThroughFlat ( Inst ) ) ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) || ( TII -> isMIMG ( Inst ) && ! Inst . mayLoad ( ) && ! Inst . mayStore ( ) ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else if ( Inst . mayStore ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; } else if ( Inst . isCall ( ) ) { if ( callWaitsOnFunctionReturn ( Inst ) ) {

ScoreBrackets -> applyWaitcnt ( <STR_LIT> :: allZero ( ST -> hasVscnt ( ) ) ) ;


<s> if ( ! TRI -> isInAllocatableClass ( Op . getReg ( ) ) || TRI -> isAGPR ( * MRI , Op . getReg ( ) ) ) return { - <NUM_LIT> , - <NUM_LIT> } ; assert ( ! Op . getSubReg ( ) || ! Op . isUndef ( ) ) ; RegInterval Result ; unsigned Reg = TRI -> getEncodingValue ( Op . getReg ( ) ) ; if ( TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ;

assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ;


<s> RegInterval WaitcntBrackets :: getRegInterval ( const MachineInstr * MI , const <ISA_LIT>InstrInfo * TII , const MachineRegisterInfo * MRI , const <ISA_LIT>RegisterInfo * TRI , unsigned OpNo ) const { const MachineOperand & Op = MI -> getOperand ( OpNo ) ; assert ( Op . isReg ( ) ) ; if ( ! TRI -> isInAllocatableClass ( Op . getReg ( ) ) || TRI -> isAGPR ( * MRI , Op . getReg ( ) ) ) return { - <NUM_LIT> , - <NUM_LIT> } ; assert ( ! Op . getSubReg ( ) || ! Op . isUndef ( ) ) ;


<s> } else { switch ( Inst . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SQ_MESSAGE , Inst ) ; break ; case <STR_LIT> : case <STR_LIT> : { int Imm = TII -> getNamedOperand ( Inst , <STR_LIT> :: tgt ) -> getImm ( ) ; if ( Imm >= <NUM_LIT> && Imm <= <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_PARAM_ACCESS , Inst ) ; else if ( Imm >= <NUM_LIT> && Imm <= <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_POS_ACCESS , Inst ) ; else ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_GPR_LOCK , Inst ) ; break ; } case <STR_LIT> : case <STR_LIT> : ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; break ; default : break ; } }


<s> if ( mayAccessLDSThroughFlat ( Inst ) ) { ++ FlatASCount ; ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; } assert ( FlatASCount ) ; if ( FlatASCount > <NUM_LIT> ) ScoreBrackets -> setPendingFlat ( ) ; } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) || ( TII -> isMIMG ( Inst ) && ! Inst . mayLoad ( ) && ! Inst . mayStore ( ) ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else if ( Inst . mayStore ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) {

ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ;


<s> else if ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; } if ( TII -> usesLGKM_CNT ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; if ( mayAccessLDSThroughFlat ( Inst ) ) ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { if ( ! ST -> hasVscnt ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; else if ( ( Inst . mayLoad ( ) && <STR_LIT> ( Inst . getOpcode ( ) ) == - <NUM_LIT> ) || ( TII -> isMIMG ( Inst ) && ! Inst . mayLoad ( ) && ! Inst . mayStore ( ) ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_READ_ACCESS , Inst ) ; else if ( Inst . mayStore ( ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_WRITE_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) {


<s> II -> eraseFromParent ( ) ; Modified = true ; } } else { assert ( II -> getOpcode ( ) == <STR_LIT> ) ; assert ( II -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) ; if ( ! TrackedWaitcntSet . count ( & * II ) ) { unsigned OldVSCnt = TII -> getNamedOperand ( * II , <STR_LIT> :: simm16 ) -> getImm ( ) ; Wait . VsCnt = std :: min ( Wait . VsCnt , OldVSCnt ) ; } if ( ! WaitcntVsCntInstr ) { WaitcntVsCntInstr = & * II ; } else { II -> eraseFromParent ( ) ; Modified = true ; } } } if ( WaitcntInstr ) { if ( Wait . hasWaitExceptVsCnt ( ) ) { unsigned NewEnc = <STR_LIT> ( IV , Wait ) ; unsigned OldEnc = WaitcntInstr -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( OldEnc != NewEnc ) { WaitcntInstr -> getOperand ( <NUM_LIT> ) . setImm ( NewEnc ) ; Modified = true ; } ScoreBrackets . applyWaitcnt ( Wait ) ; Wait . VmCnt = ~ <NUM_LIT> ; Wait . LgkmCnt = ~ <NUM_LIT> ; Wait . ExpCnt = ~ <NUM_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << MI << <STR_LIT> << * WaitcntInstr << '\n' ) ; } else { WaitcntInstr -> eraseFromParent ( ) ; Modified = true ; } } if ( WaitcntVsCntInstr ) { if ( Wait . hasWaitVsCnt ( ) ) { assert ( ST -> hasVscnt ( ) ) ;

unsigned OldVSCnt = TII -> getNamedOperand ( * WaitcntVsCntInstr , <STR_LIT> :: simm16 ) -> getImm ( ) ;


<s> RegInterval Result ; unsigned Reg = TRI -> getEncodingValue ( <STR_LIT> ( Op . getReg ( ) , * ST ) ) ; if ( TRI -> isVectorRegister ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; if ( TRI -> isAGPR ( * MRI , Op . getReg ( ) ) ) Result . first += AGPR_OFFSET ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . SGPR0 && Reg < SQ_MAX_PGM_SGPRS ) ; Result . first = Reg - RegisterEncoding . SGPR0 + NUM_ALL_VGPRS ; assert ( Result . first >= NUM_ALL_VGPRS && Result . first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS ) ; }

else return { - <NUM_LIT> , - <NUM_LIT> } ;


<s> if ( TRI -> isVectorRegister ( * MRI , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; if ( TRI -> isAGPR ( * MRI , Op . getReg ( ) ) ) Result . first += AGPR_OFFSET ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) {


<s> BlockWaitcntProcessedSet . clear ( ) ; ReversePostOrderTraversal < MachineFunction * > RPOT ( & MF ) ; bool Modified = false ; for ( ReversePostOrderTraversal < MachineFunction * > :: rpo_iterator I = RPOT . begin ( ) , E = RPOT . end ( ) , J = RPOT . begin ( ) ; I != E ; ) { MachineBasicBlock & MBB = * * I ; BlockVisitedSet . insert ( & MBB ) ; BlockWaitcntBrackets * ScoreBrackets = BlockWaitcntBracketsMap [ & MBB ] . get ( ) ; if ( ! ScoreBrackets ) { BlockWaitcntBracketsMap [ & MBB ] = llvm :: make_unique < BlockWaitcntBrackets > ( ST ) ; ScoreBrackets = BlockWaitcntBracketsMap [ & MBB ] . get ( ) ; } ScoreBrackets -> setPostOrder ( MBB . getNumber ( ) ) ; MachineLoop * ContainingLoop = MLI -> getLoopFor ( & MBB ) ; if ( ContainingLoop && LoopWaitcntDataMap [ ContainingLoop ] == nullptr ) LoopWaitcntDataMap [ ContainingLoop ] = llvm :: make_unique < LoopWaitcntData > ( ) ; if ( ContainingLoop && ContainingLoop -> getHeader ( ) == & MBB ) { unsigned Count = countNumBottomBlocks ( ContainingLoop ) ; if ( ( std :: count ( BlockWaitcntProcessedSet . begin ( ) , BlockWaitcntProcessedSet . end ( ) , & MBB ) < Count ) ) { BlockWaitcntBracketsMap [ & MBB ] -> setRevisitLoop ( true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ContainingLoop -> getHeader ( ) -> getNumber ( ) << '\n' ; ) ; } } insertWaitcntInBlock ( MF , MBB ) ; BlockWaitcntProcessedSet . push_back ( & MBB ) ; if ( ContainingLoop && isLoopBottom ( ContainingLoop , & MBB ) && std :: count ( BlockWaitcntProcessedSet . begin ( ) , BlockWaitcntProcessedSet . end ( ) , & MBB ) == <NUM_LIT> ) { MachineBasicBlock * EntryBB = ContainingLoop -> getHeader ( ) ; BlockWaitcntBrackets * EntrySB = BlockWaitcntBracketsMap [ EntryBB ] . get ( ) ; if ( EntrySB && EntrySB -> getRevisitLoop ( ) ) { EntrySB -> setRevisitLoop ( false ) ; J = I ; int32_t PostOrder = EntrySB -> getPostOrder ( ) ; for ( ReversePostOrderTraversal < MachineFunction * > :: rpo_iterator X = RPOT . begin ( ) , Y = RPOT . end ( ) ; X != Y ; ++ X ) { MachineBasicBlock & MBBX = * * X ; if ( MBBX . getNumber ( ) == PostOrder ) { I = X ; break ; } } LoopWaitcntData * WaitcntData = LoopWaitcntDataMap [ ContainingLoop ] . get ( ) ; WaitcntData -> incIterCnt ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << EntryBB -> getNumber ( ) << '\n' ; ) ; continue ; } else { LoopWaitcntData * WaitcntData = LoopWaitcntDataMap [ ContainingLoop ] . get ( ) ;


<s> unsigned Reg = TRI -> getEncodingValue ( Op . getReg ( ) ) ; if ( TRI -> isVGPR ( MRIA , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( MRIA , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . SGPR0 && Reg < SQ_MAX_PGM_SGPRS ) ; Result . first = Reg - RegisterEncoding . SGPR0 + NUM_ALL_VGPRS ; assert ( Result . first >= NUM_ALL_VGPRS && Result . first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS ) ; } else return { - <NUM_LIT> , - <NUM_LIT> } ; const MachineInstr & MIA = * MI ; const TargetRegisterClass * RC = TII -> getOpRegClass ( MIA , OpNo ) ;


<s> I != E ; ) { MachineBasicBlock & MBB = * * I ; BlockVisitedSet . insert ( & MBB ) ; BlockWaitcntBrackets * ScoreBrackets = BlockWaitcntBracketsMap [ & MBB ] . get ( ) ; if ( ! ScoreBrackets ) { BlockWaitcntBracketsMap [ & MBB ] = llvm :: make_unique < BlockWaitcntBrackets > ( ST ) ; ScoreBrackets = BlockWaitcntBracketsMap [ & MBB ] . get ( ) ; } ScoreBrackets -> setPostOrder ( MBB . getNumber ( ) ) ; MachineLoop * ContainingLoop = MLI -> getLoopFor ( & MBB ) ; if ( ContainingLoop && LoopWaitcntDataMap [ ContainingLoop ] == nullptr ) LoopWaitcntDataMap [ ContainingLoop ] = llvm :: make_unique < LoopWaitcntData > ( ) ; if ( ContainingLoop && ContainingLoop -> getHeader ( ) == & MBB ) { unsigned Count = countNumBottomBlocks ( ContainingLoop ) ; if ( ( std :: count ( BlockWaitcntProcessedSet . begin ( ) , BlockWaitcntProcessedSet . end ( ) , & MBB ) < ( int ) Count ) ) { BlockWaitcntBracketsMap [ & MBB ] -> setRevisitLoop ( true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ContainingLoop -> getHeader ( ) -> getNumber ( ) << '\n' ; ) ; } } insertWaitcntInBlock ( MF , MBB ) ; BlockWaitcntProcessedSet . push_back ( & MBB ) ; if ( ContainingLoop && isLoopBottom ( ContainingLoop , & MBB ) && std :: count ( BlockWaitcntProcessedSet . begin ( ) , BlockWaitcntProcessedSet . end ( ) , & MBB ) == <NUM_LIT> ) { MachineBasicBlock * EntryBB = ContainingLoop -> getHeader ( ) ; BlockWaitcntBrackets * EntrySB = BlockWaitcntBracketsMap [ EntryBB ] . get ( ) ; if ( EntrySB && EntrySB -> getRevisitLoop ( ) ) { EntrySB -> setRevisitLoop ( false ) ; J = I ; int32_t PostOrder = EntrySB -> getPostOrder ( ) ; for ( ReversePostOrderTraversal < MachineFunction * > :: rpo_iterator X = RPOT . begin ( ) , Y = RPOT . end ( ) ;


<s> Wait . ExpCnt = std :: min ( Wait . ExpCnt , Count ) ; break ; case LGKM_CNT : Wait . LgkmCnt = std :: min ( Wait . LgkmCnt , Count ) ; break ; default : llvm_unreachable ( <STR_LIT> ) ;


<s> void WaitcntBrackets :: applyWaitcnt ( InstCounterType T , unsigned Count ) { const uint32_t UB = getScoreUB ( T ) ; if ( Count >= UB ) return ; if ( Count != <NUM_LIT> ) { if ( counterOutOfOrder ( T ) ) return ; setScoreLB ( T , std :: max ( getScoreLB ( T ) , UB - Count ) ) ; } else {


<s> explicit BlockInfo ( MachineBasicBlock * MBB ) : MBB ( MBB ) {


<s> void clear ( ) { memset ( ScoreLBs , <NUM_LIT> , sizeof ( ScoreLBs ) ) ; memset ( ScoreUBs , <NUM_LIT> , sizeof ( ScoreUBs ) ) ; PendingEvents = <NUM_LIT> ; memset ( MixedPendingEvents , <NUM_LIT> , sizeof ( MixedPendingEvents ) ) ;

for ( auto T : inst_counter_types ( ) ) memset ( VgprScores [ T ] , <NUM_LIT> , sizeof ( VgprScores [ T ] ) ) ;


<s> return new <ISA_LIT>InsertWaitcnts ( ) ;


<s> void WaitcntBrackets :: determineWait ( InstCounterType T , uint32_t ScoreToWait , <STR_LIT> & Wait ) const { const uint32_t LB = getScoreLB ( T ) ; const uint32_t UB = getScoreUB ( T ) ; if ( ( UB >= ScoreToWait ) && ( ScoreToWait > LB ) ) { if ( ( T == VM_CNT || T == LGKM_CNT ) && hasPendingFlat ( ) && ! ST -> hasFlatLgkmVMemCountInOrder ( ) ) { addWait ( Wait , T , <NUM_LIT> ) ; } else if ( counterOutOfOrder ( T ) ) { addWait ( Wait , T , <NUM_LIT> ) ;


<s> void dump ( ) { print ( dbgs ( ) ) ;


<s> enum_iterator ( EnumT Value ) : Value ( Value ) {


<s> assert ( WaitEventMaskForInst [ EXP_CNT ] & ( <NUM_LIT> << E ) ) ;


<s> if ( E == VMEM_ACCESS ) return VM_CNT ; if ( WaitEventMaskForInst [ LGKM_CNT ] & ( <NUM_LIT> << E ) ) return LGKM_CNT ;

assert ( WaitEventMaskForInst [ EXP_CNT ] & ( <NUM_LIT> << E ) ) ;


<s> int32_t getMaxSGPR ( ) const { return SgprUB ;


<s> int32_t getMaxVGPR ( ) const { return VgprUB ;


<s> assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( MRIA , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . SGPR0 && Reg < SQ_MAX_PGM_SGPRS ) ; Result . first = Reg - RegisterEncoding . SGPR0 + NUM_ALL_VGPRS ; assert ( Result . first >= NUM_ALL_VGPRS && Result . first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS ) ;


<s> if ( TRI -> isVGPR ( MRIA , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . VGPR0 && Reg <= RegisterEncoding . VGPRL ) ; Result . first = Reg - RegisterEncoding . VGPR0 ; assert ( Result . first >= <NUM_LIT> && Result . first < SQ_MAX_PGM_VGPRS ) ; } else if ( TRI -> isSGPRReg ( MRIA , Op . getReg ( ) ) ) { assert ( Reg >= RegisterEncoding . SGPR0 && Reg < SQ_MAX_PGM_SGPRS ) ; Result . first = Reg - RegisterEncoding . SGPR0 + NUM_ALL_VGPRS ;

assert ( Result . first >= NUM_ALL_VGPRS && Result . first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS ) ;


<s> if ( GprNo < NUM_ALL_VGPRS ) { return VgprScores [ T ] [ GprNo ] ; } assert ( T == LGKM_CNT ) ; return SgprScores [ GprNo - NUM_ALL_VGPRS ] ;


<s> uint32_t getScoreLB ( InstCounterType T ) const { assert ( T < NUM_INST_CNTS ) ;

if ( T >= NUM_INST_CNTS ) return <NUM_LIT> ;


<s> assert ( T < NUM_INST_CNTS ) ; if ( T >= NUM_INST_CNTS ) return <NUM_LIT> ;


<s> static uint32_t getWaitCountMax ( InstCounterType T ) { switch ( T ) { case VM_CNT : return HardwareLimits . VmcntMax ; case LGKM_CNT : return HardwareLimits . LgkmcntMax ;


return ( ( LastFlat [ LGKM_CNT ] > ScoreLBs [ LGKM_CNT ] && LastFlat [ LGKM_CNT ] <= ScoreUBs [ LGKM_CNT ] ) || ( LastFlat [ VM_CNT ] > ScoreLBs [ VM_CNT ] && LastFlat [ VM_CNT ] <= ScoreUBs [ VM_CNT ] ) ) ;


<s> return ( ( LastFlat [ LGKM_CNT ] > ScoreLBs [ LGKM_CNT ] && LastFlat [ LGKM_CNT ] <= ScoreUBs [ LGKM_CNT ] ) || ( LastFlat [ VM_CNT ] > ScoreLBs [ VM_CNT ] && LastFlat [ VM_CNT ] <= ScoreUBs [ VM_CNT ] ) ) ;


<s> LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> << Block . getNumber ( ) << <STR_LIT> ;


<s> iterator_range < enum_iterator < InstCounterType >> inst_counter_types ( ) {

return make_range ( enum_iterator < InstCounterType > ( VM_CNT ) , enum_iterator < InstCounterType > ( NUM_INST_CNTS ) ) ;


<s> unsigned AS = Memop -> getAddrSpace ( ) ; if ( AS == <ISA_LIT>AS :: LOCAL_ADDRESS || AS == <ISA_LIT>AS :: FLAT_ADDRESS ) return true ;


<s> const uint32_t OtherEvents = Other . PendingEvents & WaitEventMaskForInst [ T ] ; if ( OtherEvents & ~ OldEvents ) StrictDom = true ; if ( Other . MixedPendingEvents [ T ] || ( OldEvents && OtherEvents && OldEvents != OtherEvents ) ) MixedPendingEvents [ T ] = true ; PendingEvents |= OtherEvents ; const uint32_t MyPending = ScoreUBs [ T ] - ScoreLBs [ T ] ; const uint32_t OtherPending = Other . ScoreUBs [ T ] - Other . ScoreLBs [ T ] ; MergeInfo M ; M . OldLB = ScoreLBs [ T ] ; M . OtherLB = Other . ScoreLBs [ T ] ; M . MyShift = OtherPending > MyPending ? OtherPending - MyPending : <NUM_LIT> ; M . OtherShift = ScoreUBs [ T ] - Other . ScoreUBs [ T ] + M . MyShift ; const uint32_t NewUB = ScoreUBs [ T ] + M . MyShift ; if ( NewUB < ScoreUBs [ T ] ) report_fatal_error ( <STR_LIT> ) ; ScoreUBs [ T ] = NewUB ; ScoreLBs [ T ] = std :: min ( M . OldLB + M . MyShift , M . OtherLB + M . OtherShift ) ; StrictDom |= mergeScore ( M , LastFlat [ T ] , Other . LastFlat [ T ] ) ; bool RegStrictDom = false ; for ( int J = <NUM_LIT> , E = std :: max ( getMaxVGPR ( ) , Other . getMaxVGPR ( ) ) + <NUM_LIT> ; J != E ; J ++ ) { RegStrictDom |= mergeScore ( M , VgprScores [ T ] [ J ] , Other . VgprScores [ T ] [ J ] ) ; } if ( T == LGKM_CNT ) { for ( int J = <NUM_LIT> , E = std :: max ( getMaxSGPR ( ) , Other . getMaxSGPR ( ) ) + <NUM_LIT> ; J != E ; J ++ ) { RegStrictDom |= mergeScore ( M , SgprScores [ J ] , Other . SgprScores [ J ] ) ; } }

if ( RegStrictDom && ! OldOutOfOrder ) StrictDom = true ;


<s> M . MyShift = OtherPending > MyPending ? OtherPending - MyPending : <NUM_LIT> ; M . OtherShift = ScoreUBs [ T ] - Other . ScoreUBs [ T ] + M . MyShift ; const uint32_t NewUB = ScoreUBs [ T ] + M . MyShift ; if ( NewUB < ScoreUBs [ T ] ) report_fatal_error ( <STR_LIT> ) ; ScoreUBs [ T ] = NewUB ; ScoreLBs [ T ] = std :: min ( M . OldLB + M . MyShift , M . OtherLB + M . OtherShift ) ; StrictDom |= mergeScore ( M , LastFlat [ T ] , Other . LastFlat [ T ] ) ; bool RegStrictDom = false ; for ( int J = <NUM_LIT> , E = std :: max ( getMaxVGPR ( ) , Other . getMaxVGPR ( ) ) + <NUM_LIT> ; J != E ; J ++ ) { RegStrictDom |= mergeScore ( M , VgprScores [ T ] [ J ] , Other . VgprScores [ T ] [ J ] ) ; } if ( T == LGKM_CNT ) {

for ( int J = <NUM_LIT> , E = std :: max ( getMaxSGPR ( ) , Other . getMaxSGPR ( ) ) + <NUM_LIT> ;


<s> OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; default : OS << <STR_LIT> << UB - LB << <STR_LIT> ; break ; } if ( LB < UB ) { for ( int J = <NUM_LIT> ; J <= getMaxVGPR ( ) ; J ++ ) { uint32_t RegScore = getRegScore ( J , T ) ; if ( RegScore <= LB ) continue ; uint32_t RelScore = RegScore - LB - <NUM_LIT> ; if ( J < SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS ) { OS << RelScore << <STR_LIT> << J << <STR_LIT> ;


<s> unsigned Opc = MI . getOpcode ( ) ; return ( Opc == <STR_LIT> || Opc == <STR_LIT> ) && ! MI . getOperand ( <NUM_LIT> ) . isUndef ( ) ;


<s> return ( Opc == <STR_LIT> || Opc == <STR_LIT> ) && ! MI . getOperand ( <NUM_LIT> ) . isUndef ( ) ;


<s> if ( DebugCounter :: isCounterSet ( ForceLgkmCounter ) && DebugCounter :: shouldExecute ( ForceLgkmCounter ) ) { ForceEmitWaitcnt [ LGKM_CNT ] = true ; } else { ForceEmitWaitcnt [ LGKM_CNT ] = false ; } if ( DebugCounter :: isCounterSet ( ForceVMCounter ) && DebugCounter :: shouldExecute ( ForceVMCounter ) ) { ForceEmitWaitcnt [ VM_CNT ] = true ; } else { ForceEmitWaitcnt [ VM_CNT ] = false ; }


<s> } if ( DebugCounter :: isCounterSet ( ForceLgkmCounter ) && DebugCounter :: shouldExecute ( ForceLgkmCounter ) ) { ForceEmitWaitcnt [ LGKM_CNT ] = true ; } else {


<s> void setPendingFlat ( ) { LastFlat [ VM_CNT ] = ScoreUBs [ VM_CNT ] ; LastFlat [ LGKM_CNT ] = ScoreUBs [ LGKM_CNT ] ;


<s> LastFlat [ VM_CNT ] = ScoreUBs [ VM_CNT ] ; LastFlat [ LGKM_CNT ] = ScoreUBs [ LGKM_CNT ] ;


<s> } else { assert ( T == LGKM_CNT ) ; if ( GprNo - NUM_ALL_VGPRS > SgprUB ) {


<s> if ( ScoreLBs [ T ] < UB && UB < ScoreUBs [ T ] ) ScoreLBs [ T ] = UB ; }


<s> <ISA_LIT>InsertWaitcnts ( ) : MachineFunctionPass ( ID ) { ( void ) ForceExpCounter ; ( void ) ForceLgkmCounter ;


<s> const uint32_t UB = getScoreUB ( T ) ; if ( Count < UB && UB - Count > LB ) return true ; Count = ~ <NUM_LIT> ;


<s> if ( Count < UB && UB - Count > LB ) return true ; Count = ~ <NUM_LIT> ;


<s> uint32_t CurrScore = getScoreUB ( T ) + <NUM_LIT> ; if ( CurrScore == <NUM_LIT> ) report_fatal_error ( <STR_LIT> ) ; if ( ! hasPendingEvent ( E ) ) { if ( PendingEvents & WaitEventMaskForInst [ T ] ) MixedPendingEvents [ T ] = true ; PendingEvents |= <NUM_LIT> << E ; } setScoreUB ( T , CurrScore ) ; if ( T == EXP_CNT ) { if ( TII -> isDS ( Inst ) && ( Inst . mayStore ( ) || Inst . mayLoad ( ) ) ) { if ( Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { setExpScore ( & Inst , TII , TRI , MRI , <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: addr ) , CurrScore ) ; } if ( Inst . mayStore ( ) ) { setExpScore ( & Inst , TII , TRI , MRI , <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data0 ) , CurrScore ) ; if ( <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data1 ) != - <NUM_LIT> ) { setExpScore ( & Inst , TII , TRI , MRI , <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data1 ) , CurrScore ) ; } } else if ( <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { for ( unsigned I = <NUM_LIT> , E = Inst . getNumOperands ( ) ; I != E ; ++ I ) { const MachineOperand & Op = Inst . getOperand ( I ) ; if ( Op . isReg ( ) && ! Op . isDef ( ) && TRI -> isVGPR ( MRIA , Op . getReg ( ) ) ) { setExpScore ( & Inst , TII , TRI , MRI , I , CurrScore ) ; } } } } else if ( TII -> isFLAT ( Inst ) ) { if ( Inst . mayStore ( ) ) { setExpScore ( & Inst , TII , TRI , MRI , <STR_LIT> ( Inst . getOpcode ( ) , <STR_LIT> :: data ) , CurrScore ) ; } else if ( <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) {


<s> ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_ACCESS , Inst ) ; ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_GPR_LOCK , Inst ) ; } else { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; } } else if ( TII -> isFLAT ( Inst ) ) { assert ( Inst . mayLoad ( ) || Inst . mayStore ( ) ) ; if ( TII -> usesVM_CNT ( Inst ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; if ( TII -> usesLGKM_CNT ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; if ( mayAccessLDSThroughFlat ( Inst ) ) ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) {

ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ;


<s> if ( TII -> isDS ( Inst ) && TII -> usesLGKM_CNT ( Inst ) ) { if ( TII -> hasModifiersSet ( Inst , <STR_LIT> :: gds ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_ACCESS , Inst ) ; ScoreBrackets -> updateByEvent ( TII , TRI , MRI , GDS_GPR_LOCK , Inst ) ; } else { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; } } else if ( TII -> isFLAT ( Inst ) ) { assert ( Inst . mayLoad ( ) || Inst . mayStore ( ) ) ; if ( TII -> usesVM_CNT ( Inst ) ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; if ( TII -> usesLGKM_CNT ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , LDS_ACCESS , Inst ) ; if ( mayAccessLDSThroughFlat ( Inst ) ) ScoreBrackets -> setPendingFlat ( ) ; } } else if ( <ISA_LIT>InstrInfo :: isVMEM ( Inst ) && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> && Inst . getOpcode ( ) != <STR_LIT> ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMEM_ACCESS , Inst ) ; if ( ST -> vmemWriteNeedsExpWaitcnt ( ) && ( Inst . mayStore ( ) || <STR_LIT> ( Inst . getOpcode ( ) ) != - <NUM_LIT> ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , VMW_GPR_LOCK , Inst ) ; } } else if ( TII -> isSMRD ( Inst ) ) { ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SMEM_ACCESS , Inst ) ; } else { switch ( Inst . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : ScoreBrackets -> updateByEvent ( TII , TRI , MRI , SQ_MESSAGE , Inst ) ; break ; case <STR_LIT> : case <STR_LIT> : { int Imm = TII -> getNamedOperand ( Inst , <STR_LIT> :: tgt ) -> getImm ( ) ;

if ( Imm >= <NUM_LIT> && Imm <= <NUM_LIT> ) ScoreBrackets -> updateByEvent ( TII , TRI , MRI , EXP_PARAM_ACCESS , Inst ) ;


<s> WaitcntBrackets ( const GCNSubtarget * SubTarget ) : ST ( SubTarget ) { for ( auto T : inst_counter_types ( ) ) memset ( VgprScores [ T ] , <NUM_LIT> , sizeof ( VgprScores [ T ] ) ) ;


<s> if ( MI . getNumOperands ( ) != <NUM_LIT> ) { assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) && <STR_LIT> ) ; const TargetRegisterClass * RC = TII -> getOpRegClass ( MI , <NUM_LIT> ) ; unsigned Size = RC -> getSize ( ) ; Result . Named . LGKM = Size > <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; } else { Result . Named . LGKM = <NUM_LIT> ;


<s> if ( Data && Op . isIdenticalTo ( * Data ) ) return true ; MachineOperand * Data0 = TII -> getNamedOperand ( MI , <STR_LIT> :: data0 ) ; if ( Data0 && Op . isIdenticalTo ( * Data0 ) ) return true ; MachineOperand * Data1 = TII -> getNamedOperand ( MI , <STR_LIT> :: data1 ) ; if ( Data1 && Op . isIdenticalTo ( * Data1 ) ) return true ; return false ; } for ( MachineInstr :: mop_iterator I = MI . operands_begin ( ) , E = MI . operands_end ( ) ; I != E ; ++ I ) { if ( I -> isReg ( ) && I -> isUse ( ) ) return Op . isIdenticalTo ( * I ) ; }


<s> for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { LastIssued . Array [ i ] += Increment . Array [ i ] ; if ( Increment . Array [ i ] ) Limit . Array [ i ] = LastIssued . Array [ i ] ; Sum += Increment . Array [ i ] ; } if ( Sum == <NUM_LIT> ) { LastOpcodeType = OTHER ; return ; } if ( MBB . getParent ( ) -> getSubtarget < <ISA_LIT>Subtarget > ( ) . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( ( LastOpcodeType == SMEM && TII -> isSMRD ( * I ) ) || ( LastOpcodeType == VMEM && Increment . Named . VM ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; LastInstWritesM0 = false ; } if ( TII -> isSMRD ( * I ) ) LastOpcodeType = SMEM ; else if ( Increment . Named . VM ) LastOpcodeType = VMEM ; } if ( Increment . Named . EXP ) { ExpInstrTypesSeen |= I -> getOpcode ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ; } for ( unsigned i = <NUM_LIT> , e = I -> getNumOperands ( ) ; i != e ; ++ i ) { MachineOperand & Op = I -> getOperand ( i ) ; if ( ! isOpRelevant ( Op ) ) continue ; const TargetRegisterClass * RC = TII -> getOpRegClass ( * I , i ) ; RegInterval Interval = getRegInterval ( RC , Op ) ; for ( unsigned j = Interval . first ; j < Interval . second ; ++ j ) { if ( Op . isDef ( ) ) DefinedRegs [ j ] = Limit ;

if ( Op . isUse ( ) ) UsedRegs [ j ] = Limit ;


<s> unsigned Sum = <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { LastIssued . Array [ i ] += Increment . Array [ i ] ; if ( Increment . Array [ i ] ) Limit . Array [ i ] = LastIssued . Array [ i ] ; Sum += Increment . Array [ i ] ; } if ( Sum == <NUM_LIT> ) { LastOpcodeType = OTHER ; return ; } if ( MBB . getParent ( ) -> getSubtarget < <ISA_LIT>Subtarget > ( ) . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { if ( ( LastOpcodeType == SMEM && TII -> isSMRD ( * I ) ) || ( LastOpcodeType == VMEM && Increment . Named . VM ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; LastInstWritesM0 = false ; } if ( TII -> isSMRD ( * I ) ) LastOpcodeType = SMEM ; else if ( Increment . Named . VM ) LastOpcodeType = VMEM ; } if ( Increment . Named . EXP ) { ExpInstrTypesSeen |= I -> getOpcode ( ) == <STR_LIT> ? <NUM_LIT> : <NUM_LIT> ;


<s> return new <ISA_LIT>InsertWaits ( ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>InsertWaitsPass ( ) {


<s> if ( MI . getNumOperands ( ) != <NUM_LIT> ) { assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) && <STR_LIT> ) ; const TargetRegisterClass * RC = TII -> getOpRegClass ( MI , <NUM_LIT> ) ; unsigned Size = TRI -> getRegSizeInBits ( * RC ) ; Result . Named . LGKM = Size > <NUM_LIT> ? <NUM_LIT> : <NUM_LIT> ; } else { Result . Named . LGKM = <NUM_LIT> ; } } else {


<s> assert ( Size >= <NUM_LIT> ) ; RegInterval Result ; Result . first = TRI -> getEncodingValue ( Reg . getReg ( ) ) ; Result . second = Result . first + Size / <NUM_LIT> ;


<s> Counts . Named . LGKM = <STR_LIT> ( ISA , Imm ) ; for ( unsigned i = <NUM_LIT> ; i < <NUM_LIT> ; ++ i ) { if ( Counts . Array [ i ] <= LastIssued . Array [ i ] ) WaitOn . Array [ i ] = LastIssued . Array [ i ] - Counts . Array [ i ] ; else WaitOn . Array [ i ] = <NUM_LIT> ; }

increaseCounters ( DelayedWaitOn , WaitOn ) ;


<s> const TargetRegisterClass * RC = TII -> getOpRegClass ( MI , i ) ; RegInterval Interval = getRegInterval ( RC , Op ) ; for ( unsigned j = Interval . first ; j < Interval . second ; ++ j ) { if ( Op . isDef ( ) ) { increaseCounters ( Result , UsedRegs [ j ] ) ; increaseCounters ( Result , DefinedRegs [ j ] ) ; } if ( Op . isUse ( ) ) increaseCounters ( Result , DefinedRegs [ j ] ) ;