<s> unsigned Opc = MI -> getOpcode ( ) ; return <ISA_LIT>InstrInfo :: isVALU ( * MI ) && Opc != <STR_LIT> && Opc != <STR_LIT> && Opc != <STR_LIT> ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ; auto * Src0 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) ; unsigned Reg = Src0 -> getReg ( ) ; bool IsUndef = Src0 -> isUndef ( ) ; BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Define | ( IsUndef ? RegState :: Dead : <NUM_LIT> ) ) . addReg ( Reg , IsUndef ? RegState :: Undef : RegState :: Kill ) ; return true


<s> if ( ! <ISA_LIT>InstrInfo :: isSALU ( * MI ) && ! <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) return false ; if ( MI -> getNumDefs ( ) == <NUM_LIT> ) return false ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; auto IsHazardFn = [ TRI , MI ] ( MachineInstr * I ) { if ( ! <ISA_LIT>InstrInfo :: isVMEM ( * I ) && ! <ISA_LIT>InstrInfo :: isDS ( * I ) && ! <ISA_LIT>InstrInfo :: isFLAT ( * I ) ) return false ; for ( const MachineOperand & Def : MI -> defs ( ) ) { MachineOperand * Op = I -> findRegisterUseOperand ( Def . getReg ( ) , false , TRI ) ; if ( ! Op || ( Op -> isImplicit ( ) && Op -> getReg ( ) == <STR_LIT> ) ) continue ; return true


<s> if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkAnyInstHazards ( MI ) > <NUM_LIT> )


<s> if ( checkFPAtomicToDenormModeHazard ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasNoDataDepHazard ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkAnyInstHazards ( MI ) > <NUM_LIT> )


<s> WaitStates = std :: max ( WaitStates , checkFPAtomicToDenormModeHazard ( MI ) ) ; if ( ST . hasNoDataDepHazard ( ) ) return WaitStates ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( MI -> isInlineAsm ( ) ) return std :: max ( WaitStates , checkInlineAsmHazards ( MI ) ) ; if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards


<s> int GCNHazardRecognizer :: checkReadM0Hazards ( MachineInstr * MI ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo (


<s> if ( ST . hasNoDataDepHazard ( ) ) return NoHazard ; if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) || <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) || <ISA_LIT>InstrInfo :: isDS ( * MI ) || <ISA_LIT>InstrInfo :: isEXP ( * MI ) ) && checkMAIVALUHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) )

&& checkReadM0Hazards ( MI ) > <NUM_LIT> ) return HazardType ;


<s> return Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode


<s> static bool isXDL ( const GCNSubtarget & ST , const MachineInstr & MI ) { unsigned Opcode = MI . getOpcode ( ) ; if ( ! <ISA_LIT>InstrInfo :: isMAI ( MI ) || isDGEMM ( Opcode ) || Opcode == <STR_LIT> ||


<s> if ( ! <ISA_LIT>InstrInfo :: isMAI ( MI ) || isDGEMM ( Opcode ) || Opcode == <STR_LIT> || Opcode == <STR_LIT> )


<s> if ( MI -> isBundle ( ) ) return <NUM_LIT> ; int WaitStates = <NUM_LIT> ; if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) return std :: max ( WaitStates , checkSMRDHazards ( MI ) ) ; if ( ST . hasNSAtoVMEMBug ( ) ) WaitStates = std :: max ( WaitStates , checkNSAtoVMEMHazard ( MI ) ) ; WaitStates = std :: max ( WaitStates , checkFPAtomicToDenormModeHazard ( MI ) ) ; if ( ST . hasNoDataDepHazard ( ) ) return WaitStates ; if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVMEMHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) || <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) || <ISA_LIT>InstrInfo :: isDS ( * MI ) || <ISA_LIT>InstrInfo :: isEXP ( * MI ) ) && checkMAIVALUHazards ( MI ) > <NUM_LIT> ) WaitStates = std :: max ( WaitStates , checkMAIVALUHazards ( MI ) ) ; if ( MI -> isInlineAsm ( ) ) return std :: max ( WaitStates , checkInlineAsmHazards ( MI ) ) ; if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI ->

getOpcode ( ) ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; unsigned NumWaitStates = TII -> getNumWaitStates ( * CurrCycleInstr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; } EmittedInstrs . resize ( getMaxLookAhead (


<s> int GCNHazardRecognizer :: checkDPPHazards ( MachineInstr * DPP ) { const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; int DppVgprWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; for ( const MachineOperand & Use : DPP -> uses ( ) ) { if ( ! Use . isReg ( ) || ! TRI -> isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; int WaitStatesNeededForUse = DppVgprWaitStates - getWaitStatesSinceDef ( Use . getReg (


<s> if ( ! Use . isReg ( ) || ! TRI -> isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; int WaitStatesNeededForUse = DppVgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; } return WaitStatesNeeded


<s> if ( ! LaneSelectOp -> isReg ( ) || ! TRI -> isSGPRReg ( MRI , LaneSelectOp -> getReg ( ) ) ) return <NUM_LIT> ; unsigned LaneSelectReg = LaneSelectOp -> getReg ( ) ; auto IsHazardFn = [ TII ] (


<s> if ( ST . getGeneration ( ) < <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return <NUM_LIT> ; std :: set < unsigned > ClauseDefs ; std :: set < unsigned > ClauseUses ; for ( MachineInstr * MI : EmittedInstrs ) { if ( ! MI || ! <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) break ; addRegsToSet ( MI -> defs ( ) , ClauseDefs ) ; addRegsToSet ( MI -> uses ( ) , ClauseUses ) ; } if ( ClauseDefs . empty ( ) ) return <NUM_LIT> ; if ( SMEM -> mayStore ( ) ) return <NUM_LIT> ; addRegsToSet ( SMEM -> defs ( ) , ClauseDefs ) ; addRegsToSet ( SMEM -> uses ( ) , ClauseUses ) ; std :: vector < unsigned > Result ( std :: max ( ClauseDefs . size (


<s> for ( MachineInstr * MI : EmittedInstrs ) { if ( ! MI || ! <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) break ; addRegsToSet ( MI -> defs ( ) , ClauseDefs ) ; addRegsToSet ( MI -> uses ( ) , ClauseUses ) ; } if ( ClauseDefs . empty ( ) ) return <NUM_LIT> ; if ( SMEM -> mayStore ( ) ) return <NUM_LIT> ; addRegsToSet ( SMEM -> defs ( ) , ClauseDefs ) ; addRegsToSet ( SMEM -> uses ( ) , ClauseUses ) ; std :: vector < unsigned > Result ( std :: max ( ClauseDefs . size ( ) , ClauseUses . size ( ) ) ) ; std :: vector < unsigned > :: iterator End ; End = std :: set_intersection ( ClauseDefs . begin ( ) , ClauseDefs . end ( ) , ClauseUses . begin ( ) , ClauseUses . end ( ) , Result . begin ( ) ) ; if ( End != Result . begin ( ) ) return <NUM_LIT> ; return <NUM_LIT>


<s> if ( ST . getGeneration ( ) != <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return WaitStatesNeeded ; int SmrdSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ; for ( const MachineOperand & Use : SMRD


<s> int WaitStatesNeeded = <NUM_LIT> ; WaitStatesNeeded = checkSMEMSoftClauseHazards ( SMRD ) ; if ( ST . getGeneration ( ) != <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return WaitStatesNeeded ; int SmrdSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) { if ( ! Use . isReg (


<s> auto IsHazardFn = [ this , Reg , TRI ] ( MachineInstr * MI ) { int DataIdx = createsVALUHazard ( * MI ) ; return DataIdx >= <NUM_LIT> && TRI -> regsOverlap ( MI -> getOperand ( DataIdx ) . getReg ( ) , Reg ) ; } ; int WaitStatesNeededForDef = VALUWaitStates - getWaitStatesSince


<s> int GCNHazardRecognizer :: checkVALUHazards ( MachineInstr * VALU ) { if ( ! ST . has12DWordStoreHazard ( ) ) return <NUM_LIT> ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const MachineRegisterInfo & MRI = VALU -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; const int VALUWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; for ( const MachineOperand & Def : VALU -> defs ( ) ) { if ( ! TRI -> isVGPR ( MRI , Def . getReg (


<s> CurrCycleInstr ( nullptr ) , MF ( MF ) , ST ( MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ) { MaxLookAhead = <NUM_LIT>


<s> if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( isVInterp ( MI -> getOpcode ( ) ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkAnyInstHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; return


<s> return Opcode == <STR_LIT> || <STR_LIT> || Opcode ==


<s> static bool isVInterp ( unsigned Opcode ) { return Opcode == <STR_LIT> || Opcode == <STR_LIT> || Opcode


<s> } if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ; if ( isSMovRel ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ; return


<s> if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasNSAtoVMEMBug ( ) && checkNSAtoVMEMHazard ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkFPAtomicToDenormModeHazard ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasNoDataDepHazard ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> mayLoadOrStore ( ) && checkMAILdStHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; return NoHazard


<s> if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> mayLoadOrStore ( ) && checkMAILdStHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; return


<s> if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) ) return std :: max ( WaitStates , checkMAIHazards ( MI ) ) ; if ( MI -> mayLoadOrStore ( ) ) return std :: max (

WaitStates , checkMAILdStHazards ( MI ) ) ;


<s> if ( ! SU -> isInstr ( ) ) return false ; MachineInstr * MAI = nullptr ; auto IsMFMAFn = [ & MAI ] ( MachineInstr * MI ) { MAI = nullptr ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && MI -> getOpcode ( ) != <STR_LIT> && MI -> getOpcode ( ) != <STR_LIT> ) MAI = MI ; return MAI != nullptr ; } ; MachineInstr * MI = SU ->


<s> if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> mayLoadOrStore ( ) && checkMAILdStHazards ( MI ) >


<s> ScheduleHazardRecognizer :: HazardType GCNHazardRecognizer :: getHazardType ( SUnit * SU , int Stalls ) { MachineInstr * MI = SU -> getInstr ( ) ; if ( MI -> isBundle ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) && checkSMRDHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasNSAtoVMEMBug ( ) && checkNSAtoVMEMHazard ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkFPAtomicToDenormModeHazard ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasNoDataDepHazard ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI


<s> if ( ST . hasNSAtoVMEMBug ( ) ) WaitStates = std :: max ( WaitStates , checkNSAtoVMEMHazard ( MI ) ) ; WaitStates = std :: max ( WaitStates , checkFPAtomicToDenormModeHazard ( MI ) ) ; if ( ST . hasNoDataDepHazard ( ) ) return WaitStates ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( MI -> isInlineAsm ( ) ) return std :: max ( WaitStates , checkInlineAsmHazards ( MI ) ) ; if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) ) return std :: max


<s> if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI


<s> if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVMEMHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( * MI ) ) return std


<s> int GCNHazardRecognizer :: checkSMRDHazards ( MachineInstr * SMRD ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; int WaitStatesNeeded = <NUM_LIT> ; WaitStatesNeeded = checkSMEMSoftClauseHazards ( SMRD ) ; if ( ST . getGeneration ( ) != <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS


<s> if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkAnyInstHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; return


<s> if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkAnyInstHazards ( MI ) > <NUM_LIT> ) return


<s> unsigned GCNHazardRecognizer :: PreEmitNoops ( MachineInstr * MI ) { int WaitStates = std :: max ( <NUM_LIT> , checkAnyInstHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) return std :: max ( WaitStates , checkSMRDHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) { WaitStates = std :: max ( WaitStates , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVMEMHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( TII . isVINTRP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ; return WaitStates ; } if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ; if ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( )

) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ;


<s> const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ; unsigned NumWaitStates = TII -> getNumWaitStates ( * CurrCycleInstr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; } EmittedInstrs . resize ( getMaxLookAhead ( ) ) ; CurrCycleInstr = nullptr


<s> if ( ! CurrCycleInstr ) return ; const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ; unsigned NumWaitStates = TII -> getNumWaitStates ( * CurrCycleInstr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( ST . getRegisterInfo ( ) ) ; int DppVgprWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; for ( const MachineOperand & Use :


<s> int GCNHazardRecognizer :: checkVMEMHazards ( MachineInstr * VMEM ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( ST . getInstrInfo ( ) ) ; if ( ST . getGeneration ( ) < <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS


<s> CurrCycleInstr ( nullptr ) , MF ( MF ) { MaxLookAhead = <NUM_LIT>


<s> if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) && checkSMRDHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) && checkVMEMHazards


<s> if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) return std :: max ( <NUM_LIT> , checkSMRDHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) ) return std :: max ( <NUM_LIT>


<s> auto IsHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isVALU ( * MI ) ; } ; auto IsBufferHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isSALU ( * MI ) ; } ; bool IsBufferSMRD = TII . isBufferSMRD ( * SMRD ) ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) { if ( ! Use . isReg ( ) ) continue ; int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn , SmrdSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; if ( IsBufferSMRD ) { int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsBufferHazardDefFn , SmrdSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded ,


<s> if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT>


<s> ++ WaitStates ; if ( ! MI || ! IsHazardDef ( MI ) ) continue ; if ( MI -> modifiesRegister ( Reg , TRI ) ) return WaitStates ; } return std :: numeric_limits < int


<s> if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) >


<s> if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; return NoHazard


<s> if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVMEMHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards


<s> if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) { int WaitStates = std :: max ( <NUM_LIT> , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVMEMHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates

= std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ;


<s> if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) || <ISA_LIT>InstrInfo :: isDS (

* MI ) ) && checkMAILdStHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> MachineInstr * MI = SU -> getInstr ( ) ; if ( MI -> isBundle ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) && checkSMRDHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasNSAtoVMEMBug ( ) && checkNSAtoVMEMHazard ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkFPAtomicToDenormModeHazard ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasNoDataDepHazard ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) || <ISA_LIT>InstrInfo :: isDS ( * MI ) )

&& checkMAILdStHazards ( MI ) > <NUM_LIT> ) return NoopHazard ;


<s> unsigned Opcode = MI -> getOpcode ( ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) continue ; } ++ WaitStates ; } return std :: numeric_limits < int > ::


<s> const TargetRegisterInfo * TRI = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getRegisterInfo ( ) ; int WaitStates = - <NUM_LIT> ; for ( MachineInstr * MI : EmittedInstrs ) { ++


<s> if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) ) return


<s> static void addRegsToSet ( const <ISA_LIT>RegisterInfo & TRI , iterator_range < MachineInstr :: const_mop_iterator > Ops , BitVector


<s> static void addRegUnits ( const <ISA_LIT>RegisterInfo & TRI , BitVector


<s> else if ( CurrCycleInstr -> isDebugInstr ( ) ) return ; unsigned NumWaitStates = TII . getNumWaitStates ( * CurrCycleInstr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front


<s> if ( MI -> isDebugInstr ( ) ) return <NUM_LIT> ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; if ( ! ST . hasSMovFedHazard ( ) ) return <NUM_LIT> ; int MovFedWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; for ( const MachineOperand & Use : MI -> uses ( ) ) { if ( ! Use . isReg ( ) || TRI -> isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; auto IsHazardFn = [ ] ( MachineInstr * MI ) { return MI -> getOpcode ( ) == <STR_LIT> ; } ; int WaitStatesNeededForUse = MovFedWaitStates - getWaitStatesSinceDef ( Use . getReg


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const int DivFMasWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ; int WaitStatesNeeded = getWaitStatesSinceDef ( <STR_LIT> , IsHazardDefFn , DivFMasWaitStates ) ; return DivFMasWaitStates -


<s> int DppExecWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ; for ( const MachineOperand & Use : DPP -> uses ( ) ) { if ( ! Use . isReg ( ) || ! TRI -> isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; int WaitStatesNeededForUse = DppVgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , [ ] ( MachineInstr * ) { return true ; } , DppVgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; } WaitStatesNeeded = std :: max ( WaitStatesNeeded , DppExecWaitStates - getWaitStatesSinceDef ( <STR_LIT> , IsHazardDefFn , DppExecWaitStates ) ) ; return WaitStatesNeeded


<s> unsigned LaneSelectReg = LaneSelectOp -> getReg ( ) ; auto IsHazardFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ; const int RWLaneWaitStates = <NUM_LIT> ; int WaitStatesSince = getWaitStatesSinceDef ( LaneSelectReg , IsHazardFn , RWLaneWaitStates ) ; return RWLaneWaitStates -


<s> int GCNHazardRecognizer :: checkRWLaneHazards ( MachineInstr * RWLane ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const MachineOperand * LaneSelectOp = TII -> getNamedOperand ( * RWLane , <STR_LIT> ::


<s> bool IsBufferSMRD = TII . isBufferSMRD ( * SMRD ) ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) { if ( ! Use . isReg ( ) ) continue ; int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn , SmrdSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; if ( IsBufferSMRD ) { int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsBufferHazardDefFn , SmrdSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; } } return WaitStatesNeeded


<s> int WaitStatesNeeded = <NUM_LIT> ; WaitStatesNeeded = checkSoftClauseHazards ( SMRD ) ; if ( ST . getGeneration ( ) != <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return WaitStatesNeeded ; int SmrdSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isVALU ( * MI ) ; } ; auto IsBufferHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isSALU ( * MI ) ; } ; bool IsBufferSMRD = TII . isBufferSMRD ( * SMRD ) ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) { if ( ! Use . isReg ( ) ) continue ; int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn , SmrdSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; if ( IsBufferSMRD ) { int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsBufferHazardDefFn , SmrdSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded


<s> if ( IsSMRD != <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) break ; addClauseInst ( * MI ) ; } if ( ClauseDefs . none ( ) ) return <NUM_LIT> ; if ( MEM -> mayStore ( ) )


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const int VALUWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; if ( ! TRI -> isVGPR ( MRI , Def . getReg ( ) ) ) return WaitStatesNeeded ; unsigned Reg = Def . getReg ( ) ; auto IsHazardFn = [ this , Reg , TRI ] ( MachineInstr * MI ) { int DataIdx = createsVALUHazard ( * MI


<s> return DataIdx >= <NUM_LIT> && TRI -> regsOverlap ( MI -> getOperand ( DataIdx ) . getReg ( ) , Reg ) ; } ; int WaitStatesNeededForDef = VALUWaitStates - getWaitStatesSince ( IsHazardFn , VALUWaitStates ) ; WaitStatesNeeded = std :: max (

WaitStatesNeeded , WaitStatesNeededForDef ) ;


<s> IsHazardRecognizerMode ( false ) , CurrCycleInstr ( nullptr ) , MF ( MF ) , ST ( MF . getSubtarget < GCNSubtarget > ( ) ) , TII ( * ST . getInstrInfo ( ) ) , TRI ( TII . getRegisterInfo ( ) ) , ClauseUses ( TRI . getNumRegUnits ( ) ) , ClauseDefs (


<s> if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) && checkSMRDHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards


<s> MachineInstr * MI = SU -> getInstr ( ) ; if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) && checkSMRDHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) >


<s> return :: getWaitStatesSince ( IsHazard , CurrCycleInstr , IsExpiredFn ) ; } int WaitStates = <NUM_LIT> ; for ( MachineInstr * MI : EmittedInstrs ) { if ( MI ) { if ( IsHazard ( MI ) ) return WaitStates ; if ( MI -> isInlineAsm ( ) ) continue ; } ++ WaitStates ; if ( WaitStates >=


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; auto IsHazardFn = [ IsHazardDef , TRI , Reg ] ( MachineInstr * MI ) { return IsHazardDef ( MI ) && MI -> modifiesRegister ( Reg ,


<s> return isSSetReg ( MI -> getOpcode ( ) ) && IsHazard ( MI ) ; } ; return getWaitStatesSince ( IsHazardFn ,


<s> if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVMEMHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards


<s> static int BUCompareLatency ( const SUnit * left , const SUnit * right ) { int LHeight = ( int ) left -> getHeight ( ) ; int RHeight = ( int ) right -> getHeight ( ) ; if ( LHeight != RHeight ) return LHeight > RHeight ? <NUM_LIT> : - <NUM_LIT> ; int LDepth = left -> getDepth (


<s> if ( LScratch != RScratch ) return LScratch > RScratch ? right : left ; bool const DisableSchedCycles = false ; if ( ! DisableSchedCycles ) { int result = BUCompareLatency ( left , right ) ; if ( result != <NUM_LIT> ) return result > <NUM_LIT> ? right : left ; return left ; } else { if ( left -> getHeight ( ) != right -> getHeight ( ) ) return ( left -> getHeight ( ) > right -> getHeight ( ) ) ? right : left ; if ( left -> getDepth ( ) != right -> getDepth ( ) ) return ( left -> getDepth ( ) < right -> getDepth ( ) ) ? right : left ; } assert ( left -> NodeQueueId && right -> NodeQueueId && <STR_LIT> ) ; return ( left -> NodeQueueId > right -> NodeQueueId ) ?


<s>
Schedule
.
reserve
(
SUnits
.
size
(
)
)
;
while
(
true
)
{
if
(
AvailQueue
.
empty
(
)
&&
!
PendingQueue
.
empty
(
)
)
{
auto
EarliestSU
=
std
::
min_element
(
PendingQueue
.
begin
(
)
,
PendingQueue
.
end
(
)
,
[
=
]
(
const
Candidate
&
C1
,
const
Candidate
&
C2
)
{
return
C1
.
SU
->
getHeight
(
)
<
C2
.
SU
->
getHeight
(
)
;
}
)
->
SU
;
advanceToCycle
(
std
::
max
(
CurCycle
+
<NUM_LIT>
,
EarliestSU
->
getHeight
(
)
)
)
;
}
if
(
AvailQueue
.
empty
(
)
)
break
;
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<STR_LIT>
;
for
(
auto
&
C
:
AvailQueue
)
dbgs
(
)
<<
' '
<<
C
.
SU
->
NodeNum
;
dbgs
(
)
<<
'\n'
;
)
;
auto
C
=
pickCandidate
(
)
;
assert
(
C
)
;
AvailQueue
.
remove
(
*
C
)


<s> CurCycle = NextCycle ; releasePending (


<s> if ( LDepth != RDepth ) { DEBUG ( dbgs ( ) << <STR_LIT> << left -> NodeNum << <STR_LIT> << LDepth << <STR_LIT> << right -> NodeNum << <STR_LIT> << RDepth << <STR_LIT> ) ; return LDepth < RDepth ?


<s> if ( LDepth != RDepth ) { DEBUG ( dbgs ( ) << <STR_LIT> << left -> NodeNum << <STR_LIT> << LDepth << <STR_LIT> << right -> NodeNum << <STR_LIT> << RDepth << <STR_LIT> ) ; return LDepth < RDepth ? <NUM_LIT> : - <NUM_LIT> ; } if ( left -> Latency != right -> Latency ) return left -> Latency > right -> Latency ? <NUM_LIT>


<s> unsigned Scratches = <NUM_LIT> ; for ( const SDep & Pred : SU -> Preds ) { if ( Pred . isCtrl ( ) ) continue ; Scratches ++


<s> if ( SethiUllmanNumber != <NUM_LIT> ) return SethiUllmanNumber ; unsigned Extra = <NUM_LIT> ; for ( const SDep & Pred : SU -> Preds ) { if ( Pred . isCtrl ( ) ) continue ; SUnit * PredSU = Pred . getSUnit ( ) ; unsigned PredSethiUllman = CalcNodeSethiUllmanNumber ( PredSU , SUNumbers ) ; if ( PredSethiUllman > SethiUllmanNumber ) { SethiUllmanNumber = PredSethiUllman ; Extra = <NUM_LIT> ; } else if ( PredSethiUllman == SethiUllmanNumber ) ++ Extra ; } SethiUllmanNumber += Extra ; if ( SethiUllmanNumber == <NUM_LIT> )


<s> static unsigned closestSucc ( const SUnit * SU ) { unsigned MaxHeight = <NUM_LIT> ; for ( const SDep & Succ : SU -> Succs ) { if ( Succ . isCtrl ( ) ) continue ; unsigned Height = Succ . getSUnit ( ) -> getHeight ( ) ; if ( Height > MaxHeight ) MaxHeight = Height ; } return


<s> if ( SU -> NumPreds == <NUM_LIT> && SU -> NumSuccs != <NUM_LIT> ) return <NUM_LIT> ; return SUNumbers [ SU ->


<s> if ( SU -> NumSuccs == <NUM_LIT> && SU -> NumPreds != <NUM_LIT> ) return <NUM_LIT> ; if ( SU -> NumPreds == <NUM_LIT> && SU -> NumSuccs != <NUM_LIT> ) return <NUM_LIT> ; return SUNumbers [ SU


<s> GCNILPScheduler S ; return S . schedule (


<s> std :: vector < const SUnit * > makeGCNILPScheduler ( ArrayRef < const SUnit * > BotRoots , const ScheduleDAG & DAG ) { GCNILPScheduler S ; return S . schedule ( BotRoots , DAG


<s> for ( auto I = PendingQueue . begin ( ) , E = PendingQueue . end ( ) ; I != E ; ) { auto & C =


<s> void GCNILPScheduler :: releasePending ( ) { for ( auto I = PendingQueue . begin ( ) , E = PendingQueue . end ( ) ; I != E ; ) { auto & C = * I ++ ; if ( C . SU -> getHeight ( ) <= CurCycle ) { PendingQueue . remove ( C ) ; AvailQueue . push_back ( C


<s> auto PredSU = PredEdge . getSUnit ( ) ; if ( PredEdge . isWeak ( ) ) continue ; assert ( PredSU -> isBoundaryNode ( ) || PredSU -> NumSuccsLeft > <NUM_LIT> ) ; PredSU -> setHeightToAtLeast ( SU -> getHeight ( ) + PredEdge . getLatency ( )


<s>
if
(
AvailQueue
.
empty
(
)
&&
!
PendingQueue
.
empty
(
)
)
{
auto
EarliestSU
=
std
::
min_element
(
PendingQueue
.
begin
(
)
,
PendingQueue
.
end
(
)
,
[
=
]
(
const
Candidate
&
C1
,
const
Candidate
&
C2
)
{
return
C1
.
SU
->
getHeight
(
)
<
C2
.
SU
->
getHeight
(
)
;
}
)
->
SU
;
advanceToCycle
(
std
::
max
(
CurCycle
+
<NUM_LIT>
,
EarliestSU
->
getHeight
(
)
)
)
;
}
if
(
AvailQueue
.
empty
(
)
)
break
;
DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<STR_LIT>
;
for
(
auto
&
C
:
AvailQueue
)
dbgs
(
)
<<
' '
<<
C
.
SU
->
NodeNum
;
dbgs
(
)
<<
'\n'
;
)
;
auto
C
=
pickCandidate
(
)
;
assert
(
C
)
;
AvailQueue
.
remove
(
*
C
)
;
auto
SU
=
C
->
SU
;
DEBUG
(
dbgs
(
)
<<
<STR_LIT>
;
SU
->
dump
(
&
DAG
)
)
;
advanceToCycle
(
SU
->
getHeight
(
)
)
;
releasePredecessors
(
SU
)
;
Schedule
.
push_back
(
SU
)
;
SU
->
isScheduled
=
true
;
}
assert
(
SUnits
.
size
(
)
==
Schedule
.
size
(
)
)
;
std
::
reverse
(
Schedule
.
begin
(
)
,
Schedule
.
end
(
)
)
;
for
(
auto
&
SU
:
SUnits
)
SU
=
SUSavedCopy
[
SU
.
NodeNum
]
;
return
Schedule


<s> Sch . BaseClass :: startBlock ( BB ) ; Sch . BaseClass :: enterRegion ( BB , R . Begin , R . End , R . NumRegionInstrs ) ; Sch . buildSchedGraph ( Sch . AA , nullptr , nullptr ,


<s> BuildDAG ( const Region & R , GCNIterativeScheduler & _Sch ) : Sch ( _Sch ) { auto BB = R . Begin -> getParent ( ) ; Sch . BaseClass :: startBlock ( BB ) ; Sch . BaseClass :: enterRegion ( BB , R . Begin


<s> const auto & D = std :: find_if ( DbgB , DbgE , [ SU ] ( decltype ( * DbgB ) & P ) { return P . second == SU -> getInstr ( ) ; } ) ; if ( D != DbgE ) Res . push_back ( D -> first ) ; } return Res


<s> switch ( Strategy ) { case SCHEDULE_MINREGONLY : scheduleMinReg ( ) ; break ; case SCHEDULE_MINREGFORCED : scheduleMinReg ( true ) ;


<s> if ( Regions . empty ( ) ) return ; switch ( Strategy ) { case SCHEDULE_MINREGONLY : scheduleMinReg (


<s> static inline MachineInstr * getMachineInstr ( const SUnit & SU


<s> static inline MachineInstr * getMachineInstr ( const SUnit


<s> auto const BBEnd = Begin -> getParent ( ) -> end ( ) ; auto const BottomMI = End == BBEnd ? std :: prev ( End ) : End ; auto AfterBottomMI = std :: next ( BottomMI ) ; if ( AfterBottomMI == BBEnd || & * AfterBottomMI != UPTracker . getLastTrackedMI ( ) ) { UPTracker . reset ( * BottomMI ) ; }


<s> } else { RPTracker . reset ( * std :: prev ( BBEnd ) ) ; } for ( auto I = Schedule . end ( ) , B = Schedule . begin ( ) ; I != B ; ) { RPTracker . recede ( * getMachineInstr ( *


<s> SUnit * pickNode ( bool & IsTopNode ) override { return


<s> OS << <STR_LIT> ; printRegion ( OS , R -> Begin , R -> End , LIS , <NUM_LIT> ) ; printLivenessInfo ( OS , R -> Begin , R -> End , LIS ) ; OS << <STR_LIT> ; R -> MaxPressure . print ( OS


<s> LLVM_DUMP_METHOD void GCNIterativeScheduler :: printSchedResult ( raw_ostream & OS , const Region * R , const GCNRegPressure & RP ) const { OS << <STR_LIT> ; printRegion ( OS , R -> Begin , R -> End , LIS ) ; printSchedRP ( OS , R -> MaxPressure ,


<s> printRegion ( OS , R -> Begin , R -> End , LIS ) ; printSchedRP ( OS , R -> MaxPressure ,


<s> LLVM_DUMP_METHOD void GCNIterativeScheduler :: printSchedRP ( raw_ostream & OS , const GCNRegPressure & Before , const GCNRegPressure & After ) const { const auto & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; OS <<


<s> if ( ! Regions . empty ( ) && Regions . back ( ) -> Begin == RegionBegin ) { dbgs ( ) << <STR_LIT> ; Regions . back ( ) -> MaxPressure . print ( dbgs ( )

, & MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ) ;


<s> void GCNIterativeScheduler :: scheduleBest ( Region & R ) { assert ( R . BestSchedule . get ( ) && <STR_LIT> ) ; scheduleRegion ( R , R . BestSchedule ->

Schedule , R . BestSchedule -> MaxPressure ) ;


<s> auto Occ = Regions . front ( ) -> MaxPressure . getOccupancy ( ST ) ; if ( TryMaximizeOccupancy && Occ < TgtOcc ) Occ = tryMaximizeOccupancy ( TgtOcc ) ; const int NumPasses = Occ < TgtOcc ? <NUM_LIT> : <NUM_LIT> ; TgtOcc = std :: min ( Occ , TgtOcc ) ; DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> << TgtOcc << '\n' ) ; GCNMaxOccupancySchedStrategy LStrgy ( Context ) ; for ( int I = <NUM_LIT> ; I < NumPasses ; ++ I ) { LStrgy . setTargetOccupancy ( I == <NUM_LIT> ? <NUM_LIT> : TgtOcc ) ; for ( auto R : Regions ) { OverrideLegacyStrategy Ovr ( * R , LStrgy , * this ) ; Ovr . schedule ( ) ; const auto RP = getRegionPressure ( * R ) ; DEBUG ( printSchedRP ( dbgs ( ) , R -> MaxPressure , RP ) ) ; if ( RP . getOccupancy ( ST ) < TgtOcc ) { DEBUG ( dbgs ( ) << <STR_LIT> << TgtOcc ) ; if ( R -> BestSchedule . get ( ) && R -> BestSchedule -> MaxPressure . getOccupancy ( ST ) >= TgtOcc ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; scheduleBest ( * R


<s> DEBUG ( if ( R -> MaxPressure . less ( ST , RP , TgtOcc ) ) { dbgs ( ) << <STR_LIT> ; printSchedRP ( dbgs ( ) , R -> MaxPressure , RP ) ; } ) ; if ( ! force && MaxPressure . less ( ST , RP , TgtOcc ) ) break ; scheduleRegion ( * R , MinSchedule , RP ) ; DEBUG ( printSchedResult ( dbgs ( ) , R , RP ) ) ; MaxPressure =


<s> const auto RP = getSchedulePressure ( * R , MinSchedule ) ; DEBUG ( if ( R -> MaxPressure . less ( ST , RP , TgtOcc ) ) { dbgs ( ) << <STR_LIT> ; printSchedRP ( dbgs ( ) , R -> MaxPressure , RP ) ; } ) ; if ( ! force && MaxPressure . less ( ST , RP , TgtOcc ) ) break ; scheduleRegion ( * R , MinSchedule


<s> void GCNIterativeScheduler :: scheduleRegion ( Region & R , Range && Schedule , const GCNRegPressure & MaxRP ) { assert ( RegionBegin == R . Begin && RegionEnd == R . End ) ; assert ( LIS != nullptr ) ; const auto SchedMaxRP = getSchedulePressure ( R , Schedule ) ; auto BB = R . Begin -> getParent ( ) ; auto Top = R . Begin ; for ( const auto & I : Schedule ) { auto MI = getMachineInstr ( I ) ; if ( MI != & * Top ) { BB -> remove ( MI ) ; BB -> insert ( Top , MI ) ; if ( ! MI -> isDebugValue ( ) ) LIS -> handleMove ( * MI , true ) ; } if ( ! MI -> isDebugValue ( ) ) { for ( auto & Op : MI -> operands ( ) ) if ( Op . isReg ( ) && Op . isDef ( ) ) Op . setIsUndef ( false ) ; RegisterOperands RegOpers ; RegOpers . collect ( * MI , * TRI , MRI , true , false ) ; auto SlotIdx = LIS -> getInstructionIndex ( * MI ) . getRegSlot ( ) ; RegOpers . adjustLaneLiveness ( * LIS , MRI , SlotIdx , MI ) ; } Top = std :: next ( MI -> getIterator ( ) ) ; } RegionBegin = getMachineInstr ( Schedule . front ( ) ) ; if ( ! std :: is_same < decltype ( * Schedule . begin ( ) ) , MachineInstr * > :: value ) { placeDebugValues ( ) ; RegionEnd = R . End ; } R . Begin = RegionBegin ; R . MaxPressure = MaxRP ; const auto RegionMaxRP = getRegionPressure ( R ) ; const auto & ST = MF .

getSubtarget < <ISA_LIT>Subtarget > ( ) ;


<s> if ( ! MI -> isDebugValue ( ) ) { for ( auto & Op : MI -> operands ( ) ) if ( Op . isReg ( ) && Op . isDef ( ) ) Op . setIsUndef ( false ) ; RegisterOperands RegOpers ; RegOpers . collect ( * MI , * TRI , MRI , true , false ) ; auto SlotIdx = LIS -> getInstructionIndex ( * MI ) . getRegSlot ( ) ; RegOpers . adjustLaneLiveness ( * LIS , MRI , SlotIdx , MI ) ; } Top = std :: next ( MI -> getIterator ( ) ) ; } RegionBegin = getMachineInstr ( Schedule . front ( ) ) ; if ( ! std :: is_same < decltype ( * Schedule . begin ( ) ) , MachineInstr * > :: value ) { placeDebugValues ( ) ; RegionEnd = R . End ; } R . Begin = RegionBegin ; R . MaxPressure = MaxRP ; const auto RegionMaxRP = getRegionPressure ( R


<s> R . BestSchedule . reset ( new TentativeSchedule { detachSchedule ( Schedule )


<s> bool shouldTrackLaneMasks ( ) const override { return false


<s> const auto & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; std :: sort ( Regions . begin ( ) , Regions . end ( ) , [ & ST , TargetOcc ] ( const Region * R1 , const Region * R2 ) { return R2 -> MaxPressure . less ( ST , R1 -> MaxPressure , TargetOcc ) ; }


<s> for ( auto R : Regions ) { if ( R -> MaxPressure . getOccupancy ( ST ) >= NewOcc ) break ; DEBUG ( printRegion ( dbgs ( ) , R -> Begin , R -> End , LIS , <NUM_LIT> ) ; printLivenessInfo ( dbgs ( ) , R -> Begin , R -> End , LIS ) ) ; BuildDAG DAG ( * R , * this ) ; const auto MinSchedule = makeMinRegSchedule ( DAG . getTopRoots (


<s> TgtOcc = std :: min ( Occ , TgtOcc ) ; DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> << TgtOcc << '\n' ) ; for ( auto R : Regions ) { BuildDAG DAG ( * R , * this ) ; const auto ILPSchedule = makeGCNILPScheduler ( DAG . getBottomRoots ( ) , * this ) ; const auto RP = getSchedulePressure ( * R , ILPSchedule ) ; DEBUG ( printSchedRP ( dbgs ( ) , R -> MaxPressure , RP ) ) ; if ( RP . getOccupancy ( ST ) < TgtOcc ) { DEBUG ( dbgs ( ) << <STR_LIT> << TgtOcc ) ; if ( R -> BestSchedule . get ( ) && R -> BestSchedule -> MaxPressure . getOccupancy ( ST ) >= TgtOcc ) { DEBUG ( dbgs ( ) << <STR_LIT> ) ; scheduleBest ( * R ) ; } } else { scheduleRegion ( * R , ILPSchedule ,


<s> if ( ! Regions . empty ( ) && Regions . back ( ) -> Begin == RegionBegin ) { dbgs ( ) << <STR_LIT> ; Regions . back ( ) -> MaxPressure . print ( dbgs ( ) , & MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ) ; } dbgs ( ) << '\n'


<s> <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; auto TgtOcc = MFI -> getMinAllowedOccupancy ( ) ; sortRegionsByPressure ( TgtOcc ) ; auto Occ = Regions . front ( ) -> MaxPressure . getOccupancy ( ST ) ; if ( TryMaximizeOccupancy && Occ < TgtOcc ) Occ = tryMaximizeOccupancy ( TgtOcc ) ; TgtOcc = std :: min ( Occ , TgtOcc ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> << TgtOcc << '\n' ) ; unsigned FinalOccupancy = std :: min ( Occ , MFI -> getOccupancy ( ) ) ; for ( auto R : Regions ) { BuildDAG DAG ( * R , * this ) ; const auto ILPSchedule = makeGCNILPScheduler ( DAG . getBottomRoots ( ) , * this ) ; const auto RP = getSchedulePressure ( * R , ILPSchedule ) ; LLVM_DEBUG ( printSchedRP ( dbgs ( ) , R -> MaxPressure , RP ) ) ; if ( RP . getOccupancy ( ST ) < TgtOcc ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TgtOcc ) ; if ( R -> BestSchedule . get ( ) && R -> BestSchedule -> MaxPressure . getOccupancy ( ST ) >= TgtOcc ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; scheduleBest ( * R ) ; } } else


<s> if ( ! force && R -> MaxPressure . less ( ST , MaxPressure , TgtOcc ) ) break ; BuildDAG DAG ( * R , * this ) ; const auto MinSchedule = makeMinRegSchedule ( DAG . getTopRoots ( ) , * this ) ; const auto RP = getSchedulePressure ( * R , MinSchedule ) ; LLVM_DEBUG ( if ( R -> MaxPressure . less ( ST , RP , TgtOcc ) ) { dbgs ( ) << <STR_LIT> ; printSchedRP ( dbgs ( ) , R -> MaxPressure , RP ) ; } ) ; if ( ! force && MaxPressure . less ( ST , RP ,


<s> if ( ! MI -> isDebugInstr ( ) ) LIS -> handleMove ( * MI , true ) ; } if ( ! MI -> isDebugInstr ( ) ) { for ( auto & Op : MI -> operands ( ) ) if ( Op . isReg ( ) && Op . isDef ( ) ) Op . setIsUndef ( false ) ; RegisterOperands RegOpers ; RegOpers . collect ( * MI , * TRI , MRI , true , false ) ; auto SlotIdx = LIS -> getInstructionIndex ( * MI ) . getRegSlot ( ) ; RegOpers . adjustLaneLiveness ( * LIS , MRI , SlotIdx , MI ) ; } Top = std :: next ( MI -> getIterator ( ) ) ; } RegionBegin = getMachineInstr ( Schedule . front ( ) ) ; if ( ! std :: is_same < decltype ( * Schedule . begin ( ) ) , MachineInstr * > :: value ) { placeDebugValues ( ) ; RegionEnd = R . End ; } R . Begin = RegionBegin ; R . MaxPressure = MaxRP ; const auto RegionMaxRP = getRegionPressure ( R ) ; const auto & ST = MF . getSubtarget < <ISA_LIT>Subtarget


<s> for ( auto R : Regions ) { if ( R -> MaxPressure . getOccupancy ( ST ) >= NewOcc ) break ; DEBUG ( printRegion ( dbgs ( ) , R -> Begin , R -> End , LIS , <NUM_LIT> ) ; printLivenessInfo ( dbgs ( ) , R -> Begin , R -> End , LIS ) ) ; BuildDAG DAG ( * R , * this ) ; const auto MinSchedule = makeMinRegSchedule ( DAG . getTopRoots ( ) , * this ) ; const auto MaxRP = getSchedulePressure ( * R , MinSchedule ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; printSchedRP ( dbgs ( ) , R -> MaxPressure , MaxRP ) ) ; NewOcc = std :: min ( NewOcc , MaxRP . getOccupancy ( ST ) ) ; if ( NewOcc <= Occ ) break ; setBestSchedule ( * R , MinSchedule , MaxRP ) ; } DEBUG ( dbgs ( ) << <STR_LIT> << NewOcc << <STR_LIT> << Occ


<s> llvm :: sort ( Regions . begin ( ) , Regions . end ( ) , [ & ST , TargetOcc ] ( const Region * R1 , const Region * R2 ) { return R2 -> MaxPressure . less ( ST , R1 -> MaxPressure ,


<s> const auto & ST = MF . getSubtarget < GCNSubtarget > ( ) ; llvm :: sort ( Regions . begin ( ) , Regions . end ( ) , [ & ST , TargetOcc ] ( const Region * R1 , const Region * R2 ) { return R2 -> MaxPressure . less ( ST , R1 -> MaxPressure , TargetOcc ) ; }


<s> Sch . BaseClass :: startBlock ( BB ) ; Sch . BaseClass :: enterRegion ( BB , R . Begin , R . End , R . NumRegionInstrs ) ; Sch . buildSchedGraph ( Sch . AA , nullptr , nullptr


<s> auto BB = R . Begin -> getParent ( ) ; Sch . BaseClass :: startBlock ( BB ) ; Sch . BaseClass :: enterRegion ( BB , R . Begin , R . End , R . NumRegionInstrs ) ; Sch . buildSchedGraph ( Sch . AA , nullptr , nullptr , nullptr , true ) ; Sch . Topo . InitDAGTopologicalSorting ( ) ; SmallVector < SUnit * , <NUM_LIT> > BotRoots ; Sch . findRootsAndBiasEdges (


<s> const auto & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; auto TgtOcc = std :: min ( ST . getOccupancyWithLocalMemSize ( MF ) , MFI -> getMaxWavesPerEU ( ) ) ; sortRegionsByPressure ( TgtOcc ) ; auto Occ = Regions . front ( ) -> MaxPressure . getOccupancy ( ST ) ; if ( TryMaximizeOccupancy && Occ < TgtOcc ) Occ = tryMaximizeOccupancy ( TgtOcc ) ; TgtOcc = std :: min ( Occ , TgtOcc ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> << TgtOcc << '\n' ) ; for ( auto R : Regions ) { BuildDAG DAG ( * R , * this ) ; const auto ILPSchedule = makeGCNILPScheduler ( DAG . getBottomRoots ( ) ,


<s> auto MaxPressure = Regions . front ( ) -> MaxPressure ; for ( auto R : Regions ) { if ( ! force && R -> MaxPressure . less ( ST , MaxPressure , TgtOcc ) ) break ; BuildDAG DAG ( * R , * this ) ; const auto MinSchedule = makeMinRegSchedule ( DAG . getTopRoots ( ) , * this


<s> GCNRegPressure getRegionPressure ( const Region &


<s> ArrayRef < const SUnit * > getTopRoots (


<s> ArrayRef < const SUnit * > getTopRoots ( ) const { return TopRoots


<s> LLVM_DUMP_METHOD void GCNIterativeScheduler :: printSchedRP ( raw_ostream & OS , const GCNRegPressure & Before , const GCNRegPressure & After ) const { const auto & ST = MF . getSubtarget < GCNSubtarget > ( ) ; OS <<


<s> LLVM_DEBUG ( printLivenessInfo ( dbgs ( ) , RegionBegin , RegionEnd , LIS ) ; if ( ! Regions . empty ( ) && Regions . back ( ) -> Begin == RegionBegin ) { dbgs ( )


<s> dbgs ( ) << <STR_LIT> ; Regions . back ( ) -> MaxPressure . print ( dbgs ( ) , & MF . getSubtarget < GCNSubtarget > ( ) ) ; } dbgs ( ) << '\n'


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> << TgtOcc << '\n' ) ; unsigned FinalOccupancy = std :: min ( Occ , MFI -> getOccupancy ( ) ) ; for ( auto R : Regions ) { BuildDAG DAG ( * R , * this ) ; const auto ILPSchedule = makeGCNILPScheduler ( DAG . getBottomRoots ( ) , * this ) ; const auto RP = getSchedulePressure ( * R , ILPSchedule ) ; LLVM_DEBUG ( printSchedRP ( dbgs ( ) , R -> MaxPressure , RP ) ) ; if ( RP . getOccupancy ( ST ) < TgtOcc ) { LLVM_DEBUG ( dbgs ( )


<s> if ( TryMaximizeOccupancy && Occ < TgtOcc ) Occ = tryMaximizeOccupancy ( TgtOcc ) ; const int NumPasses = Occ < TgtOcc ? <NUM_LIT> : <NUM_LIT> ; TgtOcc = std :: min ( Occ , TgtOcc ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> << TgtOcc << '\n' ) ; GCNMaxOccupancySchedStrategy LStrgy ( Context ) ; unsigned FinalOccupancy = std :: min ( Occ , MFI -> getOccupancy ( ) ) ; for ( int I = <NUM_LIT> ; I < NumPasses ; ++ I ) { LStrgy . setTargetOccupancy ( I == <NUM_LIT> ? <NUM_LIT> : TgtOcc ) ; for ( auto R : Regions ) { OverrideLegacyStrategy Ovr ( * R , LStrgy , * this ) ; Ovr . schedule ( ) ; const auto RP = getRegionPressure ( * R ) ; LLVM_DEBUG ( printSchedRP ( dbgs ( ) , R -> MaxPressure , RP ) ) ; if ( RP . getOccupancy ( ST ) < TgtOcc ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TgtOcc ) ; if ( R -> BestSchedule . get ( ) && R -> BestSchedule -> MaxPressure . getOccupancy ( ST ) >= TgtOcc ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; scheduleBest ( * R ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Ovr . restoreOrder ( ) ; assert ( R -> MaxPressure . getOccupancy (


<s> } if ( ! MI -> isDebugInstr ( ) ) { for ( auto & Op : MI -> operands ( ) ) if ( Op . isReg ( ) && Op . isDef ( ) ) Op . setIsUndef ( false ) ; RegisterOperands RegOpers ; RegOpers . collect ( * MI , * TRI , MRI , true , false ) ; auto SlotIdx = LIS -> getInstructionIndex ( * MI ) . getRegSlot ( ) ; RegOpers . adjustLaneLiveness ( * LIS , MRI , SlotIdx , MI ) ; } Top = std :: next ( MI -> getIterator ( ) ) ; } RegionBegin = getMachineInstr ( Schedule . front ( ) ) ; if ( ! std :: is_same < decltype ( * Schedule . begin ( ) ) , MachineInstr * > :: value ) { placeDebugValues ( ) ; RegionEnd = R . End ; } R . Begin = RegionBegin ; R . MaxPressure = MaxRP ; const auto RegionMaxRP = getRegionPressure ( R ) ; const auto & ST = MF . getSubtarget < GCNSubtarget > ( ) ; assert ( ( SchedMaxRP == RegionMaxRP && ( MaxRP . empty ( ) || SchedMaxRP == MaxRP ) ) || ( dbgs ( ) << <STR_LIT> <STR_LIT> , SchedMaxRP . print ( dbgs ( ) , & ST ) , dbgs ( ) << <STR_LIT> , MaxRP . print ( dbgs ( ) , & ST ) , dbgs ( ) << <STR_LIT> ,

RegionMaxRP . print ( dbgs ( ) , & ST ) , false ) ) ;


<s> void GCNIterativeScheduler :: sortRegionsByPressure ( unsigned TargetOcc ) { const auto & ST = MF . getSubtarget < GCNSubtarget > (


<s> llvm :: sort ( Regions , [ & ST , TargetOcc ] ( const


<s> const auto & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const auto Occ = Regions . front ( ) -> MaxPressure . getOccupancy ( ST ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << TargetOcc << <STR_LIT> << Occ << '\n' ) ; auto NewOcc = TargetOcc ; for ( auto R : Regions ) { if ( R -> MaxPressure . getOccupancy ( ST ) >= NewOcc ) break ; LLVM_DEBUG ( printRegion ( dbgs ( ) , R -> Begin , R -> End , LIS , <NUM_LIT> ) ; printLivenessInfo ( dbgs ( ) , R -> Begin , R -> End , LIS ) ) ; BuildDAG DAG ( * R , * this ) ; const auto MinSchedule = makeMinRegSchedule ( DAG . getTopRoots ( ) , * this ) ; const auto MaxRP = getSchedulePressure ( * R , MinSchedule ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; printSchedRP ( dbgs ( ) , R -> MaxPressure , MaxRP ) ) ; NewOcc = std :: min ( NewOcc , MaxRP . getOccupancy ( ST ) ) ; if ( NewOcc <= Occ ) break ; setBestSchedule ( * R , MinSchedule , MaxRP ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << NewOcc <<


<s> for ( const auto & S : SchedSU -> Succs ) { if ( S . getSUnit ( ) -> isBoundaryNode ( ) || isScheduled ( S . getSUnit ( ) ) || S . getKind ( ) != SDep :: Data ) continue ; for ( const auto & P : S . getSUnit ( ) -> Preds ) { auto PSU = P . getSUnit ( ) ; assert ( ! PSU -> isBoundaryNode ( ) ) ; if ( PSU != SchedSU && ! isScheduled ( PSU ) ) { Set . insert ( PSU ) ; } } } SmallVector < const SUnit * , <NUM_LIT> > Worklist ( Set . begin ( ) , Set . end ( ) ) ; while ( ! Worklist . empty


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << Res << <STR_LIT> << - Res << '\n' ) ; return - Res ; } ) ; if ( Num == <NUM_LIT> ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Num << '\n' ) ; Num = findMax ( Num , [ = ] ( const Candidate & C ) { auto SU = C . SU ; auto Res = getReadySuccessors ( SU ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << Res << <STR_LIT> << Res << '\n' ) ; return Res ; } ) ; if ( Num ==


<s>
std
::
vector
<
const
SUnit
*
>
GCNMinRegScheduler
::
schedule
(
ArrayRef
<
const
SUnit
*
>
TopRoots
,
const
ScheduleDAG
&
DAG
)
{
const
auto
&
SUnits
=
DAG
.
SUnits
;
std
::
vector
<
const
SUnit
*
>
Schedule
;
Schedule
.
reserve
(
SUnits
.
size
(
)
)
;
initNumPreds
(
SUnits
)
;
int
StepNo
=
<NUM_LIT>
;
for
(
auto
SU
:
TopRoots
)
{
RQ
.
push_back
(
*
new
(
Alloc
.
Allocate
(
)
)
Candidate
(
SU
,
StepNo
)
)
;
}
releaseSuccessors
(
&
DAG
.
EntrySU
,
StepNo
)
;
while
(
!
RQ
.
empty
(
)
)
{
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
StepNo
<<
<STR_LIT>
<STR_LIT>
;
for
(
auto
&
C
:
RQ
)
dbgs
(
)
<<
' '
<<
C
.
SU
->
NodeNum
<<
<STR_LIT>
<<
C
.
Priority
<<
')'
;
dbgs
(
)
<<
'\n'
;
)
;
auto
C
=
pickCandidate
(
)
;
assert
(
C


<s> if ( S . getSUnit ( ) -> isBoundaryNode ( ) || isScheduled ( S . getSUnit ( ) ) || S . getKind ( ) != SDep :: Data ) continue ; for ( const auto & P : S . getSUnit ( ) -> Preds ) { auto PSU = P . getSUnit ( ) ; assert ( ! PSU -> isBoundaryNode ( ) ) ; if ( PSU != SchedSU && ! isScheduled ( PSU ) ) { Set . insert ( PSU ) ; } } } SmallVector < const SUnit * , <NUM_LIT> > Worklist ( Set . begin ( )


<s> using T = decltype ( C ( * RQ . begin ( ) ) ) ; T Max = std :: numeric_limits < T > :: min ( ) ; unsigned NumMax = <NUM_LIT> ; for ( auto I = RQ . begin ( ) ; Num ; -- Num ) { T Cur = C ( * I ) ; if ( Cur >= Max ) { if ( Cur > Max ) { Max = Cur ; NumMax = <NUM_LIT> ; } else ++


<s>
for
(
auto
SU
:
TopRoots
)
{
RQ
.
push_back
(
*
new
(
Alloc
.
Allocate
(
)
)
Candidate
(
SU
,
StepNo
)
)
;
}
releaseSuccessors
(
&
DAG
.
EntrySU
,
StepNo
)
;
while
(
!
RQ
.
empty
(
)
)
{
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
StepNo
<<
<STR_LIT>
<STR_LIT>
;
for
(
auto
&
C
:
RQ
)
dbgs
(
)
<<
' '
<<
C
.
SU
->
NodeNum
<<
<STR_LIT>
<<
C
.
Priority
<<
')'
;
dbgs
(
)
<<
'\n'
;
)
;
auto
C
=
pickCandidate
(
)
;
assert
(
C
)


<s> SmallVector < const SUnit * , <NUM_LIT> > Worklist ( Set . begin ( ) , Set . end ( ) ) ; while ( ! Worklist . empty ( ) ) { auto SU = Worklist . pop_back_val ( ) ; assert ( ! SU -> isBoundaryNode ( ) ) ; for ( const auto & P : SU -> Preds ) { if ( ! P . getSUnit ( ) -> isBoundaryNode ( ) && ! isScheduled ( P . getSUnit ( ) ) && Set . insert ( P . getSUnit ( ) ) . second ) Worklist . push_back ( P . getSUnit ( ) ) ; } } DEBUG ( dbgs ( ) << <STR_LIT> << SchedSU -> NodeNum << <STR_LIT> << Priority << <STR_LIT> ) ; const auto SetEnd = Set . end ( ) ; for ( auto & C : RQ ) { if ( Set . find ( C . SU ) != SetEnd ) { C . Priority = Priority ; DEBUG ( dbgs ( ) << <STR_LIT> << C


<s> assert ( ! SU -> isBoundaryNode ( ) ) ; assert ( NumPreds [ SU -> NodeNum ] != std :: numeric_limits < unsigned > :: max ( ) ) ; return -- NumPreds [ SU ->


<s> unsigned GCNMinRegScheduler :: findMax ( unsigned Num , Calc C ) { assert ( ! RQ . empty ( ) && Num <= RQ . size ( ) ) ; typedef decltype ( C ( * RQ . begin ( ) ) ) T ; T Max = std :: numeric_limits < T > :: min ( ) ; unsigned NumMax = <NUM_LIT> ; for ( auto I = RQ . begin ( ) ; Num ;


<s> int GCNMinRegScheduler :: getNotReadySuccessors ( const SUnit * SU ) const { return SU -> Succs . size ( ) - getReadySuccessors


<s> int GCNMinRegScheduler :: getNotReadySuccessors ( const SUnit


<s> void GCNMinRegScheduler :: initNumPreds ( const decltype ( ScheduleDAG :: SUnits ) & SUnits ) { NumPreds . resize ( SUnits . size ( ) ) ; for ( unsigned I = <NUM_LIT> ; I < SUnits . size ( ) ; ++ I )

NumPreds [ I ] = SUnits [ I ] . NumPredsLeft ;


<s> void GCNMinRegScheduler :: initNumPreds ( const decltype ( ScheduleDAG :: SUnits ) & SUnits ) { NumPreds . resize ( SUnits . size ( ) ) ; for ( unsigned I = <NUM_LIT> ; I < SUnits . size ( ) ; ++

I ) NumPreds [ I ] = SUnits [ I ] . NumPredsLeft ;


<s> return NumPreds [ SU -> NodeNum ] == std :: numeric_limits < unsigned


<s> GCNMinRegScheduler S ; return S . schedule (


<s> DEBUG ( dbgs ( ) << <STR_LIT> << Num << '\n' ) ; Num = findMax ( Num , [ = ] ( const Candidate & C ) { return C . Priority ; } ) ; if ( Num == <NUM_LIT> ) break ; DEBUG ( dbgs ( ) << <STR_LIT> << Num << '\n' ) ; Num = findMax ( Num , [ = ] ( const Candidate & C ) { auto SU = C . SU ; int Res = getNotReadySuccessors ( SU ) ; DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << Res << <STR_LIT> << - Res << '\n' ) ; return - Res ; } ) ; if ( Num == <NUM_LIT> ) break ; DEBUG ( dbgs ( ) << <STR_LIT> << Num << '\n' ) ; Num = findMax ( Num , [ = ] ( const Candidate & C ) { auto SU = C . SU ; auto Res = getReadySuccessors ( SU ) ; DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << Res << <STR_LIT> << Res << '\n' ) ; return Res ; } ) ; if ( Num == <NUM_LIT>


<s> void GCNMinRegScheduler :: releaseSuccessors ( const SUnit * SU , int Priority ) { for ( const auto & S : SU -> Succs ) { auto SuccSU = S . getSUnit ( ) ; if ( S . isWeak ( ) ) continue ; assert ( SuccSU -> isBoundaryNode ( ) || getNumPreds (


<s>
for
(
auto
&
C
:
RQ
)
dbgs
(
)
<<
' '
<<
C
.
SU
->
NodeNum
<<
<STR_LIT>
<<
C
.
Priority
<<
')'
;
dbgs
(
)
<<
'\n'
;
)
;
auto
C
=
pickCandidate
(
)
;
assert
(
C
)
;
RQ
.
remove
(
*
C
)
;
auto
SU
=
C
->
SU
;
DEBUG
(
dbgs
(
)
<<
<STR_LIT>
;
SU
->
dump
(
&
DAG
)
)
;
releaseSuccessors
(
SU
,
StepNo
)
;
Schedule
.
push_back
(
SU
)
;
setIsScheduled
(
SU
)
;
if
(
getReadySuccessors
(
SU
)
==
<NUM_LIT>
)
bumpPredsPriority
(
SU
,
StepNo
)
;
++
StepNo
;
}
assert
(
SUnits
.
size
(
)
==
Schedule
.


<s> NumPreds [ SU -> NodeNum ] = std :: numeric_limits


<s> if ( Register :: isPhysicalRegister ( Reg ) || ! VRM -> isAssignedReg ( Reg ) ) return NSA_Status :: FIXED ; Register PhysReg = VRM -> getPhys ( Reg ) ; if ( ! Fast ) { if ( ! PhysReg ) return NSA_Status :: FIXED ; if ( MRI -> getRegClass ( Reg ) != & <STR_LIT> || Op . getSubReg ( ) ) return NSA_Status :: FIXED ; const MachineInstr * Def = MRI -> getUniqueVRegDef ( Reg ) ; if ( Def && Def -> isCopy ( ) && Def -> getOperand ( <NUM_LIT> ) . getReg ( ) == PhysReg ) return NSA_Status :: FIXED ; for ( auto U : MRI -> use_nodbg_operands (


<s> bool GCNNSAReassign :: tryAssignRegisters ( SmallVectorImpl < LiveInterval * > & Intervals , unsigned StartReg ) const { unsigned NumRegs = Intervals . size ( ) ; for ( unsigned N = <NUM_LIT> ; N < NumRegs ; ++ N ) if ( VRM -> hasPhys ( Intervals [ N ] -> reg ) ) LRM -> unassign ( * Intervals [ N ] ) ; for ( unsigned N = <NUM_LIT> ; N < NumRegs ; ++ N ) if ( LRM -> checkInterference ( * Intervals [ N ] , StartReg + N ) ) return false ; for ( unsigned N = <NUM_LIT> ; N < NumRegs ; ++ N ) LRM -> assign ( * Intervals


<s> LIS = & getAnalysis < LiveIntervals > ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MaxNumVGPRs = ST -> getMaxNumVGPRs ( MF ) ; MaxNumVGPRs = std :: min ( ST -> getMaxNumVGPRs ( MFI -> getOccupancy ( ) ) , MaxNumVGPRs ) ; CSRegs = TRI -> getCalleeSavedRegs ( & MF ) ; using Candidate = std :: pair < const MachineInstr * , bool > ; SmallVector < Candidate , <NUM_LIT> > Candidates ; for ( const MachineBasicBlock & MBB : MF ) { for ( const MachineInstr & MI : MBB ) { switch ( CheckNSA ( MI ) ) { default : continue ; case NSA_Status :: CONTIGUOUS : Candidates . push_back ( std :: make_pair ( & MI , true ) ) ; break ; case NSA_Status :: NON_CONTIGUOUS : Candidates . push_back ( std :: make_pair ( & MI , false ) ) ; ++ NumNSAInstructions ; break ; } } } bool Changed = false ; for ( auto & C : Candidates ) { if ( C . second ) continue ; const MachineInstr * MI = C . first ; if ( CheckNSA ( * MI , true ) == NSA_Status :: CONTIGUOUS ) { C . second = true ; ++ NumNSAConverted ; continue ; } const <STR_LIT> * Info = <STR_LIT> ( MI -> getOpcode ( ) ) ; int VAddr0Idx = <STR_LIT> ( MI -> getOpcode ( ) , <STR_LIT> :: vaddr0 ) ; SmallVector < LiveInterval * , <NUM_LIT> > Intervals ; SmallVector < unsigned , <NUM_LIT> > OrigRegs ; SlotIndex MinInd , MaxInd ; for ( unsigned I = <NUM_LIT> ; I < Info -> VAddrDwords ; ++ I ) { const MachineOperand & Op = MI -> getOperand ( VAddr0Idx + I ) ; unsigned Reg = Op . getReg ( ) ; LiveInterval * LI = & LIS -> getInterval ( Reg ) ; if ( llvm :: find ( Intervals , LI ) != Intervals . end ( ) ) { Intervals . clear ( ) ; break ; } Intervals . push_back (


<s> if ( llvm :: find ( Intervals , LI ) != Intervals . end ( ) ) { Intervals . clear ( ) ; break ; } Intervals . push_back ( LI ) ; OrigRegs . push_back ( VRM -> getPhys ( Reg ) ) ; MinInd = I ? std :: min ( MinInd , LI -> beginIndex ( ) ) : LI -> beginIndex ( ) ; MaxInd = I ? std :: max ( MaxInd , LI -> endIndex ( ) ) : LI -> endIndex ( ) ; } if ( Intervals . empty ( ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI << <STR_LIT> ; for ( auto * LI : Intervals ) dbgs ( ) << <STR_LIT> << llvm :: printReg ( ( VRM -> getPhys ( LI -> reg ) ) , TRI ) ; dbgs ( ) << '\n' ) ; bool Success = scavengeRegs ( Intervals ) ; if ( ! Success ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( VRM -> hasPhys ( Intervals . back ( ) -> reg ) ) continue ; } else { auto I = std :: lower_bound ( Candidates . begin ( ) , & C , MinInd , [ this ] ( const Candidate & C , SlotIndex I ) { return LIS -> getInstructionIndex ( * C . first ) < I ; } ) ; for ( auto E = Candidates . end ( ) ; Success && I != E && LIS -> getInstructionIndex ( * I -> first ) < MaxInd ; ++ I ) { if ( I -> second && CheckNSA ( * I -> first , true ) < NSA_Status :: CONTIGUOUS ) { Success =


<s> SmallVector < LiveInterval * , <NUM_LIT> > Intervals ; SmallVector < unsigned , <NUM_LIT> > OrigRegs ; SlotIndex MinInd , MaxInd ; for ( unsigned I = <NUM_LIT> ; I < Info -> VAddrDwords ; ++ I ) { const MachineOperand & Op = MI -> getOperand ( VAddr0Idx + I ) ; Register Reg = Op . getReg ( ) ; LiveInterval * LI = & LIS -> getInterval ( Reg ) ; if ( llvm :: find ( Intervals , LI ) != Intervals . end ( ) ) { Intervals . clear ( ) ; break ; } Intervals . push_back ( LI ) ; OrigRegs . push_back ( VRM -> getPhys ( Reg ) ) ; if ( LI -> empty ( ) ) { if ( I == <NUM_LIT> ) MinInd = MaxInd = LIS -> getInstructionIndex ( * MI ) ; continue ; } MinInd = I != <NUM_LIT> ? std :: min ( MinInd , LI -> beginIndex ( ) ) : LI -> beginIndex ( ) ; MaxInd = I != <NUM_LIT> ? std :: max ( MaxInd , LI -> endIndex ( ) ) : LI -> endIndex ( ) ; } if ( Intervals . empty ( ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI << <STR_LIT> ; for ( auto * LI : Intervals ) dbgs ( ) << <STR_LIT> << llvm :: printReg ( ( VRM -> getPhys ( LI -> reg ( ) ) ) , TRI ) ; dbgs ( ) << '\n' ) ; bool Success = scavengeRegs ( Intervals ) ; if ( ! Success ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( VRM -> hasPhys ( Intervals . back ( ) -> reg ( ) ) ) continue ; } else { auto I = std :: lower_bound ( Candidates . begin ( ) , & C , MinInd , [ this ] ( const Candidate & C , SlotIndex I ) { return LIS -> getInstructionIndex ( * C . first


<s> unsigned NumRegs = Intervals . size ( ) ; for ( unsigned N = <NUM_LIT> ; N < NumRegs ; ++ N ) if ( VRM -> hasPhys ( Intervals [ N ] -> reg ( ) ) ) LRM -> unassign ( * Intervals [ N ] ) ; for ( unsigned N = <NUM_LIT> ; N < NumRegs ; ++ N ) if ( LRM -> checkInterference ( * Intervals [ N ] , StartReg + N ) ) return false ; for ( unsigned N = <NUM_LIT> ; N < NumRegs ; ++ N ) LRM -> assign ( * Intervals


<s> for ( unsigned N = <NUM_LIT> ; N < NumRegs ; ++ N ) if ( VRM -> hasPhys ( Intervals [ N ] -> reg ( ) ) ) LRM -> unassign ( * Intervals [ N ] ) ; for ( unsigned N = <NUM_LIT> ; N < NumRegs ; ++ N ) if ( LRM -> checkInterference

( * Intervals [ N ] , StartReg + N ) ) return false ;


<s> if ( Reg . isPhysical ( ) || ! VRM -> isAssignedReg ( Reg ) ) return NSA_Status :: FIXED ; Register PhysReg = VRM -> getPhys ( Reg ) ; if ( ! Fast ) { if ( ! PhysReg ) return NSA_Status :: FIXED ; if ( MRI -> getRegClass ( Reg ) != & <STR_LIT> || Op . getSubReg ( ) ) return NSA_Status :: FIXED ; if ( VRM -> getPreSplitReg ( Reg ) ) return NSA_Status :: FIXED ; const MachineInstr * Def = MRI -> getUniqueVRegDef ( Reg ) ; if ( Def && Def -> isCopy ( ) && Def -> getOperand ( <NUM_LIT> ) . getReg ( ) == PhysReg ) return NSA_Status :: FIXED ; for ( auto U : MRI -> use_nodbg_operands ( Reg ) ) { if ( U . isImplicit ( ) ) return NSA_Status :: FIXED ; const MachineInstr * UseInst = U . getParent ( ) ; if ( UseInst -> isCopy ( ) && UseInst -> getOperand ( <NUM_LIT> ) . getReg ( ) == PhysReg ) return NSA_Status :: FIXED ; } if ( ! LIS -> hasInterval (


<s> SmallVector < LiveInterval * , <NUM_LIT> > Intervals ; SmallVector < unsigned , <NUM_LIT> > OrigRegs ; SlotIndex MinInd , MaxInd ; for ( unsigned I = <NUM_LIT> ; I < Info -> VAddrDwords ; ++ I ) { const MachineOperand & Op = MI -> getOperand ( VAddr0Idx + I ) ; Register Reg = Op . getReg ( ) ; LiveInterval * LI = & LIS -> getInterval ( Reg ) ; if ( llvm :: find ( Intervals , LI ) != Intervals . end ( ) ) { Intervals . clear ( ) ; break ; } Intervals . push_back ( LI ) ; OrigRegs . push_back ( VRM -> getPhys ( Reg ) ) ; MinInd = I ? std :: min ( MinInd , LI -> beginIndex ( ) ) : LI -> beginIndex ( ) ; MaxInd = I ? std :: max ( MaxInd , LI -> endIndex ( ) ) : LI -> endIndex ( ) ; } if ( Intervals . empty ( ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI << <STR_LIT> ; for ( auto * LI : Intervals ) dbgs ( ) << <STR_LIT> << llvm :: printReg ( ( VRM -> getPhys ( LI -> reg ) ) , TRI ) ; dbgs ( ) << '\n' ) ; bool Success = scavengeRegs ( Intervals ) ; if ( ! Success ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( VRM -> hasPhys ( Intervals . back ( ) -> reg ) ) continue ; } else { auto I = std :: lower_bound ( Candidates . begin ( ) , & C , MinInd , [ this ] ( const Candidate & C , SlotIndex I ) { return LIS -> getInstructionIndex ( * C . first ) < I ; } ) ; for ( auto E = Candidates . end ( ) ; Success && I != E && LIS -> getInstructionIndex ( * I -> first ) < MaxInd ; ++ I ) { if ( I -> second && CheckNSA ( * I -> first , true ) <


<s> if ( ! PhysReg ) return NSA_Status :: FIXED ; if ( MRI -> getRegClass ( Reg ) != & <STR_LIT> || Op . getSubReg ( ) ) return NSA_Status :: FIXED ; const MachineInstr * Def = MRI -> getUniqueVRegDef ( Reg ) ; if ( Def && Def -> isCopy ( ) && Def -> getOperand ( <NUM_LIT> ) . getReg ( ) == PhysReg ) return NSA_Status :: FIXED ; for ( auto U : MRI -> use_nodbg_operands ( Reg ) ) { if ( U . isImplicit ( ) ) return NSA_Status :: FIXED ; const MachineInstr * UseInst = U . getParent (


<s> initializeGCNNSAReassignPass ( * PassRegistry ::


<s> initializeGCNNSAReassignPass ( * PassRegistry :: getPassRegistry


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < LiveIntervals > (


<s> ++ NumNSAConverted ; continue ; } const <STR_LIT> * Info = <STR_LIT> ( MI -> getOpcode ( ) ) ; int VAddr0Idx = <STR_LIT> ( MI -> getOpcode ( ) , <STR_LIT> :: vaddr0 ) ; SmallVector < LiveInterval * , <NUM_LIT> > Intervals ; SmallVector < MCRegister , <NUM_LIT> > OrigRegs ; SlotIndex MinInd , MaxInd ; for ( unsigned I = <NUM_LIT> ; I < Info -> VAddrDwords ; ++ I ) { const MachineOperand & Op = MI -> getOperand ( VAddr0Idx + I ) ; Register Reg = Op . getReg ( ) ; LiveInterval * LI = & LIS -> getInterval ( Reg ) ; if ( llvm :: find ( Intervals , LI ) != Intervals . end ( ) ) { Intervals . clear ( ) ; break ; } Intervals . push_back ( LI ) ; OrigRegs . push_back ( VRM -> getPhys ( Reg ) ) ; if ( LI -> empty ( ) ) { if ( I == <NUM_LIT> ) MinInd = MaxInd = LIS -> getInstructionIndex ( * MI ) ; continue ; } MinInd = I != <NUM_LIT> ? std :: min ( MinInd , LI -> beginIndex ( ) ) : LI -> beginIndex ( ) ; MaxInd = I != <NUM_LIT> ? std :: max ( MaxInd , LI -> endIndex ( ) ) : LI -> endIndex ( ) ; } if ( Intervals . empty ( ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI << <STR_LIT> ; for ( auto * LI : Intervals ) dbgs ( ) << <STR_LIT> << llvm :: printReg ( ( VRM -> getPhys ( LI -> reg ( ) ) ) , TRI ) ; dbgs ( ) << '\n' ) ; bool Success = scavengeRegs ( Intervals ) ; if ( ! Success ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( VRM -> hasPhys ( Intervals . back ( ) -> reg ( ) ) ) continue ; } else { auto I = std :: lower_bound ( Candidates . begin ( ) , & C , MinInd , [ this ] ( const Candidate & C , SlotIndex I ) { return LIS -> getInstructionIndex ( * C . first ) < I ; } ) ; for ( auto E = Candidates . end ( ) ; Success && I != E && LIS -> getInstructionIndex ( * I -> first ) < MaxInd ; ++ I ) { if ( I -> second && CheckNSA ( * I -> first , true ) < NSA_Status :: CONTIGUOUS ) { Success = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I -> first ) ; } } } if ( ! Success


<s> unsigned MaxReg = MaxNumVGPRs - NumRegs + <STR_LIT> ; for ( unsigned Reg = <STR_LIT> ; Reg <= MaxReg ; ++ Reg ) { if ( ! canAssign ( Reg , NumRegs ) ) continue ; if ( tryAssignRegisters ( Intervals , Reg ) ) return true ; } return false


<s> } } if ( ! Def0 || ! Def1 || Def0 -> getParent ( ) != Def1 -> getParent ( ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Def0 << <STR_LIT> << * Def1 << <STR_LIT> ) ; if ( SlotIndex :: isEarlierInstr ( LIS -> getInstructionIndex ( * Def1 ) , LIS -> getInstructionIndex ( * Def0 ) ) ) std :: swap ( Def0 , Def1 ) ; LIS -> RemoveMachineInstrFromMaps ( * Def0 ) ; LIS -> RemoveMachineInstrFromMaps ( * Def1 ) ; auto NewI = BuildMI ( * Def0 -> getParent ( ) , * Def0 , Def0 -> getDebugLoc ( ) , TII

-> get ( <STR_LIT> ) , Reg ) . addImm ( Init ) ;


<s> uint64_t Init = <NUM_LIT> ; for ( MachineInstr & I : MRI -> def_instructions ( Reg ) ) { if ( I . getOpcode ( ) != <STR_LIT> || I . getOperand ( <NUM_LIT> ) . getReg ( ) != Reg || ! I . getOperand ( <NUM_LIT> ) . isImm ( ) || I . getNumOperands ( ) != <NUM_LIT> ) return false ; switch ( I . getOperand ( <NUM_LIT> ) . getSubReg ( ) ) { default : return false ; case <STR_LIT> : if ( Def0 ) return false ; Def0 = & I ; Init |= I . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> ; break ; case <STR_LIT> : if ( Def1 ) return false ; Def1 = & I ; Init |= static_cast < uint64_t > ( I . getOperand ( <NUM_LIT> ) . getImm ( ) ) << <NUM_LIT> ; break ; } } if ( ! Def0 || ! Def1 || Def0 -> getParent ( ) != Def1 -> getParent ( ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Def0 << <STR_LIT> << * Def1 << <STR_LIT> ) ; if ( SlotIndex :: isEarlierInstr ( LIS -> getInstructionIndex ( * Def1 ) , LIS -> getInstructionIndex ( * Def0 ) ) ) std :: swap ( Def0 , Def1 ) ; LIS -> RemoveMachineInstrFromMaps ( * Def0 ) ; LIS -> RemoveMachineInstrFromMaps ( * Def1 ) ; auto NewI = BuildMI ( * Def0 -> getParent ( ) , * Def0 , Def0 -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Reg ) . addImm ( Init ) ; Def0 -> eraseFromParent ( ) ; Def1 -> eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * NewI ) ; LIS -> removeInterval ( Reg ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; LLVM_DEBUG ( dbgs ( ) <<


<s> FunctionPass * llvm :: createGCNPreRAOptimizationsPass ( ) { return new GCNPreRAOptimizations (


<s> initializeGCNPreRAOptimizationsPass ( * PassRegistry ::


<s> AU . addRequired < LiveIntervals > ( ) ; AU . setPreservesAll


<s> StringRef getPassName ( ) const override { return <STR_LIT>


<s> if ( I . getOperand ( <NUM_LIT> ) . getReg ( ) != Reg || ! I . getOperand ( <NUM_LIT> ) . isImm ( ) || I . getNumOperands ( ) != <NUM_LIT> ) return false ; switch ( I . getOperand ( <NUM_LIT> ) . getSubReg ( ) ) { default : return false ; case <STR_LIT> : if ( Def0 ) return false ; Def0 = & I ; Init |= I . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> ; break ; case <STR_LIT> : if ( Def1 ) return false ; Def1 = & I ; Init |= static_cast < uint64_t > ( I . getOperand ( <NUM_LIT> ) . getImm ( ) ) << <NUM_LIT> ; break ; } break ; } } if ( IsAGPRDst ) { if ( Changed ) { for ( Register RegToUpdate : ModifiedRegs ) { LIS -> removeInterval ( RegToUpdate ) ; LIS -> createAndComputeVirtRegInterval ( RegToUpdate ) ; } } return Changed ; } if ( ! Def0 || ! Def1 || Def0 -> getParent ( ) != Def1 -> getParent ( ) ) return Changed ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Def0 << <STR_LIT> << * Def1 << <STR_LIT> ) ; if ( SlotIndex :: isEarlierInstr ( LIS -> getInstructionIndex ( * Def1 ) , LIS -> getInstructionIndex ( * Def0 ) ) ) std :: swap ( Def0 , Def1 ) ; LIS -> RemoveMachineInstrFromMaps ( * Def0 ) ; LIS -> RemoveMachineInstrFromMaps ( * Def1 ) ; auto NewI = BuildMI ( * Def0 -> getParent ( ) , * Def0 , Def0 -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Reg ) . addImm ( Init ) ; Def0 -> eraseFromParent ( ) ; Def1 -> eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( *


<s> TII = ST . getInstrInfo ( ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; TRI = ST . getRegisterInfo ( ) ; bool Changed = false ; for ( unsigned I = <NUM_LIT> , E = MRI -> getNumVirtRegs ( ) ; I != E ; ++ I ) { Register Reg = Register :: index2VirtReg ( I ) ; if ( ! LIS -> hasInterval ( Reg ) ) continue ; const TargetRegisterClass * RC = MRI -> getRegClass ( Reg ) ; if ( ( RC -> MC -> getSizeInBits ( ) != <NUM_LIT> || ! TRI -> isSGPRClass ( RC ) ) && ( ST . hasGFX90AInsts ( )

|| ! TRI -> isAGPRClass ( RC ) ) ) continue ;


<s> if ( Bank != - <NUM_LIT> && R == Reg && Op . getSubReg ( ) ) { unsigned LM = TRI -> getSubRegIndexLaneMask ( Op . getSubReg ( ) ) . getAsInteger ( ) ; if ( ! ( LM & <NUM_LIT> ) && ( Bank < NUM_VGPR_BANKS ) ) { if ( countPopulation ( LM ) >= NUM_VGPR_BANKS ) continue ; ShiftedBank = ( Bank + countTrailingZeros ( LM ) ) % NUM_VGPR_BANKS ; } else if ( ! ( LM & <NUM_LIT> ) && ( Bank >= SGPR_BANK_OFFSET ) ) { if ( countPopulation ( LM ) / <NUM_LIT> >= NUM_SGPR_BANKS ) continue ; ShiftedBank = SGPR_BANK_OFFSET + ( Bank - SGPR_BANK_OFFSET + ( countTrailingZeros ( LM ) >> <NUM_LIT> ) ) % NUM_SGPR_BANKS ; } } unsigned Mask = getRegBankMask ( R , Op . getSubReg ( ) , ( Reg == R ) ? ShiftedBank : - <NUM_LIT> ) ; StallCycles += countPopulation ( UsedBanks & Mask ) ; UsedBanks |= Mask ; OperandMasks . push_back ( OperandMask ( Op . getReg ( ) , Op . getSubReg ( )


<s> Candidates . sort ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; for ( auto C : Candidates ) C . dump ( this ) ; dbgs ( ) << <STR_LIT> ) ; unsigned CyclesSaved = <NUM_LIT> ; while ( ! Candidates . empty ( ) ) { Candidate C = Candidates . back ( ) ; unsigned LocalCyclesSaved = tryReassign ( C ) ; CyclesSaved += LocalCyclesSaved ; if ( VerifyStallCycles > <NUM_LIT> && ! verifyCycles ( MF , StallCycles , CyclesSaved ) ) report_fatal_error ( <STR_LIT> ) ; Candidates . pop_back ( ) ; if ( LocalCyclesSaved ) { removeCandidates ( C . Reg ) ; computeStallCycles ( C . Reg , <STR_LIT> , - <NUM_LIT> , true ) ; Candidates . sort ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; for ( auto C : Candidates ) C . dump ( this ) ; dbgs ( ) << <STR_LIT> ) ; } } NumStallsRecovered +=


<s> unsigned Occupancy = MFI -> getOccupancy ( ) ; MaxNumVGPRs = ST -> getMaxNumVGPRs ( MF ) ; MaxNumSGPRs = ST -> getMaxNumSGPRs ( MF ) ; MaxNumVGPRs = std :: min ( ST -> getMaxNumVGPRs ( Occupancy ) , MaxNumVGPRs ) ; MaxNumSGPRs = std :: min ( ST -> getMaxNumSGPRs ( Occupancy , true ) , MaxNumSGPRs ) ; CSRegs = TRI -> getCalleeSavedRegs ( & MF ) ; RegsUsed . resize ( <STR_LIT> . getNumRegs ( ) + TRI -> getEncodingValue ( <STR_LIT> ) / <NUM_LIT> + <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MF . getName ( ) << '\n' ) ; unsigned StallCycles = collectCandidates ( MF ) ; NumStallsDetected += StallCycles ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << StallCycles << <STR_LIT> <STR_LIT> << MF . getName ( ) << '\n' ) ; Candidates . sort ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; for ( auto C : Candidates ) C . dump ( this ) ; dbgs ( ) << <STR_LIT> ) ; unsigned CyclesSaved = <NUM_LIT> ; while ( ! Candidates . empty ( ) ) { Candidate C = Candidates . back ( ) ; unsigned LocalCyclesSaved = tryReassign ( C ) ; CyclesSaved += LocalCyclesSaved ; if ( VerifyStallCycles > <NUM_LIT> && ! verifyCycles ( MF , StallCycles , CyclesSaved ) ) report_fatal_error ( <STR_LIT> ) ; Candidates . pop_back ( ) ; if ( LocalCyclesSaved ) { removeCandidates ( C . Reg ) ; computeStallCycles ( C . Reg , <STR_LIT> , - <NUM_LIT> , true ) ; Candidates . sort ( ) ; LLVM_DEBUG ( dbgs ( )


<s> if ( ! Op . isReg ( ) || Op . isUndef ( ) ) continue ; Register R = Op . getReg ( ) ; if ( TRI -> hasAGPRs ( TRI -> getRegClassForReg ( * MRI , R ) ) ) continue ; unsigned ShiftedBank = Bank ; if ( Bank != - <NUM_LIT> && R == Reg && Op . getSubReg ( ) ) { unsigned Offset = TRI -> getChannelFromSubReg ( Op . getSubReg ( ) ) ; LaneBitmask LM = TRI -> getSubRegIndexLaneMask ( Op . getSubReg ( ) ) ; if ( Offset && Bank < NUM_VGPR_BANKS ) { if ( TRI -> getNumCoveredRegs ( LM ) >= NUM_VGPR_BANKS ) continue ; ShiftedBank = ( Bank + Offset ) % NUM_VGPR_BANKS ; } else if ( Offset > <NUM_LIT> && Bank >= SGPR_BANK_OFFSET ) { if ( TRI -> getNumCoveredRegs ( LM ) / <NUM_LIT> >= NUM_SGPR_BANKS ) continue ; ShiftedBank = SGPR_BANK_OFFSET + ( Bank - SGPR_BANK_OFFSET + ( Offset >> <NUM_LIT> ) ) % NUM_SGPR_BANKS ; } } uint32_t Mask = getRegBankMask ( R , Op . getSubReg ( ) ,

( Reg == R ) ? ShiftedBank : - <NUM_LIT> ) ;


<s> else dbgs ( ) << MBB . getName ( ) ; dbgs ( ) << <STR_LIT> ; } ) ; for ( MachineInstr & MI : MBB . instrs ( ) ) { if ( MI . isBundle ( ) ) continue ; unsigned StallCycles ; unsigned UsedBanks ; std :: tie ( StallCycles , UsedBanks ) = analyzeInst ( MI ) ; if ( Collect ) collectCandidates ( MI , UsedBanks , StallCycles ) ; TotalStallCycles += StallCycles


<s> unsigned TotalStallCycles = <NUM_LIT> ; SmallSet < const MachineInstr * , <NUM_LIT> > Visited ; for ( auto & MI : MRI -> use_nodbg_instructions ( SrcReg ) ) { if ( MI . isBundle ( ) ) continue ; if ( ! Visited . insert ( & MI ) . second ) continue ; unsigned StallCycles ; unsigned


<s> unsigned GCNRegBankReassign :: getPhysRegBank ( unsigned Reg ) const { assert ( Register :: isPhysicalRegister ( Reg ) ) ; const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg ) ; unsigned Size = TRI -> getRegSizeInBits ( * RC ) ; if ( Size == <NUM_LIT> ) Reg = TRI -> get32BitRegister ( Reg ) ; else if ( Size > <NUM_LIT> ) Reg = TRI -> getSubReg ( Reg , <STR_LIT> ) ; if ( TRI -> hasVGPRs ( RC ) ) { Reg -= <STR_LIT> ; return Reg % NUM_VGPR_BANKS ; } Reg = TRI -> getEncodingValue ( Reg )


<s> if ( TRI -> hasVGPRs ( RC ) ) { Reg -= <STR_LIT> ; uint32_t Mask = maskTrailingOnes < uint32_t > ( Size ) ; unsigned Used = <NUM_LIT> ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) if ( RegsUsed . test ( Reg + I ) ) Used |= <NUM_LIT> << I ; RegsUsed . set ( Reg , Reg + Size ) ; Mask &= ~ Used ; Mask <<= ( Bank == - <NUM_LIT> ) ? Reg % NUM_VGPR_BANKS : uint32_t ( Bank ) ; return ( Mask | ( Mask >> NUM_VGPR_BANKS ) ) & VGPR_BANK_MASK ; } Reg = TRI -> getEncodingValue ( Reg ) / <NUM_LIT> ; unsigned StartBit = <STR_LIT> . getNumRegs ( ) ; if ( Reg + StartBit >= RegsUsed . size ( ) ) return <NUM_LIT> ; if ( Size > <NUM_LIT> ) Size /= <NUM_LIT> ; unsigned Mask = ( <NUM_LIT> << Size ) - <NUM_LIT> ; unsigned Used = <NUM_LIT> ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) if ( RegsUsed . test ( StartBit + Reg + I )


<s> const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg ) ; unsigned Size = TRI -> getRegSizeInBits ( * RC ) ; if ( Size == <NUM_LIT> ) { Reg = TRI -> get32BitRegister ( Reg ) ; Size = <NUM_LIT> ; } else { Size /= <NUM_LIT> ; if ( Size > <NUM_LIT> ) Reg = TRI -> getSubReg ( Reg , <STR_LIT> ) ; } if ( TRI -> hasVGPRs ( RC ) ) { Reg -= <STR_LIT> ; uint32_t Mask = maskTrailingOnes < uint32_t > ( Size ) ; unsigned Used = <NUM_LIT> ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) if ( RegsUsed . test ( Reg + I ) ) Used |= <NUM_LIT> << I ; RegsUsed . set ( Reg , Reg + Size ) ; Mask &= ~ Used ; Mask <<= ( Bank == - <NUM_LIT> ) ? Reg % NUM_VGPR_BANKS : uint32_t ( Bank ) ; return ( Mask | ( Mask >> NUM_VGPR_BANKS ) )


<s> Register PhysReg = VRM -> getPhys ( Reg ) ; if ( Def && Def -> isCopy ( ) && Def -> getOperand ( <NUM_LIT> ) . getReg ( ) == PhysReg ) return false ; for ( auto U : MRI -> use_nodbg_operands ( Reg ) ) { if ( U . isImplicit ( ) ) return false ; const MachineInstr * UseInst = U . getParent ( ) ; if ( UseInst -> isCopy ( ) && UseInst -> getOperand ( <NUM_LIT> ) . getReg ( ) == PhysReg ) return false ; } const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( PhysReg ) ; unsigned Size = TRI -> getRegSizeInBits ( * RC ) ; if ( Size < <NUM_LIT> ) return false ; if ( TRI -> hasVGPRs ( RC ) ) return true ; if ( Size == <NUM_LIT> ) return


<s> bool GCNRegBankReassign :: isReassignable ( unsigned Reg ) const { if ( Register :: isPhysicalRegister ( Reg ) || ! VRM -> isAssignedReg ( Reg ) ) return false ; const MachineInstr * Def = MRI -> getUniqueVRegDef ( Reg ) ; Register PhysReg = VRM -> getPhys ( Reg ) ; if ( Def && Def -> isCopy ( ) && Def -> getOperand ( <NUM_LIT> ) . getReg ( ) == PhysReg ) return false ; for ( auto U : MRI -> use_nodbg_operands ( Reg ) ) { if ( U . isImplicit ( ) ) return false ; const MachineInstr * UseInst = U . getParent ( ) ; if ( UseInst -> isCopy ( ) && UseInst -> getOperand ( <NUM_LIT> ) . getReg ( ) == PhysReg ) return false ; } const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( PhysReg ) ; unsigned Size = TRI -> getRegSizeInBits ( * RC ) ; if ( Size < <NUM_LIT>


<s> if ( countPopulation ( LM ) / <NUM_LIT> >= NUM_SGPR_BANKS ) continue ; ShiftedBank = SGPR_BANK_OFFSET + ( Bank - SGPR_BANK_OFFSET + ( countTrailingZeros ( LM ) >> <NUM_LIT> ) ) % NUM_SGPR_BANKS ; } } unsigned Mask = getRegBankMask ( R , Op . getSubReg ( ) , ( Reg == R ) ? ShiftedBank : - <NUM_LIT> ) ; StallCycles += countPopulation ( UsedBanks & Mask ) ; UsedBanks |= Mask ; OperandMasks . push_back ( OperandMask ( Op . getReg ( ) , Op . getSubReg ( ) , Mask


<s> if ( ! Reg ) return <NUM_LIT> ; if ( SubReg ) Reg = TRI -> getSubReg ( Reg , SubReg ) ; } const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg ) ; unsigned Size = TRI -> getRegSizeInBits ( * RC ) / <NUM_LIT> ; if ( Size > <NUM_LIT> ) Reg = TRI -> getSubReg ( Reg , <STR_LIT> ) ; if ( TRI -> hasVGPRs ( RC ) ) { Reg -= <STR_LIT> ; unsigned Mask = ( <NUM_LIT> << Size ) - <NUM_LIT> ; unsigned Used = <NUM_LIT> ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) if ( RegsUsed . test ( Reg + I ) ) Used |= <NUM_LIT> << I ; RegsUsed . set ( Reg , Reg + Size ) ; Mask &= ~ Used ; Mask <<= ( Bank == - <NUM_LIT> ) ? Reg % NUM_VGPR_BANKS : unsigned ( Bank ) ; return ( Mask | ( Mask >> NUM_VGPR_BANKS ) ) & VGPR_BANK_MASK ; } Reg = TRI -> getEncodingValue ( Reg ) / <NUM_LIT> ; unsigned StartBit = <STR_LIT> . getNumRegs ( ) ; if ( Reg + StartBit >= RegsUsed . size ( ) ) return <NUM_LIT> ; if ( Size > <NUM_LIT> ) Size /= <NUM_LIT> ; unsigned Mask = ( <NUM_LIT> << Size ) - <NUM_LIT> ; unsigned Used = <NUM_LIT> ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) if ( RegsUsed . test ( StartBit + Reg + I ) ) Used |= <NUM_LIT> << I ; RegsUsed . set ( StartBit + Reg , StartBit + Reg + Size


<s> unsigned Stalls ; } ; SmallVector < BankStall , <NUM_LIT> > BankStalls ; for ( int Bank = <NUM_LIT> ; Bank < NUM_BANKS ; ++ Bank ) { if ( C . FreeBanks & ( <NUM_LIT> << Bank ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printBank ( Bank ) << '\n' ) ; unsigned Stalls = computeStallCycles ( C . Reg , C . Reg , Bank ) ; if ( Stalls < OrigStalls ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printBank ( Bank ) << <STR_LIT> << Stalls << '\n' ) ; BankStalls . push_back ( BankStall ( ( unsigned ) Bank , Stalls ) ) ; } } } std :: sort ( BankStalls . begin ( ) , BankStalls . end ( ) ) ; Register OrigReg = VRM -> getPhys ( C . Reg ) ; LRM -> unassign ( LI ) ; while ( ! BankStalls . empty ( ) ) { BankStall BS = BankStalls . pop_back_val ( ) ; unsigned Reg = scavengeReg ( LI , BS


<s> if ( ! LIS -> hasInterval ( C . Reg ) ) return <NUM_LIT> ; LiveInterval & LI = LIS -> getInterval ( C . Reg ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( C . Reg ) << <STR_LIT> ; C . MI -> dump ( ) ; LI . dump ( ) ) ; unsigned OrigStalls = computeStallCycles ( C . Reg ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << OrigStalls << '\n' ) ; if ( ! OrigStalls ) return <NUM_LIT> ; struct BankStall { BankStall ( unsigned b , unsigned s ) : Bank ( b ) , Stalls ( s ) { } ; bool operator < ( const BankStall & RHS ) const { return Stalls > RHS . Stalls ; } unsigned Bank ; unsigned Stalls ; } ; SmallVector < BankStall , <NUM_LIT> > BankStalls ; for ( int Bank = <NUM_LIT> ; Bank < NUM_BANKS ; ++ Bank ) { if ( C . FreeBanks & ( <NUM_LIT> <<


<s> Register R = Op . getReg ( ) ; if ( TRI -> hasAGPRs ( TRI -> getRegClassForReg ( * MRI , R ) ) ) continue ; unsigned ShiftedBank = Bank ; if ( Bank != - <NUM_LIT> && R == Reg && Op . getSubReg ( ) ) { unsigned LM = TRI -> getSubRegIndexLaneMask ( Op . getSubReg ( ) ) . getAsInteger ( ) ; if ( ! ( LM & <NUM_LIT> ) && ( Bank < NUM_VGPR_BANKS ) ) { if ( countPopulation ( LM ) >= NUM_VGPR_BANKS ) continue ; ShiftedBank = ( Bank + countTrailingZeros ( LM ) ) % NUM_VGPR_BANKS ; } else if ( ! ( LM & <NUM_LIT> ) && ( Bank >= SGPR_BANK_OFFSET ) ) { if ( countPopulation ( LM ) / <NUM_LIT> >= NUM_SGPR_BANKS ) continue ; ShiftedBank = SGPR_BANK_OFFSET + ( Bank - SGPR_BANK_OFFSET + ( countTrailingZeros ( LM ) >> <NUM_LIT> ) ) % NUM_SGPR_BANKS ; } } uint32_t Mask = getRegBankMask ( R , Op . getSubReg ( ) , ( Reg == R ) ? ShiftedBank : - <NUM_LIT> ) ; StallCycles += countPopulation (


<s> if ( MI . isDebugValue ( ) ) return <NUM_LIT> ; RegsUsed . reset ( ) ; OperandMasks . clear ( ) ; for ( const auto & Op : MI . explicit_uses ( ) ) { if ( ! Op . isReg ( ) || Op . isUndef ( ) ) continue ; Register R = Op . getReg ( ) ; if ( TRI -> hasAGPRs ( TRI -> getRegClassForReg ( * MRI , R ) ) ) continue ; unsigned ShiftedBank = Bank ; if ( Bank != - <NUM_LIT> && R == Reg && Op . getSubReg ( ) ) { unsigned Offset = TRI -> getChannelFromSubReg ( Op . getSubReg ( ) ) ; LaneBitmask LM = TRI -> getSubRegIndexLaneMask ( Op . getSubReg ( ) ) ; if ( Offset && Bank < NUM_VGPR_BANKS ) { if ( TRI -> getNumCoveredRegs ( LM ) >= NUM_VGPR_BANKS


<s> unsigned Shift = Offset ; if ( Shift >= NUM_VGPR_BANKS ) return <NUM_LIT> ; unsigned VB = FreeBanks & VGPR_BANK_MASK ; FreeBanks = ( ( VB >> Shift ) | ( VB << ( NUM_VGPR_BANKS - Shift ) ) ) & VGPR_BANK_MASK ; } else if ( Offset > <NUM_LIT> && ( Mask & SGPR_BANK_MASK ) ) { unsigned Shift = Offset >> <NUM_LIT> ; if ( Shift >= NUM_SGPR_BANKS ) return <NUM_LIT> ; unsigned SB = FreeBanks >> SGPR_BANK_OFFSET ; FreeBanks = ( ( SB >> Shift ) | ( SB << ( NUM_SGPR_BANKS - Shift ) ) ) & SGPR_BANK_SHIFTED_MASK ; FreeBanks <<=


<s> assert ( Register :: isPhysicalRegister ( Reg ) ) ; const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg ) ; unsigned Size = TRI -> getRegSizeInBits ( * RC ) ; if ( Size > <NUM_LIT> ) Reg = TRI -> getSubReg ( Reg ,