<s> Reg = TRI -> getEncodingValue ( Reg ) / <NUM_LIT> ; unsigned StartBit = <STR_LIT> . getNumRegs ( ) ; if ( Reg + StartBit >= RegsUsed . size ( ) ) return <NUM_LIT> ; if ( Size > <NUM_LIT> ) Size /= <NUM_LIT> ; unsigned Mask = ( <NUM_LIT> << Size ) - <NUM_LIT> ; unsigned Used = <NUM_LIT> ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) if ( RegsUsed . test ( StartBit + Reg + I ) ) Used |= <NUM_LIT> << I ; RegsUsed . set ( StartBit + Reg , StartBit + Reg + Size ) ; Mask &= ~ Used ; Mask <<= ( Bank == - <NUM_LIT> ) ? Reg % NUM_SGPR_BANKS : unsigned ( Bank - SGPR_BANK_OFFSET ) ; Mask = ( Mask | ( Mask >> NUM_SGPR_BANKS ) ) & SGPR_BANK_SHIFTED_MASK ; return Mask


<s> const MachineInstr * UseInst = U . getParent ( ) ; if ( UseInst -> isCopy ( ) && UseInst -> getOperand ( <NUM_LIT> ) . getReg ( ) == PhysReg ) return false ; } const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( PhysReg ) ; if ( TRI -> hasVGPRs ( RC ) ) return true ; unsigned Size = TRI -> getRegSizeInBits ( * RC ) ; if ( Size > <NUM_LIT> ) PhysReg = TRI -> getSubReg ( PhysReg , <STR_LIT> ) ; return <STR_LIT> . contains


<s> if ( U . isImplicit ( ) ) return false ; const MachineInstr * UseInst = U . getParent ( ) ; if ( UseInst -> isCopy ( ) && UseInst -> getOperand ( <NUM_LIT> ) . getReg ( ) == PhysReg ) return false ; } const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( PhysReg ) ; if ( TRI -> hasVGPRs ( RC )


<s> if ( Register :: isPhysicalRegister ( Reg ) ) { OS << llvm :: printReg ( Reg , TRI ) ; return ; } if ( ! VRM -> isAssignedReg ( Reg ) ) OS << <STR_LIT> << llvm :: printReg ( Reg , TRI ) ; else OS << llvm :: printReg ( Reg , TRI ) << '(' << llvm :: printReg ( VRM -> getPhys ( Reg ) , TRI


<s> unsigned Shift = Offset ; if ( Shift >= NUM_VGPR_BANKS ) return <NUM_LIT> ; unsigned VB = FreeBanks & VGPR_BANK_MASK ; FreeBanks = ( ( VB >> Shift ) | ( VB << ( NUM_VGPR_BANKS - Shift ) ) ) & VGPR_BANK_MASK ; } else if ( Offset > <NUM_LIT> && ( Mask & SGPR_BANK_MASK ) ) { unsigned Shift = Offset >> <NUM_LIT> ; if ( Shift >= NUM_SGPR_BANKS ) return <NUM_LIT> ; unsigned SB = FreeBanks >> SGPR_BANK_OFFSET ; FreeBanks = ( ( SB >> Shift ) | ( SB << ( NUM_SGPR_BANKS - Shift ) ) ) & SGPR_BANK_SHIFTED_MASK ; FreeBanks <<=


<s> MachineBasicBlock :: const_instr_iterator Def ( MI . getIterator ( ) ) ; MachineBasicBlock :: const_instr_iterator B ( MI . getParent ( ) -> instr_begin ( ) ) ; for ( unsigned S = StallCycles ; S && Def != B && Defs != <NUM_LIT> ; -- S ) { if ( MI . isDebugInstr (


<s> for ( unsigned S = StallCycles ; S && Def != B && Defs != <NUM_LIT> ; -- S ) { if ( MI . isDebugInstr ( ) ) continue ; -- Def ; if ( Def -> getOpcode ( ) == TargetOpcode :: IMPLICIT_DEF ) continue ; if ( Def -> modifiesRegister ( Reg1 , TRI ) )


<s> unsigned Size = TRI -> getRegSizeInBits ( * RC ) ; if ( Size == <NUM_LIT> ) Reg = TRI -> get32BitRegister ( Reg ) ; else if ( Size > <NUM_LIT> ) { if ( SubReg ) { const TargetRegisterClass * SubRC = TRI -> getSubRegClass ( RC , SubReg ) ; Reg = TRI -> getSubReg ( Reg , SubReg ) ; if ( TRI -> getRegSizeInBits ( * SubRC ) > <NUM_LIT> )

Reg = TRI -> getSubReg ( Reg , <STR_LIT> ) ;


<s> if ( ! Reg ) return <NUM_LIT> ; if ( SubReg ) Reg = TRI -> getSubReg ( Reg , SubReg ) ; } const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg ) ; unsigned Size = TRI -> getRegSizeInBits ( * RC ) ; if ( Size == <NUM_LIT> ) { Reg = TRI -> get32BitRegister ( Reg ) ; Size = <NUM_LIT> ; } else { Size /= <NUM_LIT> ; if ( Size > <NUM_LIT> ) Reg = TRI -> getSubReg ( Reg , <STR_LIT> ) ; } if ( TRI -> hasVGPRs ( RC ) ) { unsigned RegNo = Reg - <STR_LIT> ; uint32_t Mask = maskTrailingOnes < uint32_t > ( Size ) ; unsigned Used = <NUM_LIT> ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) if ( RegsUsed . test ( RegNo + I ) ) Used |= <NUM_LIT> << I ; RegsUsed . set ( RegNo , RegNo + Size ) ; Mask &= ~ Used ; Mask <<= ( Bank == - <NUM_LIT> ) ? RegNo % NUM_VGPR_BANKS : uint32_t ( Bank ) ; return ( Mask | ( Mask >> NUM_VGPR_BANKS ) ) & VGPR_BANK_MASK ; } unsigned RegNo = TRI -> getEncodingValue ( <STR_LIT> ( Reg , * ST ) ) / <NUM_LIT> ; unsigned StartBit = <STR_LIT> . getNumRegs ( ) ; if ( RegNo + StartBit >= RegsUsed . size ( ) ) return <NUM_LIT> ; if ( Size > <NUM_LIT> ) Size /= <NUM_LIT> ; unsigned Mask = ( <NUM_LIT> << Size ) - <NUM_LIT> ; unsigned Used = <NUM_LIT> ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) if ( RegsUsed . test ( StartBit + RegNo + I ) ) Used |= <NUM_LIT> << I ; RegsUsed . set ( StartBit + RegNo , StartBit + RegNo + Size ) ; Mask &=


<s> void pop_back ( ) { rbegin ( ) -> second . pop_back ( ) ; if ( rbegin ( ) -> second . empty ( ) ) erase (


<s> if ( rbegin ( ) -> second . empty ( ) ) erase ( rbegin (


<s> if ( TRI -> isSubRegisterEq ( Reg , MaxReg ) ) break ; if ( ! MRI -> isAllocatable ( Reg ) || getPhysRegBank ( Reg , SubReg ) != Bank ) continue ; for ( unsigned I = <NUM_LIT> ; CSRegs [ I ] ; ++ I ) if ( TRI -> isSubRegisterEq ( Reg , CSRegs [ I ] ) && ! LRM -> isPhysRegUsed ( CSRegs [ I ] ) ) return MCRegister :: from ( <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg ) << '\n' ) ; if ( ! LRM -> checkInterference ( LI , Reg


<s> for ( const auto & Op : MI . explicit_uses ( ) ) { if ( ! Op . isReg ( ) || Op . isUndef ( ) ) continue ; const Register R = Op . getReg ( ) ; const TargetRegisterClass * RC = TRI -> getRegClassForReg ( * MRI , R ) ; if ( TRI -> hasAGPRs ( RC ) ) continue ; if ( Op . getSubReg ( ) ) { LaneBitmask LM = TRI -> getSubRegIndexLaneMask ( Op . getSubReg ( ) ) ; if ( TRI -> hasVGPRs ( RC ) ) { if ( TRI -> getNumCoveredRegs ( LM ) >= NUM_VGPR_BANKS ) continue ; } else { if ( TRI -> getNumCoveredRegs ( LM ) / <NUM_LIT> >= NUM_SGPR_BANKS ) continue ; } } unsigned ShiftedBank = Bank ; if ( Bank != - <NUM_LIT> && R == Reg && ( Op . getSubReg ( ) || SubReg ) ) { unsigned RegOffset = TRI -> getChannelFromSubReg ( SubReg ? SubReg : ( unsigned ) <STR_LIT> ) ; unsigned Offset = TRI -> getChannelFromSubReg ( Op . getSubReg ( ) ? Op . getSubReg ( ) : ( unsigned ) <STR_LIT> ) ; if ( Bank < NUM_VGPR_BANKS ) { unsigned Shift = ( ( NUM_VGPR_BANKS + Offset ) - RegOffset ) ; ShiftedBank = ( Bank + Shift ) % NUM_VGPR_BANKS ; } else if ( Bank >= SGPR_BANK_OFFSET ) { unsigned Shift = ( NUM_SGPR_BANKS + ( Offset >> <NUM_LIT> ) ) - ( RegOffset >> <NUM_LIT> ) ; ShiftedBank = SGPR_BANK_OFFSET + ( Bank -

SGPR_BANK_OFFSET + Shift ) % NUM_SGPR_BANKS ;


<s> if ( Op . getSubReg ( ) ) { LaneBitmask LM = TRI -> getSubRegIndexLaneMask ( Op . getSubReg ( ) ) ; if ( TRI -> hasVGPRs ( RC ) ) { if ( TRI -> getNumCoveredRegs ( LM ) >= NUM_VGPR_BANKS ) continue ; } else { if ( TRI -> getNumCoveredRegs ( LM ) / <NUM_LIT> >= NUM_SGPR_BANKS ) continue ; } } unsigned ShiftedBank = Bank ; if ( Bank != - <NUM_LIT> && R == Reg && ( Op . getSubReg ( ) || SubReg ) ) { unsigned RegOffset = TRI -> getChannelFromSubReg ( SubReg ? SubReg : ( unsigned ) <STR_LIT> ) ; unsigned Offset = TRI -> getChannelFromSubReg ( Op . getSubReg ( ) ? Op . getSubReg ( ) : ( unsigned ) <STR_LIT> ) ; if ( Bank < NUM_VGPR_BANKS ) { unsigned Shift = ( ( NUM_VGPR_BANKS +


<s> if ( ! Visited . insert ( & MI ) . second ) continue ; unsigned StallCycles ; unsigned UsedBanks ; std :: tie ( StallCycles , UsedBanks ) = analyzeInst ( MI ,


<s> if ( Size == <NUM_LIT> ) { Reg = TRI -> get32BitRegister ( Reg ) ; Size = <NUM_LIT> ; } else { Size /= <NUM_LIT> ; if ( Size > <NUM_LIT> ) Reg = TRI -> getSubReg ( Reg , <STR_LIT> ) ; } if ( TRI -> hasVGPRs ( RC ) ) { unsigned RegNo = Reg - <STR_LIT> ; uint32_t Mask = maskTrailingOnes < uint32_t > ( Size ) ; unsigned Used = <NUM_LIT> ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) if ( RegsUsed . test ( RegNo + I ) ) Used |= <NUM_LIT> << I ; RegsUsed . set ( RegNo , RegNo + Size ) ; Mask &= ~ Used ; Mask <<= ( Bank == - <NUM_LIT> ) ? RegNo % NUM_VGPR_BANKS : uint32_t ( Bank ) ; return ( Mask | ( Mask >> NUM_VGPR_BANKS ) ) & VGPR_BANK_MASK ; } unsigned RegNo = TRI -> getEncodingValue ( Reg ) / <NUM_LIT> ; unsigned StartBit = <STR_LIT> . getNumRegs ( ) ; if ( RegNo + StartBit >= RegsUsed . size ( ) ) return <NUM_LIT> ; if ( Size > <NUM_LIT> ) Size /= <NUM_LIT> ; unsigned Mask = ( <NUM_LIT> << Size ) - <NUM_LIT> ; unsigned Used = <NUM_LIT> ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) if ( RegsUsed . test ( StartBit + RegNo + I ) ) Used |= <NUM_LIT> << I ; RegsUsed . set ( StartBit + RegNo , StartBit + RegNo + Size ) ; Mask &= ~ Used ; Mask <<= ( Bank == - <NUM_LIT> ) ? RegNo % NUM_SGPR_BANKS : unsigned ( Bank - SGPR_BANK_OFFSET ) ; Mask = ( Mask | ( Mask >> NUM_SGPR_BANKS ) ) & SGPR_BANK_SHIFTED_MASK ; return Mask << SGPR_BANK_OFFSET


<s> ST = & MF . getSubtarget < GCNSubtarget > ( ) ; if ( ! ST -> hasRegisterBanking ( ) || skipFunction ( MF . getFunction ( ) ) ) return false ; MRI = & MF . getRegInfo ( ) ; TRI = ST -> getRegisterInfo ( ) ; MLI = & getAnalysis < MachineLoopInfo > ( ) ; VRM = & getAnalysis < VirtRegMap > ( ) ; LRM = & getAnalysis < LiveRegMatrix > ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned Occupancy = MFI -> getOccupancy ( ) ; MaxNumVGPRs = ST -> getMaxNumVGPRs ( MF ) ; MaxNumSGPRs = ST -> getMaxNumSGPRs ( MF ) ; MaxNumVGPRs = std :: min ( ST -> getMaxNumVGPRs (


<s> unsigned OrigStalls = computeStallCycles ( C . Reg ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << OrigStalls << '\n' ) ; if ( ! OrigStalls ) return <NUM_LIT> ; struct BankStall { BankStall ( unsigned b , unsigned s ) : Bank ( b ) , Stalls ( s ) { } ; bool operator < ( const BankStall & RHS ) const { if ( Stalls == RHS . Stalls ) return Bank < RHS . Bank ; return Stalls > RHS . Stalls ; } unsigned Bank ; unsigned Stalls ; } ; SmallVector < BankStall , <NUM_LIT> > BankStalls ; for ( int Bank = <NUM_LIT> ; Bank < NUM_BANKS ; ++ Bank ) { if ( C . FreeBanks & ( <NUM_LIT> << Bank ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printBank ( Bank ) << '\n' ) ; unsigned Stalls = computeStallCycles ( C . Reg , C . Reg , C . SubReg , Bank ) ; if ( Stalls < OrigStalls ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printBank ( Bank ) << <STR_LIT> << Stalls << '\n' ) ; BankStalls . push_back ( BankStall ( ( unsigned ) Bank , Stalls ) ) ; } } } llvm :: sort ( BankStalls ) ; Register OrigReg = VRM -> getPhys ( C . Reg ) ; LRM -> unassign ( LI ) ; while ( ! BankStalls . empty ( ) ) { BankStall BS = BankStalls . pop_back_val ( ) ; Register Reg = scavengeReg ( LI , BS . Bank , C . SubReg ) ; if ( Reg == <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printBank ( BS . Bank ) << '\n' ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg ) << ( LRM -> isPhysRegUsed ( Reg ) ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << printBank ( BS . Bank ) << '\n' ) ; LRM -> assign ( LI , Reg ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << OrigStalls - BS . Stalls << '\n' ) ; return OrigStalls - BS . Stalls


<s> unsigned StallCycles = <NUM_LIT> ; UsedBanks = <NUM_LIT> ; if ( MI . isDebugValue ( ) ) return <NUM_LIT> ; RegsUsed . reset ( ) ; OperandMasks . clear ( ) ; for ( const auto & Op : MI . explicit_uses ( ) ) { if ( ! Op . isReg ( ) || Op . isUndef ( ) ) continue ; unsigned R = Op . getReg ( ) ; if ( TRI -> hasAGPRs ( TRI -> getRegClassForReg ( * MRI , R ) ) ) continue ; unsigned ShiftedBank = Bank ; if ( Bank != - <NUM_LIT> && R == Reg && Op . getSubReg ( ) ) { unsigned LM = TRI -> getSubRegIndexLaneMask ( Op . getSubReg ( ) ) . getAsInteger ( ) ; if ( ! ( LM & <NUM_LIT> ) && ( Bank < NUM_VGPR_BANKS ) ) { if ( countPopulation ( LM ) >= NUM_VGPR_BANKS ) continue ; ShiftedBank = ( Bank + countTrailingZeros ( LM ) ) % NUM_VGPR_BANKS ; } else if ( ! ( LM & <NUM_LIT> ) && ( Bank >= SGPR_BANK_OFFSET ) ) { if ( countPopulation ( LM ) / <NUM_LIT> >= NUM_SGPR_BANKS ) continue ; ShiftedBank = SGPR_BANK_OFFSET + ( Bank - SGPR_BANK_OFFSET + ( countTrailingZeros ( LM ) >> <NUM_LIT> ) ) % NUM_SGPR_BANKS ; } } unsigned Mask = getRegBankMask ( R , Op . getSubReg ( ) , ( Reg == R ) ? ShiftedBank : - <NUM_LIT> ) ; StallCycles += countPopulation ( UsedBanks & Mask ) ; UsedBanks |= Mask ; OperandMasks . push_back ( OperandMask ( Op . getReg ( ) , Op . getSubReg ( ) , Mask ) ) ; } return StallCycles


<s> BankStall ( unsigned b , unsigned s ) : Bank ( b )


<s> BankStall ( unsigned b , unsigned s ) : Bank (


<s> for ( auto & MI : MRI -> use_nodbg_instructions ( SrcReg ) ) { if ( MI . isBundle ( ) ) continue ; if ( ! Visited . insert ( & MI ) . second


<s> void dump ( const GCNRegBankReassign * P ) const { MI -> dump ( ) ; dbgs ( ) << P -> printReg


<s> for ( unsigned L = <NUM_LIT> ; L < NUM_BANKS ; ++ L ) if ( FreeBanks & ( <NUM_LIT>

<<
L
)
)
dbgs
(
)
<<
printBank
(
L
)
<<
' '
;


<s> if ( ! ( LM & <NUM_LIT> ) && ( Mask & VGPR_BANK_MASK ) ) { unsigned Shift = countTrailingZeros ( LM ) ; if ( Shift >= NUM_VGPR_BANKS ) return <NUM_LIT> ; unsigned VB = FreeBanks & VGPR_BANK_MASK ; FreeBanks = ( ( VB >> Shift ) | ( VB << ( NUM_VGPR_BANKS - Shift ) ) ) & VGPR_BANK_MASK ; } else if ( ! ( LM & <NUM_LIT> ) && ( Mask & SGPR_BANK_MASK ) ) { unsigned Shift = countTrailingZeros ( LM ) >> <NUM_LIT> ; if ( Shift >= NUM_SGPR_BANKS ) return <NUM_LIT> ; unsigned SB = FreeBanks >> SGPR_BANK_OFFSET ; FreeBanks = ( ( SB >> Shift ) | ( SB << ( NUM_SGPR_BANKS - Shift ) ) ) & SGPR_BANK_SHIFTED_MASK ; FreeBanks <<= SGPR_BANK_OFFSET ; } LLVM_DEBUG ( if ( FreeBanks ) { dbgs ( ) << <STR_LIT> << printReg ( Reg , SubReg ) << <STR_LIT> ; dumpFreeBanks (


<s> StringRef getPassName ( ) const override { return <STR_LIT>


<s> const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg ) ; unsigned Size = TRI -> getRegSizeInBits ( * RC ) ; if ( Size > <NUM_LIT> ) Reg = TRI -> getSubReg ( Reg , <STR_LIT> ) ; if ( TRI -> hasVGPRs ( RC ) ) { Reg -=


<s> if ( ! Reg ) return <NUM_LIT> ; if ( SubReg ) Reg = TRI -> getSubReg ( Reg , SubReg ) ; } const TargetRegisterClass * RC = TRI -> getMinimalPhysRegClass ( Reg ) ; unsigned Size = TRI -> getRegSizeInBits ( * RC ) / <NUM_LIT> ; if ( Size > <NUM_LIT> ) Reg = TRI -> getSubReg ( Reg , <STR_LIT> ) ; if ( TRI -> hasVGPRs ( RC ) ) { Reg -= <STR_LIT> ; unsigned Mask = ( <NUM_LIT> << Size ) - <NUM_LIT> ; unsigned Used = <NUM_LIT> ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) if ( RegsUsed . test ( Reg + I ) ) Used |= <NUM_LIT> << I ; RegsUsed . set ( Reg , Reg + Size ) ; Mask &= ~ Used ; Mask <<= ( Bank == - <NUM_LIT> ) ? Reg % NUM_VGPR_BANKS : unsigned ( Bank ) ; return ( Mask | ( Mask >> NUM_VGPR_BANKS ) ) & VGPR_BANK_MASK ; } Reg = TRI -> getEncodingValue ( Reg ) / <NUM_LIT> ; unsigned StartBit = <STR_LIT> . getNumRegs ( ) ; if ( Reg + StartBit >= RegsUsed . size ( ) ) return <NUM_LIT> ; if ( Size > <NUM_LIT> ) Size /= <NUM_LIT> ; unsigned Mask = ( <NUM_LIT> << Size ) -


<s> LLVM_DEBUG ( if ( Collect ) { dbgs ( ) << '\n'


<s> if ( TargetRegisterInfo :: isPhysicalRegister ( Reg ) || ! VRM -> isAssignedReg ( Reg ) ) return false ; const MachineInstr * Def = MRI -> getUniqueVRegDef ( Reg ) ; unsigned PhysReg = VRM -> getPhys ( Reg ) ; if ( Def && Def -> isCopy ( ) && Def -> getOperand ( <NUM_LIT> ) . getReg ( ) == PhysReg


<s> OS << ( ( Bank >= SGPR_BANK_OFFSET ) ? Bank -


<s> OS << ( ( Bank >= SGPR_BANK_OFFSET ) ? Bank - SGPR_BANK_OFFSET : Bank ) ; }


<s> if ( C . Weight ) push_back ( C ) ; else push_front ( C )


<s> MaxNumSGPRs = ST -> getMaxNumSGPRs ( MF ) ; MaxNumVGPRs = std :: min ( ST -> getMaxNumVGPRs ( Occupancy ) , MaxNumVGPRs ) ; MaxNumSGPRs = std :: min ( ST -> getMaxNumSGPRs ( Occupancy , true ) , MaxNumSGPRs ) ; CSRegs = MRI -> getCalleeSavedRegs ( ) ; RegsUsed . resize ( <STR_LIT> . getNumRegs ( ) + TRI -> getEncodingValue ( <STR_LIT> ) / <NUM_LIT> + <NUM_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MF . getName ( ) << '\n' ) ; unsigned StallCycles = collectCandidates ( MF ) ; NumStallsDetected += StallCycles ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << StallCycles << <STR_LIT> <STR_LIT> << MF . getName ( ) << '\n' ) ; Candidates . sort ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; for ( auto C : Candidates ) C . dump ( this ) ; dbgs ( ) << <STR_LIT> ) ; unsigned CyclesSaved = <NUM_LIT> ; while ( ! Candidates . empty ( ) ) { Candidate C = Candidates . back ( ) ; unsigned LocalCyclesSaved = tryReassign ( C ) ; CyclesSaved += LocalCyclesSaved ; if ( VerifyStallCycles > <NUM_LIT> && ! verifyCycles ( MF , StallCycles , CyclesSaved ) ) report_fatal_error ( <STR_LIT> ) ; Candidates . pop_back ( ) ; if ( LocalCyclesSaved ) { removeCandidates ( C


<s> for ( unsigned I = <NUM_LIT> ; CSRegs [ I ] ; ++ I ) if ( TRI -> isSubRegisterEq ( Reg , CSRegs [ I ] ) && ! LRM -> isPhysRegUsed ( CSRegs [ I ] ) ) return <STR_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg ) << '\n' ) ; if ( ! LRM -> checkInterference ( LI , Reg )


<s> LRM -> unassign ( LI ) ; while ( ! BankStalls . empty ( ) ) { BankStall BS = BankStalls . pop_back_val ( ) ; unsigned Reg = scavengeReg ( LI , BS . Bank ) ; if ( Reg == <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printBank ( BS . Bank ) << '\n' ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg ) << ( LRM -> isPhysRegUsed ( Reg ) ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << printBank ( BS . Bank ) << '\n' ) ; LRM -> assign ( LI , Reg ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << OrigStalls - BS . Stalls << '\n' ) ; return OrigStalls - BS .


<s> unsigned Bank ; unsigned Stalls ; } ; SmallVector < BankStall , <NUM_LIT> > BankStalls ; for ( int Bank = <NUM_LIT> ; Bank < NUM_BANKS ; ++ Bank ) { if ( C . FreeBanks & ( <NUM_LIT> << Bank ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printBank ( Bank ) << '\n' ) ; unsigned Stalls = computeStallCycles ( C . Reg , C . Reg , Bank ) ; if ( Stalls < OrigStalls ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printBank ( Bank ) << <STR_LIT> << Stalls << '\n' ) ; BankStalls . push_back ( BankStall ( ( unsigned ) Bank , Stalls ) ) ; } } } std :: sort ( BankStalls . begin ( ) , BankStalls . end ( ) ) ; unsigned OrigReg = VRM -> getPhys ( C . Reg ) ; LRM -> unassign ( LI ) ; while ( ! BankStalls . empty ( ) ) { BankStall BS = BankStalls . pop_back_val ( ) ; unsigned Reg = scavengeReg ( LI , BS . Bank ) ; if ( Reg == <STR_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printBank ( BS . Bank ) << '\n' ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg ) << ( LRM -> isPhysRegUsed ( Reg ) ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << printBank ( BS . Bank ) << '\n' ) ; LRM -> assign ( LI , Reg ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << OrigStalls - BS . Stalls << '\n' ) ; return OrigStalls -


<s> bool GCNRegBankReassign :: verifyCycles ( MachineFunction & MF , unsigned OriginalCycles , unsigned CyclesSaved ) { unsigned StallCycles = collectCandidates ( MF , false ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << StallCycles << <STR_LIT> ) ; return StallCycles + CyclesSaved


<s> bool higherOccupancy ( const <ISA_LIT>Subtarget & ST , const GCNRegPressure & O ) const { return getOccupancy ( ST ) > O . getOccupancy


<s> std :: sort ( Indexes . begin ( ) , Indexes . end ( ) ) ; auto & MRI = ( * R . begin ( ) ) -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; DenseMap < MachineInstr * , GCNRPTracker :: LiveRegSet > LiveRegMap ; SmallVector < SlotIndex , <NUM_LIT> > LiveIdxs , SRLiveIdxs ; for ( unsigned I = <NUM_LIT> , E = MRI . getNumVirtRegs ( ) ; I != E ; ++ I ) { auto Reg = Register :: index2VirtReg ( I ) ; if ( ! LIS . hasInterval ( Reg ) ) continue ; auto & LI = LIS . getInterval ( Reg ) ; LiveIdxs . clear (


<s> } std :: sort ( Indexes . begin ( ) , Indexes . end ( ) ) ; auto & MRI = ( * R . begin ( ) ) -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; DenseMap < MachineInstr * , GCNRPTracker :: LiveRegSet > LiveRegMap ; SmallVector < SlotIndex , <NUM_LIT> > LiveIdxs , SRLiveIdxs ; for ( unsigned I = <NUM_LIT> , E = MRI . getNumVirtRegs ( ) ; I != E ; ++ I ) { auto Reg = Register :: index2VirtReg ( I ) ; if ( ! LIS . hasInterval ( Reg ) ) continue ; auto & LI = LIS . getInterval ( Reg ) ; LiveIdxs . clear ( ) ; if ( ! LI . findIndexesLiveAt ( Indexes , std :: back_inserter ( LiveIdxs ) ) ) continue ; if ( ! LI . hasSubRanges ( ) ) { for ( auto <ISA_LIT> : LiveIdxs ) LiveRegMap [ <ISA_LIT>I . getInstructionFromIndex ( <ISA_LIT> ) ] [ Reg ] = MRI . getMaxLaneMaskForVReg ( Reg ) ; } else for ( const auto & S : LI . subranges ( ) ) { SRLiveIdxs . clear ( ) ; S . findIndexesLiveAt ( LiveIdxs , std :: back_inserter ( SRLiveIdxs ) ) ; for ( auto <ISA_LIT> : SRLiveIdxs ) LiveRegMap [ <ISA_LIT>I . getInstructionFromIndex ( <ISA_LIT> ) ] [ Reg ] |= S . LaneMask ; } } return


<s> unsigned GCNRegPressure :: getRegKind ( unsigned Reg , const MachineRegisterInfo & MRI ) { assert ( TargetRegisterInfo :: isVirtualRegister ( Reg ) ) ; const auto RC = MRI . getRegClass ( Reg ) ; auto STI = static_cast < const <ISA_LIT>RegisterInfo *


<s> unsigned getVGPRNum ( ) const { return std :: max ( Value [ VGPR32 ] ,


<s> return std :: max ( Value [ VGPR32 ] ,


<s> LLVM_DUMP_METHOD void GCNRegPressure :: print ( raw_ostream & OS , const GCNSubtarget * ST ) const { OS << <STR_LIT> << getVGPRNum ( ) ; if ( ST ) OS << <STR_LIT> << ST -> getOccupancyWithNumVGPRs ( getVGPRNum


<s> NextMI = skipDebugInstructionsForward ( NextMI , MBBEnd ) ; for ( const auto & MO : LastTrackedMI -> operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . isDef ( ) ) continue ; Register Reg = MO . getReg ( ) ; if ( ! Reg . isVirtual ( )


<s> auto I = llvm :: find_if ( Res , [ Reg ] ( const RegisterMaskPair & RM ) { return RM . RegUnit == Reg ; } ) ; if ( I != Res . end ( ) ) I -> LaneMask |= UsedMask ; else Res . push_back ( RegisterMaskPair ( Reg ,


<s> return getSGRPNum ( ) == <NUM_LIT> &&


<s> return std :: min ( ST . getOccupancyWithNumSGPRs ( getSGRPNum ( )

) , ST . getOccupancyWithNumVGPRs ( getVGRPNum ( ) ) ) ;


<s> return std :: min ( ST . getOccupancyWithNumSGPRs ( getSGRPNum ( ) ) , ST .

getOccupancyWithNumVGPRs ( getVGRPNum ( ) ) ) ;


<s> if ( Occ != OtherOcc ) return Occ > OtherOcc ; bool SGPRImportant = SGPROcc < VGPROcc ; const bool OtherSGPRImportant = OtherSGPROcc < OtherVGPROcc ; if ( SGPRImportant != OtherSGPRImportant ) { SGPRImportant = false ; } bool SGPRFirst = SGPRImportant ; for ( int I = <NUM_LIT> ; I > <NUM_LIT> ; -- I , SGPRFirst = ! SGPRFirst ) { if ( SGPRFirst ) { auto SW = getSGPRTuplesWeight ( ) ; auto OtherSW = O . getSGPRTuplesWeight ( ) ; if ( SW != OtherSW ) return SW < OtherSW ; } else { auto VW = getVGPRTuplesWeight ( ) ; auto OtherVW = O .


<s> } bool SGPRFirst = SGPRImportant ; for ( int I = <NUM_LIT> ; I > <NUM_LIT> ; -- I , SGPRFirst = ! SGPRFirst ) { if ( SGPRFirst ) { auto SW = getSGPRTuplesWeight ( ) ; auto OtherSW = O . getSGPRTuplesWeight ( ) ; if ( SW != OtherSW ) return SW < OtherSW ; } else { auto VW = getVGPRTuplesWeight ( ) ; auto OtherVW = O . getVGPRTuplesWeight ( ) ; if ( VW != OtherVW ) return VW < OtherVW ; } } return SGPRImportant ? ( getSGPRNum ( ) < O . getSGPRNum ( ) ) : ( getVGPRNum ( ST . hasGFX90AInsts ( ) ) < O .

getVGPRNum ( ST . hasGFX90AInsts ( ) ) ) ;


<s> if ( ST ) OS << <STR_LIT> << ST -> getOccupancyWithNumVGPRs ( getVGPRNum ( ST -> hasGFX90AInsts ( ) ) ) << ')' ; OS << <STR_LIT> << getSGPRNum ( ) ; if ( ST ) OS << <STR_LIT> << ST -> getOccupancyWithNumSGPRs ( getSGPRNum ( ) ) << ')' ; OS << <STR_LIT> << getVGPRTuplesWeight ( ) << <STR_LIT> << getSGPRTuplesWeight ( ) ; if ( ST ) OS << <STR_LIT> << getOccupancy ( * ST ) ; OS <<


<s> if ( ! MO . isReg ( ) || ! MO . isDef ( ) ) continue ; Register Reg = MO . getReg ( ) ; if ( ! Register :: isVirtualRegister ( Reg ) ) continue ; auto & LiveMask = LiveRegs [ Reg ] ; auto PrevMask = LiveMask ; LiveMask |= getDefRegMask ( MO


<s> if ( ! MO . isUse ( ) || ! MO . readsReg ( ) ) continue ; auto const UsedMask = getUsedRegMask ( MO , MRI , LIS ) ; auto Reg = MO . getReg ( ) ; auto I = std :: find_if ( Res . begin ( ) , Res . end ( ) , [ Reg ] ( const RegisterMaskPair & RM ) { return RM . RegUnit ==


<s> auto Reg = MO . getReg ( ) ; auto I = std :: find_if ( Res . begin ( ) , Res . end ( ) , [ Reg ] ( const RegisterMaskPair & RM ) { return RM . RegUnit == Reg ; } ) ; if ( I != Res . end ( ) ) I -> LaneMask |= UsedMask ; else Res . push_back ( RegisterMaskPair ( Reg , UsedMask ) ) ; } return


<s> static LaneBitmask getDefRegMask ( const MachineOperand & MO , const MachineRegisterInfo & MRI ) { assert ( MO . isDef ( ) && MO . isReg ( ) && Register :: isVirtualRegister ( MO . getReg ( ) ) ) ; return MO . getSubReg ( ) == <NUM_LIT> ? MRI

. getMaxLaneMaskForVReg ( MO . getReg ( ) ) :


<s> return STI -> isSGPRClass ( RC ) ? ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ? SGPR32 : SGPR_TUPLE ) : STI -> hasAGPRs ( RC ) ? ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ? AGPR32 : AGPR_TUPLE ) : ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ? VGPR32


<s> return STI -> isSGPRClass ( RC ) ? ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ? SGPR32 : SGPR_TUPLE ) : STI -> hasAGPRs ( RC ) ? ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ? AGPR32 : AGPR_TUPLE ) : ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ?


<s> static LaneBitmask getUsedRegMask ( const MachineOperand & MO , const MachineRegisterInfo & MRI , const LiveIntervals & LIS ) { assert ( MO . isUse ( ) && MO . isReg ( )

&& Register :: isVirtualRegister ( MO . getReg ( ) ) ) ;


<s> void GCNUpwardRPTracker :: recede ( const MachineInstr & MI ) { assert ( MRI && <STR_LIT> ) ; LastTrackedMI = & MI ; if ( MI . isDebugInstr ( ) ) return ; auto const RegUses = collectVirtualRegUses ( MI , LIS , * MRI ) ; auto AtMIPressure = CurPressure ; for ( const auto & U : RegUses ) { auto LiveMask = LiveRegs [ U . RegUnit ] ; AtMIPressure . inc ( U . RegUnit , LiveMask , LiveMask | U . LaneMask , * MRI ) ; } MaxPressure = max ( AtMIPressure , MaxPressure )


<s> if ( S1 . size ( ) != S2 . size ( ) ) return false ; for ( const auto & P : S1 ) { auto I = S2 . find ( P . first ) ; if ( I == S2 . end ( ) || I -> second != P . second ) return false ; } return true


<s> bool GCNRegPressure :: less ( const <ISA_LIT>Subtarget & ST , const GCNRegPressure & O , unsigned MaxOccupancy ) const { const auto SGPROcc = std :: min ( MaxOccupancy , ST . getOccupancyWithNumSGPRs ( getSGPRNum ( ) ) ) ; const auto VGPROcc = std :: min ( MaxOccupancy , ST . getOccupancyWithNumVGPRs ( getVGPRNum ( ) ) ) ; const auto OtherSGPROcc = std :: min ( MaxOccupancy , ST . getOccupancyWithNumSGPRs ( O . getSGPRNum ( ) ) ) ; const auto OtherVGPROcc = std :: min ( MaxOccupancy , ST . getOccupancyWithNumVGPRs ( O . getVGPRNum ( ) ) ) ; const auto Occ = std :: min ( SGPROcc , VGPROcc ) ; const auto OtherOcc = std :: min ( OtherSGPROcc , OtherVGPROcc ) ; if ( Occ != OtherOcc ) return Occ > OtherOcc ; bool SGPRImportant = SGPROcc < VGPROcc ; const bool OtherSGPRImportant = OtherSGPROcc < OtherVGPROcc ; if ( SGPRImportant != OtherSGPRImportant ) { SGPRImportant = false ; } bool SGPRFirst = SGPRImportant ; for ( int I = <NUM_LIT> ; I > <NUM_LIT> ; -- I , SGPRFirst = ! SGPRFirst ) { if ( SGPRFirst ) { auto SW = getSGPRTuplesWeight ( ) ; auto OtherSW = O . getSGPRTuplesWeight ( ) ; if ( SW != OtherSW ) return SW < OtherSW ; } else { auto VW = getVGPRTuplesWeight ( ) ; auto OtherVW = O . getVGPRTuplesWeight ( ) ; if ( VW != OtherVW )


<s> OS << <STR_LIT> << getVGPRNum ( ) ; if ( ST ) OS << <STR_LIT> << ST -> getOccupancyWithNumVGPRs ( getVGPRNum ( ) ) << ')' ; OS << <STR_LIT> << getSGPRNum ( ) ; if ( ST ) OS << <STR_LIT> << ST -> getOccupancyWithNumSGPRs (


<s> if ( ST ) OS << <STR_LIT> << ST -> getOccupancyWithNumVGPRs ( getVGPRNum ( ) ) << ')' ; OS << <STR_LIT> << getSGPRNum ( ) ; if ( ST ) OS << <STR_LIT> << ST -> getOccupancyWithNumSGPRs ( getSGPRNum ( ) ) << ')' ; OS << <STR_LIT> << getVGPRTuplesWeight ( ) << <STR_LIT>


<s> void GCNRPTracker :: printLiveRegs ( raw_ostream & OS , const LiveRegSet & LiveRegs , const MachineRegisterInfo & MRI ) { const TargetRegisterInfo * TRI = MRI . getTargetRegisterInfo ( ) ; for ( unsigned I = <NUM_LIT> , E = MRI . getNumVirtRegs ( ) ; I != E ; ++ I ) { unsigned Reg = TargetRegisterInfo :: index2VirtReg ( I ) ; auto It = LiveRegs . find


<s> auto const RegUses = collectVirtualRegUses ( MI , LIS , * MRI ) ; auto AtMIPressure = CurPressure ; for ( const auto & U : RegUses ) { auto LiveMask = LiveRegs [ U . RegUnit ] ; AtMIPressure . inc ( U . RegUnit , LiveMask , LiveMask | U . LaneMask , * MRI ) ; } MaxPressure = max ( AtMIPressure , MaxPressure ) ; for ( const auto & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) || ! TargetRegisterInfo :: isVirtualRegister ( MO . getReg ( ) ) || MO . isDead ( ) ) continue ; auto Reg = MO . getReg ( ) ; auto I = LiveRegs . find ( Reg ) ; if ( I == LiveRegs . end ( ) ) continue ; auto & LiveMask = I -> second ; auto PrevMask = LiveMask ; LiveMask &= ~ getDefRegMask ( MO , * MRI ) ; CurPressure . inc ( Reg , PrevMask ,


<s> LastTrackedMI = & MI ; if ( MI . isDebugValue ( ) ) return ; auto const RegUses = collectVirtualRegUses ( MI , LIS , * MRI ) ; auto AtMIPressure = CurPressure ; for ( const auto & U : RegUses ) { auto LiveMask = LiveRegs [ U . RegUnit ] ; AtMIPressure . inc ( U . RegUnit , LiveMask , LiveMask | U . LaneMask , * MRI ) ; } MaxPressure = max ( AtMIPressure , MaxPressure ) ; for ( const auto & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) || ! TargetRegisterInfo :: isVirtualRegister ( MO . getReg ( ) ) || MO . isDead ( ) ) continue ; auto Reg = MO . getReg ( ) ; auto I = LiveRegs . find ( Reg ) ; if ( I == LiveRegs . end ( ) ) continue ; auto & LiveMask = I -> second ; auto PrevMask = LiveMask ; LiveMask &= ~ getDefRegMask ( MO , * MRI ) ; CurPressure . inc ( Reg , PrevMask , LiveMask , * MRI ) ; if ( LiveMask . none ( ) ) LiveRegs . erase ( I ) ; } for ( const auto & U : RegUses ) { auto & LiveMask = LiveRegs [ U . RegUnit ] ; auto PrevMask = LiveMask


<s> LastTrackedMI = nullptr ; MBBEnd = MI . getParent ( ) -> end ( ) ; NextMI = & MI ; NextMI = skipDebugInstructionsForward ( NextMI , MBBEnd ) ; if ( NextMI == MBBEnd ) return false ; if ( LiveRegsCopy ) { if ( & LiveRegs != LiveRegsCopy ) LiveRegs = * LiveRegsCopy ; } else { LiveRegs = getLiveRegsBefore ( * NextMI , LIS ) ; } MaxPressure = CurPressure = getRegPressure ( *


<s> auto & LiveMask = LiveRegs [ Reg ] ; auto PrevMask = LiveMask ; LiveMask |= getDefRegMask ( MO , * MRI ) ; CurPressure . inc ( Reg , PrevMask , LiveMask , *


<s> bool empty ( ) const { return getSGPRNum ( ) == <NUM_LIT> &&


<s> unsigned getAGPRNum ( ) const { return Value [


<s> assert ( MO . isDef ( ) && MO . isReg (

) && MO . getReg ( ) . isVirtual ( ) ) ;


<s> auto LiveMask = getLiveLaneMask ( Reg , <ISA_LIT> , LIS , MRI ) ; if ( LiveMask . any ( ) ) LiveRegs [ Reg ] = LiveMask ; } return LiveRegs


<s> GCNRPTracker :: LiveRegSet llvm :: getLiveRegs ( SlotIndex <ISA_LIT> , const LiveIntervals & LIS , const MachineRegisterInfo & MRI ) { GCNRPTracker :: LiveRegSet LiveRegs ; for ( unsigned I = <NUM_LIT> , E = MRI . getNumVirtRegs ( )


<s> unsigned getOccupancy ( const GCNSubtarget & ST ) const { return std :: min ( ST . getOccupancyWithNumSGPRs ( getSGPRNum ( ) ) , ST

. getOccupancyWithNumVGPRs ( getVGPRNum ( ST . hasGFX90AInsts ( ) ) ) ) ;


<s> auto STI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; return STI -> isSGPRClass ( RC ) ? ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ? SGPR32 : SGPR_TUPLE ) : STI -> hasAGPRs ( RC ) ? ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT>


<s> static LaneBitmask getUsedRegMask ( const MachineOperand & MO , const MachineRegisterInfo & MRI , const LiveIntervals & LIS ) { assert ( MO . isUse ( ) && MO . isReg ( ) && MO . getReg ( ) . isVirtual ( ) ) ; if ( auto SubReg = MO . getSubReg ( ) ) return MRI . getTargetRegisterInfo ( ) -> getSubRegIndexLaneMask ( SubReg ) ; auto MaxMask = MRI . getMaxLaneMaskForVReg ( MO . getReg (


<s> if ( UnifiedVGPRFile ) { return Value [ AGPR32 ] ? alignTo ( Value [ VGPR32 ] , <NUM_LIT> ) + Value [ AGPR32 ] : Value [ VGPR32


<s> unsigned getVGPRTuplesWeight ( ) const { return std :: max ( Value [ VGPR_TUPLE ] , Value [ AGPR_TUPLE ]


<s> unsigned getVGPRTuplesWeight ( ) const { return std :: max ( Value [ VGPR_TUPLE ] , Value


<s> case AGPR32 : Value [ Kind ] += Sign ; break ; case SGPR_TUPLE : case VGPR_TUPLE : case AGPR_TUPLE : assert ( PrevMask < NewMask ) ; Value [ Kind == SGPR_TUPLE ? SGPR32 : Kind == AGPR_TUPLE ? AGPR32 : VGPR32 ] += Sign * <ISA_LIT>RegisterInfo :: getNumCoveredRegs ( ~ PrevMask & NewMask ) ; if ( PrevMask . none ( ) ) { assert ( NewMask . any


<s>
if
(
It
!=
LiveRegs
.
end
(
)
&&
It
->
second
.
any
(
)
)
OS
<<
' '
<<
printVRegOrUnit
(
Reg
,
TRI
)
<<
':'
<<
PrintLaneMask
(
It
->
second
)
;
}
OS
<<


<s> void GCNRPTracker :: printLiveRegs ( raw_ostream & OS , const LiveRegSet & LiveRegs , const MachineRegisterInfo & MRI ) { const TargetRegisterInfo * TRI = MRI . getTargetRegisterInfo ( ) ; for ( unsigned I = <NUM_LIT> , E = MRI . getNumVirtRegs ( ) ; I != E ; ++ I ) { unsigned Reg = Register :: index2VirtReg ( I


<s> LastTrackedMI = & MI ; if ( MI . isDebugInstr ( ) ) return ; auto const RegUses = collectVirtualRegUses ( MI , LIS , * MRI ) ; auto AtMIPressure = CurPressure ; for ( const auto & U : RegUses ) { auto LiveMask = LiveRegs [ U . RegUnit ] ; AtMIPressure . inc ( U . RegUnit , LiveMask , LiveMask | U . LaneMask , * MRI ) ; } MaxPressure = max ( AtMIPressure , MaxPressure ) ; for ( const auto & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . isDef ( ) || ! MO . getReg ( ) . isVirtual ( ) || MO . isDead ( ) ) continue ; auto Reg = MO . getReg ( ) ; auto I = LiveRegs . find ( Reg


<s> LaneBitmask GCNUpwardRPTracker :: getDefRegMask ( const MachineOperand & MO ) const { assert ( MO . isDef ( ) && MO . isReg ( ) && TargetRegisterInfo :: isVirtualRegister ( MO . getReg ( ) ) ) ; return MO . getSubReg ( ) == <NUM_LIT> ? MRI -> getMaxLaneMaskForVReg


<s> assert ( ! MRI . reg_nodbg_empty ( Reg ) ) ; LaneBitmask LiveMask ; const auto & LI = LIS . getInterval ( Reg ) ; if ( LI . hasSubRanges ( ) ) { for ( const auto & S : LI . subranges ( ) ) if ( S . liveAt ( <ISA_LIT> ) ) { LiveMask |= S . LaneMask ; assert ( LiveMask < MRI . getMaxLaneMaskForVReg ( Reg ) || LiveMask == MRI . getMaxLaneMaskForVReg ( Reg ) ) ; } } else if ( LI . liveAt ( <ISA_LIT> ) ) { LiveMask = MRI . getMaxLaneMaskForVReg (


<s> if ( MRI . reg_nodbg_empty ( Reg ) ) continue ; auto LiveMask = getLiveLaneMask ( Reg , <ISA_LIT> , LIS , MRI ) ; if ( LiveMask . any ( ) ) LiveRegs [ Reg ] = LiveMask ; } return LiveRegs


<s> return STI -> isSGPRClass ( RC ) ? ( RC -> getSize ( ) == <NUM_LIT> ? SGPR32 : SGPR_TUPLE ) : ( RC -> getSize ( ) == <NUM_LIT> ? VGPR32


<s> LaneBitmask GCNUpwardRPTracker :: getUsedRegMask ( const MachineOperand & MO ) const { assert ( MO . isUse ( ) && MO . isReg ( ) && TargetRegisterInfo :: isVirtualRegister ( MO . getReg ( ) ) ) ; if ( auto SubReg = MO . getSubReg ( ) ) return MRI -> getTargetRegisterInfo ( ) -> getSubRegIndexLaneMask ( SubReg ) ; auto MaxMask = MRI -> getMaxLaneMaskForVReg ( MO . getReg ( ) ) ; if ( MaxMask . getAsInteger ( ) == <NUM_LIT> )


<s> const bool OtherSGPRImportant = OtherSGPROcc < OtherVGPROcc ; if ( SGPRImportant != OtherSGPRImportant ) { SGPRImportant = false ; } bool SGPRFirst = SGPRImportant ; for ( int I = <NUM_LIT> ; I > <NUM_LIT> ; -- I , SGPRFirst = ! SGPRFirst ) { if ( SGPRFirst ) { auto SW = getSGPRTuplesWeight ( ) ; auto OtherSW = O . getSGPRTuplesWeight ( ) ; if ( SW != OtherSW ) return SW < OtherSW ; } else { auto VW = getVGPRTuplesWeight ( ) ; auto OtherVW = O . getVGPRTuplesWeight (


<s> dbgs ( ) << <STR_LIT> << <ISA_LIT> << <STR_LIT> << * LIS . getInstructionFromIndex ( <ISA_LIT> ) ; unsigned Num = <NUM_LIT> ; for ( unsigned I = <NUM_LIT> , E = MRI . getNumVirtRegs ( ) ; I != E ; ++ I ) { const unsigned Reg = TargetRegisterInfo :: index2VirtReg ( I ) ; if ( MRI . reg_nodbg_empty ( Reg ) ) continue ; const auto & LI = LIS . getInterval ( Reg ) ; if ( LI . hasSubRanges ( ) ) { bool firstTime = true ; for ( const auto & S : LI . subranges ( ) ) { if ( ! S . liveAt ( <ISA_LIT> ) ) continue ; if ( firstTime ) { dbgs ( ) << <STR_LIT> << PrintReg ( Reg , MRI . getTargetRegisterInfo ( ) ) << '\n' ; firstTime = false ; } dbgs ( ) << <STR_LIT> << S <<


<s> if ( LI . hasSubRanges ( ) ) { bool firstTime = true ; for ( const auto & S : LI . subranges ( ) ) { if ( ! S . liveAt ( <ISA_LIT> ) ) continue ; if ( firstTime ) { dbgs ( ) << <STR_LIT> << PrintReg ( Reg , MRI . getTargetRegisterInfo ( ) ) << '\n' ; firstTime = false ; } dbgs ( ) << <STR_LIT> << S << '\n' ; ++ Num ; } } else if ( LI . liveAt ( <ISA_LIT> ) ) { dbgs ( ) << <STR_LIT> << LI


<s> auto Reg = MO . getReg ( ) ; auto & LiveMask = LiveRegs [ Reg ] ; auto PrevMask = LiveMask ; LiveMask &= ~ getDefRegMask ( MO ) ; CurPressure . inc ( Reg , PrevMask , LiveMask , * MRI ) ; } for ( const auto & MO : MI . uses ( ) ) { if ( ! MO . isReg ( ) || ! MO . readsReg ( ) || ! TargetRegisterInfo :: isVirtualRegister ( MO . getReg ( ) ) ) continue ; auto Reg = MO . getReg ( ) ; auto & LiveMask = LiveRegs [ Reg ] ; auto PrevMask = LiveMask ; LiveMask |= getUsedRegMask ( MO ) ; CurPressure . inc ( Reg , PrevMask , LiveMask , * MRI ) ; } MaxPressure = max (


<s> auto PrevMask = LiveMask ; LiveMask &= ~ getDefRegMask ( MO ) ; CurPressure . inc ( Reg , PrevMask , LiveMask , * MRI ) ; } for ( const auto & MO : MI . uses ( ) ) { if ( ! MO . isReg ( ) || ! MO . readsReg ( ) || ! TargetRegisterInfo :: isVirtualRegister ( MO . getReg ( ) ) ) continue ; auto Reg = MO . getReg ( ) ; auto & LiveMask = LiveRegs [ Reg ] ; auto PrevMask =


<s> void GCNUpwardRPTracker :: reset ( const MachineInstr & MI ) { MRI = & MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; LiveRegs = getLiveRegsAfter ( MI , LIS ) ; MaxPressure = CurPressure = getRegPressure ( * MRI


<s> for ( const auto & MO : LastTrackedMI -> defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; Register Reg = MO .


<s> auto MaxMask = MRI . getMaxLaneMaskForVReg ( MO . getReg ( ) ) ; if ( MaxMask == LaneBitmask :: getLane ( <NUM_LIT> ) ) return MaxMask ; auto <ISA_LIT> = LIS . getInstructionIndex ( * MO . getParent ( ) ) .


<s> LiveMask &= ~ getDefRegMask ( MO , * MRI ) ; CurPressure . inc ( Reg , PrevMask , LiveMask , * MRI ) ; if ( LiveMask . none ( ) ) LiveRegs . erase ( I ) ; } for ( const auto & U : RegUses ) { auto & LiveMask = LiveRegs [ U . RegUnit ] ; auto PrevMask = LiveMask ; LiveMask |= U . LaneMask ; CurPressure . inc ( U . RegUnit , PrevMask , LiveMask , * MRI ) ; } assert ( CurPressure == getRegPressure ( * MRI ,


<s> unsigned GCNRegPressure :: getRegKind ( Register Reg , const MachineRegisterInfo & MRI ) { assert ( Reg . isVirtual ( ) ) ; const auto RC = MRI . getRegClass ( Reg ) ; auto STI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; return STI -> isSGPRClass ( RC ) ? ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ? SGPR32 : SGPR_TUPLE ) : STI -> isAGPRClass ( RC ) ? ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ? AGPR32 :

AGPR_TUPLE ) : ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ? VGPR32 : VGPR_TUPLE ) ;


<s> return STI -> isSGPRClass ( RC ) ? ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ? SGPR32 : SGPR_TUPLE ) : STI -> isAGPRClass ( RC ) ? ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ? AGPR32 : AGPR_TUPLE ) :

( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ? VGPR32 : VGPR_TUPLE ) ;


<s> auto MaxMask = MRI . getMaxLaneMaskForVReg ( MO . getReg ( ) ) ; if ( MaxMask . getAsInteger ( ) == <NUM_LIT> ) return MaxMask ; auto <ISA_LIT> = LIS . getInstructionIndex ( * MO . getParent ( ) ) . getBaseIndex ( ) ; return getLiveLaneMask ( MO . getReg ( ) , <ISA_LIT> ,


<s>
void
GCNRPTracker
::
printLiveRegs
(
raw_ostream
&
OS
,
const
LiveRegSet
&
LiveRegs
,
const
MachineRegisterInfo
&
MRI
)
{
const
TargetRegisterInfo
*
TRI
=
MRI
.
getTargetRegisterInfo
(
)
;
for
(
unsigned
I
=
<NUM_LIT>
,
E
=
MRI
.
getNumVirtRegs
(
)
;
I
!=
E
;
++
I
)
{
Register
Reg
=
Register
::
index2VirtReg
(
I
)
;
auto
It
=
LiveRegs
.
find
(
Reg
)
;
if
(
It
!=
LiveRegs
.
end
(
)
&&
It
->
second
.
any
(
)
)
OS
<<
' '
<<
printVRegOrUnit
(
Reg
,
TRI
)
<<
':'
<<


<s>
if
(
It
!=
LiveRegs
.
end
(
)
&&
It
->
second
.
any
(
)
)
OS
<<
' '
<<
printVRegOrUnit
(
Reg
,
TRI
)
<<
':'
<<
PrintLaneMask
(
It
->
second
)
;
}
OS
<<
'\n'


<s> for ( const auto & S : LI . subranges ( ) ) { if ( ! S . liveAt ( <ISA_LIT> ) ) continue ; if ( firstTime ) { dbgs ( ) << <STR_LIT> << printReg ( Reg , MRI . getTargetRegisterInfo ( ) ) << '\n' ; firstTime = false ; } dbgs ( ) << <STR_LIT> << S << '\n' ; ++ Num


<s> } else if ( ! LI . liveAt ( <ISA_LIT> ) ) { auto PrevMask = It . second ; It . second = LaneBitmask :: getNone ( ) ; CurPressure . inc ( It . first , PrevMask , It . second , * MRI ) ; } if ( It . second . none ( ) ) LiveRegs . erase ( It . first ) ; } MaxPressure = max ( MaxPressure , CurPressure


<s> } } } else if ( ! LI . liveAt ( <ISA_LIT> ) ) { auto PrevMask = It . second ; It . second = LaneBitmask :: getNone ( ) ; CurPressure . inc ( It . first , PrevMask , It . second , * MRI ) ; } if ( It . second . none ( ) ) LiveRegs . erase ( It . first ) ; } MaxPressure = max ( MaxPressure , CurPressure ) ; return


<s> std :: fill ( & Value [ <NUM_LIT> ] , & Value [ TOTAL_KINDS


<s> bool empty ( ) const { return getSGPRNum ( ) == <NUM_LIT> &&


<s> return MO . getSubReg ( ) == <NUM_LIT> ? MRI . getMaxLaneMaskForVReg ( MO . getReg ( ) ) : MRI . getTargetRegisterInfo ( ) -> getSubRegIndexLaneMask (


<s> const MachineInstr * getLastTrackedMI


<s> const auto & LI = LIS . getInterval ( Reg ) ; if ( LI . hasSubRanges ( ) ) { for ( const auto & S : LI . subranges ( ) ) if ( S . liveAt ( <ISA_LIT> ) ) { LiveMask |= S


<s> inline GCNRPTracker :: LiveRegSet getLiveRegsAfter ( const MachineInstr & MI , const LiveIntervals & LIS ) { return getLiveRegs ( LIS . getInstructionIndex ( MI ) . getDeadSlot ( ) , LIS , MI . getParent ( ) -> getParent (


<s> return std :: min ( ST . getOccupancyWithNumSGPRs ( getSGPRNum ( ) ) , ST . getOccupancyWithNumVGPRs (


<s> STI -> hasAGPRs ( RC ) ? ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ? AGPR32 : AGPR_TUPLE ) : ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ? VGPR32


<s> STI -> hasAGPRs ( RC ) ? ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT> ? AGPR32 : AGPR_TUPLE ) : ( STI -> getRegSizeInBits ( * RC ) == <NUM_LIT>


<s> GCNRegPressure getRegPressure ( const MachineRegisterInfo & MRI , Range && LiveRegs ) { GCNRegPressure Res ; for ( const auto & RM : LiveRegs ) Res . inc ( RM . first , LaneBitmask :: getNone ( ) , RM . second , MRI ) ; return


<s> unsigned getSGPRTuplesWeight ( ) const { return Value [


<s> static LaneBitmask getUsedRegMask ( const MachineOperand & MO , const MachineRegisterInfo & MRI , const LiveIntervals & LIS ) { assert ( MO . isUse ( ) && MO . isReg ( ) && TargetRegisterInfo :: isVirtualRegister ( MO . getReg ( ) ) ) ; if ( auto SubReg = MO . getSubReg ( ) ) return MRI . getTargetRegisterInfo ( ) -> getSubRegIndexLaneMask ( SubReg ) ; auto MaxMask = MRI . getMaxLaneMaskForVReg ( MO . getReg


<s> bool higherOccupancy ( const GCNSubtarget & ST ,


<s> for ( const auto & P : S1 ) { auto I = S2 . find ( P . first ) ; if ( I == S2 . end ( ) || I -> second != P . second ) return false ; } return


<s> const auto & TrackedLR = LiveRegs ; if ( ! isEqual ( LISLR , TrackedLR ) ) { dbgs ( ) << <STR_LIT> <STR_LIT> ; printLivesAt ( <ISA_LIT> , LIS , * MRI ) ; reportMismatch ( LISLR , TrackedLR , MRI


<s> const auto SGPROcc = std :: min ( MaxOccupancy , ST . getOccupancyWithNumSGPRs ( getSGPRNum ( ) ) ) ; const auto VGPROcc = std :: min ( MaxOccupancy , ST . getOccupancyWithNumVGPRs ( getVGPRNum ( ) ) ) ; const auto OtherSGPROcc = std :: min ( MaxOccupancy , ST . getOccupancyWithNumSGPRs ( O . getSGPRNum ( ) ) ) ; const auto OtherVGPROcc = std :: min ( MaxOccupancy , ST . getOccupancyWithNumVGPRs ( O . getVGPRNum ( ) ) ) ; const auto Occ = std :: min ( SGPROcc , VGPROcc ) ; const auto OtherOcc = std :: min ( OtherSGPROcc , OtherVGPROcc ) ; if ( Occ != OtherOcc ) return Occ > OtherOcc ; bool SGPRImportant = SGPROcc < VGPROcc ; const bool OtherSGPRImportant = OtherSGPROcc < OtherVGPROcc ; if ( SGPRImportant != OtherSGPRImportant ) { SGPRImportant = false ; } bool SGPRFirst = SGPRImportant ; for ( int I = <NUM_LIT> ; I > <NUM_LIT> ; -- I , SGPRFirst = ! SGPRFirst ) { if ( SGPRFirst ) { auto SW =


<s> auto Res = MaxPressure ; MaxPressure . clear ( ) ; return


<s>
OS
<<
<STR_LIT>
<<
Value
[
VGPR32
]
<<
' '
;
OS
<<
<STR_LIT>
<<
Value
[
AGPR32
]
;
if
(
ST
)
OS
<<
<STR_LIT>
<<
ST
->
getOccupancyWithNumVGPRs
(
getVGPRNum
(
)
)
<<
')'
;
OS
<<
<STR_LIT>
<<
getSGPRNum
(
)
;
if
(
ST
)
OS
<<
<STR_LIT>
<<
ST
->
getOccupancyWithNumSGPRs
(
getSGPRNum
(
)
)
<<
')'
;
OS
<<
<STR_LIT>
<<
getVGPRTuplesWeight
(
)
<<
<STR_LIT>
<<
getSGPRTuplesWeight
(
)
;
if
(
ST
)
OS
<<
<STR_LIT>
<<
getOccupancy
(
*
ST
)
;
OS
<<


<s> OS << <STR_LIT> << getSGPRNum ( ) ; if ( ST ) OS << <STR_LIT> << ST -> getOccupancyWithNumSGPRs ( getSGPRNum ( ) ) << ')' ; OS << <STR_LIT> << getVGPRTuplesWeight ( ) << <STR_LIT> << getSGPRTuplesWeight ( ) ; if ( ST ) OS << <STR_LIT> << getOccupancy


<s>
for
(
unsigned
I
=
<NUM_LIT>
,
E
=
MRI
.
getNumVirtRegs
(
)
;
I
!=
E
;
++
I
)
{
unsigned
Reg
=
TargetRegisterInfo
::
index2VirtReg
(
I
)
;
auto
It
=
LiveRegs
.
find
(
Reg
)
;
if
(
It
!=
LiveRegs
.
end
(
)
&&
It
->
second
.
any
(
)
)
OS
<<
' '
<<
printVRegOrUnit
(
Reg
,
TRI
)
<<
':'
<<
PrintLaneMask
(
It
->
second
)
;
}
OS
<<


<s> auto LiveMask = LiveRegs [ U . RegUnit ] ; AtMIPressure . inc ( U . RegUnit , LiveMask , LiveMask | U . LaneMask , * MRI ) ; } MaxPressure = max ( AtMIPressure , MaxPressure ) ; for ( const auto & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) || ! TargetRegisterInfo :: isVirtualRegister ( MO . getReg ( ) ) || MO . isDead ( ) ) continue ; auto Reg = MO . getReg ( ) ; auto I = LiveRegs . find ( Reg ) ; if ( I == LiveRegs . end ( ) ) continue ; auto & LiveMask = I -> second ; auto PrevMask = LiveMask ; LiveMask &= ~ getDefRegMask ( MO , * MRI ) ; CurPressure . inc ( Reg , PrevMask , LiveMask , * MRI ) ; if ( LiveMask . none ( ) ) LiveRegs . erase ( I ) ; } for ( const auto & U : RegUses ) { auto & LiveMask = LiveRegs [ U .


<s> } MaxPressure = max ( AtMIPressure , MaxPressure ) ; for ( const auto & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) || ! TargetRegisterInfo :: isVirtualRegister ( MO . getReg ( ) ) || MO . isDead ( ) ) continue ; auto Reg = MO . getReg ( ) ; auto I = LiveRegs . find ( Reg ) ; if ( I == LiveRegs . end ( ) ) continue ; auto & LiveMask = I -> second ; auto PrevMask = LiveMask ; LiveMask &= ~ getDefRegMask ( MO , * MRI ) ; CurPressure . inc ( Reg , PrevMask , LiveMask , * MRI ) ; if ( LiveMask . none ( ) ) LiveRegs . erase ( I ) ; } for ( const auto & U : RegUses ) { auto & LiveMask = LiveRegs [ U


<s> else if ( I -> second != P . second ) { dbgs ( ) << <STR_LIT> << printReg ( P . first , TRI ) << <STR_LIT> << PrintLaneMask ( I -> second ) << <STR_LIT> << PrintLaneMask ( P . second ) << '\n' ; } } for ( auto const & P :


<s> dbgs ( ) << <STR_LIT> << printReg ( P . first , TRI ) << <STR_LIT> << PrintLaneMask ( P . second ) << <STR_LIT> ; } else if ( I -> second != P . second ) { dbgs ( ) << <STR_LIT> << printReg ( P . first , TRI ) << <STR_LIT> << PrintLaneMask ( I -> second ) << <STR_LIT> << PrintLaneMask ( P . second ) << '\n' ; } } for ( auto const & P : LISLR ) { auto I = TrackedLR . find ( P . first ) ; if ( I == TrackedLR . end


<s> MRI = & MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; LastTrackedMI = nullptr ; MBBEnd = MI . getParent ( )


<s> } RegionBegin = Region . first ; RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( ) ; MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == InitialSchedule ) computeBlockPressure ( MBB ) ; } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << MF . getName ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> << * begin ( ) << <STR_LIT> ; if ( RegionEnd != MBB -> end ( ) ) dbgs ( ) << * RegionEnd ; else dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> << NumRegionInstrs << '\n' ) ; schedule ( ) ; exitRegion ( ) ; ++ RegionIdx


<s> GenericScheduler ( C ) , TargetOccupancy ( <NUM_LIT> ) , HasClusteredNodes ( false ) ,

HasExcessPressure ( false ) , MF ( nullptr ) {


<s> GCNMaxOccupancySchedStrategy :: GCNMaxOccupancySchedStrategy ( const MachineSchedContext * C


<s> unsigned MinRegOccupancy = std :: min ( ST . getOccupancyWithNumSGPRs ( SGPRs ) , ST . getOccupancyWithNumVGPRs ( VGPRs ) ) ; return std :: min ( MinRegOccupancy , ST . getOccupancyWithLocalMemSize ( MFI ->

getLDSSize ( ) , * MF . getFunction ( ) ) ) ;


<s> const int ErrorMargin = <NUM_LIT> ; SGPRExcessLimit = Context -> RegClassInfo -> getNumAllocatableRegs ( & <STR_LIT> ) - ErrorMargin ; VGPRExcessLimit = Context -> RegClassInfo -> getNumAllocatableRegs ( & <STR_LIT> ) - ErrorMargin ; SGPRCriticalLimit = SRI -> getRegPressureSetLimit ( DAG -> MF , SRI

-> getSGPRPressureSet ( ) ) - ErrorMargin ;


<s> SUnit * GCNMaxOccupancySchedStrategy :: pickNode ( bool & IsTopNode ) { if ( DAG -> top ( ) == DAG -> bottom ( ) ) { assert ( Top . Available . empty ( ) && Top . Pending . empty ( ) && Bot . Available . empty ( ) && Bot . Pending . empty ( ) && <STR_LIT> ) ; return nullptr ; } SUnit * SU ; do { if ( RegionPolicy . OnlyTopDown ) { SU = Top . pickOnlyChoice ( ) ; if ( ! SU ) { CandPolicy NoPolicy ; TopCand . reset ( NoPolicy ) ; pickNodeFromQueue ( Top , NoPolicy , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopCand . Reason != NoCand && <STR_LIT> ) ; SU = TopCand . SU ; } IsTopNode = true ; } else if ( RegionPolicy . OnlyBottomUp ) { SU = Bot . pickOnlyChoice ( ) ; if ( ! SU ) { CandPolicy NoPolicy ; BotCand . reset ( NoPolicy ) ; pickNodeFromQueue ( Bot , NoPolicy , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotCand . Reason != NoCand && <STR_LIT> ) ; SU = BotCand . SU ; } IsTopNode =


<s> GenericScheduler :: tryCandidate ( Cand , TopCand , nullptr ) ; if ( TopCand . Reason != NoCand ) { Cand . setBest ( TopCand ) ; } else { TopCand . Reason = TopReason ; } } else { if ( TopCand . Reason == RegExcess && TopCand . RPDelta . Excess . getUnitInc ( ) <= <NUM_LIT> ) { Cand = TopCand ; } else if ( BotCand . Reason == RegExcess && BotCand . RPDelta . Excess . getUnitInc ( ) <= <NUM_LIT> ) { Cand = BotCand ; } else if ( TopCand . Reason == RegCritical && TopCand . RPDelta . CriticalMax . getUnitInc ( ) <= <NUM_LIT> ) { Cand = TopCand ; } else if ( BotCand . Reason == RegCritical && BotCand . RPDelta . CriticalMax . getUnitInc ( ) <= <NUM_LIT> ) { Cand = BotCand ; } else { int TopRank = getBidirectionalReasonRank ( TopCand . Reason ) ; int BotRank = getBidirectionalReasonRank ( BotCand . Reason ) ; if ( TopRank > BotRank ) { Cand = TopCand ; } else


<s> } CandPolicy BotPolicy ; setPolicy ( BotPolicy , false , Bot , & Top ) ; CandPolicy TopPolicy ; setPolicy ( TopPolicy , false , Top , & Bot ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( ! BotCand . isValid ( ) || BotCand . SU -> isScheduled || BotCand . Policy != BotPolicy ) { BotCand . reset ( CandPolicy ( ) ) ; pickNodeFromQueue ( Bot , BotPolicy , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotCand . Reason != NoCand && <STR_LIT> ) ; } else { DEBUG ( traceCandidate ( BotCand ) ) ; } DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( ! TopCand . isValid ( ) || TopCand . SU -> isScheduled || TopCand . Policy != TopPolicy ) { TopCand . reset ( CandPolicy ( ) ) ; pickNodeFromQueue ( Top , TopPolicy , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopCand . Reason != NoCand && <STR_LIT> ) ; } else { DEBUG ( traceCandidate ( TopCand ) ) ; } DEBUG ( dbgs ( ) << <STR_LIT> ; traceCandidate ( TopCand ) ; dbgs ( ) << <STR_LIT> ; traceCandidate ( BotCand ) ; ) ; SchedCandidate Cand ; if ( TopCand . Reason == BotCand . Reason ) { Cand = BotCand ; GenericSchedulerBase :: CandReason TopReason = TopCand . Reason ; TopCand . Reason


<s> MachineInstr * Def = Op -> getParent ( ) ; if ( Op -> getSubReg ( ) != <NUM_LIT> || ! isTriviallyReMaterializable ( * Def , AA ) ) continue ; MachineInstr * UseI = & * MRI . use_instr_nodbg_begin ( Reg ) ; if ( Def -> getParent ( ) == UseI -> getParent ( ) ) continue ; bool AddedToRematList = false ; for ( unsigned I = <NUM_LIT> , E = Regions . size ( ) ; I != E ; ++ I ) { auto It = LiveIns [ I ] . find ( Reg ) ; if ( It != LiveIns [ I ] . end ( ) && ! It -> second . none ( ) ) { if ( RegionsWithMinOcc [ I ] ) { RematerializableInsts [ I ] [ Def ] = UseI ; AddedToRematList = true ; } RematDefToLiveInRegions [ Def ] . push_back ( I ) ; } } if ( ! AddedToRematList ) RematDefToLiveInRegions . erase ( Def


<s> assert ( isEqual ( getLiveRegsBefore ( * NonDbgMI , * LIS ) , LRS ) ) ; RPTracker . reset ( * I , & LRS ) ; } for ( ; ; ) { I = RPTracker . getNext ( ) ; if ( Regions [ CurRegion ] . first == I || NonDbgMI == I ) { LiveIns [ CurRegion ] = RPTracker . getLiveRegs ( ) ; RPTracker . clearMaxPressure ( ) ; } if ( Regions [ CurRegion ] . second == I ) { Pressure [ CurRegion ] = RPTracker . moveMaxPressure ( ) ; if ( CurRegion -- == RegionIdx ) break ; } RPTracker . advanceToNext ( ) ; RPTracker . advanceBeforeNext ( ) ; } if ( OnlySucc ) { if ( I != MBB -> end ( ) ) { RPTracker . advanceToNext ( ) ; RPTracker . advance ( MBB -> end ( ) ) ; } RPTracker . reset ( * OnlySucc -> begin ( ) , & RPTracker .


<s> if ( Stage == ClusteredLowOccupancyReschedule ) { if ( StartingOccupancy <= MinOccupancy ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; } if ( Stage == PreRARematerialize ) { if ( RegionsWithMinOcc . none ( ) || Regions . size ( ) == <NUM_LIT> ) break ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const TargetInstrInfo * TII = MF . getSubtarget ( ) . getInstrInfo ( ) ; if ( ST . computeOccupancy ( MF . getFunction ( ) , MFI . getLDSSize ( ) ) == MinOccupancy ) break ; static_assert ( LastStage == PreRARematerialize , <STR_LIT> ) ; collectRematerializableInstructions ( ) ; if ( RematerializableInsts . empty ( ) || ! sinkTriviallyRematInsts ( ST , TII ) ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; } } if ( Stage == UnclusteredReschedule ) SavedMutations . swap ( Mutations ) ; for ( auto Region : Regions ) { if ( ( ( Stage == UnclusteredReschedule || Stage == PreRARematerialize ) && ! RescheduleRegions [ RegionIdx ] ) || ( Stage == ClusteredLowOccupancyReschedule && ! RegionsWithClusters [ RegionIdx ] && ! RegionsWithHighRP [ RegionIdx ] ) ) { ++ RegionIdx ; continue ; } RegionBegin = Region . first ; RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( ) ; MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == InitialSchedule ) computeBlockPressure ( MBB ) ; } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; ++ RegionIdx ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << MF . getName ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> << * begin ( ) << <STR_LIT> ; if ( RegionEnd != MBB -> end ( ) ) dbgs ( ) << * RegionEnd ; else dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> << NumRegionInstrs << '\n' ) ; schedule ( ) ; exitRegion ( ) ; ++ RegionIdx ; } finishBlock ( ) ; if ( Stage == UnclusteredReschedule ) SavedMutations . swap (


<s> if ( Stage == UnclusteredReschedule ) { if ( RescheduleRegions . none ( ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } if ( Stage == ClusteredLowOccupancyReschedule ) { if ( StartingOccupancy <= MinOccupancy ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; } if ( Stage == PreRARematerialize ) { if ( RegionsWithMinOcc . none ( ) || Regions . size ( ) == <NUM_LIT> ) break ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const TargetInstrInfo * TII = MF . getSubtarget ( ) . getInstrInfo ( ) ; if ( ST . computeOccupancy ( MF . getFunction ( ) , MFI . getLDSSize ( ) ) == MinOccupancy ) break ; static_assert ( LastStage == PreRARematerialize , <STR_LIT> ) ; collectRematerializableInstructions ( ) ; if ( RematerializableInsts . empty ( ) || ! sinkTriviallyRematInsts ( ST , TII ) ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; } } if ( Stage == UnclusteredReschedule ) SavedMutations . swap ( Mutations ) ; for ( auto Region : Regions ) { if ( ( ( Stage == UnclusteredReschedule || Stage == PreRARematerialize ) && ! RescheduleRegions [ RegionIdx ] ) || ( Stage == ClusteredLowOccupancyReschedule && ! RegionsWithClusters [ RegionIdx ] && ! RegionsWithHighRP [ RegionIdx ] ) ) { ++ RegionIdx


<s> void GCNMaxOccupancySchedStrategy :: initialize ( ScheduleDAGMI * DAG ) { GenericScheduler :: initialize ( DAG ) ; MF = & DAG -> MF ; const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; const unsigned ErrorMargin = <NUM_LIT> ; SGPRExcessLimit = Context -> RegClassInfo -> getNumAllocatableRegs ( & <STR_LIT> ) ; VGPRExcessLimit = Context -> RegClassInfo -> getNumAllocatableRegs ( & <STR_LIT> ) ; <ISA_LIT>MachineFunctionInfo & MFI = * MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; TargetOccupancy = MFI . getOccupancy ( ) ; SGPRCriticalLimit = std :: min ( ST . getMaxNumSGPRs ( TargetOccupancy , true ) , SGPRExcessLimit ) ; VGPRCriticalLimit = std :: min ( ST .

getMaxNumVGPRs ( TargetOccupancy ) , VGPRExcessLimit ) ;


<s> SGPRExcessLimit = Context -> RegClassInfo -> getNumAllocatableRegs ( & <STR_LIT> ) ; VGPRExcessLimit = Context -> RegClassInfo -> getNumAllocatableRegs ( & <STR_LIT> ) ; <ISA_LIT>MachineFunctionInfo & MFI = * MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; TargetOccupancy = MFI . getOccupancy ( ) ; SGPRCriticalLimit = std :: min ( ST . getMaxNumSGPRs ( TargetOccupancy , true ) , SGPRExcessLimit ) ; VGPRCriticalLimit = std :: min ( ST . getMaxNumVGPRs ( TargetOccupancy ) , VGPRExcessLimit ) ; SGPRCriticalLimit = std :: min ( SGPRCriticalLimit


<s> bool GCNScheduleDAGMILive :: isTriviallyReMaterializable ( const MachineInstr & MI , AAResults * AA ) { if ( ! TII -> isTriviallyReMaterializable ( MI , AA ) ) return false ; for ( const MachineOperand & MO : MI . operands ( ) ) if ( MO . isReg ( ) && MO . isUse ( ) && MO . getReg


<s> if ( ! TII -> isTriviallyReMaterializable ( MI , AA ) ) return false ; for ( const MachineOperand & MO : MI . operands ( ) ) if ( MO . isReg ( ) && MO . isUse ( ) && MO . getReg ( ) . isVirtual ( ) ) return false ; return


<s> } GCNRegPressure PressureBefore ; if ( LIS ) { PressureBefore = Pressure [ RegionIdx ] ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; GCNRPTracker :: printLiveRegs ( dbgs ( ) , LiveIns [ RegionIdx ] , MRI ) ; dbgs ( ) << <STR_LIT> ; llvm :: getRegPressure ( MRI , LiveIns [ RegionIdx ] ) . print ( dbgs ( ) ) ; dbgs ( ) << <STR_LIT> ; PressureBefore . print ( dbgs ( ) ) ) ; } GCNMaxOccupancySchedStrategy & S = ( GCNMaxOccupancySchedStrategy & ) * SchedImpl ; S . HasClusteredNodes = Stage > InitialSchedule ; S . HasExcessPressure = false ; ScheduleDAGMILive :: schedule ( ) ; Regions [ RegionIdx ] = std :: make_pair ( RegionBegin , RegionEnd ) ; RescheduleRegions [ RegionIdx ] = false ; if ( Stage == InitialSchedule && S . HasClusteredNodes ) RegionsWithClusters [ RegionIdx ] = true ; if ( S . HasExcessPressure ) RegionsWithHighRP [ RegionIdx ] = true ; if ( ! LIS ) return ; auto PressureAfter = getRealRegPressure ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; PressureAfter . print ( dbgs ( ) ) ) ; if ( PressureAfter . getSGPRNum ( ) <= S . SGPRCriticalLimit && PressureAfter . getVGPRNum ( ST . hasGFX90AInsts ( ) ) <= S . VGPRCriticalLimit ) { Pressure [ RegionIdx ] = PressureAfter ; RegionsWithMinOcc [ RegionIdx ] = PressureAfter . getOccupancy ( ST ) == MinOccupancy ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } unsigned WavesAfter = std :: min ( S . TargetOccupancy , PressureAfter . getOccupancy ( ST ) ) ; unsigned WavesBefore = std :: min ( S . TargetOccupancy , PressureBefore . getOccupancy ( ST ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << WavesBefore << <STR_LIT> << WavesAfter << <STR_LIT> ) ; unsigned NewOccupancy = std :: max ( WavesAfter , WavesBefore ) ; if ( WavesAfter < WavesBefore && WavesAfter < MinOccupancy && WavesAfter >= MFI . getMinAllowedOccupancy ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MFI . getMinAllowedOccupancy ( ) << <STR_LIT> ) ; NewOccupancy = WavesAfter ; } if ( NewOccupancy < MinOccupancy ) { MinOccupancy = NewOccupancy ; MFI . limitOccupancy ( MinOccupancy ) ; RegionsWithMinOcc . reset ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; } unsigned MaxVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned MaxSGPRs = ST . getMaxNumSGPRs ( MF ) ; if ( PressureAfter . getVGPRNum ( false ) > MaxVGPRs || PressureAfter . getAGPRNum ( ) > MaxVGPRs || PressureAfter . getSGPRNum ( ) > MaxSGPRs ) { RescheduleRegions [ RegionIdx ] = true ; RegionsWithHighRP [ RegionIdx ] = true ; } if ( WavesAfter >= MinOccupancy )


<s> TotalSinkableRegs += <ISA_LIT>RegisterInfo :: getNumCoveredRegs ( NewLiveIns [ I ] [ DefReg ] ) ; } int VGPRsAfterSink = VGPRUsage - TotalSinkableRegs ; unsigned OptimisticOccupancy = ST . getOccupancyWithNumVGPRs ( VGPRsAfterSink ) ; if ( OptimisticOccupancy <= MinOccupancy ) break ; unsigned ImproveOccupancy = <NUM_LIT> ; SmallVector < MachineInstr * , <NUM_LIT> > SinkedDefs ; for ( auto & It : RematerializableInsts [ I ] ) { MachineInstr * Def = It . first ; MachineBasicBlock :: iterator InsertPos = MachineBasicBlock :: iterator ( It . second ) ; Register Reg = Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ; TII -> reMaterialize ( * InsertPos -> getParent ( ) , InsertPos , Reg , Def -> getOperand ( <NUM_LIT> ) . getSubReg ( ) , * Def , * TRI ) ; MachineInstr * NewMI = & * ( -- InsertPos ) ; LIS -> InsertMachineInstrInMaps ( * NewMI ) ; LIS -> removeInterval ( Reg ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; InsertedMIToOldDef [ NewMI ] = Def ; updateRegionBoundaries ( NewRegions , Def , nullptr , true ) ; updateRegionBoundaries ( NewRegions , InsertPos , NewMI ) ; LaneBitmask PrevMask = NewLiveIns [ I ] [ Reg ] ; for ( auto Idx : RematDefToLiveInRegions [ Def ] ) { NewLiveIns [ Idx ] . erase ( Reg ) ; if ( InsertPos -> getParent ( ) != Regions [ Idx ] . first -> getParent ( ) ) { NewPressure [ Idx ] . inc ( Reg , PrevMask , LaneBitmask :: getNone ( ) , MRI ) ; } else { GCNDownwardRPTracker RPT ( * LIS ) ; auto * NonDbgMI = & * skipDebugInstructionsForward ( NewRegions [ Idx ] . first , NewRegions [ Idx ] . second ) ; RPT . reset ( * NonDbgMI , & NewLiveIns [ Idx ] ) ; RPT . advance ( NewRegions [ Idx ] . second ) ; NewPressure [ Idx ] = RPT . moveMaxPressure ( ) ; } } SinkedDefs . push_back ( Def ) ; ImproveOccupancy = NewPressure [ I ] . getOccupancy ( ST ) ; if ( ImproveOccupancy > MinOccupancy ) break ; } for ( auto & Def : SinkedDefs ) for ( auto TrackedIdx : RematDefToLiveInRegions [ Def ] ) RematerializableInsts [ TrackedIdx ] . erase ( Def ) ; if ( ImproveOccupancy <= MinOccupancy ) break ; NewRescheduleRegions [ I ] = true ; Improved = true ; } if ( ! Improved ) { for ( auto & Entry : InsertedMIToOldDef ) { MachineInstr * MI = Entry . first ; MachineInstr * OldMI = Entry . second ; Register Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; LIS -> RemoveMachineInstrFromMaps ( * MI ) ; MI -> eraseFromParent ( ) ; OldMI -> clearRegisterDeads ( Reg ) ; LIS -> removeInterval ( Reg ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; } return false ; } for ( auto & Entry : InsertedMIToOldDef


<s> if ( MI == RegionBoundaries [ I ] . first ) { if ( Removing ) RegionBoundaries [ I ] = std :: make_pair ( std :: next ( MI ) , RegionBoundaries [ I ] . second ) ; else RegionBoundaries [ I ] = std :: make_pair ( MachineBasicBlock :: iterator ( NewMI ) , RegionBoundaries [ I ] . second ) ; return ; } if ( Removing && MI == RegionBoundaries [ I ] . second


<s> MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == InitialSchedule ) computeBlockPressure ( MBB ) ; } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << MF . getName ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> << * begin ( ) << <STR_LIT> ; if ( RegionEnd != MBB -> end ( ) ) dbgs ( ) << * RegionEnd ; else dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> << NumRegionInstrs << '\n' ) ; schedule ( ) ; exitRegion ( ) ; ++ RegionIdx ; } finishBlock ( ) ; if ( Stage == UnclusteredReschedule ) SavedMutations . swap ( Mutations ) ; } while ( Stage !=


<s> SchedCandidate TCand ; TCand . reset ( CandPolicy ( ) ) ; pickNodeFromQueue ( Top , TopPolicy , DAG -> getTopRPTracker ( ) , TCand ) ; assert ( TCand . SU == TopCand . SU && <STR_LIT> ) ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; traceCandidate ( TopCand ) ; dbgs ( ) << <STR_LIT> ; traceCandidate ( BotCand ) ; ) ; SchedCandidate Cand ; if ( TopCand . Reason == BotCand . Reason ) { Cand = BotCand ; GenericSchedulerBase :: CandReason TopReason = TopCand . Reason ; TopCand . Reason = NoCand ; GenericScheduler :: tryCandidate ( Cand , TopCand , nullptr ) ; if ( TopCand . Reason != NoCand ) { Cand . setBest ( TopCand ) ; } else { TopCand . Reason = TopReason ; } } else { if ( TopCand . Reason == RegExcess && TopCand . RPDelta . Excess . getUnitInc ( ) <= <NUM_LIT> ) { Cand = TopCand ; } else if ( BotCand . Reason == RegExcess && BotCand . RPDelta . Excess . getUnitInc ( ) <= <NUM_LIT> ) { Cand = BotCand ; } else if ( TopCand . Reason == RegCritical && TopCand . RPDelta . CriticalMax . getUnitInc ( ) <= <NUM_LIT> ) { Cand = TopCand ; } else if ( BotCand . Reason == RegCritical && BotCand . RPDelta . CriticalMax . getUnitInc ( ) <= <NUM_LIT> ) { Cand = BotCand ; } else { if ( BotCand . Reason > TopCand . Reason ) { Cand = TopCand ; } else { Cand =


<s> unsigned NewSGPRPressure = Pressure [ SRI -> getSGPRPressureSet ( ) ] ; unsigned NewVGPRPressure = Pressure [ SRI -> getVGPRPressureSet ( ) ] ; const unsigned MaxVGPRPressureInc = <NUM_LIT> ; bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit ; bool ShouldTrackSGPRs = ! ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit ; if ( ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit ) { Cand . RPDelta . Excess = PressureChange ( SRI -> getVGPRPressureSet ( ) ) ; Cand . RPDelta . Excess . setUnitInc ( NewVGPRPressure - VGPRExcessLimit ) ; } if ( ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit ) { Cand . RPDelta . Excess = PressureChange ( SRI -> getSGPRPressureSet ( ) ) ; Cand . RPDelta . Excess . setUnitInc ( NewSGPRPressure - SGPRExcessLimit ) ; } int SGPRDelta = NewSGPRPressure


<s> unsigned VGPRPressure = Pressure [ SRI -> getVGPRPressureSet ( ) ] ; ReadyQueue & Q = Zone . Available ; for ( SUnit * SU : Q ) { SchedCandidate TryCand ( ZonePolicy ) ; initCandidate ( TryCand , SU , Zone . isTop ( ) , RPTracker , SRI , SGPRPressure , VGPRPressure ) ; SchedBoundary * ZoneArg = Cand . AtTop == TryCand . AtTop ? & Zone : nullptr ; GenericScheduler :: tryCandidate ( Cand , TryCand ,


<s> LiveIns . resize ( Regions . size ( ) ) ; Pressure . resize ( Regions . size ( ) ) ; do { Stage ++ ; RegionIdx = <NUM_LIT> ; MachineBasicBlock * MBB = nullptr ; if ( Stage > <NUM_LIT> ) { if ( ! LIS || StartingOccupancy <= MinOccupancy ) break ; DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; S . setTargetOccupancy ( MinOccupancy ) ; } for ( auto Region : Regions ) { RegionBegin = Region . first ; RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( ) ; MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == <NUM_LIT> ) computeBlockPressure ( MBB ) ; } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ) ; DEBUG ( dbgs ( ) << MF . getName ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> << * begin ( ) << <STR_LIT> ; if ( RegionEnd != MBB -> end ( ) ) dbgs ( ) << * RegionEnd ; else dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> << NumRegionInstrs << '\n' ) ; schedule ( ) ; exitRegion ( ) ; ++ RegionIdx ; } finishBlock ( )


<s> SGPRExcessLimit = Context -> RegClassInfo -> getNumAllocatableRegs ( & <STR_LIT> ) - ErrorMargin ; VGPRExcessLimit = Context -> RegClassInfo -> getNumAllocatableRegs ( & <STR_LIT> ) - ErrorMargin ; if ( TargetOccupancy ) { SGPRCriticalLimit = ST . getMaxNumSGPRs ( TargetOccupancy , true ) ; VGPRCriticalLimit = ST . getMaxNumVGPRs ( TargetOccupancy ) ; } else { SGPRCriticalLimit = SRI -> getRegPressureSetLimit ( DAG -> MF , SRI -> getSGPRPressureSet ( ) ) ; VGPRCriticalLimit = SRI -> getRegPressureSetLimit ( DAG -> MF , SRI -> getVGPRPressureSet ( ) ) ; } SGPRCriticalLimit -=


<s> SUnit * GCNMaxOccupancySchedStrategy :: pickNodeBidirectional ( bool & IsTopNode ) { if ( SUnit * SU = Bot . pickOnlyChoice ( ) ) { IsTopNode = false ; return SU ; } if ( SUnit * SU = Top . pickOnlyChoice ( ) ) { IsTopNode = true ; return SU ; } CandPolicy BotPolicy ; setPolicy ( BotPolicy , false , Bot , & Top ) ; CandPolicy TopPolicy ; setPolicy ( TopPolicy , false , Top , & Bot ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( ! BotCand . isValid ( ) || BotCand . SU -> isScheduled || BotCand . Policy != BotPolicy ) { BotCand . reset ( CandPolicy ( ) ) ; pickNodeFromQueue ( Bot , BotPolicy , DAG -> getBotRPTracker ( ) , BotCand ) ; assert ( BotCand . Reason != NoCand && <STR_LIT> ) ; } else { DEBUG ( traceCandidate ( BotCand ) ) ; } DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( ! TopCand . isValid ( ) || TopCand . SU -> isScheduled || TopCand . Policy != TopPolicy ) { TopCand . reset ( CandPolicy


<s> GCNRPTracker :: printLiveRegs ( dbgs ( ) , LiveIns [ RegionIdx ] , MRI ) ; dbgs ( ) << <STR_LIT> ; llvm :: getRegPressure ( MRI , LiveIns [ RegionIdx ] ) . print ( dbgs ( ) ) ; dbgs ( ) << <STR_LIT> ; PressureBefore . print ( dbgs ( ) ) ) ; } ScheduleDAGMILive :: schedule ( ) ; Regions [ RegionIdx ] = std :: make_pair ( RegionBegin , RegionEnd ) ; if ( ! LIS ) return ; GCNMaxOccupancySchedStrategy & S = ( GCNMaxOccupancySchedStrategy & ) * SchedImpl ; auto PressureAfter = getRealRegPressure ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; PressureAfter . print ( dbgs ( ) ) ) ; if ( PressureAfter . getSGPRNum ( ) <= S . SGPRCriticalLimit && PressureAfter . getVGPRNum ( ) <= S . VGPRCriticalLimit ) { Pressure [ RegionIdx ] = PressureAfter ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } unsigned WavesAfter = getMaxWaves ( PressureAfter . getSGPRNum ( ) , PressureAfter . getVGPRNum ( ) , MF ) ; unsigned WavesBefore = getMaxWaves ( PressureBefore . getSGPRNum ( ) , PressureBefore . getVGPRNum ( ) , MF ) ; DEBUG ( dbgs ( ) << <STR_LIT> << WavesBefore << <STR_LIT> << WavesAfter << <STR_LIT> ) ; unsigned NewOccupancy = std :: max ( WavesAfter , WavesBefore ) ; if ( NewOccupancy


<s> GCNScheduleDAGMILive :: GCNScheduleDAGMILive ( MachineSchedContext * C , std :: unique_ptr < MachineSchedStrategy > S ) : ScheduleDAGMILive ( C , std :: move ( S ) ) , ST ( MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ) , MFI ( * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ) , StartingOccupancy ( MFI . getOccupancy ( ) ) , MinOccupancy ( StartingOccupancy ) , Stage ( <NUM_LIT> ) , RegionIdx ( <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << StartingOccupancy <<


<s> Pressure . resize ( Regions . size ( ) ) ; do { Stage ++ ; RegionIdx = <NUM_LIT> ; MachineBasicBlock * MBB = nullptr ; if ( Stage > <NUM_LIT> ) { if ( ! LIS || StartingOccupancy <= MinOccupancy ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; S . setTargetOccupancy ( MinOccupancy ) ; } for ( auto Region : Regions ) { RegionBegin = Region . first ; RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( ) ; MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == <NUM_LIT> ) computeBlockPressure ( MBB ) ; } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << MF . getName ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> << * begin ( ) << <STR_LIT> ; if ( RegionEnd != MBB -> end ( ) ) dbgs ( ) << * RegionEnd ; else dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> << NumRegionInstrs << '\n' ) ; schedule ( ) ; exitRegion ( ) ; ++ RegionIdx ; } finishBlock ( )


<s> ScheduleDAGMILive ( C , std :: move ( S ) ) , ST ( MF . getSubtarget < GCNSubtarget > ( ) ) , MFI ( * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ) , StartingOccupancy ( MFI . getOccupancy ( ) ) , MinOccupancy ( StartingOccupancy ) ,

Stage ( <NUM_LIT> ) , RegionIdx ( <NUM_LIT> ) {


<s> ScheduleDAGMILive ( C , std :: move ( S ) ) , ST ( MF . getSubtarget < GCNSubtarget > ( ) ) , MFI ( * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ) , StartingOccupancy ( MFI

. getOccupancy ( ) ) , MinOccupancy ( StartingOccupancy ) , Stage ( <NUM_LIT> ) , RegionIdx ( <NUM_LIT> ) {


<s> Cand . AtTop = AtTop ; RegPressureTracker & TempTracker = const_cast < RegPressureTracker & > ( RPTracker ) ; std :: vector < unsigned > Pressure ; std :: vector < unsigned > MaxPressure ; if ( AtTop ) TempTracker . getDownwardPressure ( SU -> getInstr ( ) , Pressure , MaxPressure ) ; else { TempTracker . getUpwardPressure ( SU -> getInstr ( ) , Pressure , MaxPressure ) ; } unsigned NewSGPRPressure = Pressure [ SRI -> getSGPRPressureSet ( ) ] ; unsigned NewVGPRPressure = Pressure [ SRI -> getVGPRPressureSet ( ) ] ; const unsigned MaxVGPRPressureInc = <NUM_LIT> ; bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit ; bool ShouldTrackSGPRs = ! ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit ; if ( ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit ) { Cand . RPDelta . Excess = PressureChange ( SRI -> getVGPRPressureSet ( ) ) ; Cand . RPDelta . Excess . setUnitInc ( NewVGPRPressure - VGPRExcessLimit ) ; } if ( ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit ) { Cand . RPDelta . Excess = PressureChange ( SRI -> getSGPRPressureSet ( ) ) ; Cand . RPDelta . Excess . setUnitInc ( NewSGPRPressure - SGPRExcessLimit ) ; } int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit ; int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit ; if ( SGPRDelta >= <NUM_LIT> || VGPRDelta >= <NUM_LIT> ) { if ( SGPRDelta > VGPRDelta ) { Cand . RPDelta . CriticalMax = PressureChange ( SRI -> getSGPRPressureSet ( ) ) ; Cand . RPDelta . CriticalMax . setUnitInc ( SGPRDelta ) ; } else { Cand . RPDelta . CriticalMax = PressureChange ( SRI -> getVGPRPressureSet ( ) ) ; Cand . RPDelta . CriticalMax . setUnitInc ( VGPRDelta


<s> const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; const int ErrorMargin = <NUM_LIT> ; SGPRExcessLimit = Context -> RegClassInfo -> getNumAllocatableRegs ( & <STR_LIT> ) - ErrorMargin ; VGPRExcessLimit = Context -> RegClassInfo -> getNumAllocatableRegs ( & <STR_LIT> ) - ErrorMargin ; if ( TargetOccupancy ) { SGPRCriticalLimit = ST . getMaxNumSGPRs ( TargetOccupancy , true ) ; VGPRCriticalLimit = ST . getMaxNumVGPRs


<s> } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( ! TopCand . isValid ( ) || TopCand . SU -> isScheduled || TopCand . Policy != TopPolicy ) { TopCand . reset ( CandPolicy ( ) ) ; pickNodeFromQueue ( Top , TopPolicy , DAG -> getTopRPTracker ( ) , TopCand ) ; assert ( TopCand . Reason != NoCand && <STR_LIT> ) ; } else { LLVM_DEBUG ( traceCandidate ( TopCand ) ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; traceCandidate ( TopCand ) ; dbgs ( ) << <STR_LIT> ; traceCandidate ( BotCand ) ; ) ; SchedCandidate Cand ; if ( TopCand . Reason == BotCand . Reason ) { Cand = BotCand ; GenericSchedulerBase :: CandReason TopReason = TopCand . Reason ; TopCand . Reason = NoCand ; GenericScheduler :: tryCandidate ( Cand , TopCand , nullptr ) ; if ( TopCand . Reason != NoCand ) { Cand . setBest ( TopCand ) ; } else { TopCand . Reason = TopReason ; } } else { if ( TopCand . Reason == RegExcess && TopCand . RPDelta . Excess . getUnitInc ( ) <= <NUM_LIT> ) { Cand = TopCand ; } else if ( BotCand . Reason == RegExcess && BotCand . RPDelta . Excess . getUnitInc ( ) <= <NUM_LIT> ) { Cand = BotCand ; } else if ( TopCand . Reason == RegCritical && TopCand . RPDelta . CriticalMax . getUnitInc ( ) <= <NUM_LIT> ) { Cand = TopCand ; } else if ( BotCand . Reason == RegCritical && BotCand . RPDelta . CriticalMax . getUnitInc ( ) <= <NUM_LIT> ) { Cand = BotCand ; } else { if ( BotCand . Reason > TopCand . Reason ) { Cand = TopCand ; } else


<s> unsigned MinRegOccupancy = std :: min ( ST . getOccupancyWithNumSGPRs ( SGPRs ) , ST . getOccupancyWithNumVGPRs ( VGPRs ) ) ; return std :: min ( MinRegOccupancy , ST . getOccupancyWithLocalMemSize ( MFI -> getLDSSize (


<s> unsigned MinRegOccupancy = std :: min ( ST . getOccupancyWithNumSGPRs ( SGPRs ) , ST . getOccupancyWithNumVGPRs ( VGPRs ) ) ; return std :: min ( MinRegOccupancy , ST . getOccupancyWithLocalMemSize (


<s> RegPressureTracker & TempTracker = const_cast < RegPressureTracker & > ( RPTracker ) ; std :: vector < unsigned > Pressure ; std :: vector < unsigned > MaxPressure ; if ( AtTop ) TempTracker . getDownwardPressure ( SU -> getInstr ( ) , Pressure , MaxPressure ) ; else { TempTracker . getUpwardPressure ( SU -> getInstr ( ) , Pressure , MaxPressure ) ; } int NewSGPRPressure = Pressure [ SRI -> getSGPRPressureSet ( ) ] ; int NewVGPRPressure = Pressure [ SRI -> getVGPRPressureSet ( ) ] ; const int MaxVGPRPressureInc = <NUM_LIT> ; bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit ; bool ShouldTrackSGPRs = ! ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit ; const int ErrorMargin = <NUM_LIT> ; VGPRExcessLimit -= ErrorMargin ; SGPRExcessLimit -= ErrorMargin ; if ( ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit ) { Cand . RPDelta . Excess = PressureChange ( SRI -> getVGPRPressureSet ( ) ) ; Cand . RPDelta . Excess . setUnitInc ( NewVGPRPressure - VGPRExcessLimit ) ; } if ( ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit ) { Cand . RPDelta . Excess = PressureChange ( SRI -> getSGPRPressureSet ( ) ) ; Cand . RPDelta . Excess . setUnitInc ( NewSGPRPressure = SGPRExcessLimit ) ; } VGPRCriticalLimit -= ErrorMargin ; SGPRCriticalLimit -= ErrorMargin ; int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit ; int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit ; if ( SGPRDelta >= <NUM_LIT> || VGPRDelta >= <NUM_LIT> ) { if ( SGPRDelta


<s> ) ; SchedCandidate Cand ; if ( TopCand . Reason == BotCand . Reason ) { Cand = BotCand ; GenericSchedulerBase :: CandReason TopReason = TopCand . Reason ; TopCand . Reason = NoCand ; GenericScheduler :: tryCandidate ( Cand , TopCand , nullptr ) ; if ( TopCand . Reason != NoCand ) { Cand . setBest ( TopCand ) ; } else { TopCand . Reason = TopReason ; } } else { if ( TopCand . Reason == RegExcess && TopCand . RPDelta . Excess . getUnitInc ( ) <= <NUM_LIT> ) { Cand = TopCand ; } else if ( BotCand . Reason == RegExcess && BotCand . RPDelta . Excess . getUnitInc ( ) <= <NUM_LIT> ) { Cand = BotCand ; } else if ( TopCand . Reason == RegCritical && TopCand . RPDelta . CriticalMax . getUnitInc ( ) <= <NUM_LIT> ) { Cand = TopCand ; } else if ( BotCand . Reason == RegCritical && BotCand . RPDelta . CriticalMax . getUnitInc ( ) <= <NUM_LIT> ) { Cand = BotCand ; } else { int TopRank = getBidirectionalReasonRank ( TopCand . Reason ) ; int BotRank = getBidirectionalReasonRank ( BotCand . Reason ) ; if ( TopRank > BotRank ) { Cand = TopCand ; } else { Cand = BotCand ; } } } DEBUG ( dbgs ( ) << <STR_LIT> ; traceCandidate ( Cand )


<s> ArrayRef < unsigned > Pressure = RPTracker . getRegSetPressureAtPos ( ) ; unsigned SGPRPressure = Pressure [ SRI -> getSGPRPressureSet ( ) ] ; unsigned VGPRPressure = Pressure [ SRI -> getVGPRPressureSet ( ) ] ; unsigned SGPRExcessLimit = Context -> RegClassInfo -> getNumAllocatableRegs ( & <STR_LIT> ) ; unsigned VGPRExcessLimit = Context -> RegClassInfo -> getNumAllocatableRegs ( & <STR_LIT> ) ; unsigned MaxWaves = getMaxWaves ( SGPRPressure , VGPRPressure , DAG -> MF ) ; unsigned SGPRCriticalLimit = SRI -> getMaxNumSGPRs ( ST , MaxWaves , true ) ; unsigned VGPRCriticalLimit = SRI -> getMaxNumVGPRs ( MaxWaves


<s> unsigned SGPRExcessLimit = Context -> RegClassInfo -> getNumAllocatableRegs ( & <STR_LIT> ) ; unsigned VGPRExcessLimit = Context -> RegClassInfo -> getNumAllocatableRegs ( & <STR_LIT> ) ; unsigned MaxWaves = getMaxWaves ( SGPRPressure , VGPRPressure , DAG -> MF ) ; unsigned SGPRCriticalLimit = SRI -> getMaxNumSGPRs ( ST , MaxWaves , true ) ; unsigned VGPRCriticalLimit = SRI -> getMaxNumVGPRs ( MaxWaves ) ; ReadyQueue & Q = Zone . Available ; for ( SUnit * SU : Q ) { SchedCandidate TryCand ( ZonePolicy ) ; initCandidate ( TryCand , SU , Zone . isTop ( ) , RPTracker , SRI , SGPRPressure , VGPRPressure , SGPRExcessLimit , VGPRExcessLimit , SGPRCriticalLimit , VGPRCriticalLimit ) ; SchedBoundary * ZoneArg = Cand . AtTop == TryCand . AtTop ? & Zone : nullptr ; GenericScheduler :: tryCandidate ( Cand , TryCand


<s> } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ) ; DEBUG ( dbgs ( ) << MF . getName ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> << * begin ( ) << <STR_LIT> ; if ( RegionEnd != MBB -> end ( ) ) dbgs ( ) <<


<s> GCNScheduleDAGMILive :: GCNScheduleDAGMILive ( MachineSchedContext * C , std :: unique_ptr < MachineSchedStrategy > S ) : ScheduleDAGMILive ( C , std :: move ( S ) ) , ST ( MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ) , MFI ( * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ) , StartingOccupancy ( ST . getOccupancyWithLocalMemSize ( MFI . getLDSSize ( ) , * MF . getFunction ( ) ) ) , MinOccupancy ( StartingOccupancy ) , Stage ( <NUM_LIT> ) { DEBUG ( dbgs ( ) << <STR_LIT> <<


<s> } else if ( ! LI . liveAt ( <ISA_LIT> ) ) { setMask ( MRI , SRI , It . first , It . second , LaneBitmask :: getNone ( ) , SGPRs , VGPRs ) ; } } for ( const auto & MO : MI . defs ( ) ) { if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( Reg ) ) continue ; unsigned SubRegIdx = MO . getSubReg ( ) ; LaneBitmask LaneMask = SubRegIdx != <NUM_LIT> ? TRI -> getSubRegIndexLaneMask ( SubRegIdx ) : MRI . getMaxLaneMaskForVReg ( Reg ) ; LaneBitmask & LM = LiveRegs [ Reg ] ; setMask ( MRI , SRI , Reg , LM , LM | LaneMask , SGPRs , VGPRs ) ; } MaxSGPRs = std :: max ( MaxSGPRs , SGPRs ) ; MaxVGPRs = std :: max ( MaxVGPRs , VGPRs ) ; } DEBUG ( dbgs ( ) << <STR_LIT> << MaxSGPRs << <STR_LIT> << MaxVGPRs << '\n' ) ; return std :: make_pair ( MaxSGPRs


<s> if ( MI -> getIterator ( ) != RegionEnd ) { BB -> remove ( MI ) ; BB -> insert ( RegionEnd , MI ) ; LIS -> handleMove ( * MI , true ) ; } for ( auto & Op : MI -> operands ( ) ) if ( Op . isReg ( ) && Op . isDef ( ) ) Op . setIsUndef ( false ) ; RegisterOperands RegOpers ; RegOpers . collect ( * MI , * TRI , MRI , ShouldTrackLaneMasks , false ) ; if ( ShouldTrackLaneMasks ) { SlotIndex SlotIdx = LIS -> getInstructionIndex ( * MI ) . getRegSlot ( ) ; RegOpers . adjustLaneLiveness ( * LIS , MRI , SlotIdx , MI ) ; } else { RegOpers . detectDeadDefs ( * MI , * LIS ) ; } RegionEnd = MI -> getIterator ( ) ; ++ RegionEnd ; DEBUG ( dbgs ( ) << <STR_LIT> << *


<s> int NewRegs = countPopulation ( NewMask . getAsInteger ( ) ) - countPopulation ( PrevMask . getAsInteger ( ) ) ; if ( SRI -> isSGPRReg ( MRI , Reg


<s> void GCNScheduleDAGMILive :: finalizeSchedule ( ) { GCNMaxOccupancySchedStrategy & S = ( GCNMaxOccupancySchedStrategy & ) * SchedImpl ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; LiveIns . resize ( Regions . size ( ) ) ; Pressure . resize ( Regions . size ( ) ) ; do { Stage ++ ; RegionIdx = <NUM_LIT> ; MachineBasicBlock * MBB = nullptr ; if ( Stage > <NUM_LIT> ) { if ( ! LIS || StartingOccupancy <= MinOccupancy ) break ; DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; S . setTargetOccupancy ( MinOccupancy ) ; } for ( auto Region : Regions ) { RegionBegin = Region . first ; RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( )


<s> do { Stage ++ ; RegionIdx = <NUM_LIT> ; MachineBasicBlock * MBB = nullptr ; if ( Stage > <NUM_LIT> ) { if ( ! LIS || StartingOccupancy <= MinOccupancy ) break ; DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; S . setTargetOccupancy ( MinOccupancy ) ; } for ( auto Region : Regions ) { RegionBegin = Region . first ; RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( ) ; MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == <NUM_LIT> ) computeBlockPressure ( MBB ) ; } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> ) ; DEBUG ( dbgs ( ) << MF . getName ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> <<


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned MinRegOccupancy = std :: min ( ST . getOccupancyWithNumSGPRs ( SGPRs ) , ST .


<s> } ScheduleDAGMILive :: schedule ( ) ; Regions [ RegionIdx ] = std :: make_pair ( RegionBegin , RegionEnd ) ; if ( ! LIS ) return ; GCNMaxOccupancySchedStrategy & S = ( GCNMaxOccupancySchedStrategy & ) * SchedImpl ; auto PressureAfter = getRealRegPressure ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; PressureAfter . print ( dbgs ( ) ) ) ; if ( PressureAfter . getSGPRNum ( ) <= S . SGPRCriticalLimit && PressureAfter . getVGPRNum ( ) <= S . VGPRCriticalLimit ) { Pressure [ RegionIdx ] = PressureAfter ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } unsigned WavesAfter = getMaxWaves ( PressureAfter . getSGPRNum ( ) , PressureAfter . getVGPRNum ( ) , MF ) ; unsigned WavesBefore = getMaxWaves ( PressureBefore . getSGPRNum ( ) , PressureBefore . getVGPRNum ( ) , MF ) ; DEBUG ( dbgs ( ) << <STR_LIT> << WavesBefore << <STR_LIT> << WavesAfter << <STR_LIT> ) ; unsigned NewOccupancy = std :: max ( WavesAfter , WavesBefore ) ; if ( NewOccupancy < MinOccupancy ) { MinOccupancy = NewOccupancy ; DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT>


<s> ScheduleDAGMILive :: schedule ( ) ; Regions [ RegionIdx ] = std :: make_pair ( RegionBegin , RegionEnd ) ; if ( ! LIS ) return ; GCNMaxOccupancySchedStrategy & S = ( GCNMaxOccupancySchedStrategy & ) * SchedImpl ; auto PressureAfter = getRealRegPressure ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; PressureAfter . print ( dbgs ( ) ) ) ; if ( PressureAfter . getSGPRNum ( ) <= S . SGPRCriticalLimit && PressureAfter . getVGPRNum ( ) <= S . VGPRCriticalLimit ) { Pressure [ RegionIdx ] = PressureAfter ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } unsigned WavesAfter = getMaxWaves ( PressureAfter . getSGPRNum ( ) , PressureAfter . getVGPRNum ( ) , MF ) ; unsigned WavesBefore = getMaxWaves ( PressureBefore . getSGPRNum ( ) , PressureBefore . getVGPRNum ( ) , MF ) ; DEBUG ( dbgs ( ) << <STR_LIT> << WavesBefore << <STR_LIT> << WavesAfter << <STR_LIT> ) ; unsigned NewOccupancy = std :: max ( WavesAfter , WavesBefore ) ; if ( NewOccupancy < MinOccupancy ) { MinOccupancy = NewOccupancy ; DEBUG ( dbgs ( ) << <STR_LIT>


<s> GCNScheduleDAGMILive ( MachineSchedContext * C , std

:: unique_ptr < MachineSchedStrategy > S ) :


<s> GCNScheduleDAGMILive ( MachineSchedContext * C , std :: unique_ptr < MachineSchedStrategy


<s> unsigned WavesBefore = std :: min ( Occ , PressureBefore . getOccupancy ( ST ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << WavesBefore << <STR_LIT> << WavesAfter << <STR_LIT> ) ; unsigned NewOccupancy = std :: max ( WavesAfter , WavesBefore ) ; if ( WavesAfter < WavesBefore && WavesAfter < MinOccupancy && WavesAfter >= MFI . getMinAllowedOccupancy ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MFI . getMinAllowedOccupancy ( ) << <STR_LIT> ) ; NewOccupancy = WavesAfter ; } if ( NewOccupancy < MinOccupancy ) { MinOccupancy = NewOccupancy ; MFI . limitOccupancy ( MinOccupancy ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; } if ( WavesAfter >= MinOccupancy ) { unsigned TotalVGPRs = <STR_LIT> :: getAddressableNumVGPRs ( & ST ) ; unsigned TotalSGPRs = <STR_LIT> :: getAddressableNumSGPRs ( & ST ) ; if ( WavesAfter > MFI . getMinWavesPerEU ( ) || PressureAfter . less ( ST , PressureBefore ) || ( TotalVGPRs >= PressureAfter . getVGPRNum ( ) && TotalSGPRs >= PressureAfter . getSGPRNum ( ) ) ) { Pressure [ RegionIdx ] = PressureAfter ; return ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; RegionEnd = RegionBegin ; for ( MachineInstr * MI : Unsched ) { if ( MI -> isDebugInstr ( ) ) continue ; if ( MI -> getIterator ( ) != RegionEnd ) { BB -> remove ( MI ) ; BB -> insert ( RegionEnd , MI ) ; if ( ! MI -> isDebugInstr ( ) ) LIS -> handleMove ( * MI , true ) ; } for ( auto & Op : MI -> operands ( ) ) if ( Op . isReg ( ) && Op . isDef ( ) ) Op . setIsUndef ( false ) ; RegisterOperands RegOpers ; RegOpers . collect ( * MI , * TRI , MRI , ShouldTrackLaneMasks , false ) ; if ( ! MI -> isDebugInstr ( ) ) { if ( ShouldTrackLaneMasks ) { SlotIndex SlotIdx = LIS -> getInstructionIndex ( * MI ) . getRegSlot ( ) ; RegOpers . adjustLaneLiveness ( * LIS , MRI , SlotIdx , MI ) ; } else { RegOpers . detectDeadDefs ( * MI , * LIS ) ; } } RegionEnd = MI -> getIterator ( ) ; ++ RegionEnd


<s> RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( ) ; MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == InitialSchedule ) computeBlockPressure ( MBB ) ; } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << MF . getName ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> << * begin ( ) << <STR_LIT> ; if ( RegionEnd != MBB -> end ( ) ) dbgs ( ) << * RegionEnd ; else dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> << NumRegionInstrs << '\n' ) ; schedule ( ) ; exitRegion ( ) ; ++ RegionIdx ; } finishBlock (


<s> } ScheduleDAGMILive :: schedule ( ) ; Regions [ RegionIdx ] = std :: make_pair ( RegionBegin , RegionEnd ) ; if ( ! LIS ) return ; GCNMaxOccupancySchedStrategy & S = ( GCNMaxOccupancySchedStrategy & ) * SchedImpl ; auto PressureAfter = getRealRegPressure ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; PressureAfter . print ( dbgs ( ) ) ) ; if ( PressureAfter . getSGPRNum ( ) <= S . SGPRCriticalLimit && PressureAfter . getVGPRNum ( ) <= S . VGPRCriticalLimit ) { Pressure [ RegionIdx ] = PressureAfter ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return ; } unsigned WavesAfter = getMaxWaves ( PressureAfter . getSGPRNum ( ) , PressureAfter . getVGPRNum ( ) , MF ) ; unsigned WavesBefore = getMaxWaves ( PressureBefore . getSGPRNum ( ) , PressureBefore . getVGPRNum ( ) , MF ) ; WavesAfter = std :: min ( WavesAfter , MFI . getMaxWavesPerEU ( ) ) ; WavesBefore = std :: min ( WavesBefore , MFI . getMaxWavesPerEU ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << WavesBefore << <STR_LIT> << WavesAfter << <STR_LIT> ) ; unsigned NewOccupancy = std :: max ( WavesAfter , WavesBefore ) ; if ( NewOccupancy < MinOccupancy ) { MinOccupancy = NewOccupancy ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; } if ( WavesAfter >= WavesBefore ) { Pressure [ RegionIdx ] = PressureAfter ; return ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; RegionEnd = RegionBegin ; for ( MachineInstr * MI : Unsched ) { if ( MI -> isDebugInstr ( ) ) continue ; if ( MI -> getIterator ( ) != RegionEnd ) { BB -> remove ( MI ) ; BB -> insert ( RegionEnd , MI ) ; if ( ! MI -> isDebugInstr ( ) ) LIS -> handleMove ( * MI , true ) ; } for ( auto & Op : MI -> operands ( ) ) if ( Op . isReg ( ) && Op . isDef ( ) ) Op . setIsUndef ( false ) ; RegisterOperands RegOpers ; RegOpers . collect ( * MI , * TRI , MRI , ShouldTrackLaneMasks , false ) ; if ( ! MI -> isDebugInstr ( ) ) { if ( ShouldTrackLaneMasks ) { SlotIndex SlotIdx = LIS -> getInstructionIndex ( * MI ) . getRegSlot ( ) ; RegOpers . adjustLaneLiveness ( * LIS , MRI , SlotIdx , MI ) ; } else { RegOpers . detectDeadDefs ( * MI


<s> TempTracker . getUpwardPressure ( SU -> getInstr ( ) , Pressure , MaxPressure ) ; } int NewSGPRPressure = Pressure [ SRI -> getSGPRPressureSet ( ) ] ; int NewVGPRPressure = Pressure [ SRI -> getVGPRPressureSet ( ) ] ; const int MaxVGPRPressureInc = <NUM_LIT> ; bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit ; bool ShouldTrackSGPRs = ! ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit ; const int ErrorMargin = <NUM_LIT> ; VGPRExcessLimit -= ErrorMargin ; SGPRExcessLimit -= ErrorMargin ; if ( ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit ) { Cand . RPDelta . Excess = PressureChange ( SRI -> getVGPRPressureSet ( ) ) ; Cand . RPDelta . Excess . setUnitInc ( NewVGPRPressure - VGPRExcessLimit ) ; } if ( ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit


<s> Register Reg = Register :: index2VirtReg ( I ) ; if ( ! LIS -> hasInterval ( Reg ) ) continue ; if ( ! SRI -> isVGPRClass ( MRI . getRegClass ( Reg ) ) || ! MRI . hasOneDef ( Reg ) || ! MRI . hasOneUse ( Reg ) ) continue ; if ( HighRPLiveIns . find ( Reg ) == HighRPLiveIns . end ( ) ) continue ; MachineInstr * Def = MRI . getOneDef ( Reg ) -> getParent ( ) ; if ( ! Def || Def -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <NUM_LIT> || ! isTriviallyReMaterializable ( * Def , AA ) ) continue ; MachineInstr * UseI = & * MRI . use_instr_begin ( Reg ) ; if ( Def -> getParent ( ) == UseI -> getParent ( ) ) continue ; RematerializableInsts . push_back ( std :: make_pair ( Def , UseI


<s> if ( RegionsWithMinOcc . count ( ) != <NUM_LIT> || Regions . size ( ) == <NUM_LIT> ) break ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const TargetInstrInfo * TII = MF . getSubtarget ( ) . getInstrInfo ( ) ; if ( ST . computeOccupancy ( MF . getFunction ( ) , MFI . getLDSSize ( ) ) == MinOccupancy ) break ; static_assert ( LastStage == PreRARematerialize , <STR_LIT> ) ; unsigned HighRPIdx = RegionsWithMinOcc . find_first ( ) ; collectRematerializableInstructions ( HighRPIdx ) ; if ( RematerializableInsts . empty ( ) || ! sinkTriviallyRematInsts ( ST , TII , HighRPIdx ) ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MinOccupancy << <STR_LIT> ) ; } } if ( Stage == UnclusteredReschedule ) SavedMutations . swap ( Mutations ) ; for ( auto Region : Regions ) { if ( ( ( Stage == UnclusteredReschedule || Stage == PreRARematerialize ) && ! RescheduleRegions [ RegionIdx ] ) || ( Stage == ClusteredLowOccupancyReschedule && ! RegionsWithClusters [ RegionIdx ] && ! RegionsWithHighRP [ RegionIdx ] ) ) { ++ RegionIdx ; continue ; } RegionBegin = Region . first ; RegionEnd = Region . second ; if ( RegionBegin -> getParent ( ) != MBB ) { if ( MBB ) finishBlock ( ) ; MBB = RegionBegin -> getParent ( ) ; startBlock ( MBB ) ; if ( Stage == InitialSchedule ) computeBlockPressure ( MBB ) ; } unsigned NumRegionInstrs = std :: distance ( begin ( ) , end ( ) ) ; enterRegion ( MBB , begin ( ) , end ( ) , NumRegionInstrs ) ; if ( begin ( ) == end ( ) || begin ( ) == std :: prev ( end ( ) ) ) { exitRegion ( ) ; ++ RegionIdx ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << MF . getName ( ) << <STR_LIT> << printMBBReference ( * MBB ) << <STR_LIT> << MBB -> getName ( ) << <STR_LIT> << * begin ( ) << <STR_LIT> ; if ( RegionEnd != MBB -> end ( ) ) dbgs ( ) << * RegionEnd ; else dbgs ( )


<s> GCNRegPressure RPAfterSinking = RPT . moveMaxPressure ( ) ; ImproveOccupancy = RPAfterSinking . getOccupancy ( ST ) ; if ( ImproveOccupancy > MinOccupancy ) break ; } if ( ImproveOccupancy <= MinOccupancy ) { for ( auto & Entry : InsertedMIToOldDef ) { MachineInstr * MI = Entry . first ; MachineInstr * OldMI = Entry . second ; Register Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; LIS -> RemoveMachineInstrFromMaps ( * MI ) ; MI -> eraseFromParent ( ) ; OldMI -> clearRegisterDeads ( Reg ) ; LIS -> removeInterval ( Reg ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; } return false ; } for ( auto & Entry : InsertedMIToOldDef ) { MachineInstr * MI = Entry . first ; MachineInstr * OldMI = Entry . second ; updateRegionBoundaries ( OldMI , nullptr , true ) ; BBLiveInMap . erase ( OldMI ) ; Register Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; LIS -> RemoveMachineInstrFromMaps ( * OldMI ) ; OldMI -> eraseFromParent ( ) ; LIS -> removeInterval ( Reg ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; MachineBasicBlock :: iterator InsertPos = std :: next ( MachineBasicBlock :: iterator ( MI ) ) ; updateRegionBoundaries ( InsertPos , MI ) ; } LiveIns [ HighRPIdx ] . copyFrom ( NewLiveIns ) ; MBBLiveIns . erase ( Regions [ HighRPIdx ] . first -> getParent ( ) ) ; GCNDownwardRPTracker RPTracker ( * LIS ) ; RPTracker . advance ( Regions [ HighRPIdx ] . first , Regions [ HighRPIdx ] . second , & LiveIns [ HighRPIdx ] ) ; Pressure [ HighRPIdx ] = RPTracker . moveMaxPressure ( ) ; <ISA_LIT>MachineFunctionInfo & MFI = * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MFI . increaseOccupancy ( MF ,


<s> -- CurRegion ; auto I = MBB -> begin ( ) ; auto LiveInIt = MBBLiveIns . find ( MBB ) ; if ( LiveInIt != MBBLiveIns . end ( ) ) { auto LiveIn = std :: move ( LiveInIt -> second ) ; RPTracker . reset ( * MBB -> begin ( ) , & LiveIn ) ; MBBLiveIns . erase ( LiveInIt ) ; } else { auto & Rgn = Regions [ CurRegion ] ; I = Rgn . first ; auto * NonDbgMI = & * skipDebugInstructionsForward ( Rgn . first , Rgn . second ) ; auto LRS = BBLiveInMap . lookup ( NonDbgMI ) ; assert ( isEqual ( getLiveRegsBefore ( * NonDbgMI , * LIS ) , LRS ) ) ; RPTracker . reset ( *