<s>
if
(
!
full
)
return
;
dbgs
(
)
<<
<STR_LIT>
<<
HighLatencyBlock
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
<ISA_LIT>ScheduleBlock
*
P
:
Preds
)
{
P
->
printDebug
(
false
)
;
}
dbgs
(
)
<<
<STR_LIT>
;
for
(
std
::
pair
<
<ISA_LIT>ScheduleBlock
*
,
<ISA_LIT>ScheduleBlockLinkKind
>
S
:
Succs
)
{
if
(
S
.
second
==
<STR_LIT>
)
dbgs
(
)
<<
<STR_LIT>
;
S
.
first
->
printDebug
(
false
)
;
}
if
(
Scheduled
)
{
dbgs
(
)
<<
<STR_LIT>
<<
LiveInPressure
[
<STR_LIT>
::
SReg_32
]
<<
' '
<<
LiveInPressure
[
<STR_LIT>
::
VGPR_32
]
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
LiveOutPressure
[
<STR_LIT>
::
SReg_32
]
<<
' '
<<
LiveOutPressure
[
<STR_LIT>
::
VGPR_32
]
<<
<STR_LIT>
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveInRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveOutRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;


<s> for ( unsigned SUNum : DAG -> BottomUpIndex2SU ) { SUnit * SU = & DAG -> SUnits [ SUNum ] ; bool hasSuccessor = false ; if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; hasSuccessor = true ;


<s> int GroupID = NextNonReservedID ++ ; for ( unsigned SUNum : DAG -> BottomUpIndex2SU ) { SUnit * SU = & DAG -> SUnits [ SUNum ] ; bool hasSuccessor = false ; if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; hasSuccessor = true ; } if ( ! hasSuccessor ) CurrentColoring [ SU -> NodeNum ] = GroupID ;


<s> void <ISA_LIT>ScheduleBlockScheduler :: releaseBlockSuccs ( <ISA_LIT>ScheduleBlock * Parent ) { for ( const auto & Block : Parent -> getSuccs ( ) ) {

if ( -- BlockNumPredsLeft [ Block . first -> getID ( ) ] == <NUM_LIT> ) ReadyBlocks . push_back ( Block . first ) ;


<s> if ( -- BlockNumPredsLeft [ Block . first -> getID ( ) ] == <NUM_LIT> ) ReadyBlocks . push_back ( Block . first ) ; if ( Parent -> isHighLatencyBlock ( ) && Block . second == <STR_LIT> ) LastPosHighLatencyParentScheduled [ Block . first -> getID ( ) ] = NumBlockScheduled ;


<s> void <ISA_LIT>ScheduleBlock :: releaseSucc ( SUnit * SU , SDep * SuccEdge ) { SUnit * SuccSU = SuccEdge -> getSUnit ( ) ; if ( SuccEdge -> isWeak ( ) ) { -- SuccSU -> WeakPredsLeft ; return ;


<s> } if ( SuccSU -> NumPredsLeft == <NUM_LIT> ) { dbgs ( ) << <STR_LIT> ; DAG -> dumpNode ( * SuccSU ) ;


<s> SUnit * SuccSU = Succ . getSUnit ( ) ; if ( SuccSU -> NodeNum >= DAG -> SUnits . size ( ) ) continue ; if ( BC -> isSUInBlock ( SuccSU , ID ) != InOrOutBlock ) continue ; releaseSucc ( SU , & Succ ) ;

if ( SuccSU -> NumPredsLeft == <NUM_LIT> && InOrOutBlock ) TopReadySUs . push_back ( SuccSU ) ;


<s> void <ISA_LIT>ScheduleDAGMI :: restoreSULinksLeft ( ) { for ( unsigned i = <NUM_LIT> , e = SUnits . size ( ) ; i != e ; ++ i ) { SUnits [ i ] . isScheduled = false ; SUnits [ i ] . WeakPredsLeft = SUnitsLinksBackup [ i ] . WeakPredsLeft ; SUnits [ i ] . NumPredsLeft = SUnitsLinksBackup [ i ] . NumPredsLeft ; SUnits [ i ] . WeakSuccsLeft = SUnitsLinksBackup [ i ] . WeakSuccsLeft ;

SUnits [ i ] . NumSuccsLeft = SUnitsLinksBackup [ i ] . NumSuccsLeft ;


<s> SUnits [ i ] . isScheduled = false ; SUnits [ i ] . WeakPredsLeft = SUnitsLinksBackup [ i ] . WeakPredsLeft ; SUnits [ i ] . NumPredsLeft = SUnitsLinksBackup [ i ] . NumPredsLeft ; SUnits [ i ] . WeakSuccsLeft = SUnitsLinksBackup [ i ] . WeakSuccsLeft ; SUnits [ i ] . NumSuccsLeft = SUnitsLinksBackup [ i ] . NumSuccsLeft ; }


<s> if ( <ISA_LIT>TII -> getMemOperandWithOffset ( * SU -> getInstr ( ) , BaseLatOp , OffLatReg , OffsetIsScalable , TRI ) ) LowLatencyOffset [ i ] = OffLatReg ; } else if ( <ISA_LIT>TII -> isHighLatencyDef ( SU -> getInstr ( ) -> getOpcode ( ) ) ) IsHighLatencySU [ i ] = <NUM_LIT> ; } <ISA_LIT>Scheduler Scheduler ( this ) ; Best = Scheduler . scheduleVariant ( <STR_LIT> , <STR_LIT> ) ; if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsageLatency } , { LatenciesGrouped , BlockLatencyRegUsage } , { LatenciesAlonePlusConsecutive , BlockLatencyRegUsage } , } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , { LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , { LatenciesAlonePlusConsecutive , BlockRegUsage } } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } ScheduledSUnits = Best . SUs ; ScheduledSUnitsInv . resize ( SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { ScheduledSUnitsInv [ ScheduledSUnits [ i ] ] = i ; } moveLowLatencies ( ) ; assert ( TopRPTracker . getPos ( ) == RegionBegin && <STR_LIT> ) ; TopRPTracker . setPos ( CurrentTop ) ; for ( unsigned I : ScheduledSUnits ) { SUnit * SU = & SUnits [ I ] ; scheduleMI ( SU , true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << * SU -> getInstr ( ) ) ; } assert ( CurrentTop == CurrentBottom && <STR_LIT> ) ; placeDebugValues ( ) ; LLVM_DEBUG ( {

dbgs ( ) << <STR_LIT> << printMBBReference ( * begin ( ) -> getParent ( ) ) << <STR_LIT> ;


<s> MachineBasicBlock :: iterator CurrentTopFastSched = DAG -> getCurrentTop ( ) ; std :: vector < MachineBasicBlock :: iterator > PosOld ; std :: vector < MachineBasicBlock :: iterator > PosNew ; PosOld . reserve ( DAG -> SUnits . size ( ) ) ; PosNew . reserve ( DAG -> SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { int BlockIndice = TopDownIndex2Block [ i ] ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; std :: vector < SUnit * > SUs = Block -> getScheduledUnits ( ) ; for ( SUnit * SU : SUs ) { MachineInstr * MI = SU -> getInstr ( ) ; MachineBasicBlock :: iterator Pos = MI ; PosOld . push_back ( Pos ) ; if ( & * CurrentTopFastSched == MI ) { PosNew . push_back ( Pos ) ; CurrentTopFastSched = nextIfDebug ( ++ CurrentTopFastSched , DAG -> getCurrentBottom ( ) ) ; } else { DAG -> getBB ( ) -> splice ( CurrentTopFastSched , DAG -> getBB ( ) , MI ) ; DAG -> getLIS ( ) -> handleMove ( * MI , true ) ;

PosNew . push_back ( CurrentTopFastSched ) ;


<s> struct <ISA_LIT>ScheduleBlockResult <STR_LIT> ( <ISA_LIT>SchedulerBlockCreatorVariant BlockVariant , <ISA_LIT>SchedulerBlockSchedulerVariant ScheduleVariant ) { <ISA_LIT>ScheduleBlocks Blocks = BlockCreator . getBlocks ( BlockVariant ) ; <ISA_LIT>ScheduleBlockScheduler Scheduler ( DAG , ScheduleVariant , Blocks ) ; std :: vector < <ISA_LIT>ScheduleBlock * > ScheduledBlocks ;

struct <ISA_LIT>ScheduleBlockResult Res ;


<s> <ISA_LIT>ScheduleBlockCreator :: <ISA_LIT>ScheduleBlockCreator ( <ISA_LIT>ScheduleDAGMI * DAG ) : DAG ( DAG ) {


<s> int topoInd = - <NUM_LIT> ; for ( <ISA_LIT>ScheduleBlock * Pred : Block -> getPreds ( ) ) { std :: set < unsigned > PredOutRegs = Pred -> getOutRegs ( ) ; std :: set < unsigned > :: iterator RegPos = PredOutRegs . find ( Reg ) ; if ( RegPos != PredOutRegs . end ( ) ) { Found = true ; if ( topoInd < BlocksStruct . TopDownBlock2Index [ Pred -> getID ( ) ] ) { topoInd = BlocksStruct . TopDownBlock2Index [ Pred -> getID ( ) ] ; } } } if ( ! Found ) continue ; int PredID = BlocksStruct . TopDownIndex2Block [ topoInd ] ; ++ LiveOutRegsNumUsages [ PredID ] [ Reg ] ; } } LastPosHighLatencyParentScheduled . resize ( Blocks . size ( ) , <NUM_LIT> ) ; BlockNumPredsLeft . resize ( Blocks . size ( ) ) ; BlockNumSuccsLeft . resize ( Blocks . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = Blocks [ i ] ; BlockNumPredsLeft [ i ] = Block -> getPreds ( ) . size ( ) ; BlockNumSuccsLeft [ i ] = Block -> getSuccs ( ) . size ( ) ; } for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = Blocks [ i ] ; assert ( Block -> getID ( ) == i ) ; } std :: set < unsigned > InRegs = DAG -> getInRegs ( ) ; addLiveRegs ( InRegs ) ; for ( unsigned Reg : DAG -> getOutRegs ( ) ) { for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { int ID = BlocksStruct . TopDownIndex2Block [ Blocks . size ( ) - <NUM_LIT> - i ] ; <ISA_LIT>ScheduleBlock * Block = Blocks [ ID ] ; const std :: set < unsigned > & OutRegs = Block -> getOutRegs ( ) ; if ( OutRegs . find ( Reg ) == OutRegs . end ( ) ) continue ; ++ LiveOutRegsNumUsages [ ID ] [ Reg ] ; break ; } } for ( <ISA_LIT>ScheduleBlock * Block : Blocks ) { for ( unsigned Reg : Block -> getInRegs ( ) ) { bool Found = false ; for ( <ISA_LIT>ScheduleBlock * Pred : Block -> getPreds ( ) ) { std :: set < unsigned > PredOutRegs = Pred -> getOutRegs ( ) ; std :: set < unsigned > :: iterator RegPos = PredOutRegs . find ( Reg ) ;


<s> <ISA_LIT>TII = static_cast < const <ISA_LIT>InstrInfo * > ( TII ) ;

<ISA_LIT>TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( TRI ) ;


<s> BottomUpIndex2SU = std :: vector < int > ( Topo . rbegin ( ) , Topo . rend ( ) ) ;


<s> if ( TryCand . IsHighLatency && <STR_LIT> ( TryCand . Height , Cand . Height , TryCand , Cand , Depth ) ) return true ; if ( <STR_LIT> ( TryCand . NumHighLatencySuccessors , Cand . NumHighLatencySuccessors , TryCand , Cand , Successor ) ) return true ;


<s> if ( <STR_LIT> ( TryCand . VGPRUsageDiff > <NUM_LIT> , Cand . VGPRUsageDiff > <NUM_LIT> , TryCand , Cand , RegUsage ) ) return true ; if ( <STR_LIT> ( TryCand . NumSuccessors > <NUM_LIT> , Cand . NumSuccessors > <NUM_LIT> , TryCand , Cand , Successor ) ) return true ; if ( <STR_LIT> ( TryCand . Height , Cand . Height , TryCand , Cand , Depth ) ) return true ; if ( <STR_LIT> ( TryCand . VGPRUsageDiff , Cand . VGPRUsageDiff , TryCand , Cand , RegUsage ) ) return true ;


<s> if ( <STR_LIT> ( TryCand . NumSuccessors > <NUM_LIT> , Cand . NumSuccessors > <NUM_LIT> , TryCand , Cand , Successor ) ) return true ; if ( <STR_LIT> ( TryCand . Height , Cand . Height , TryCand , Cand , Depth ) ) return true ; if ( <STR_LIT> ( TryCand . VGPRUsageDiff , Cand . VGPRUsageDiff , TryCand , Cand , RegUsage ) ) return true ;


<s> TryCand . Reason = NodeOrder ; return ; } if ( Cand . SGPRUsage > <NUM_LIT> && <STR_LIT> ( TryCand . SGPRUsage , Cand . SGPRUsage , TryCand , Cand , RegUsage ) ) return ; if ( <STR_LIT> ( TryCand . HasLowLatencyNonWaitedParent , Cand . HasLowLatencyNonWaitedParent , TryCand , Cand , <STR_LIT> ) ) return ;

if ( <STR_LIT> ( TryCand . IsLowLatency , Cand . IsLowLatency , TryCand , Cand , <STR_LIT> ) ) return ;


<s> TryCand . Reason = NodeOrder ; return ; } if ( Cand . SGPRUsage > <NUM_LIT> && <STR_LIT> ( TryCand . SGPRUsage , Cand . SGPRUsage , TryCand , Cand , RegUsage ) ) return ; if ( <STR_LIT> ( TryCand . HasLowLatencyNonWaitedParent , Cand . HasLowLatencyNonWaitedParent , TryCand , Cand , <STR_LIT> ) ) return ; if ( <STR_LIT> ( TryCand . IsLowLatency , Cand . IsLowLatency , TryCand , Cand , <STR_LIT> ) ) return ; if ( TryCand . IsLowLatency && <STR_LIT> ( TryCand . LowLatencyOffset , Cand . LowLatencyOffset , TryCand , Cand , <STR_LIT> ) ) return ; if ( <STR_LIT> ( TryCand . VGPRUsage , Cand . VGPRUsage , TryCand , Cand , RegUsage ) ) return ; if ( TryCand . SU -> NodeNum < Cand . SU -> NodeNum ) {


<s> if ( TryVal < CandVal ) { TryCand . Reason = Reason ; return true ; }


<s> void <ISA_LIT>ScheduleBlock :: undoReleaseSucc ( SUnit * SU , SDep * SuccEdge ) { SUnit * SuccSU = SuccEdge -> getSUnit ( ) ; if ( SuccEdge -> isWeak ( ) ) { ++ SuccSU -> WeakPredsLeft ; return ;


<s> for ( SUnit * SU : SUnits ) { SU -> isScheduled = false ;


if ( PredDep . isWeak ( ) || Pred -> NodeNum >= DAGSize ) continue ;


<s> if ( PredID == P -> getID ( ) ) return ; } Preds . push_back ( Pred ) ; assert ( none_of ( Succs , [ = ] ( <ISA_LIT>ScheduleBlock * S ) { return PredID == S -> getID ( ) ; } ) && <STR_LIT> ) ;


<s> void <ISA_LIT>ScheduleBlock :: addPred ( <ISA_LIT>ScheduleBlock * Pred ) { unsigned PredID = Pred -> getID ( ) ; for ( <ISA_LIT>ScheduleBlock * P : Preds ) { if ( PredID == P -> getID ( ) ) return ; }


<s> if ( SuccID == S -> getID ( ) ) return ; } if ( Succ -> isHighLatencyBlock ( ) ) ++ NumHighLatencySuccessors ; Succs . push_back ( Succ ) ; assert ( none_of ( Preds , [ = ] ( <ISA_LIT>ScheduleBlock * P ) { return SuccID == P -> getID ( ) ; } ) && <STR_LIT> ) ;


<s> decreaseLiveRegs ( Block , Block -> getInRegs ( ) ) ; addLiveRegs ( Block -> getOutRegs ( ) ) ; releaseBlockSuccs ( Block ) ; for ( std :: map < unsigned , unsigned > :: iterator RegI = LiveOutRegsNumUsages [ Block -> getID ( ) ] . begin ( ) , E = LiveOutRegsNumUsages [ Block -> getID ( ) ] . end ( ) ; RegI != E ; ++ RegI ) { std :: pair < unsigned , unsigned > RegP = * RegI ; if ( LiveRegsConsumers . find ( RegP . first ) == LiveRegsConsumers . end ( ) ) LiveRegsConsumers [ RegP . first ] = RegP . second ; else { assert ( LiveRegsConsumers [ RegP . first ] == <NUM_LIT> ) ; LiveRegsConsumers [ RegP . first ] += RegP . second ;


<s> void <ISA_LIT>ScheduleBlockScheduler :: blockScheduled ( <ISA_LIT>ScheduleBlock * Block ) { decreaseLiveRegs ( Block , Block -> getInRegs ( ) ) ; addLiveRegs ( Block -> getOutRegs ( ) ) ; releaseBlockSuccs ( Block ) ; for ( std :: map < unsigned , unsigned > :: iterator RegI = LiveOutRegsNumUsages [ Block -> getID ( ) ] . begin ( ) , E = LiveOutRegsNumUsages [ Block -> getID ( ) ] . end ( ) ; RegI != E ; ++ RegI ) { std :: pair < unsigned , unsigned > RegP = * RegI ; if ( LiveRegsConsumers . find ( RegP . first ) == LiveRegsConsumers . end ( ) ) LiveRegsConsumers [ RegP . first ] = RegP . second ; else { assert ( LiveRegsConsumers [ RegP . first ] == <NUM_LIT> ) ; LiveRegsConsumers [ RegP . first ] += RegP . second ; } }

if ( LastPosHighLatencyParentScheduled [ Block -> getID ( ) ] > ( unsigned ) LastPosWaitedHighLatency ) LastPosWaitedHighLatency = LastPosHighLatencyParentScheduled [ Block -> getID ( ) ] ;


if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ;


<s> if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ; if ( ColorCount [ color ] > <NUM_LIT> ) continue ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ; } if ( SUColors . size ( ) == <NUM_LIT> && * SUColors . begin ( ) != color ) {


<s> void <ISA_LIT>ScheduleBlockCreator :: fillStats ( ) { unsigned DAGSize = CurrentBlocks . size ( ) ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { int BlockIndice = TopDownIndex2Block [ i ] ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; if ( Block -> getPreds ( ) . empty ( ) ) Block -> Depth = <NUM_LIT> ;


<s>
if
(
ReadyBlocks
.
empty
(
)
)
return
nullptr
;
DAG
->
fillVgprSgprCost
(
LiveRegs
.
begin
(
)
,
LiveRegs
.
end
(
)
,
VregCurrentUsage
,
SregCurrentUsage
)
;
if
(
VregCurrentUsage
>
maxVregUsage
)
maxVregUsage
=
VregCurrentUsage
;
if
(
VregCurrentUsage
>
maxSregUsage
)
maxSregUsage
=
VregCurrentUsage
;
DEBUG
(
dbgs
(
)
<<
<STR_LIT>
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
<ISA_LIT>ScheduleBlock
*
Block
:
ReadyBlocks
)
dbgs
(
)
<<
Block
->
getID
(
)
<<
' '
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveRegs
)
dbgs
(
)
<<
PrintVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
dbgs
(
)
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
VregCurrentUsage
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
SregCurrentUsage
<<
'\n'
;
)
;
Cand
.
Block
=
nullptr
;
for
(
std
::
vector
<
<ISA_LIT>ScheduleBlock
*
>
::
iterator
I
=
ReadyBlocks
.
begin
(
)
,
E
=
ReadyBlocks
.
end
(
)
;
I
!=
E
;
++
I
)
{
<ISA_LIT>BlockSchedCandidate
TryCand
;
TryCand
.
Block
=
*
I
;
TryCand
.
IsHighLatency
=
TryCand
.
Block
->
isHighLatencyBlock
(
)
;
TryCand
.
VGPRUsageDiff
=
checkRegUsageImpact
(
TryCand
.
Block
->
getInRegs
(
)
,
TryCand
.
Block
->
getOutRegs
(
)
)
[
DAG
->
getVGPRSetID
(
)
]
;
TryCand
.
NumSuccessors
=
TryCand
.
Block
->
getSuccs
(
)
.
size
(
)
;
TryCand
.
NumHighLatencySuccessors
=
TryCand
.
Block
->
getNumHighLatencySuccessors
(
)
;
TryCand
.
LastPosHighLatParentScheduled
=
(
unsigned
int
)
std
::
max
<
int
>
(
<NUM_LIT>
,
LastPosHighLatencyParentScheduled
[
TryCand
.
Block
->
getID
(
)
]
LastPosWaitedHighLatency
)
;

TryCand . Height = TryCand . Block -> Height ;


<s>
for
(
unsigned
Reg
:
LiveInRegs
)
dbgs
(
)
<<
PrintVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveOutRegs
)
dbgs
(
)
<<
PrintVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
}
dbgs
(
)
<<
<STR_LIT>
;
if
(
!
Scheduled
)
{
for
(
SUnit
*
SU
:
SUnits
)
{
SU
->
dump
(
DAG
)
;
}
}
else
{


<s> for ( <ISA_LIT>ScheduleBlock * Block : Parent -> getSuccs ( ) ) { -- BlockNumPredsLeft [ Block -> getID ( ) ] ; if ( BlockNumPredsLeft [ Block -> getID ( ) ] == <NUM_LIT> ) { ReadyBlocks . push_back ( Block ) ; } if ( Parent -> isHighLatencyBlock ( ) ) LastPosHighLatencyParentScheduled [ Block -> getID ( ) ] = NumBlockScheduled ;


<s> std :: vector < unsigned > & getInternalAdditionalRegUsage ( ) {


<s> std :: vector < unsigned > & getInternalAdditionalRegUsage ( ) { return InternalAdditionalPressure ;


<s>
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
<ISA_LIT>ScheduleBlock
*
Block
:
ReadyBlocks
)
dbgs
(
)
<<
Block
->
getID
(
)
<<
' '
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
dbgs
(
)
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
VregCurrentUsage
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
SregCurrentUsage
<<
'\n'
;
)
;
Cand
.
Block
=
nullptr
;
for
(
std
::
vector
<
<ISA_LIT>ScheduleBlock
*
>
::
iterator
I
=
ReadyBlocks
.
begin
(
)
,
E
=
ReadyBlocks
.
end
(
)
;
I
!=
E
;
++
I
)
{
<ISA_LIT>BlockSchedCandidate
TryCand
;
TryCand
.
Block
=
*
I
;
TryCand
.
IsHighLatency
=
TryCand
.
Block
->
isHighLatencyBlock
(
)
;
TryCand
.
VGPRUsageDiff
=
checkRegUsageImpact
(
TryCand
.
Block
->
getInRegs
(
)
,
TryCand
.
Block
->
getOutRegs
(
)
)
[
DAG
->
getVGPRSetID
(
)
]
;
TryCand
.
NumSuccessors
=
TryCand
.
Block
->
getSuccs
(
)
.
size
(
)
;
TryCand
.
NumHighLatencySuccessors
=
TryCand
.
Block
->
getNumHighLatencySuccessors
(
)
;
TryCand
.
LastPosHighLatParentScheduled
=
(
unsigned
int
)
std
::
max
<
int
>
(
<NUM_LIT>
,
LastPosHighLatencyParentScheduled
[
TryCand
.
Block
->
getID
(
)
]
LastPosWaitedHighLatency
)
;
TryCand
.
Height
=
TryCand
.
Block
->
Height
;
if
(
VregCurrentUsage
>
<NUM_LIT>
||
Variant
!=
<STR_LIT>
)
{
if
(
!
tryCandidateRegUsage
(
Cand
,
TryCand
)
&&
Variant
!=
<STR_LIT>
)
tryCandidateLatency
(
Cand
,
TryCand
)
;
}
else
{
if
(
!
tryCandidateLatency
(
Cand
,
TryCand
)
)
tryCandidateRegUsage
(
Cand
,
TryCand
)
;
}


<s>
dbgs
(
)
<<
<STR_LIT>
<<
LiveOutPressure
[
DAG
->
getSGPRSetID
(
)
]
<<
' '
<<
LiveOutPressure
[
DAG
->
getVGPRSetID
(
)
]
<<
<STR_LIT>
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveInRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveOutRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
}


<s> for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } ScheduledSUnits = Best . SUs ; ScheduledSUnitsInv . resize ( SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { ScheduledSUnitsInv [ ScheduledSUnits [ i ] ] = i ; } moveLowLatencies ( ) ; assert ( TopRPTracker . getPos ( ) == RegionBegin && <STR_LIT> ) ; TopRPTracker . setPos ( CurrentTop ) ; for ( std :: vector < unsigned > :: iterator I = ScheduledSUnits . begin ( ) , E = ScheduledSUnits . end ( ) ; I != E ; ++ I ) { SUnit * SU = & SUnits [ * I ] ; scheduleMI ( SU , true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << * SU -> getInstr ( ) ) ;


<s> for ( <ISA_LIT>ScheduleBlock * S : Succs ) { if ( PredID == S -> getID ( ) ) assert ( ! <STR_LIT> ) ;


<s> if ( SuccID == S -> getID ( ) ) return ; } if ( Succ -> isHighLatencyBlock ( ) ) ++ NumHighLatencySuccessors ; Succs . push_back ( Succ ) ; for ( <ISA_LIT>ScheduleBlock * P : Preds ) { if ( SuccID == P -> getID ( ) ) assert ( <STR_LIT> ) ; }


<s> if ( SU -> Preds . size ( ) > <NUM_LIT> && ! DAG -> IsLowLatencySU [ SU -> NodeNum ] ) continue ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ; } if ( SUColors . size ( ) == <NUM_LIT> ) CurrentColoring [ SU -> NodeNum ] = * SUColors . begin ( ) ; }


<s> for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ; } if ( SUColors . size ( ) == <NUM_LIT> ) CurrentColoring [ SU -> NodeNum ] = * SUColors . begin ( ) ; }


<s> if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ; } if ( SUColors . size ( ) == <NUM_LIT> ) CurrentColoring [ SU -> NodeNum ] = * SUColors . begin ( ) ;


<s> SUnit * SU = & DAG -> SUnits [ DAG -> BottomUpIndex2SU [ i ] ] ; std :: set < unsigned > SUColors ; if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ; }

if ( SUColors . size ( ) == <NUM_LIT> && * SUColors . begin ( ) <= DAGSize ) CurrentColoring [ SU -> NodeNum ] = * SUColors . begin ( ) ;


<s> for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { SUnit * SU = & DAG -> SUnits [ DAG -> BottomUpIndex2SU [ i ] ] ; unsigned color = CurrentColoring [ SU -> NodeNum ] ; std :: map < unsigned , unsigned > :: iterator Pos = ColorCount . find ( color ) ; if ( Pos != ColorCount . end ( ) ) { ++ ColorCount [ color ] ; } else { ColorCount [ color ] = <NUM_LIT> ; } } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { SUnit * SU = & DAG -> SUnits [ DAG -> BottomUpIndex2SU [ i ] ] ; unsigned color = CurrentColoring [ SU -> NodeNum ] ; std :: set < unsigned > SUColors ; if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ; if ( ColorCount [ color ] > <NUM_LIT> ) continue ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ; } if ( SUColors . size ( ) == <NUM_LIT> && * SUColors . begin ( ) != color ) { -- ColorCount [ color ] ; CurrentColoring [ SU -> NodeNum ] = * SUColors . begin ( ) ; ++ ColorCount [ * SUColors . begin ( ) ] ; }


<s> ScheduleDAGInstrs * llvm :: create<ISA_LIT>MachineScheduler ( MachineSchedContext * C ) {

return new <ISA_LIT>ScheduleDAGMI ( C ) ;


<s> } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { int BlockIndice = BottomUpIndex2Block [ i ] ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; if ( Block -> getSuccs ( ) . size ( ) == <NUM_LIT> ) Block -> Height = <NUM_LIT> ; else { unsigned Height = <NUM_LIT> ; for ( <ISA_LIT>ScheduleBlock * Succ : Block -> getSuccs ( ) ) {

if ( Height < Succ -> Height + <NUM_LIT> ) Height = Succ -> Height + <NUM_LIT> ;


<s> RegPressureTracker RPTracker ( Pressure ) , BotRPTracker ( BotPressure ) ; LiveIntervals * LIS = DAG -> getLIS ( ) ; MachineRegisterInfo * MRI = DAG -> getMRI ( ) ; DAG -> initRPTracker ( TopRPTracker ) ; DAG -> initRPTracker ( BotRPTracker ) ; DAG -> initRPTracker ( RPTracker ) ; for ( SUnit * SU : ScheduledSUnits ) { RPTracker . setPos ( SU -> getInstr ( ) ) ; RPTracker . advance ( ) ; } RPTracker . closeRegion ( ) ; TopRPTracker . addLiveRegs ( RPTracker . getPressure ( ) . LiveInRegs ) ; BotRPTracker . addLiveRegs ( RPTracker . getPressure ( ) . LiveOutRegs ) ; for ( unsigned Reg : RPTracker . getPressure ( ) . LiveInRegs ) { if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) ) LiveInRegs . insert ( Reg ) ; } LiveOutRegs . clear ( ) ; for ( unsigned Reg : RPTracker . getPressure ( ) . LiveOutRegs ) { if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) && isDefBetween ( Reg , LIS -> getInstructionIndex ( BeginBlock ) . getRegSlot ( ) , LIS -> getInstructionIndex ( EndBlock ) . getRegSlot ( ) , MRI , LIS ) ) { LiveOutRegs . insert ( Reg ) ;


<s> if ( MI -> isDebugValue ( ) ) continue ; SlotIndex InstSlot = LIS -> getInstructionIndex ( MI ) . getRegSlot ( ) ; if ( InstSlot >= First && InstSlot <= Last ) return true ; }


<s> IsLowLatencyUser = true ; } if ( Pred -> NodeNum >= DAGSize ) continue ; unsigned PredPos = ScheduledSUnitsInv [ Pred -> NodeNum ] ; if ( PredPos >= MinPos ) MinPos = PredPos + <NUM_LIT> ; } if ( <ISA_LIT>TII -> isLowLatencyInstruction ( SU -> getInstr ( ) ) ) { unsigned BestPos = LastLowLatencyUser + <NUM_LIT> ; if ( ( int ) BestPos <= LastLowLatencyPos ) BestPos = LastLowLatencyPos + <NUM_LIT> ; if ( BestPos < MinPos ) BestPos = MinPos ; if ( BestPos < i ) { for ( unsigned u = i ; u > BestPos ; -- u ) { ++ ScheduledSUnitsInv [ ScheduledSUnits [ u - <NUM_LIT> ] ] ; ScheduledSUnits [ u ] = ScheduledSUnits [ u - <NUM_LIT> ] ; } ScheduledSUnits [ BestPos ] = SU -> NodeNum ; ScheduledSUnitsInv [ SU -> NodeNum ] = BestPos ; } LastLowLatencyPos = BestPos ; if ( IsLowLatencyUser ) LastLowLatencyUser = BestPos ; } else if ( IsLowLatencyUser ) { LastLowLatencyUser = i ; } else if ( SU -> getInstr ( ) -> getOpcode ( ) == <STR_LIT> ) { bool CopyForLowLat = false ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( <ISA_LIT>TII -> isLowLatencyInstruction ( Succ -> getInstr ( ) ) ) { CopyForLowLat = true ; } } if ( ! CopyForLowLat ) continue ; if ( MinPos < i ) { for ( unsigned u = i ; u > MinPos ; -- u ) { ++ ScheduledSUnitsInv [ ScheduledSUnits [ u - <NUM_LIT> ] ] ; ScheduledSUnits [ u ] = ScheduledSUnits [ u - <NUM_LIT> ] ; } ScheduledSUnits [ MinPos ] = SU -> NodeNum ; ScheduledSUnitsInv [ SU -> NodeNum ] = MinPos ; } }


<s> static MachineBasicBlock :: iterator nextIfDebug ( MachineBasicBlock :: iterator I , MachineBasicBlock :: const_iterator End ) {

return MachineBasicBlock :: instr_iterator ( const_cast < MachineInstr * > ( & * nextIfDebug ( MachineBasicBlock :: const_iterator ( I ) , End ) ) ) ;


<s> if ( HasLowLatencyNonWaitedParent [ NodeNum2Index [ SU -> NodeNum ] ] ) HasLowLatencyNonWaitedParent . assign ( SUnits . size ( ) , <NUM_LIT> ) ; if ( DAG -> IsLowLatencySU [ SU -> NodeNum ] ) { for ( SDep & Succ : SU -> Succs ) {

std :: map < unsigned , unsigned > :: iterator I = NodeNum2Index . find ( Succ . getSUnit ( ) -> NodeNum ) ;


<s> llvm_unreachable ( nullptr ) ; } TopReadySUs . erase ( I ) ; releaseSuccessors ( SU , true ) ; if ( HasLowLatencyNonWaitedParent [ NodeNum2Index [ SU -> NodeNum ] ] ) HasLowLatencyNonWaitedParent . assign ( SUnits . size ( ) , <NUM_LIT> ) ; if ( DAG -> IsLowLatencySU [ SU -> NodeNum ] ) { for ( SDep & Succ : SU -> Succs ) {


<s> void <ISA_LIT>ScheduleBlockCreator :: regroupNoUserInstructions ( ) { unsigned DAGSize = DAG -> SUnits . size ( ) ; int GroupID = NextNonReservedID ++ ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { SUnit * SU = & DAG -> SUnits [ DAG -> BottomUpIndex2SU [ i ] ] ; bool hasSuccessor = false ; if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ;


<s> if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ;


<s> SUnit * SuccSU = Succ . getSUnit ( ) ;

if ( BC -> isSUInBlock ( SuccSU , ID ) != InOrOutBlock ) continue ;


<s> SUnit * SuccSU = Succ . getSUnit ( ) ; if ( BC -> isSUInBlock ( SuccSU , ID ) != InOrOutBlock ) continue ; releaseSucc ( SU , & Succ ) ; if ( SuccSU -> NumPredsLeft == <NUM_LIT> && InOrOutBlock ) TopReadySUs . push_back ( SuccSU ) ; }


<s> { LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , { LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , { LatenciesAlonePlusConsecutive , BlockRegUsage } } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } ScheduledSUnits = Best . SUs ; ScheduledSUnitsInv . resize ( SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { ScheduledSUnitsInv [ ScheduledSUnits [ i ] ] = i ; } moveLowLatencies ( ) ; assert ( TopRPTracker . getPos ( ) == RegionBegin && <STR_LIT> ) ; TopRPTracker . setPos ( CurrentTop ) ; for ( std :: vector < unsigned > :: iterator I = ScheduledSUnits . begin ( ) , E = ScheduledSUnits . end ( ) ; I != E ; ++ I ) { SUnit * SU = & SUnits [ * I ] ; scheduleMI ( SU , true ) ; DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << * SU -> getInstr ( ) ) ; } assert ( CurrentTop == CurrentBottom && <STR_LIT> ) ; placeDebugValues ( ) ; DEBUG ( { unsigned BBNum = begin ( ) -> getParent ( ) -> getNumber ( ) ; dbgs ( ) << <STR_LIT> << BBNum << <STR_LIT> ; dumpSchedule ( ) ;


<s> if ( & * CurrentTopFastSched == MI ) { PosNew . push_back ( Pos ) ; CurrentTopFastSched = nextIfDebug ( ++ CurrentTopFastSched , DAG -> getCurrentBottom ( ) ) ; } else { DAG -> getBB ( ) -> splice ( CurrentTopFastSched , DAG -> getBB ( ) , MI ) ; DAG -> getLIS ( ) -> handleMove ( MI , true ) ; PosNew . push_back ( CurrentTopFastSched ) ; } } } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ i ] ; std :: vector < SUnit * > SUs = Block -> getScheduledUnits ( ) ; Block -> schedule ( ( * SUs . begin ( ) ) -> getInstr ( ) , ( * SUs . rbegin ( ) ) -> getInstr ( ) ) ; } DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( unsigned i = PosOld . size ( ) , e = <NUM_LIT> ; i != e ; -- i ) { MachineBasicBlock :: iterator POld = PosOld [ i - <NUM_LIT> ] ; MachineBasicBlock :: iterator PNew = PosNew [ i - <NUM_LIT> ] ; if ( PNew != POld ) { DAG -> getBB ( ) -> splice ( POld , DAG -> getBB ( ) , PNew ) ; DAG -> getLIS ( ) -> handleMove ( POld , true ) ; } } DEBUG ( for ( unsigned i = <NUM_LIT> , e = CurrentBlocks . size ( ) ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ i ] ;


<s> <ISA_LIT>TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( TRI ) ; VGPRSetID = <ISA_LIT>TRI -> getVGPR32PressureSet ( ) ; SGPRSetID = <ISA_LIT>TRI -> getSGPR32PressureSet ( ) ;


<s> <ISA_LIT>TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( TRI ) ;

VGPRSetID = <ISA_LIT>TRI -> getVGPR32PressureSet ( ) ;


<s> if ( SU -> NodeNum < DAGSize && ! -- TopDownSU2Index [ SU -> NodeNum ] ) WorkList . push_back ( SU ) ; } } BottomUpIndex2SU = std :: vector < int > ( TopDownIndex2SU . rbegin ( ) , TopDownIndex2SU . rend ( ) ) ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { SUnit * SU = & SUnits [ i ] ; for ( SDep & Pred : SU -> Preds ) { if ( Pred . getSUnit ( ) -> NodeNum >= DAGSize ) continue ; assert ( TopDownSU2Index [ SU -> NodeNum ] > TopDownSU2Index [ Pred . getSUnit ( ) -> NodeNum ] && <STR_LIT> ) ; } } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { SUnit * SU = & SUnits [ i ] ; for ( SDep & Succ : SU -> Succs ) { if ( Succ . getSUnit ( ) -> NodeNum >= DAGSize ) continue ;

assert ( TopDownSU2Index [ SU -> NodeNum ] < TopDownSU2Index [ Succ . getSUnit ( ) -> NodeNum ] && <STR_LIT> ) ;


<s> <ISA_LIT>ScheduleBlockCreator :: ~ <ISA_LIT>ScheduleBlockCreator ( ) {


<s> <ISA_LIT>ScheduleDAGMI :: ~ <ISA_LIT>ScheduleDAGMI ( ) {


if ( Node2CurrentBlock [ Pred -> NodeNum ] != SUID ) CurrentBlocks [ SUID ] -> addPred ( CurrentBlocks [ Node2CurrentBlock [ Pred -> NodeNum ] ] ) ;


<s> unsigned getSGPRSetID ( ) const { return SGPRSetID ;


<s> const std :: vector < <ISA_LIT>ScheduleBlock * > & getSuccs ( ) const { return Succs ;


<s> bool OffsetIsScalable ; if ( <ISA_LIT>TII -> getMemOperandWithOffset ( * SU -> getInstr ( ) , BaseLatOp , OffLatReg , OffsetIsScalable , TRI ) ) LowLatencyOffset [ i ] = OffLatReg ; } else if ( <ISA_LIT>TII -> isHighLatencyDef ( SU -> getInstr ( ) -> getOpcode ( ) ) ) IsHighLatencySU [ i ] = <NUM_LIT> ; } <ISA_LIT>Scheduler Scheduler ( this ) ; Best = Scheduler . scheduleVariant ( <STR_LIT> , <STR_LIT> ) ; if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsageLatency } , { LatenciesGrouped , BlockLatencyRegUsage } , { LatenciesAlonePlusConsecutive , BlockLatencyRegUsage } , } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , { LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , { LatenciesAlonePlusConsecutive , BlockRegUsage } } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } ScheduledSUnits = Best . SUs ; ScheduledSUnitsInv . resize ( SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { ScheduledSUnitsInv [ ScheduledSUnits [ i ] ] = i ; } moveLowLatencies ( ) ; assert ( TopRPTracker . getPos ( ) == RegionBegin && <STR_LIT> ) ; TopRPTracker . setPos ( CurrentTop ) ; for ( std :: vector < unsigned > :: iterator I = ScheduledSUnits . begin ( ) , E = ScheduledSUnits . end ( ) ; I != E ; ++ I ) { SUnit * SU = & SUnits [ * I ] ; scheduleMI ( SU , true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << * SU -> getInstr ( ) ) ; }

assert ( CurrentTop == CurrentBottom && <STR_LIT> ) ;


<s> { LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , { LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , { LatenciesAlonePlusConsecutive , BlockRegUsage } } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } ScheduledSUnits = Best . SUs ; ScheduledSUnitsInv . resize ( SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { ScheduledSUnitsInv [ ScheduledSUnits [ i ] ] = i ; } moveLowLatencies ( ) ; assert ( TopRPTracker . getPos ( ) == RegionBegin && <STR_LIT> ) ; TopRPTracker . setPos ( CurrentTop ) ; for ( std :: vector < unsigned > :: iterator I = ScheduledSUnits . begin ( ) , E = ScheduledSUnits . end ( ) ; I != E ; ++ I ) { SUnit * SU = & SUnits [ * I ] ; scheduleMI ( SU , true ) ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << * SU -> getInstr ( ) ) ;


<s> for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ i ] ; Block -> fastSchedule ( ) ; } MachineBasicBlock :: iterator CurrentTopFastSched = DAG -> getCurrentTop ( ) ; std :: vector < MachineBasicBlock :: iterator > PosOld ; std :: vector < MachineBasicBlock :: iterator > PosNew ; PosOld . reserve ( DAG -> SUnits . size ( ) ) ; PosNew . reserve ( DAG -> SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { int BlockIndice = TopDownIndex2Block [ i ] ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; std :: vector < SUnit * > SUs = Block -> getScheduledUnits ( ) ; for ( SUnit * SU : SUs ) { MachineInstr * MI = SU -> getInstr ( ) ; MachineBasicBlock :: iterator Pos = MI ; PosOld . push_back ( Pos ) ; if ( & * CurrentTopFastSched == MI ) { PosNew . push_back ( Pos ) ; CurrentTopFastSched = nextIfDebug ( ++ CurrentTopFastSched , DAG -> getCurrentBottom ( ) ) ; } else { DAG -> getBB ( ) -> splice ( CurrentTopFastSched , DAG -> getBB ( ) , MI ) ; DAG -> getLIS ( ) -> handleMove ( * MI , true ) ; PosNew . push_back ( CurrentTopFastSched ) ; } } } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ i ] ; std :: vector < SUnit * > SUs = Block -> getScheduledUnits ( ) ; Block -> schedule ( ( * SUs . begin ( ) ) -> getInstr ( ) , ( * SUs . rbegin ( ) ) -> getInstr ( ) ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( unsigned i = PosOld . size ( ) , e = <NUM_LIT> ; i != e ; -- i ) {

MachineBasicBlock :: iterator POld = PosOld [ i - <NUM_LIT> ] ;


<s> <ISA_LIT>BlockSchedCandidate ( ) : Block ( nullptr ) {


<s> <ISA_LIT>SchedCandidate ( ) : SU ( nullptr ) {


<s> <ISA_LIT>ScheduleBlock ( <ISA_LIT>ScheduleDAGMI * DAG , <ISA_LIT>ScheduleBlockCreator * BC , unsigned ID ) :

DAG ( DAG ) , BC ( BC ) , SUnits ( ) , TopReadySUs ( ) , ScheduledSUnits ( ) , TopRPTracker ( TopPressure ) , Scheduled ( false ) , HighLatencyBlock ( false ) , ID ( ID ) , Preds ( ) , Succs ( ) , NumHighLatencySuccessors ( <NUM_LIT> ) { }


<s> DAG ( DAG ) , Variant ( Variant ) , Blocks ( BlocksStruct . Blocks ) , LastPosWaitedHighLatency ( <NUM_LIT> ) , NumBlockScheduled ( <NUM_LIT> ) , VregCurrentUsage ( <NUM_LIT> ) , SregCurrentUsage ( <NUM_LIT> ) , maxVregUsage ( <NUM_LIT> ) , maxSregUsage ( <NUM_LIT> ) { LiveOutRegsNumUsages . resize ( Blocks . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = Blocks [ i ] ; for ( unsigned Reg : Block -> getInRegs ( ) ) { bool Found = false ; int topoInd = - <NUM_LIT> ; for ( <ISA_LIT>ScheduleBlock * Pred : Block -> getPreds ( ) ) { std :: set < unsigned > PredOutRegs = Pred -> getOutRegs ( ) ; std :: set < unsigned > :: iterator RegPos = PredOutRegs . find ( Reg ) ; if ( RegPos != PredOutRegs . end ( ) ) { Found = true ; if ( topoInd < BlocksStruct . TopDownBlock2Index [ Pred -> getID ( ) ] ) { topoInd = BlocksStruct . TopDownBlock2Index [ Pred -> getID ( ) ] ; } } } if ( ! Found ) continue ; int PredID = BlocksStruct . TopDownIndex2Block [ topoInd ] ; ++ LiveOutRegsNumUsages [ PredID ] [ Reg ] ; } } LastPosHighLatencyParentScheduled . resize ( Blocks . size ( ) , <NUM_LIT> ) ; BlockNumPredsLeft . resize ( Blocks . size ( ) ) ; BlockNumSuccsLeft . resize ( Blocks . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = Blocks [ i ] ; BlockNumPredsLeft [ i ] = Block -> getPreds ( ) . size ( ) ; BlockNumSuccsLeft [ i ] = Block -> getSuccs ( ) . size ( ) ; } for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = Blocks [ i ] ; assert ( Block -> getID ( ) == i ) ; } std :: set < unsigned > InRegs = DAG -> getInRegs ( ) ; addLiveRegs ( InRegs ) ; for ( unsigned Reg : DAG -> getOutRegs ( ) ) { for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { int ID = BlocksStruct . TopDownIndex2Block [ Blocks . size ( ) - <NUM_LIT> - i ] ; <ISA_LIT>ScheduleBlock * Block = Blocks [ ID ] ; const std :: set < unsigned > & OutRegs = Block -> getOutRegs ( ) ; if ( OutRegs . find ( Reg ) == OutRegs . end ( ) ) continue ; ++ LiveOutRegsNumUsages [ ID ] [ Reg ] ; break ; }


<s> <ISA_LIT>SchedulerCandidate ( ) : Reason ( NoCand ) , RepeatReasonSet ( <NUM_LIT> ) {


<s> if ( ! Register :: isVirtualRegister ( Reg ) ) continue ; ( void ) LiveRegs . insert ( Reg ) ; }


<s> for ( unsigned Reg : OutRegs ) { if ( ! Register :: isVirtualRegister ( Reg ) ) continue ; PSetIterator PSetI = DAG -> getMRI ( ) -> getPressureSets ( Reg ) ; for ( ; PSetI . isValid ( ) ; ++ PSetI ) { DiffSetPressure [ * PSetI ] += PSetI . getWeight ( ) ; }


<s> void <ISA_LIT>ScheduleDAGMI :: fillVgprSgprCost ( _Iterator First , _Iterator End , unsigned & VgprUsage , unsigned & SgprUsage ) { VgprUsage = <NUM_LIT> ; SgprUsage = <NUM_LIT> ; for ( _Iterator RegI = First ; RegI != End ; ++ RegI ) { unsigned Reg = * RegI ; if ( ! Register :: isVirtualRegister ( Reg ) ) continue ; PSetIterator PSetI = MRI . getPressureSets ( Reg ) ; for ( ; PSetI . isValid ( ) ; ++ PSetI ) { if ( * PSetI == <STR_LIT> :: VGPR_32 ) VgprUsage += PSetI . getWeight ( ) ; else if ( * PSetI == <STR_LIT> :: SReg_32 ) SgprUsage += PSetI . getWeight ( ) ;


<s> void <ISA_LIT>ScheduleDAGMI :: fillVgprSgprCost ( _Iterator First , _Iterator End , unsigned & VgprUsage , unsigned & SgprUsage ) { VgprUsage = <NUM_LIT> ; SgprUsage = <NUM_LIT> ; for ( _Iterator RegI = First ; RegI != End ; ++ RegI ) { unsigned Reg = * RegI ; if ( ! Register :: isVirtualRegister ( Reg ) ) continue ; PSetIterator PSetI = MRI . getPressureSets ( Reg ) ; for ( ; PSetI . isValid ( ) ; ++ PSetI ) { if ( * PSetI == <STR_LIT> :: VGPR_32 ) VgprUsage += PSetI . getWeight ( ) ;

else if ( * PSetI == <STR_LIT> :: SReg_32 ) SgprUsage += PSetI . getWeight ( ) ;


<s> RegPressureTracker RPTracker ( Pressure ) , BotRPTracker ( BotPressure ) ; LiveIntervals * LIS = DAG -> getLIS ( ) ; MachineRegisterInfo * MRI = DAG -> getMRI ( ) ; DAG -> initRPTracker ( TopRPTracker ) ; DAG -> initRPTracker ( BotRPTracker ) ; DAG -> initRPTracker ( RPTracker ) ; for ( SUnit * SU : ScheduledSUnits ) { RPTracker . setPos ( SU -> getInstr ( ) ) ; RPTracker . advance ( ) ; } RPTracker . closeRegion ( ) ; TopRPTracker . addLiveRegs ( RPTracker . getPressure ( ) . LiveInRegs ) ; BotRPTracker . addLiveRegs ( RPTracker . getPressure ( ) . LiveOutRegs ) ; for ( const auto & RegMaskPair : RPTracker . getPressure ( ) . LiveInRegs ) { if ( Register :: isVirtualRegister ( RegMaskPair . RegUnit ) ) LiveInRegs . insert ( RegMaskPair . RegUnit ) ; } LiveOutRegs . clear ( ) ; for ( const auto & RegMaskPair : RPTracker . getPressure ( ) . LiveOutRegs ) { unsigned Reg = RegMaskPair . RegUnit ; if ( Register :: isVirtualRegister ( Reg ) && isDefBetween ( Reg , LIS -> getInstructionIndex ( * BeginBlock ) . getRegSlot ( ) , LIS -> getInstructionIndex ( * EndBlock ) . getRegSlot ( ) , MRI , LIS ) ) { LiveOutRegs . insert ( Reg ) ; }


<s> RPTracker . advance ( ) ; } RPTracker . closeRegion ( ) ; TopRPTracker . addLiveRegs ( RPTracker . getPressure ( ) . LiveInRegs ) ; BotRPTracker . addLiveRegs ( RPTracker . getPressure ( ) . LiveOutRegs ) ; for ( const auto & RegMaskPair : RPTracker . getPressure ( ) . LiveInRegs ) { if ( TargetRegisterInfo :: isVirtualRegister ( RegMaskPair . RegUnit ) ) LiveInRegs . insert ( RegMaskPair . RegUnit ) ; } LiveOutRegs . clear ( ) ; for ( const auto & RegMaskPair : RPTracker . getPressure ( ) . LiveOutRegs ) { unsigned Reg = RegMaskPair . RegUnit ;

if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) && isDefBetween ( Reg , LIS -> getInstructionIndex ( BeginBlock ) . getRegSlot ( ) , LIS -> getInstructionIndex ( EndBlock ) . getRegSlot ( ) , MRI , LIS ) ) {


<s> void <ISA_LIT>ScheduleBlock :: initRegPressure ( MachineBasicBlock :: iterator BeginBlock , MachineBasicBlock :: iterator EndBlock ) { IntervalPressure Pressure , BotPressure ; RegPressureTracker RPTracker ( Pressure ) , BotRPTracker ( BotPressure ) ; LiveIntervals * LIS = DAG -> getLIS ( ) ; MachineRegisterInfo * MRI = DAG -> getMRI ( ) ; DAG -> initRPTracker ( TopRPTracker ) ; DAG -> initRPTracker ( BotRPTracker ) ; DAG -> initRPTracker ( RPTracker ) ; for ( SUnit * SU : ScheduledSUnits ) { RPTracker . setPos ( SU -> getInstr ( ) ) ; RPTracker . advance ( ) ; } RPTracker . closeRegion ( ) ;

TopRPTracker . addLiveRegs ( RPTracker . getPressure ( ) . LiveInRegs ) ;


<s> for ( std :: vector < <ISA_LIT>ScheduleBlock * > :: iterator I = ReadyBlocks . begin ( ) , E = ReadyBlocks . end ( ) ; I != E ; ++ I ) { <ISA_LIT>BlockSchedCandidate TryCand ; TryCand . Block = * I ; TryCand . IsHighLatency = TryCand . Block -> isHighLatencyBlock ( ) ; TryCand . VGPRUsageDiff = checkRegUsageImpact ( TryCand . Block -> getInRegs ( ) , TryCand . Block -> getOutRegs ( ) ) [ DAG -> getVGPRSetID ( ) ] ; TryCand . NumSuccessors = TryCand . Block -> getSuccs ( ) . size ( ) ; TryCand . NumHighLatencySuccessors = TryCand . Block -> getNumHighLatencySuccessors ( ) ; TryCand . LastPosHighLatParentScheduled = ( unsigned int ) std :: max < int > ( <NUM_LIT> , LastPosHighLatencyParentScheduled [ TryCand . Block -> getID ( ) ] - LastPosWaitedHighLatency ) ; TryCand . Height = TryCand . Block -> Height ; if ( VregCurrentUsage > <NUM_LIT> || Variant != <STR_LIT> ) { if ( ! tryCandidateRegUsage ( Cand , TryCand ) && Variant != <STR_LIT> ) tryCandidateLatency ( Cand , TryCand ) ; } else { if ( ! tryCandidateLatency ( Cand , TryCand ) ) tryCandidateRegUsage ( Cand , TryCand ) ; } if ( TryCand . Reason != NoCand ) {


<s>
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
dbgs
(
)
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
VregCurrentUsage
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
SregCurrentUsage
<<
'\n'
;
)
;
Cand
.
Block
=
nullptr
;
for
(
std
::
vector
<
<ISA_LIT>ScheduleBlock
*
>
::
iterator
I
=
ReadyBlocks
.
begin
(
)
,
E
=
ReadyBlocks
.
end
(
)
;
I
!=
E
;
++
I
)
{
<ISA_LIT>BlockSchedCandidate
TryCand
;
TryCand
.
Block
=
*
I
;
TryCand
.
IsHighLatency
=
TryCand
.
Block
->
isHighLatencyBlock
(
)
;
TryCand
.
VGPRUsageDiff
=
checkRegUsageImpact
(
TryCand
.
Block
->
getInRegs
(
)
,
TryCand
.
Block
->
getOutRegs
(
)
)
[
DAG
->
getVGPRSetID
(
)
]
;
TryCand
.
NumSuccessors
=
TryCand
.
Block
->
getSuccs
(
)
.
size
(
)
;
TryCand
.
NumHighLatencySuccessors
=
TryCand
.
Block
->
getNumHighLatencySuccessors
(
)
;
TryCand
.
LastPosHighLatParentScheduled
=
(
unsigned
int
)
std
::
max
<
int
>
(
<NUM_LIT>
,
LastPosHighLatencyParentScheduled
[
TryCand
.
Block
->
getID
(
)
]
LastPosWaitedHighLatency
)
;
TryCand
.
Height
=
TryCand
.
Block
->
Height
;
if
(
VregCurrentUsage
>
<NUM_LIT>
||
Variant
!=
<STR_LIT>
)
{
if
(
!
tryCandidateRegUsage
(
Cand
,
TryCand
)
&&
Variant
!=
<STR_LIT>
)
tryCandidateLatency
(
Cand
,
TryCand
)
;
}
else
{
if
(
!
tryCandidateLatency
(
Cand
,
TryCand
)
)
tryCandidateRegUsage
(
Cand
,
TryCand
)
;
}
if
(
TryCand
.
Reason
!=
NoCand
)
{
Cand
.
setBest
(
TryCand
)
;
Best
=
I
;
DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
Cand
.
Block
->
getID
(
)
<<
' '
<<
getReasonStr
(
Cand
.
Reason
)
<<
'\n'
)
;
}
}

DEBUG ( dbgs ( ) << <STR_LIT> << Cand . Block -> getID ( ) << '\n' ;


<s> static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsageLatency } , { LatenciesGrouped , BlockLatencyRegUsage } , { LatenciesAlonePlusConsecutive , BlockLatencyRegUsage } , } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , { LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , { LatenciesAlonePlusConsecutive , BlockRegUsage } } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } ScheduledSUnits = Best . SUs ; ScheduledSUnitsInv . resize ( SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { ScheduledSUnitsInv [ ScheduledSUnits [ i ] ] = i ; } moveLowLatencies ( ) ; assert ( TopRPTracker . getPos ( ) == RegionBegin && <STR_LIT> ) ; TopRPTracker . setPos ( CurrentTop ) ; for ( std :: vector < unsigned > :: iterator I = ScheduledSUnits . begin ( ) , E = ScheduledSUnits . end ( ) ; I != E ; ++ I ) { SUnit * SU = & SUnits [ * I ] ; scheduleMI ( SU , true ) ; DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << * SU -> getInstr ( ) ) ; } assert ( CurrentTop == CurrentBottom && <STR_LIT> ) ;


<s> buildDAGWithRegPressure ( ) ; DEBUG ( for ( SUnit & SU : SUnits ) SU . dumpAll ( this ) ) ; topologicalSort ( ) ; findRootsAndBiasEdges ( TopRoots , BotRoots ) ; SchedImpl -> initialize ( this ) ; initQueues ( TopRoots , BotRoots ) ; SUnitsLinksBackup = SUnits ; IsLowLatencySU . clear ( ) ; LowLatencyOffset . clear ( ) ; IsHighLatencySU . clear ( ) ; IsLowLatencySU . resize ( SUnits . size ( ) , <NUM_LIT> ) ; LowLatencyOffset . resize ( SUnits . size ( ) , <NUM_LIT> ) ; IsHighLatencySU . resize ( SUnits . size ( ) , <NUM_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { SUnit * SU = & SUnits [ i ] ; unsigned BaseLatReg ; int64_t OffLatReg ; if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * SU -> getInstr ( ) ) ) { IsLowLatencySU [ i ] = <NUM_LIT> ; if ( <ISA_LIT>TII -> getMemOpBaseRegImmOfs ( * SU -> getInstr ( ) , BaseLatReg , OffLatReg , TRI ) ) LowLatencyOffset [ i ] = OffLatReg ; } else if ( <ISA_LIT>TII -> isHighLatencyInstruction ( * SU -> getInstr ( ) ) ) IsHighLatencySU [ i ] = <NUM_LIT> ; } <ISA_LIT>Scheduler Scheduler ( this ) ; Best = Scheduler . scheduleVariant ( <STR_LIT> , <STR_LIT> ) ; if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsageLatency } , { LatenciesGrouped , BlockLatencyRegUsage } , { LatenciesAlonePlusConsecutive , BlockLatencyRegUsage } , } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = {

{ LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , { LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , {


<s> } ScheduledSUnits = Best . SUs ; ScheduledSUnitsInv . resize ( SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { ScheduledSUnitsInv [ ScheduledSUnits [ i ] ] = i ; } moveLowLatencies ( ) ; assert ( TopRPTracker . getPos ( ) == RegionBegin && <STR_LIT> ) ; TopRPTracker . setPos ( CurrentTop ) ; for ( std :: vector < unsigned > :: iterator I = ScheduledSUnits . begin ( ) , E = ScheduledSUnits . end ( ) ; I != E ; ++ I ) { SUnit * SU = & SUnits [ * I ] ; scheduleMI ( SU , true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << * SU -> getInstr ( ) ) ; } assert ( CurrentTop == CurrentBottom && <STR_LIT> ) ; placeDebugValues ( ) ; LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> << printMBBReference ( * begin ( ) -> getParent ( ) ) << <STR_LIT> ; dumpSchedule ( ) ;


<s>
dbgs
(
)
<<
<STR_LIT>
;
for
(
std
::
pair
<
<ISA_LIT>ScheduleBlock
*
,
<ISA_LIT>ScheduleBlockLinkKind
>
S
:
Succs
)
{
if
(
S
.
second
==
<STR_LIT>
)
dbgs
(
)
<<
<STR_LIT>
;
S
.
first
->
printDebug
(
false
)
;
}
if
(
Scheduled
)
{
dbgs
(
)
<<
<STR_LIT>
<<
LiveInPressure
[
DAG
->
getSGPRSetID
(
)
]
<<
' '
<<
LiveInPressure
[
DAG
->
getVGPRSetID
(
)
]
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
LiveOutPressure
[
DAG
->
getSGPRSetID
(
)
]
<<
' '
<<
LiveOutPressure
[
DAG
->
getVGPRSetID
(
)
]
<<
<STR_LIT>
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveInRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveOutRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
}
dbgs
(
)
<<
<STR_LIT>
;
if
(
!
Scheduled
)
{
for
(
SUnit
*
SU
:
SUnits
)
{
SU
->
dump
(
DAG
)
;
}
}
else
{
for
(
SUnit
*
SU
:
SUnits
)
{
SU
->
dump
(
DAG
)
;
}
}


<s> { LatenciesAlone , BlockRegUsageLatency } , { LatenciesGrouped , BlockLatencyRegUsage } , { LatenciesAlonePlusConsecutive , BlockLatencyRegUsage } , } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , { LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , { LatenciesAlonePlusConsecutive , BlockRegUsage } } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } ScheduledSUnits = Best . SUs ; ScheduledSUnitsInv . resize ( SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { ScheduledSUnitsInv [ ScheduledSUnits [ i ] ] = i ; } moveLowLatencies ( ) ; assert ( TopRPTracker . getPos ( ) == RegionBegin && <STR_LIT> ) ; TopRPTracker . setPos ( CurrentTop ) ; for ( std :: vector < unsigned > :: iterator I = ScheduledSUnits . begin ( ) , E = ScheduledSUnits . end ( ) ; I != E ; ++ I ) { SUnit * SU = & SUnits [ * I ] ; scheduleMI ( SU , true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << * SU -> getInstr ( ) ) ; } assert ( CurrentTop == CurrentBottom && <STR_LIT> ) ; placeDebugValues ( ) ; LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> << printMBBReference ( * begin ( ) -> getParent ( ) ) << <STR_LIT> ; dumpSchedule ( ) ; dbgs ( ) << '\n' ;


<s> if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , { LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , { LatenciesAlonePlusConsecutive , BlockRegUsage } } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } ScheduledSUnits = Best . SUs ; ScheduledSUnitsInv . resize ( SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { ScheduledSUnitsInv [ ScheduledSUnits [ i ] ] = i ; } moveLowLatencies ( ) ; assert ( TopRPTracker . getPos ( ) == RegionBegin && <STR_LIT> ) ; TopRPTracker . setPos ( CurrentTop ) ; for ( std :: vector < unsigned > :: iterator I = ScheduledSUnits . begin ( ) , E = ScheduledSUnits . end ( ) ; I != E ; ++ I ) { SUnit * SU = & SUnits [ * I ] ; scheduleMI ( SU , true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << * SU -> getInstr ( ) ) ;


<s> if ( I == TopReadySUs . end ( ) ) { dbgs ( ) << <STR_LIT> ; llvm_unreachable ( nullptr ) ; } TopReadySUs . erase ( I ) ; releaseSuccessors ( SU , true ) ; if ( HasLowLatencyNonWaitedParent [ NodeNum2Index [ SU -> NodeNum ] ] ) HasLowLatencyNonWaitedParent . assign ( SUnits . size ( ) , <NUM_LIT> ) ; if ( DAG -> IsLowLatencySU [ SU -> NodeNum ] ) {


<s> buildDAGWithRegPressure ( ) ; DEBUG ( for ( SUnit & SU : SUnits ) SU . dumpAll ( this ) ) ; topologicalSort ( ) ; findRootsAndBiasEdges ( TopRoots , BotRoots ) ; SchedImpl -> initialize ( this ) ; initQueues ( TopRoots , BotRoots ) ; SUnitsLinksBackup = SUnits ; IsLowLatencySU . clear ( ) ; LowLatencyOffset . clear ( ) ; IsHighLatencySU . clear ( ) ; IsLowLatencySU . resize ( SUnits . size ( ) , <NUM_LIT> ) ; LowLatencyOffset . resize ( SUnits . size ( ) , <NUM_LIT> ) ; IsHighLatencySU . resize ( SUnits . size ( ) , <NUM_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { SUnit * SU = & SUnits [ i ] ; unsigned BaseLatReg ; int64_t OffLatReg ; if ( <ISA_LIT>TII -> isLowLatencyInstruction ( SU -> getInstr ( ) ) ) { IsLowLatencySU [ i ] = <NUM_LIT> ; if ( <ISA_LIT>TII -> getMemOpBaseRegImmOfs ( SU -> getInstr ( ) , BaseLatReg , OffLatReg , TRI ) ) LowLatencyOffset [ i ] = OffLatReg ; } else if ( <ISA_LIT>TII -> isHighLatencyInstruction ( SU -> getInstr ( ) ) ) IsHighLatencySU [ i ] = <NUM_LIT> ; } <ISA_LIT>Scheduler Scheduler ( this ) ; Best = Scheduler . scheduleVariant ( <STR_LIT> , <STR_LIT> ) ; if ( Best . MaxVGPRUsage > <NUM_LIT> ) { std :: vector < std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant >> Variants = { { LatenciesAlone , BlockRegUsageLatency } , { LatenciesGrouped , BlockLatencyRegUsage } , { LatenciesAlonePlusConsecutive , BlockLatencyRegUsage } , } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } if ( Best . MaxVGPRUsage > <NUM_LIT> ) { std :: vector < std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant >> Variants = { { LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , { LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , { LatenciesAlonePlusConsecutive , BlockRegUsage } } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } }


<s> SgprUsage = <NUM_LIT> ; for ( _Iterator RegI = First ; RegI != End ; ++ RegI ) { unsigned Reg = * RegI ; if ( ! Register :: isVirtualRegister ( Reg ) ) continue ; PSetIterator PSetI = MRI . getPressureSets ( Reg ) ;

for ( ; PSetI . isValid ( ) ; ++ PSetI ) {


<s> <ISA_LIT>TII = static_cast < const <ISA_LIT>InstrInfo * > ( TII ) ;

<ISA_LIT>TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( TRI ) ;


<s>
void
<ISA_LIT>ScheduleBlock
::
printDebug
(
bool
full
)
{
dbgs
(
)
<<
<STR_LIT>
<<
ID
<<
<STR_LIT>
;
if
(
!
full
)
return
;
dbgs
(
)
<<
<STR_LIT>
<<
HighLatencyBlock
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
<ISA_LIT>ScheduleBlock
*
P
:
Preds
)
{
P
->
printDebug
(
false
)
;
}
dbgs
(
)
<<
<STR_LIT>
;
for
(
std
::
pair
<
<ISA_LIT>ScheduleBlock
*
,
<ISA_LIT>ScheduleBlockLinkKind
>
S
:
Succs
)
{
if
(
S
.
second
==
<STR_LIT>
)
dbgs
(
)
<<
<STR_LIT>
;
S
.
first
->
printDebug
(
false
)
;
}
if
(
Scheduled
)
{
dbgs
(
)
<<
<STR_LIT>
<<
LiveInPressure
[
DAG
->
getSGPRSetID
(
)
]
<<
' '
<<
LiveInPressure
[
DAG
->
getVGPRSetID
(
)
]
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
LiveOutPressure
[
DAG
->
getSGPRSetID
(
)
]
<<
' '
<<
LiveOutPressure
[
DAG
->
getVGPRSetID
(
)
]
<<
<STR_LIT>
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveInRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveOutRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;


<s> DAG ( DAG ) , BC ( BC ) , TopRPTracker ( TopPressure ) , Scheduled ( false ) , HighLatencyBlock ( false ) , ID ( ID ) , NumHighLatencySuccessors ( <NUM_LIT> ) {


<s> <ISA_LIT>ScheduleBlock ( <ISA_LIT>ScheduleDAGMI * DAG , <ISA_LIT>ScheduleBlockCreator * BC , unsigned ID ) :

DAG ( DAG ) , BC ( BC ) , TopRPTracker ( TopPressure ) , Scheduled ( false ) , HighLatencyBlock ( false ) , ID ( ID ) , NumHighLatencySuccessors ( <NUM_LIT> ) {


<s> std :: set < unsigned > InRegs ( RPTracker . getPressure ( ) . LiveInRegs . begin ( ) , RPTracker . getPressure ( ) . LiveInRegs . end ( ) ) ;


<s> if ( ! TargetRegisterInfo :: isVirtualRegister ( Reg ) ) continue ;


<s> for ( std :: map < unsigned , unsigned > :: iterator RegI = LiveOutRegsNumUsages [ Block -> getID ( ) ] . begin ( ) , E = LiveOutRegsNumUsages [ Block -> getID ( ) ] . end ( ) ; RegI != E ; ++ RegI ) { std :: pair < unsigned , unsigned > RegP = * RegI ; assert ( LiveRegsConsumers . find ( RegP . first ) == LiveRegsConsumers . end ( ) || LiveRegsConsumers [ RegP . first ] == <NUM_LIT> ) ; LiveRegsConsumers [ RegP . first ] += RegP . second ;


<s> if ( ! TargetRegisterInfo :: isVirtualRegister ( Reg ) ) continue ; if ( LiveRegsConsumers [ Reg ] > <NUM_LIT> ) continue ; PSetIterator PSetI = DAG -> getMRI ( ) -> getPressureSets ( Reg ) ; for ( ; PSetI . isValid ( ) ; ++ PSetI ) { DiffSetPressure [ * PSetI ] -= PSetI . getWeight ( ) ;


<s> DiffSetPressure . assign ( DAG -> getTRI ( ) -> getNumRegPressureSets ( ) , <NUM_LIT> ) ; for ( unsigned Reg : InRegs ) { if ( ! TargetRegisterInfo :: isVirtualRegister ( Reg ) ) continue ; if ( LiveRegsConsumers [ Reg ] > <NUM_LIT> ) continue ;

PSetIterator PSetI = DAG -> getMRI ( ) -> getPressureSets ( Reg ) ;


CurrentColoring [ SU -> NodeNum ] = Pos -> second ;


<s> } else { for ( unsigned k : SubGraph ) { if ( DAG -> IsHighLatencySU [ k ] || ( CurrentColoring [ k ] != ProposedColor && CurrentColoring [ k ] != <NUM_LIT> ) ) { CompatibleGroup = false ; break ; } if ( hasDataDependencyPred ( DAG -> SUnits [ k ] , DAG -> SUnits [ j ] ) ) { CompatibleGroup = false ; break ; } } if ( ! CompatibleGroup ) break ; if ( hasDataDependencyPred ( SU , DAG -> SUnits [ j ] ) ) { CompatibleGroup = false ; break ; } AdditionalElements . insert ( AdditionalElements . end ( ) , SubGraph . begin ( ) , SubGraph . end ( ) ) ; } } if ( CompatibleGroup ) { FormingGroup . insert ( SU . NodeNum ) ; for ( unsigned j : AdditionalElements ) CurrentColoring [ j ] = ProposedColor ; CurrentColoring [ SU . NodeNum ] = ProposedColor ; ++ Count ; } if ( ! CompatibleGroup ) { FormingGroup . clear ( ) ; Color = ++ NextReservedID ; ProposedColor = Color ; FormingGroup . insert ( SU . NodeNum ) ; CurrentColoring [ SU . NodeNum ] = ProposedColor ; Count = <NUM_LIT> ; } else if ( Count == GroupSize ) { FormingGroup . clear ( ) ; Color = ++ NextReservedID ;


<s> for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { SUnit * SU = & DAG -> SUnits [ i ] ; if ( DAG -> IsHighLatencySU [ SU -> NodeNum ] ) ++ NumHighLatencies ; } if ( NumHighLatencies == <NUM_LIT> ) return ; if ( NumHighLatencies <= <NUM_LIT> ) GroupSize = <NUM_LIT> ; else if ( NumHighLatencies <= <NUM_LIT> ) GroupSize = <NUM_LIT> ; else GroupSize = <NUM_LIT> ; for ( unsigned SUNum : DAG -> TopDownIndex2SU ) { const SUnit & SU = DAG -> SUnits [ SUNum ] ; if ( DAG -> IsHighLatencySU [ SU . NodeNum ] ) { unsigned CompatibleGroup = true ; int ProposedColor = Color ; std :: vector < int > AdditionalElements ; for ( unsigned j : FormingGroup ) { bool HasSubGraph ; std :: vector < int > SubGraph ; SubGraph = DAG -> GetTopo ( ) -> GetSubGraph ( SU , DAG -> SUnits [ j ] , HasSubGraph ) ; assert ( ! HasSubGraph ) ; SubGraph = DAG -> GetTopo ( ) -> GetSubGraph ( DAG -> SUnits [ j ] , SU , HasSubGraph ) ; if ( ! HasSubGraph ) continue ; else if ( SubGraph . size ( ) > <NUM_LIT> ) { CompatibleGroup = false ; break ; } else { for ( unsigned k : SubGraph ) { if ( DAG -> IsHighLatencySU [ k ] || ( CurrentColoring [ k ] != ProposedColor && CurrentColoring [ k ] != <NUM_LIT> ) ) { CompatibleGroup = false ; break ; } if ( hasDataDependencyPred ( DAG -> SUnits [ k ] , DAG -> SUnits [ j ] ) ) { CompatibleGroup = false ; break ; } } if ( ! CompatibleGroup ) break ; if ( hasDataDependencyPred ( SU , DAG -> SUnits [ j ] ) ) { CompatibleGroup = false ; break ; } AdditionalElements . insert ( AdditionalElements . end ( ) , SubGraph . begin ( ) , SubGraph . end ( ) ) ; } } if ( CompatibleGroup ) { FormingGroup . insert ( SU . NodeNum ) ; for ( unsigned j : AdditionalElements ) CurrentColoring [ j ] = ProposedColor ;

CurrentColoring [ SU . NodeNum ] = ProposedColor ;


<s> for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { int BlockIndice = TopDownIndex2Block [ i ] ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; if ( Block -> getPreds ( ) . empty ( ) ) Block -> Depth = <NUM_LIT> ; else { unsigned Depth = <NUM_LIT> ; for ( <ISA_LIT>ScheduleBlock * Pred : Block -> getPreds ( ) ) { if ( Depth < Pred -> Depth + <NUM_LIT> ) Depth = Pred -> Depth + <NUM_LIT> ; } Block -> Depth = Depth ; } } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { int BlockIndice = BottomUpIndex2Block [ i ] ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; if ( Block -> getSuccs ( ) . empty ( ) ) Block -> Height = <NUM_LIT> ; else { unsigned Height = <NUM_LIT> ; for ( const auto & Succ : Block -> getSuccs ( ) ) Height = std :: min ( Height , Succ . first -> Height + <NUM_LIT> ) ;


<s> void <ISA_LIT>ScheduleBlockCreator :: fillStats ( ) { unsigned DAGSize = CurrentBlocks . size ( ) ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { int BlockIndice = TopDownIndex2Block [ i ] ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; if ( Block -> getPreds ( ) . empty ( ) ) Block -> Depth = <NUM_LIT> ; else { unsigned Depth = <NUM_LIT> ; for ( <ISA_LIT>ScheduleBlock * Pred : Block -> getPreds ( ) ) { if ( Depth < Pred -> Depth + <NUM_LIT> ) Depth = Pred -> Depth + <NUM_LIT> ; } Block -> Depth = Depth ; } } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) {

int BlockIndice = BottomUpIndex2Block [ i ] ;


<s> if ( ! TargetRegisterInfo :: isVirtualRegister ( Reg ) ) continue ; PSetIterator PSetI = MRI . getPressureSets ( Reg ) ; for ( ; PSetI . isValid ( ) ; ++ PSetI ) {

if ( * PSetI == VGPRSetID ) VgprUsage += PSetI . getWeight ( ) ;


<s> std :: vector < <ISA_LIT>ScheduleBlock * > getBlocks ( ) { return BlocksScheduled ;


<s> MachineBasicBlock :: iterator getCurrentTop ( ) { return CurrentTop ;


<s> MachineBasicBlock :: iterator getCurrentTop ( ) {


<s> SUnit & getEntrySU ( ) { return EntrySU ;


<s> unsigned getID ( ) const { return ID ;


for ( const auto & RegMaskPair : RPTracker . getPressure ( ) . LiveInRegs ) {


<s> for ( const auto & RegMaskPair : RPTracker . getPressure ( ) . LiveInRegs ) { InRegs . insert ( RegMaskPair . RegUnit ) ;


<s> std :: vector < unsigned > & getInternalAdditionnalRegUsage ( ) {


<s> std :: vector < SUnit * > getScheduledUnits ( ) {


<s> ArrayRef < std :: pair < <ISA_LIT>ScheduleBlock * , <ISA_LIT>ScheduleBlockLinkKind >> getSuccs ( ) const {


<s> ArrayRef < std :: pair < <ISA_LIT>ScheduleBlock * , <ISA_LIT>ScheduleBlockLinkKind >> getSuccs ( ) const { return Succs ;


<s> ScheduleDAGTopologicalSort * GetTopo ( ) { return & Topo ;


<s> const TargetRegisterInfo * getTRI ( ) { return TRI ;


<s> for ( SUnit * SU : ScheduledSUnits ) { RPTracker . setPos ( SU -> getInstr ( ) ) ; RPTracker . advance ( ) ; } RPTracker . closeRegion ( ) ; TopRPTracker . addLiveRegs ( RPTracker . getPressure ( ) . LiveInRegs ) ; BotRPTracker . addLiveRegs ( RPTracker . getPressure ( ) . LiveOutRegs ) ; for ( const auto & RegMaskPair : RPTracker . getPressure ( ) . LiveInRegs ) { if ( TargetRegisterInfo :: isVirtualRegister ( RegMaskPair . RegUnit ) ) LiveInRegs . insert ( RegMaskPair . RegUnit ) ; } LiveOutRegs . clear ( ) ; for ( const auto & RegMaskPair : RPTracker . getPressure ( ) . LiveOutRegs ) { unsigned Reg = RegMaskPair . RegUnit ; if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) && isDefBetween ( Reg , LIS -> getInstructionIndex ( * BeginBlock ) . getRegSlot ( ) , LIS -> getInstructionIndex ( * EndBlock ) . getRegSlot ( ) , MRI , LIS ) ) { LiveOutRegs . insert ( Reg ) ; } } LiveInPressure = TopPressure . MaxSetPressure ;

LiveOutPressure = BotPressure . MaxSetPressure ;


<s> RPTracker . init ( & MF , RegClassInfo , LIS , BB , RegionBegin , false , false ) ;


<s> void initRPTracker ( RegPressureTracker & RPTracker ) { RPTracker . init ( & MF , RegClassInfo , LIS , BB , RegionBegin , false , false ) ;


<s> bool isRepeat ( <ISA_LIT>ScheduleCandReason R ) { return RepeatReasonSet & ( <NUM_LIT> << R ) ;


<s> SUnit * Pred = PredDep . getSUnit ( ) ; if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * Pred -> getInstr ( ) ) ) { IsLowLatencyUser = true ; } if ( Pred -> NodeNum >= DAGSize ) continue ; unsigned PredPos = ScheduledSUnitsInv [ Pred -> NodeNum ] ; if ( PredPos >= MinPos ) MinPos = PredPos + <NUM_LIT> ; } if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * SU -> getInstr ( ) ) ) { unsigned BestPos = LastLowLatencyUser + <NUM_LIT> ; if ( ( int ) BestPos <= LastLowLatencyPos ) BestPos = LastLowLatencyPos + <NUM_LIT> ; if ( BestPos < MinPos ) BestPos = MinPos ; if ( BestPos < i ) { for ( unsigned u = i ; u > BestPos ; -- u ) { ++ ScheduledSUnitsInv [ ScheduledSUnits [ u - <NUM_LIT> ] ] ; ScheduledSUnits [ u ] = ScheduledSUnits [ u - <NUM_LIT> ] ; } ScheduledSUnits [ BestPos ] = SU -> NodeNum ; ScheduledSUnitsInv [ SU -> NodeNum ] = BestPos ; } LastLowLatencyPos = BestPos ; if ( IsLowLatencyUser ) LastLowLatencyUser = BestPos ; } else if ( IsLowLatencyUser ) { LastLowLatencyUser = i ; } else if ( SU -> getInstr ( ) -> getOpcode ( ) == <STR_LIT> ) { bool CopyForLowLat = false ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ;

if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * Succ -> getInstr ( ) ) ) {


<s> for ( unsigned u = i ; u > BestPos ; -- u ) { ++ ScheduledSUnitsInv [ ScheduledSUnits [ u - <NUM_LIT> ] ] ; ScheduledSUnits [ u ] = ScheduledSUnits [ u - <NUM_LIT> ] ; } ScheduledSUnits [ BestPos ] = SU -> NodeNum ; ScheduledSUnitsInv [ SU -> NodeNum ] = BestPos ; } LastLowLatencyPos = BestPos ; if ( IsLowLatencyUser ) LastLowLatencyUser = BestPos ; } else if ( IsLowLatencyUser ) { LastLowLatencyUser = i ; } else if ( SU -> getInstr ( ) -> getOpcode ( ) == <STR_LIT> ) { bool CopyForLowLat = false ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * Succ -> getInstr ( ) ) ) { CopyForLowLat = true ; } } if ( ! CopyForLowLat ) continue ; if ( MinPos < i ) {

for ( unsigned u = i ; u > MinPos ; -- u ) {


<s> static MachineBasicBlock :: iterator nextIfDebug ( MachineBasicBlock :: iterator I , MachineBasicBlock :: const_iterator End ) { for ( ; I != End ; ++ I ) { if ( ! I -> isDebugValue ( ) ) break ; } return I ;


<s> TryCand . SGPRUsage = pressure [ DAG -> getSGPRSetID ( ) ] ; TryCand . VGPRUsage = pressure [ DAG -> getVGPRSetID ( ) ] ; TryCand . IsLowLatency = DAG -> IsLowLatencySU [ SU -> NodeNum ] ; TryCand . LowLatencyOffset = DAG -> LowLatencyOffset [ SU -> NodeNum ] ; TryCand . HasLowLatencyNonWaitedParent = HasLowLatencyNonWaitedParent [ NodeNum2Index [ SU -> NodeNum ] ] ; tryCandidateTopDown ( TopCand , TryCand ) ; if ( TryCand . Reason != NoCand ) TopCand . setBest ( TryCand ) ; } return TopCand . SU ;


<s> TryCand . SGPRUsage = pressure [ DAG -> getSGPRSetID ( ) ] ; TryCand . VGPRUsage = pressure [ DAG -> getVGPRSetID ( ) ] ; TryCand . IsLowLatency = DAG -> IsLowLatencySU [ SU -> NodeNum ] ; TryCand . LowLatencyOffset = DAG -> LowLatencyOffset [ SU -> NodeNum ] ; TryCand . HasLowLatencyNonWaitedParent = HasLowLatencyNonWaitedParent [ NodeNum2Index [ SU -> NodeNum ] ] ; tryCandidateTopDown ( TopCand , TryCand ) ; if ( TryCand . Reason != NoCand ) TopCand . setBest ( TryCand ) ;


<s> -- SuccSU -> WeakPredsLeft ; return ; } if ( SuccSU -> NumPredsLeft == <NUM_LIT> ) { dbgs ( ) << <STR_LIT> ; SuccSU -> dump ( DAG ) ; dbgs ( ) << <STR_LIT> ; llvm_unreachable ( nullptr ) ;


<s> buildDAGWithRegPressure ( ) ; DEBUG ( for ( SUnit & SU : SUnits ) SU . dumpAll ( this ) ) ; topologicalSort ( ) ; findRootsAndBiasEdges ( TopRoots , BotRoots ) ; SchedImpl -> initialize ( this ) ; initQueues ( TopRoots , BotRoots ) ; SUnitsLinksBackup = SUnits ; IsLowLatencySU . clear ( ) ; LowLatencyOffset . clear ( ) ; IsHighLatencySU . clear ( ) ; IsLowLatencySU . resize ( SUnits . size ( ) , <NUM_LIT> ) ; LowLatencyOffset . resize ( SUnits . size ( ) , <NUM_LIT> ) ; IsHighLatencySU . resize ( SUnits . size ( ) , <NUM_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { SUnit * SU = & SUnits [ i ] ; unsigned BaseLatReg ; int64_t OffLatReg ; if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * SU -> getInstr ( ) ) ) { IsLowLatencySU [ i ] = <NUM_LIT> ; if ( <ISA_LIT>TII -> getMemOpBaseRegImmOfs ( * SU -> getInstr ( ) , BaseLatReg , OffLatReg , TRI ) ) LowLatencyOffset [ i ] = OffLatReg ; } else if ( <ISA_LIT>TII -> isHighLatencyInstruction ( * SU -> getInstr ( ) ) ) IsHighLatencySU [ i ] = <NUM_LIT> ; } <ISA_LIT>Scheduler Scheduler ( this ) ; Best = Scheduler . scheduleVariant ( <STR_LIT> , <STR_LIT> ) ; if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsageLatency } , { LatenciesGrouped , BlockLatencyRegUsage } , { LatenciesAlonePlusConsecutive , BlockLatencyRegUsage } , } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } }


<s> <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; std :: vector < SUnit * > SUs = Block -> getScheduledUnits ( ) ; for ( SUnit * SU : SUs ) { MachineInstr * MI = SU -> getInstr ( ) ; MachineBasicBlock :: iterator Pos = MI ; PosOld . push_back ( Pos ) ; if ( & * CurrentTopFastSched == MI ) { PosNew . push_back ( Pos ) ; CurrentTopFastSched = nextIfDebug ( ++ CurrentTopFastSched , DAG -> getCurrentBottom ( ) ) ; } else { DAG -> getBB ( ) -> splice ( CurrentTopFastSched , DAG -> getBB ( ) , MI ) ; DAG -> getLIS ( ) -> handleMove ( * MI , true ) ; PosNew . push_back ( CurrentTopFastSched ) ; } } } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ i ] ; std :: vector < SUnit * > SUs = Block -> getScheduledUnits ( ) ; Block -> schedule ( ( * SUs . begin ( ) ) -> getInstr ( ) , ( * SUs . rbegin ( ) ) -> getInstr ( ) ) ; } DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( unsigned i = PosOld . size ( ) , e = <NUM_LIT> ; i != e ; -- i ) { MachineBasicBlock :: iterator POld = PosOld [ i - <NUM_LIT> ] ; MachineBasicBlock :: iterator PNew = PosNew [ i - <NUM_LIT> ] ; if ( PNew != POld ) { DAG -> getBB ( ) -> splice ( POld , DAG -> getBB ( ) , PNew ) ; DAG -> getLIS ( ) -> handleMove ( * POld , true ) ; } } DEBUG ( for ( unsigned i = <NUM_LIT> , e = CurrentBlocks . size ( ) ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ i ] ; Block -> printDebug ( true ) ;


<s> <ISA_LIT>ScheduleBlock * Block = ScheduledBlocks [ b ] ; std :: vector < SUnit * > SUs = Block -> getScheduledUnits ( ) ; for ( SUnit * SU : SUs ) Res . SUs . push_back ( SU -> NodeNum ) ; } Res . MaxSGPRUsage = Scheduler . getSGPRUsage ( ) ; Res . MaxVGPRUsage = Scheduler . getVGPRUsage ( ) ;


<s> <ISA_LIT>ScheduleBlockScheduler Scheduler ( DAG , ScheduleVariant , Blocks ) ; std :: vector < <ISA_LIT>ScheduleBlock * > ScheduledBlocks ; struct <ISA_LIT>ScheduleBlockResult Res ; ScheduledBlocks = Scheduler . getBlocks ( ) ;

for ( unsigned b = <NUM_LIT> ; b < ScheduledBlocks . size ( ) ; ++ b ) {


<s> DAG ( DAG ) , BC ( BC ) , TopRPTracker ( TopPressure ) , ID ( ID ) {


<s> <ISA_LIT>ScheduleBlock ( <ISA_LIT>ScheduleDAGMI * DAG , <ISA_LIT>ScheduleBlockCreator * BC , unsigned ID ) : DAG ( DAG ) , BC ( BC ) , TopRPTracker ( TopPressure ) , ID ( ID ) {


<s> <ISA_LIT>ScheduleBlockCreator :: <ISA_LIT>ScheduleBlockCreator ( <ISA_LIT>ScheduleDAGMI * DAG ) : DAG ( DAG ) {


<s> } for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = Blocks [ i ] ; assert ( Block -> getID ( ) == i ) ; } std :: set < unsigned > InRegs = DAG -> getInRegs ( ) ; addLiveRegs ( InRegs ) ; for ( unsigned Reg : DAG -> getOutRegs ( ) ) { for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { int ID = BlocksStruct . TopDownIndex2Block [ Blocks . size ( ) - <NUM_LIT> - i ] ; <ISA_LIT>ScheduleBlock * Block = Blocks [ ID ] ; const std :: set < unsigned > & OutRegs = Block -> getOutRegs ( ) ; if ( OutRegs . find ( Reg ) == OutRegs . end ( ) ) continue ; ++ LiveOutRegsNumUsages [ ID ] [ Reg ] ; break ; } } for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = Blocks [ i ] ; for ( unsigned Reg : Block -> getInRegs ( ) ) { bool Found = false ; for ( <ISA_LIT>ScheduleBlock * Pred : Block -> getPreds ( ) ) { std :: set < unsigned > PredOutRegs = Pred -> getOutRegs ( ) ; std :: set < unsigned > :: iterator RegPos = PredOutRegs . find ( Reg ) ; if ( RegPos != PredOutRegs . end ( ) ) { Found = true ; break ; } } if ( ! Found ) ++ LiveRegsConsumers [ Reg ] ; } } for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = Blocks [ i ] ; if ( BlockNumPredsLeft [ i ] == <NUM_LIT> ) { ReadyBlocks . push_back ( Block ) ;


<s> ScheduleDAGMILive ( C , llvm :: make_unique < GenericScheduler > ( C ) ) { <ISA_LIT>TII = static_cast < const <ISA_LIT>InstrInfo * > ( TII ) ; <ISA_LIT>TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( TRI ) ; VGPRSetID = <ISA_LIT>TRI -> getVGPRPressureSet ( ) ; SGPRSetID = <ISA_LIT>TRI -> getSGPRPressureSet ( ) ;


<s> <ISA_LIT>TII = static_cast < const <ISA_LIT>InstrInfo * > ( TII ) ; <ISA_LIT>TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( TRI ) ; VGPRSetID = <ISA_LIT>TRI -> getVGPRPressureSet ( ) ; SGPRSetID = <ISA_LIT>TRI -> getSGPRPressureSet ( ) ;


<s> <ISA_LIT>Scheduler ( <ISA_LIT>ScheduleDAGMI * DAG ) : DAG ( DAG ) , BlockCreator ( DAG ) {


<s> int i = WorkList . back ( ) ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ i ] ; WorkList . pop_back ( ) ; TopDownBlock2Index [ i ] = -- Id ; TopDownIndex2Block [ Id ] = i ; for ( <ISA_LIT>ScheduleBlock * Pred : Block -> getPreds ( ) ) { if ( ! -- TopDownBlock2Index [ Pred -> getID ( ) ] ) WorkList . push_back ( Pred -> getID ( ) ) ; } } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ i ] ; for ( <ISA_LIT>ScheduleBlock * Pred : Block -> getPreds ( ) ) {

assert ( TopDownBlock2Index [ i ] > TopDownBlock2Index [ Pred -> getID ( ) ] && <STR_LIT> ) ;


<s> if ( TryCand . IsHighLatency && tryGreater ( TryCand . Height , Cand . Height , TryCand , Cand , Depth ) ) return true ; if ( tryGreater ( TryCand . NumHighLatencySuccessors , Cand . NumHighLatencySuccessors , TryCand , Cand , Successor ) ) return true ;


<s> if ( TryCand . IsHighLatency && tryGreater ( TryCand . Height , Cand . Height , TryCand , Cand , Depth ) ) return true ;

if ( tryGreater ( TryCand . NumHighLatencySuccessors , Cand . NumHighLatencySuccessors , TryCand , Cand , Successor ) ) return true ;


<s> if ( ! Cand . isValid ( ) ) { TryCand . Reason = NodeOrder ; return true ; } if ( tryLess ( TryCand . VGPRUsageDiff > <NUM_LIT> , Cand . VGPRUsageDiff > <NUM_LIT> , TryCand , Cand , RegUsage ) ) return true ; if ( tryGreater ( TryCand . NumSuccessors > <NUM_LIT> , Cand . NumSuccessors > <NUM_LIT> , TryCand , Cand , Successor ) ) return true ; if ( tryGreater ( TryCand . Height , Cand . Height , TryCand , Cand , Depth ) ) return true ; if ( tryLess ( TryCand . VGPRUsageDiff , Cand . VGPRUsageDiff , TryCand , Cand , RegUsage ) ) return true ; return false ;


<s> void <ISA_LIT>ScheduleBlock :: tryCandidateTopDown ( <ISA_LIT>SchedCandidate & Cand , <ISA_LIT>SchedCandidate & TryCand ) { if ( ! Cand . isValid ( ) ) { TryCand . Reason = NodeOrder ; return ; } if ( Cand . SGPRUsage > <NUM_LIT> && tryLess ( TryCand . SGPRUsage , Cand . SGPRUsage , TryCand , Cand , RegUsage ) ) return ; if ( tryLess ( TryCand . HasLowLatencyNonWaitedParent , Cand . HasLowLatencyNonWaitedParent , TryCand , Cand , <STR_LIT> ) ) return ; if ( tryGreater ( TryCand . IsLowLatency , Cand . IsLowLatency , TryCand , Cand , <STR_LIT> ) ) return ; if ( TryCand . IsLowLatency && tryLess ( TryCand . LowLatencyOffset , Cand . LowLatencyOffset , TryCand , Cand , <STR_LIT> ) ) return ; if ( tryLess ( TryCand . VGPRUsage , Cand . VGPRUsage , TryCand , Cand , RegUsage ) ) return ;

if ( TryCand . SU -> NodeNum < Cand . SU -> NodeNum ) {


<s> } if ( bytes > <NUM_LIT> ) return ; for ( unsigned i = <NUM_LIT> , e = Desc . getNumOperands ( ) ; i < e ; ++ i ) { if ( ! <STR_LIT> ( Desc , i ) ) continue ; const MCOperand & Op = MI . getOperand ( i ) ; if ( getLitEncoding ( Op , Desc . OpInfo [ i ] , STI ) != <NUM_LIT> ) continue ; int64_t Imm = <NUM_LIT> ; if ( Op . isImm ( ) ) Imm = Op . getImm ( ) ; else if ( Op . isExpr ( ) ) { if ( const auto * C = dyn_cast < MCConstantExpr > ( Op . getExpr ( ) ) ) Imm = C -> getValue ( ) ; } else if ( ! Op . isExpr ( ) ) llvm_unreachable ( <STR_LIT> ) ;

for ( unsigned j = <NUM_LIT> ; j < <NUM_LIT> ; j ++ ) {


<s> int64_t Imm = <NUM_LIT> ; if ( Op . isImm ( ) ) Imm = Op . getImm ( ) ; else if ( Op . isExpr ( ) ) { if ( const auto * C = dyn_cast < MCConstantExpr > ( Op . getExpr ( ) ) ) Imm = C -> getValue ( ) ; } else if ( ! Op . isExpr ( ) ) llvm_unreachable ( <STR_LIT> ) ; for ( unsigned j = <NUM_LIT> ; j < <NUM_LIT> ; j ++ ) { OS . write ( ( uint8_t ) ( ( Imm >> ( <NUM_LIT> * j ) ) & <NUM_LIT> ) ) ; }


<s> if ( Val == <NUM_LIT> ) return <NUM_LIT> ; if ( Val == <NUM_LIT> ) return <NUM_LIT> ; if ( Val == <NUM_LIT> ) return <NUM_LIT> ; if ( Val == <NUM_LIT> ) return <NUM_LIT> ; if ( Val == <NUM_LIT> ) return <NUM_LIT> ; if ( Val == <NUM_LIT> ) return <NUM_LIT> ; if ( Val == <NUM_LIT> && STI . getFeatureBits ( ) [ <STR_LIT> ] ) return <NUM_LIT> ;


<s> if ( Val == FloatToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == FloatToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == FloatToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == FloatToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ;

if ( Val == <NUM_LIT> && STI . getFeatureBits ( ) [ <STR_LIT> ] ) return <NUM_LIT> ;


<s> static uint32_t getLit64Encoding ( uint64_t Val , const MCSubtargetInfo & STI ) { uint32_t IntImm = getIntInlineImmEncoding ( static_cast < int64_t > ( Val ) ) ; if ( IntImm != <NUM_LIT> ) return IntImm ; if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ;


<s> if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ;


<s> const auto * C = dyn_cast < MCConstantExpr > ( MO . getExpr ( ) ) ; if ( ! C ) return <NUM_LIT> ; Imm = C -> getValue ( ) ; } else { assert ( ! MO . isFPImm ( ) ) ; if ( ! MO . isImm ( ) ) return ~ <NUM_LIT> ; Imm = MO . getImm ( ) ; } switch ( OpInfo . OperandType ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit64Encoding ( static_cast < uint64_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : {


<s> Imm = MO . getImm ( ) ; } switch ( OpInfo . OperandType ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit64Encoding ( static_cast < uint64_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : {


<s> const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; if ( <STR_LIT> ( Desc , OpNo ) ) { uint32_t Enc = getLitEncoding ( MO , Desc . OpInfo [ OpNo ] , STI ) ; if ( Enc != ~ <NUM_LIT> && ( Enc != <NUM_LIT> || Desc . getSize ( ) == <NUM_LIT> ) ) return Enc ;

} else if ( MO . isImm ( ) ) return MO . getImm ( ) ;


<s> RegEnc |= SDWA9EncValues :: SRC_SGPR_MASK ; } return RegEnc ; } else { const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; uint32_t Enc = getLitEncoding ( MO , Desc . OpInfo [ OpNo ] , STI ) ; if ( Enc != ~ <NUM_LIT> && Enc != <NUM_LIT> ) { return Enc | SDWA9EncValues :: SRC_SGPR_MASK ; } }


<s> RegEnc |= SDWA9EncValues :: SRC_SGPR_MASK ; } return RegEnc ; } else { const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; uint32_t Enc = getLitEncoding ( MO , Desc . OpInfo [ OpNo ] , STI ) ; if ( Enc != ~ <NUM_LIT> && Enc != <NUM_LIT> ) { return Enc | SDWA9EncValues :: SRC_SGPR_MASK ;


<s> unsigned <ISA_LIT>MCCodeEmitter :: getSDWAVopcDstEncoding ( const MCInst & MI , unsigned OpNo , SmallVectorImpl < MCFixup > & Fixups , const MCSubtargetInfo & STI ) const { using namespace <STR_LIT> ; uint64_t RegEnc = <NUM_LIT> ; const MCOperand & MO = MI . getOperand ( OpNo ) ; unsigned Reg = MO . getReg ( ) ; if ( Reg != <STR_LIT> ) { RegEnc |= MRI . getEncodingValue ( Reg ) ;

RegEnc &= SDWA9EncValues :: VOPC_DST_SGPR_MASK ;


<s> auto * BE = cast < MCBinaryExpr > ( Expr ) ; if ( BE -> getOpcode ( ) == MCBinaryExpr :: Sub ) return false ; return needsPCRel ( BE -> getLHS ( ) ) || needsPCRel ( BE -> getRHS ( ) ) ; } case MCExpr :: Unary : return needsPCRel ( cast < MCUnaryExpr > ( Expr ) -> getSubExpr ( ) ) ;


<s> <ISA_LIT>MCCodeEmitter ( const MCInstrInfo & mcii , const MCRegisterInfo & mri , MCContext & ctx ) : <ISA_LIT>MCCodeEmitter ( mcii ) , MRI ( mri ) {


<s> MCCodeEmitter * llvm :: create<ISA_LIT>MCCodeEmitter ( const MCInstrInfo & MCII , const MCRegisterInfo & MRI , MCContext & Ctx ) { return new <ISA_LIT>MCCodeEmitter ( MCII , MRI , Ctx ) ;


<s> } if ( bytes > <NUM_LIT> ) return ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i < e ; ++ i ) { if ( ! isSrcOperand ( Desc , i ) ) continue ; int RCID = Desc . OpInfo [ i ] . RegClass ; const MCRegisterClass & RC = MRI . getRegClass ( RCID ) ; const MCOperand & Op = MI . getOperand ( i ) ; if ( getLitEncoding ( Op , RC . getSize ( ) ) != <NUM_LIT> ) continue ; int64_t Imm = <NUM_LIT> ; if ( Op . isImm ( ) ) Imm = Op . getImm ( ) ;

else if ( ! Op . isExpr ( ) ) llvm_unreachable ( <STR_LIT> ) ;


<s> if ( Imm >= - <NUM_LIT> && Imm <= - <NUM_LIT> ) return <NUM_LIT> + std :: abs ( Imm ) ;


<s> if ( Imm >= <NUM_LIT> && Imm <= <NUM_LIT> ) return <NUM_LIT> + Imm ; if ( Imm >= - <NUM_LIT> && Imm <= - <NUM_LIT> ) return <NUM_LIT> + std :: abs ( Imm ) ;


<s> if ( Val == FloatToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == FloatToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == FloatToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == FloatToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == FloatToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == FloatToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ;


<s> if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( <NUM_LIT> ) ) return <NUM_LIT> ; if ( Val == DoubleToBits ( - <NUM_LIT> ) ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> assert ( ! MO . isFPImm ( ) ) ; if ( ! MO . isImm ( ) ) return ~ <NUM_LIT> ; if ( OpSize == <NUM_LIT> ) return getLit32Encoding ( static_cast < uint32_t > ( MO . getImm ( ) ) ) ; assert ( OpSize == <NUM_LIT> ) ;

return getLit64Encoding ( static_cast < uint64_t > ( MO . getImm ( ) ) ) ;


<s> unsigned <ISA_LIT>MCCodeEmitter :: getSOPPBrEncoding ( const MCInst & MI , unsigned OpNo , SmallVectorImpl < MCFixup > & Fixups , const MCSubtargetInfo & STI ) const { const MCOperand & MO = MI . getOperand ( OpNo ) ;


<s> unsigned OpType = Desc . OpInfo [ OpNo ] . OperandType ; return OpType == <STR_LIT> || OpType == <STR_LIT> ;


<s> <ISA_LIT>MCCodeEmitter ( const MCInstrInfo & mcii , const MCRegisterInfo & mri , MCContext & ctx ) : MCII ( mcii ) , MRI ( mri ) {


<s> ~ <ISA_LIT>MCCodeEmitter ( ) override {


<s> for ( unsigned i = <NUM_LIT> ; i < NumPadding ; ++ i ) OS . write ( <NUM_LIT> ) ; } if ( ( bytes > <NUM_LIT> && STI . getFeatureBits ( ) [ <STR_LIT> ] ) || ( bytes > <NUM_LIT> && ! STI . getFeatureBits ( ) [ <STR_LIT> ] ) ) return ; int ImmLitIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: imm ) ; if ( ImmLitIdx != - <NUM_LIT> ) return ; for ( unsigned i = <NUM_LIT> , e = Desc . getNumOperands ( ) ; i < e ; ++ i ) { if ( ! <STR_LIT> ( Desc , i ) ) continue ; const MCOperand & Op = MI . getOperand ( i ) ; if ( getLitEncoding ( Op , Desc . OpInfo [ i ] , STI ) != <NUM_LIT> ) continue ; int64_t Imm = <NUM_LIT> ; if ( Op . isImm ( ) ) Imm = Op . getImm ( ) ; else if ( Op . isExpr ( ) ) { if ( const auto * C = dyn_cast < MCConstantExpr > ( Op . getExpr ( ) ) ) Imm = C -> getValue ( ) ; } else if ( ! Op . isExpr ( ) ) llvm_unreachable ( <STR_LIT> ) ; for ( unsigned j = <NUM_LIT> ; j < <NUM_LIT> ; j ++ ) { OS . write ( ( uint8_t ) ( ( Imm >> ( <NUM_LIT> * j ) ) & <NUM_LIT> ) ) ; } break ; }


<s> if ( MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) ) Enc |= <NUM_LIT> ;


<s> if ( MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) ) Enc |= <NUM_LIT> ; return Enc ;


<s> if ( <STR_LIT> ( Opcode , op_sel_hi ) != - <NUM_LIT> ) { if ( <STR_LIT> ( Opcode , src2 ) != - <NUM_LIT> ) return <NUM_LIT> ; if ( <STR_LIT> ( Opcode , src1 ) != - <NUM_LIT> ) return OP_SEL_HI_2 ; if ( <STR_LIT> ( Opcode , src0 ) != - <NUM_LIT> ) return OP_SEL_HI_1 | OP_SEL_HI_2 ; }


<s> uint16_t IntImm = getIntInlineImmEncoding ( static_cast < int16_t > ( Val ) ) ; return IntImm == <NUM_LIT> ? <NUM_LIT> : IntImm ;


<s> for ( unsigned e = MI . getNumOperands ( ) ; OpNo < e ; ++ OpNo ) { if ( & MO == & MI . getOperand ( OpNo ) ) break ; } const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; if ( <STR_LIT> ( Desc , OpNo ) ) { uint32_t Enc = getLitEncoding ( MO , Desc . OpInfo [ OpNo ] , STI ) ;


<s> using namespace <STR_LIT> ; uint64_t RegEnc = <NUM_LIT> ; const MCOperand & MO = MI . getOperand ( OpNo ) ; unsigned Reg = MO . getReg ( ) ; if ( Reg != <STR_LIT> && Reg != <STR_LIT> ) { RegEnc |= MRI . getEncodingValue ( Reg ) ; RegEnc &= SDWA9EncValues :: VOPC_DST_SGPR_MASK ; RegEnc |= SDWA9EncValues :: VOPC_DST_VCC_MASK ; }


<s> unsigned <ISA_LIT>MCCodeEmitter :: getSDWAVopcDstEncoding ( const MCInst & MI , unsigned OpNo , SmallVectorImpl < MCFixup > & Fixups , const MCSubtargetInfo & STI ) const { using namespace <STR_LIT> ; uint64_t RegEnc = <NUM_LIT> ; const MCOperand & MO = MI . getOperand ( OpNo ) ; unsigned Reg = MO . getReg ( ) ;

if ( Reg != <STR_LIT> && Reg != <STR_LIT> ) {


<s> auto Offset = MI . getOperand ( OpNo ) . getImm ( ) ;

assert ( ! <STR_LIT> ( STI ) || isUInt < <NUM_LIT> > ( Offset ) ) ;


<s> unsigned <ISA_LIT>MCCodeEmitter :: getSMEMOffsetEncoding ( const MCInst & MI , unsigned OpNo , SmallVectorImpl < MCFixup > & Fixups , const MCSubtargetInfo & STI ) const { auto Offset = MI . getOperand ( OpNo ) . getImm ( ) ; assert ( ! <STR_LIT> ( STI ) || isUInt < <NUM_LIT> > ( Offset ) ) ;


<s> auto * BE = cast < MCBinaryExpr > ( Expr ) ; if ( BE -> getOpcode ( ) == MCBinaryExpr :: Sub ) return false ; return needsPCRel ( BE -> getLHS ( ) ) || needsPCRel ( BE -> getRHS ( ) ) ; } case MCExpr :: Unary : return needsPCRel ( cast < MCUnaryExpr > ( Expr ) -> getSubExpr ( ) ) ; case MCExpr :: Target : case MCExpr :: Constant : return false ; } llvm_unreachable ( <STR_LIT> ) ;


<s> const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; unsigned bytes = Desc . getSize ( ) ; for ( unsigned i = <NUM_LIT> ; i < bytes ; i ++ ) { OS . write ( ( uint8_t ) ( ( Encoding >> ( <NUM_LIT> * i ) ) & <NUM_LIT> ) ) ; } if ( <STR_LIT> ( STI ) && Desc . TSFlags & <STR_LIT> ) { int vaddr0 = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr0 ) ; int srsrc = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: srsrc ) ; assert ( vaddr0 >= <NUM_LIT> && srsrc > vaddr0 ) ; unsigned NumExtraAddrs = srsrc - vaddr0 - <NUM_LIT> ; unsigned NumPadding = ( - NumExtraAddrs ) & <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < NumExtraAddrs ; ++ i ) OS . write ( ( uint8_t ) getMachineOpValue ( MI , MI . getOperand ( vaddr0 + <NUM_LIT> + i ) , Fixups , STI ) ) ; for ( unsigned i = <NUM_LIT> ; i < NumPadding ; ++ i ) OS . write ( <NUM_LIT> ) ; } if ( ( bytes > <NUM_LIT> && STI . getFeatureBits ( ) [ <STR_LIT> ] ) || ( bytes > <NUM_LIT> && ! STI . getFeatureBits ( ) [ <STR_LIT> ] ) ) return ; for ( unsigned i = <NUM_LIT> , e = Desc . getNumOperands ( ) ; i < e ; ++ i ) { if ( ! <STR_LIT> ( Desc , i ) ) continue ; const MCOperand & Op = MI . getOperand ( i ) ; if ( getLitEncoding ( Op , Desc . OpInfo [ i ] , STI ) != <NUM_LIT> ) continue ; int64_t Imm = <NUM_LIT> ; if ( Op . isImm ( ) ) Imm = Op . getImm ( ) ; else if ( Op . isExpr ( ) ) { if ( const auto * C = dyn_cast < MCConstantExpr > ( Op . getExpr ( ) ) ) Imm = C -> getValue ( ) ; } else if ( ! Op . isExpr ( ) ) llvm_unreachable ( <STR_LIT> ) ; for ( unsigned j = <NUM_LIT> ; j < <NUM_LIT> ; j ++ ) { OS . write ( ( uint8_t ) ( ( Imm >> ( <NUM_LIT> * j ) ) & <NUM_LIT> ) ) ; } break ;


<s> unsigned Reg = MI . getOperand ( OpNo ) . getReg ( ) ; uint64_t Enc = MRI . getEncodingValue ( Reg ) ; if ( MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) || MRI . getRegClass ( <STR_LIT> ) . contains ( Reg ) ) Enc |= <NUM_LIT> ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit64Encoding ( static_cast < uint64_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return getLit16Encoding ( static_cast < uint16_t > ( Imm ) , STI ) ; case <STR_LIT> : case <STR_LIT> : if ( ! isUInt < <NUM_LIT> > ( Imm ) && STI . getFeatureBits ( ) [ <STR_LIT> ] ) return getLit32Encoding ( static_cast < uint32_t > ( Imm ) , STI ) ; LLVM_FALLTHROUGH ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> uint64_t <ISA_LIT>MCCodeEmitter :: getMachineOpValue ( const MCInst & MI , const MCOperand & MO , SmallVectorImpl < MCFixup > & Fixups , const MCSubtargetInfo & STI ) const { if ( MO . isReg ( ) ) return MRI . getEncodingValue ( MO . getReg ( ) ) ; if ( MO . isExpr ( ) && MO . getExpr ( ) -> getKind ( ) != MCExpr :: Constant ) { MCFixupKind Kind ; if ( needsPCRel ( MO . getExpr ( ) ) ) Kind = FK_PCRel_4 ;


<s> if ( needsPCRel ( MO . getExpr ( ) ) ) Kind = FK_PCRel_4 ; else Kind = FK_Data_4 ; const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; uint32_t Offset = Desc . getSize ( ) ; assert ( Offset == <NUM_LIT> || Offset == <NUM_LIT> ) ; Fixups . push_back ( MCFixup :: create ( Offset , MO . getExpr ( ) , Kind , MI . getLoc ( ) ) ) ; } unsigned OpNo = <NUM_LIT> ; for ( unsigned e = MI . getNumOperands ( ) ; OpNo < e ; ++ OpNo ) { if ( & MO == & MI . getOperand ( OpNo ) ) break ; }


<s> verifyInstructionPredicates ( MI , computeAvailableFeatures ( STI . getFeatureBits ( ) ) ) ; uint64_t Encoding = getBinaryCodeForInstr ( MI , Fixups , STI ) ; const MCInstrDesc & Desc = MCII . get ( MI . getOpcode ( ) ) ; unsigned bytes = Desc . getSize ( ) ; for ( unsigned i = <NUM_LIT> ; i < bytes ; i ++ ) { OS . write ( ( uint8_t ) ( ( Encoding >> ( <NUM_LIT> * i ) ) & <NUM_LIT> ) ) ; } if ( <STR_LIT> ( STI ) && Desc . TSFlags & <STR_LIT> ) { int vaddr0 = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr0 ) ; int srsrc = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: srsrc ) ; assert ( vaddr0 >= <NUM_LIT> && srsrc > vaddr0 ) ; unsigned NumExtraAddrs = srsrc - vaddr0 - <NUM_LIT> ; unsigned NumPadding = ( - NumExtraAddrs ) & <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i < NumExtraAddrs ; ++ i ) OS . write ( ( uint8_t ) getMachineOpValue ( MI , MI . getOperand ( vaddr0 + <NUM_LIT> + i ) , Fixups , STI ) ) ; for ( unsigned i = <NUM_LIT> ; i < NumPadding ; ++ i ) OS . write ( <NUM_LIT> ) ; }

if ( ( bytes > <NUM_LIT> && STI . getFeatureBits ( ) [ <STR_LIT> ] ) || ( bytes > <NUM_LIT> && ! STI . getFeatureBits ( ) [ <STR_LIT> ] ) ) return ;