<s> for ( auto & Op : I -> operands ( ) ) { if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } auto Next = std :: next ( I ) ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; I -> eraseFromParent ( ) ; I = Next ; Changed = true ; } if ( I != E ) continue ; for ( auto * Pred : CurBB -> predecessors ( ) ) { if ( Pred -> succ_size ( ) == <NUM_LIT> ) Blocks . push_back ( Pred ) ; } } continue ; } auto Lead = MBB . begin ( ) , E = MBB . end ( ) ; if ( MBB . succ_size ( ) != <NUM_LIT> || Lead == E || ! isEndCF ( * Lead , TRI ) ) continue ; const MachineBasicBlock * Succ = * MBB . succ_begin ( ) ; if ( ! MBB . isLayoutSuccessor ( Succ ) ) continue ; auto I = std :: next ( Lead ) ; for ( ; I != E ; ++ I ) if ( ! TII -> isSALU ( * I ) || I -> readsRegister ( <STR_LIT> , TRI ) ) break ; if ( I != E ) continue ; const auto NextLead = Succ -> begin ( ) ; if ( NextLead == Succ -> end ( ) || ! isEndCF ( * NextLead , TRI ) || ! getOrExecSource ( * NextLead , * TII , MRI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Lead << '\n' ) ; auto SaveExec = getOrExecSource ( * Lead , * TII , MRI ) ; unsigned SaveExecReg = getOrNonExecReg ( *


<s> MachineOperand * Op2 = TII -> getNamedOperand ( * Cmp , <STR_LIT> :: src1 ) ; if ( Op1 -> isImm ( ) && Op2 -> isReg ( ) ) std :: swap ( Op1 , Op2 ) ; if ( ! Op1 -> isReg ( ) || ! Op2 -> isImm ( ) || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; unsigned SelReg = Op1 -> getReg ( ) ; auto * Sel = TRI -> findReachingDef ( SelReg , Op1 -> getSubReg ( ) , * Cmp , MRI , LIS ) ; if ( ! Sel || Sel -> getOpcode ( ) != <STR_LIT> ) return <STR_LIT> ; Op1 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src0 ) ; Op2 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src1 ) ; MachineOperand * CC = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src2 ) ; if ( ! Op1 -> isImm ( ) || ! Op2 -> isImm ( ) || ! CC -> isReg ( ) || Op1 -> getImm ( ) != <NUM_LIT> || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Sel << '\t' << * Cmp << '\t' << * And ) ; unsigned CCReg = CC -> getReg ( ) ; LIS -> RemoveMachineInstrFromMaps ( * And ) ; MachineInstr * Andn2 = BuildMI ( MBB , * And , And -> getDebugLoc ( ) , TII -> get ( Andn2Opc ) , And -> getOperand ( <NUM_LIT> ) . getReg ( ) ) . addReg ( ExecReg ) . addReg ( CCReg , CC -> getSubReg ( ) ) ; And -> eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * Andn2 ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Andn2 << '\n' ) ; if ( ( TargetRegisterInfo :: isVirtualRegister ( CmpReg ) && MRI . use_nodbg_empty ( CmpReg ) ) || ( CmpReg == CondReg && std :: none_of ( std :: next ( Cmp -> getIterator ( ) ) , Andn2 -> getIterator ( ) , [ & ] ( const MachineInstr & MI ) { return MI . readsRegister ( CondReg , TRI ) ; } ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Cmp << '\n' ) ; LIS -> RemoveMachineInstrFromMaps ( * Cmp ) ; Cmp -> eraseFromParent ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SelReg ) && MRI . use_nodbg_empty ( SelReg ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Sel << '\n' ) ; LIS -> RemoveMachineInstrFromMaps ( * Sel ) ; Sel -> eraseFromParent (


<s> DenseSet < unsigned > RecalcRegs ( { <STR_LIT> , <STR_LIT> } ) ; bool Changed = false ; for ( MachineBasicBlock & MBB : MF ) { if ( unsigned Reg = optimizeVcndVcmpPair ( MBB , ST , MRI , LIS ) ) { RecalcRegs . insert ( Reg ) ; RecalcRegs . insert ( <STR_LIT> ) ; RecalcRegs . insert ( <STR_LIT> ) ; RecalcRegs . insert ( <STR_LIT> ) ; Changed = true ; } if ( MBB . succ_empty ( ) ) { if ( MBB . empty ( ) ) continue ; MachineInstr & Term = MBB . back ( ) ; if ( Term . getOpcode ( ) != <STR_LIT> || Term . getNumOperands ( ) != <NUM_LIT> ) continue ; SmallVector < MachineBasicBlock * , <NUM_LIT> > Blocks ( { & MBB } ) ; while ( ! Blocks . empty ( ) ) { auto CurBB = Blocks . pop_back_val ( ) ; auto I = CurBB -> rbegin ( ) , E = CurBB -> rend ( ) ; if ( I != E ) { if ( I -> isUnconditionalBranch ( ) || I -> getOpcode ( ) == <STR_LIT> ) ++ I ; else if ( I -> isBranch ( ) ) continue ; } while ( I != E ) { if ( I -> isDebugInstr ( ) ) { I = std :: next ( I ) ; continue ; } if ( I -> mayStore ( ) || I -> isBarrier ( ) || I -> isCall ( ) || I -> hasUnmodeledSideEffects ( ) || I -> hasOrderedMemoryRef ( ) ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I << '\n' ) ; for ( auto & Op : I -> operands ( ) ) { if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } auto Next = std :: next ( I ) ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; I -> eraseFromParent ( ) ; I = Next ; Changed = true ; } if ( I != E ) continue ; for ( auto * Pred : CurBB -> predecessors ( ) ) { if ( Pred -> succ_size ( ) == <NUM_LIT> ) Blocks . push_back ( Pred ) ; } } continue ; } auto Lead = MBB . begin ( ) , E = MBB . end ( ) ; if ( MBB . succ_size ( ) != <NUM_LIT> || Lead == E || ! isEndCF ( * Lead , TRI ) ) continue ; const MachineBasicBlock * Succ = * MBB . succ_begin ( ) ; if ( ! MBB . isLayoutSuccessor ( Succ ) ) continue ; auto I = std ::


<s> if ( MI . isCopy ( ) && MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) { assert ( MI . isFullCopy ( ) ) ; return true ; } return false


<s> if ( TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src0_modifiers ) || TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src1_modifiers ) ) return <STR_LIT> ; Op1 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src0 ) ; Op2 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src1 ) ; MachineOperand * CC = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src2 ) ; if ( ! Op1 -> isImm ( ) || ! Op2 -> isImm ( ) || ! CC -> isReg ( ) || Op1 -> getImm ( ) != <NUM_LIT> || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Sel << '\t' << * Cmp << '\t' << * And ) ; unsigned CCReg = CC -> getReg ( ) ; LIS -> RemoveMachineInstrFromMaps ( * And ) ; MachineInstr * Andn2 = BuildMI ( MBB , * And , And -> getDebugLoc ( ) , TII -> get ( Andn2Opc ) , And -> getOperand ( <NUM_LIT> ) . getReg ( ) ) . addReg ( ExecReg ) . addReg ( CCReg , <NUM_LIT> , CC -> getSubReg ( ) ) ; And -> eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * Andn2 ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Andn2 << '\n' ) ; if ( ( TargetRegisterInfo :: isVirtualRegister ( CmpReg ) && MRI . use_nodbg_empty ( CmpReg ) ) || ( CmpReg == CondReg && std :: none_of ( std :: next ( Cmp -> getIterator ( ) ) ,

Andn2 -> getIterator ( ) , [ & ] ( const MachineInstr & MI ) {


<s> if ( ! And || And -> getOpcode ( ) != AndOpc || ! And -> getOperand ( <NUM_LIT> ) . isReg ( ) || ! And -> getOperand ( <NUM_LIT> ) . isReg ( ) ) return <STR_LIT> ; MachineOperand * AndCC = & And -> getOperand ( <NUM_LIT> ) ; unsigned CmpReg = AndCC -> getReg ( ) ; unsigned CmpSubReg = AndCC -> getSubReg ( ) ; if ( CmpReg == ExecReg ) { AndCC = & And -> getOperand ( <NUM_LIT> ) ; CmpReg = AndCC -> getReg ( ) ; CmpSubReg = AndCC -> getSubReg ( ) ; } else if ( And -> getOperand ( <NUM_LIT> ) . getReg ( ) != ExecReg ) { return <STR_LIT> ; } auto * Cmp = TRI -> findReachingDef ( CmpReg , CmpSubReg , * And , MRI , LIS ) ; if ( ! Cmp || ! ( Cmp -> getOpcode ( ) == <STR_LIT> || Cmp -> getOpcode ( ) == <STR_LIT> ) || Cmp -> getParent ( ) != And -> getParent ( ) ) return <STR_LIT> ; MachineOperand * Op1 = TII -> getNamedOperand ( * Cmp , <STR_LIT> :: src0 ) ; MachineOperand * Op2 = TII -> getNamedOperand ( * Cmp , <STR_LIT> :: src1 ) ; if ( Op1 -> isImm ( ) && Op2 -> isReg ( ) ) std :: swap ( Op1 , Op2 ) ; if ( ! Op1 -> isReg ( ) || ! Op2 -> isImm ( ) || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; unsigned SelReg = Op1 -> getReg ( ) ; auto * Sel = TRI -> findReachingDef ( SelReg , Op1 -> getSubReg ( ) , * Cmp , MRI , LIS ) ; if ( ! Sel || Sel -> getOpcode ( ) != <STR_LIT> ) return <STR_LIT> ; if ( TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src0_modifiers ) || TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src1_modifiers ) ) return <STR_LIT> ; Op1 = TII -> getNamedOperand ( * Sel , <STR_LIT>


<s> const MachineBasicBlock * Succ = * MBB . succ_begin ( ) ; if ( ! MBB . isLayoutSuccessor ( Succ ) ) continue ; auto I = std :: next ( Lead ) ; for ( ; I != E ; ++ I ) if ( ! TII -> isSALU ( * I ) || I -> readsRegister ( <STR_LIT> , TRI ) ) break ; if ( I != E ) continue ; const auto NextLead = Succ -> begin ( ) ; if ( NextLead == Succ -> end ( ) || ! isEndCF ( * NextLead , TRI ) || ! getOrExecSource ( * NextLead , * TII , MRI ) ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * Lead << '\n' ) ; auto SaveExec = getOrExecSource ( * Lead , * TII , MRI ) ; unsigned SaveExecReg = getOrNonExecReg ( * Lead , * TII ) ; for ( auto & Op : Lead -> operands ( ) ) { if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } LIS -> RemoveMachineInstrFromMaps ( * Lead ) ; Lead -> eraseFromParent ( ) ; if ( SaveExecReg ) { LIS -> removeInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( SaveExecReg ) ; } Changed = true ; if ( ! SaveExec || ! SaveExec -> isFullCopy ( ) ) continue ; unsigned SavedExec = SaveExec -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool SafeToReplace = true ; for ( auto & U : MRI . use_nodbg_instructions ( SavedExec ) ) { if ( U . getParent ( ) != SaveExec -> getParent ( ) ) { SafeToReplace = false ; break ; } DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExec << '\n' ) ; } if ( SafeToReplace ) { LIS -> RemoveMachineInstrFromMaps ( * SaveExec ) ; SaveExec -> eraseFromParent ( ) ; MRI . replaceRegWith ( SavedExec , <STR_LIT> ) ; LIS -> removeInterval ( SavedExec ) ; } } if ( Changed )


<s> auto Lead = MBB . begin ( ) , E = MBB . end ( ) ; if ( MBB . succ_size ( ) != <NUM_LIT> || Lead == E || ! isEndCF ( * Lead , TRI ) ) continue ; const MachineBasicBlock * Succ = * MBB . succ_begin ( ) ; if ( ! MBB . isLayoutSuccessor ( Succ ) ) continue ; auto I = std :: next ( Lead ) ; for ( ; I != E ; ++ I ) if ( ! TII -> isSALU ( * I ) || I -> readsRegister ( <STR_LIT> , TRI ) ) break ; if ( I != E ) continue ; const auto NextLead = Succ -> begin ( ) ; if ( NextLead == Succ -> end ( ) || ! isEndCF ( * NextLead , TRI ) || ! getOrExecSource ( * NextLead , * TII , MRI ) ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * Lead << '\n' ) ; auto SaveExec = getOrExecSource ( * Lead , * TII , MRI ) ; unsigned SaveExecReg = getOrNonExecReg ( * Lead , * TII ) ; for ( auto & Op : Lead -> operands ( ) ) { if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } LIS -> RemoveMachineInstrFromMaps ( * Lead ) ; Lead -> eraseFromParent ( ) ; if ( SaveExecReg ) { LIS -> removeInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( SaveExecReg ) ; } Changed = true ; if ( ! SaveExec || ! SaveExec -> isFullCopy ( ) ) continue ; unsigned SavedExec = SaveExec -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool SafeToReplace = true ; for ( auto & U : MRI . use_nodbg_instructions ( SavedExec ) ) { if ( U . getParent ( ) != SaveExec -> getParent ( ) ) { SafeToReplace = false ; break ; } DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExec << '\n' ) ; } if ( SafeToReplace ) { LIS -> RemoveMachineInstrFromMaps ( * SaveExec


<s> return new <ISA_LIT>OptimizeExecMaskingPreRA ( )


<s> StringRef getPassName ( ) const override { return


<s> I -- ; } if ( ! AndExecMI ) return false ; SlotIndex StartIdx = LIS -> getInstructionIndex ( SaveExecMI ) ; SlotIndex EndIdx = LIS -> getInstructionIndex ( * AndExecMI ) ; for ( MCRegUnitIterator UI ( ExecReg , TRI ) ; UI . isValid ( ) ; ++ UI ) { LiveRange & RegUnit = LIS -> getRegUnit ( * UI ) ; if ( RegUnit . find ( StartIdx ) != std :: prev ( RegUnit . find ( EndIdx ) ) ) return false ; } LIS -> removeInterval ( SavedExecReg ) ; LIS -> removeInterval ( DstReg ) ; SaveExecMI . getOperand ( <NUM_LIT> ) . setReg ( DstReg ) ; LIS -> RemoveMachineInstrFromMaps ( * AndExecMI ) ; AndExecMI -> eraseFromParent ( ) ; LIS -> createAndComputeVirtRegInterval ( DstReg ) ; return true


<s> <ISA_LIT>OptimizeExecMaskingPreRA ( ) : MachineFunctionPass


<s> for ( unsigned N = <NUM_LIT> ; N <= InstLimit && I != E ; ++ I , ++ N ) { Register CopyFromExec = isCopyFromExec ( * I , ST ) ; if ( CopyFromExec . isValid


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && Dst . getReg ( ) == ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) && MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; break ; } case <STR_LIT> : case <STR_LIT> : llvm_unreachable ( <STR_LIT> )


<s> case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && Dst . getReg ( ) == ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) && MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; break ; } case <STR_LIT> : case <STR_LIT> : llvm_unreachable ( <STR_LIT> ) ; } return Register


<s> static bool isLiveOut ( const MachineBasicBlock & MBB , unsigned Reg ) { for ( MachineBasicBlock * Succ : MBB . successors (


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Src1 = MI . getOperand ( <NUM_LIT> ) ; if ( Src1 . isReg ( ) && Src1 . getReg ( ) == <STR_LIT> ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const MachineOperand & Src2 = MI . getOperand ( <NUM_LIT> ) ; if ( Src2 . isReg ( ) && Src2 . getReg ( ) == <STR_LIT> ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT>


<s> LivePhysRegs LR ( * TRI ) ; LR . addLiveOuts ( * Stop . getParent ( ) ) ; for ( auto A = Stop . getParent ( )


<s> Register MoveDest = SaveExecInstr . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock :: instr_iterator InsertPosIt = SaveExecInstr . getIterator ( ) ; if ( ! SaveExecInstr . uses ( ) . empty ( ) ) { bool isSGPR32 = TRI -> getRegSizeInBits ( MoveDest , MRI ) == <NUM_LIT> ; unsigned MovOpcode = isSGPR32 ? <STR_LIT> : <STR_LIT> ; BuildMI ( * SaveExecInstr . getParent ( ) , InsertPosIt , SaveExecInstr . getDebugLoc ( ) , TII -> get ( MovOpcode ) , MoveDest ) . addReg ( Exec ) ; } auto Builder = BuildMI ( * VCmp . getParent ( ) , std :: next ( InsertPosIt ) , VCmp . getDebugLoc ( ) , TII -> get ( NewOpcode ) ) ; auto TryAddImmediateValueFromNamedOperand = [ & ] ( unsigned OperandName ) -> void { if ( auto * Mod = TII -> getNamedOperand ( VCmp , OperandName ) ) Builder


<s> bool Changed = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: reverse_iterator I = fixTerminators ( * TII , MBB ) ; MachineBasicBlock :: reverse_iterator E = MBB . rend ( ) ; if ( I == E ) continue ; Register CopyToExec ; unsigned SearchCount = <NUM_LIT> ; const unsigned SearchLimit = <NUM_LIT> ; while ( I != E && SearchCount ++ < SearchLimit ) { CopyToExec = isCopyToExec ( * I , ST ) ; if ( CopyToExec ) break ; ++ I ; } if ( ! CopyToExec ) continue ; auto CopyToExecInst = & * I ; auto CopyFromExecInst = findExecCopy ( * TII , ST , MBB , I , CopyToExec ) ; if ( CopyFromExecInst == E ) { auto PrepareExecInst = std :: next ( I ) ; if ( PrepareExecInst == E ) continue ; if ( CopyToExecInst -> getOperand ( <NUM_LIT> ) . isKill ( ) && isLogicalOpOnExec ( * PrepareExecInst ) == CopyToExec ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PrepareExecInst ) ; PrepareExecInst -> getOperand ( <NUM_LIT> ) . setReg ( Exec ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PrepareExecInst << '\n' ) ; CopyToExecInst -> eraseFromParent ( ) ; Changed = true ; } continue ; } if ( isLiveOut ( MBB , CopyToExec ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } Register CopyFromExec = CopyFromExecInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * SaveExecInst = nullptr ; SmallVector < MachineInstr * , <NUM_LIT> > OtherUseInsts ; for ( MachineBasicBlock :: iterator J = std :: next ( CopyFromExecInst -> getIterator ( ) ) , JE = I -> getIterator ( ) ; J != JE ; ++ J ) { if ( SaveExecInst && J -> readsRegister ( Exec , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; SaveExecInst = nullptr ; break ; } bool ReadsCopyFromExec = J -> readsRegister ( CopyFromExec , TRI ) ; if ( J -> modifiesRegister ( CopyToExec , TRI ) ) { if ( SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( CopyToExec , TRI ) << '\n' ) ; SaveExecInst = nullptr ; break ; } unsigned SaveExecOp = getSaveExecOp ( J -> getOpcode ( ) ) ; if ( SaveExecOp == <STR_LIT> ) break ; if ( ReadsCopyFromExec ) { SaveExecInst = &


<s> for ( auto & O : make_early_inc_range ( MRI -> use_operands ( Reg ) ) ) { auto * UseMI = O . getParent ( ) ; auto * UseBlock = UseMI -> getParent ( ) ; if ( UseBlock == Loop ) O . setReg ( NewReg ) ; } MachineInstrBuilder PHI = BuildMI ( * Loop , Loop -> getFirstNonPHI ( ) , DebugLoc ( ) , TII -> get ( TargetOpcode :: PHI ) , NewReg ) ; for ( auto * Pred : Loop -> predecessors ( ) ) { if ( Pred == Loop ) PHI . addReg ( UndefReg , RegState :: Undef ) . addMBB (


<s>
MDT
=
&
getAnalysis
<
MachineDominatorTree
>
(
)
;
Loops
=
&
getAnalysis
<
MachineLoopInfo
>
(
)
;
LV
=
&
getAnalysis
<
LiveVariables
>
(
)
;
MRI
=
&
MF
.
getRegInfo
(
)
;
if
(
skipFunction
(
MF
.
getFunction
(
)
)
)
return
false
;
bool
MadeChange
=
false
;
for
(
MachineBasicBlock
&
MBB
:
MF
)
{
for
(
auto
&
MI
:
MBB
.
terminators
(
)
)
{
if
(
MI
.
getOpcode
(
)
==
<STR_LIT>
)
{
MachineBasicBlock
*
IfTarget
=
MI
.
getOperand
(
<NUM_LIT>
)
.
getMBB
(
)
;
auto
*
Endif
=
getElseTarget
(
IfTarget
)
;
if
(
!
Endif
)
continue
;
SmallSetVector
<
MachineBasicBlock
*
,
<NUM_LIT>
>
ElseBlocks
;
SmallVector
<
Register
>
CandidateRegs
;
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
printMBBReference
(
MBB
)
<<
' '
<<
printMBBReference
(
*
IfTarget
)
<<
' '
<<
printMBBReference
(
*
Endif
)
<<
'\n'
)
;
collectElseRegionBlocks
(
IfTarget
,
Endif
,
ElseBlocks
)
;
collectCandidateRegisters
(
&
MBB
,
IfTarget
,
Endif
,
ElseBlocks
,
CandidateRegs
)
;
MadeChange
|=
!
CandidateRegs
.
empty
(
)
;
for
(
auto
Reg
:
CandidateRegs
)
optimizeLiveRange
(
Reg
,
&
MBB
,
IfTarget
,
Endif
,
ElseBlocks
)
;
}
else
if
(
MI
.
getOpcode
(
)
==
<STR_LIT>
)
{
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
printMBBReference
(
MBB
)
<<
'\n'
)
;
SmallSetVector
<
Register
,
<NUM_LIT>
>
CandidateRegs
;
collectWaterfallCandidateRegisters
(
&
MBB
,
CandidateRegs
)
;
MadeChange
|=
!
CandidateRegs
.
empty
(
)
;
for
(
auto
Reg
:
CandidateRegs
)
optimizeWaterfallLiveRange
(
Reg
,
&
MBB
)
;
}
}
}
return


<s>
for
(
auto
&
MI
:
MBB
.
terminators
(
)
)
{
if
(
MI
.
getOpcode
(
)
==
<STR_LIT>
)
{
MachineBasicBlock
*
IfTarget
=
MI
.
getOperand
(
<NUM_LIT>
)
.
getMBB
(
)
;
auto
*
Endif
=
getElseTarget
(
IfTarget
)
;
if
(
!
Endif
)
continue
;
SmallSetVector
<
MachineBasicBlock
*
,
<NUM_LIT>
>
ElseBlocks
;
SmallVector
<
Register
>
CandidateRegs
;
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
printMBBReference
(
MBB
)
<<
' '
<<
printMBBReference
(
*
IfTarget
)
<<
' '
<<
printMBBReference
(
*
Endif
)
<<
'\n'
)
;
collectElseRegionBlocks
(
IfTarget
,
Endif
,
ElseBlocks
)
;
collectCandidateRegisters
(
&
MBB
,
IfTarget
,
Endif
,
ElseBlocks
,
CandidateRegs
)
;
MadeChange
|=
!
CandidateRegs
.
empty
(
)
;
for
(
auto
Reg
:
CandidateRegs
)
optimizeLiveRange
(
Reg
,
&
MBB
,
IfTarget
,
Endif
,
ElseBlocks
)
;
}
else
if
(
MI
.
getOpcode
(
)
==
<STR_LIT>
)
{
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
printMBBReference
(


<s> } for ( auto I = MRI -> use_nodbg_begin ( Reg ) , E = MRI -> use_nodbg_end ( ) ; I != E ; ++ I ) { auto * UseMI = I -> getParent ( ) ; if ( UseMI -> isPHI ( ) && I -> readsReg ( ) ) { if ( Visited . contains ( UseMI -> getParent ( ) ) ) PHIIncoming . insert ( UseMI -> getOperand ( I . getOperandNo ( ) + <NUM_LIT> ) . getMBB ( ) ) ; } } Visited . clear ( ) ; for ( MachineBasicBlock * MBB : depth_first_ext ( ThenEntry , Visited ) ) { if ( MBB == Flow ) break ; SmallVector < MachineInstr * > Uses ; findNonPHIUsesInBlock ( Reg , MBB , Uses ) ; if ( Uses . size ( ) == <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MBB )


<s> if ( ( VI . AliveBlocks . test ( If -> getNumber ( ) ) || DefMBB == If ) && Loops -> getLoopFor ( DefMBB ) == Loops -> getLoopFor ( If ) ) KillsInElse . insert ( Reg ) ; } } auto IsLiveThroughThen = [ & ] ( Register Reg ) { for ( auto I = MRI -> use_nodbg_begin ( Reg ) , E = MRI -> use_nodbg_end ( ) ; I != E ; ++ I ) { if ( ! I -> readsReg ( ) ) continue ; auto * UseMI = I -> getParent ( ) ; auto * UseMBB = UseMI -> getParent ( ) ; if ( UseMBB == Flow || UseMBB == Endif ) { if ( ! UseMI -> isPHI ( ) ) return true ; auto * IncomingMBB = UseMI -> getOperand ( I . getOperandNo ( ) + <NUM_LIT> ) . getMBB ( ) ; if ( ( UseMBB == Flow && IncomingMBB != If ) || ( UseMBB == Endif && IncomingMBB == Flow ) ) return true ; } } return false ; } ; for ( auto Reg : KillsInElse ) { if ( ! IsLiveThroughThen ( Reg )


<s> if ( ! VI . isLiveIn ( * Endif , MOReg , * MRI ) ) { KillsInElse . insert ( MOReg ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( MOReg , TRI ) << <STR_LIT> ) ; } } } } } } for ( auto & MI : Endif -> phis ( ) ) { for ( unsigned Idx = <NUM_LIT> ; Idx < MI . getNumOperands ( ) ; Idx += <NUM_LIT> ) { auto & MO = MI . getOperand ( Idx ) ; auto * Pred = MI . getOperand ( Idx + <NUM_LIT> ) . getMBB ( ) ; if ( Pred == Flow ) continue ; assert ( ElseBlocks . contains ( Pred ) && <STR_LIT> ) ; if ( ! MO . isReg ( ) || ! MO . getReg ( ) || MO . isUndef ( ) ) continue ; Register Reg = MO . getReg ( ) ; if ( Reg . isPhysical ( ) || ! TRI -> isVectorRegister ( * MRI , Reg ) ) continue ; LiveVariables :: VarInfo & VI = LV -> getVarInfo ( Reg ) ; if ( VI . isLiveIn ( * Endif , Reg , * MRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ; continue ; } const MachineBasicBlock * DefMBB = MRI -> getVRegDef ( Reg ) -> getParent ( ) ; if ( ( VI . AliveBlocks . test ( If -> getNumber ( ) ) || DefMBB == If ) && Loops -> getLoopFor ( DefMBB ) == Loops -> getLoopFor ( If ) ) KillsInElse . insert ( Reg ) ; } } auto IsLiveThroughThen = [ & ] ( Register Reg ) { for ( auto I = MRI -> use_nodbg_begin ( Reg ) , E = MRI -> use_nodbg_end ( ) ; I != E ; ++ I ) { if ( ! I -> readsReg ( ) ) continue ; auto * UseMI = I -> getParent ( ) ; auto * UseMBB = UseMI -> getParent ( ) ; if ( UseMBB == Flow || UseMBB == Endif ) { if ( ! UseMI -> isPHI ( ) ) return true ; auto * IncomingMBB = UseMI -> getOperand ( I . getOperandNo ( ) +


<s> unsigned Cur = <NUM_LIT> ; while ( MBB ) { for ( auto * Pred : MBB -> predecessors ( ) ) { if ( Pred != Flow && ! Blocks . contains ( Pred ) ) Blocks . insert ( Pred ) ; } if ( Cur < Blocks . size ( ) ) MBB = Blocks [ Cur ++ ] ; else MBB = nullptr ; } LLVM_DEBUG


<s> for ( auto * Pred : MBB -> predecessors ( ) ) { if ( Pred != Flow && ! Blocks . contains ( Pred ) ) Blocks . insert ( Pred ) ; } if ( Cur < Blocks . size ( ) ) MBB = Blocks [ Cur ++ ] ; else MBB = nullptr ; } LLVM_DEBUG


<s> if ( MBB == LoopEnd ) break ; assert ( MBB -> pred_size ( ) == <NUM_LIT> || ( MBB == LoopHeader && MBB -> pred_size ( ) == <NUM_LIT> ) ) ; assert ( MBB -> succ_size ( ) == <NUM_LIT> ) ; MBB = * MBB -> succ_begin ( ) ; } for ( auto * I : Instructions ) { auto & MI = * I ; for ( auto & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . getReg ( ) || MO . isDef ( ) ) continue ; Register MOReg = MO . getReg ( ) ; if ( MOReg . isPhysical ( ) || ! TRI -> isVectorRegister ( * MRI , MOReg ) ) continue ; if ( MO . readsReg ( ) ) { MachineBasicBlock * DefMBB = MRI -> getVRegDef ( MOReg ) -> getParent ( ) ; if ( ! Blocks . contains ( DefMBB ) && ! CandidateRegs . contains ( MOReg ) ) { LiveVariables :: VarInfo & OldVarInfo = LV -> getVarInfo ( MOReg ) ; bool IsUsed = false ; for ( auto * Succ : LoopEnd -> successors (


<s> AU . addRequired < MachineDominatorTree > ( ) ; AU . addRequired < MachineLoopInfo > ( ) ; AU . addPreserved < LiveVariables > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ; AU . addPreserved < MachineLoopInfo > (


<s> AU . addPreserved < LiveVariables > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ; AU . addPreserved <


<s> MachineFunctionProperties getRequiredProperties (


<s> const auto * RC = MRI -> getRegClass ( Reg ) ; Register NewReg = MRI -> createVirtualRegister ( RC ) ; Register UndefReg = MRI -> createVirtualRegister ( RC ) ; MachineInstrBuilder PHI = BuildMI ( * Flow , Flow -> getFirstNonPHI ( ) , DebugLoc ( ) , TII -> get ( TargetOpcode :: PHI ) , NewReg ) ; for ( auto * Pred : Flow -> predecessors ( ) ) { if ( Pred == If ) PHI . addReg ( Reg ) . addMBB ( Pred ) ; else PHI . addReg ( UndefReg , RegState :: Undef ) . addMBB ( Pred ) ; } for ( auto & O : make_early_inc_range ( MRI -> use_operands ( Reg ) ) ) { auto * UseMI = O . getParent ( ) ; auto * UseBlock = UseMI -> getParent ( ) ; if ( UseBlock == Endif ) { assert ( UseMI -> isPHI ( ) && <STR_LIT>


<s>
SmallSetVector
<
MachineBasicBlock
*
,
<NUM_LIT>
>
ElseBlocks
;
SmallVector
<
Register
>
CandidateRegs
;
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
printMBBReference
(
MBB
)
<<
' '
<<
printMBBReference
(
*
IfTarget
)
<<
' '
<<
printMBBReference
(
*
Endif
)
<<
'\n'
)
;
collectElseRegionBlocks
(
IfTarget
,
Endif
,
ElseBlocks
)
;
collectCandidateRegisters
(
&
MBB
,
IfTarget
,
Endif
,
ElseBlocks
,
CandidateRegs
)
;
MadeChange
|=
!
CandidateRegs
.
empty
(
)
;
for
(
auto
Reg
:
CandidateRegs
)
optimizeLiveRange
(
Reg
,
&
MBB
,
IfTarget
,
Endif
,
ElseBlocks
)
;
}
else
if
(
MI
.
getOpcode
(
)
==
<STR_LIT>
)
{
auto
*
LoopHeader
=
MI
.
getOperand
(
<NUM_LIT>
)
.
getMBB
(
)
;
auto
*
LoopEnd
=
&
MBB
;
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
printMBBReference
(
*
LoopHeader
)
<<
'\n'
)
;
SmallSetVector
<
Register
,
<NUM_LIT>
>
CandidateRegs
;
SmallVector
<
MachineInstr
*
,
<NUM_LIT>
>
Instructions
;
SmallSetVector
<
MachineBasicBlock
*
,
<NUM_LIT>
>
Blocks
;
collectWaterfallCandidateRegisters
(
LoopHeader
,
LoopEnd
,
CandidateRegs
,
Blocks
,
Instructions
)
;
MadeChange
|=
!
CandidateRegs
.
empty
(
)
;
for
(
auto
Reg
:
CandidateRegs
)
optimizeWaterfallLiveRange
(
Reg
,
LoopHeader
,
Blocks
,
Instructions
)
;
}
}
}
return


<s> NewVarInfo . AliveBlocks . set ( BBNum ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MBB ) << '\n' ) ; OldVarInfo . AliveBlocks . reset ( BBNum ) ; } } auto I = OldVarInfo . Kills . begin ( ) ; while ( I != OldVarInfo . Kills . end ( ) ) { if ( ElseBlocks . contains ( ( * I ) -> getParent ( ) ) ) { NewVarInfo . Kills . push_back ( * I ) ; I = OldVarInfo . Kills . erase ( I ) ; } else


<s> void <ISA_LIT>OptimizeVGPRLiveRange :: updateLiveRangeInElseRegion ( Register Reg , Register NewReg , MachineBasicBlock * Flow , MachineBasicBlock * Endif , SmallSetVector < MachineBasicBlock * , <NUM_LIT> > & ElseBlocks ) const { LiveVariables :: VarInfo & NewVarInfo = LV -> getVarInfo ( NewReg ) ; LiveVariables :: VarInfo & OldVarInfo = LV -> getVarInfo ( Reg ) ; for ( auto * MBB : ElseBlocks ) { unsigned BBNum = MBB -> getNumber ( ) ; if ( OldVarInfo . AliveBlocks . test ( BBNum ) ) { NewVarInfo . AliveBlocks . set ( BBNum ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <<


<s> uint64_t SDWASrcOperand :: getSrcMods ( const <ISA_LIT>InstrInfo * TII , const MachineOperand * SrcOp ) const { uint64_t Mods = <NUM_LIT> ; const auto * MI = SrcOp -> getParent ( ) ; if ( TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) == SrcOp ) { if ( auto * Mod = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0_modifiers ) ) { Mods = Mod -> getImm ( ) ; } } else if ( TII -> getNamedOperand ( * MI , <STR_LIT> :: src1 ) == SrcOp ) { if ( auto * Mod = TII -> getNamedOperand ( * MI , <STR_LIT> :: src1_modifiers ) ) { Mods = Mod -> getImm ( ) ; } } if ( Abs || Neg ) { assert ( ! Sext && <STR_LIT> ) ; Mods |= Abs ? <STR_LIT> : <NUM_LIT> ; Mods ^= Neg ? <STR_LIT> : <NUM_LIT> ; } else if ( Sext ) { Mods |= <STR_LIT> ; } return


<s> if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) Opc = <STR_LIT> ( Opc ) ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) return false ; if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && ( SDst -> getReg ( ) != <STR_LIT> && SDst

-> getReg ( ) != <STR_LIT> ) ) return false ;


<s> void <ISA_LIT>PeepholeSDWA :: pseudoOpConvertToVOP2 ( MachineInstr & MI , const GCNSubtarget & ST ) const { int Opc = MI . getOpcode ( ) ; assert ( ( Opc == <STR_LIT> || Opc == <STR_LIT> ) && <STR_LIT> ) ; if ( ! TII -> canShrink ( MI , * MRI ) ) return ; Opc = <STR_LIT> ( Opc ) ; const MachineOperand * Sdst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( ! Sdst ) return ; MachineOperand * NextOp = findSingleRegUse ( Sdst , MRI ) ; if ( ! NextOp ) return ; MachineInstr & MISucc = * NextOp -> getParent ( ) ; if ( ! TII -> canShrink ( MISucc , * MRI ) ) return ; int SuccOpc = <STR_LIT> ( MISucc . getOpcode ( ) ) ; MachineOperand * CarryIn = TII -> getNamedOperand ( MISucc , <STR_LIT> :: src2 ) ; if ( ! CarryIn ) return ; MachineOperand * CarryOut = TII -> getNamedOperand ( MISucc , <STR_LIT> :: sdst ) ; if ( ! CarryOut ) return ; if ( ! MRI -> hasOneUse ( CarryIn -> getReg ( ) ) || ! MRI -> use_empty ( CarryOut -> getReg ( ) ) ) return ; MachineBasicBlock & MBB = * MI . getParent ( ) ; auto Liveness = MBB . computeRegisterLiveness ( TRI , <STR_LIT> , MI , <NUM_LIT> ) ; if ( Liveness != MachineBasicBlock :: LQR_Dead ) return ; for ( auto I = std :: next ( MI . getIterator ( ) ) , E = MISucc . getIterator ( ) ; I != E ; ++ I ) { if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return ; } auto NewMI = BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII


<s> if ( ! MRI -> hasOneUse ( CarryIn -> getReg ( ) ) || ! MRI -> use_empty ( CarryOut -> getReg ( ) ) ) return ; MachineBasicBlock & MBB = * MI . getParent ( ) ; auto Liveness = MBB . computeRegisterLiveness ( TRI , <STR_LIT> , MI , <NUM_LIT> ) ; if ( Liveness != MachineBasicBlock :: LQR_Dead ) return ; for ( auto I = std :: next ( MI . getIterator ( ) ) , E = MISucc . getIterator ( ) ; I != E ; ++ I ) { if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return ; } auto NewMI = BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) ) ; NewMI . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) ; NewMI . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) ; NewMI . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ) ; MI . eraseFromParent ( ) ; auto NewInst = BuildMI ( MBB , MISucc , MISucc . getDebugLoc ( ) , TII -> get ( SuccOpc ) ) ; NewInst . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: vdst ) ) ; NewInst . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src0 ) ) ; NewInst . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src1 ) ) ; MISucc . eraseFromParent


<s> } SDWAOperands . clear ( ) ; matchSDWAOperands ( MBB ) ; for ( const auto & OperandPair : SDWAOperands ) { const auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI && isConvertibleToSDWA ( * PotentialMI , ST ) ) { PotentialMatches [ PotentialMI ] . push_back ( Operand . get ( ) ) ; } } for ( auto & PotentialPair : PotentialMatches ) { MachineInstr & PotentialMI = * PotentialPair . first ; convertToSDWA ( PotentialMI , PotentialPair . second ) ; } PotentialMatches . clear ( ) ; SDWAOperands . clear ( )


<s> assert ( Dst && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: sdst ) != - <NUM_LIT> ) ; SDWAInst . add ( * Dst ) ; } else { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: sdst ) != - <NUM_LIT> ) ; SDWAInst . addReg ( <STR_LIT> , RegState :: Define ) ; } MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; assert ( Src0 && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ) ; if ( auto * Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ) SDWAInst . addImm ( Mod -> getImm ( ) ) ; else SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src0 ) ; MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_modifiers ) != - <NUM_LIT> ) ; if ( auto * Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ) SDWAInst . addImm ( Mod -> getImm ( ) ) ; else SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src1 ) ; } if ( SDWAOpcode == <STR_LIT> || SDWAOpcode == <STR_LIT> ) { MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; assert ( Src2 ) ; SDWAInst . add ( * Src2 ) ; } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: clamp ) != - <NUM_LIT> ) ; MachineOperand * Clamp = TII -> getNamedOperand ( MI , <STR_LIT> :: clamp ) ; if ( Clamp ) { SDWAInst . add ( * Clamp ) ; } else { SDWAInst . addImm ( <NUM_LIT> ) ; } if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: omod ) != - <NUM_LIT> ) { MachineOperand * OMod = TII -> getNamedOperand ( MI , <STR_LIT> :: omod ) ; if ( OMod ) { SDWAInst . add ( * OMod ) ; } else { SDWAInst . addImm ( <NUM_LIT> ) ; } } if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_sel


<s> int Opc = MI . getOpcode ( ) ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) Opc = <STR_LIT> ( Opc ) ; if ( Opc == - <NUM_LIT> || <STR_LIT> ( Opc ) == - <NUM_LIT> ) return false ; if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> ::


<s> if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ; } if ( ! ST . hasSDWAOutModsVOPC ( ) && ( TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) || TII -> hasModifiersSet (

MI , <STR_LIT> :: omod ) ) ) return false ;


<s> auto SDWADst = make_unique < SDWADstOperand > ( Dst , Src1 , * Imm == <NUM_LIT> ? WORD_1 : BYTE_3 , UNUSED_PAD ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWADst << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWADst ) ; ++ NumSDWAPatternsFound ; } else { auto SDWASrc = make_unique < SDWASrcOperand > ( Src1 , Dst , * Imm == <NUM_LIT> ? WORD_1 : BYTE_3 , false , false , Opcode != <STR_LIT> && Opcode != <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWASrc << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWASrc ) ; ++ NumSDWAPatternsFound ; } break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; auto Imm = foldToImm ( * Src0 ) ; if ( ! Imm || * Imm != <NUM_LIT> ) break ; MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; MachineOperand * Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ; if ( TRI -> isPhysicalRegister ( Src1 -> getReg ( ) ) || TRI -> isPhysicalRegister ( Dst -> getReg ( ) ) ) break ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { auto SDWADst = make_unique < SDWADstOperand > ( Dst , Src1 , BYTE_1 , UNUSED_PAD ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWADst << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWADst ) ; ++ NumSDWAPatternsFound ; } else { auto SDWASrc = make_unique < SDWASrcOperand > ( Src1 , Dst , BYTE_1 , false , false , Opcode != <STR_LIT> && Opcode != <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWASrc << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWASrc ) ; ++ NumSDWAPatternsFound ; } break ; } case <STR_LIT> : case <STR_LIT> : { MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> ::


<s> } else if ( ( Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ) ) { assert ( Dst && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: sdst ) != - <NUM_LIT> ) ; SDWAInst . add ( * Dst ) ; } else { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: sdst ) != - <NUM_LIT> ) ; SDWAInst . addReg ( <STR_LIT> , RegState :: Define ) ; } MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; assert ( Src0 && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ) ; if ( auto * Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ) SDWAInst . addImm ( Mod -> getImm ( ) ) ; else SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src0 ) ; MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_modifiers ) != - <NUM_LIT> ) ; if ( auto * Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ) SDWAInst . addImm ( Mod -> getImm ( ) ) ; else SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src1 ) ; } if ( SDWAOpcode == <STR_LIT> || SDWAOpcode == <STR_LIT> ) { MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; assert ( Src2 ) ; SDWAInst . add ( * Src2 ) ; } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: clamp ) != - <NUM_LIT> ) ; MachineOperand * Clamp = TII -> getNamedOperand ( MI , <STR_LIT> :: clamp ) ; if ( Clamp ) { SDWAInst . add ( * Clamp ) ; } else { SDWAInst . addImm ( <NUM_LIT> ) ; } if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: omod ) != - <NUM_LIT> ) { MachineOperand * OMod = TII -> getNamedOperand ( MI , <STR_LIT> :: omod ) ; if ( OMod


<s> const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ; } if ( ! ST . hasSDWAOutModsVOPC ( ) && ( TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) ) return false ; } else if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) || ! TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) { return false ; } if ( ! ST . hasSDWAMac ( ) && ( Opc

== <STR_LIT> || Opc == <STR_LIT> ) ) return false ;


<s> if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ; } if ( ! ST . hasSDWAOutModsVOPC ( ) && ( TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) ) return false ; } else if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) || ! TII -> getNamedOperand ( MI , <STR_LIT>


<s> TRI = ST . getRegisterInfo ( ) ; TII = ST . getInstrInfo ( ) ; bool Ret = false ; for ( MachineBasicBlock & MBB : MF ) { bool Changed = false ; do { matchSDWAOperands ( MBB ) ; for ( const auto & OperandPair : SDWAOperands ) { const auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI && isConvertibleToSDWA ( * PotentialMI , ST ) ) { PotentialMatches [ PotentialMI ] . push_back ( Operand . get ( ) ) ; } } for ( auto & PotentialPair : PotentialMatches ) { MachineInstr & PotentialMI = * PotentialPair . first ; convertToSDWA ( PotentialMI , PotentialPair . second ) ; } PotentialMatches . clear ( ) ; SDWAOperands . clear (


<s> Mods = Mod -> getImm ( ) ; } } else if ( TII -> getNamedOperand ( * MI , <STR_LIT> :: src1 ) == SrcOp ) { if ( auto * Mod = TII -> getNamedOperand ( * MI , <STR_LIT> :: src1_modifiers ) ) { Mods = Mod -> getImm ( ) ; } } if ( Abs || Neg ) { assert ( ! Sext && <STR_LIT> ) ; Mods |= Abs ? <STR_LIT> : <NUM_LIT> ; Mods ^= Neg ? <STR_LIT> : <NUM_LIT> ; } else if ( Sext ) { Mods |=


<s> if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) Opc = <STR_LIT> ( Opc ) ; if ( Opc == - <NUM_LIT> || <STR_LIT> ( Opc ) == - <NUM_LIT> ) return false ; if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT>


<s> if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ; } if ( ! ST . hasSDWAClampVOPC ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) ) return false ; } else if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ) { return false ; } if ( ! ST . hasSDWAMac ( ) && ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ) return false ; return true


<s> if ( SuperReg . getReg ( ) != SubReg . getReg ( ) ) return false ; LaneBitmask SuperMask = TRI -> getSubRegIndexLaneMask ( SuperReg . getSubReg ( ) ) ; LaneBitmask SubMask = TRI -> getSubRegIndexLaneMask ( SubReg . getSubReg ( ) ) ; SuperMask |= ~


<s> static bool isSubregOf ( const MachineOperand & SubReg , const MachineOperand & SuperReg , const TargetRegisterInfo * TRI ) { if ( ! SuperReg . isReg ( ) || ! SubReg . isReg ( ) ) return false ; if ( isSameReg ( SuperReg , SubReg ) ) return true ; if ( SuperReg . getReg ( ) != SubReg . getReg ( ) ) return false ; LaneBitmask SuperMask = TRI -> getSubRegIndexLaneMask ( SuperReg . getSubReg ( ) ) ; LaneBitmask SubMask = TRI -> getSubRegIndexLaneMask ( SubReg . getSubReg ( ) ) ; SuperMask |= ~ SubMask ; return SuperMask .


<s> if ( ! isSubregOf ( * Replaced , PotentialMO , MRI -> getTargetRegisterInfo ( ) ) ) continue ; if ( ! isSameReg ( * Replaced , PotentialMO ) ) return nullptr ; for ( MachineOperand & UseMO : MRI -> use_operands ( PotentialMO . getReg ( ) ) ) { if ( isSubregOf ( UseMO , PotentialMO , MRI -> getTargetRegisterInfo ( ) ) && UseMO . getParent ( ) != ParentMI ) { return


<s> const auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI && isConvertibleToSDWA ( * PotentialMI , ST ) ) { PotentialMatches [ PotentialMI ] . push_back ( Operand . get ( ) ) ; } } for ( auto & PotentialPair : PotentialMatches ) { MachineInstr & PotentialMI = * PotentialPair . first ; convertToSDWA ( PotentialMI , PotentialPair . second ) ; } PotentialMatches . clear (


<s> if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_sel ) != - <NUM_LIT> ) { MachineOperand * DstSel = TII -> getNamedOperand ( MI , <STR_LIT> :: dst_sel ) ; if ( DstSel ) { SDWAInst . add ( * DstSel ) ; } else { SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; } } if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_unused ) != - <NUM_LIT> ) { MachineOperand * DstUnused = TII -> getNamedOperand ( MI , <STR_LIT> :: dst_unused ) ; if ( DstUnused ) { SDWAInst . add ( * DstUnused ) ; } else { SDWAInst . addImm ( <STR_LIT> :: DstUnused :: UNUSED_PAD ) ; } } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_sel ) != - <NUM_LIT> ) ; MachineOperand * Src0Sel = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_sel ) ; if ( Src0Sel ) { SDWAInst . add ( * Src0Sel ) ; } else { SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; } if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_sel ) != - <NUM_LIT> ) ; MachineOperand * Src1Sel = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_sel ) ; if ( Src1Sel ) { SDWAInst . add ( * Src1Sel ) ; } else { SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; } } auto DstUnused = TII -> getNamedOperand ( MI , <STR_LIT> :: dst_unused ) ; if ( DstUnused && DstUnused -> getImm ( ) == <STR_LIT> :: DstUnused :: UNUSED_PRESERVE ) { assert ( Dst && Dst -> isTied ( ) ) ; assert ( Opcode == static_cast < unsigned int > ( SDWAOpcode ) ) ; auto PreserveDstIdx = <STR_LIT> ( SDWAOpcode , <STR_LIT> :: vdst ) ; assert ( PreserveDstIdx != - <NUM_LIT> ) ; auto TiedIdx = MI . findTiedOperandIdx ( PreserveDstIdx ) ; auto Tied = MI . getOperand ( TiedIdx ) ; SDWAInst . add ( Tied ) ; SDWAInst -> tieOperands ( PreserveDstIdx , SDWAInst -> getNumOperands ( ) - <NUM_LIT> ) ; } bool Converted = false ; for ( auto & Operand : SDWAOperands ) { DEBUG ( dbgs ( ) << * SDWAInst << <STR_LIT> << * Operand ) ; if ( PotentialMatches . count ( Operand -> getParentInst ( ) ) == <NUM_LIT> ) Converted |= Operand -> convertToSDWA ( * SDWAInst , TII ) ; } if ( Converted


<s> int SDWAOpcode = <STR_LIT> ( MI . getOpcode ( ) ) ; assert ( SDWAOpcode != - <NUM_LIT> ) ; const MCInstrDesc & SDWADesc = TII -> get ( SDWAOpcode ) ; MachineInstrBuilder SDWAInst = BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , SDWADesc ) ; MachineOperand * Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ; if ( Dst ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: vdst ) != - <NUM_LIT> ) ; SDWAInst . add ( * Dst ) ; } else { assert ( TII -> isVOPC ( MI ) ) ; } MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; assert ( Src0 && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src0 ) ; MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_modifiers ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src1 ) ; } else { assert ( TII -> isVOP1 ( MI ) ) ; } if ( SDWAOpcode == <STR_LIT> || SDWAOpcode == <STR_LIT> ) { MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; assert ( Src2 ) ; SDWAInst . add ( * Src2 ) ; } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: clamp ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <NUM_LIT> ) ; if ( Dst ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_sel ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_unused ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; SDWAInst . addImm ( <STR_LIT> :: DstUnused :: UNUSED_PAD ) ; } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_sel ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_sel ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <STR_LIT> :: SdwaSel ::


<s> MachineInstrBuilder SDWAInst = BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , SDWADesc ) ; MachineOperand * Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ; if ( Dst ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: vdst ) != - <NUM_LIT> ) ; SDWAInst . add ( * Dst ) ; } else { assert ( TII -> isVOPC ( MI ) ) ; } MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; assert ( Src0 && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src0 ) ; MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_modifiers ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src1 ) ; } else { assert ( TII -> isVOP1 ( MI ) ) ; } if ( SDWAOpcode == <STR_LIT> || SDWAOpcode == <STR_LIT> ) { MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; assert ( Src2 ) ; SDWAInst . add ( * Src2 ) ; } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: clamp ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <NUM_LIT> ) ; if ( Dst ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_sel ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_unused


<s> uint64_t Mods = <NUM_LIT> ; if ( Abs || Neg ) { assert ( ! Sext && <STR_LIT> ) ; Mods |= Abs ? <STR_LIT> : <NUM_LIT> ; Mods |= Neg ? <STR_LIT> : <NUM_LIT> ; } else if ( Sext ) { Mods |= <STR_LIT>


<s> else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = WORD_0 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = DWORD ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = BYTE_1 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = BYTE_2 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = WORD_1 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = BYTE_3 ; else break ; MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; MachineOperand * Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ; if ( TRI -> isPhysicalRegister ( Src0 -> getReg ( ) ) || TRI -> isPhysicalRegister ( Dst -> getReg ( ) ) ) break ; auto SDWASrc = make_unique < SDWASrcOperand > ( Src0 , Dst , SrcSel , false , false , Opcode == <STR_LIT> ? false : true ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWASrc << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWASrc ) ; ++ NumSDWAPatternsFound ; break ; } case <STR_LIT> : { MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; auto Imm = foldToImm ( * Src0 ) ; if ( ! Imm ) break ; if ( * Imm != <NUM_LIT> && * Imm != <NUM_LIT> ) break ; MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; MachineOperand * Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ; if ( TRI -> isPhysicalRegister ( Src1 -> getReg ( ) ) || TRI -> isPhysicalRegister ( Dst -> getReg ( ) ) ) break ; auto SDWASrc = make_unique < SDWASrcOperand > ( Src1 , Dst , *


<s> auto SDWADst = make_unique < SDWADstOperand > ( Dst , Src1 , BYTE_1 , UNUSED_PAD ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWADst << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWADst ) ; ++ NumSDWAPatternsFound ; } else { auto SDWASrc = make_unique < SDWASrcOperand > ( Src1 , Dst , BYTE_1 , false , false , Opcode == <STR_LIT> ? false : true ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWASrc << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWASrc ) ; ++ NumSDWAPatternsFound ; } break ; } case <STR_LIT> : case <STR_LIT> : { MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; auto Offset = foldToImm ( * Src1 ) ; if ( ! Offset ) break ; MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; auto Width = foldToImm ( * Src2 ) ; if ( ! Width ) break ; SdwaSel SrcSel = DWORD ; if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = BYTE_0 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = WORD_0 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = DWORD ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = BYTE_1 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = BYTE_2 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = WORD_1 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = BYTE_3 ; else break ; MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; MachineOperand * Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ; if ( TRI -> isPhysicalRegister ( Src0 -> getReg ( ) ) || TRI -> isPhysicalRegister ( Dst -> getReg ( ) ) ) break ; auto SDWASrc = make_unique < SDWASrcOperand > ( Src0 , Dst , SrcSel , false , false , Opcode == <STR_LIT> ? false : true ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWASrc << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWASrc ) ; ++ NumSDWAPatternsFound ; break ; } case <STR_LIT> : { MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> ::


<s> TRI = ST . getRegisterInfo ( ) ; TII = ST . getInstrInfo ( ) ; std :: unordered_map < MachineInstr * , SDWAOperandsVector > PotentialMatches ; matchSDWAOperands ( MF ) ; for ( auto & OperandPair : SDWAOperands ) { auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI ) { PotentialMatches [ PotentialMI ] . push_back ( std


<s> void <ISA_LIT>PeepholeSDWA :: matchSDWAOperands ( MachineFunction & MF ) { for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : MBB ) { if ( auto Operand = matchSDWAOperand


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ! ST . hasSDWA ( ) || skipFunction ( MF . getFunction ( ) ) ) return false ; MRI = & MF . getRegInfo ( ) ; TRI = ST . getRegisterInfo ( ) ; TII = ST . getInstrInfo ( ) ; bool Changed = false ; bool Ret = false ; do { matchSDWAOperands ( MF ) ; for ( const auto & OperandPair : SDWAOperands ) { const auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI && isConvertibleToSDWA ( * PotentialMI , ST ) ) { PotentialMatches [ PotentialMI ] . push_back ( Operand . get ( )


<s> bool <ISA_LIT>PeepholeSDWA :: isConvertibleToSDWA ( MachineInstr & MI , const GCNSubtarget & ST ) const { unsigned Opc = MI . getOpcode ( ) ; if ( TII -> isSDWA ( Opc ) ) return true ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) Opc = <STR_LIT> ( Opc ) ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) return false ; if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ; } if ( ! ST . hasSDWAOutModsVOPC ( ) && ( TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) ) return false ; } else if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) || ! TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) { return false ; } if ( ! ST . hasSDWAMac ( ) && ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) ) return false ; if ( TII -> pseudoToMCOpcode ( Opc ) == - <NUM_LIT> ) return false ; if ( Opc == <STR_LIT>


<s> const MCInstrDesc & Desc = TII -> get ( MI . getOpcode ( ) ) ; unsigned ConstantBusCount = <NUM_LIT> ; for ( MachineOperand & Op : MI . explicit_uses ( ) ) { if ( ! Op . isImm ( ) && ! ( Op . isReg ( ) && ! TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) ) continue ; unsigned I = MI . getOperandNo ( & Op ) ; if ( Desc . OpInfo [ I ] . RegClass == - <NUM_LIT> || ! TRI -> hasVGPRs ( TRI -> getRegClass ( Desc . OpInfo [ I ] . RegClass ) ) ) continue ; if ( ST . hasSDWAScalar ( ) && ConstantBusCount == <NUM_LIT> && Op . isReg ( ) && TRI -> isSGPRReg ( * MRI


<s> if ( Op . isReg ( ) ) { for ( const MachineOperand & Def : MRI -> def_operands ( Op . getReg ( ) ) ) { if ( ! isSameReg ( Op , Def ) ) continue ; const MachineInstr * DefInst = Def . getParent


<s> static bool isSameBB ( const MachineInstr * FirstMI , const MachineInstr


<s> MachineInstr * SDWADstOperand :: potentialToConvert ( const <ISA_LIT>InstrInfo * TII ) { MachineRegisterInfo * MRI = getMRI ( ) ; MachineInstr * ParentMI = getParentInst ( ) ; MachineOperand * Replaced = getReplacedOperand ( ) ; assert ( Replaced -> isReg ( ) ) ; for ( MachineOperand & PotentialMO : MRI -> def_operands ( Replaced -> getReg ( ) ) ) { if ( ! isSubregOf ( * Replaced , PotentialMO , MRI -> getTargetRegisterInfo ( ) ) ) continue ; if ( ! isSameBB ( getParentInst ( ) , PotentialMO . getParent ( ) ) || ! isSameReg ( * Replaced , PotentialMO ) ) return nullptr ; for ( MachineOperand & UseMO : MRI -> use_operands ( PotentialMO . getReg ( ) ) ) { if ( isSubregOf ( UseMO , PotentialMO , MRI -> getTargetRegisterInfo ( ) ) && UseMO . getParent ( ) != ParentMI ) { return


<s> SDWAOpcode = <STR_LIT> ( Opcode ) ; if ( SDWAOpcode == - <NUM_LIT> ) SDWAOpcode = <STR_LIT> ( <STR_LIT> ( Opcode ) ) ; } assert ( SDWAOpcode != - <NUM_LIT> ) ; const MCInstrDesc & SDWADesc = TII -> get ( SDWAOpcode ) ; MachineInstrBuilder SDWAInst = BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , SDWADesc ) ; MachineOperand * Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ; if ( Dst ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: vdst ) != - <NUM_LIT> ) ; SDWAInst . add ( * Dst ) ; } else if ( ( Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ) ) { assert ( Dst && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: sdst ) != - <NUM_LIT> ) ; SDWAInst . add ( * Dst ) ; } else { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: sdst ) != - <NUM_LIT> ) ; SDWAInst . addReg ( <STR_LIT> , RegState :: Define ) ; } MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; assert ( Src0 && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ) ; if ( auto * Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ) SDWAInst . addImm ( Mod -> getImm ( ) ) ; else SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src0 ) ; MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_modifiers ) != - <NUM_LIT> ) ; if ( auto * Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ) SDWAInst . addImm ( Mod -> getImm ( ) ) ; else SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src1 ) ; } if ( SDWAOpcode == <STR_LIT> || SDWAOpcode == <STR_LIT> ) { MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; assert ( Src2 ) ; SDWAInst . add ( * Src2 ) ; } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: clamp ) != - <NUM_LIT> ) ; MachineOperand * Clamp = TII -> getNamedOperand ( MI , <STR_LIT> :: clamp ) ; if ( Clamp ) { SDWAInst . add ( * Clamp ) ; } else { SDWAInst . addImm ( <NUM_LIT> ) ; } if ( <STR_LIT> ( SDWAOpcode , <STR_LIT>


<s> MachineOperand & Op = MI . getOperand ( I ) ; if ( ! Op . isImm ( ) && ! ( Op . isReg ( ) && ! TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) ) continue ; if ( Desc . OpInfo [ I ] . RegClass == - <NUM_LIT> || ! TRI -> hasVGPRs ( TRI -> getRegClass ( Desc . OpInfo [ I ] . RegClass ) ) ) continue ; unsigned VGPR = MRI -> createVirtualRegister ( & <STR_LIT> ) ; auto Copy = BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , VGPR ) ; if ( Op . isImm ( ) ) Copy . addImm ( Op . getImm ( ) ) ; else if ( Op . isReg ( ) ) Copy . addReg ( Op . getReg ( ) , Op . isKill ( ) ? RegState :: Kill : <NUM_LIT> , Op .


<s> bool <ISA_LIT>PeepholeSDWA :: runOnMachineFunction ( MachineFunction & MF ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ! ST . hasSDWA ( ) || ! <STR_LIT> ( ST ) ) { return false ; } MRI = & MF . getRegInfo ( ) ; TRI = ST . getRegisterInfo ( ) ; TII = ST . getInstrInfo ( ) ; matchSDWAOperands ( MF


<s> if ( ! NextOp ) return ; MachineInstr & MISucc = * NextOp -> getParent ( ) ; if ( ! TII -> canShrink ( MISucc , * MRI ) ) return ; int SuccOpc = <STR_LIT> ( MISucc . getOpcode ( ) ) ; MachineOperand * CarryIn = TII -> getNamedOperand ( MISucc , <STR_LIT> :: src2 ) ; if ( ! CarryIn ) return ; MachineOperand * CarryOut = TII -> getNamedOperand ( MISucc , <STR_LIT> :: sdst ) ; if ( ! CarryOut ) return ; if ( ! MRI -> hasOneUse ( CarryIn -> getReg ( ) ) || ! MRI -> use_empty ( CarryOut -> getReg ( ) ) ) return ; MachineBasicBlock & MBB = * MI . getParent ( ) ; auto Liveness = MBB . computeRegisterLiveness ( TRI , <STR_LIT> , MI , <NUM_LIT> ) ; if ( Liveness != MachineBasicBlock :: LQR_Dead ) return ; for ( auto I = std :: next ( MI . getIterator ( ) ) , E = MISucc . getIterator ( ) ; I != E ; ++ I ) { if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return ; } BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ) . setMIFlags ( MI . getFlags ( ) ) ; MI . eraseFromParent ( ) ; BuildMI ( MBB , MISucc , MISucc . getDebugLoc ( ) , TII -> get ( SuccOpc ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: vdst ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src0 ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src1 ) ) . setMIFlags ( MISucc . getFlags ( ) ) ; MISucc . eraseFromParent


<s> bool <ISA_LIT>PeepholeSDWA :: isConvertibleToSDWA ( const MachineInstr & MI , const GCNSubtarget & ST ) const { unsigned Opc = MI . getOpcode ( ) ; if ( TII -> isSDWA ( Opc ) ) return true ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) Opc = <STR_LIT> ( Opc ) ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) return false ; if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ; } if ( ! ST . hasSDWAOutModsVOPC ( ) && ( TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) ) return false ; } else if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) || ! TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) { return false ; } if ( ! ST . hasSDWAMac ( ) && ( Opc == <STR_LIT> || Opc == <STR_LIT> )


<s> matchSDWAOperands ( MBB ) ; for ( const auto & OperandPair : SDWAOperands ) { const auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI && isConvertibleToSDWA ( * PotentialMI , ST ) ) { PotentialMatches [ PotentialMI ] . push_back ( Operand . get ( ) ) ; } } for ( auto & PotentialPair : PotentialMatches ) { MachineInstr & PotentialMI = * PotentialPair . first ; convertToSDWA ( PotentialMI , PotentialPair . second ) ; } PotentialMatches . clear ( ) ; SDWAOperands . clear ( ) ; Changed = ! ConvertedInstructions . empty (


<s> for ( MachineOperand & Op : MI . explicit_uses ( ) ) { if ( ! Op . isImm ( ) && ! ( Op . isReg ( ) && ! TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) ) continue ; unsigned I = MI . getOperandNo ( & Op ) ; if ( Desc . OpInfo [ I ] . RegClass == - <NUM_LIT> || ! TRI -> isVGPRClass ( TRI -> getRegClass ( Desc . OpInfo [ I ] . RegClass ) ) ) continue ; if ( ST . hasSDWAScalar ( ) && ConstantBusCount == <NUM_LIT> && Op . isReg ( ) && TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { ++ ConstantBusCount ; continue ; } Register VGPR = MRI -> createVirtualRegister ( & <STR_LIT> ) ; auto Copy = BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , VGPR ) ; if ( Op . isImm ( ) ) Copy . addImm ( Op . getImm ( ) ) ; else if ( Op . isReg ( ) ) Copy . addReg ( Op . getReg ( ) , Op . isKill ( )

? RegState :: Kill : <NUM_LIT> , Op . getSubReg ( ) ) ;


<s> if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ; } if ( ! ST . hasSDWAOutModsVOPC ( ) && ( TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) ) return false ; } else if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) || ! TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) { return false ; } if ( ! ST . hasSDWAMac ( ) && ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ) return false ; if ( Opc == <STR_LIT> ) return false ; return


<s> if ( DstSel ) { SDWAInst . add ( * DstSel ) ; } else { SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; } } if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_unused ) != - <NUM_LIT> ) { MachineOperand * DstUnused = TII -> getNamedOperand ( MI , <STR_LIT> :: dst_unused ) ; if ( DstUnused ) { SDWAInst . add ( * DstUnused ) ; } else { SDWAInst . addImm ( <STR_LIT> :: DstUnused :: UNUSED_PAD ) ; } } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_sel ) != - <NUM_LIT> ) ; MachineOperand * Src0Sel = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_sel ) ; if ( Src0Sel ) { SDWAInst . add ( * Src0Sel ) ; } else { SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; } if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_sel ) != - <NUM_LIT> ) ; MachineOperand * Src1Sel = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_sel ) ; if ( Src1Sel ) { SDWAInst . add ( * Src1Sel ) ; } else { SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; } } auto DstUnused = TII -> getNamedOperand ( MI , <STR_LIT> :: dst_unused ) ; if ( DstUnused && DstUnused -> getImm ( ) == <STR_LIT> :: DstUnused :: UNUSED_PRESERVE ) { assert ( Dst && Dst -> isTied ( ) ) ; assert ( Opcode == static_cast < unsigned int > ( SDWAOpcode ) ) ; auto PreserveDstIdx = <STR_LIT> ( SDWAOpcode , <STR_LIT> :: vdst ) ; assert ( PreserveDstIdx != - <NUM_LIT> ) ; auto TiedIdx = MI . findTiedOperandIdx ( PreserveDstIdx ) ; auto Tied = MI . getOperand ( TiedIdx ) ; SDWAInst . add ( Tied ) ; SDWAInst -> tieOperands ( PreserveDstIdx , SDWAInst -> getNumOperands ( ) - <NUM_LIT> ) ; } bool Converted = false ; for ( auto & Operand : SDWAOperands ) { LLVM_DEBUG ( dbgs ( ) << * SDWAInst << <STR_LIT> << * Operand ) ; if ( PotentialMatches . count ( Operand -> getParentInst ( ) ) == <NUM_LIT> ) Converted |= Operand -> convertToSDWA ( * SDWAInst , TII ) ; } if ( Converted ) { ConvertedInstructions . push_back ( SDWAInst ) ; } else


<s> To . setSubReg ( From . getSubReg ( ) ) ; To . setIsUndef ( From . isUndef ( ) ) ; if ( To . isUse ( ) ) { To . setIsKill ( From . isKill ( ) ) ; } else { To . setIsDead ( From . isDead (


<s> if ( ! DefInstr ) return nullptr ; for ( auto & DefMO : DefInstr -> defs ( ) ) { if ( DefMO . isReg ( ) && DefMO . getReg ( ) == Reg -> getReg ( ) ) return & DefMO ; } return nullptr


<s> if ( ! Reg -> isReg ( ) || ! Reg -> isDef ( ) ) return nullptr ; MachineOperand * ResMO = nullptr ; for ( MachineOperand & UseMO : MRI -> use_nodbg_operands ( Reg -> getReg ( ) ) ) { if ( ! isSameReg ( UseMO , * Reg ) ) return nullptr ; if ( ! ResMO ) { ResMO = &


<s> MachineOperand * getPreservedOperand ( ) const { return Preserve


<s> MachineOperand * getReplacedOperand ( ) const { return Replaced


<s> bool getSext ( ) const { return Sext


<s> uint64_t Mods = <NUM_LIT> ; const auto * MI = SrcOp -> getParent ( ) ; if ( TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) == SrcOp ) { if ( auto * Mod = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0_modifiers ) ) { Mods = Mod -> getImm ( ) ; } } else if ( TII -> getNamedOperand ( * MI , <STR_LIT> :: src1 ) == SrcOp ) { if ( auto * Mod = TII -> getNamedOperand ( * MI , <STR_LIT> :: src1_modifiers ) ) { Mods = Mod -> getImm ( ) ; } } if ( Abs || Neg ) { assert ( ! Sext && <STR_LIT> ) ; Mods |= Abs ? <STR_LIT> : <NUM_LIT> ; Mods ^= Neg ? <STR_LIT>


<s> MachineOperand * getTargetOperand ( ) const { return Target


<s> bool <ISA_LIT>PeepholeSDWA :: isConvertibleToSDWA ( MachineInstr & MI , const GCNSubtarget & ST ) const { unsigned Opc = MI . getOpcode ( ) ; if ( TII -> isSDWA ( Opc ) ) return true ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) Opc = <STR_LIT> ( Opc ) ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) return false ; if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && ( SDst -> getReg ( ) != <STR_LIT> && SDst -> getReg ( ) != <STR_LIT> ) ) return false ; } if ( ! ST . hasSDWAOutModsVOPC ( ) && ( TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) ) return false ; } else if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) || ! TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) { return false ; } if ( ! ST . hasSDWAMac ( ) && ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) ) return false ; if ( TII -> pseudoToMCOpcode ( Opc ) == - <NUM_LIT> ) return false ; if ( Opc == <STR_LIT> )


<s> return LHS . isReg ( ) && RHS . isReg ( ) && LHS . getReg ( ) == RHS . getReg (

) && LHS . getSubReg ( ) == RHS . getSubReg ( ) ;


<s> if ( ! Op . isImm ( ) && ! ( Op . isReg ( ) && ! TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) ) continue ; unsigned I = MI . getOperandNo ( & Op ) ; if ( Desc . OpInfo [ I ] . RegClass == - <NUM_LIT> || ! TRI -> isVSSuperClass ( TRI -> getRegClass ( Desc . OpInfo [ I ] . RegClass ) ) ) continue ; if ( ST . hasSDWAScalar ( ) && ConstantBusCount == <NUM_LIT> && Op . isReg ( ) && TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) { ++ ConstantBusCount ; continue ; } Register VGPR = MRI -> createVirtualRegister ( & <STR_LIT> ) ; auto Copy = BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT>


<s> if ( ! Op . isImm ( ) && ! ( Op . isReg ( ) && ! TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) ) continue ; unsigned I = MI . getOperandNo ( & Op ) ; if ( Desc . OpInfo [ I ] . RegClass == - <NUM_LIT> || ! TRI -> isVSSuperClass ( TRI -> getRegClass ( Desc . OpInfo [ I ] . RegClass ) ) ) continue ; if ( ST . hasSDWAScalar ( ) && ConstantBusCount == <NUM_LIT> && Op . isReg ( ) && TRI -> isSGPRReg ( * MRI , Op .


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * Operand << '\n' ) ; SDWAOperands [ & MI ] = std


<s> OS << <STR_LIT> << * getTargetOperand ( ) << <STR_LIT> << getDstSel ( ) <<

<STR_LIT> << * getPreservedOperand ( ) << '\n' ;


<s> if ( Liveness != MachineBasicBlock :: LQR_Dead ) return ; for ( auto I = std :: next ( MI . getIterator ( ) ) , E = MISucc . getIterator ( ) ; I != E ; ++ I ) { if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return ; } BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ) . setMIFlags ( MI . getFlags ( ) ) ; MI . eraseFromParent ( ) ; BuildMI ( MBB , MISucc , MISucc . getDebugLoc ( ) , TII -> get ( SuccOpc ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: vdst ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src0 ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src1 ) ) . setMIFlags


<s> if ( Liveness != MachineBasicBlock :: LQR_Dead ) return ; for ( auto I = std :: next ( MI . getIterator ( ) ) , E = MISucc . getIterator ( ) ; I != E ; ++ I ) { if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return ; } BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ) . setMIFlags ( MI . getFlags ( ) ) ; MI . eraseFromParent ( ) ; BuildMI ( MBB , MISucc , MISucc . getDebugLoc ( ) , TII -> get ( SuccOpc ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: vdst ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src0 ) ) . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src1 ) ) . setMIFlags ( MISucc . getFlags ( ) ) ; MISucc . eraseFromParent (


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; if ( ! ST . hasSDWA ( ) || skipFunction ( MF . getFunction ( ) ) ) return false ; MRI = & MF . getRegInfo ( ) ; TRI = ST . getRegisterInfo ( ) ; TII = ST . getInstrInfo ( ) ; bool Ret = false ; for ( MachineBasicBlock & MBB : MF ) { bool Changed = false ; do { matchSDWAOperands ( MBB ) ; for ( const auto & OperandPair : SDWAOperands ) { const auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI && ( PotentialMI -> getOpcode ( ) == <STR_LIT> || PotentialMI -> getOpcode ( ) == <STR_LIT> ) ) pseudoOpConvertToVOP2 ( * PotentialMI , ST ) ; } SDWAOperands . clear ( ) ; matchSDWAOperands ( MBB ) ; for ( const auto & OperandPair : SDWAOperands ) { const auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI && isConvertibleToSDWA ( * PotentialMI , ST ) ) { PotentialMatches [ PotentialMI ] . push_back ( Operand . get ( ) ) ; } } for ( auto & PotentialPair : PotentialMatches ) { MachineInstr & PotentialMI = * PotentialPair . first ; convertToSDWA ( PotentialMI , PotentialPair . second ) ; } PotentialMatches . clear ( ) ; SDWAOperands . clear (


<s> SDWAOperand ( MachineOperand * TargetOp , MachineOperand * ReplacedOp ) : Target ( TargetOp ) , Replaced ( ReplacedOp ) { assert ( Target -> isReg ( )


<s> initialize<ISA_LIT>PeepholeSDWAPass ( * PassRegistry :: getPassRegistry (


<s> Register Reg = Op . getReg ( ) ; assert ( ! Op . getSubReg ( ) && <STR_LIT> ) ; for ( MCRegUnitIterator Units ( Reg , TRI ) ; Units . isValid ( ) ; ++ Units ) UsedRegUnits . set


<s> const uint64_t IMemFlags = MI . getDesc ( ) .


<s> if ( ! MI . mayLoad ( ) ) return false ; for ( const MachineOperand & Op : MI . explicit_operands ( ) ) { if ( ! Op . isReg ( ) ) continue ; Register Reg = Op . getReg


<s> return new <ISA_LIT>PostRABundler ( )


<s> StringRef getPassName ( ) const override { return <STR_LIT>


<s> for ( const MachineOperand & Op : MI . explicit_operands ( ) ) { if ( ! Op . isReg ( ) ) continue ; Register Reg = Op . getReg ( ) ; for ( Register Def : Defs ) if ( TRI -> regsOverlap ( Reg , Def ) ) return true ; } return false


<s> MachineBasicBlock :: instr_iterator B = MBB . instr_begin ( ) ; MachineBasicBlock :: instr_iterator E = MBB . instr_end ( ) ; for ( auto I = B ; I != E ; I = Next ) { Next = std :: next ( I ) ; if ( I -> isBundled ( ) || ! I -> mayLoadOrStore ( ) || B -> mayLoad ( ) != I -> mayLoad ( ) || B -> mayStore ( ) != I -> mayStore ( ) || ( B -> getDesc ( ) . TSFlags & MemFlags ) != ( I -> getDesc ( ) . TSFlags & MemFlags ) || isDependentLoad ( * I ) ) { if ( B != I ) { if ( std :: next ( B ) != I ) { finalizeBundle ( MBB , B , I ) ; Changed = true


<s> initialize<ISA_LIT>PostRABundlerPass ( * PassRegistry ::


<s> if ( ! MO . isReg ( ) ) return false ; Register Reg = MO . getReg ( ) ; if ( Reg . isPhysical ( ) ) return false ; if ( ! TRI -> isVGPR ( * MRI , Reg ) ) return false ; if ( VRM -> hasPhys ( Reg ) ) return false ; LiveInterval & LI = LIS -> getInterval ( Reg ) ; for ( MCRegister PhysReg : RegClassInfo . getOrder ( MRI -> getRegClass ( Reg ) ) ) { if ( ! MRI -> isPhysRegUsed ( PhysReg ) && Matrix -> checkInterference ( LI , PhysReg ) == LiveRegMatrix :: IK_Free ) { Matrix -> assign ( LI , PhysReg


<s> LiveInterval & LI = LIS -> getInterval ( Reg ) ; for ( MCRegister PhysReg : RegClassInfo . getOrder ( MRI -> getRegClass ( Reg ) ) ) { if ( ! MRI -> isPhysRegUsed ( PhysReg ) && Matrix -> checkInterference ( LI , PhysReg ) == LiveRegMatrix :: IK_Free ) { Matrix -> assign ( LI , PhysReg ) ; assert ( PhysReg != <NUM_LIT> ) ; RegsToRewrite . push_back ( Reg ) ; return true ; } } llvm_unreachable ( <STR_LIT>


<s> unsigned Opc = MI . getOpcode ( ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { dbgs ( ) << <STR_LIT> ; } else { assert ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ; dbgs ( ) << <STR_LIT> ; } if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) { dbgs ( )


<s> if ( ! VRM -> hasPhys ( VirtReg ) ) continue ; Register PhysReg = VRM -> getPhys ( VirtReg ) ; const unsigned SubReg = MO . getSubReg ( ) ; if ( SubReg != <NUM_LIT> ) { PhysReg = TRI -> getSubReg ( PhysReg , SubReg ) ; MO . setSubReg ( <NUM_LIT> ) ; } MO . setReg ( PhysReg ) ; MO . setIsRenamable ( false ) ; } } } <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; for ( unsigned Reg : RegsToRewrite ) { LIS -> removeInterval ( Reg ) ; const Register PhysReg = VRM -> getPhys ( Reg ) ; assert ( PhysReg != <NUM_LIT> ) ; if ( ! MFI -> WWMReservedRegs . count ( PhysReg ) ) { Optional < int > FI ; if ( ! MFI -> isEntryFunction ( ) ) { const TargetRegisterClass * RC = TRI -> getPhysRegClass ( PhysReg ) ; FI = FrameInfo . CreateSpillStackObject ( TRI -> getSpillSize ( * RC ) , TRI -> getSpillAlign ( * RC ) ) ; } MFI -> reserveWWMRegister ( PhysReg , FI ) ; } } RegsToRewrite . clear


<s> RegClassInfo . runOnMachineFunction ( MF ) ; bool RegsAssigned = false ; ReversePostOrderTraversal < MachineFunction * > RPOT ( & MF ) ; for ( MachineBasicBlock * MBB : RPOT ) { bool InWWM = false ; for ( MachineInstr & MI : * MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) RegsAssigned |= processDef ( MI . getOperand ( <NUM_LIT> ) ) ; if ( MI . getOpcode ( ) == <STR_LIT> ||


<s> if ( ! TRI -> isVGPR ( * MRI , Reg ) ) return false ; if ( VRM -> hasPhys ( Reg ) ) return false ; LiveInterval & LI = LIS -> getInterval ( Reg ) ; for ( MCRegister PhysReg : RegClassInfo . getOrder ( MRI -> getRegClass ( Reg ) ) ) { if ( ! MRI -> isPhysRegUsed ( PhysReg ) && Matrix -> checkInterference ( LI , PhysReg ) == LiveRegMatrix :: IK_Free ) { Matrix -> assign ( LI , PhysReg ) ; assert ( PhysReg != <NUM_LIT> ) ; RegsToRewrite . push_back (


<s> if ( Reg . isPhysical ( ) ) return false ; if ( VRM -> hasPhys ( Reg ) ) return false ; LiveInterval & LI = LIS -> getInterval ( Reg ) ; for ( unsigned PhysReg : RegClassInfo . getOrder ( MRI -> getRegClass ( Reg ) ) ) { if ( ! MRI -> isPhysRegUsed ( PhysReg ) && Matrix -> checkInterference ( LI , PhysReg ) == LiveRegMatrix :: IK_Free ) { Matrix -> assign ( LI , PhysReg ) ; assert ( PhysReg != <NUM_LIT> ) ; RegsToRewrite . push_back ( Reg ) ; return true ; } } llvm_unreachable ( <STR_LIT> ) ; return false


<s> FunctionPass * llvm :: create<ISA_LIT>PreAllocateWWMRegsPass ( ) { return new <ISA_LIT>PreAllocateWWMRegs ( )


<s> FunctionPass * llvm :: create<ISA_LIT>PreAllocateWWMRegsPass ( ) { return new <ISA_LIT>PreAllocateWWMRegs


<s> AU . addRequired < VirtRegMap > ( ) ; AU . addRequired < LiveRegMatrix > ( ) ; AU . addPreserved < SlotIndexes >


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < LiveIntervals > ( ) ; AU . addPreserved < LiveIntervals > ( ) ; AU . addRequired < VirtRegMap > ( ) ; AU . addRequired < LiveRegMatrix > ( ) ; AU . addPreserved < SlotIndexes > ( ) ; AU . setPreservesCFG ( ) ; MachineFunctionPass :: getAnalysisUsage (


<s> if ( ! MO . isReg ( ) ) return false ; Register Reg = MO . getReg ( ) ; if ( ! TRI -> isVGPR ( * MRI , Reg ) ) return false ; if ( Register :: isPhysicalRegister ( Reg ) ) return


<s> MO . setReg ( PhysReg ) ; MO . setIsRenamable ( false ) ; } } } <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; for ( unsigned Reg : RegsToRewrite ) { LIS -> removeInterval ( Reg ) ; const Register PhysReg = VRM -> getPhys ( Reg ) ; assert ( PhysReg != <NUM_LIT> ) ; MFI -> ReserveWWMRegister ( PhysReg ) ; } RegsToRewrite . clear


<s> void <ISA_LIT>PreAllocateWWMRegs :: rewriteRegs ( MachineFunction & MF ) { for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : MBB ) { for ( MachineOperand & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) ) continue ; const Register VirtReg = MO . getReg ( ) ; if ( Register :: isPhysicalRegister ( VirtReg ) ) continue ; if ( ! VRM -> hasPhys ( VirtReg ) ) continue ; Register PhysReg = VRM -> getPhys ( VirtReg ) ; const unsigned SubReg = MO . getSubReg ( ) ; if ( SubReg != <NUM_LIT> ) { PhysReg = TRI -> getSubReg ( PhysReg , SubReg ) ; MO . setSubReg ( <NUM_LIT> ) ; } MO . setReg ( PhysReg


<s> <ISA_LIT>PreAllocateWWMRegs ( ) : MachineFunctionPass (


<s> Register IdxReg = Idx -> isReg ( ) ? Idx -> getReg ( ) : Register ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > ToRemove ; bool IdxOn = true ; if ( ! MI . isIdenticalTo ( First ) ) return false ; for ( MachineBasicBlock :: iterator I = std :: next ( First . getIterator ( ) ) , E = MI . getIterator ( ) ; I != E ; ++ I ) { switch ( I -> getOpcode ( ) ) { case <STR_LIT> : return false ; case <STR_LIT> : IdxOn = false ; ToRemove . push_back ( & * I ) ; break ; default : if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return false ; if ( IdxReg && I -> modifiesRegister ( IdxReg , TRI ) ) return false ; if ( llvm :: any_of ( I -> operands ( ) , [ & MRI , this ] ( const MachineOperand & MO ) { return MO . isReg ( ) && TRI -> isVectorRegister ( MRI , MO . getReg ( ) ) ; } ) ) { if ( ! IdxOn || ! ( ( I -> getOpcode ( ) == <STR_LIT> && I -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) || I -> getOpcode ( ) == <STR_LIT> ) ) return false ; } } } MI . eraseFromParent


<s> MI . eraseFromParent ( ) ; MBBE = MBB . getFirstTerminator ( ) ; } break ; default : break ; } } if ( ! ST . hasVGPRIndexMode ( ) ) continue ; MachineInstr * SetGPRMI = nullptr ; const unsigned Threshold = <NUM_LIT> ; unsigned Count = <NUM_LIT> ; for ( MachineBasicBlock :: iterator MBBI = MBB . begin ( ) ; MBBI != MBBE ; ) { MachineInstr & MI = * MBBI ; ++ MBBI ; if ( Count == Threshold ) SetGPRMI = nullptr ; else ++ Count ; if ( MI . getOpcode ( ) != <STR_LIT> ) continue ; Count = <NUM_LIT> ; if ( !


<s> if ( M -> definesRegister ( SReg , TRI ) ) break ; if ( M -> modifiesRegister ( SReg , TRI ) ) return Changed ; ReadsSreg |= M -> readsRegister ( SReg , TRI ) ; } if ( M == E || ! M -> isMoveImmediate ( ) || ! M -> getOperand ( <NUM_LIT> ) . isImm ( ) || M -> getOperand ( <NUM_LIT> ) . getImm ( ) != - <NUM_LIT> ) return Changed ; if ( ! ReadsSreg && Op2 . isKill ( ) ) { A -> getOperand ( <NUM_LIT> ) . ChangeToImmediate ( - <NUM_LIT> ) ; M -> eraseFromParent ( ) ; } } if ( ! ReadsCond && A -> registerDefIsDead ( <STR_LIT> ) && MI . killsRegister ( CondReg , TRI ) ) A -> eraseFromParent ( ) ; bool IsVCCZ = MI . getOpcode ( ) == <STR_LIT> ; if ( SReg == ExecReg ) { if ( IsVCCZ ) { MI . eraseFromParent ( ) ; return true ; } MI . setDesc ( TII -> get ( <STR_LIT>


<s> if ( A -> modifiesRegister ( ExecReg , TRI ) ) return false ; if ( A -> modifiesRegister ( CondReg , TRI ) ) { if ( ! A -> definesRegister ( CondReg , TRI ) || A -> getOpcode ( ) != And ) return false ; break ; } ReadsCond |= A -> readsRegister ( CondReg , TRI ) ; } if ( A == E ) return false ; MachineOperand & Op1 = A -> getOperand ( <NUM_LIT> ) ; MachineOperand & Op2 = A -> getOperand ( <NUM_LIT> ) ; if ( Op1 . getReg ( ) != ExecReg && Op2 . isReg ( ) && Op2 . getReg ( ) == ExecReg ) { TII -> commuteInstruction ( * A ) ; Changed = true ; } if ( Op1 . getReg ( ) != ExecReg ) return Changed ; if ( Op2 . isImm ( ) && Op2 . getImm ( ) != - <NUM_LIT> ) return Changed ; Register SReg ; if ( Op2 . isReg ( ) ) { SReg = Op2 . getReg ( ) ; auto M = std :: next ( A ) ; bool ReadsSreg = false ; for ( ; M != E ; ++ M ) { if ( M -> definesRegister ( SReg , TRI ) ) break ; if ( M -> modifiesRegister ( SReg , TRI ) ) return Changed ; ReadsSreg |= M -> readsRegister ( SReg , TRI ) ; } if ( M == E || ! M -> isMoveImmediate ( ) || ! M -> getOperand ( <NUM_LIT> ) . isImm ( ) || M -> getOperand ( <NUM_LIT> ) . getImm ( ) != - <NUM_LIT> ) return Changed ; if ( ! ReadsSreg && Op2 . isKill ( ) ) { A -> getOperand ( <NUM_LIT> ) . ChangeToImmediate ( - <NUM_LIT> ) ; M -> eraseFromParent ( ) ; } } if ( ! ReadsCond && A -> registerDefIsDead ( <STR_LIT> ) && MI . killsRegister ( CondReg , TRI ) ) A -> eraseFromParent ( ) ; bool IsVCCZ = MI . getOpcode ( ) == <STR_LIT> ; if ( SReg == ExecReg ) { if ( IsVCCZ ) { MI . eraseFromParent ( ) ; return true ; } MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; } else { MI . setDesc ( TII -> get ( IsVCCZ ? <STR_LIT> : <STR_LIT> ) ) ; } MI . RemoveOperand ( MI . findRegisterUseOperandIdx ( CondReg , false


<s> bool <ISA_LIT>PreEmitPeephole :: runOnMachineFunction ( MachineFunction & MF ) { const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; bool Changed


<s> if ( M -> modifiesRegister ( SReg , TRI ) ) return Changed ; ReadsSreg |= M -> readsRegister ( SReg , TRI ) ; } if ( M == E || ! M -> isMoveImmediate ( ) || ! M -> getOperand ( <NUM_LIT> ) . isImm ( ) || ( M -> getOperand ( <NUM_LIT> ) . getImm ( ) != - <NUM_LIT> && M -> getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ) ) return Changed ; MaskValue = M -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( ! ReadsSreg && Op2 . isKill ( ) ) { A -> getOperand ( <NUM_LIT> ) . ChangeToImmediate ( MaskValue ) ; M -> eraseFromParent ( ) ; } } else if ( Op2 . isImm ( ) ) { MaskValue = Op2 . getImm ( ) ; } else { llvm_unreachable ( <STR_LIT> ) ; } assert ( MaskValue == <NUM_LIT> || MaskValue == - <NUM_LIT> ) ; if ( A -> getOpcode ( ) == AndN2 ) MaskValue = ~ MaskValue ; if ( ! ReadsCond && A -> registerDefIsDead ( <STR_LIT> ) && MI . killsRegister ( CondReg , TRI ) ) A -> eraseFromParent ( ) ; bool IsVCCZ = MI . getOpcode ( ) == <STR_LIT> ; if ( SReg == ExecReg ) { if ( IsVCCZ ) { MI . eraseFromParent ( ) ; return true ; } MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; } else if ( IsVCCZ && MaskValue == <NUM_LIT> ) { MachineBasicBlock * Parent = MI . getParent ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > ToRemove ; bool Found = false ; for ( MachineInstr & Term : Parent -> terminators ( ) ) { if ( Found ) { if ( Term . isBranch ( ) ) ToRemove . push_back ( & Term ) ; } else { Found = Term . isIdenticalTo ( MI ) ; } } assert ( Found && <STR_LIT> ) ; for ( auto BranchMI : ToRemove ) { MachineOperand & Dst = BranchMI -> getOperand ( <NUM_LIT> ) ; assert ( Dst . isMBB ( ) && <STR_LIT> ) ; Parent -> removeSuccessor ( Dst . getMBB ( ) ) ; BranchMI -> eraseFromParent ( ) ; } if ( MachineBasicBlock * Succ = Parent -> getFallThrough ( ) ) { Parent -> removeSuccessor ( Succ ) ; } MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; } else if ( ! IsVCCZ && MaskValue == <NUM_LIT> ) { MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; assert ( Dst . isMBB ( ) && <STR_LIT>


<s> const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MachineOperand * Idx = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; Register IdxReg = Idx -> isReg ( ) ? Idx -> getReg ( ) : Register ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > ToRemove ; bool IdxOn = true ; if ( ! MI . isIdenticalTo ( First ) ) return false ; for ( MachineBasicBlock :: instr_iterator I = std :: next ( First . getIterator ( ) ) , E = MI . getIterator ( ) ; I != E ; ++ I ) { if ( I -> isBundle ( ) ) continue ; switch ( I -> getOpcode ( ) ) { case <STR_LIT> : return false ; case <STR_LIT> : IdxOn = false ; ToRemove . push_back ( & * I ) ; break ; default : if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return false ; if ( IdxReg && I -> modifiesRegister ( IdxReg , TRI ) ) return false ; if ( llvm :: any_of ( I -> operands ( ) , [ & MRI , this ] ( const MachineOperand & MO ) { return MO . isReg ( ) && TRI -> isVectorRegister ( MRI , MO . getReg ( ) ) ; } ) ) { if ( ! IdxOn || ! ( ( I -> getOpcode ( ) == <STR_LIT> && I -> hasRegisterImplicitUseOperand

( <STR_LIT> ) ) || I -> getOpcode ( ) == <STR_LIT> ) ) return false ;


<s> Register IdxReg = Idx -> isReg ( ) ? Idx -> getReg ( ) : Register ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > ToRemove ; bool IdxOn = true ; if ( ! MI . isIdenticalTo ( First ) ) return false ; for ( MachineBasicBlock :: instr_iterator I = std :: next ( First . getIterator ( ) ) , E = MI . getIterator ( ) ; I != E ; ++ I ) { if ( I -> isBundle ( ) ) continue ; switch ( I -> getOpcode ( ) ) { case <STR_LIT> : return false ; case <STR_LIT> : IdxOn = false ; ToRemove . push_back ( & * I ) ; break ; default : if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return false ; if ( IdxReg && I -> modifiesRegister ( IdxReg , TRI ) ) return false ; if ( llvm :: any_of ( I -> operands ( ) , [ & MRI , this ] ( const MachineOperand & MO ) { return MO . isReg ( ) && TRI -> isVectorRegister ( MRI , MO . getReg ( ) ) ; } )


<s> bool <ISA_LIT>PreEmitPeephole :: runOnMachineFunction ( MachineFunction & MF ) { const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; bool Changed = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator MBBE = MBB . getFirstTerminator ( ) ; if ( MBBE != MBB . end ( ) ) { MachineInstr & MI = * MBBE ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : Changed |= optimizeVccBranch ( MI ) ; continue ; default : break ; } } if ( ! ST . hasVGPRIndexMode ( ) ) continue ; MachineInstr * SetGPRMI = nullptr ; const unsigned Threshold = <NUM_LIT> ; unsigned Count = <NUM_LIT> ; for ( MachineBasicBlock :: iterator MBBI = MBB . begin ( ) ; MBBI != MBBE ; ) { MachineInstr & MI = * MBBI ; ++ MBBI ; if ( Count == Threshold ) SetGPRMI


<s> bool <ISA_LIT>PreEmitPeephole :: getBlockDestinations ( MachineBasicBlock & SrcMBB , MachineBasicBlock * & TrueMBB , MachineBasicBlock * & FalseMBB , SmallVectorImpl < MachineOperand > & Cond ) { if ( TII -> analyzeBranch ( SrcMBB , TrueMBB , FalseMBB , Cond


<s> if ( I -> isConditionalBranch ( ) ) return true ; if ( TII -> hasUnwantedEffectsWhenEXECEmpty ( * I ) ) return true ; if ( TII -> isSMRD ( * I ) || TII -> isVMEM ( * I ) || TII -> isFLAT ( * I ) || TII -> isDS ( * I ) || I -> getOpcode ( ) == <STR_LIT> ) return true ; ++ NumInstr ; if ( NumInstr >= SkipThreshold )


<s> Register IdxReg = Idx -> isReg ( ) ? Idx -> getReg ( ) : Register ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > ToRemove ; bool IdxOn = true ; if ( ! MI . isIdenticalTo ( First ) ) return false ; for ( MachineBasicBlock :: instr_iterator I = std :: next ( First . getIterator ( ) ) , E = MI . getIterator ( ) ; I != E ; ++ I ) { if ( I -> isBundle ( ) ) continue ; switch ( I -> getOpcode ( ) ) { case <STR_LIT> : return false ; case <STR_LIT> : IdxOn = false ; ToRemove . push_back ( & * I ) ; break ; default : if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return false ; if ( IdxReg && I -> modifiesRegister ( IdxReg , TRI )


<s> if ( ! ReadsSreg && Op2 . isKill ( ) ) { A -> getOperand ( <NUM_LIT> ) . ChangeToImmediate ( MaskValue ) ; M -> eraseFromParent ( ) ; } } else if ( Op2 . isImm ( ) ) { MaskValue = Op2 . getImm ( ) ; } else { llvm_unreachable ( <STR_LIT> ) ; } assert ( MaskValue == <NUM_LIT> || MaskValue == - <NUM_LIT> ) ; if ( A -> getOpcode ( ) == AndN2 ) MaskValue = ~ MaskValue ; if ( ! ReadsCond && A -> registerDefIsDead ( <STR_LIT> ) ) { if ( ! MI . killsRegister ( CondReg , TRI ) ) { if ( MaskValue == <NUM_LIT> ) { BuildMI ( * A -> getParent ( ) , * A , A -> getDebugLoc ( ) , TII -> get ( Mov ) , CondReg ) . addImm ( <NUM_LIT> ) ; } else { BuildMI ( * A -> getParent ( ) , * A , A -> getDebugLoc ( ) , TII -> get ( Mov ) , CondReg ) . addReg ( ExecReg ) ; } } A -> eraseFromParent ( ) ; } bool IsVCCZ = MI . getOpcode ( ) == <STR_LIT> ; if ( SReg == ExecReg ) { if ( IsVCCZ ) { MI . eraseFromParent ( ) ; return true ; } MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; } else if ( IsVCCZ && MaskValue == <NUM_LIT> ) { MachineBasicBlock * Parent = MI . getParent ( ) ; SmallVector < MachineInstr * , <NUM_LIT> > ToRemove ; bool Found = false ; for ( MachineInstr & Term : Parent -> terminators ( ) ) { if ( Found ) { if ( Term . isBranch ( ) ) ToRemove . push_back ( & Term ) ; } else { Found = Term . isIdenticalTo ( MI ) ; } } assert ( Found && <STR_LIT> ) ; for ( auto BranchMI : ToRemove ) { MachineOperand & Dst = BranchMI -> getOperand (


<s> if ( ( SrcMBB . getNumber ( ) >= TrueMBB -> getNumber ( ) ) || mustRetainExeczBranch ( * FalseMBB , * TrueMBB ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MI . eraseFromParent ( ) ; SrcMBB . removeSuccessor ( TrueMBB ) ; return true


<s> <ISA_LIT>PreEmitPeephole ( ) : MachineFunctionPass ( ID


<s> RS . enterBasicBlock ( & MBB ) ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { MachineInstr & MI = * I ; RS . forward ( I ) ; DebugLoc DL = MI . getDebugLoc ( ) ; if ( ! TII -> isVGPRSpill ( MI . getOpcode ( ) ) ) continue ; unsigned ScratchRsrcReg = RS . scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; unsigned Rsrc0 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc1 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Rsrc0 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Rsrc1 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Rsrc2 ) . addImm ( Rsrc23 & <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Rsrc3 ) . addImm ( Rsrc23 >> <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; if ( ScratchOffsetReg == <STR_LIT> ) { ScratchOffsetReg = RS . scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchOffsetReg ) . addFrameIndex ( ScratchOffsetFI ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) ; } else if ( ! MBB . isLiveIn ( ScratchOffsetReg ) ) { MBB . addLiveIn ( ScratchOffsetReg ) ; } if ( ScratchRsrcReg == <STR_LIT> || ScratchOffsetReg == <STR_LIT> ) { LLVMContext & Ctx = MF . getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> ) ; ScratchRsrcReg = <STR_LIT> ; ScratchOffsetReg = <STR_LIT> ; } MI . getOperand ( <NUM_LIT> ) . setReg ( ScratchRsrcReg ) ; MI . getOperand ( <NUM_LIT> ) . setIsKill ( true ) ; MI . getOperand ( <NUM_LIT> ) . setIsUndef (


<s> BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Rsrc3 ) . addImm ( Rsrc23 >> <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; if ( ScratchOffsetReg == <STR_LIT> ) { ScratchOffsetReg = RS . scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchOffsetReg ) . addFrameIndex ( ScratchOffsetFI ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) ; } else if ( ! MBB . isLiveIn ( ScratchOffsetReg ) ) { MBB . addLiveIn ( ScratchOffsetReg ) ; } if ( ScratchRsrcReg == <STR_LIT> || ScratchOffsetReg == <STR_LIT> ) { LLVMContext & Ctx = MF . getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> ) ; ScratchRsrcReg = <STR_LIT> ; ScratchOffsetReg = <STR_LIT> ; } MI . getOperand ( <NUM_LIT> ) . setReg ( ScratchRsrcReg ) ; MI . getOperand ( <NUM_LIT> ) . setIsKill ( true ) ; MI . getOperand ( <NUM_LIT> ) . setIsUndef ( false ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( ScratchOffsetReg ) ; MI . getOperand ( <NUM_LIT> ) . setIsUndef ( false ) ; MI . getOperand ( <NUM_LIT> ) . setIsKill ( false ) ; MI . addOperand ( MachineOperand :: CreateReg ( Rsrc0 , false , true , true ) ) ; MI . addOperand ( MachineOperand :: CreateReg ( Rsrc1 , false , true , true ) ) ; MI . addOperand ( MachineOperand :: CreateReg ( Rsrc2 , false , true , true ) ) ; MI . addOperand ( MachineOperand :: CreateReg ( Rsrc3 , false , true , true ) ) ; } } return


<s> unsigned ScratchPtrPreloadReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: SCRATCH_PTR ) ; unsigned ScratchOffsetPreloadReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: SCRATCH_WAVE_OFFSET ) ; if ( ! Entry -> isLiveIn ( ScratchPtrPreloadReg ) ) Entry -> addLiveIn ( ScratchPtrPreloadReg ) ; if ( ! Entry -> isLiveIn ( ScratchOffsetPreloadReg ) ) Entry -> addLiveIn ( ScratchOffsetPreloadReg ) ; unsigned ScratchOffsetReg = TRI -> findUnusedRegister ( MRI , & <STR_LIT> ) ; int ScratchOffsetFI = - <NUM_LIT> ; if ( ScratchOffsetReg != <STR_LIT> ) { BuildMI ( * Entry , I , DL , TII -> get ( <STR_LIT> ) , ScratchOffsetReg ) . addReg ( ScratchOffsetPreloadReg ) ; } else { ScratchOffsetFI = FrameInfo -> CreateSpillStackObject ( <NUM_LIT> , <NUM_LIT> ) ; BuildMI ( * Entry , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( ScratchOffsetPreloadReg ) . addFrameIndex ( ScratchOffsetFI ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) ; } RegScavenger RS ; unsigned ScratchRsrcFI = FrameInfo -> CreateSpillStackObject ( <NUM_LIT> , <NUM_LIT> ) ; RS . addScavengingFrameIndex ( ScratchRsrcFI ) ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; if ( ! MBB . isLiveIn ( ScratchOffsetReg ) && ScratchOffsetReg != <STR_LIT> ) MBB . addLiveIn ( ScratchOffsetReg ) ; RS . enterBasicBlock ( & MBB ) ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { MachineInstr & MI = * I ; RS . forward ( I ) ; DebugLoc DL = MI . getDebugLoc ( ) ; if ( ! TII -> isVGPRSpill ( MI . getOpcode ( ) ) ) continue ; unsigned ScratchRsrcReg = RS . scavengeRegister ( & <STR_LIT> , <NUM_LIT> ) ; uint64_t Rsrc = <STR_LIT> | <STR_LIT> |


<s> FunctionPass * llvm :: create<ISA_LIT>PrepareScratchRegs ( ) { return new <ISA_LIT>PrepareScratchRegs (


<s> return new <ISA_LIT>PrepareScratchRegs (


<s> return S_00B848_VGPRS ( VGPRBlocks ) | S_00B848_SGPRS ( SGPRBlocks ) | S_00B848_PRIORITY ( Priority ) | S_00B848_FLOAT_MODE ( FloatMode ) | S_00B848_PRIV ( Priv ) | S_00B848_DX10_CLAMP ( DX10Clamp ) |

S_00B848_DEBUG_MODE ( DebugMode ) | S_00B848_IEEE_MODE ( IEEEMode ) | S_00B848_WGP_MODE ( WgpMode ) | S_00B848_MEM_ORDERED ( MemOrdered ) ;


<s> return getComputePGMRSrc1 ( ) ; } uint64_t Reg = S_00B848_VGPRS ( VGPRBlocks ) | S_00B848_SGPRS ( SGPRBlocks ) | S_00B848_PRIORITY ( Priority ) | S_00B848_FLOAT_MODE ( FloatMode ) | S_00B848_PRIV ( Priv ) | S_00B848_DX10_CLAMP ( DX10Clamp ) | S_00B848_DEBUG_MODE ( DebugMode ) | S_00B848_IEEE_MODE ( IEEEMode ) ; switch ( CC ) { case CallingConv :: <ISA_LIT>_PS : Reg |= S_00B028_MEM_ORDERED ( MemOrdered ) ; break ; case CallingConv :: <ISA_LIT>_VS : Reg |= S_00B128_MEM_ORDERED ( MemOrdered ) ; break ; case CallingConv :: <ISA_LIT>_GS : Reg |= S_00B228_WGP_MODE ( WgpMode ) | S_00B228_MEM_ORDERED ( MemOrdered ) ; break ; case CallingConv :: <ISA_LIT>_HS : Reg |= S_00B428_WGP_MODE ( WgpMode ) | S_00B428_MEM_ORDERED ( MemOrdered ) ; break ; default : break ; } return Reg


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; if ( ST . isXNACKEnabled ( ) ) reserveRegisterTuples ( Reserved , <STR_LIT> ) ; } if ( ST . hasSGPRInitBug ( ) ) { unsigned NumSGPRs = <STR_LIT> . getNumRegs ( ) ; unsigned Limit = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> ; if ( ST . isXNACKEnabled ( ) ) Limit -= <NUM_LIT> ; for ( unsigned i = Limit ; i < NumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; if ( ScratchWaveOffsetReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved ,


<s> if ( ST . hasSGPRInitBug ( ) ) { unsigned BaseIdx = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> - <NUM_LIT> ; if ( ST . isXNACKEnabled ( ) ) BaseIdx -= <NUM_LIT> ; unsigned BaseReg ( <STR_LIT> . getRegister ( BaseIdx ) ) ; return getMatchingSuperReg ( BaseReg , <STR_LIT> , & <STR_LIT> ) ; } if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { return <STR_LIT> ; } return <STR_LIT>