<s> MachineOperand & ImpDefSCC = MI . getOperand ( <NUM_LIT> ) ; assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ; bool SimpleIf = isSimpleIf ( MI , MRI ) ; if ( SimpleIf ) { auto UseMI = MRI -> use_instr_nodbg_begin ( SaveExecReg ) ; SimpleIf = ! hasKill ( MI . getParent ( ) , UseMI -> getParent ( ) ) ; } Register CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( Exec ) . addReg ( Exec , RegState :: ImplicitDefine ) ; LoweredIf . insert ( CopyReg ) ; Register Tmp = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( AndOpc ) , Tmp ) . addReg ( CopyReg ) . add ( Cond ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = nullptr ; if ( ! SimpleIf )


<s> if ( ! SkipAnding ) { Register AndReg = MRI -> createVirtualRegister ( BoolRC ) ; And = BuildMI ( MBB , & MI , DL , TII -> get ( AndOpc ) , AndReg ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . addReg ( AndReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> createAndComputeVirtRegInterval ( AndReg ) ; } else Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . add

( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ;


<s> void <ISA_LIT>LowerControlFlow :: findMaskOperands ( MachineInstr & MI , unsigned OpNo , SmallVectorImpl < MachineOperand > & Src ) const { MachineOperand & Op = MI . getOperand ( OpNo ) ; if ( ! Op . isReg ( ) || ! Op . getReg ( ) . isVirtual ( ) ) { Src . push_back ( Op ) ; return ; } MachineInstr * Def = MRI -> getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Def || Def -> getParent ( ) != MI . getParent ( ) || ! ( Def -> isFullCopy ( ) || ( Def -> getOpcode ( ) == MI . getOpcode ( ) ) ) ) return ; for ( auto I = Def -> getIterator ( ) ; I != MI . getIterator ( ) ; ++ I ) if ( I -> modifiesRegister ( <STR_LIT> , TRI ) && ! ( I -> isCopy ( ) && I -> getOperand ( <NUM_LIT> ) . getReg ( ) != Exec ) ) return ; for ( const auto & SrcOp : Def -> explicit_operands ( ) ) if ( SrcOp . isReg ( ) && SrcOp . isUse ( ) && ( SrcOp . getReg

( ) . isVirtual ( ) || SrcOp . getReg ( ) == Exec ) ) Src . push_back ( SrcOp ) ;


<s> Register SaveExecReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; auto U = MRI -> use_instr_nodbg_begin ( SaveExecReg


<s> FirstMI = & * std :: next ( FirstMI -> getIterator ( ) ) ; } } } const DebugLoc DL = MI . getDebugLoc ( ) ; const unsigned WavefrontSize = ST . getWavefrontSize ( ) ; const unsigned Mask = ( WavefrontSize << <NUM_LIT> ) - <NUM_LIT> ; Register CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; auto BfeMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( <STR_LIT> ) , CountReg ) . addReg ( InputReg ) . addImm ( ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) & Mask ) | <NUM_LIT> ) ; auto BfmMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( CountReg ) . addImm ( <NUM_LIT> ) ; auto CmpMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( CountReg , RegState :: Kill ) . addImm ( WavefrontSize ) ; auto CmovMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addImm ( - <NUM_LIT> ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * BfeMI ) ; LIS -> InsertMachineInstrInMaps ( *


<s> for ( MachineInstr * MI : LoweredEndCf ) { MachineBasicBlock & MBB = * MI -> getParent ( ) ; auto Next = skipIgnoreExecInstsTrivialSucc ( MBB , std :: next ( MI -> getIterator ( ) ) ) ; if ( Next == MBB . end ( ) || !

LoweredEndCf . count ( & * Next ) ) continue ;


<s> emitIfBreak ( MI ) ; break ; case <STR_LIT> : emitLoop ( MI ) ; break ; case <STR_LIT> : MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; break ; case <STR_LIT> : SplitBB = emitEndCf ( MI ) ; break ; default : assert ( false && <STR_LIT> ) ; break ; } MachineBasicBlock :: iterator Next ; for ( I = Prev ? Prev -> getIterator ( ) : MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MaskMI = * I ; switch ( MaskMI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : combineMasks ( MaskMI ) ; break ; default : I = MBB .


<s> XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; OrOpc = <STR_LIT> ; XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } const bool CanDemote = MF . getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ; for ( auto & MBB : MF ) { bool IsKillBlock = false ; for ( auto & Term : MBB . terminators ( ) ) { if ( TII -> isKillTerminator ( Term . getOpcode ( ) ) ) { KillBlocks . insert ( & MBB ) ; IsKillBlock = true ; break ; } } if ( CanDemote && ! IsKillBlock ) { for ( auto & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) { KillBlocks . insert ( & MBB ) ; break ; } } } } MachineFunction :: iterator NextBB ; for ( MachineFunction :: iterator BI = MF . begin ( ) ; BI != MF . end ( ) ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock * MBB = & * BI ; MachineBasicBlock :: iterator I , E , Next ; E = MBB -> end ( ) ; for ( I = MBB -> begin ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; MachineBasicBlock * SplitMBB = MBB ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT>


<s> Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } const bool CanDemote = MF . getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ; for ( auto & MBB : MF ) { bool IsKillBlock = false ; for ( auto & Term : MBB . terminators ( ) ) { if ( TII -> isKillTerminator ( Term . getOpcode ( ) ) ) { KillBlocks . insert ( & MBB ) ; IsKillBlock = true ; break ; } } if ( CanDemote && ! IsKillBlock ) { for ( auto & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) { KillBlocks . insert ( & MBB ) ; break ; } } } } MachineFunction :: iterator NextBB ; for ( MachineFunction :: iterator BI = MF . begin ( ) ; BI != MF . end ( ) ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock * MBB = & * BI ; MachineBasicBlock :: iterator I , E , Next ; E = MBB -> end ( ) ; for ( I = MBB -> begin ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; MachineBasicBlock * SplitMBB = MBB


<s> static void setImpSCCDefDead ( MachineInstr & MI , bool IsDead ) { MachineOperand & ImpDefSCC = MI . getOperand ( <NUM_LIT> ) ; assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ; ImpDefSCC . setIsDead (


<s> SmallSet < const MachineBasicBlock * , <NUM_LIT> > Visited ; MachineBasicBlock * B = & MBB ; do { if ( ! Visited . insert ( B ) . second ) return MBB . end ( ) ; auto E = B -> end ( ) ; for ( ; It != E ; ++ It ) { if ( TII -> mayReadEXEC ( * MRI , * It ) ) break ; } if ( It != E ) return It ; if ( B -> succ_size ( ) != <NUM_LIT> ) return MBB . end ( ) ; MachineBasicBlock * Succ = * B -> succ_begin ( ) ; It = Succ -> begin ( ) ; B =


<s> for ( ; It != E ; ++ It ) { if ( TII -> mayReadEXEC ( * MRI , * It ) ) break ; } if ( It != E ) return It ; if ( B -> succ_size ( ) != <NUM_LIT> ) return MBB . end ( ) ; MachineBasicBlock * Succ = * B ->


<s> assert ( I -> isTerminator ( ) ) ; MachineBasicBlock :: iterator End = MBB . end ( ) ; while ( I != End && ! I -> isUnconditionalBranch ( ) ) ++ I ; return I


<s> bool SkipAnding = false ; if ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) { if ( MachineInstr * Def = MRI -> getUniqueVRegDef ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { SkipAnding = Def -> getParent ( ) == MI . getParent ( ) && <ISA_LIT>InstrInfo :: isVALU ( * Def ) ; } } MachineInstr * And = nullptr , * Or = nullptr ; if ( ! SkipAnding ) { And = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( <STR_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Or = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else Or = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI .


<s> return ; } MachineInstr * Def = MRI -> getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Def || Def -> getParent ( ) != MI . getParent ( ) || ! ( Def -> isFullCopy ( ) || ( Def -> getOpcode ( ) == MI . getOpcode ( ) ) ) ) return ; for ( auto I = Def -> getIterator ( ) ; I != MI . getIterator ( ) ; ++ I ) if ( I -> modifiesRegister ( <STR_LIT> , TRI ) && ! ( I -> isCopy ( ) && I -> getOperand (

<NUM_LIT> ) . getReg ( ) != <STR_LIT> ) ) return ;


<s> MachineInstr * Or = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( <STR_LIT> ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * Or ) ; MI . eraseFromParent


<s> void <ISA_LIT>LowerControlFlow :: emitElse ( MachineInstr & MI ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) == <STR_LIT> ) ; bool ExecModified = MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ; MachineBasicBlock :: iterator Start = MBB . begin ( ) ; unsigned CopyReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; unsigned SaveReg = ExecModified ? MRI -> createVirtualRegister ( & <STR_LIT> ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( <STR_LIT> ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS


<s> const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator InsPt = MBB . begin ( ) ; MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * NewMI


<s> assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ; unsigned CopyReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> , RegState :: ImplicitDefine ) ; unsigned Tmp = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Tmp ) . addReg ( CopyReg ) . addReg ( Cond . getReg ( ) ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Tmp , RegState :: Kill ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; if ( !


<s> MachineInstr * AndN2 = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; MachineInstr * Branch = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) { LIS -> ReplaceMachineInstrInMaps ( MI , * AndN2 ) ; LIS -> InsertMachineInstrInMaps ( * Branch


<s> <ISA_LIT>LowerControlFlow ( ) : MachineFunctionPass ( ID ) , TRI ( nullptr ) , TII ( nullptr ) , LIS ( nullptr )


<s> Register SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( XorTermrOpc ) , Exec ) . addReg ( Exec ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps


<s> Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) == <STR_LIT> ) ; bool ExecModified = MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ; MachineBasicBlock :: iterator Start = MBB . begin ( ) ; Register CopyReg = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Register SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( XorTermrOpc ) , Exec ) . addReg ( Exec ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT>


<s> void <ISA_LIT>LowerControlFlow :: emitEndCf ( MachineInstr & MI ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator InsPt = MBB . begin ( ) ; MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( OrOpc ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * NewMI ) ; MI . eraseFromParent ( ) ; if ( LIS ) LIS -> handleMove ( *


<s> Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . addReg ( Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS


<s> if ( Ops . size ( ) != <NUM_LIT> ) return ; unsigned UniqueOpndIdx ; if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else return ; Register Reg = MI . getOperand ( OpToReplace ) . getReg ( ) ; MI . removeOperand ( OpToReplace ) ; MI . addOperand ( Ops [ UniqueOpndIdx ] ) ; if ( MRI -> use_empty ( Reg ) ) MRI -> getUniqueVRegDef (


<s> LV = getAnalysisIfAvailable < LiveVariables > ( ) ; MDT = getAnalysisIfAvailable < MachineDominatorTree > ( ) ; MRI = & MF . getRegInfo ( ) ; BoolRC = TRI -> getBoolRC ( ) ; if ( ST . isWave32 ( ) ) { AndOpc = <STR_LIT> ; OrOpc = <STR_LIT> ; XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; OrOpc = <STR_LIT> ; XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } const bool CanDemote = MF . getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ; for ( auto & MBB : MF ) { bool IsKillBlock = false ; for ( auto & Term : MBB . terminators ( ) ) { if ( TII -> isKillTerminator ( Term . getOpcode ( ) ) ) { KillBlocks . insert ( & MBB ) ; IsKillBlock = true ; break ; } } if ( CanDemote && ! IsKillBlock ) { for ( auto & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) { KillBlocks . insert ( & MBB ) ; break ; } } } } bool Changed


<s> } MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( Opcode ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; LoweredEndCf . insert ( NewMI ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * NewMI ) ; MI . eraseFromParent ( ) ; if ( LIS ) LIS -> handleMove ( * NewMI ) ; return SplitBB


<s> unsigned Opcode = OrOpc ; MachineBasicBlock * SplitBB = & MBB ; if ( NeedBlockSplit ) { SplitBB = MBB . splitAt ( MI , true , LIS ) ; Opcode = OrTermrOpc ; InsPt = MI ; } MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( Opcode ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; LoweredEndCf . insert


<s> if ( ! I . isDebugInstr ( ) && ! I . isUnconditionalBranch ( ) ) return false ; } assert ( MBB . succ_size ( ) == <NUM_LIT> && <STR_LIT> ) ; MachineBasicBlock * Succ = * MBB . succ_begin ( ) ; MachineBasicBlock * FallThrough = nullptr ; while ( ! MBB . predecessors ( ) . empty ( ) ) { MachineBasicBlock * P = * MBB . pred_begin ( ) ; if ( GetFallThroughSucc ( P ) == & MBB ) FallThrough = P ; P -> ReplaceUsesOfBlockWith ( & MBB , Succ ) ; } MBB . removeSuccessor ( Succ ) ; if ( LIS ) { for ( auto & I : MBB . instrs ( ) ) LIS -> RemoveMachineInstrFromMaps ( I ) ; } MBB . clear ( ) ; MBB . eraseFromParent ( ) ; if ( FallThrough && ! FallThrough -> isLayoutSuccessor ( Succ ) ) { if ( ! GetFallThroughSucc ( Succ ) ) { MachineFunction * MF = FallThrough -> getParent ( ) ; MachineFunction :: iterator FallThroughPos ( FallThrough


<s> OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; OrOpc = <STR_LIT> ; XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } const bool CanDemote = MF . getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ; for ( auto & MBB : MF ) { bool IsKillBlock = false ; for ( auto & Term : MBB . terminators ( ) ) { if ( TII -> isKillTerminator ( Term . getOpcode ( ) ) ) { KillBlocks . insert ( & MBB ) ; IsKillBlock = true ; break ; } } if ( CanDemote && ! IsKillBlock ) { for ( auto & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) { KillBlocks . insert ( & MBB ) ; break ; } } } } MachineFunction :: iterator NextBB ; for ( MachineFunction :: iterator BI = MF . begin ( ) ; BI != MF . end ( ) ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock * MBB = & * BI ; MachineBasicBlock :: iterator I , E , Next ; E = MBB -> end (


<s> } EndCf ( MI ) ; break ; case <STR_LIT> : if ( Depth == <NUM_LIT> ) { if ( skipIfDead ( MI , * NextBB ) ) { NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; } } else HaveKill = true ; Kill ( MI ) ; break ; case <STR_LIT> : Branch ( MI ) ; break ; case <STR_LIT> : { assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( BI != -- MF . end ( ) || I != MBB . getFirstTerminator ( ) ) { if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB . addSuccessor ( EmptyMBBAtEnd ) ; BuildMI ( * BI , I , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; I -> eraseFromParent (


<s> MachineFunctionPass ( ID ) , TRI ( nullptr ) , TII (

nullptr ) , SkipThreshold ( <NUM_LIT> ) {


<s> Register CopyReg = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Register SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc )

, DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ;


<s> MachineInstr * CopyExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Register SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( XorTermrOpc ) , Exec ) . addReg ( Exec ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> InsertMachineInstrInMaps ( * OrSaveExec ) ; LIS -> InsertMachineInstrInMaps (


<s> MachineOperand & ImpDefSCC = MI . getOperand ( <NUM_LIT> ) ; assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ; bool SimpleIf = isSimpleIf ( MI , MRI , TII ) ; Register CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( Exec ) . addReg ( Exec , RegState :: ImplicitDefine ) ; Register Tmp = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( AndOpc ) , Tmp ) . addReg ( CopyReg ) . add ( Cond ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = nullptr


<s> MachineBasicBlock * MBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( MBB == MI . getParent ( ) -> getNextNode ( ) ) MI


<s> void <ISA_LIT>LowerControlFlow :: Else ( MachineInstr & MI , bool ExecModified ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , MBB . getFirstNonPHI ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Src ) ; if ( ExecModified ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( <STR_LIT> ) . addReg ( Dst ) ; } BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Dst ) ; Skip ( MI , MI . getOperand ( <NUM_LIT> ) ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) )

. addOperand ( MI . getOperand ( <NUM_LIT> ) ) . addReg ( Dst ) ;


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , MBB . getFirstNonPHI ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Src ) ; if ( ExecModified ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( <STR_LIT> ) . addReg ( Dst ) ; } BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Dst ) ; Skip ( MI , MI . getOperand ( <NUM_LIT> ) ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) . addReg ( Dst ) ; MI . eraseFromParent


<s> BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( <STR_LIT> ) . addReg ( IdxReg . getReg ( ) , getUndefRegState ( IdxReg . isUndef ( ) ) ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; if ( Offset != <NUM_LIT> ) { BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addImm ( Offset ) ; } LoopBB . insert ( I , MovRel ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( &


<s> BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( <STR_LIT> ) . addReg ( IdxReg . getReg ( ) , getUndefRegState ( IdxReg . isUndef ( ) ) ) ; BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; if ( Offset != <NUM_LIT> ) { BuildMI ( LoopBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg


<s> DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Vcc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( Vcc ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( <STR_LIT> ) . addReg ( Reg ) ; Skip ( MI , MI . getOperand ( <NUM_LIT> ) ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) . addReg ( Reg ) ; MI . eraseFromParent


<s> unsigned Reg ; const MachineOperand * Val = TII -> getNamedOperand ( MI , <STR_LIT> :: val ) ; std :: tie ( Reg , Offset ) = computeIndirectRegAndOffset ( Dst , Offset ) ; MachineOperand * Idx = TII -> getNamedOperand ( MI , <STR_LIT> :: idx ) ; if ( Idx -> getReg ( ) == <STR_LIT> ) { BuildMI ( MBB , MI . getIterator ( ) , DL , TII -> get ( <STR_LIT> ) , Reg )


<s> BuildMI ( MBB , MI . getIterator ( ) , DL , TII -> get ( <STR_LIT> ) , Reg ) . addOperand ( * Val ) ; MI . eraseFromParent ( ) ; return false ; } MachineInstr * MovRel = BuildMI ( * MBB . getParent ( ) , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( Val -> getReg ( ) , getUndefRegState ( Val ->

isUndef ( ) ) ) . addReg ( Dst , RegState :: Implicit ) ;


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const MachineOperand * SrcVec = TII -> getNamedOperand ( MI , <STR_LIT> :: src ) ; int Offset = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> getImm ( ) ; unsigned Reg ; std :: tie ( Reg , Offset ) = computeIndirectRegAndOffset ( SrcVec -> getReg ( )


<s> CallingConv :: ID CallConv = MBB . getParent ( ) -> getFunction ( ) -> getCallingConv ( ) ; assert ( CallConv == CallingConv :: <ISA_LIT>_PS || CallConv == CallingConv :: <ISA_LIT>_GS ) ; if ( ( Op . isImm ( ) ) ) { if ( Op . getImm ( ) &


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; CallingConv :: ID CallConv = MBB . getParent ( ) -> getFunction ( ) -> getCallingConv ( ) ; assert ( CallConv == CallingConv :: <ISA_LIT>_PS || CallConv == CallingConv :: <ISA_LIT>_GS ) ; if ( ( Op . isImm ( ) ) ) { if ( Op . getImm ( ) & <NUM_LIT> ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addImm ( <NUM_LIT> ) ; } } else { BuildMI ( MBB , & MI , DL , TII -> get (

<STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addOperand ( Op ) ;


<s> BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Idx -> getReg ( ) , getUndefRegState ( Idx -> isUndef ( ) ) ) . addImm ( Offset ) ; } else { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Idx -> getReg ( ) , getUndefRegState ( Idx -> isUndef ( ) ) ) ; } MBB . insert ( I , MovRel ) ; MI . eraseFromParent ( ) ; return false ; } MachineOperand * SaveOp = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; SaveOp -> setIsDead ( false ) ; unsigned Save = SaveOp -> getReg ( ) ; assert ( <STR_LIT> . contains ( Save ) && <STR_LIT> . contains ( Idx -> getReg ( ) ) ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Save ) . addReg ( <STR_LIT> ) ; LivePhysRegs RemainderLiveRegs ( TRI ) ; RemainderLiveRegs . addLiveOuts ( MBB ) ; MachineBasicBlock * LoopBB ; MachineBasicBlock * RemainderBB ; std :: tie ( LoopBB , RemainderBB ) = splitBlock ( MBB , I ) ; for ( const MachineInstr & Inst : reverse ( * RemainderBB ) ) RemainderLiveRegs . stepBackward ( Inst ) ; MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; LoopBB -> addSuccessor


<s> MachineInstr & MI = * I ; if ( TII -> isFLAT ( MI ) ) NeedFlat = true ; if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) ExecModified = true ; switch ( MI . getOpcode ( ) ) { default : break ; case <STR_LIT> : ++ Depth ; If ( MI ) ; break ; case <STR_LIT> : Else ( MI , ExecModified ) ; break ; case <STR_LIT> : Break ( MI ) ; break ; case <STR_LIT> : IfBreak ( MI ) ; break ; case <STR_LIT> : ElseBreak ( MI ) ; break ; case <STR_LIT> : ++ Depth ; Loop ( MI ) ; break ; case <STR_LIT> : if ( -- Depth == <NUM_LIT> && HaveKill ) { HaveKill = false ; } EndCf ( MI ) ; break ; case <STR_LIT> : if ( Depth == <NUM_LIT> ) { if ( skipIfDead ( MI , * NextBB ) ) { NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; } } else HaveKill = true ; Kill ( MI ) ; break ; case <STR_LIT> : Branch ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( indirectSrc ( MI ) ) { NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; Next = MBB . end ( ) ; } break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( indirectDst ( MI ) ) { NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; Next = MBB . end ( ) ; } break ; case <STR_LIT> : { assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( BI != -- MF . end ( ) || I != MBB . getFirstTerminator ( ) ) { if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB . addSuccessor ( EmptyMBBAtEnd ) ; BuildMI ( * BI , I , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; I -> eraseFromParent ( ) ; } break ; } } } } if ( NeedFlat && MFI -> IsKernel ) { if ( NeedFlat ) MFI -> setHasFlatInstructions ( true ) ; } return


<s> Break ( MI ) ; break ; case <STR_LIT> : IfBreak ( MI ) ; break ; case <STR_LIT> : ElseBreak ( MI ) ; break ; case <STR_LIT> : ++ Depth ; Loop ( MI ) ; break ; case <STR_LIT> : if ( -- Depth == <NUM_LIT> && HaveKill ) { HaveKill = false ; } EndCf ( MI ) ; break ; case <STR_LIT> : if ( Depth == <NUM_LIT> ) { if ( skipIfDead ( MI , * NextBB ) ) { NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; } } else HaveKill = true ; Kill ( MI ) ; break ; case <STR_LIT> : Branch ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( indirectSrc ( MI ) ) { NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; Next = MBB . end ( ) ; } break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( indirectDst (


<s> unsigned NumInstr = <NUM_LIT> ; MachineFunction * MF = From -> getParent ( ) ; for ( MachineFunction :: iterator MBBI ( From ) , ToI ( To ) , End = MF -> end ( ) ; MBBI != End && MBBI != ToI ; ++ MBBI ) { MachineBasicBlock & MBB = * MBBI ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; NumInstr < SkipThreshold && I != E ; ++ I ) { if ( opcodeEmitsNoInsts ( I -> getOpcode ( )


<s> MBBI != End && MBBI != ToI ; ++ MBBI ) { MachineBasicBlock & MBB = * MBBI ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; NumInstr < SkipThreshold && I != E ; ++ I ) { if ( opcodeEmitsNoInsts ( I -> getOpcode ( ) ) ) continue ; if ( I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) return true ; if ( I -> isInlineAsm ( )


<s> MachineBasicBlock :: iterator Insert = SkipBB -> begin ( ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) ; return true


<s> bool <ISA_LIT>LowerControlFlow :: skipIfDead ( MachineInstr & MI , MachineBasicBlock & NextBB ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; MachineFunction * MF = MBB . getParent ( ) ; if ( MF -> getFunction ( ) -> getCallingConv ( ) != CallingConv :: <ISA_LIT>_PS || ! shouldSkip ( & MBB , & MBB . getParent ( ) -> back ( ) ) ) return false ; MachineBasicBlock * SkipBB = insertSkipBlock ( MBB , MI . getIterator (


<s> MachineBasicBlock * LoopBB = MF -> CreateMachineBasicBlock ( ) ; MachineBasicBlock * RemainderBB = MF -> CreateMachineBasicBlock ( ) ; MachineFunction :: iterator MBBI ( MBB ) ; ++ MBBI ; MF -> insert ( MBBI , LoopBB ) ; MF -> insert ( MBBI


<s> std :: pair < MachineBasicBlock * , MachineBasicBlock * > <ISA_LIT>LowerControlFlow :: splitBlock ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator I ) { MachineFunction * MF = MBB . getParent ( ) ; MachineBasicBlock * LoopBB = MF -> CreateMachineBasicBlock ( ) ; MachineBasicBlock * RemainderBB = MF -> CreateMachineBasicBlock ( ) ; MachineFunction :: iterator MBBI ( MBB ) ; ++ MBBI ; MF -> insert ( MBBI , LoopBB


<s> void <ISA_LIT>LowerControlFlow :: splitLoadM0BlockLiveIns ( LivePhysRegs & RemainderLiveRegs , const MachineRegisterInfo & MRI , const MachineInstr & MI , MachineBasicBlock & LoopBB , MachineBasicBlock & RemainderBB , unsigned SaveReg , const MachineOperand & IdxReg ) { RemainderLiveRegs . addReg ( SaveReg ) ; if ( const MachineOperand * Val = TII -> getNamedOperand


<s> MachineBasicBlock :: iterator Start = MBB . begin ( ) ; Register CopyReg = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Register SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified


<s> MachineInstr * CopyExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Register SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS ->


<s> MachineInstr * Def = MRI . getUniqueVRegDef ( CFMask ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator InsPt = Def && Def -> getParent ( ) == & MBB ? std :: next ( MachineBasicBlock :: iterator ( Def ) ) : MBB . begin ( ) ; MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( OrOpc ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; LoweredEndCf . insert ( NewMI ) ; MachineInstr * CleanUpMI = nullptr ; if ( NeedsKillCleanup . count ( & MI ) ) CleanUpMI = BuildMI ( MBB , InsPt , DL , TII -> get ( <STR_LIT> ) ) ; if ( LIS ) { LIS -> ReplaceMachineInstrInMaps ( MI , * NewMI


<s> const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator InsPt = Def && Def -> getParent ( ) == & MBB ? std :: next ( MachineBasicBlock :: iterator ( Def ) ) : MBB . begin ( ) ; MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( OrOpc ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; LoweredEndCf . insert ( NewMI ) ; MachineInstr * CleanUpMI = nullptr ; if ( NeedsKillCleanup . count ( & MI ) ) CleanUpMI = BuildMI ( MBB , InsPt , DL , TII -> get ( <STR_LIT> ) ) ; if ( LIS ) { LIS -> ReplaceMachineInstrInMaps ( MI


<s> for ( MachineInstr * MI : LoweredEndCf ) { MachineBasicBlock & MBB = * MI -> getParent ( ) ; auto Next = skipIgnoreExecInstsTrivialSucc ( MBB , std :: next ( MI -> getIterator ( ) ) ) ; if ( Next == MBB . end ( ) || ! LoweredEndCf .


<s> MachineBasicBlock & MBB = * MI -> getParent ( ) ; auto Next = skipIgnoreExecInstsTrivialSucc ( MBB , std :: next ( MI -> getIterator ( ) ) ) ; if ( Next == MBB . end ( ) || ! LoweredEndCf . count ( & * Next ) ) continue ; Register SavedExec = TII -> getNamedOperand ( * Next , <STR_LIT> :: src1 ) -> getReg ( ) ; assert ( SavedExec . isVirtual ( ) && <STR_LIT> ) ; const MachineInstr * Def = MRI -> getUniqueVRegDef ( SavedExec ) ; if ( Def && LoweredIf . count ( SavedExec ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI


<s> const MachineInstr * Def = MRI . getUniqueVRegDef ( SavedExec ) ; if ( Def && ( Def -> getOpcode ( ) == <STR_LIT> || LoweredIf . count ( SavedExec ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ) ; if ( LIS ) LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; return ; } } } MachineBasicBlock :: iterator InsPt = Def && Def -> getParent ( ) == & MBB ? std :: next ( MachineBasicBlock :: iterator ( Def ) ) : MBB . begin ( ) ; MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( OrOpc ) , Exec ) .

addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ;


<s> assert ( Cond . getSubReg ( ) == <STR_LIT> ) ; MachineOperand & ImpDefSCC = MI . getOperand ( <NUM_LIT> ) ; assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ; bool SimpleIf = isSimpleIf ( MI , MRI , TII ) ; Register CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( Exec ) . addReg ( Exec , RegState :: ImplicitDefine ) ; LoweredIf . insert ( CopyReg ) ; Register Tmp = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( AndOpc ) , Tmp ) . addReg ( CopyReg ) . add ( Cond ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor =


<s> XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; OrOpc = <STR_LIT> ; XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } MachineFunction :: iterator NextBB ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next , Last ; for ( I = MBB . begin ( ) , Last = MBB . end ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : emitIf ( MI ) ; break ; case <STR_LIT>


<s> XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; OrOpc = <STR_LIT> ; XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } MachineFunction :: iterator NextBB ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next , Last ; for ( I = MBB . begin ( ) , Last = MBB . end ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : emitIf ( MI ) ; break ; case <STR_LIT> : emitElse ( MI ) ; break ; case <STR_LIT> : emitIfBreak ( MI ) ; break ; case <STR_LIT> : emitLoop ( MI ) ; break ; case <STR_LIT> : emitEndCf ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : combineMasks ( MI ) ; Last = I ; continue ; default : Last = I ; continue ; } Next = ( Last == MBB . end ( ) ) ? MBB . begin ( ) : Last ; } } LoweredEndCf . clear ( ) ; LoweredIf . clear (


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator I ( & MI ) ; Register SaveExecReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineOperand & Cond = MI . getOperand ( <NUM_LIT> ) ; assert ( Cond . getSubReg ( ) == <STR_LIT> ) ; MachineOperand & ImpDefSCC = MI . getOperand ( <NUM_LIT> ) ; assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ; bool SimpleIf = isSimpleIf ( MI , MRI , TII ) ; Register CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( Exec ) . addReg ( Exec , RegState :: ImplicitDefine ) ; Register Tmp = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( AndOpc ) , Tmp ) . addReg ( CopyReg ) . add ( Cond ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = nullptr ; if ( ! SimpleIf ) { Xor = BuildMI ( MBB , I , DL , TII -> get ( XorOpc ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; } MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( MovTermOpc ) , Exec ) . addReg ( Tmp , RegState :: Kill ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * And ) ; if ( ! SimpleIf ) LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps (


<s> MachineRegisterInfo & MRI = MBB . getParent ( ) -> getRegInfo ( ) ; Register CFMask = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * Def = MRI . getUniqueVRegDef ( CFMask ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator InsPt = Def && Def -> getParent ( ) == & MBB ? std :: next ( MachineBasicBlock :: iterator ( Def ) ) : MBB . begin ( ) ; MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( OrOpc ) , Exec ) .


<s> void <ISA_LIT>LowerControlFlow :: Else ( MachineInstr & MI , bool ExecModified ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , MBB . getFirstNonPHI ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Src ) ; if ( ExecModified ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> )

, Dst ) . addReg ( <STR_LIT> ) . addReg ( Dst ) ;


<s> BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( <STR_LIT> ) . addReg ( Dst ) ; } BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Dst ) ; Skip ( MI , MI . getOperand ( <NUM_LIT> ) ) ; MI . eraseFromParent ( )


<s> unsigned NumInstr = <NUM_LIT> ; MachineFunction * MF = From -> getParent ( ) ; for ( MachineFunction :: iterator MBBI ( From ) , ToI ( To ) , End = MF -> end ( ) ; MBBI != End && MBBI != ToI ; ++ MBBI ) { MachineBasicBlock & MBB = * MBBI ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; NumInstr < SkipThreshold && I != E ; ++ I ) { if ( opcodeEmitsNoInsts ( I -> getOpcode ( ) ) ) continue ; if ( I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) return true ; if ( ++ NumInstr >= SkipThreshold ) return true ; } } return


<s> DebugLoc DL = MI . getDebugLoc ( ) ; if ( MBB . getParent ( ) -> getFunction ( ) -> getCallingConv ( ) != CallingConv :: <ISA_LIT>_PS || ! shouldSkip ( & MBB , & MBB . getParent ( ) -> back ( ) ) ) return ; MachineBasicBlock :: iterator Insert = & MI ; ++ Insert ; BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT>

) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ;


<s> MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( <STR_LIT> ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps (


<s> MachineInstr * CopyExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; unsigned SaveReg = ExecModified ? MRI -> createVirtualRegister ( & <STR_LIT> ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( <STR_LIT> ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> InsertMachineInstrInMaps ( * OrSaveExec ) ; LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * Branch ) ; LIS -> removeInterval ( DstReg ) ; LIS -> createAndComputeVirtRegInterval ( DstReg ) ; LIS -> createAndComputeVirtRegInterval


<s> setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = nullptr ; if ( ! SimpleIf ) { Xor = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; } MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Tmp , RegState :: Kill ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * And ) ; if ( ! SimpleIf ) LIS -> InsertMachineInstrInMaps ( *


<s> Xor = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; } MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Tmp , RegState :: Kill ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * And ) ; if ( ! SimpleIf ) LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * SetExec ) ; LIS -> InsertMachineInstrInMaps ( * NewBr ) ; LIS -> removeAllRegUnitsForPhysReg ( <STR_LIT> ) ; MI . eraseFromParent ( ) ; LIS -> removeInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( SaveExecReg


<s> } } } MachineFunction :: iterator NextBB ; for ( MachineFunction :: iterator BI = MF . begin ( ) ; BI != MF . end ( ) ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock * MBB = & * BI ; MachineBasicBlock :: iterator I , E , Next ; E = MBB -> end ( ) ; for ( I = MBB -> begin ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; MachineBasicBlock * SplitMBB = MBB ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : SplitMBB = process ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : SplitMBB = process ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : lowerInitExec ( MBB , MI ) ; if ( LIS ) LIS ->

removeAllRegUnitsForPhysReg ( <STR_LIT> ) ;


<s> const <ISA_LIT>MachineFunctionInfo * MFI = MBB . getParent ( ) -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; assert ( MFI -> getShaderType ( ) == ShaderType :: PIXEL || MFI -> getShaderType ( ) == ShaderType :: GEOMETRY ) ; if ( ( Op . isImm ( ) ) ) { if ( Op . getImm ( ) & <NUM_LIT> ) { BuildMI ( MBB , & MI , DL , TII -> get

( <STR_LIT> ) , <STR_LIT> ) . addImm ( <NUM_LIT> ) ;


<s> BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Idx ) ; } MBB . insert ( I , MovRel ) ; } else { assert ( <STR_LIT> . contains ( Save ) ) ; assert ( <STR_LIT> . contains ( Idx ) ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Save ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Idx ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Idx ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; if ( Offset ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addImm ( Offset ) ; } MBB . insert ( I , MovRel ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( - <NUM_LIT> ) .


<s> if ( <STR_LIT> . contains ( Idx ) ) { if ( Offset ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Idx ) . addImm ( Offset ) ; } else { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Idx ) ; } MBB . insert ( I , MovRel ) ; } else { assert ( <STR_LIT> . contains ( Save ) ) ; assert ( <STR_LIT> . contains ( Idx ) ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Save ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Idx ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Idx ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; if ( Offset ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addImm ( Offset ) ; } MBB . insert ( I , MovRel ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( - <NUM_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT>


<s> unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) .


<s> default : break ; case <STR_LIT> : ++ Depth ; If ( MI ) ; break ; case <STR_LIT> : Else ( MI ) ; break ; case <STR_LIT> : Break ( MI ) ; break ; case <STR_LIT> : IfBreak ( MI ) ; break ; case <STR_LIT> : ElseBreak ( MI ) ; break ; case <STR_LIT> : ++ Depth ; Loop ( MI ) ; break ; case <STR_LIT> : if ( -- Depth == <NUM_LIT> && HaveKill ) { SkipIfDead ( MI ) ; HaveKill = false ; } EndCf ( MI ) ; break ; case <STR_LIT> : if ( Depth == <NUM_LIT> ) SkipIfDead ( MI ) ; else HaveKill = true ; Kill ( MI ) ; break ; case <STR_LIT> : Branch ( MI ) ; break ; case <STR_LIT> : IndirectSrc ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : IndirectDst ( MI ) ; break ; } } } if ( NeedWQM && MFI -> getShaderType ( ) == ShaderType :: PIXEL ) { MachineBasicBlock & MBB = MF . front ( ) ; BuildMI ( MBB , MBB . getFirstNonPHI ( ) , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; } if ( NeedFlat && MFI -> IsKernel


<s> void <ISA_LIT>LowerControlFlowPass :: Skip ( MachineInstr & From , MachineOperand & To ) { if ( ! shouldSkip ( * From . getParent ( ) -> succ_begin ( ) , To . getMBB ( ) ) ) return ; DebugLoc DL = From . getDebugLoc


<s> MachineBasicBlock :: iterator Insert = & MI ; ++ Insert ; BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , Insert , DL , TII


<s> void <ISA_LIT>LowerControlFlow :: emitElseBreak ( MachineInstr


<s> SmallVector < MachineBasicBlock * , <NUM_LIT> > Worklist ( Begin -> successors ( ) ) ; while ( ! Worklist . empty ( ) ) { MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; if ( MBB == End || ! Visited . insert ( MBB ) . second ) continue ; for ( auto & Term : MBB -> terminators ( ) ) if ( TII -> isKillTerminator ( Term . getOpcode ( ) ) ) return true ; Worklist . append ( MBB -> succ_begin ( ) , MBB -> succ_end ( )


<s> for ( MachineFunction :: iterator BI = MF . begin ( ) ; BI != MF . end ( ) ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock * MBB = & * BI ; MachineBasicBlock :: iterator I , E , Next ; E = MBB -> end ( ) ; for ( I = MBB -> begin ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; MachineBasicBlock * SplitMBB = MBB ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : SplitMBB = process ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( InsertKillCleanups ) Worklist . push_back ( & MI ) ; else SplitMBB = process ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : lowerInitExec ( MBB , MI ) ; if ( LIS ) LIS -> removeAllRegUnitsForPhysReg ( <STR_LIT>


<s> MachineBasicBlock * EmptyMBBAtEnd = nullptr ; MachineBasicBlock :: iterator I , Next ; bool ExecModified = false ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; if ( TII -> isFLAT ( MI ) ) NeedFlat = true ; if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) ExecModified = true ; switch ( MI . getOpcode ( ) ) { default : break ; case <STR_LIT> : ++ Depth ; If ( MI ) ; break ; case <STR_LIT> : Else ( MI , ExecModified ) ; break ; case <STR_LIT> : Break ( MI ) ; break ; case <STR_LIT> : IfBreak ( MI ) ; break ; case <STR_LIT> : ElseBreak ( MI ) ; break ; case <STR_LIT> : ++ Depth ; Loop ( MI ) ; break ; case <STR_LIT> : if ( -- Depth == <NUM_LIT> && HaveKill ) { SkipIfDead ( MI ) ; HaveKill = false ; } EndCf ( MI ) ; break ; case <STR_LIT> : if ( Depth == <NUM_LIT> ) SkipIfDead ( MI ) ; else HaveKill = true ; Kill ( MI ) ; break ; case <STR_LIT> : Branch ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( indirectSrc ( MI ) ) { NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; Next = MBB . end ( ) ; } break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( indirectDst (


<s> void <ISA_LIT>LowerControlFlow :: splitBlockLiveIns ( const MachineBasicBlock & MBB , const MachineInstr & MI , MachineBasicBlock & LoopBB , MachineBasicBlock & RemainderBB , unsigned SaveReg , const MachineOperand & IdxReg ) { LivePhysRegs RemainderLiveRegs ( TRI ) ; RemainderLiveRegs . addLiveOuts ( MBB ) ; for ( MachineBasicBlock :: const_reverse_iterator I = MBB . rbegin ( ) , E ( & MI ) ; I != E ; ++ I ) { RemainderLiveRegs . stepBackward ( * I ) ; } RemainderLiveRegs . addReg ( SaveReg ) ; if ( const MachineOperand * Val = TII -> getNamedOperand ( MI , <STR_LIT> :: val ) ) { if ( ! Val ->


<s> MachineOperand & Cond = MI . getOperand ( <NUM_LIT> ) ; assert ( SaveExec . getSubReg ( ) == <STR_LIT> && Cond . getSubReg ( ) == <STR_LIT> ) ; unsigned SaveExecReg = SaveExec . getReg ( ) ; MachineOperand & ImpDefSCC = MI . getOperand ( <NUM_LIT> ) ; assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ; unsigned CopyReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> , RegState :: ImplicitDefine ) ; unsigned Tmp = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , Tmp ) . addReg ( CopyReg ) . addReg ( Cond . getReg ( ) ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Tmp , RegState :: Kill ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> InsertMachineInstrInMaps (


<s> assert ( MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) == <STR_LIT> ) ; bool ExecModified = MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ; MachineBasicBlock :: iterator Start = MBB . begin ( ) ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( <STR_LIT> ) . addReg ( DstReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( DstReg ) ; MachineBasicBlock :: iterator Term = MBB . getFirstTerminator ( ) ; MachineInstr * Branch = BuildMI ( MBB , Term , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) . addReg ( DstReg ) ; if ( !


<s> void <ISA_LIT>LowerControlFlow :: emitElse ( MachineInstr & MI ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) == <STR_LIT> ) ; bool ExecModified = MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ; MachineBasicBlock :: iterator Start = MBB . begin ( ) ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( <STR_LIT> ) . addReg ( DstReg ) ; if ( LIS ) LIS ->


<s> MachineOperand & SaveExec = MI . getOperand ( <NUM_LIT> ) ; MachineOperand & Cond = MI . getOperand ( <NUM_LIT> ) ; assert ( SaveExec . getSubReg ( ) == <STR_LIT> && Cond . getSubReg ( ) == <STR_LIT> ) ; unsigned SaveExecReg = SaveExec . getReg ( ) ; MachineInstr * AndSaveExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SaveExecReg ) . addOperand ( Cond ) ; MachineInstr * Xor = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SaveExecReg ) . addReg ( <STR_LIT> ) . addReg ( SaveExecReg ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) . addReg ( SaveExecReg , getKillRegState ( SaveExec . isKill ( ) ) ) ; if ( ! LIS


<s> const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineInstr * AndN2 = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; MachineInstr * Branch = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS )


<s> AU . addPreserved < SlotIndexes > ( ) ; AU . setPreservesCFG ( ) ; MachineFunctionPass :: getAnalysisUsage ( AU


<s> MachineFunctionPass ( ID ) , TRI ( nullptr ) , TII ( nullptr ) , LIS (


<s> void <ISA_LIT>LowerControlFlow :: emitIfBreak ( MachineInstr & MI ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; auto Dst = getSaveExec ( & MI ) ; bool SkipAnding = false ; if ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) { if ( MachineInstr * Def = MRI -> getUniqueVRegDef ( MI . getOperand ( <NUM_LIT> )


<s> And = BuildMI ( MBB , & MI , DL , TII -> get ( AndOpc ) , Dst ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . addReg ( Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS


<s> for ( ; I != E ; I ++ ) { if ( I -> isBranch ( ) && TII -> getBranchDestBlock ( * I ) == S ) return nullptr ; } } return S ; } ; for ( auto & I : MBB . instrs ( ) ) { if ( ! I . isDebugInstr ( ) && ! I . isUnconditionalBranch ( ) ) return false ; } assert ( MBB . succ_size ( ) == <NUM_LIT> && <STR_LIT> ) ; MachineBasicBlock * Succ = * MBB . succ_begin ( ) ; MachineBasicBlock * FallThrough = nullptr ; while ( ! MBB . predecessors ( ) . empty ( ) ) { MachineBasicBlock * P = * MBB . pred_begin ( ) ; if ( GetFallThroughSucc ( P ) == & MBB ) FallThrough = P ; P -> ReplaceUsesOfBlockWith ( & MBB , Succ ) ; } MBB . removeSuccessor ( Succ ) ; if ( LIS ) { for ( auto & I : MBB . instrs ( ) ) LIS -> RemoveMachineInstrFromMaps ( I ) ; } if ( MDT ) MDT -> eraseNode ( & MBB ) ; MBB . clear ( ) ; MBB . eraseFromParent ( ) ; if ( FallThrough && ! FallThrough ->


<s> MachineInstr * Xor = nullptr ; if ( ! SimpleIf ) { Xor = BuildMI ( MBB , I , DL , TII -> get ( XorOpc ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; } MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( MovTermOpc ) , Exec ) . addReg ( Tmp , RegState :: Kill ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * And ) ; if ( ! SimpleIf ) LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * SetExec ) ; LIS -> InsertMachineInstrInMaps ( * NewBr ) ; LIS -> removeAllRegUnitsForPhysReg ( <STR_LIT> ) ; MI . eraseFromParent ( ) ; LIS -> removeInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( Tmp )


<s> MachineOperand & Op = MI . getOperand ( OpNo ) ; if ( ! Op . isReg ( ) || ! TargetRegisterInfo :: isVirtualRegister ( Op . getReg ( ) ) ) { Src . push_back ( Op ) ; return ; } MachineInstr * Def = MRI -> getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Def || Def -> getParent ( ) != MI . getParent ( ) || ! ( Def -> isFullCopy ( ) || ( Def -> getOpcode ( ) == MI . getOpcode ( ) ) ) ) return ; for ( auto I = Def -> getIterator ( ) ; I != MI . getIterator ( ) ; ++ I ) if ( I -> modifiesRegister ( <STR_LIT> , TRI ) && ! ( I -> isCopy ( ) &&

I -> getOperand ( <NUM_LIT> ) . getReg ( ) != Exec ) ) return ;


<s> MachineOperand & Op = MI . getOperand ( OpNo ) ; if ( ! Op . isReg ( ) || ! TargetRegisterInfo :: isVirtualRegister ( Op . getReg ( ) ) ) { Src . push_back ( Op ) ; return ; } MachineInstr * Def = MRI -> getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Def || Def -> getParent ( ) != MI . getParent ( ) || ! ( Def -> isFullCopy ( ) || ( Def -> getOpcode ( ) == MI . getOpcode ( ) ) ) ) return ; for ( auto I = Def -> getIterator ( ) ; I != MI . getIterator ( ) ; ++ I ) if ( I -> modifiesRegister ( <STR_LIT> , TRI ) && ! ( I -> isCopy ( ) && I -> getOperand ( <NUM_LIT> ) . getReg ( ) != Exec ) ) return ; for ( const auto & SrcOp : Def -> explicit_operands ( ) ) if ( SrcOp . isReg ( ) && SrcOp . isUse ( ) && ( TargetRegisterInfo :: isVirtualRegister ( SrcOp . getReg ( ) ) || SrcOp . getReg (

) == Exec ) ) Src . push_back ( SrcOp ) ;


<s> MachineBasicBlock :: iterator Start = MBB . begin ( ) ; unsigned CopyReg = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; unsigned SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( XorTermrOpc ) , Exec ) . addReg ( Exec ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> InsertMachineInstrInMaps ( * OrSaveExec ) ; LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * Branch ) ; LIS -> removeInterval ( DstReg )


<s> UseMI = MRI -> use_instr_nodbg_begin ( NextExecReg ) ; } if ( UseMI -> getOpcode ( ) == <STR_LIT> ) { if ( hasKill ( MI . getParent ( ) , UseMI -> getParent ( ) , TII ) ) { NeedsKillCleanup . insert ( & * UseMI ) ; SimpleIf = false ; } } } else if ( SimpleIf ) { auto UseMI = MRI -> use_instr_nodbg_begin ( SaveExecReg ) ; SimpleIf = ! hasKill ( MI . getParent ( ) , UseMI -> getParent ( ) , TII ) ; } Register CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( Exec ) . addReg ( Exec , RegState :: ImplicitDefine ) ; LoweredIf . insert ( CopyReg ) ; Register Tmp = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( AndOpc ) , Tmp ) . addReg ( CopyReg ) . add ( Cond ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = nullptr ; if ( ! SimpleIf ) { Xor = BuildMI ( MBB , I , DL , TII -> get ( XorOpc ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; } MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( MovTermOpc ) , Exec ) . addReg ( Tmp , RegState :: Kill ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * And


<s> MachineBasicBlock :: iterator I , Next , Last ; for ( I = MBB . begin ( ) , Last = MBB . end ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : emitIf ( MI ) ; break ; case <STR_LIT> : emitElse ( MI ) ; break ; case <STR_LIT> : emitBreak ( MI ) ; break ; case <STR_LIT> : emitIfBreak ( MI ) ; break ; case <STR_LIT> : emitElseBreak ( MI ) ; break ; case <STR_LIT> : emitLoop ( MI ) ; break ; case <STR_LIT> : emitEndCf ( MI ) ; break ; case <STR_LIT>


<s> switch ( MI . getOpcode ( ) ) { default : break ; case <STR_LIT> : ++ Depth ; If ( MI ) ; break ; case <STR_LIT> : Else ( MI , ExecModified ) ; break ; case <STR_LIT> : Break ( MI ) ; break ; case <STR_LIT> : IfBreak ( MI ) ; break ; case <STR_LIT> : ElseBreak ( MI ) ; break ; case <STR_LIT> : ++ Depth ; Loop ( MI ) ; break ; case <STR_LIT> : if ( -- Depth == <NUM_LIT> && HaveKill ) { HaveKill = false ; if ( skipIfDead ( MI , * NextBB ) ) { NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; Next = MBB . end ( ) ; } } EndCf ( MI ) ; break ; case <STR_LIT> : if ( Depth == <NUM_LIT> ) { if ( skipIfDead ( MI , * NextBB ) ) { NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; Next = MBB . end ( ) ; } } else HaveKill = true ; Kill ( MI ) ; break ; case <STR_LIT> : Branch ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( indirectSrc ( MI ) ) { NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; Next = MBB . end ( ) ; } break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( indirectDst ( MI ) ) { NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; Next = MBB . end ( ) ; } break ; case <STR_LIT> : { assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( BI != -- MF . end ( ) || I != MBB . getFirstTerminator ( ) ) { if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB . addSuccessor ( EmptyMBBAtEnd ) ; BuildMI ( * BI , I , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) )


<s> if ( I -> isInlineAsm ( ) ) { const MCAsmInfo * MAI = MF -> getTarget ( ) . getMCAsmInfo ( ) ; const char * AsmStr = I -> getOperand ( <NUM_LIT> ) . getSymbolName ( ) ; uint64_t MaxAsmSize = TII -> getInlineAsmLength ( AsmStr , * MAI ) ; NumInstr += MaxAsmSize / MAI -> getMaxInstLength ( ) ; } else { ++ NumInstr ; } if ( NumInstr >= SkipThreshold ) return true ; } } return false


<s> MachineFunction * MF = From -> getParent ( ) ; for ( MachineFunction :: iterator MBBI ( From ) , ToI ( To ) , End = MF -> end ( ) ; MBBI != End && MBBI != ToI ; ++ MBBI ) { MachineBasicBlock & MBB = * MBBI ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; NumInstr < SkipThreshold && I != E ; ++ I ) { if ( opcodeEmitsNoInsts ( I -> getOpcode ( ) ) ) continue ; if ( I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) return true ; if ( I -> isInlineAsm ( ) ) { const MCAsmInfo * MAI = MF -> getTarget ( ) . getMCAsmInfo ( ) ; const char * AsmStr = I -> getOperand ( <NUM_LIT> ) . getSymbolName ( ) ; uint64_t MaxAsmSize = TII -> getInlineAsmLength ( AsmStr , * MAI ) ; NumInstr += MaxAsmSize / MAI -> getMaxInstLength ( ) ; } else { ++


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; MachineFunction * MF = MBB . getParent ( ) ; if ( MF -> getFunction ( ) -> getCallingConv ( ) != CallingConv :: <ISA_LIT>_PS || ! shouldSkip ( & MBB , & MBB . getParent ( ) -> back ( ) ) ) return false ; MachineBasicBlock * SkipBB = insertSkipBlock ( MBB , MI . getIterator ( ) ) ; SkipBB -> addSuccessor ( & NextBB


<s> MachineBasicBlock :: iterator Insert = SkipBB -> begin ( ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) ; BuildMI ( * SkipBB , Insert , DL , TII ->


<s> void <ISA_LIT>LowerControlFlowPass :: IndirectDst ( MachineInstr & MI ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; int Off = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; unsigned Val = MI . getOperand ( <NUM_LIT>


<s> computeIndirectRegAndOffset ( Vec , Reg , Off ) ; MachineInstr * MovRel = BuildMI ( * MBB . getParent ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Reg ) . addReg (

<STR_LIT> , RegState :: Implicit ) . addReg ( Vec , RegState :: Implicit ) ;


<s> int Off = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; unsigned Reg ; computeIndirectRegAndOffset ( Vec , Reg , Off ) ; MachineInstr * MovRel = BuildMI ( * MBB . getParent ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Reg ) . addReg ( <STR_LIT> , RegState :: Implicit ) . addReg


<s> bool isVreg1 ( unsigned Reg ) const { return Register :: isVirtualRegister ( Reg ) && MRI -> getRegClass ( Reg )


<s> return Register :: isVirtualRegister ( Reg ) && MRI -> getRegClass


<s> MachineSSAUpdater SSAUpdater ( * MF ) ; LoopFinder LF ( * DT , * PDT ) ; SmallVector < MachineInstr * , <NUM_LIT> > DeadCopies ; for ( MachineBasicBlock & MBB : * MF ) { LF . initialize ( MBB ) ; for ( MachineInstr & MI : MBB ) { if ( MI . getOpcode ( ) != <STR_LIT> && MI . getOpcode ( ) != <STR_LIT> ) continue ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! isVreg1 ( DstReg ) ) continue ; if ( MRI -> use_empty ( DstReg ) ) { DeadCopies . push_back ( & MI ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MRI -> setRegClass ( DstReg , IsWave32 ? & <STR_LIT> : & <STR_LIT> ) ; if ( MI . getOpcode ( ) == <STR_LIT> ) continue ; DebugLoc DL = MI . getDebugLoc ( ) ; Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( ! MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ; if ( ! Register :: isVirtualRegister ( SrcReg ) || ( ! isLaneMaskReg ( SrcReg ) && ! isVreg1 ( SrcReg ) ) ) { assert ( TII -> getRegisterInfo ( ) . getRegSizeInBits ( SrcReg , * MRI ) == <NUM_LIT> ) ; unsigned TmpReg = createLaneMaskReg ( * MF ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SrcReg ) . addImm ( <NUM_LIT> ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( TmpReg ) ; SrcReg = TmpReg ; } std :: vector < MachineBasicBlock * > DomBlocks = { & MBB } ; for ( MachineInstr & Use : MRI -> use_instructions ( DstReg ) ) DomBlocks . push_back ( Use . getParent ( ) ) ; MachineBasicBlock * PostDomBound = PDT -> findNearestCommonDominator ( DomBlocks


<s> MachineInstr * IncomingDef = MRI -> getUniqueVRegDef ( IncomingReg ) ; if ( IncomingDef -> getOpcode ( ) == <STR_LIT> ) { IncomingReg = IncomingDef -> getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( isLaneMaskReg ( IncomingReg ) || isVreg1 ( IncomingReg ) ) ; assert ( ! IncomingDef -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ; } else if ( IncomingDef -> getOpcode ( ) == <STR_LIT> ) { continue ; } else { assert ( IncomingDef -> isPHI ( ) || PhiRegisters . count ( IncomingReg ) ) ; } IncomingBlocks . push_back ( IncomingMBB ) ; IncomingRegs . push_back ( IncomingReg ) ; } PhiRegisters . insert ( DstReg ) ; std :: vector < MachineBasicBlock * > DomBlocks = { & MBB } ; for ( MachineInstr & Use : MRI -> use_instructions ( DstReg ) ) DomBlocks . push_back ( Use . getParent ( ) ) ; MachineBasicBlock * PostDomBound = PDT -> findNearestCommonDominator ( DomBlocks ) ; unsigned FoundLoopLevel = LF . findLoop ( PostDomBound ) ; SSAUpdater . Initialize ( DstReg ) ; if ( FoundLoopLevel ) { LF . addLoopEntries ( FoundLoopLevel , SSAUpdater , IncomingBlocks ) ; for ( unsigned i = <NUM_LIT> ; i < IncomingRegs . size ( ) ; ++ i ) { IncomingUpdated . push_back ( createLaneMaskReg ( * MF ) ) ; SSAUpdater . AddAvailableValue ( IncomingBlocks [ i ] , IncomingUpdated . back ( ) ) ; } for ( unsigned i = <NUM_LIT> ; i < IncomingRegs . size ( )


<s> MRI = & MF -> getRegInfo ( ) ; DT = & getAnalysis < MachineDominatorTree > ( ) ; PDT = & getAnalysis < MachinePostDominatorTree > ( ) ; ST = & MF -> getSubtarget < GCNSubtarget > ( ) ; TII = ST -> getInstrInfo ( ) ; IsWave32 = ST -> isWave32 ( ) ; if ( IsWave32 ) { ExecReg = <STR_LIT> ; MovOp = <STR_LIT> ; AndOp = <STR_LIT> ; OrOp = <STR_LIT> ; XorOp = <STR_LIT> ; AndN2Op = <STR_LIT> ; OrN2Op = <STR_LIT> ; } else { ExecReg = <STR_LIT> ; MovOp = <STR_LIT> ; AndOp = <STR_LIT> ; OrOp = <STR_LIT> ; XorOp = <STR_LIT> ; AndN2Op = <STR_LIT> ; OrN2Op = <STR_LIT> ; } lowerCopiesFromI1 ( ) ; lowerPhis ( ) ; lowerCopiesToI1 ( ) ; for ( unsigned Reg : ConstrainRegs ) MRI -> constrainRegClass ( Reg , & <STR_LIT> ) ; ConstrainRegs . clear ( ) ; return


<s> Divergent = true ; break ; } } if ( Divergent && PDT . dominates ( & DefBlock , MBB ) ) append_range ( Stack , MBB -> successors ( ) ) ; } while ( ! Stack . empty ( ) ) { MachineBasicBlock * MBB = Stack . pop_back_val ( ) ; if ( ! ReachableMap . try_emplace ( MBB , false ) . second ) continue ; ReachableOrdered . push_back ( MBB ) ; append_range ( Stack , MBB -> successors ( ) ) ; } for ( MachineBasicBlock * MBB : ReachableOrdered ) { bool HaveReachablePred = false ; for ( MachineBasicBlock * Pred : MBB -> predecessors ( ) ) { if ( ReachableMap . count ( Pred ) ) { HaveReachablePred = true


<s> BuildMI ( MBB , I , DL , TII -> get ( XorOp ) , DstReg ) . addReg ( ExecReg ) . addImm ( - <NUM_LIT> ) ; } return ; } unsigned PrevMaskedReg = <NUM_LIT> ; unsigned CurMaskedReg = <NUM_LIT> ; if ( ! PrevConstant ) { if ( CurConstant && CurVal ) { PrevMaskedReg = PrevReg ; } else { PrevMaskedReg = createLaneMaskReg ( * MF ) ; BuildMI ( MBB , I , DL , TII -> get ( AndN2Op ) , PrevMaskedReg ) . addReg ( PrevReg ) . addReg ( ExecReg ) ; } } if ( ! CurConstant ) { if ( PrevConstant && PrevVal ) { CurMaskedReg = CurReg ; } else { CurMaskedReg = createLaneMaskReg ( * MF ) ; BuildMI ( MBB , I , DL , TII -> get ( AndOp ) , CurMaskedReg ) . addReg ( CurReg ) . addReg ( ExecReg ) ; } } if ( PrevConstant && ! PrevVal ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( CurMaskedReg ) ; } else if ( CurConstant && ! CurVal ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( PrevMaskedReg ) ; } else if ( PrevConstant && PrevVal ) { BuildMI ( MBB , I , DL , TII -> get ( OrN2Op ) , DstReg ) . addReg ( CurMaskedReg ) . addReg ( ExecReg ) ; } else


<s> if ( DomIt != Visited . end ( ) && DomIt -> second <= LoopLevel ) return true ; if ( llvm :: is_contained ( Blocks , & MBB ) )


<s> bool inLoopLevel ( MachineBasicBlock & MBB , unsigned LoopLevel , ArrayRef < MachineBasicBlock * > Blocks ) const { auto DomIt = Visited . find ( & MBB ) ; if ( DomIt != Visited . end ( )

&& DomIt -> second <= LoopLevel ) return true ;


<s> bool Changed = false ; MachineSSAUpdater SSAUpdater ( * MF ) ; LoopFinder LF ( * DT , * PDT ) ; SmallVector < MachineInstr * , <NUM_LIT> > DeadCopies ; for ( MachineBasicBlock & MBB : * MF ) { LF . initialize ( MBB ) ; for ( MachineInstr & MI : MBB ) { if ( MI . getOpcode ( ) != <STR_LIT> && MI . getOpcode ( ) != <STR_LIT> ) continue ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! isVreg1 ( DstReg ) ) continue ; Changed = true ; if ( MRI -> use_empty ( DstReg ) ) { DeadCopies . push_back ( & MI ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MRI -> setRegClass ( DstReg , IsWave32 ? & <STR_LIT> : & <STR_LIT> ) ; if ( MI . getOpcode ( ) == <STR_LIT> ) continue ; DebugLoc DL = MI . getDebugLoc ( ) ; Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( ! MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ; if ( ! SrcReg . isVirtual ( ) || ( ! isLaneMaskReg ( SrcReg ) && ! isVreg1 ( SrcReg ) ) ) { assert ( TII -> getRegisterInfo ( ) . getRegSizeInBits ( SrcReg , * MRI ) == <NUM_LIT> ) ; unsigned TmpReg = createLaneMaskReg ( * MF ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SrcReg ) . addImm ( <NUM_LIT> ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( TmpReg ) ; SrcReg = TmpReg ; } std :: vector < MachineBasicBlock * > DomBlocks = { & MBB } ; for ( MachineInstr & Use : MRI -> use_instructions ( DstReg ) ) DomBlocks . push_back ( Use . getParent ( ) ) ; MachineBasicBlock * PostDomBound = PDT -> findNearestCommonDominator ( DomBlocks ) ; unsigned FoundLoopLevel = LF . findLoop ( PostDomBound ) ; if ( FoundLoopLevel ) { SSAUpdater . Initialize ( DstReg ) ; SSAUpdater . AddAvailableValue ( & MBB , DstReg ) ; LF . addLoopEntries ( FoundLoopLevel , SSAUpdater ) ; buildMergeLaneMasks ( MBB , MI , DL , DstReg , SSAUpdater . GetValueInMiddleOfBlock ( & MBB ) , SrcReg ) ; DeadCopies . push_back ( & MI ) ; } } for ( MachineInstr * MI : DeadCopies ) MI -> eraseFromParent ( ) ; DeadCopies . clear


<s> assert ( IncomingDef -> isPHI ( ) || PhiRegisters . count ( IncomingReg ) ) ; } IncomingBlocks . push_back ( IncomingMBB ) ; IncomingRegs . push_back ( IncomingReg ) ; } PhiRegisters . insert ( DstReg ) ; std :: vector < MachineBasicBlock * > DomBlocks = { & MBB } ; for ( MachineInstr & Use : MRI -> use_instructions ( DstReg ) ) DomBlocks . push_back ( Use . getParent ( ) ) ; MachineBasicBlock * PostDomBound = PDT -> findNearestCommonDominator ( DomBlocks ) ; unsigned FoundLoopLevel = LF . findLoop ( PostDomBound ) ; SSAUpdater . Initialize ( DstReg ) ; if ( FoundLoopLevel ) { LF . addLoopEntries ( FoundLoopLevel , SSAUpdater , IncomingBlocks ) ; for ( unsigned i = <NUM_LIT> ; i < IncomingRegs . size ( ) ; ++ i ) { IncomingUpdated . push_back ( createLaneMaskReg ( * MF ) ) ; SSAUpdater . AddAvailableValue ( IncomingBlocks [ i ] , IncomingUpdated . back ( ) ) ; } for ( unsigned i = <NUM_LIT> ; i < IncomingRegs . size ( ) ; ++ i ) { MachineBasicBlock & IMBB = * IncomingBlocks [ i ] ; buildMergeLaneMasks ( IMBB , getSaluInsertionAtEnd ( IMBB ) , { } , IncomingUpdated [ i ] , SSAUpdater . GetValueInMiddleOfBlock ( & IMBB ) , IncomingRegs [ i ] ) ; } } else { PIA . analyze ( MBB , IncomingBlocks ) ; for ( MachineBasicBlock * MBB : PIA . predecessors ( ) ) SSAUpdater . AddAvailableValue ( MBB , insertUndefLaneMask ( * MBB ) ) ; for ( unsigned i = <NUM_LIT> ; i < IncomingRegs . size ( ) ; ++ i ) { MachineBasicBlock & IMBB = * IncomingBlocks [ i ] ; if ( PIA . isSource ( IMBB ) ) { IncomingUpdated . push_back ( <NUM_LIT> ) ; SSAUpdater . AddAvailableValue ( & IMBB , IncomingRegs [ i ] ) ; } else { IncomingUpdated . push_back ( createLaneMaskReg


<s> TII = ST -> getInstrInfo ( ) ; IsWave32 = ST -> isWave32 ( ) ; if ( IsWave32 ) { ExecReg = <STR_LIT> ; MovOp = <STR_LIT> ; AndOp = <STR_LIT> ; OrOp = <STR_LIT> ; XorOp = <STR_LIT> ; AndN2Op = <STR_LIT> ; OrN2Op = <STR_LIT> ; } else { ExecReg = <STR_LIT> ; MovOp = <STR_LIT> ; AndOp = <STR_LIT> ; OrOp = <STR_LIT> ; XorOp = <STR_LIT> ; AndN2Op = <STR_LIT> ; OrN2Op = <STR_LIT> ; } bool Changed = false ; Changed |= lowerCopiesFromI1 (


<s> if ( ! isLaneMaskReg ( Reg ) ) return false ; } if ( MI -> getOpcode ( ) != MovOp ) return false ; if ( ! MI -> getOperand ( <NUM_LIT> ) . isImm ( ) ) return false ; int64_t Imm = MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( Imm == <NUM_LIT> ) { Val = false ; return true ; } if ( Imm == - <NUM_LIT> ) { Val =


<s> SmallVector < MachineInstr * , <NUM_LIT> > DeadCopies ; for ( MachineBasicBlock & MBB : * MF ) { for ( MachineInstr & MI : MBB ) { if ( MI . getOpcode ( ) != <STR_LIT> ) continue ; unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) || MRI -> getRegClass ( SrcReg ) != & <STR_LIT> ) continue ; if ( isLaneMaskReg ( DstReg ) || ( TargetRegisterInfo :: isVirtualRegister ( DstReg ) && MRI -> getRegClass ( DstReg ) == & <STR_LIT> ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; DebugLoc DL = MI . getDebugLoc ( ) ; assert ( TII -> getRegisterInfo ( ) . getRegSizeInBits ( DstReg , * MRI ) == <NUM_LIT> ) ; assert ( ! MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ; ConstrainRegs . insert ( SrcReg ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) . addReg ( SrcReg ) ; DeadCopies . push_back ( &


<s> if ( MI . getOpcode ( ) != <STR_LIT> ) continue ; unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) || MRI -> getRegClass ( SrcReg ) != & <STR_LIT> ) continue ; if ( isLaneMaskReg ( DstReg ) || ( TargetRegisterInfo :: isVirtualRegister ( DstReg ) && MRI -> getRegClass ( DstReg ) == & <STR_LIT> ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; DebugLoc DL = MI . getDebugLoc (


<s> MachineSSAUpdater SSAUpdater ( * MF ) ; LoopFinder LF ( * DT , * PDT ) ; SmallVector < MachineInstr * , <NUM_LIT> > DeadCopies ; for ( MachineBasicBlock & MBB : * MF ) { LF . initialize ( MBB ) ; for ( MachineInstr & MI : MBB ) { if ( MI . getOpcode ( ) != <STR_LIT> && MI . getOpcode ( ) != <STR_LIT> ) continue ; unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( DstReg ) || MRI -> getRegClass ( DstReg ) != & <STR_LIT> ) continue ; if ( MRI -> use_empty ( DstReg ) ) { DeadCopies . push_back ( & MI ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MRI -> setRegClass ( DstReg , IsWave32 ? & <STR_LIT> : & <STR_LIT> ) ; if ( MI . getOpcode ( ) == <STR_LIT> ) continue ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( ! MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) || ! isLaneMaskReg ( SrcReg ) ) { assert ( TII -> getRegisterInfo ( ) . getRegSizeInBits ( SrcReg , * MRI ) == <NUM_LIT> ) ; unsigned TmpReg = createLaneMaskReg ( * MF ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SrcReg ) . addImm ( <NUM_LIT> ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( TmpReg ) ; SrcReg = TmpReg ; } MachineBasicBlock * PostDomBound = & MBB ; for ( MachineInstr & Use : MRI -> use_instructions ( DstReg ) ) { PostDomBound = PDT -> findNearestCommonDominator ( PostDomBound , Use . getParent ( ) ) ; } unsigned FoundLoopLevel = LF . findLoop ( PostDomBound ) ; if ( FoundLoopLevel


<s> unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC = MRI . getRegClass ( Reg ) ; if ( RC == & <STR_LIT> ) MRI . setRegClass ( Reg , & <STR_LIT> ) ; continue ; } if ( MI . getOpcode ( ) != <STR_LIT> ) continue ; const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; const MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( Src . getReg ( ) ) || ! TargetRegisterInfo :: isVirtualRegister ( Dst . getReg ( ) ) ) continue ; const TargetRegisterClass * DstRC = MRI . getRegClass ( Dst . getReg ( ) ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( Src . getReg ( ) ) ; DebugLoc DL = MI . getDebugLoc ( ) ; MachineInstr * DefInst = MRI . getUniqueVRegDef ( Src . getReg ( ) ) ; if ( DstRC == & <STR_LIT> && TRI -> getCommonSubClass ( SrcRC , & <STR_LIT> ) ) { I1Defs . push_back ( Dst . getReg ( ) ) ; if ( DefInst -> getOpcode ( ) == <STR_LIT> ) { if ( DefInst -> getOperand ( <NUM_LIT> ) . isImm ( ) ) { I1Defs . push_back ( Dst . getReg ( ) ) ; int64_t Val = DefInst -> getOperand ( <NUM_LIT> )


<s> const TargetRegisterClass * DstRC = MRI . getRegClass ( Dst . getReg ( ) ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( Src . getReg ( ) ) ; if ( DstRC == & <STR_LIT> && TRI -> getCommonSubClass ( SrcRC , & <STR_LIT> ) ) { I1Defs . push_back ( Dst . getReg ( ) ) ; DebugLoc DL = MI . getDebugLoc ( ) ; MachineInstr * DefInst = MRI . getUniqueVRegDef ( Src . getReg ( ) ) ; if ( DefInst -> getOpcode ( ) == <STR_LIT> ) { if ( DefInst -> getOperand ( <NUM_LIT> ) . isImm ( ) ) { I1Defs . push_back ( Dst . getReg ( ) ) ; int64_t Val = DefInst -> getOperand ( <NUM_LIT> ) . getImm ( ) ; assert ( Val == <NUM_LIT> || Val == - <NUM_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( Dst ) . addImm ( Val ) ; MI . eraseFromParent ( ) ; continue ; } } BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( Dst ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) . addOperand ( Src ) ; MI . eraseFromParent ( ) ; } else if ( TRI -> getCommonSubClass ( DstRC , & <STR_LIT> ) && SrcRC == & <STR_LIT> ) { BuildMI ( MBB , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addOperand ( Dst ) . addOperand ( Src ) . addImm ( <NUM_LIT> ) ; MI . eraseFromParent ( )


<s> for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; if ( MI . getOpcode ( ) == <STR_LIT> ) { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC = MRI . getRegClass ( Reg ) ; if ( RC == & <STR_LIT> ) MRI . setRegClass ( Reg , & <STR_LIT> ) ; continue ; } if ( MI . getOpcode ( ) != <STR_LIT> ) continue ; const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; const MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( Src . getReg ( ) ) || ! TargetRegisterInfo :: isVirtualRegister ( Dst . getReg ( ) ) ) continue ; const TargetRegisterClass * DstRC = MRI . getRegClass ( Dst . getReg ( ) ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( Src . getReg ( ) ) ; if ( DstRC == & <STR_LIT> && TRI -> getCommonSubClass ( SrcRC , & <STR_LIT> ) ) { I1Defs . push_back ( Dst . getReg ( ) ) ; DebugLoc DL = MI . getDebugLoc ( ) ; MachineInstr * DefInst = MRI . getUniqueVRegDef ( Src . getReg ( ) ) ; if ( DefInst -> getOpcode ( ) == <STR_LIT> ) { if ( DefInst -> getOperand ( <NUM_LIT> ) . isImm ( ) ) { I1Defs . push_back ( Dst . getReg ( ) ) ; int64_t Val = DefInst -> getOperand ( <NUM_LIT> ) . getImm ( ) ; assert ( Val == <NUM_LIT> || Val == - <NUM_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( Dst ) . addImm ( Val ) ; MI . eraseFromParent ( ) ; continue ; } } BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( Dst ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) . addOperand ( Src ) ; MI . eraseFromParent


<s> VisitedDom = DefBlock ; Stack . push_back ( DefBlock ) ; } else { VisitedPostDom = PDT . getNode ( VisitedPostDom ) -> getIDom ( ) -> getBlock ( ) ; VisitedDom = CommonDominators . back ( ) ; for ( unsigned i = <NUM_LIT> ; i < NextLevel . size ( ) ; ) { if ( PDT . dominates ( VisitedPostDom , NextLevel [ i ] ) ) { Stack . push_back ( NextLevel [ i ] ) ; NextLevel [ i ] = NextLevel . back ( ) ; NextLevel . pop_back ( ) ; } else { i ++ ; } } } unsigned Level = CommonDominators . size ( ) ; while ( ! Stack . empty ( ) ) { MachineBasicBlock * MBB = Stack . pop_back_val ( ) ; if ( ! PDT . dominates ( VisitedPostDom , MBB ) ) NextLevel . push_back ( MBB ) ; Visited [ MBB ] = Level ; VisitedDom = DT . findNearestCommonDominator ( VisitedDom , MBB ) ; for ( MachineBasicBlock * Succ : MBB -> successors ( ) ) { if ( Succ ==


<s> BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( CurReg ) ; } else if ( CurVal ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( ExecReg ) ; } else { BuildMI ( MBB , I , DL , TII -> get ( XorOp ) , DstReg ) . addReg ( ExecReg ) . addImm ( - <NUM_LIT> ) ; } return ; } unsigned PrevMaskedReg = <NUM_LIT> ; unsigned CurMaskedReg = <NUM_LIT> ; if ( ! PrevConstant ) { if ( CurConstant && CurVal ) { PrevMaskedReg = PrevReg ; } else { PrevMaskedReg = createLaneMaskReg ( * MF ) ; BuildMI ( MBB , I , DL , TII -> get ( AndN2Op ) , PrevMaskedReg ) . addReg ( PrevReg ) . addReg ( ExecReg ) ; } } if ( ! CurConstant ) { if ( PrevConstant && PrevVal ) { CurMaskedReg = CurReg ; } else { CurMaskedReg = createLaneMaskReg ( * MF ) ; BuildMI ( MBB , I , DL , TII -> get ( AndOp ) , CurMaskedReg ) . addReg ( CurReg ) . addReg ( ExecReg ) ; } } if ( PrevConstant && ! PrevVal ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( CurMaskedReg ) ; } else if ( CurConstant && ! CurVal ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( PrevMaskedReg ) ; } else if ( PrevConstant && PrevVal ) { BuildMI ( MBB , I , DL , TII -> get ( OrN2Op ) , DstReg

) . addReg ( CurMaskedReg ) . addReg ( ExecReg ) ;


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; return MRI . createVirtualRegister ( ST . isWave32 ( ) ? &


<s> static unsigned createLaneMaskReg ( MachineFunction & MF ) { const GCNSubtarget & ST = MF . getSubtarget <


<s> if ( ! VisitedPostDom ) advanceLevel ( ) ; unsigned Level = <NUM_LIT> ; while ( PDNode -> getBlock ( ) != PostDom ) { if ( PDNode -> getBlock ( ) == VisitedPostDom ) advanceLevel ( ) ; PDNode = PDNode -> getIDom ( ) ; Level ++ ; if ( FoundLoopLevel == Level )


<s> if ( PDNode -> getBlock ( ) == VisitedPostDom ) advanceLevel ( ) ; PDNode = PDNode -> getIDom ( ) ; Level ++ ; if ( FoundLoopLevel == Level ) return Level ; } return


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . setPreservesCFG ( ) ; AU . addRequired < MachineDominatorTree > ( ) ; AU . addRequired < MachinePostDominatorTree > ( ) ; MachineFunctionPass :: getAnalysisUsage ( AU


<s> AU . addRequired < MachineDominatorTree > ( ) ; AU . addRequired < MachinePostDominatorTree > ( ) ; MachineFunctionPass :: getAnalysisUsage (


<s> while ( InsertionPt != MBB . begin ( ) ) { InsertionPt -- ; bool DefSCC , UseSCC ; instrDefsUsesSCC ( * InsertionPt , DefSCC , UseSCC ) ; if ( DefSCC ) return InsertionPt ; } llvm_unreachable (


<s> Stack . clear ( ) ; NextLevel . clear ( ) ; VisitedPostDom =


<s> bool inLoopLevel ( MachineBasicBlock & MBB , unsigned LoopLevel , ArrayRef < MachineBasicBlock * > Blocks ) const { auto DomIt = Visited . find ( & MBB ) ; if ( DomIt != Visited . end ( ) && DomIt -> second <= LoopLevel


<s> auto DomIt = Visited . find ( & MBB ) ; if ( DomIt != Visited . end ( ) && DomIt -> second


<s> unsigned UndefReg = createLaneMaskReg ( MF ) ; BuildMI ( MBB , MBB . getFirstTerminator ( ) , { } ,


<s> Use = false ; for ( const MachineOperand & MO : MI . operands ( ) ) { if ( MO . isReg ( ) && MO . getReg ( ) == <STR_LIT> ) { if ( MO . isUse ( )


<s> for ( ; ; ) { MI = MRI -> getUniqueVRegDef ( Reg ) ; if ( MI -> getOpcode ( ) != <STR_LIT> ) break ; Reg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! Reg . isVirtual ( ) ) return false ; if ( ! isLaneMaskReg ( Reg ) ) return false ; } if ( MI -> getOpcode ( ) != MovOp ) return false ; if ( ! MI -> getOperand ( <NUM_LIT> ) . isImm ( ) ) return false ; int64_t Imm = MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( Imm == <NUM_LIT> ) { Val = false ; return true ; } if ( Imm == - <NUM_LIT> ) { Val = true ; return


<s> return TII -> getRegisterInfo ( ) . isSGPRReg ( * MRI , Reg ) && TII -> getRegisterInfo ( ) .

getRegSizeInBits ( Reg , * MRI ) == ST -> getWavefrontSize ( ) ;


<s> return ReachableMap . find ( & MBB ) ->


<s> bool isSource ( MachineBasicBlock & MBB ) const { return ReachableMap . find ( & MBB )


<s> static bool isVRegCompatibleReg ( const <ISA_LIT>RegisterInfo & TRI , const MachineRegisterInfo & MRI , Register Reg ) { unsigned Size = TRI . getRegSizeInBits ( Reg


<s> BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( - <NUM_LIT> ) . addReg ( SrcReg ) ; DeadCopies . push_back ( & MI ) ; } for ( MachineInstr * MI : DeadCopies ) MI -> eraseFromParent ( ) ; DeadCopies . clear


<s> SmallVector < MachineInstr * , <NUM_LIT> > DeadCopies ; for ( MachineBasicBlock & MBB : * MF ) { for ( MachineInstr & MI : MBB ) { if ( MI . getOpcode ( ) != <STR_LIT> ) continue ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! isVreg1 ( SrcReg )


<s> continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MRI -> setRegClass ( DstReg , IsWave32 ? & <STR_LIT> : & <STR_LIT> ) ; if ( MI . getOpcode ( ) == <STR_LIT> ) continue ; DebugLoc DL = MI . getDebugLoc ( ) ; Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( ! MI . getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ; if ( ! SrcReg . isVirtual ( ) || ( ! isLaneMaskReg ( SrcReg ) && ! isVreg1 ( SrcReg ) ) ) { assert ( TII -> getRegisterInfo ( ) . getRegSizeInBits ( SrcReg , * MRI ) == <NUM_LIT> ) ; unsigned TmpReg = createLaneMaskReg ( * MF ) ; BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SrcReg ) . addImm ( <NUM_LIT> ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( TmpReg ) ; SrcReg = TmpReg ; } std :: vector < MachineBasicBlock * > DomBlocks = { & MBB } ; for ( MachineInstr & Use : MRI -> use_instructions ( DstReg ) ) DomBlocks . push_back ( Use . getParent ( ) ) ; MachineBasicBlock * PostDomBound = PDT -> findNearestCommonDominator ( DomBlocks ) ; unsigned FoundLoopLevel = LF . findLoop ( PostDomBound ) ; if ( FoundLoopLevel ) { SSAUpdater . Initialize ( DstReg