<s> DT = & getAnalysis < DominatorTreeWrapperPass > ( ) . getDomTree ( ) ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; for ( df_iterator < BasicBlock * > I = df_begin ( & F . getEntryBlock ( ) ) , E = df_end ( & F . getEntryBlock ( ) ) ; I != E ; ++ I ) { BranchInst * Term = dyn_cast < BranchInst > ( ( * I ) -> getTerminator ( ) ) ; if ( ! Term || Term -> isUnconditional ( ) ) { if ( isTopOfStack ( * I ) ) closeControlFlow ( * I ) ; continue ; } if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( * I ) ) closeControlFlow ( * I ) ; handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( * I ) ) { PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( ) ) ; if ( Phi && Phi -> getParent ( ) == * I && isElse ( Phi ) ) { insertElse (


<s> bool <ISA_LIT>AnnotateControlFlow :: runOnFunction ( Function & F ) { DT = & getAnalysis < DominatorTreeWrapperPass > ( ) . getDomTree ( ) ; LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; for ( df_iterator < BasicBlock * > I = df_begin ( & F . getEntryBlock ( ) ) , E = df_end ( & F . getEntryBlock ( ) ) ; I != E ; ++ I ) { BranchInst * Term = dyn_cast < BranchInst > ( ( * I ) -> getTerminator ( ) ) ; if ( ! Term || Term -> isUnconditional ( ) ) { if ( isTopOfStack ( * I ) ) closeControlFlow ( * I ) ; continue ; } if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( * I ) ) closeControlFlow ( * I ) ; handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( * I ) ) { PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( ) ) ; if ( Phi && Phi -> getParent ( ) == * I && isElse ( Phi ) ) { insertElse ( Term )


<s> SmallVector < BasicBlock * , <NUM_LIT> > Preds ; for ( BasicBlock * Pred : predecessors ( BB ) ) { if ( ! is_contained ( Latches , Pred ) ) Preds . push_back ( Pred ) ; } BB = SplitBlockPredecessors ( BB , Preds , <STR_LIT> , DT , LI , nullptr , false ) ; } Value * Exec = popSaved ( ) ; Instruction * FirstInsertionPt = & * BB ->


<s> AU . addRequired < LoopInfoWrapperPass > ( ) ; AU . addRequired < DominatorTreeWrapperPass > ( ) ; AU . addRequired < LegacyDivergenceAnalysis > ( ) ; AU . addPreserved < LoopInfoWrapperPass > ( ) ; AU . addPreserved < DominatorTreeWrapperPass >


<s> AU . addRequired < LegacyDivergenceAnalysis > ( ) ; AU . addPreserved < LoopInfoWrapperPass > ( ) ; AU . addPreserved < DominatorTreeWrapperPass > ( ) ; AU . addRequired < TargetPassConfig > ( ) ; FunctionPass :: getAnalysisUsage ( AU


<s> bool <ISA_LIT>AnnotateControlFlow :: handleLoop ( BranchInst * Term ) { if ( isUniform ( Term ) ) return false ; BasicBlock * BB = Term -> getParent ( ) ; llvm :: Loop * L = LI -> getLoopFor ( BB ) ; if ( ! L ) return false ; BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( IntMask , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) { Value * PHIValue = IntMaskZero


<s> BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( IntMask , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) { Value * PHIValue = IntMaskZero ; if ( Pred == BB ) PHIValue = Arg ; else if ( L -> contains ( Pred ) && DT -> dominates ( Pred , BB ) ) PHIValue = Broken ; Broken -> addIncoming ( PHIValue , Pred ) ; } Term -> setCondition ( CallInst :: Create ( Loop , Arg , <STR_LIT> , Term ) ) ; push ( Term -> getSuccessor ( <NUM_LIT> ) , Arg ) ; return


<s> Insert = Parent -> getTerminator ( ) ; } else { Insert = L -> getHeader ( ) -> getFirstNonPHIOrDbgOrLifetime ( ) ; } Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak , Args , <STR_LIT> , Insert ) ; } if ( isa < Constant > ( Cond ) ) { Instruction * Insert = Cond == BoolTrue ? Term : L -> getHeader ( ) -> getTerminator ( ) ; Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak , Args , <STR_LIT> ,


<s> bool <ISA_LIT>AnnotateControlFlow :: hasKill ( const BasicBlock * BB ) { for ( const Instruction & I : * BB ) { if ( const CallInst * CI = dyn_cast < CallInst > ( & I ) ) if ( CI -> getIntrinsicID (


<s> LLVMContext & Context = M . getContext ( ) ; Void = Type :: getVoidTy ( Context ) ; Boolean = Type :: getInt1Ty ( Context ) ; IntMask = ST . isWave32 ( ) ? Type :: getInt32Ty ( Context ) : Type :: getInt64Ty ( Context ) ; ReturnStruct = StructType :: get ( Boolean , IntMask ) ; BoolTrue = ConstantInt :: getTrue ( Context ) ; BoolFalse = ConstantInt :: getFalse ( Context ) ; BoolUndef = UndefValue :: get ( Boolean ) ; IntMaskZero = ConstantInt :: get ( IntMask , <NUM_LIT> ) ; If = <STR_LIT> ( & M , <STR_LIT> , { IntMask } ) ; Else = <STR_LIT> ( & M , <STR_LIT> , { IntMask , IntMask } ) ; IfBreak = <STR_LIT> ( & M ,


<s> bool <ISA_LIT>AnnotateControlFlow :: insertElse ( BranchInst * Term ) { if ( isUniform ( Term ) ) { return false ; } Value * Ret = CallInst :: Create ( Else , popSaved ( ) , <STR_LIT> , Term ) ; Term -> setCondition ( ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ; push ( Term -> getSuccessor ( <NUM_LIT> ) , ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ; return


<s> return DA -> isUniform ( T ) || T -> getMetadata


<s> if ( isUniform ( Term ) ) return false ; Value * Ret = CallInst :: Create ( If ,

Term -> getCondition ( ) , <STR_LIT> , Term ) ;


<s> AU . addPreserved < DominatorTreeWrapperPass > ( ) ; AU . addRequired < TargetPassConfig > ( ) ; FunctionPass :: getAnalysisUsage ( AU )


<s> llvm :: Loop * L = LI -> getLoopFor ( BB ) ; if ( ! L ) return ; BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( IntMask , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) { Value * PHIValue = IntMaskZero ; if ( Pred == BB ) PHIValue = Arg ; else if ( L -> contains ( Pred ) && DT -> dominates ( Pred , BB ) ) PHIValue = Broken ; Broken -> addIncoming ( PHIValue , Pred ) ; } Term -> setCondition ( CallInst :: Create ( Loop , Arg , <STR_LIT> , Term ) ) ; push ( Term -> getSuccessor ( <NUM_LIT>


<s> Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak , Args , <STR_LIT> , Insert ) ; } if ( isa < Constant > ( Cond ) ) { Instruction * Insert = Cond == BoolTrue ? Term : L -> getHeader ( ) -> getTerminator ( ) ; Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak , Args , <STR_LIT> , Insert ) ; } llvm_unreachable ( <STR_LIT> )


<s> } else { Insert = L -> getHeader ( ) -> getFirstNonPHIOrDbgOrLifetime ( ) ; } Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak , Args , <STR_LIT> , Insert ) ; } if ( isa < Constant > ( Cond


<s> LI = & getAnalysis < LoopInfoWrapperPass > ( ) . getLoopInfo ( ) ; DA = & getAnalysis < LegacyDivergenceAnalysis > ( ) ; TargetPassConfig & TPC = getAnalysis < TargetPassConfig > ( ) ; const TargetMachine & TM = TPC . getTM < TargetMachine > ( ) ; initialize ( * F . getParent ( ) , TM . getSubtarget < GCNSubtarget > ( F ) ) ; for ( df_iterator < BasicBlock * > I = df_begin ( & F . getEntryBlock ( ) ) , E = df_end ( & F . getEntryBlock ( ) ) ; I != E ; ++ I ) { BasicBlock * BB = * I ; BranchInst * Term = dyn_cast < BranchInst > ( BB -> getTerminator ( ) ) ; if ( ! Term || Term -> isUnconditional ( ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; continue ; } if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; if ( DT -> dominates ( Term -> getSuccessor ( <NUM_LIT> ) , BB ) ) handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( BB ) ) { PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( ) ) ; if ( Phi && Phi -> getParent ( ) == BB && isElse ( Phi ) ) { insertElse ( Term ) ; eraseIfUnused ( Phi ) ; continue ; } closeControlFlow ( BB ) ; } openIf ( Term ) ; } if ( ! Stack . empty ( ) ) { report_fatal_error ( <STR_LIT>


<s> L -> getLoopLatches ( Latches ) ; std :: vector < BasicBlock * > Preds ; for ( pred_iterator PI = pred_begin ( BB ) , PE = pred_end ( BB ) ; PI != PE ; ++ PI ) { if ( std :: find ( Latches . begin ( ) , Latches . end ( ) , * PI ) == Latches . end ( ) ) Preds . push_back ( * PI ) ; } BB = llvm :: SplitBlockPredecessors ( BB , Preds , <STR_LIT> , DT , LI , false ) ; } Value * Exec = popSaved ( ) ; if ( ! isa < UndefValue > ( Exec ) ) CallInst :: Create ( EndCf

, Exec , <STR_LIT> , & * BB -> getFirstInsertionPt ( ) ) ;


<s> BasicBlock * BB = Term -> getParent ( ) ; llvm :: Loop * L = LI -> getLoopFor ( BB ) ; if ( ! L ) return ; BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term ->


<s> Value * Ret = NewPhi ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( isa < ConstantInt > ( Incoming ) ) { NewPhi -> addIncoming ( Broken , From ) ; continue ; } Phi -> setIncomingValue ( i , BoolFalse ) ; Value * PhiArg = handleLoopCondition ( Incoming , Broken , L , Term ) ; NewPhi -> addIncoming ( PhiArg , From ) ; } BasicBlock * IDom = DT -> getNode ( Parent ) -> getIDom ( ) -> getBlock ( ) ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; if ( Incoming != BoolTrue ) continue ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( From == IDom ) { CallInst * OldEnd = dyn_cast < CallInst > ( Parent -> getFirstInsertionPt ( ) ) ; while ( OldEnd && OldEnd -> getCalledFunction ( ) != EndCf ) OldEnd = dyn_cast < CallInst > ( OldEnd -> getNextNode ( ) ) ; if ( OldEnd && OldEnd -> getCalledFunction ( ) == EndCf ) { Value * Args [ ] = { OldEnd -> getArgOperand ( <NUM_LIT> ) , NewPhi } ; Ret = CallInst :: Create ( ElseBreak , Args , <STR_LIT> , OldEnd ) ; continue ; } } TerminatorInst * Insert = From -> getTerminator ( ) ; Value * PhiArg = CallInst :: Create ( Break , Broken , <STR_LIT> , Insert ) ; NewPhi -> setIncomingValue ( i , PhiArg ) ; } eraseIfUnused ( Phi ) ; return Ret ; } else if ( Instruction * Inst = dyn_cast < Instruction > ( Cond ) ) { BasicBlock * Parent = Inst -> getParent ( ) ; Instruction * Insert ; if ( L -> contains ( Inst ) ) { Insert = Parent -> getTerminator ( ) ; } else { Insert = L -> getHeader ( ) -> getFirstNonPHIOrDbgOrLifetime ( ) ; } Value * Args [ ] = { Cond


<s> Value * Ret = CallInst :: Create ( If , Term -> getCondition ( ) , <STR_LIT> , Term ) ; Term -> setCondition ( ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT>


<s> continue ; } if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( * I ) ) closeControlFlow ( * I ) ; handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( * I ) ) { PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( ) ) ; if ( Phi && Phi -> getParent ( ) == * I && isElse ( Phi ) ) { insertElse ( Term )


<s> BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ; for ( pred_iterator PI = pred_begin ( Target ) , PE = pred_end ( Target ) ; PI != PE ; ++


<s> BasicBlock * Parent = Phi -> getParent ( ) ; PHINode * NewPhi = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Parent -> front ( ) ) ; Value * Ret = NewPhi ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( isa < ConstantInt > ( Incoming ) ) { NewPhi -> addIncoming ( Broken , From ) ; continue ; } Phi -> setIncomingValue ( i , BoolFalse ) ; Value * PhiArg = handleLoopCondition ( Incoming , Broken , L , Term ) ; NewPhi -> addIncoming ( PhiArg , From ) ; } BasicBlock * IDom = DT -> getNode ( Parent ) -> getIDom ( ) -> getBlock ( ) ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; if ( Incoming != BoolTrue ) continue ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( From == IDom ) { CallInst * OldEnd = dyn_cast < CallInst > ( Parent -> getFirstInsertionPt ( ) ) ; while ( OldEnd && OldEnd -> getCalledFunction ( ) != EndCf ) OldEnd = dyn_cast < CallInst > ( OldEnd -> getNextNode ( ) ) ; if ( OldEnd && OldEnd -> getCalledFunction ( ) == EndCf ) { Value * Args [ ] = { OldEnd -> getArgOperand ( <NUM_LIT> ) , NewPhi } ; Ret = CallInst :: Create ( ElseBreak , Args , <STR_LIT> , OldEnd ) ; continue ; } } TerminatorInst * Insert = From -> getTerminator ( ) ; Value * PhiArg = CallInst :: Create ( Break , Broken , <STR_LIT> , Insert ) ; NewPhi -> setIncomingValue ( i


<s> std :: vector < BasicBlock * > Preds ; for ( pred_iterator PI = pred_begin ( BB ) , PE = pred_end ( BB ) ; PI != PE ; ++ PI ) { if ( std :: find ( Latches . begin ( ) , Latches . end ( ) , * PI ) == Latches . end ( ) ) Preds . push_back ( * PI ) ; } BB = llvm :: SplitBlockPredecessors ( BB , Preds , <STR_LIT> , nullptr , DT , LI , false ) ; } CallInst :: Create ( EndCf , popSaved ( ) , <STR_LIT> , BB -> getFirstInsertionPt


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < LoopInfoWrapperPass > ( ) ; AU . addRequired < DominatorTreeWrapperPass


<s> void <ISA_LIT>AnnotateControlFlow :: handleLoop ( BranchInst * Term ) { if ( isUniform ( Term ) ) return ; BasicBlock * BB = Term -> getParent ( ) ; llvm :: Loop * L = LI -> getLoopFor ( BB ) ; if ( ! L ) return ; BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) { Value * PHIValue = Int64Zero ; if ( Pred == BB ) PHIValue


<s> if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; continue ; } if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( BB ) ) { PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( ) ) ; if ( Phi && Phi -> getParent ( ) == BB && isElse ( Phi ) ) { insertElse ( Term ) ; eraseIfUnused ( Phi ) ; continue ; } closeControlFlow ( BB


<s> BranchInst * Term = dyn_cast < BranchInst > ( BB -> getTerminator ( ) ) ; if ( ! Term || Term -> isUnconditional ( ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; continue ; } if ( I . nodeVisited ( Term -> getSuccessor ( <NUM_LIT> ) ) ) { if ( isTopOfStack ( BB ) ) closeControlFlow ( BB ) ; handleLoop ( Term ) ; continue ; } if ( isTopOfStack ( BB ) ) { PHINode * Phi = dyn_cast < PHINode > ( Term -> getCondition ( )


<s> llvm :: Loop * L = LI -> getLoopFor ( BB ) ; assert ( Stack . back ( ) . first == BB ) ; if ( L && L -> getHeader ( ) == BB ) { SmallVector < BasicBlock * , <NUM_LIT> > Latches ; L -> getLoopLatches ( Latches ) ; std :: vector < BasicBlock * > Preds ; for ( pred_iterator PI = pred_begin ( BB ) , PE = pred_end ( BB ) ; PI != PE ; ++ PI ) { if ( ! is_contained ( Latches , * PI ) ) Preds . push_back ( * PI ) ; } BB = llvm :: SplitBlockPredecessors ( BB , Preds , <STR_LIT> , DT , LI , false ) ; } Value * Exec = popSaved ( ) ; if ( ! isa < UndefValue > ( Exec ) ) CallInst :: Create ( EndCf , Exec , <STR_LIT> , &


<s> Else = M . getOrInsertFunction ( ElseIntrinsic , ReturnStruct , Int64 , ( Type * ) nullptr ) ; Break = M . getOrInsertFunction ( BreakIntrinsic , Int64 , Int64 , ( Type * ) nullptr ) ; cast < Function > ( Break ) -> setDoesNotAccessMemory ( ) ; IfBreak = M . getOrInsertFunction ( IfBreakIntrinsic , Int64 , Boolean , Int64 , ( Type * ) nullptr ) ; cast < Function > ( IfBreak ) -> setDoesNotAccessMemory ( ) ; ; ElseBreak = M . getOrInsertFunction ( ElseBreakIntrinsic , Int64 , Int64 , Int64 , ( Type * ) nullptr ) ; cast < Function > ( ElseBreak ) -> setDoesNotAccessMemory ( ) ; Loop = M . getOrInsertFunction ( LoopIntrinsic , Boolean


<s> void <ISA_LIT>AnnotateControlFlow :: handleLoop ( BranchInst * Term ) { if ( isUniform ( Term ) ) return ; BasicBlock * BB = Term -> getParent ( ) ; llvm :: Loop * L = LI -> getLoopFor ( BB ) ; if ( ! L ) return ; BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) Broken -> addIncoming ( Pred ==


<s> void <ISA_LIT>AnnotateControlFlow :: handleLoop ( BranchInst * Term ) { if ( isUniform ( Term ) ) return ; BasicBlock * BB = Term -> getParent ( ) ; llvm :: Loop * L = LI -> getLoopFor ( BB ) ; if ( ! L ) return ; BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) Broken -> addIncoming ( Pred == BB ? Arg : Int64Zero


<s> ReturnStruct = StructType :: get ( Boolean , Int64 , ( Type * ) nullptr ) ; BoolTrue = ConstantInt :: getTrue ( Context ) ; BoolFalse = ConstantInt :: getFalse ( Context ) ; BoolUndef = UndefValue :: get ( Boolean ) ; Int64Zero = ConstantInt :: get ( Int64 , <NUM_LIT> ) ; If = <STR_LIT> ( & M , <STR_LIT> ) ; Else = <STR_LIT> ( & M , <STR_LIT> ) ; Break = <STR_LIT> ( & M , <STR_LIT> ) ; IfBreak = <STR_LIT> ( & M , <STR_LIT> ) ; ElseBreak = <STR_LIT> ( & M , <STR_LIT> ) ; Loop = <STR_LIT> ( & M , <STR_LIT> ) ; EndCf = <STR_LIT> ( & M , <STR_LIT> ) ; return


<s> void <ISA_LIT>AnnotateControlFlow :: eraseIfUnused ( PHINode * Phi


<s> Value * Arg = handleLoopCondition ( Cond , Broken , L , Term , LoopPhiConditions ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) Broken -> addIncoming ( Pred == BB ? Arg : Int64Zero , Pred ) ; Term -> setCondition ( CallInst :: Create ( Loop , Arg , <STR_LIT> , Term ) ) ; for ( WeakVH Val : reverse ( LoopPhiConditions ) ) { if ( PHINode * Cond = cast_or_null < PHINode > (


<s> } Phi -> setIncomingValue ( i , BoolFalse ) ; Value * PhiArg = handleLoopCondition ( Incoming , Broken , L , Term , LoopPhiConditions ) ; NewPhi -> addIncoming ( PhiArg , From ) ; } BasicBlock * IDom = DT -> getNode ( Parent ) -> getIDom ( ) -> getBlock ( ) ; for ( unsigned i = <NUM_LIT> , e = Phi -> getNumIncomingValues ( ) ; i != e ; ++ i ) { Value * Incoming = Phi -> getIncomingValue ( i ) ; if ( Incoming != BoolTrue ) continue ; BasicBlock * From = Phi -> getIncomingBlock ( i ) ; if ( From == IDom ) { CallInst * OldEnd = dyn_cast < CallInst > ( Parent -> getFirstInsertionPt ( ) ) ; while ( OldEnd && OldEnd -> getCalledFunction ( ) != EndCf ) OldEnd = dyn_cast < CallInst > ( OldEnd -> getNextNode ( ) ) ; if ( OldEnd && OldEnd -> getCalledFunction ( ) == EndCf ) { Value * Args [ ] = { OldEnd -> getArgOperand ( <NUM_LIT> ) , NewPhi } ; Ret = CallInst :: Create ( ElseBreak , Args , <STR_LIT> , OldEnd ) ; continue ; } } TerminatorInst * Insert = From -> getTerminator ( ) ; Value * PhiArg = CallInst :: Create ( Break , Broken , <STR_LIT> , Insert ) ; NewPhi -> setIncomingValue ( i , PhiArg ) ; } LoopPhiConditions . push_back ( WeakVH ( Phi ) ) ; return Ret ; } if ( Instruction * Inst = dyn_cast < Instruction


<s> if ( From == IDom ) { CallInst * OldEnd = dyn_cast < CallInst > ( Parent -> getFirstInsertionPt ( ) ) ; while ( OldEnd && OldEnd -> getCalledFunction ( ) != EndCf ) OldEnd = dyn_cast < CallInst > ( OldEnd -> getNextNode ( ) ) ; if ( OldEnd && OldEnd -> getCalledFunction ( ) == EndCf ) { Value * Args [ ] = { OldEnd -> getArgOperand ( <NUM_LIT> ) , NewPhi } ; Ret = CallInst :: Create ( ElseBreak , Args , <STR_LIT> , OldEnd ) ; continue ; } } TerminatorInst * Insert = From -> getTerminator ( ) ; Value * PhiArg = CallInst :: Create ( Break , Broken , <STR_LIT> , Insert ) ; NewPhi -> setIncomingValue ( i , PhiArg ) ; } LoopPhiConditions . push_back ( WeakVH ( Phi ) ) ; return Ret ; } if ( Instruction * Inst = dyn_cast < Instruction > ( Cond ) ) { BasicBlock * Parent = Inst -> getParent ( ) ; Instruction * Insert ; if ( L -> contains ( Inst ) ) { Insert = Parent -> getTerminator ( ) ; } else { Insert = L -> getHeader ( ) -> getFirstNonPHIOrDbgOrLifetime ( ) ; } Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak , Args , <STR_LIT> , Insert ) ; } if ( isa < Constant > ( Cond ) ) { Instruction * Insert = Cond == BoolTrue ? Term : L -> getHeader ( ) -> getTerminator ( ) ; Value * Args [ ] = { Cond , Broken } ; return CallInst :: Create ( IfBreak ,


<s> if ( Stack . back ( ) . first != BB ) return ; if ( L && L -> getHeader ( ) == BB ) { SmallVector < BasicBlock * , <NUM_LIT> > Latches ; L -> getLoopLatches ( Latches ) ; std :: vector < BasicBlock * > Preds ; for ( pred_iterator PI = pred_begin ( BB ) , PE = pred_end ( BB ) ; PI != PE ; ++ PI ) { if ( std :: find ( Latches . begin ( ) , Latches . end ( ) , * PI

) == Latches . end ( ) ) Preds . push_back ( * PI ) ;


<s> void <ISA_LIT>AnnotateControlFlow :: handleLoop ( BranchInst * Term ) { if ( DA -> isUniform ( Term -> getCondition ( ) ) ) { return ; } BasicBlock * BB = Term -> getParent ( ) ; llvm :: Loop * L = LI -> getLoopFor ( BB ) ; BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term


<s> Value * Ret = CallInst :: Create ( Else , popSaved ( ) , <STR_LIT> , Term ) ; Term -> setCondition ( ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ; push ( Term -> getSuccessor ( <NUM_LIT> ) , ExtractValueInst :: Create ( Ret , <NUM_LIT>


<s> Value * Ret = CallInst :: Create ( Else , popSaved ( ) , <STR_LIT> , Term ) ; Term -> setCondition ( ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT> , Term ) ) ; push ( Term -> getSuccessor ( <NUM_LIT> ) , ExtractValueInst :: Create


<s> Value * Ret = CallInst :: Create ( If , Term -> getCondition ( ) , <STR_LIT> , Term ) ; Term -> setCondition ( ExtractValueInst :: Create ( Ret , <NUM_LIT> , <STR_LIT>


<s> void <ISA_LIT>AnnotateControlFlow :: openIf ( BranchInst * Term ) { if ( DA -> isUniform ( Term -> getCondition ( ) ) ) { return ; } Value * Ret = CallInst :: Create ( If , Term -> getCondition ( ) ,


<s> BasicBlock * Target = Term -> getSuccessor ( <NUM_LIT> ) ; PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L


<s> Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term , LoopPhiConditions ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) Broken -> addIncoming ( Pred == BB ? Arg : Int64Zero , Pred ) ; Term -> setCondition ( CallInst :: Create ( Loop , Arg , <STR_LIT>


<s> PHINode * Broken = PHINode :: Create ( Int64 , <NUM_LIT> , <STR_LIT> , & Target -> front ( ) ) ; SmallVector < WeakTrackingVH , <NUM_LIT> > LoopPhiConditions ; Value * Cond = Term -> getCondition ( ) ; Term -> setCondition ( BoolTrue ) ; Value * Arg = handleLoopCondition ( Cond , Broken , L , Term , LoopPhiConditions ) ; for ( BasicBlock * Pred : predecessors ( Target ) ) Broken -> addIncoming ( Pred == BB


<s> StringRef getPassName ( ) const override { return PASS_NAME


<s> StringRef getPassName ( ) const override { return


<s> auto DL = MI -> getDebugLoc ( ) ; if ( NopInserted . find ( DL . getLine ( ) ) == NopInserted . end ( ) ) { BuildMI ( MBB , * MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; NopInserted . insert ( DL


<s> if ( ! ST . debuggerInsertNops ( ) ) return false ; if ( ! MF . getMMI ( ) . hasDebugInfo ( ) ) return false ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; DenseSet < unsigned > NopInserted ; for ( auto & MBB : MF ) { for ( auto MI = MBB . begin ( ) ; MI != MBB . end ( ) ; ++ MI ) { if ( MI -> isDebugInstr ( ) || ! MI -> getDebugLoc ( ) ) continue ; auto DL = MI -> getDebugLoc ( ) ; if ( NopInserted . find ( DL . getLine ( )


<s> if ( ! ST . debuggerInsertNops ( ) ) return false ; if ( ! MF . getMMI ( ) . hasDebugInfo ( ) ) return false ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; DenseSet < unsigned > NopInserted ; for ( auto & MBB : MF ) { for ( auto MI = MBB . begin ( ) ; MI != MBB . end


<s> DenseSet < unsigned > NopInserted ; for ( auto & MBB : MF ) { for ( auto MI = MBB . begin ( ) ; MI != MBB . end ( ) ; ++ MI ) { if ( MI -> isDebugValue ( ) || ! MI -> getDebugLoc ( ) ) continue ; auto DL = MI -> getDebugLoc ( ) ; if ( NopInserted . find ( DL . getLine ( ) ) == NopInserted . end ( ) ) { BuildMI ( MBB , * MI , DL , TII -> get ( <STR_LIT>


<s> FunctionPass * llvm :: create<ISA_LIT>DebuggerInsertNopsPass ( ) { return new <ISA_LIT>DebuggerInsertNops ( )


<s> return new <ISA_LIT>DebuggerInsertNops


<s> return new <ISA_LIT>FixControlFlowLiveIntervals (


<s> FunctionPass * llvm :: create<ISA_LIT>FixSGPRCopiesPass ( ) { return new <ISA_LIT>FixSGPRCopies


<s> unsigned SrcReg = Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * SrcRC = TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ? MRI . getRegClass ( SrcReg ) : TRI . getPhysRegClass ( SrcReg ) ; const TargetRegisterClass * DstRC = TargetRegisterInfo :: isVirtualRegister ( DstReg ) ? MRI . getRegClass ( DstReg ) : TRI . getPhysRegClass ( DstReg ) ; return std :: make_pair ( SrcRC ,


<s> static std :: pair < const TargetRegisterClass * , const TargetRegisterClass * > getCopyRegClasses ( const MachineInstr & Copy , const <ISA_LIT>RegisterInfo & TRI , const MachineRegisterInfo & MRI ) { unsigned DstReg = Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned SrcReg = Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * SrcRC = TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ? MRI . getRegClass ( SrcReg ) : TRI . getPhysRegClass ( SrcReg )


<s> const MachineRegisterInfo & MRI = MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( ! MI . getOperand ( i ) . isReg ( ) || ! TargetRegisterInfo :: isVirtualRegister ( MI . getOperand ( i ) . getReg ( ) ) ) continue ; if ( TRI -> hasVGPRs ( MRI . getRegClass ( MI . getOperand ( i ) . getReg


<s> static bool isSGPRToVGPRCopy ( const TargetRegisterClass * SrcRC , const TargetRegisterClass * DstRC , const <ISA_LIT>RegisterInfo &


<s> static bool isVGPRToSGPRCopy ( const TargetRegisterClass * SrcRC , const TargetRegisterClass * DstRC , const <ISA_LIT>RegisterInfo & TRI ) { return TRI . isSGPRClass ( DstRC ) && TRI . hasVGPRs


<s> return TRI . isSGPRClass ( DstRC ) && TRI . hasVGPRs


<s> switch ( MI . getOpcode ( ) ) { default : continue ; case <STR_LIT> : { if ( ! TargetRegisterInfo :: isVirtualRegister ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( MI , * TRI , MRI ) ; if ( isVGPRToSGPRCopy ( SrcRC , DstRC , * TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool SGPRBranch = false ; bool HasBreakDef = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; if ( TRI -> hasVGPRs ( MRI . getRegClass ( Reg ) ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefInstr = MRI . getUniqueVRegDef ( Reg ) ; assert ( DefInstr ) ; switch ( DefInstr -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : HasBreakDef = true ; break ; } } if ( ! SGPRBranch && ! HasBreakDef ) TII -> moveToVALU ( MI ) ; break ; } case <STR_LIT>


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < MachineDominatorTree > (


<s> StringRef getPassName ( ) const override { return <STR_LIT>


<s> for ( MachineBasicBlock :: const_iterator I = MBB . getFirstTerminator ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { if ( I -> modifiesRegister ( <STR_LIT> , & TRI ) ) return true ; } return false


<s> static bool isReachable ( const MachineInstr * From , const MachineInstr * To , const MachineBasicBlock * CutOff , MachineDominatorTree & MDT ) { if ( MDT . dominates ( From , To ) ) return true ; const MachineBasicBlock * MBBFrom = From -> getParent (


<s> for ( unsigned i = <NUM_LIT> ; i < PHI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned Reg = PHI . getOperand ( i ) . getReg ( ) ; if ( Visited . count ( Reg ) ) continue ; Visited . insert ( Reg ) ; MachineInstr * DefInstr = MRI . getVRegDef ( Reg ) ; switch ( DefInstr -> getOpcode ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : case


<s> static bool phiHasVGPROperands ( const MachineInstr & PHI , const MachineRegisterInfo & MRI , const <ISA_LIT>RegisterInfo * TRI , const <ISA_LIT>InstrInfo


<s> return searchPredecessors ( MBB , nullptr , [ TRI ] ( MachineBasicBlock


<s> return searchPredecessors ( MBB , nullptr , [ TRI ] ( MachineBasicBlock * MBB ) { return hasTerminatorThatModifiesExec ( * MBB ,


<s> case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( phiHasVGPROperands ( MI , MRI , TRI , TII ) || ! phiHasBreakDef ( MI , MRI , Visited ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; break ; } case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg (


<s> SmallVector < MachineBasicBlock * , <NUM_LIT> > Worklist ( MBB -> pred_begin ( ) , MBB -> pred_end ( ) ) ; while ( ! Worklist . empty ( ) ) { MachineBasicBlock * MBB = Worklist


<s> unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( DstReg ) ) ) return false ; if ( ! MRI . hasOneUse ( DstReg ) ) return false ; MachineInstr & CopyUse = * MRI . use_instr_begin ( DstReg ) ; if ( ! CopyUse . isCopy ( ) ) return false ; if ( TargetRegisterInfo :: isPhysicalRegister ( CopyUse . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) return false ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( CopyUse , * TRI , MRI ) ; if ( ! isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) return false ; if ( tryChangeVGPRtoSGPRinCopy ( CopyUse , TRI , TII ) ) return true ; unsigned SubReg = CopyUse . getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( SubReg != <STR_LIT> ) return false ; MRI . setRegClass ( DstReg , DstRC ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( CopyUse . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { unsigned SrcReg = MI . getOperand ( I ) . getReg ( ) ; unsigned SrcSubReg = MI . getOperand ( I ) . getSubReg ( ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( SrcReg ) ; assert ( TRI -> isSGPRClass ( SrcRC ) && <STR_LIT> ) ; SrcRC = TRI -> getSubRegClass ( SrcRC ,


<s> for ( auto I2 = std :: next ( I1 ) ; I2 != E ; ) { MachineInstr * MI2 = * I2 ; auto intereferes = [ & ] ( MachineBasicBlock :: iterator From , MachineBasicBlock :: iterator To ) -> bool { assert ( MDT . dominates ( & * To , & * From ) ) ; auto interferes = [ & MDT , From , To ] ( MachineInstr * & Clobber ) -> bool { const MachineBasicBlock * MBBFrom = From -> getParent ( ) ; const MachineBasicBlock * MBBTo = To -> getParent ( ) ; bool MayClobberFrom = isReachable ( Clobber , & * From , MBBTo , MDT ) ; bool MayClobberTo = isReachable ( Clobber , & * To , MBBTo , MDT ) ; if ( ! MayClobberFrom && ! MayClobberTo ) return false ; if ( ( MayClobberFrom && ! MayClobberTo ) || ( ! MayClobberFrom && MayClobberTo ) ) return true ; return ! ( ( MBBFrom == MBBTo && MDT . dominates ( Clobber , & * From ) && MDT . dominates ( Clobber , & * To ) ) || MDT . properlyDominates ( Clobber -> getParent ( ) , MBBTo ) ) ; } ; return ( llvm :: any_of ( Clobbers , interferes ) ) || ( llvm :: any_of ( Inits , [ & ] ( InitListMap :: value_type & C ) { return C . first != Init . first && llvm :: any_of ( C . second , interferes ) ; } ) ) ; } ; if ( MDT . dominates ( MI1 , MI2 ) ) { if ( ! intereferes ( MI2 , MI1 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MI2 -> getParent ( ) ) << <STR_LIT> << * MI2 ) ; MI2 -> eraseFromParent ( ) ; Defs . erase ( I2 ++ ) ; Changed = true ; continue ; } } else if ( MDT . dominates ( MI2 , MI1 ) ) { if ( ! intereferes ( MI1 , MI2 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MI1 -> getParent ( ) ) << <STR_LIT> << * MI1 ) ; MI1 -> eraseFromParent ( ) ; Defs . erase ( I1 ++ ) ; Changed = true ; break ; } } else { auto * MBB = MDT . findNearestCommonDominator ( MI1 -> getParent ( ) , MI2 -> getParent ( ) ) ; if ( ! MBB ) { ++ I2 ; continue ; } MachineBasicBlock :: iterator I = MBB -> getFirstNonPHI ( ) ; if ( ! intereferes ( MI1 , I ) && ! intereferes ( MI2 , I ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MI1 -> getParent ( ) ) << <STR_LIT> << * MI1 << <STR_LIT> << printMBBReference ( * MI2 -> getParent ( ) ) <<

<STR_LIT> << printMBBReference ( * I -> getParent ( ) ) << <STR_LIT> << * MI2 ) ;


<s> static bool isSGPRToVGPRCopy ( const TargetRegisterClass * SrcRC , const TargetRegisterClass * DstRC , const <ISA_LIT>RegisterInfo & TRI ) { return DstRC != & <STR_LIT> && TRI . isSGPRClass ( SrcRC ) && TRI


<s> return DstRC != & <STR_LIT> && TRI . isSGPRClass (


<s> return SrcRC != & <STR_LIT> && TRI . isSGPRClass ( DstRC ) && TRI . hasVGPRs


<s> break ; } TII -> moveToVALU ( MI , MDT ) ; } else if ( isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) { tryChangeVGPRtoSGPRinCopy ( MI , TRI , TII ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool HasVGPROperand = phiHasVGPROperands ( MI , MRI , TRI , TII ) ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> && ! HasVGPROperand ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( HasVGPROperand || ! phiHasBreakDef ( MI , MRI , Visited ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI , MDT ) ; } break ; } case <STR_LIT> : if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI , MDT ) ; break ; case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( TRI -> isSGPRClass ( DstRC ) && ( TRI -> hasVGPRs ( Src0RC ) || TRI -> hasVGPRs ( Src1RC ) ) ) { LLVM_DEBUG ( dbgs ( )


<s> SmallVector < MachineBasicBlock * , <NUM_LIT> > Worklist ( MBB -> pred_begin ( ) , MBB -> pred_end ( ) ) ; while ( ! Worklist . empty ( ) ) { MachineBasicBlock * MBB = Worklist . pop_back_val


<s> const auto * UseMI = MO . getParent ( ) ; if ( UseMI == & MI ) continue ; if ( MO . isDef ( ) || UseMI -> getParent ( ) != MI . getParent ( ) || UseMI -> getOpcode ( ) <= TargetOpcode :: GENERIC_OP_END || ! TII -> isOperandLegal ( * UseMI , UseMI -> getOperandNo ( & MO ) , & Src ) ) return false ; } MRI . setRegClass ( DstReg , TRI -> getEquivalentSGPRClass ( MRI . getRegClass ( DstReg ) ) ) ; return


<s> if ( MO . isDef ( ) || UseMI -> getParent ( ) != MI . getParent ( ) || UseMI -> getOpcode ( ) <= TargetOpcode :: GENERIC_OP_END || ! TII -> isOperandLegal ( * UseMI , UseMI -> getOperandNo ( & MO ) , & Src ) ) return false ; } MRI . setRegClass ( DstReg , TRI -> getEquivalentSGPRClass (


<s> unsigned DstReg = Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned SrcReg = Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * SrcRC = TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ? MRI . getRegClass ( SrcReg ) : TRI . getPhysRegClass ( SrcReg ) ; const TargetRegisterClass * DstRC = TargetRegisterInfo :: isVirtualRegister ( DstReg ) ? MRI . getRegClass ( DstReg ) : TRI . getPhysRegClass (


<s> bool <ISA_LIT>FixSGPRCopies :: isSGPRToVGPRCopy ( const TargetRegisterClass * SrcRC , const TargetRegisterClass * DstRC , const <ISA_LIT>RegisterInfo & TRI ) const { return TRI . isSGPRClass ( SrcRC ) && TRI


<s> for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { default : continue ; case <STR_LIT> : { if ( ! TargetRegisterInfo :: isVirtualRegister ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( MI , * TRI , MRI ) ; if ( isVGPRToSGPRCopy ( SrcRC , DstRC , * TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool SGPRBranch = false ; bool HasBreakDef = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; if ( TRI -> hasVGPRs ( MRI . getRegClass ( Reg ) ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefInstr = MRI . getUniqueVRegDef ( Reg ) ; assert ( DefInstr ) ; switch ( DefInstr -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case


<s> bool IsAGPR = TRI -> isAGPRClass ( DstRC ) ; for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { Register SrcReg = MI . getOperand ( I ) . getReg ( ) ; unsigned SrcSubReg = MI . getOperand ( I ) . getSubReg ( ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( SrcReg ) ; assert ( TRI -> isSGPRClass ( SrcRC ) && <STR_LIT> ) ; SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ; const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentVGPRClass ( SrcRC ) ; Register TmpReg = MRI . createVirtualRegister ( NewSrcRC ) ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . add ( MI . getOperand ( I ) ) ; if ( IsAGPR ) { const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentAGPRClass ( SrcRC ) ; Register TmpAReg = MRI . createVirtualRegister ( NewSrcRC ) ; unsigned Opc = NewSrcRC == & <STR_LIT> ? <STR_LIT> : <STR_LIT> ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) , TmpAReg ) . addReg ( TmpReg , RegState :: Kill ) ; TmpReg =


<s> assert ( MI . isRegSequence ( ) ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( DstReg ) ) ) return false ; if ( ! MRI . hasOneUse ( DstReg ) ) return false ; MachineInstr & CopyUse = * MRI . use_instr_begin ( DstReg ) ; if ( ! CopyUse . isCopy ( ) ) return false ; if ( CopyUse . getOperand ( <NUM_LIT> ) . getReg ( ) . isPhysical ( ) ) return false ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( CopyUse , * TRI , MRI ) ; if ( ! isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) return false ; if ( tryChangeVGPRtoSGPRinCopy ( CopyUse , TRI , TII ) ) return true ; unsigned SubReg = CopyUse . getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( SubReg != <STR_LIT> ) return false ; MRI . setRegClass ( DstReg , DstRC ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( CopyUse . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; bool IsAGPR = TRI -> isAGPRClass ( DstRC ) ; for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { Register SrcReg = MI . getOperand ( I ) . getReg ( ) ; unsigned SrcSubReg = MI . getOperand ( I ) . getSubReg ( ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( SrcReg ) ; assert ( TRI -> isSGPRClass ( SrcRC ) && <STR_LIT> ) ; SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ; const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentVGPRClass ( SrcRC ) ; Register TmpReg = MRI . createVirtualRegister ( NewSrcRC ) ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . add ( MI . getOperand ( I ) ) ; if ( IsAGPR ) { const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentAGPRClass ( SrcRC ) ; Register TmpAReg = MRI . createVirtualRegister ( NewSrcRC ) ; unsigned Opc = NewSrcRC == & <STR_LIT> ? <STR_LIT> : <STR_LIT> ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) ,

TmpAReg ) . addReg ( TmpReg , RegState :: Kill ) ;


<s> Register SrcReg = Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * SrcRC = SrcReg . isVirtual ( ) ? MRI . getRegClass ( SrcReg ) : TRI . getPhysRegClass ( SrcReg ) ; const TargetRegisterClass * DstRC = DstReg . isVirtual ( ) ? MRI . getRegClass ( DstReg ) : TRI . getPhysRegClass


<s> while ( I != MBB -> end ( ) && TII -> isBasicBlockPrologue ( * I ) ) ++ I ; return I


<s> for ( const MachineOperand & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . getReg ( ) .


<s> if ( MDT . dominates ( From , To ) ) return true ; const MachineBasicBlock * MBBFrom = From -> getParent ( ) ; const MachineBasicBlock * MBBTo = To


<s> static bool isSGPRToVGPRCopy ( const TargetRegisterClass * SrcRC , const TargetRegisterClass * DstRC , const <ISA_LIT>RegisterInfo & TRI ) { return DstRC != & <STR_LIT> && TRI . isSGPRClass ( SrcRC ) && TRI . hasVectorRegisters


SrcRC ) && TRI . hasVectorRegisters ( DstRC ) ;


<s> static bool isVGPRToSGPRCopy ( const TargetRegisterClass * SrcRC , const TargetRegisterClass * DstRC , const <ISA_LIT>RegisterInfo & TRI ) { return SrcRC != & <STR_LIT> && TRI . isSGPRClass ( DstRC ) && TRI . hasVectorRegisters (


<s> Register PHIRes = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC0 = MRI -> getRegClass ( PHIRes ) ; if ( AllAGPRUses && numVGPRUses && ! TRI -> isAGPRClass ( RC0 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MRI -> setRegClass ( PHIRes , TRI -> getEquivalentAGPRClass ( RC0 ) ) ; for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { MachineInstr * DefMI = MRI -> getVRegDef ( MI . getOperand ( I ) . getReg ( ) ) ; if ( DefMI && DefMI -> isPHI ( ) ) PHIOperands . insert ( DefMI ) ; } } bool hasVGPRInput = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { Register InputReg = MI . getOperand ( i ) . getReg ( ) ; MachineInstr * Def = MRI -> getVRegDef ( InputReg ) ; if ( TRI -> isVectorRegister ( * MRI , InputReg ) ) { if ( Def -> isCopy ( ) ) { Register SrcReg = Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC = TRI -> getRegClassForReg ( * MRI , SrcReg ) ; if ( TRI -> isSGPRClass ( RC ) ) continue ; } hasVGPRInput = true ; break ; } else if ( Def -> isCopy ( ) && TRI -> isVectorRegister ( * MRI , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Register SrcReg = Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * SrcDef = MRI -> getVRegDef ( SrcReg ) ; unsigned SMovOp ; int64_t Imm ; if ( ! isSafeToFoldImmIntoCopy ( Def , SrcDef , TII , SMovOp , Imm ) ) { hasVGPRInput = true ; break ; } else { MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; Def -> getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; Def -> addImplicitDefUseOperands ( * MF ) ; Def -> setDesc ( TII -> get ( SMovOp ) ) ; } } } if ( ( ! TRI -> isVectorRegister ( * MRI , PHIRes ) && RC0 != & <STR_LIT> ) && ( hasVGPRInput || numVGPRUses > <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; CreatedBB = TII -> moveToVALU ( MI ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> legalizeOperands ( MI , MDT ) ; } while ( ! PHIOperands . empty ( ) ) { processPHINode ( * PHIOperands .


<s> auto & Src = MI . getOperand ( <NUM_LIT> ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register SrcReg = Src . getReg ( ) ; if ( ! SrcReg . isVirtual ( ) || ! DstReg . isVirtual ( ) ) return false ; for ( const auto & MO : MRI .


<s> if ( ! SrcReg . isVirtual ( ) || ! DstReg . isVirtual ( ) ) return false ; for ( const auto & MO : MRI . reg_nodbg_operands ( DstReg ) ) { const auto * UseMI = MO . getParent ( ) ; if ( UseMI == & MI ) continue ; if ( MO . isDef ( ) || UseMI -> getParent ( ) != MI . getParent ( ) || UseMI -> getOpcode ( ) <= TargetOpcode :: GENERIC_OP_END ) return false ; unsigned OpIdx = UseMI -> getOperandNo ( &


<s> bool MayClobberFrom = isReachable ( Clobber , & * From , MBBTo , MDT ) ; bool MayClobberTo = isReachable ( Clobber , & * To , MBBTo , MDT ) ; if ( ! MayClobberFrom && ! MayClobberTo ) return false ; if ( ( MayClobberFrom && ! MayClobberTo ) || ( ! MayClobberFrom && MayClobberTo ) ) return true ; return ! ( ( MBBFrom == MBBTo && MDT . dominates ( Clobber , & * From ) && MDT . dominates ( Clobber , & * To ) ) || MDT . properlyDominates ( Clobber -> getParent ( ) , MBBTo ) ) ; } ; return ( llvm :: any_of ( Clobbers , interferes ) ) || ( llvm :: any_of ( Inits , [ & ] ( InitListMap :: value_type & C ) { return C . first != Init . first && llvm :: any_of ( C . second , interferes ) ; } ) ) ; } ; if ( MDT . dominates ( MI1 , MI2 ) ) { if ( ! intereferes ( MI2 , MI1 ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI2 -> getParent ( ) -> getNumber ( ) << <STR_LIT> << * MI2 ) ; MI2 -> eraseFromParent ( ) ; Defs . erase ( I2 ++ ) ; Changed = true ; continue ; } } else if ( MDT . dominates ( MI2 , MI1 ) ) { if ( ! intereferes ( MI1 , MI2 ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI1 -> getParent ( ) -> getNumber ( ) << <STR_LIT> << * MI1 ) ; MI1 -> eraseFromParent ( ) ; Defs . erase ( I1 ++ ) ; Changed = true ; break ; } } else { auto * MBB = MDT . findNearestCommonDominator ( MI1 -> getParent ( ) , MI2 -> getParent ( ) ) ; if ( ! MBB ) { ++ I2 ; continue ; } MachineBasicBlock :: iterator I = MBB -> getFirstNonPHI ( ) ; if ( ! intereferes ( MI1 , I ) && ! intereferes ( MI2 , I ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI1 -> getParent ( ) -> getNumber ( ) << <STR_LIT> << * MI1 << <STR_LIT> << MI2 -> getParent ( ) -> getNumber ( ) << <STR_LIT> << I -> getParent ( ) -> getNumber ( ) << <STR_LIT> << * MI2 ) ; I -> getParent ( ) -> splice ( I , MI2 -> getParent ( ) , MI2 ) ; MI1 -> eraseFromParent ( ) ; Defs . erase ( I1 ++ ) ; Changed = true ; break ; } } ++ I2 ; } ++ I1 ; } } if ( Changed ) MRI . clearKillFlags ( Reg


<s> if ( ! MayClobberFrom && ! MayClobberTo ) return false ; if ( ( MayClobberFrom && ! MayClobberTo ) || ( ! MayClobberFrom && MayClobberTo ) ) return true ; return ! ( ( MBBFrom == MBBTo && MDT . dominates ( Clobber , & * From ) && MDT . dominates ( Clobber , & * To ) ) || MDT . properlyDominates ( Clobber -> getParent ( ) , MBBTo ) ) ; } ; return ( llvm :: any_of ( Clobbers , interferes ) ) || ( llvm :: any_of ( Inits , [ & ] ( InitListMap :: value_type & C ) { return C . first != Init . first && llvm :: any_of ( C . second , interferes ) ; } ) ) ; } ; if ( MDT . dominates ( MI1 , MI2 ) ) { if ( ! intereferes ( MI2 , MI1 ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI2 -> getParent ( ) -> getNumber ( ) << <STR_LIT> << * MI2 ) ; MI2 -> eraseFromParent ( ) ; Defs . erase ( I2 ++ ) ; Changed = true ; continue ; } } else if ( MDT . dominates ( MI2 , MI1 ) ) { if ( ! intereferes ( MI1 , MI2 ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI1 -> getParent ( ) -> getNumber ( ) << <STR_LIT> << * MI1 ) ; MI1 -> eraseFromParent ( ) ; Defs . erase ( I1 ++ ) ; Changed = true ; break ; } } else


<s> for ( unsigned i = <NUM_LIT> ; i < PHI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned Reg = PHI . getOperand ( i ) . getReg ( ) ; if ( Visited . count ( Reg ) ) continue ; Visited . insert ( Reg ) ; MachineInstr * DefInstr = MRI . getUniqueVRegDef ( Reg ) ; assert ( DefInstr ) ; switch ( DefInstr -> getOpcode ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; case <STR_LIT> : if ( phiHasBreakDef ( * DefInstr , MRI , Visited ) ) return true ; } } return false


<s> int64_t Imm ; if ( isSafeToFoldImmIntoCopy ( & MI , DefMI , TII , SMovOp , Imm ) ) { MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( phiHasVGPROperands ( MI , MRI , TRI , TII ) || ! phiHasBreakDef ( MI , MRI , Visited ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; break ; } case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand


<s> for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { unsigned SrcReg = MI . getOperand ( I ) . getReg ( ) ; unsigned SrcSubReg = MI . getOperand ( I ) . getSubReg ( ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( SrcReg ) ; assert ( TRI -> isSGPRClass ( SrcRC ) && <STR_LIT> ) ; SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ; const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentVGPRClass ( SrcRC ) ; unsigned TmpReg = MRI . createVirtualRegister ( NewSrcRC ) ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . addOperand ( MI . getOperand ( I ) ) ; MI . getOperand ( I ) . setReg ( TmpReg ) ; } CopyUse . eraseFromParent


<s> break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( phiHasVGPROperands ( MI , MRI , TRI , TII ) || ! phiHasBreakDef ( MI , MRI , Visited ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; break ; } case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( TRI -> isSGPRClass ( DstRC ) && ( TRI -> hasVGPRs ( Src0RC ) || TRI -> hasVGPRs ( Src1RC ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI


<s> SmallSet < unsigned , <NUM_LIT> > Visited ; if ( phiHasVGPROperands ( MI , MRI , TRI , TII ) || ! phiHasBreakDef ( MI , MRI , Visited ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; break ; } case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( TRI -> isSGPRClass ( DstRC ) && ( TRI -> hasVGPRs ( Src0RC ) || TRI -> hasVGPRs ( Src1RC )


<s> assert ( TRI -> isSGPRClass ( SrcRC ) && <STR_LIT> ) ; SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ; const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentVGPRClass ( SrcRC ) ; Register TmpReg = MRI . createVirtualRegister ( NewSrcRC ) ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . add ( MI . getOperand ( I ) ) ; if ( IsAGPR ) { const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentAGPRClass ( SrcRC ) ; Register TmpAReg = MRI . createVirtualRegister ( NewSrcRC ) ; unsigned Opc = NewSrcRC == & <STR_LIT> ? <STR_LIT> : <STR_LIT> ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) , TmpAReg ) . addReg ( TmpReg , RegState :: Kill ) ; TmpReg = TmpAReg ; } MI . getOperand ( I ) . setReg ( TmpReg ) ; } CopyUse . eraseFromParent ( ) ; return true


<s> assert ( MI . isRegSequence ( ) ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( DstReg ) ) ) return false ; if ( ! MRI . hasOneUse ( DstReg ) ) return false ; MachineInstr & CopyUse = * MRI . use_instr_begin ( DstReg ) ; if ( ! CopyUse . isCopy ( ) ) return false ; if ( Register :: isPhysicalRegister ( CopyUse . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) return false ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( CopyUse , * TRI , MRI ) ; if ( ! isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) return false ; if ( tryChangeVGPRtoSGPRinCopy ( CopyUse , TRI , TII ) ) return true ; unsigned SubReg = CopyUse . getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( SubReg != <STR_LIT> ) return false ; MRI . setRegClass ( DstReg , DstRC ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( CopyUse . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; bool IsAGPR = TRI -> hasAGPRs ( DstRC ) ; for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { Register SrcReg = MI . getOperand ( I ) . getReg ( ) ; unsigned SrcSubReg = MI . getOperand ( I ) . getSubReg ( ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( SrcReg ) ; assert ( TRI -> isSGPRClass ( SrcRC ) && <STR_LIT> ) ; SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ; const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentVGPRClass ( SrcRC ) ; Register TmpReg = MRI . createVirtualRegister ( NewSrcRC ) ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . add ( MI . getOperand ( I ) ) ; if ( IsAGPR ) { const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentAGPRClass ( SrcRC ) ; Register TmpAReg = MRI . createVirtualRegister ( NewSrcRC ) ; unsigned Opc = NewSrcRC == & <STR_LIT> ? <STR_LIT> : <STR_LIT> ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) , TmpAReg ) . addReg


<s> Register SrcReg = Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * SrcRC = Register :: isVirtualRegister ( SrcReg ) ? MRI . getRegClass ( SrcReg ) : TRI . getPhysRegClass ( SrcReg ) ; const TargetRegisterClass * DstRC = Register :: isVirtualRegister ( DstReg ) ? MRI . getRegClass ( DstReg ) : TRI .


<s> const MachineRegisterInfo & MRI = MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( ! MI . getOperand ( i ) . isReg ( ) || ! Register :: isVirtualRegister ( MI . getOperand ( i ) . getReg ( ) ) ) continue ; if ( TRI -> hasVectorRegisters ( MRI . getRegClass ( MI . getOperand ( i ) . getReg ( ) ) ) ) return true ; } return


<s> break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( HasVGPROperand || ! phiHasBreakDef ( MI , MRI , Visited ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI , MDT ) ; } break ; } case <STR_LIT> : if ( TRI -> hasVectorRegisters ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVectorOperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI , MDT ) ; break ; case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( TRI -> isSGPRClass ( DstRC ) && ( TRI -> hasVectorRegisters ( Src0RC ) || TRI -> hasVectorRegisters ( Src1RC ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI , MDT ) ; } break ; } } } } if ( MF . getTarget ( ) . getOptLevel ( ) > CodeGenOpt :: None && EnableM0Merge ) hoistAndMergeSGPRInits ( <STR_LIT> , MRI , TRI , * MDT , TII ) ; return true


<s> if ( UseMI == & MI ) continue ; if ( MO . isDef ( ) || UseMI -> getParent ( ) != MI . getParent ( ) || UseMI -> getOpcode ( ) <= TargetOpcode :: GENERIC_OP_END || ! TII -> isOperandLegal ( * UseMI , UseMI -> getOperandNo ( & MO ) , & Src ) ) return false ; } MRI . setRegClass ( DstReg , TRI -> getEquivalentSGPRClass ( MRI . getRegClass ( DstReg ) ) ) ; return


<s> SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ; const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentVGPRClass ( SrcRC ) ; Register TmpReg = MRI . createVirtualRegister ( NewSrcRC ) ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . add ( MI . getOperand ( I ) ) ; if ( IsAGPR ) { const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentAGPRClass ( SrcRC ) ; Register TmpAReg = MRI . createVirtualRegister ( NewSrcRC ) ; unsigned Opc = NewSrcRC == & <STR_LIT> ? <STR_LIT> : <STR_LIT> ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) , TmpAReg ) . addReg ( TmpReg , RegState :: Kill ) ; TmpReg = TmpAReg ; } MI . getOperand ( I ) . setReg ( TmpReg ) ; } CopyUse . eraseFromParent ( ) ; return


<s> } if ( UseMI -> isPHI ( ) ) { const TargetRegisterClass * UseRC = MRI -> getRegClass ( Use . getReg ( ) ) ; if ( ! TRI -> isSGPRReg ( * MRI , Use . getReg ( ) ) && UseRC != & <STR_LIT> ) numVGPRUses ++ ; continue ; } const TargetRegisterClass * OpRC = TII -> getOpRegClass ( * UseMI , UseMI -> getOperandNo ( & Use ) ) ; if ( ! TRI -> isSGPRClass ( OpRC ) && OpRC != & <STR_LIT> && OpRC != & <STR_LIT> ) { numVGPRUses ++ ; } } } Register PHIRes = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC0 = MRI -> getRegClass ( PHIRes ) ; if ( AllAGPRUses && numVGPRUses && ! TRI -> hasAGPRs ( RC0 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MRI -> setRegClass ( PHIRes , TRI -> getEquivalentAGPRClass ( RC0 ) ) ; for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { MachineInstr * DefMI = MRI -> getVRegDef ( MI . getOperand ( I ) . getReg ( ) ) ; if ( DefMI && DefMI -> isPHI ( ) ) PHIOperands . insert ( DefMI ) ; } } bool hasVGPRInput = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { Register InputReg = MI . getOperand ( i ) . getReg ( ) ; MachineInstr * Def = MRI -> getVRegDef ( InputReg ) ; if ( TRI -> isVectorRegister ( * MRI , InputReg ) ) { if ( Def -> isCopy ( ) ) { Register SrcReg = Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC = TRI -> getRegClassForReg


<s> if ( UseMI -> isCopy ( ) && UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) . isPhysical ( ) && ! TRI -> isSGPRReg ( * MRI , UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { numVGPRUses ++ ; } if ( Visited . insert ( UseMI ) . second ) worklist . insert ( UseMI ) ; continue ; } if ( UseMI -> isPHI ( ) ) { const TargetRegisterClass * UseRC = MRI -> getRegClass ( Use . getReg ( ) ) ; if ( ! TRI -> isSGPRReg ( * MRI , Use . getReg ( ) ) && UseRC != & <STR_LIT> ) numVGPRUses ++ ; continue ; } const TargetRegisterClass * OpRC = TII -> getOpRegClass ( * UseMI , UseMI -> getOperandNo ( & Use ) ) ; if ( ! TRI -> isSGPRClass ( OpRC ) && OpRC != & <STR_LIT> && OpRC != & <STR_LIT> ) { numVGPRUses ++ ; } } } Register PHIRes = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC0 = MRI


<s> if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( DstReg ) ) ) return false ; if ( ! MRI . hasOneUse ( DstReg ) ) return false ; MachineInstr & CopyUse = * MRI . use_instr_begin ( DstReg ) ; if ( ! CopyUse . isCopy ( ) ) return false ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( CopyUse , * TRI , MRI ) ; if ( ! isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) return false ; unsigned SubReg = CopyUse . getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( SubReg != <STR_LIT> )


<s> if ( ! MI . getOperand ( i ) . isReg ( ) || ! TargetRegisterInfo :: isVirtualRegister ( MI . getOperand ( i ) . getReg ( ) ) ) continue ; if ( TRI -> hasVectorRegisters ( MRI . getRegClass ( MI . getOperand ( i ) . getReg ( ) ) ) ) return true ; } return false


<s> MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI , MDT ) ; } else if ( isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) { tryChangeVGPRtoSGPRinCopy ( MI , TRI , TII ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool HasVGPROperand = phiHasVGPROperands ( MI , MRI , TRI , TII ) ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> && ! HasVGPROperand ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( HasVGPROperand || ! phiHasBreakDef ( MI , MRI , Visited ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU (


<s> bool AllAGPRUses = true ; SetVector < const MachineInstr * > worklist ; SmallSet < const MachineInstr * , <NUM_LIT> > Visited ; SetVector < MachineInstr * > PHIOperands ; worklist . insert ( & MI ) ; Visited . insert ( & MI ) ; while ( ! worklist . empty ( ) ) { const MachineInstr * Instr = worklist . pop_back_val ( ) ; unsigned Reg = Instr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( const auto & Use : MRI -> use_operands ( Reg ) ) { const MachineInstr * UseMI = Use . getParent ( ) ; AllAGPRUses &= ( UseMI -> isCopy ( ) && TRI -> isAGPR ( * MRI , UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) || TRI -> isAGPR ( * MRI , Use . getReg ( ) ) ; if ( UseMI -> isCopy ( ) || UseMI -> isRegSequence ( ) ) { if ( UseMI -> isCopy ( ) && UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) . isPhysical ( ) && ! TRI -> isSGPRReg ( * MRI , UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { numVGPRUses ++ ; } if ( Visited . insert ( UseMI ) . second )


<s> SetVector < const MachineInstr * > worklist ; SmallSet < const MachineInstr * , <NUM_LIT> > Visited ; worklist . insert ( & MI ) ; Visited . insert ( & MI ) ; while ( ! worklist . empty ( ) ) { const MachineInstr * Instr = worklist . pop_back_val ( ) ; unsigned Reg = Instr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( const auto & Use : MRI -> use_operands ( Reg ) ) { const MachineInstr * UseMI = Use . getParent ( ) ; if ( UseMI -> isCopy ( ) || UseMI -> isRegSequence ( ) ) { if ( UseMI -> isCopy ( ) && UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) . isPhysical ( ) && ! TRI -> isSGPRReg ( * MRI , UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { numVGPRUses ++ ; } if ( Visited . insert ( UseMI ) . second ) worklist . insert ( UseMI ) ; continue ; } if ( UseMI -> isPHI ( ) ) { const TargetRegisterClass * UseRC = MRI -> getRegClass ( Use . getReg ( ) ) ; if ( ! TRI -> isSGPRReg ( * MRI , Use . getReg ( ) ) && UseRC != & <STR_LIT> ) numVGPRUses ++ ; continue ; } const TargetRegisterClass * OpRC = TII -> getOpRegClass ( * UseMI , UseMI -> getOperandNo ( & Use ) ) ; if ( ! TRI -> isSGPRClass ( OpRC ) && OpRC != & <STR_LIT> && OpRC != & <STR_LIT> ) { numVGPRUses ++ ; } } } bool hasVGPRInput = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned InputReg = MI . getOperand ( i ) . getReg ( ) ; MachineInstr * Def = MRI -> getVRegDef ( InputReg ) ; if ( TRI -> isVGPR ( * MRI , InputReg ) ) { if ( Def -> isCopy ( ) ) { unsigned SrcReg = Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC = TRI -> getRegClassForReg ( * MRI , SrcReg ) ; if ( TRI -> isSGPRClass (


<s> unsigned SubReg = CopyUse . getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( SubReg != <STR_LIT> ) return false ; MRI . setRegClass ( DstReg , DstRC ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( CopyUse . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; bool IsAGPR = TRI -> hasAGPRs ( DstRC ) ; for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { Register SrcReg = MI . getOperand ( I ) . getReg ( ) ; unsigned SrcSubReg = MI . getOperand ( I ) . getSubReg ( ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( SrcReg ) ; assert ( TRI -> isSGPRClass ( SrcRC ) && <STR_LIT> ) ; SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ; const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentVGPRClass ( SrcRC ) ; Register TmpReg = MRI . createVirtualRegister ( NewSrcRC ) ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . add ( MI . getOperand ( I ) ) ; if ( IsAGPR ) { const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentAGPRClass ( SrcRC ) ; Register TmpAReg = MRI . createVirtualRegister ( NewSrcRC ) ; unsigned Opc = NewSrcRC == & <STR_LIT> ?


<s> const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( MI , * TRI , MRI ) ; if ( isVGPRToSGPRCopy ( SrcRC , DstRC , * TRI ) ) { unsigned SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefMI = MRI . getVRegDef ( SrcReg ) ; unsigned SMovOp ; int64_t Imm ; if ( isSafeToFoldImmIntoCopy ( & MI , DefMI , TII , SMovOp , Imm ) ) { MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI ) ; } else if ( isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) { tryChangeVGPRtoSGPRinCopy ( MI , TRI , TII ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT>


<s> const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( MI , * TRI , MRI ) ; if ( isVGPRToSGPRCopy ( SrcRC , DstRC , * TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool SGPRBranch = false ; bool HasBreakDef = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; if ( TRI -> hasVGPRs ( MRI . getRegClass ( Reg ) ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefInstr = MRI . getUniqueVRegDef ( Reg ) ; assert ( DefInstr ) ; switch ( DefInstr -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : HasBreakDef = true ; break ; } } if ( ! SGPRBranch && ! HasBreakDef ) TII -> moveToVALU ( MI ) ; break ; } case <STR_LIT> : { if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT>


<s> MachineDomTreeNode * sentinel = MPDT -> getNode ( & * B ) -> getIDom ( ) ; while ( runner && runner != sentinel ) { PDF [ runner -> getBlock ( ) ] . insert (


<s> AU . addPreserved < MachineDominatorTree > ( ) ; AU . addRequired < MachinePostDominatorTree > ( ) ; AU . addPreserved < MachinePostDominatorTree > ( ) ; AU . setPreservesCFG (


<s> InitListMap Inits ; SmallVector < MachineInstr * , <NUM_LIT> > Clobbers ; bool Changed = false ; for ( auto & MI : MRI . def_instructions ( Reg ) ) { MachineOperand * Imm = nullptr ; for ( auto & MO : MI . operands ( ) ) { if ( ( MO . isReg ( ) && ( ( MO . isDef ( ) && MO . getReg ( ) != Reg ) || ! MO . isDef ( ) ) ) || ( ! MO . isImm ( ) && ! MO . isReg ( ) ) || ( MO . isImm ( ) && Imm ) ) { Imm = nullptr ; break ; } else if ( MO . isImm ( ) ) Imm = & MO ; } if ( Imm ) Inits [ Imm -> getImm ( ) ] . push_front ( & MI ) ; else Clobbers . push_back ( & MI ) ; } for ( auto & Init : Inits ) { auto & Defs = Init . second ; for ( auto I1 = Defs . begin ( ) , E = Defs . end ( ) ; I1 != E ; ) { MachineInstr * MI1 = * I1 ; for ( auto I2 = std :: next ( I1 ) ; I2 != E ; ) { MachineInstr * MI2 = * I2 ; auto intereferes = [ & ] ( MachineBasicBlock :: iterator From , MachineBasicBlock :: iterator To ) -> bool { assert ( MDT . dominates ( & * To , & * From ) ) ; auto interferes = [ & MDT , From , To ] ( MachineInstr * & Clobber ) -> bool { const MachineBasicBlock * MBBFrom = From


<s> if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefMI = MRI . getVRegDef ( SrcReg ) ; unsigned SMovOp ; int64_t Imm ; if ( isSafeToFoldImmIntoCopy ( & MI , DefMI , TII , SMovOp , Imm ) ) { MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI ) ; } else if ( isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) { tryChangeVGPRtoSGPRinCopy ( MI , TRI , TII ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool HasVGPROperand = phiHasVGPROperands ( MI , MRI , TRI , TII ) ; if ( ! HasVGPROperand ) { bool Uniform = true ; MachineBasicBlock * Join = MI . getParent ( ) ; for ( auto & O : MI . explicit_operands ( ) ) { if ( O . isMBB ( ) ) { MachineBasicBlock * Source = O . getMBB ( ) ; SetVector < MachineBasicBlock * > & SourcePDF = PDF [ Source ] ; SetVector < MachineBasicBlock * > & JoinPDF = PDF [ Join ] ; SetVector < MachineBasicBlock * > CDList ; for ( auto & I : SourcePDF ) { if ( ! JoinPDF . count ( I ) || MDT -> dominates ( Join , I ) ) { if ( hasTerminatorThatModifiesExec ( * I , * TRI ) ) Uniform = false ; } } } } if ( Uniform ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( HasVGPROperand || ! phiHasBreakDef ( MI , MRI , Visited ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; break ; case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg


<s> bool <ISA_LIT>FixSGPRCopies :: runOnMachineFunction ( MachineFunction & MF ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; MPDT = & getAnalysis < MachinePostDominatorTree > ( ) ; PDF . clear ( ) ; computePDF ( & MF ) ; DEBUG ( printPDF ( ) ) ; SmallVector < MachineInstr * , <NUM_LIT> > Worklist ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { default : continue ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( ! TargetRegisterInfo :: isVirtualRegister ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( MI , * TRI , MRI ) ; if ( isVGPRToSGPRCopy ( SrcRC , DstRC , * TRI ) ) { unsigned SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefMI = MRI . getVRegDef ( SrcReg ) ; unsigned SMovOp ; int64_t Imm


<s> static bool predsHasDivergentTerminator ( MachineBasicBlock * MBB , const TargetRegisterInfo * TRI ) { DenseSet < MachineBasicBlock * > Visited ; SmallVector < MachineBasicBlock * , <NUM_LIT> > Worklist ( MBB -> pred_begin ( ) , MBB -> pred_end ( ) ) ; while ( ! Worklist . empty ( ) ) { MachineBasicBlock * mbb = Worklist . back ( ) ; Worklist . pop_back ( ) ; if ( ! Visited . insert ( mbb ) . second ) continue ; if ( hasTerminatorThatModifiesExec ( * mbb , * TRI


<s> if ( isVGPRToSGPRCopy ( SrcRC , DstRC , * TRI ) ) { MachineInstr * DefMI = MRI . getVRegDef ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; unsigned SMovOp ; int64_t Imm ; if ( isSafeToFoldImmIntoCopy ( & MI , DefMI , TII , SMovOp , Imm ) ) { MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( phiHasVGPROperands ( MI , MRI , TRI , TII ) || ! phiHasBreakDef ( MI , MRI , Visited ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU (


<s> return new <ISA_LIT>FixSGPRCopies ( tm )


<s> } break ; } case <STR_LIT> : { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool SGPRBranch = false ; bool HasBreakDef = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; if ( TRI -> hasVGPRs ( MRI . getRegClass ( Reg ) ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefInstr = MRI . getUniqueVRegDef ( Reg ) ; assert ( DefInstr ) ; switch ( DefInstr -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : HasBreakDef = true ; break ; } } if ( ! SGPRBranch && ! HasBreakDef ) TII -> moveToVALU


<s> AU . addRequired < MachinePostDominatorTree > ( ) ; AU . setPreservesCFG ( ) ; AU . addPreserved < LiveIntervals >


<s> for ( std :: pair < unsigned , LiveRange * > RegLR : SGPRLiveRanges ) { unsigned Reg = RegLR . first ; LiveRange * LR = RegLR . second ; bool LiveInToA = LIS -> isLiveInToMBB ( * LR , SuccA ) ; bool LiveInToB = LIS -> isLiveInToMBB ( * LR , SuccB ) ; if ( ! LiveInToA && ! LiveInToB ) { DEBUG ( dbgs ( ) << PrintReg ( Reg , TRI , <NUM_LIT> ) << <STR_LIT> ) ; continue ; } if ( LiveInToA && LiveInToB ) { DEBUG ( dbgs ( ) << PrintReg ( Reg , TRI , <NUM_LIT> ) << <STR_LIT> ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Reg , TRI , <NUM_LIT> ) << <STR_LIT> << * LR << <STR_LIT> << SuccA -> getNumber ( ) << <STR_LIT> << SuccB -> getNumber ( ) << <STR_LIT> << NCD -> getNumber ( ) << '\n' ) ; assert ( TargetRegisterInfo :: isVirtualRegister ( Reg ) && <STR_LIT> ) ; MachineInstr * NCDSGPRUse = BuildMI ( * NCD , NCD -> getFirstNonPHI ( ) , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Implicit ) ; MadeChange = true ; SlotIndex <ISA_LIT> = LIS -> InsertMachineInstrInMaps ( NCDSGPRUse


<s> AU . addRequired < LiveVariables > ( ) ; AU . addPreserved < LiveVariables > ( ) ; AU . addRequired < MachinePostDominatorTree > ( ) ; AU . addPreserved < MachinePostDominatorTree > ( ) ; AU . setPreservesCFG ( ) ; MachineFunctionPass :: getAnalysisUsage ( AU


<s> const char * getPassName ( ) const override { return <STR_LIT>


<s> <ISA_LIT>FixSGPRLiveRanges ( ) : MachineFunctionPass


<s> MachineOperand * WOp = Worklist . pop_back_val ( ) ; if ( ! WOp -> isReg ( ) || ! Register :: isVirtualRegister ( WOp -> getReg ( ) ) ) continue ; MachineInstr * DefInst = MRI . getUniqueVRegDef ( WOp -> getReg ( ) ) ; switch ( DefInst -> getOpcode ( ) ) { default : continue ; case <STR_LIT> : Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : if ( DefInst -> getNumOperands ( ) != <NUM_LIT> ) continue ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : if ( DefInst -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <STR_LIT> ) continue ; BaseReg = DefInst -> getOperand ( <NUM_LIT> ) . getReg


<s> if ( DefInst -> getNumOperands ( ) != <NUM_LIT> ) continue ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : if ( DefInst -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <STR_LIT> ) continue ; BaseReg = DefInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( DefInst -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <STR_LIT> ) continue ; IndexReg = DefInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * MI = MRI . getUniqueVRegDef ( IndexReg ) ; if ( ! MI || ! MI -> isCopy ( ) ) continue ; const TargetRegisterClass * IdxRC , * BaseRC ; IdxRC = MRI . getRegClass ( MI -> getOperand ( <NUM_LIT> ) .


<s> MachineInstr * NewGlob = nullptr ; NewGlob = BuildMI ( MBB , I , MI . getDebugLoc ( ) , TII -> get ( NewOpcd ) ) ; if ( HasVdst ) NewGlob -> addOperand ( MF , MI . getOperand ( <NUM_LIT> ) ) ; NewGlob -> addOperand ( MF , MachineOperand :: CreateReg ( IndexReg , false ) ) ; if ( VData ) NewGlob -> addOperand ( MF , * VData ) ; NewGlob -> addOperand ( MF , MachineOperand :: CreateReg ( BaseReg , false ) ) ; NewGlob -> addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) ) ; MachineOperand * Glc = TII -> getNamedOperand ( MI , <STR_LIT> :: glc ) ; if ( Glc ) NewGlob -> addOperand ( MF , * Glc ) ; MachineOperand * DLC = TII -> getNamedOperand ( MI , <STR_LIT> :: dlc ) ; if ( DLC ) NewGlob -> addOperand ( MF , * DLC ) ; NewGlob -> addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: slc ) ) ; MachineOperand * VDstInOp = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst_in ) ; if ( VDstInOp ) NewGlob -> addOperand ( MF , * VDstInOp ) ; NewGlob -> copyImplicitOps ( MF , MI ) ; NewGlob -> cloneMemRefs ( MF


<s> FunctionPass * llvm :: create<ISA_LIT>FixupVectorISelPass


<s> return new <ISA_LIT>FixupVectorISel ( )


<s> break ; case <STR_LIT> : if ( DefInst -> getNumOperands ( ) != <NUM_LIT> ) continue ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : if ( DefInst -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <STR_LIT> ) continue ; BaseReg = DefInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( DefInst -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <STR_LIT> ) continue ; IndexReg = DefInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * MI = MRI . getUniqueVRegDef ( IndexReg ) ; if ( ! MI || ! MI -> isCopy ( ) ) continue ; const TargetRegisterClass * IdxRC , * BaseRC ; IdxRC = MRI . getRegClass ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( <STR_LIT> ( IdxRC -> getID ( ) ) != <NUM_LIT> ) continue ; IndexReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MI = MRI . getUniqueVRegDef ( BaseReg ) ; if ( ! MI || ! MI -> isCopy ( ) ) continue ; BaseReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; BaseRC = MRI . getRegClass ( BaseReg ) ; if ( <STR_LIT> ( BaseRC -> getID ( ) ) != <NUM_LIT> ) continue ; if ( ! TRI -> isSGPRReg ( MRI , BaseReg ) ) return false ; if ( ! TRI -> hasVGPRs ( MRI . getRegClass ( IndexReg


<s> Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : if ( DefInst -> getNumOperands ( ) != <NUM_LIT> ) continue ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : if ( DefInst -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <STR_LIT> ) continue ; BaseReg = DefInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( DefInst -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <STR_LIT> ) continue ; IndexReg = DefInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * MI = MRI . getUniqueVRegDef ( IndexReg ) ; if ( ! MI || ! MI -> isCopy ( ) ) continue ; const TargetRegisterClass * IdxRC , * BaseRC ; IdxRC = MRI . getRegClass ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( <STR_LIT> ( IdxRC -> getID ( ) ) != <NUM_LIT> ) continue ; IndexReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MI = MRI . getUniqueVRegDef ( BaseReg ) ; if ( ! MI || ! MI -> isCopy ( ) ) continue ; BaseReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; BaseRC = MRI . getRegClass ( BaseReg ) ; if ( <STR_LIT> ( BaseRC -> getID ( ) ) != <NUM_LIT>


<s> NewGlob = BuildMI ( MBB , I , MI . getDebugLoc ( ) , TII -> get ( NewOpcd ) ) ; if ( HasVdst ) NewGlob -> addOperand ( MF , MI . getOperand ( <NUM_LIT> ) ) ; NewGlob -> addOperand ( MF , MachineOperand :: CreateReg ( IndexReg , false ) ) ; if ( VData ) NewGlob -> addOperand ( MF , * VData ) ; NewGlob -> addOperand ( MF , MachineOperand :: CreateReg ( BaseReg , false ) ) ; NewGlob -> addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) ) ; MachineOperand * Glc = TII -> getNamedOperand ( MI , <STR_LIT> :: glc ) ; if ( Glc ) NewGlob -> addOperand ( MF , * Glc ) ; NewGlob -> addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: slc ) ) ; MachineOperand * VDstInOp = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst_in ) ; if ( VDstInOp ) NewGlob -> addOperand ( MF , * VDstInOp ) ; NewGlob -> copyImplicitOps ( MF ,


<s> initialize<ISA_LIT>FixupVectorISelPass ( * PassRegistry ::


<s> StringRef getPassName ( ) const override { return


<s> <ISA_LIT>FixVGPRCopies ( ) : MachineFunctionPass


<s> AU . addRequiredID ( MachineLoopInfoID ) ; AU . addPreserved < SlotIndexes > ( ) ; AU . addPreserved < LiveIntervals > ( ) ; AU . addPreservedID ( LiveVariablesID


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequiredID ( MachineDominatorsID ) ; AU . addRequiredID ( MachineLoopInfoID ) ; AU . addPreserved < SlotIndexes > ( ) ; AU . addPreserved < LiveIntervals > ( ) ; AU . addPreservedID ( LiveVariablesID ) ; AU . addPreservedID


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * WWM ) ; ContainsWWM = true ; break ; } } if ( ! ContainsWWM ) return false ; unsigned Reg = DefOpnd -> getReg ( ) ; for ( auto Pred : Loop -> getHeader ( ) -> predecessors ( ) ) { if ( ! Loop -> contains ( Pred ) ) { auto ImplicitDef = BuildMI ( * Pred , Pred -> getFirstTerminator ( ) , DebugLoc ( ) , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , Reg ) ; LLVM_DEBUG ( dbgs ( ) << * ImplicitDef ) ; ( void ) ImplicitDef ; } } DefOpnd -> getParent ( ) -> addOperand ( MachineOperand :: CreateReg ( Reg , false , true ) ) ; LLVM_DEBUG ( dbgs ( ) << * DefOpnd -> getParent ( ) ) ; return true


<s> } } if ( ! ContainsWWM ) return false ; unsigned Reg = DefOpnd -> getReg ( ) ; for ( auto Pred : Loop -> getHeader ( ) -> predecessors ( ) ) { if ( ! Loop -> contains ( Pred ) ) { auto ImplicitDef = BuildMI ( * Pred , Pred -> getFirstTerminator ( ) , DebugLoc ( ) , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , Reg ) ; LLVM_DEBUG ( dbgs ( ) << * ImplicitDef ) ; ( void )


<s> bool ContainsWWM = false ; for ( auto WWM : WWMs ) { if ( Loop -> contains ( WWM -> getParent ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * WWM ) ; ContainsWWM = true ; break ; } } if ( ! ContainsWWM ) return false ; unsigned Reg = DefOpnd -> getReg ( ) ; for ( auto & Use : MRI -> use_operands ( Reg ) ) Use . setIsKill ( false ) ; SmallVector < MachineInstr * , <NUM_LIT> > Defs ; for ( auto & Def : MRI -> def_instructions ( Reg ) ) Defs . push_back ( & Def ) ; for ( auto Def : Defs ) { if ( DefOpnd -> getParent ( ) == Def ) continue ; Def -> addOperand ( MachineOperand :: CreateReg ( Reg , false


<s> TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; DomTree = & getAnalysis < MachineDominatorTree > ( ) ; LoopInfo = & getAnalysis < MachineLoopInfo > ( ) ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) WWMs . push_back ( & MI ) ; else { for ( MachineOperand & DefOpnd : MI . defs ( ) ) { if ( DefOpnd . isReg ( ) ) { unsigned Reg = DefOpnd . getReg ( ) ; if ( TRI -> isVGPR ( * MRI , Reg ) ) processDef ( DefOpnd ) ; } } } } } if ( ! WWMs . empty ( ) ) { for ( auto ThenDef : ThenDefs ) Modified |= processThenDef ( ThenDef ) ; for ( auto LoopExitDef : LoopExitDefs ) Modified |= processLoopExitDef ( LoopExitDef . first , LoopExitDef . second ) ; for ( auto LoopPhiDef : LoopPhiDefs ) Modified |= processLoopPhiDef ( LoopPhiDef . first , LoopPhiDef . second ) ; } WWMs . clear ( ) ; ThenDefs . clear


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI


<s> FunctionPass * llvm :: create<ISA_LIT>FixWWMLivenessPass (


<s> FunctionPass * llvm :: create<ISA_LIT>FixWWMLivenessPass ( ) { return new <ISA_LIT>FixWWMLiveness (


<s> AU . addPreservedID ( LiveVariablesID ) ; AU . addPreservedID ( MachineLoopInfoID ) ; AU . addPreservedID ( MachineDominatorsID ) ; AU . setPreservesCFG ( ) ; MachineFunctionPass :: getAnalysisUsage ( AU


<s> UseMI -> setDesc ( TII -> get ( MovOp ) ) ; CopiesToReplace . push_back ( UseMI ) ; } else { const MCInstrDesc & UseDesc = UseMI -> getDesc ( ) ; if ( UseDesc . isVariadic ( ) || UseDesc . OpInfo [ UseOpIdx ] . RegClass == - <NUM_LIT> ) return ; } if ( ! FoldingImm ) { tryAddToFoldList ( FoldList , UseMI , UseOpIdx , & OpToFold , TII ) ; return ; } const MCInstrDesc & FoldDesc = OpToFold . getParent ( ) -> getDesc ( ) ; const TargetRegisterClass * FoldRC = TRI -> getRegClass ( FoldDesc . OpInfo [ <NUM_LIT> ] . RegClass ) ; if ( UseOp . getSubReg ( ) && <STR_LIT> ( FoldRC -> getID ( ) ) == <NUM_LIT> ) { unsigned UseReg = UseOp . getReg ( ) ; const TargetRegisterClass * UseRC = TargetRegisterInfo :: isVirtualRegister ( UseReg ) ? MRI -> getRegClass ( UseReg ) : TRI -> getPhysRegClass ( UseReg ) ; if ( <STR_LIT> ( UseRC -> getID ( ) ) != <NUM_LIT> ) return ; APInt Imm ( <NUM_LIT> , OpToFold . getImm ( ) ) ; if ( UseOp . getSubReg ( ) == <STR_LIT> ) { Imm = Imm . getLoBits ( <NUM_LIT> ) ; }


<s> if ( Op . getSubReg ( ) != <STR_LIT> ) return & Op ; MachineInstr * Def = MRI . getVRegDef ( Op . getReg ( ) ) ; if ( Def && Def ->


<s> unsigned CommuteIdx0 = TargetInstrInfo :: CommuteAnyOperandIndex ; unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( * MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) OpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) OpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( *


<s> } int Src1Idx = <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ; if ( Src1Idx == - <NUM_LIT> ) return false ; int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; MachineOperand * Src0 = getImmOrMaterializedImm ( MRI , MI -> getOperand ( Src0Idx ) ) ; MachineOperand * Src1 = getImmOrMaterializedImm ( MRI , MI -> getOperand ( Src1Idx ) ) ; if ( ! Src0 -> isImm ( ) && ! Src1 -> isImm ( ) ) return false ; if ( Src0 -> isImm ( ) && Src1 -> isImm ( ) ) { int32_t NewImm ; if ( ! evalBinaryInstruction ( Opc , NewImm , Src0 -> getImm ( ) , Src1 -> getImm ( ) ) ) return false ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; bool IsSGPR = TRI . isSGPRReg ( MRI , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MI -> getOperand ( Src0Idx ) . ChangeToImmediate ( NewImm ) ; MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( IsSGPR ) ) ) ; return true ; } if ( ! MI -> isCommutable ( ) ) return false ; if ( Src0 -> isImm ( ) && ! Src1 -> isImm ( ) ) { std :: swap ( Src0 , Src1 ) ; std :: swap ( Src0Idx


<s> if ( ! evalBinaryInstruction ( Opc , NewImm , Src0 -> getImm ( ) , Src1 -> getImm ( ) ) ) return false ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; bool IsSGPR = TRI . isSGPRReg ( MRI , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MI -> getOperand ( Src0Idx ) . ChangeToImmediate ( NewImm ) ; MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( IsSGPR ) ) ) ; return true ; } if ( ! MI -> isCommutable ( ) ) return false ; if ( Src0 -> isImm ( ) && ! Src1 -> isImm ( ) ) { std :: swap ( Src0 , Src1 ) ; std :: swap ( Src0Idx , Src1Idx ) ; } int32_t Src1Val = static_cast < int32_t > ( Src1 -> getImm ( ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) { if ( Src1Val == <NUM_LIT> ) { MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( <STR_LIT> ) ) ; } else if ( Src1Val == - <NUM_LIT> ) { MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( Opc == <STR_LIT> ) ) ) ; } else return false ; return true ; } if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { if ( Src1Val == <NUM_LIT> ) { MI -> RemoveOperand ( Src0Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( Opc == <STR_LIT> ) ) ) ; } else if ( Src1Val == - <NUM_LIT> ) { MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( <STR_LIT> ) ) ; stripExtraCopyOperands ( * MI ) ; } else return false ; return true ; } if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { if ( Src1Val == <NUM_LIT> ) { MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( <STR_LIT> ) ) ; return true ; } } return


<s> MachineInstr * Def = MRI -> getVRegDef ( ClampSrc -> getReg ( ) ) ; if ( TII -> getClampMask ( * Def ) != TII -> getClampMask ( MI ) ) return false ; MachineOperand * DefClamp = TII -> getNamedOperand ( * Def , <STR_LIT> :: clamp ) ; if ( ! DefClamp ) return false ; DEBUG ( dbgs ( ) << <STR_LIT> << * DefClamp << <STR_LIT> << * Def << '\n' ) ; DefClamp -> setImm ( <NUM_LIT> ) ; MRI -> replaceRegWith ( MI . getOperand ( <NUM_LIT> ) . getReg ( )

, Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> bool <ISA_LIT>FoldOperands :: tryFoldClamp ( MachineInstr & MI ) { const MachineOperand * ClampSrc = isClamp ( MI ) ; if ( ! ClampSrc || ! hasOneNonDBGUseInst ( * MRI , ClampSrc -> getReg ( ) ) ) return false ; MachineInstr * Def = MRI -> getVRegDef ( ClampSrc -> getReg ( ) ) ; if ( TII -> getClampMask ( * Def ) != TII -> getClampMask ( MI ) ) return false ; MachineOperand * DefClamp = TII -> getNamedOperand ( * Def , <STR_LIT> :: clamp ) ; if ( ! DefClamp ) return false ; DEBUG ( dbgs ( ) << <STR_LIT> << *

DefClamp << <STR_LIT> << * Def << '\n' ) ;


<s> const MachineOperand * Src0 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src1 ) ; if ( Src1 -> isIdenticalTo ( * Src0 ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * MI << <STR_LIT> ) ; int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( Src2Idx != - <NUM_LIT> ) MI -> RemoveOperand ( Src2Idx ) ; MI -> RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ) ; mutateCopyOp ( * MI , TII -> get ( Src0 -> isReg ( ) ? ( unsigned ) <STR_LIT> : getMovOpc ( false ) ) ) ; DEBUG ( dbgs ( ) << * MI << '\n' ) ; return


<s> const MachineOperand * RegOp ; int OMod ; std :: tie ( RegOp , OMod ) = isOMod ( MI ) ; if ( OMod == <STR_LIT> || ! RegOp -> isReg ( ) || RegOp -> getSubReg ( ) != <STR_LIT> || ! hasOneNonDBGUseInst ( * MRI , RegOp -> getReg ( ) ) ) return false ; MachineInstr * Def = MRI -> getVRegDef ( RegOp


<s> MachineOperand * New = Fold . OpToFold ; if ( TargetRegisterInfo :: isVirtualRegister ( Old . getReg ( ) ) && TargetRegisterInfo :: isVirtualRegister ( New -> getReg ( ) ) ) { Old . substVirtReg ( New -> getReg ( ) , New -> getSubReg ( ) , TRI ) ; Old . setIsUndef ( New -> isUndef (


<s> Old . ChangeToImmediate ( Fold . ImmToFold ) ; return true ; } if ( Fold . isFI ( ) ) { Old . ChangeToFrameIndex ( Fold . FrameIndexToFold ) ; return true ; } MachineOperand * New = Fold . OpToFold ; if ( TargetRegisterInfo :: isVirtualRegister ( Old . getReg ( ) ) && TargetRegisterInfo :: isVirtualRegister ( New -> getReg ( )


<s> static void appendFoldCandidate ( SmallVectorImpl < FoldCandidate > & FoldList , MachineInstr * MI , unsigned OpNo , MachineOperand * FoldOp , bool Commuted =


<s> } else if ( FoldOp -> isFI ( ) ) { FrameIndexToFold = FoldOp -> getIndex ( ) ; } else { assert ( FoldOp -> isReg ( ) || FoldOp -> isGlobal


<s> if ( TII -> isMUBUF ( UseMI ) ) return OpNo == <STR_LIT> ( UseMI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; if ( ! TII -> isFLATScratch ( UseMI ) ) return false ; int <ISA_LIT>dx = <STR_LIT> ( UseMI . getOpcode ( ) , <STR_LIT> :: saddr ) ; if ( OpNo == <ISA_LIT>dx


<s> return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; default : return <STR_LIT> ; } } case <STR_LIT> : { switch ( static_cast < uint16_t > ( Val ) ) { case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case


<s> case <STR_LIT> : { switch ( Val ) { case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; default : return <STR_LIT> ; } } case <STR_LIT> : { switch ( static_cast < uint32_t > ( Val ) ) { case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case


<s> static bool getRegSeqInit ( SmallVectorImpl < std :: pair < MachineOperand * , unsigned >> & Defs , Register UseReg , uint8_t OpTy , const <ISA_LIT>InstrInfo * TII , const MachineRegisterInfo & MRI ) { MachineInstr * Def = MRI . getVRegDef ( UseReg ) ; if ( ! Def || ! Def -> isRegSequence ( ) ) return false ; for ( unsigned I = <NUM_LIT> , E = Def -> getNumExplicitOperands ( ) ; I < E ; I += <NUM_LIT> ) { MachineOperand * Sub = & Def -> getOperand ( I ) ; assert ( Sub -> isReg ( ) ) ; for ( MachineInstr * SubDef = MRI . getVRegDef ( Sub -> getReg ( ) ) ; SubDef && Sub -> isReg ( ) && Sub -> getReg ( ) . isVirtual ( ) && ! Sub -> getSubReg ( ) && TII


<s> const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( ! Src0 -> isReg ( ) || ! Src1 -> isReg ( ) || Src0 -> getReg ( ) != Src1 -> getReg ( ) || Src0 -> getSubReg ( ) != Src1 -> getSubReg ( ) || Src0 -> getSubReg ( ) != <STR_LIT> ) return nullptr ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return nullptr ; unsigned Src0Mods = TII -> getNamedOperand ( MI , <STR_LIT> ::