<s> ReverseVector ( orderedBlks ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; printOrderedBlocks ( ) ; ) ; int NumIter = <NUM_LIT> ; bool Finish = false ; MachineBasicBlock * MBB ; bool MakeProgress = false ; int NumRemainedBlk = countActiveBlock ( OrderedBlks . begin ( ) , OrderedBlks . end ( ) ) ; do { ++ NumIter ; DEBUG ( dbgs ( ) << <STR_LIT> << NumIter << <STR_LIT> << NumRemainedBlk << <STR_LIT> ; ) ; SmallVectorImpl < MachineBasicBlock * > :: const_iterator It = OrderedBlks . begin ( ) ; SmallVectorImpl < MachineBasicBlock * > :: const_iterator E = OrderedBlks . end ( ) ; SmallVectorImpl < MachineBasicBlock * > :: const_iterator SccBeginIter = It ; MachineBasicBlock * SccBeginMBB = nullptr ; int SccNumBlk = <NUM_LIT> ; int SccNumIter ; while ( It != E ) { MBB = * It ; if ( ! SccBeginMBB ) { SccBeginIter = It ; SccBeginMBB = MBB ; SccNumIter = <NUM_LIT> ; SccNumBlk = NumRemainedBlk ; DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( SccBeginMBB ) ; dbgs ( ) << <STR_LIT> ; ) ; } if ( ! isRetiredBlock ( MBB ) ) patternMatch ( MBB ) ; ++ It ; bool ContNextScc = true ; if ( It == E || getSCCNum ( SccBeginMBB ) != getSCCNum ( * It ) ) { ++ SccNumIter ; int sccRemainedNumBlk = countActiveBlock ( SccBeginIter , It ) ; if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk >= SccNumBlk ) { DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter ;


<s> TII = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; DEBUG ( MF . dump ( ) ; ) ; OrderedBlks . clear ( ) ; Visited . clear ( ) ; FuncRep = & MF ; MLI = & getAnalysis < MachineLoopInfo > ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; PrintLoopinfo ( * MLI ) ; ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; DEBUG ( MDT -> print ( dbgs ( ) , ( const llvm :: Module * ) nullptr ) ; ) ; PDT = & getAnalysis < MachinePostDominatorTree > ( ) ; DEBUG ( PDT -> print ( dbgs ( ) ) ; ) ; prepare ( ) ; run ( ) ; DEBUG ( MF . dump ( ) ; ) ; return true ;


<s> for ( auto & It : * MLI ) for ( MachineLoop * ML : depth_first ( It ) ) NestedLoops . push_front ( ML ) ; if ( NestedLoops . empty ( ) ) return <NUM_LIT> ; int Num = <NUM_LIT> ; for ( MachineLoop * ExaminedLoop : NestedLoops ) { if ( ExaminedLoop -> getNumBlocks ( ) == <NUM_LIT> || Visited [ ExaminedLoop ] ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> ; ExaminedLoop -> dump ( ) ; ) ; int NumBreak = mergeLoop ( ExaminedLoop ) ; if ( NumBreak == - <NUM_LIT> ) break ; Num += NumBreak ; }


<s> if ( ExaminedLoop -> getNumBlocks ( ) == <NUM_LIT> || Visited [ ExaminedLoop ] ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> ; ExaminedLoop -> dump ( ) ; ) ; int NumBreak = mergeLoop ( ExaminedLoop ) ; if ( NumBreak == - <NUM_LIT> ) break ; Num += NumBreak ; }


<s> for ( ; PI != PE ; PI ++ ) { if ( LoopRep -> contains ( * PI ) ) LatchBlks . push_back ( * PI ) ; } for ( unsigned i = <NUM_LIT> , e = ExitingMBBs . size ( ) ; i < e ; ++ i ) mergeLoopbreakBlock ( ExitingMBBs [ i ] , ExitBlk ) ; for ( unsigned i = <NUM_LIT> , e = LatchBlks . size ( ) ; i < e ; ++ i ) settleLoopcontBlock ( LatchBlks [ i ] , LoopHeader ) ; int Match = <NUM_LIT> ; do { Match = <NUM_LIT> ; Match += serialPatternMatch ( LoopHeader ) ; Match += ifPatternMatch ( LoopHeader ) ; } while ( Match > <NUM_LIT> ) ; mergeLooplandBlock ( LoopHeader , ExitBlk ) ; MachineLoop * ParentLoop = LoopRep -> getParentLoop ( ) ;

if ( ParentLoop ) MLI -> changeLoopFor ( LoopHeader , ParentLoop ) ;


<s> MBBVector ExitBlks ; LoopRep -> getExitBlocks ( ExitBlks ) ; SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > ExitBlkSet ; for ( unsigned i = <NUM_LIT> , e = ExitBlks . size ( ) ; i < e ; ++ i ) ExitBlkSet . insert ( ExitBlks [ i ] ) ; assert ( ExitBlkSet . size ( ) == <NUM_LIT> ) ; MachineBasicBlock * ExitBlk = * ExitBlks . begin ( ) ; assert ( ExitBlk && <STR_LIT> ) ; MBBVector LatchBlks ; typedef GraphTraits < Inverse < MachineBasicBlock * >> InvMBBTraits ; InvMBBTraits :: ChildIteratorType PI = InvMBBTraits :: child_begin ( LoopHeader ) , PE = InvMBBTraits :: child_end ( LoopHeader ) ; for ( ; PI != PE ; PI ++ ) { if ( LoopRep -> contains ( * PI ) ) LatchBlks . push_back ( * PI ) ; } for ( unsigned i = <NUM_LIT> , e = ExitingMBBs . size ( ) ; i < e ; ++ i ) mergeLoopbreakBlock ( ExitingMBBs [ i ] , ExitBlk ) ; for ( unsigned i = <NUM_LIT> , e = LatchBlks . size ( ) ; i < e ; ++ i ) settleLoopcontBlock ( LatchBlks [ i ] , LoopHeader ) ; int Match = <NUM_LIT> ; do { Match = <NUM_LIT> ; Match += serialPatternMatch ( LoopHeader ) ; Match += ifPatternMatch ( LoopHeader ) ; } while ( Match > <NUM_LIT> ) ; mergeLooplandBlock ( LoopHeader , ExitBlk ) ; MachineLoop * ParentLoop = LoopRep -> getParentLoop ( ) ; if ( ParentLoop ) MLI -> changeLoopFor ( LoopHeader , ParentLoop ) ; else MLI -> removeBlock ( LoopHeader ) ;


<s> void <ISA_LIT>CFGStructurizer :: addDummyExitBlock ( SmallVectorImpl < MachineBasicBlock * > & RetMBB ) { MachineBasicBlock * DummyExitBlk = FuncRep -> CreateMachineBasicBlock ( ) ; FuncRep -> push_back ( DummyExitBlk ) ; insertInstrEnd ( DummyExitBlk , <STR_LIT> ) ; for ( SmallVectorImpl < MachineBasicBlock * > :: iterator It = RetMBB . begin ( ) , E = RetMBB . end ( ) ; It != E ; ++ It ) { MachineBasicBlock * MBB = * It ; MachineInstr * MI = getReturnInstr ( MBB ) ; if ( MI ) MI -> eraseFromParent ( ) ; MBB -> addSuccessor ( DummyExitBlk ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ;


<s> if ( MI ) MI -> eraseFromParent ( ) ; MBB -> addSuccessor ( DummyExitBlk ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ; } SHOWNEWBLK ( DummyExitBlk , <STR_LIT> ) ;


<s> MachineBasicBlock * LandBlk = * LandMBBPtr ; assert ( ( ! TrueMBB || TrueMBB -> succ_size ( ) <= <NUM_LIT> ) && ( ! FalseMBB || FalseMBB -> succ_size ( ) <= <NUM_LIT> ) ) ; if ( TrueMBB == FalseMBB ) return <NUM_LIT> ; MigrateTrue = needMigrateBlock ( TrueMBB ) ; MigrateFalse = needMigrateBlock ( FalseMBB ) ; if ( ! MigrateTrue && ! MigrateFalse ) return <NUM_LIT> ; if ( ! MigrateTrue && TrueMBB && TrueMBB -> pred_size ( ) > <NUM_LIT> ) MigrateTrue = true ; if ( ! MigrateFalse && FalseMBB && FalseMBB -> pred_size ( ) > <NUM_LIT> ) MigrateFalse = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; showImproveSimpleJumpintoIf ( HeadMBB , TrueMBB , FalseMBB , LandBlk , <NUM_LIT> ) ; ) ; const TargetRegisterClass * I32RC = TRI -> getCFGStructurizerRegClass ( <STR_LIT> ) ; if ( ! MigrateTrue || ! MigrateFalse ) { return <NUM_LIT> ; } int NumNewBlk = <NUM_LIT> ; bool LandBlkHasOtherPred = ( LandBlk -> pred_size ( ) > <NUM_LIT> ) ; MachineBasicBlock :: iterator I = insertInstrBefore ( LandBlk , <STR_LIT> ) ; if ( LandBlkHasOtherPred ) { report_fatal_error ( <STR_LIT> ) ; Register CmpResReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) ; dbgs ( ) << <STR_LIT> ; if ( TrueMBB ) { dbgs ( ) << <STR_LIT> << TrueMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; if ( FalseMBB ) { dbgs ( ) << <STR_LIT> << FalseMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; if ( ! LandMBB ) { dbgs ( ) << <STR_LIT> ; } else { dbgs ( ) << <STR_LIT> << LandMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; ) ; int OldOpcode = BranchMI -> getOpcode ( ) ; DebugLoc BranchDL = BranchMI -> getDebugLoc ( ) ; MachineBasicBlock :: iterator I = BranchMI ; insertCondBranchBefore ( I , getBranchNzeroOpcode ( OldOpcode ) , BranchDL ) ; if ( TrueMBB ) { MBB -> splice ( I , TrueMBB , TrueMBB -> begin ( ) , TrueMBB -> end ( ) ) ; MBB -> removeSuccessor ( TrueMBB , true ) ; if ( LandMBB && TrueMBB -> succ_size ( ) != <NUM_LIT> ) TrueMBB -> removeSuccessor ( LandMBB , true ) ; retireBlock ( TrueMBB ) ; MLI -> removeBlock ( TrueMBB ) ; } if ( FalseMBB ) { insertInstrBefore ( I , <STR_LIT> ) ; MBB -> splice ( I , FalseMBB , FalseMBB -> begin ( ) , FalseMBB -> end ( ) ) ; MBB -> removeSuccessor ( FalseMBB , true ) ; if ( LandMBB && FalseMBB -> succ_size ( ) != <NUM_LIT> ) FalseMBB -> removeSuccessor ( LandMBB , true ) ; retireBlock ( FalseMBB ) ; MLI -> removeBlock ( FalseMBB ) ; } insertInstrBefore ( I , <STR_LIT> ) ;


<s> assert ( TrueMBB ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) ; dbgs ( ) << <STR_LIT> ; if ( TrueMBB ) { dbgs ( ) << <STR_LIT> << TrueMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; if ( FalseMBB ) { dbgs ( ) << <STR_LIT> << FalseMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; if ( ! LandMBB ) { dbgs ( ) << <STR_LIT> ; } else { dbgs ( ) << <STR_LIT> << LandMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; ) ; int OldOpcode = BranchMI -> getOpcode ( ) ; DebugLoc BranchDL = BranchMI -> getDebugLoc ( ) ; MachineBasicBlock :: iterator I = BranchMI ;

insertCondBranchBefore ( I , getBranchNzeroOpcode ( OldOpcode ) , BranchDL ) ;


<s> assert ( ! ExitingMBBs . empty ( ) && <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << ExitingMBBs . size ( ) << <STR_LIT> ; ) ; MBBVector ExitBlks ; LoopRep -> getExitBlocks ( ExitBlks ) ; SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > ExitBlkSet ; for ( unsigned i = <NUM_LIT> , e = ExitBlks . size ( ) ; i < e ; ++ i ) ExitBlkSet . insert ( ExitBlks [ i ] ) ; assert ( ExitBlkSet . size ( ) == <NUM_LIT> ) ; MachineBasicBlock * ExitBlk = * ExitBlks . begin ( ) ; assert ( ExitBlk && <STR_LIT> ) ; MBBVector LatchBlks ; for ( auto * LB : inverse_children < MachineBasicBlock * > ( LoopHeader ) ) if ( LoopRep -> contains ( LB ) ) LatchBlks . push_back ( LB ) ; for ( unsigned i = <NUM_LIT> , e = ExitingMBBs . size ( ) ; i < e ; ++ i ) mergeLoopbreakBlock ( ExitingMBBs [ i ] , ExitBlk ) ; for ( unsigned i = <NUM_LIT> , e = LatchBlks . size ( ) ; i < e ; ++ i ) settleLoopcontBlock ( LatchBlks [ i ] , LoopHeader ) ; int Match = <NUM_LIT> ; do { Match = <NUM_LIT> ; Match += serialPatternMatch ( LoopHeader ) ; Match += ifPatternMatch ( LoopHeader ) ; } while ( Match > <NUM_LIT> ) ; mergeLooplandBlock ( LoopHeader , ExitBlk ) ; MachineLoop * ParentLoop = LoopRep -> getParentLoop ( ) ; if ( ParentLoop ) MLI -> changeLoopFor ( LoopHeader , ParentLoop ) ;

else MLI -> removeBlock ( LoopHeader ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI ) ; SpliceEnd = BranchMI ; } DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> size ( ) << <STR_LIT> << SrcMBB -> size ( ) << <STR_LIT> ; ) ; DstMBB -> splice ( I , SrcMBB , SrcMBB -> begin ( ) , SpliceEnd ) ; DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> size ( ) << <STR_LIT> << SrcMBB -> size ( ) << '\n' ; ) ;


<s> MachineInstr * BranchMI = getNormalBlockBranchInstr ( SrcMBB ) ; if ( ! BranchMI ) { DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; SpliceEnd = SrcMBB -> end ( ) ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI ) ; SpliceEnd = BranchMI ; } DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> size ( ) << <STR_LIT> << SrcMBB -> size ( ) << <STR_LIT> ; ) ; DstMBB -> splice ( I , SrcMBB , SrcMBB -> begin ( ) , SpliceEnd ) ; DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> size ( ) << <STR_LIT> << SrcMBB -> size ( ) << '\n' ; ) ;


<s> } } for ( SmallVectorImpl < MachineBasicBlock * > :: iterator It = ContMBB . begin ( ) , E = ContMBB . end ( ) ; It != E ; ++ It ) { ( * It ) -> removeSuccessor ( LoopHeader , true ) ; } numLoopcontPatternMatch += NumCont ; return NumCont ;


<s> int <ISA_LIT>CFGStructurizer :: loopcontPatternMatch ( MachineLoop * LoopRep , MachineBasicBlock * LoopHeader ) { int NumCont = <NUM_LIT> ; SmallVector < MachineBasicBlock * , DEFAULT_VEC_SLOTS > ContMBB ; typedef GraphTraits < Inverse < MachineBasicBlock * > > GTIM ; GTIM :: ChildIteratorType It = GTIM :: child_begin ( LoopHeader ) , E = GTIM :: child_end ( LoopHeader ) ; for ( ; It != E ; ++ It ) { MachineBasicBlock * MBB = * It ; if ( LoopRep -> contains ( MBB ) ) { handleLoopcontBlock ( MBB , MLI -> getLoopFor ( MBB ) , LoopHeader , LoopRep ) ; ContMBB . push_back ( MBB ) ; ++ NumCont ; } } for ( SmallVectorImpl < MachineBasicBlock * > :: iterator It = ContMBB . begin ( ) , E = ContMBB . end ( ) ; It != E ; ++ It ) { ( * It ) -> removeSuccessor ( LoopHeader , true ) ;


<s> if ( TrueBranch != LandMBB ) reversePredicateSetter ( I ) ; insertCondBranchBefore ( ExitingMBB , I , <STR_LIT> , <STR_LIT> , DL ) ; insertInstrBefore ( I , <STR_LIT> ) ; insertInstrBefore ( I , <STR_LIT> ) ; BranchMI -> eraseFromParent ( ) ; ExitingMBB -> removeSuccessor ( LandMBB , true ) ;


<s> Finish = true ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } else { int NewnumRemainedBlk = countActiveBlock ( OrderedBlks . begin ( ) , OrderedBlks . end ( ) ) ; if ( NewnumRemainedBlk == <NUM_LIT> || NewnumRemainedBlk < NumRemainedBlk ) { MakeProgress = true ; NumRemainedBlk = NewnumRemainedBlk ; } else { MakeProgress = false ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } } } while ( ! Finish && MakeProgress ) ; wrapup ( & * GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ) ; for ( MBBInfoMap :: iterator It = BlockInfoMap . begin ( ) , E = BlockInfoMap . end ( ) ; It != E ; ++ It ) { if ( ( * It ) . second && ( * It ) . second -> IsRetired ) { assert ( ( ( * It ) . first ) -> getNumber ( ) != - <NUM_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << ( ( * It ) . first ) -> getNumber ( ) << <STR_LIT> ; ) ; ( * It ) . first -> eraseFromParent ( ) ; } delete ( * It ) . second ; }


<s> if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk >= SccNumBlk ) { DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter ; dbgs ( ) << <STR_LIT> ; ) ; ContNextScc = true ; } else if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk < SccNumBlk ) { SccNumBlk = sccRemainedNumBlk ; It = SccBeginIter ; ContNextScc = false ; DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter << '\n' ; ) ; } else { ContNextScc = true ; } } else { ContNextScc = false ; } if ( ContNextScc ) SccBeginMBB = nullptr ; } MachineBasicBlock * EntryMBB = & * GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ; if ( EntryMBB -> succ_size ( ) == <NUM_LIT> ) { Finish = true ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } else { int NewnumRemainedBlk = countActiveBlock ( OrderedBlks . begin ( ) , OrderedBlks . end ( ) ) ; if ( NewnumRemainedBlk == <NUM_LIT> || NewnumRemainedBlk < NumRemainedBlk ) { MakeProgress = true ; NumRemainedBlk = NewnumRemainedBlk ; } else { MakeProgress = false ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; }


<s> SccBeginMBB = MBB ; SccNumIter = <NUM_LIT> ; SccNumBlk = NumRemainedBlk ; DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( SccBeginMBB ) ; dbgs ( ) << <STR_LIT> ; ) ; } if ( ! isRetiredBlock ( MBB ) ) patternMatch ( MBB ) ; ++ It ; bool ContNextScc = true ; if ( It == E || getSCCNum ( SccBeginMBB ) != getSCCNum ( * It ) ) { ++ SccNumIter ; int sccRemainedNumBlk = countActiveBlock ( SccBeginIter , It ) ; if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk >= SccNumBlk ) { DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter ; dbgs ( ) << <STR_LIT> ; ) ; ContNextScc = true ; } else if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk < SccNumBlk ) { SccNumBlk = sccRemainedNumBlk ; It = SccBeginIter ; ContNextScc = false ; DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter << '\n' ; ) ; } else { ContNextScc = true ; } } else { ContNextScc = false ; } if ( ContNextScc ) SccBeginMBB = nullptr ; } MachineBasicBlock * EntryMBB = & * GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ; if ( EntryMBB -> succ_size ( ) == <NUM_LIT> ) { Finish = true ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } else { int NewnumRemainedBlk = countActiveBlock ( OrderedBlks . begin ( ) , OrderedBlks . end ( ) ) ; if ( NewnumRemainedBlk == <NUM_LIT> || NewnumRemainedBlk < NumRemainedBlk ) { MakeProgress = true ; NumRemainedBlk = NewnumRemainedBlk ; } else { MakeProgress = false ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } } } while ( ! Finish && MakeProgress ) ; wrapup ( & * GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ) ; for ( MBBInfoMap :: iterator It = BlockInfoMap . begin ( ) , E = BlockInfoMap . end ( ) ; It != E ; ++ It ) { if ( ( * It ) . second && ( * It ) . second -> IsRetired ) { assert ( ( ( * It ) . first ) -> getNumber ( ) != - <NUM_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << ( ( * It ) . first ) -> getNumber ( ) << <STR_LIT> ; ) ; ( * It ) . first -> eraseFromParent ( ) ; } delete ( * It ) . second ; }


<s> MachineBasicBlock * LoopLatch = LoopRep -> getLoopLatch ( ) ; if ( ! LoopHeader || ! LoopLatch ) return nullptr ; MachineInstr * BranchMI = getLoopendBlockBranchInstr ( LoopLatch ) ; if ( ! BranchMI || ! isUncondBranch ( BranchMI ) ) return nullptr ; MachineBasicBlock * DummyExitBlk = FuncRep -> CreateMachineBasicBlock ( ) ; FuncRep -> push_back ( DummyExitBlk ) ; SHOWNEWBLK ( DummyExitBlk , <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI << <STR_LIT> ; ) ; LLVMContext & Ctx = LoopHeader -> getParent ( ) -> getFunction ( ) . getContext ( ) ; Ctx . emitError ( <STR_LIT> ) ; return nullptr ;


<s> return ; case OPCODE_IS_ZERO : static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . setImm ( OPCODE_IS_NOT_ZERO ) ; return ; case OPCODE_IS_NOT_ZERO : static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . setImm ( OPCODE_IS_ZERO ) ; return ; default : llvm_unreachable ( <STR_LIT> ) ;


<s> if ( I -> getOpcode ( ) == <STR_LIT> ) { switch ( static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . getImm ( ) ) { case OPCODE_IS_ZERO_INT : static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . setImm ( OPCODE_IS_NOT_ZERO_INT ) ; return ; case OPCODE_IS_NOT_ZERO_INT : static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . setImm ( OPCODE_IS_ZERO_INT ) ; return ; case OPCODE_IS_ZERO :

static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . setImm ( OPCODE_IS_NOT_ZERO ) ;


<s> ) ; SpliceEnd = SrcMBB -> end ( ) ; } else { DEBUG ( dbgs ( ) << <STR_LIT> ; BranchMI -> dump ( ) ; ) ; SpliceEnd = BranchMI ; } DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> size ( ) << <STR_LIT> << SrcMBB -> size ( ) << <STR_LIT> ; ) ;

DstMBB -> splice ( I , SrcMBB , SrcMBB -> begin ( ) , SpliceEnd ) ;


<s> if ( MBB -> succ_size ( ) != <NUM_LIT> ) return ; MachineBasicBlock * MBB1 = * MBB -> succ_begin ( ) ; MachineBasicBlock * MBB2 = * std :: next ( MBB -> succ_begin ( ) ) ;


<s> MachineBasicBlock * MBB2 = * std :: next ( MBB -> succ_begin ( ) ) ; if ( MBB1 != MBB2 ) return ; MachineInstr * BranchMI = getNormalBlockBranchInstr ( MBB ) ; assert ( BranchMI && isCondBranch ( BranchMI ) ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; BranchMI -> dump ( ) ; ) ; BranchMI -> eraseFromParent ( ) ;


while ( ( BranchMI = getLoopendBlockBranchInstr ( MBB ) ) && isUncondBranch ( BranchMI ) ) {


<s> while ( ( BranchMI = getLoopendBlockBranchInstr ( MBB ) ) && isUncondBranch ( BranchMI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> ; BranchMI -> dump ( ) ; ) ; BranchMI -> eraseFromParent ( ) ;


<s> if ( ExitingMBBs . size ( ) == <NUM_LIT> ) { MachineBasicBlock * DummyExitBlk = normalizeInfiniteLoopExit ( LoopRep ) ; if ( DummyExitBlk ) RetBlks . push_back ( DummyExitBlk ) ; } } for ( MachineBasicBlock * MBB : OrderedBlks ) { removeUnconditionalBranch ( MBB ) ; removeRedundantConditionalBranch ( MBB ) ; if ( isReturnBlock ( MBB ) ) { RetBlks . push_back ( MBB ) ;


<s> static void PrintLoopinfo ( const MachineLoopInfo & LoopInfo ) { for ( const MachineLoop * L : LoopInfo ) L -> print ( dbgs ( ) ) ;


<s> ++ NumIter ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << NumIter << <STR_LIT> << NumRemainedBlk << <STR_LIT> ; ) ; SmallVectorImpl < MachineBasicBlock * > :: const_iterator It = OrderedBlks . begin ( ) ; SmallVectorImpl < MachineBasicBlock * > :: const_iterator E = OrderedBlks . end ( ) ; SmallVectorImpl < MachineBasicBlock * > :: const_iterator SccBeginIter = It ; MachineBasicBlock * SccBeginMBB = nullptr ; int SccNumBlk = <NUM_LIT> ; int SccNumIter ; while ( It != E ) { MBB = * It ; if ( ! SccBeginMBB ) { SccBeginIter = It ; SccBeginMBB = MBB ; SccNumIter = <NUM_LIT> ; SccNumBlk = NumRemainedBlk ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( SccBeginMBB ) ; dbgs ( ) << <STR_LIT> ; ) ; } if ( ! isRetiredBlock ( MBB ) ) patternMatch ( MBB ) ; ++ It ; bool ContNextScc = true ; if ( It == E || getSCCNum ( SccBeginMBB ) != getSCCNum ( * It ) ) { ++ SccNumIter ; int sccRemainedNumBlk = countActiveBlock ( SccBeginIter , It ) ; if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk >= SccNumBlk ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter ; dbgs ( ) << <STR_LIT> ; ) ; ContNextScc = true ; } else if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk < SccNumBlk ) { SccNumBlk = sccRemainedNumBlk ; It = SccBeginIter ; ContNextScc = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter << '\n' ; ) ; } else { ContNextScc = true ; } } else {


<s> int <ISA_LIT>CFGStructurizer :: improveSimpleJumpintoIf ( MachineBasicBlock * HeadMBB , MachineBasicBlock * TrueMBB , MachineBasicBlock * FalseMBB , MachineBasicBlock * * LandMBBPtr ) { bool MigrateTrue = false ; bool MigrateFalse = false ; MachineBasicBlock * LandBlk = * LandMBBPtr ; assert ( ( ! TrueMBB || TrueMBB -> succ_size ( ) <= <NUM_LIT> ) && ( ! FalseMBB || FalseMBB -> succ_size ( ) <= <NUM_LIT> ) ) ; if ( TrueMBB == FalseMBB ) return <NUM_LIT> ; MigrateTrue = needMigrateBlock ( TrueMBB ) ; MigrateFalse = needMigrateBlock ( FalseMBB ) ; if ( ! MigrateTrue && ! MigrateFalse ) return <NUM_LIT> ; if ( ! MigrateTrue && TrueMBB && TrueMBB -> pred_size ( ) > <NUM_LIT> ) MigrateTrue = true ; if ( ! MigrateFalse && FalseMBB && FalseMBB -> pred_size ( ) > <NUM_LIT> ) MigrateFalse = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; showImproveSimpleJumpintoIf ( HeadMBB , TrueMBB , FalseMBB , LandBlk , <NUM_LIT> ) ; ) ; const TargetRegisterClass * I32RC = TRI -> getCFGStructurizerRegClass ( <STR_LIT> ) ; if ( ! MigrateTrue || ! MigrateFalse ) { return <NUM_LIT> ; } int NumNewBlk = <NUM_LIT> ; bool LandBlkHasOtherPred = ( LandBlk -> pred_size ( ) > <NUM_LIT> ) ; MachineBasicBlock :: iterator I = insertInstrBefore ( LandBlk , <STR_LIT> ) ; if ( LandBlkHasOtherPred ) { report_fatal_error ( <STR_LIT> ) ; unsigned CmpResReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; report_fatal_error ( <STR_LIT> ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , CmpResReg , DebugLoc ( ) ) ; } unsigned InitReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , InitReg , DebugLoc ( ) ) ; if ( MigrateTrue ) { migrateInstruction ( TrueMBB , LandBlk , I ) ; report_fatal_error ( <STR_LIT> ) ; } insertInstrBefore ( I , <STR_LIT> ) ; if ( MigrateFalse ) { migrateInstruction ( FalseMBB , LandBlk , I ) ; report_fatal_error ( <STR_LIT> ) ; } if ( LandBlkHasOtherPred ) {


<s> MachineBasicBlock * MBB = * It ; MachineInstr * MI = getReturnInstr ( MBB ) ; if ( MI ) MI -> eraseFromParent ( ) ; MBB -> addSuccessor ( DummyExitBlk ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ;


<s> <ISA_LIT>CFGStructurizer ( ) : MachineFunctionPass ( ID ) , TII ( nullptr ) , TRI ( nullptr ) {

initialize<ISA_LIT>CFGStructurizerPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> MachineFunctionPass ( ID ) , TII ( nullptr ) , TRI ( nullptr ) {


<s> BlockInformation ( ) : IsRetired ( false ) , SccNum ( INVALIDSCCNUM ) {


<s> for ( MachineBasicBlock :: iterator It = MBB -> begin ( ) , E = MBB -> end ( ) ; It != E ; ++ It ) {

MachineInstr * MI = Func -> CloneMachineInstr ( It ) ;


<s> MachineBasicBlock * <ISA_LIT>CFGStructurizer :: clone ( MachineBasicBlock * MBB ) { MachineFunction * Func = MBB -> getParent ( ) ; MachineBasicBlock * NewMBB = Func -> CreateMachineBasicBlock ( ) ;


<s> PredMBB -> addSuccessor ( CloneMBB ) ; cloneSuccessorList ( CloneMBB , MBB ) ; numClonedInstr += MBB -> size ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << MBB -> size ( ) << <STR_LIT> ; ) ; SHOWNEWBLK ( CloneMBB , <STR_LIT> ) ;


<s> MachineBasicBlock * <ISA_LIT>CFGStructurizer :: cloneBlockForPredecessor ( MachineBasicBlock * MBB , MachineBasicBlock * PredMBB ) { assert ( PredMBB -> isSuccessor ( MBB ) && <STR_LIT> ) ; MachineBasicBlock * CloneMBB = clone ( MBB ) ; replaceInstrUseOfBlockWith ( PredMBB , MBB , CloneMBB ) ; PredMBB -> removeSuccessor ( MBB ) ; PredMBB -> addSuccessor ( CloneMBB ) ; cloneSuccessorList ( CloneMBB , MBB ) ; numClonedInstr += MBB -> size ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << MBB -> size ( ) << <STR_LIT> ;


<s> assert ( SrcMBB -> succ_size ( ) == <NUM_LIT> ) ; if ( SrcMBB -> pred_size ( ) > <NUM_LIT> ) { SrcMBB = cloneBlockForPredecessor ( SrcMBB , PreMBB ) ; ++ Cloned ; } PreMBB = SrcMBB ; SrcMBB = * SrcMBB -> succ_begin ( ) ; } return Cloned ;


<s> void <ISA_LIT>CFGStructurizer :: cloneSuccessorList ( MachineBasicBlock * DstMBB , MachineBasicBlock * SrcMBB ) { for ( MachineBasicBlock :: succ_iterator It = SrcMBB -> succ_begin ( ) , iterEnd = SrcMBB -> succ_end ( ) ; It != iterEnd ; ++ It ) DstMBB -> addSuccessor ( * It ) ;


<s> int <ISA_LIT>CFGStructurizer :: countActiveBlock ( MBBVector :: const_iterator It , MBBVector :: const_iterator E ) const { int Count = <NUM_LIT> ; while ( It != E ) { if ( ! isRetiredBlock ( * It ) ) ++ Count ; ++ It ;


<s> int <ISA_LIT>CFGStructurizer :: countActiveBlock ( MBBVector :: const_iterator It , MBBVector :: const_iterator E ) const {


<s> return new <ISA_LIT>CFGStructurizer ( ) ;


<s> std :: set < MachineBasicBlock * > :: const_iterator It = MBBs . begin ( ) ; std :: set < MachineBasicBlock * > :: const_iterator E = MBBs . end ( ) ;

for ( CommonDom = * It ; It != E && CommonDom ; ++ It ) {


<s> std :: set < MachineBasicBlock * > :: const_iterator E = MBBs . end ( ) ; for ( CommonDom = * It ; It != E && CommonDom ; ++ It ) { MachineBasicBlock * MBB = * It ; if ( MBB != CommonDom ) CommonDom = findNearestCommonPostDom ( MBB , CommonDom ) ; } DEBUG ( dbgs ( ) << <STR_LIT> ; if ( CommonDom ) dbgs ( ) << <STR_LIT> << CommonDom -> getNumber ( ) << <STR_LIT> ;


<s> AU . addPreserved < MachineFunctionAnalysis > ( ) ; AU . addRequired < MachineFunctionAnalysis > ( ) ; AU . addRequired < MachineDominatorTree > ( ) ; AU . addRequired < MachinePostDominatorTree > ( ) ; AU . addRequired < MachineLoopInfo > ( ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addPreserved < MachineFunctionAnalysis > ( ) ; AU . addRequired < MachineFunctionAnalysis > ( ) ;

AU . addRequired < MachineDominatorTree > ( ) ;


<s> int <ISA_LIT>CFGStructurizer :: getBranchNzeroOpcode ( int OldOpcode ) { switch ( OldOpcode ) { case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ;


<s> case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ;

default : llvm_unreachable ( <STR_LIT> ) ;


<s> case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; } return - <NUM_LIT> ;


<s> MachineBasicBlock :: reverse_iterator It = MBB -> rbegin ( ) ;


<s> int <ISA_LIT>CFGStructurizer :: getContinueNzeroOpcode ( int OldOpcode ) { switch ( OldOpcode ) { case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ;


<s> case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; } ;


<s> case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; } return - <NUM_LIT> ;


<s> switch ( OldOpcode ) { case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ;


<s> MachineBasicBlock * TrueBranch = getTrueBranch ( MI ) ; MachineBasicBlock :: succ_iterator It = MBB -> succ_begin ( ) ; MachineBasicBlock :: succ_iterator Next = It ;


<s> MachineBasicBlock * TrueBranch = getTrueBranch ( MI ) ; MachineBasicBlock :: succ_iterator It = MBB -> succ_begin ( ) ;

MachineBasicBlock :: succ_iterator Next = It ;


<s> MachineInstr * instr = & ( * It ) ; if ( instr -> getDebugLoc ( ) ) DL = instr -> getDebugLoc ( ) ; }


<s> ++ It ) { MachineInstr * instr = & ( * It ) ; if ( instr -> getDebugLoc ( ) ) DL = instr -> getDebugLoc ( ) ;


<s> return LoopRep ? LoopRep -> getLoopDepth ( ) : <NUM_LIT> ;


<s> unsigned <ISA_LIT>CFGStructurizer :: getLoopDepth ( MachineLoop * LoopRep ) { return LoopRep ? LoopRep -> getLoopDepth ( ) : <NUM_LIT> ;


<s> for ( MachineBasicBlock :: reverse_iterator It = MBB -> rbegin ( ) , E = MBB -> rend ( ) ; It != E ; ++ It ) { MachineInstr * MI = & * It ; if ( MI ) { if ( isCondBranch ( MI ) || isUncondBranch ( MI ) ) return MI ; else if ( ! TII -> isMov ( MI -> getOpcode ( ) ) ) break ;


<s> if ( It == LLInfoMap . end ( ) ) return nullptr ;


<s> MachineBasicBlock * <ISA_LIT>CFGStructurizer :: getLoopLandInfo ( MachineLoop * LoopRep ) const { LoopLandInfoMap :: const_iterator It = LLInfoMap . find ( LoopRep ) ; if ( It == LLInfoMap . end ( ) ) return nullptr ;


<s> MachineInstr * MI = & * It ; if ( MI && ( isCondBranch ( MI ) || isUncondBranch ( MI ) ) ) return MI ;


if ( MI && ( isCondBranch ( MI ) || isUncondBranch ( MI ) ) ) return MI ;


<s> const char * getPassName ( ) const override {


<s> if ( It != MBB -> rend ( ) ) { MachineInstr * instr = & ( * It ) ; if ( instr -> getOpcode ( ) == <STR_LIT> ) return instr ; }


<s> MachineBasicBlock :: reverse_iterator It = MBB -> rbegin ( ) ; if ( It != MBB -> rend ( ) ) { MachineInstr * instr = & ( * It ) ; if ( instr -> getOpcode ( ) == <STR_LIT> ) return instr ; } return nullptr ;


<s> if ( It == BlockInfoMap . end ( ) ) return INVALIDSCCNUM ; return ( * It ) . second -> SccNum ;


<s> MBBInfoMap :: const_iterator It = BlockInfoMap . find ( MBB ) ;

if ( It == BlockInfoMap . end ( ) ) return INVALIDSCCNUM ;


<s> MachineBasicBlock * <ISA_LIT>CFGStructurizer :: getTrueBranch ( MachineInstr * MI ) {

return MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ;


<s> return MI -> getOperand ( <NUM_LIT> ) . getMBB ( ) ;


<s> int <ISA_LIT>CFGStructurizer :: handleJumpintoIf ( MachineBasicBlock * HeadMBB , MachineBasicBlock * TrueMBB , MachineBasicBlock * FalseMBB ) { int Num = handleJumpintoIfImp ( HeadMBB , TrueMBB , FalseMBB ) ; if ( Num == <NUM_LIT> ) {

DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> ;


<s> while ( DownBlk ) { DEBUG ( dbgs ( ) << <STR_LIT> << DownBlk -> getNumber ( ) ; ) ; if ( singlePathTo ( FalseMBB , DownBlk ) == SinglePath_InPath ) { DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; Num += cloneOnSideEntryTo ( HeadMBB , TrueMBB , DownBlk ) ; Num += cloneOnSideEntryTo ( HeadMBB , FalseMBB , DownBlk ) ; numClonedBlock += Num ; Num += serialPatternMatch ( * HeadMBB -> succ_begin ( ) ) ; Num += serialPatternMatch ( * std :: next ( HeadMBB -> succ_begin ( ) ) ) ; Num += ifPatternMatch ( HeadMBB ) ; assert ( Num > <NUM_LIT> ) ; break ; } DEBUG ( dbgs ( ) << <STR_LIT> ;


<s> int <ISA_LIT>CFGStructurizer :: handleJumpintoIfImp ( MachineBasicBlock * HeadMBB , MachineBasicBlock * TrueMBB , MachineBasicBlock * FalseMBB ) { int Num = <NUM_LIT> ; MachineBasicBlock * DownBlk ; DownBlk = TrueMBB ; DEBUG ( dbgs ( ) << <STR_LIT> << HeadMBB -> getNumber ( ) << <STR_LIT> << TrueMBB -> getNumber ( ) << <STR_LIT> << TrueMBB -> succ_size ( ) << <STR_LIT> << FalseMBB -> getNumber ( ) << <STR_LIT> ; ) ; while ( DownBlk ) { DEBUG ( dbgs ( ) << <STR_LIT> << DownBlk -> getNumber ( ) ; ) ; if ( singlePathTo ( FalseMBB , DownBlk ) == SinglePath_InPath ) { DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; Num += cloneOnSideEntryTo ( HeadMBB , TrueMBB , DownBlk ) ; Num += cloneOnSideEntryTo ( HeadMBB , FalseMBB , DownBlk ) ; numClonedBlock += Num ; Num += serialPatternMatch ( * HeadMBB -> succ_begin ( ) ) ; Num += serialPatternMatch ( * std :: next ( HeadMBB -> succ_begin ( ) ) ) ; Num += ifPatternMatch ( HeadMBB ) ; assert ( Num > <NUM_LIT> ) ; break ; } DEBUG ( dbgs ( ) << <STR_LIT> ; ) ;

DownBlk = ( DownBlk -> succ_size ( ) == <NUM_LIT> ) ? ( * DownBlk -> succ_begin ( ) ) : nullptr ;


<s> void <ISA_LIT>CFGStructurizer :: handleLoopcontBlock ( MachineBasicBlock * ContingMBB , MachineLoop * ContingLoop , MachineBasicBlock * ContMBB , MachineLoop * ContLoop ) {

DEBUG ( dbgs ( ) << <STR_LIT> << ContingMBB -> getNumber ( ) << <STR_LIT> << ContMBB -> getNumber ( ) << <STR_LIT> ;


<s> dbgs ( ) << <STR_LIT> << getLoopDepth ( ContLoop ) << <STR_LIT> << getLoopDepth ( ContingLoop ) << <STR_LIT> ; ) ;

settleLoopcontBlock ( ContingMBB , ContMBB ) ;


<s> MachineBasicBlock * LoopHeader = LoopRep -> getHeader ( ) ; return MBB -> isSuccessor ( LoopHeader ) ;


<s> MachineBasicBlock * LoopHeader = LoopRep -> getHeader ( ) ;

return MBB -> isSuccessor ( LoopHeader ) ;


<s> MachineInstr * BranchMI = getNormalBlockBranchInstr ( MBB ) ; if ( ! BranchMI ) return <NUM_LIT> ; assert ( isCondBranch ( BranchMI ) ) ; int NumMatch = <NUM_LIT> ; MachineBasicBlock * TrueMBB = getTrueBranch ( BranchMI ) ; NumMatch += serialPatternMatch ( TrueMBB ) ; NumMatch += ifPatternMatch ( TrueMBB ) ; MachineBasicBlock * FalseMBB = getFalseBranch ( MBB , BranchMI ) ; NumMatch += serialPatternMatch ( FalseMBB ) ; NumMatch += ifPatternMatch ( FalseMBB ) ; MachineBasicBlock * LandBlk ; int Cloned = <NUM_LIT> ; assert ( ! TrueMBB -> succ_empty ( ) || ! FalseMBB -> succ_empty ( ) ) ; if ( TrueMBB -> succ_size ( ) == <NUM_LIT> && FalseMBB -> succ_size ( ) == <NUM_LIT> && * TrueMBB -> succ_begin ( ) == * FalseMBB -> succ_begin ( ) ) { LandBlk = * TrueMBB -> succ_begin ( ) ; } else if ( TrueMBB -> succ_size ( ) == <NUM_LIT> && * TrueMBB -> succ_begin ( ) == FalseMBB ) { LandBlk = FalseMBB ; FalseMBB = nullptr ; } else if ( FalseMBB -> succ_size ( ) == <NUM_LIT> && * FalseMBB -> succ_begin ( ) == TrueMBB ) { std :: swap ( TrueMBB , FalseMBB ) ; reversePredicateSetter ( MBB -> end ( ) ) ; LandBlk = FalseMBB ; FalseMBB = nullptr ;

} else if ( FalseMBB -> succ_size ( ) == <NUM_LIT> && isSameloopDetachedContbreak ( TrueMBB , FalseMBB ) ) {


<s> bool MigrateTrue = false ; bool MigrateFalse = false ; MachineBasicBlock * LandBlk = * LandMBBPtr ; assert ( ( ! TrueMBB || TrueMBB -> succ_size ( ) <= <NUM_LIT> ) && ( ! FalseMBB || FalseMBB -> succ_size ( ) <= <NUM_LIT> ) ) ; if ( TrueMBB == FalseMBB ) return <NUM_LIT> ; MigrateTrue = needMigrateBlock ( TrueMBB ) ; MigrateFalse = needMigrateBlock ( FalseMBB ) ; if ( ! MigrateTrue && ! MigrateFalse ) return <NUM_LIT> ; if ( ! MigrateTrue && TrueMBB && TrueMBB -> pred_size ( ) > <NUM_LIT> ) MigrateTrue = true ; if ( ! MigrateFalse && FalseMBB && FalseMBB -> pred_size ( ) > <NUM_LIT> ) MigrateFalse = true ; DEBUG ( dbgs ( ) << <STR_LIT> ; showImproveSimpleJumpintoIf ( HeadMBB , TrueMBB , FalseMBB , LandBlk , <NUM_LIT> ) ; ) ; const TargetRegisterClass * I32RC = TRI -> getCFGStructurizerRegClass ( <STR_LIT> ) ; if ( ! MigrateTrue || ! MigrateFalse ) { return <NUM_LIT> ; } int NumNewBlk = <NUM_LIT> ; bool LandBlkHasOtherPred = ( LandBlk -> pred_size ( ) > <NUM_LIT> ) ; MachineBasicBlock :: iterator I = insertInstrBefore ( LandBlk , <STR_LIT> ) ; if ( LandBlkHasOtherPred ) { llvm_unreachable ( <STR_LIT> ) ; unsigned CmpResReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; llvm_unreachable ( <STR_LIT> ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , CmpResReg , DebugLoc ( ) ) ; } unsigned InitReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , InitReg , DebugLoc ( ) ) ; if ( MigrateTrue ) { migrateInstruction ( TrueMBB , LandBlk , I ) ; llvm_unreachable ( <STR_LIT> ) ; } insertInstrBefore ( I , <STR_LIT> ) ; if ( MigrateFalse ) {

migrateInstruction ( FalseMBB , LandBlk , I ) ;


<s> ) ; const TargetRegisterClass * I32RC = TRI -> getCFGStructurizerRegClass ( <STR_LIT> ) ; if ( ! MigrateTrue || ! MigrateFalse ) { return <NUM_LIT> ; } int NumNewBlk = <NUM_LIT> ; bool LandBlkHasOtherPred = ( LandBlk -> pred_size ( ) > <NUM_LIT> ) ; MachineBasicBlock :: iterator I = insertInstrBefore ( LandBlk , <STR_LIT> ) ; if ( LandBlkHasOtherPred ) { llvm_unreachable ( <STR_LIT> ) ; unsigned CmpResReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; llvm_unreachable ( <STR_LIT> ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , CmpResReg , DebugLoc ( ) ) ; } unsigned InitReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , InitReg , DebugLoc ( ) ) ; if ( MigrateTrue ) { migrateInstruction ( TrueMBB , LandBlk , I ) ; llvm_unreachable ( <STR_LIT> ) ; } insertInstrBefore ( I , <STR_LIT> ) ; if ( MigrateFalse ) { migrateInstruction ( FalseMBB , LandBlk , I ) ; llvm_unreachable ( <STR_LIT> ) ; } if ( LandBlkHasOtherPred ) { insertInstrBefore ( I , <STR_LIT> ) ;

for ( MachineBasicBlock :: pred_iterator PI = LandBlk -> pred_begin ( ) , PE = LandBlk -> pred_end ( ) ; PI != PE ; ++ PI ) {


<s> MachineFunction * MF = blk -> getParent ( ) ; MachineInstr * NewInstr = MF -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DL ) ; blk -> insert ( I , NewInstr ) ; MachineInstrBuilder ( * MF , NewInstr ) . addReg ( RegNum , false ) ;


<s> void <ISA_LIT>CFGStructurizer :: insertCondBranchEnd ( MachineBasicBlock * MBB , int NewOpcode , int RegNum ) { MachineFunction * MF = MBB -> getParent ( ) ; MachineInstr * NewInstr = MF -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DebugLoc ( ) ) ;


<s> void <ISA_LIT>CFGStructurizer :: insertCondBranchEnd ( MachineBasicBlock * MBB , int NewOpcode , int RegNum ) { MachineFunction * MF = MBB -> getParent ( ) ; MachineInstr * NewInstr = MF -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DebugLoc ( ) ) ; MBB -> push_back ( NewInstr ) ;

MachineInstrBuilder ( * MF , NewInstr ) . addReg ( RegNum , false ) ;


<s> MachineInstr * <ISA_LIT>CFGStructurizer :: insertInstrBefore ( MachineBasicBlock :: iterator I , int NewOpcode ) {


<s> MachineInstr * <ISA_LIT>CFGStructurizer :: insertInstrBefore ( MachineBasicBlock :: iterator I , int NewOpcode ) { MachineInstr * OldMI = & ( * I ) ; MachineBasicBlock * MBB = OldMI -> getParent ( ) ; MachineInstr * NewMBB = MBB -> getParent ( ) -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DebugLoc ( ) ) ; MBB -> insert ( I , NewMBB ) ; SHOWNEWINSTR ( NewMBB ) ;


<s> void <ISA_LIT>CFGStructurizer :: insertInstrEnd ( MachineBasicBlock * MBB , int NewOpcode , DebugLoc DL ) { MachineInstr * MI = MBB -> getParent ( ) -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DL ) ; MBB -> push_back ( MI ) ; SHOWNEWINSTR ( MI ) ;


<s> MachineInstr * MI = MBB -> getParent ( ) -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DL ) ; MBB -> push_back ( MI ) ; SHOWNEWINSTR ( MI ) ;


<s> if ( ! isRetiredBlock ( LoopLand ) ) return true ; LoopRep = LoopRep -> getParentLoop ( ) ; } return false ;


<s> MachineLoop * LoopRep = MLI -> getLoopFor ( MBB ) ; while ( LoopRep && LoopRep -> getHeader ( ) == MBB ) { MachineBasicBlock * LoopLand = getLoopLandInfo ( LoopRep ) ; if ( ! LoopLand ) return true ; if ( ! isRetiredBlock ( LoopLand ) ) return true ;

LoopRep = LoopRep -> getParentLoop ( ) ;


<s> bool <ISA_LIT>CFGStructurizer :: isRetiredBlock ( MachineBasicBlock * MBB ) const { MBBInfoMap :: const_iterator It = BlockInfoMap . find ( MBB ) ;

if ( It == BlockInfoMap . end ( ) ) return false ;


<s> bool <ISA_LIT>CFGStructurizer :: isRetiredBlock ( MachineBasicBlock * MBB ) const { MBBInfoMap :: const_iterator It = BlockInfoMap . find ( MBB ) ; if ( It == BlockInfoMap . end ( ) ) return false ; return ( * It ) . second -> IsRetired ;


<s> else if ( IsReturn ) DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ; return IsReturn ;


<s> MachineInstr * MI = getReturnInstr ( MBB ) ; bool IsReturn = ( MBB -> succ_size ( ) == <NUM_LIT> ) ; if ( MI ) assert ( IsReturn ) ; else if ( IsReturn ) DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ; return IsReturn ;


<s> if ( Src1MBB -> succ_size ( ) == <NUM_LIT> ) { MachineLoop * LoopRep = MLI -> getLoopFor ( Src1MBB ) ; if ( LoopRep && LoopRep == MLI -> getLoopFor ( Src2MBB ) ) { MachineBasicBlock * & TheEntry = LLInfoMap [ LoopRep ] ; if ( TheEntry ) { DEBUG ( dbgs ( ) << <STR_LIT> << Src1MBB -> getNumber ( ) << <STR_LIT> << Src2MBB -> getNumber ( ) << <STR_LIT> ; ) ; return true ;


<s> switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : return true ;


<s> for ( SmallVectorImpl < MachineBasicBlock * > :: iterator It = ContMBB . begin ( ) , E = ContMBB . end ( ) ; It != E ; ++ It ) { ( * It ) -> removeSuccessor ( LoopHeader ) ; } numLoopcontPatternMatch += NumCont ;


<s> if ( NestedLoops . size ( ) == <NUM_LIT> ) return <NUM_LIT> ; int Num = <NUM_LIT> ; for ( MachineLoop * ExaminedLoop : NestedLoops ) { if ( ExaminedLoop -> getNumBlocks ( ) == <NUM_LIT> || Visited [ ExaminedLoop ] ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> ; ExaminedLoop -> dump ( ) ; ) ; int NumBreak = mergeLoop ( ExaminedLoop ) ; if ( NumBreak == - <NUM_LIT> ) break ; Num += NumBreak ; } return Num ;


<s> assert ( ! ExitingMBBs . empty ( ) && <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << ExitingMBBs . size ( ) << <STR_LIT> ; ) ; MBBVector ExitBlks ; LoopRep -> getExitBlocks ( ExitBlks ) ; SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > ExitBlkSet ; for ( unsigned i = <NUM_LIT> , e = ExitBlks . size ( ) ; i < e ; ++ i ) ExitBlkSet . insert ( ExitBlks [ i ] ) ; assert ( ExitBlkSet . size ( ) == <NUM_LIT> ) ; MachineBasicBlock * ExitBlk = * ExitBlks . begin ( ) ;


<s> insertInstrBefore ( DstBlk , <STR_LIT> , DebugLoc ( ) ) ; insertInstrEnd ( DstBlk , <STR_LIT> , DebugLoc ( ) ) ; DstBlk -> addSuccessor ( LandMBB ) ; DstBlk -> removeSuccessor ( DstBlk ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> getNumber ( ) << <STR_LIT> << SrcMBB -> getNumber ( ) << <STR_LIT> ; ) ;

DstMBB -> splice ( DstMBB -> end ( ) , SrcMBB , SrcMBB -> begin ( ) , SrcMBB -> end ( ) ) ;


<s> ) ; SpliceEnd = SrcMBB -> end ( ) ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI ) ; SpliceEnd = BranchMI ; }

DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> size ( ) << <STR_LIT> << SrcMBB -> size ( ) << <STR_LIT> ;


<s> bool <ISA_LIT>CFGStructurizer :: needMigrateBlock ( MachineBasicBlock * MBB ) const { unsigned BlockSizeThreshold = <NUM_LIT> ; unsigned CloneInstrThreshold = <NUM_LIT> ; bool MultiplePreds = MBB && ( MBB -> pred_size ( ) > <NUM_LIT> ) ;


<s> bool <ISA_LIT>CFGStructurizer :: needMigrateBlock ( MachineBasicBlock * MBB ) const { unsigned BlockSizeThreshold = <NUM_LIT> ; unsigned CloneInstrThreshold = <NUM_LIT> ; bool MultiplePreds = MBB && ( MBB -> pred_size ( ) > <NUM_LIT> ) ; if ( ! MultiplePreds ) return false ;


<s> MachineBasicBlock * <ISA_LIT>CFGStructurizer :: normalizeInfiniteLoopExit ( MachineLoop * LoopRep ) { MachineBasicBlock * LoopHeader = LoopRep -> getHeader ( ) ; MachineBasicBlock * LoopLatch = LoopRep -> getLoopLatch ( ) ; if ( ! LoopHeader || ! LoopLatch ) return nullptr ; MachineInstr * BranchMI = getLoopendBlockBranchInstr ( LoopLatch ) ; if ( ! BranchMI || ! isUncondBranch ( BranchMI ) ) return nullptr ; MachineBasicBlock * DummyExitBlk = FuncRep -> CreateMachineBasicBlock ( ) ; FuncRep -> push_back ( DummyExitBlk ) ; SHOWNEWBLK ( DummyExitBlk , <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI << <STR_LIT> ; ) ; LLVMContext & Ctx = LoopHeader -> getParent ( ) -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> ) ;


<s> MachineBasicBlock * <ISA_LIT>CFGStructurizer :: normalizeInfiniteLoopExit ( MachineLoop * LoopRep ) { MachineBasicBlock * LoopHeader = LoopRep -> getHeader ( ) ; MachineBasicBlock * LoopLatch = LoopRep -> getLoopLatch ( ) ; if ( ! LoopHeader || ! LoopLatch ) return nullptr ; MachineInstr * BranchMI = getLoopendBlockBranchInstr ( LoopLatch ) ;

if ( ! BranchMI || ! isUncondBranch ( BranchMI ) ) return nullptr ;


<s> recordSccnum ( MBB , SccNum ) ; } } typedef GraphTraits < MachineFunction * > GTM ; MachineFunction :: iterator It = GTM :: nodes_begin ( MF ) , E = GTM :: nodes_end ( MF ) ; for ( ; It != E ; ++ It ) { MachineBasicBlock * MBB = & ( * It ) ; SccNum = getSCCNum ( MBB ) ; if ( SccNum == INVALIDSCCNUM ) dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << NumMatch << <STR_LIT> ; ) ; return NumMatch ;


<s> NumMatch += loopendPatternMatch ( ) ; NumMatch += serialPatternMatch ( MBB ) ;


<s> bool Changed = false ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; orderBlocks ( FuncRep ) ; SmallVector < MachineBasicBlock * , DEFAULT_VEC_SLOTS > RetBlks ; for ( MachineLoopInfo :: iterator It = MLI -> begin ( ) , E = MLI -> end ( ) ; It != E ; ++ It ) { MachineLoop * LoopRep = ( * It ) ; MBBVector ExitingMBBs ; LoopRep -> getExitingBlocks ( ExitingMBBs ) ; if ( ExitingMBBs . size ( ) == <NUM_LIT> ) { MachineBasicBlock * DummyExitBlk = normalizeInfiniteLoopExit ( LoopRep ) ; if ( DummyExitBlk ) RetBlks . push_back ( DummyExitBlk ) ; } } for ( SmallVectorImpl < MachineBasicBlock * > :: const_iterator It = OrderedBlks . begin ( ) , E = OrderedBlks . end ( ) ; It != E ; ++ It ) { MachineBasicBlock * MBB = * It ; removeUnconditionalBranch ( MBB ) ; removeRedundantConditionalBranch ( MBB ) ; if ( isReturnBlock ( MBB ) ) { RetBlks . push_back ( MBB ) ; } assert ( MBB -> succ_size ( ) <= <NUM_LIT> ) ; } if ( RetBlks . size ( ) >= <NUM_LIT> ) { addDummyExitBlock ( RetBlks ) ;


<s> static void PrintLoopinfo ( const MachineLoopInfo & LoopInfo ) {

for ( MachineLoop :: iterator iter = LoopInfo . begin ( ) , iterEnd = LoopInfo . end ( ) ; iter != iterEnd ; ++ iter ) {


<s> void printOrderedBlocks ( ) const { size_t i = <NUM_LIT> ; for ( MBBVector :: const_iterator iterBlk = OrderedBlks . begin ( ) , iterBlkEnd = OrderedBlks . end ( ) ; iterBlk != iterBlkEnd ; ++ iterBlk , ++ i ) { dbgs ( ) << <STR_LIT> << ( * iterBlk ) -> getNumber ( ) ; dbgs ( ) << <STR_LIT> << getSCCNum ( * iterBlk ) << <STR_LIT> << ( * iterBlk ) -> size ( ) << <STR_LIT> ; if ( i != <NUM_LIT> && i % <NUM_LIT> == <NUM_LIT> ) { dbgs ( ) << <STR_LIT> ;


<s> for ( MBBVector :: const_iterator iterBlk = OrderedBlks . begin ( ) , iterBlkEnd = OrderedBlks . end ( ) ; iterBlk != iterBlkEnd ; ++ iterBlk , ++ i ) { dbgs ( ) << <STR_LIT> << ( * iterBlk ) -> getNumber ( ) ; dbgs ( ) << <STR_LIT> << getSCCNum ( * iterBlk ) << <STR_LIT> << ( * iterBlk ) -> size ( ) << <STR_LIT> ; if ( i != <NUM_LIT> && i % <NUM_LIT> == <NUM_LIT> ) { dbgs ( ) << <STR_LIT> ; } else {


<s> BlockInformation * & srcBlkInfo = BlockInfoMap [ MBB ] ; if ( ! srcBlkInfo ) srcBlkInfo = new BlockInformation ( ) ;


<s> BlockInformation * & srcBlkInfo = BlockInfoMap [ MBB ] ;

if ( ! srcBlkInfo ) srcBlkInfo = new BlockInformation ( ) ;


<s> MachineBasicBlock * MBB2 = * std :: next ( MBB -> succ_begin ( ) ) ; if ( MBB1 != MBB2 ) return ; MachineInstr * BranchMI = getNormalBlockBranchInstr ( MBB ) ; assert ( BranchMI && isCondBranch ( BranchMI ) ) ;

DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI ) ;


<s> MachineInstr * BranchMI = getNormalBlockBranchInstr ( MBB ) ; assert ( BranchMI && isCondBranch ( BranchMI ) ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI ) ; BranchMI -> eraseFromParent ( ) ; SHOWNEWBLK ( MBB1 , <STR_LIT> ) ; MBB -> removeSuccessor ( MBB1 ) ;


<s> void <ISA_LIT>CFGStructurizer :: removeSuccessor ( MachineBasicBlock * MBB ) { while ( MBB -> succ_size ( ) ) MBB -> removeSuccessor ( * MBB -> succ_begin ( ) ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI ) ;


<s> void <ISA_LIT>CFGStructurizer :: replaceInstrUseOfBlockWith ( MachineBasicBlock * SrcMBB , MachineBasicBlock * OldMBB , MachineBasicBlock * NewBlk ) { MachineInstr * BranchMI = getLoopendBlockBranchInstr ( SrcMBB ) ;

if ( BranchMI && isCondBranch ( BranchMI ) && getTrueBranch ( BranchMI ) == OldMBB ) setTrueBranch ( BranchMI , NewBlk ) ;


<s> if ( ! SrcBlkInfo ) SrcBlkInfo = new BlockInformation ( ) ; SrcBlkInfo -> IsRetired = true ; assert ( MBB -> succ_size ( ) == <NUM_LIT> && MBB -> pred_size ( ) == <NUM_LIT> && <STR_LIT> ) ;


<s> DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ; BlockInformation * & SrcBlkInfo = BlockInfoMap [ MBB ] ; if ( ! SrcBlkInfo ) SrcBlkInfo = new BlockInformation ( ) ;


<s> while ( I -- ) { if ( I -> getOpcode ( ) == <STR_LIT> ) { switch ( static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . getImm ( ) ) { case OPCODE_IS_ZERO_INT : static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . setImm ( OPCODE_IS_NOT_ZERO_INT ) ; return ; case OPCODE_IS_NOT_ZERO_INT : static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . setImm ( OPCODE_IS_ZERO_INT ) ; return ;


<s> size_t sz = Src . size ( ) ; for ( size_t i = <NUM_LIT> ; i < sz / <NUM_LIT> ; ++ i ) { NodeT * t = Src [ i ] ; Src [ i ] = Src [ sz - i - <NUM_LIT> ] ; Src [ sz - i - <NUM_LIT> ] = t ; }


<s> void ReverseVector ( SmallVectorImpl < NodeT * > & Src ) { size_t sz = Src . size ( ) ; for ( size_t i = <NUM_LIT> ; i < sz / <NUM_LIT> ; ++ i ) { NodeT * t = Src [ i ] ; Src [ i ] = Src [ sz - i - <NUM_LIT> ] ;


<s> if ( ! SccBeginMBB ) { SccBeginIter = It ; SccBeginMBB = MBB ; SccNumIter = <NUM_LIT> ; SccNumBlk = NumRemainedBlk ; DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( SccBeginMBB ) ; dbgs ( ) << <STR_LIT> ; ) ; } if ( ! isRetiredBlock ( MBB ) ) patternMatch ( MBB ) ; ++ It ; bool ContNextScc = true ; if ( It == E || getSCCNum ( SccBeginMBB ) != getSCCNum ( * It ) ) { ++ SccNumIter ; int sccRemainedNumBlk = countActiveBlock ( SccBeginIter , It ) ; if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk >= SccNumBlk ) { DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter ; dbgs ( ) << <STR_LIT> ; ) ; ContNextScc = true ; } else if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk < SccNumBlk ) { SccNumBlk = sccRemainedNumBlk ; It = SccBeginIter ; ContNextScc = false ; DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter << '\n' ; ) ; } else { ContNextScc = true ; } } else { ContNextScc = false ; } if ( ContNextScc ) SccBeginMBB = nullptr ; } MachineBasicBlock * EntryMBB = GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ; if ( EntryMBB -> succ_size ( ) == <NUM_LIT> ) { Finish = true ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } else { int NewnumRemainedBlk = countActiveBlock ( OrderedBlks . begin ( ) , OrderedBlks . end ( ) ) ; if ( NewnumRemainedBlk == <NUM_LIT> || NewnumRemainedBlk < NumRemainedBlk ) { MakeProgress = true ; NumRemainedBlk = NewnumRemainedBlk ; } else { MakeProgress = false ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } } } while ( ! Finish && MakeProgress ) ; wrapup ( GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ) ; for ( MBBInfoMap :: iterator It = BlockInfoMap . begin ( ) , E = BlockInfoMap . end ( ) ; It != E ; ++ It ) { if ( ( * It ) . second && ( * It ) . second -> IsRetired ) { assert ( ( ( * It ) . first ) -> getNumber ( ) != - <NUM_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << ( ( * It ) . first ) -> getNumber ( ) << <STR_LIT> ; ) ; ( * It ) . first -> eraseFromParent ( ) ; } delete ( * It ) . second ; } BlockInfoMap . clear ( ) ; LLInfoMap . clear ( ) ; if ( ! Finish ) { DEBUG ( FuncRep -> viewCFG ( ) ) ; llvm_unreachable ( <STR_LIT> ) ;


<s> TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ; TRI = & TII -> getRegisterInfo ( ) ; DEBUG ( MF . dump ( ) ; ) ; OrderedBlks . clear ( ) ; Visited . clear ( ) ;


<s> int <ISA_LIT>CFGStructurizer :: serialPatternMatch ( MachineBasicBlock * MBB ) { if ( MBB -> succ_size ( ) != <NUM_LIT> ) return <NUM_LIT> ; MachineBasicBlock * childBlk = * MBB -> succ_begin ( ) ;

if ( childBlk -> pred_size ( ) != <NUM_LIT> || isActiveLoophead ( childBlk ) ) return <NUM_LIT> ;


<s> TheEntry = MBB ; DEBUG ( dbgs ( ) << <STR_LIT> << loopRep -> getHeader ( ) -> getNumber ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ;


<s> int OldOpcode = MI -> getOpcode ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; bool UseContinueLogical = ( ( & * ContingMBB -> rbegin ( ) ) == MI ) ; if ( ! UseContinueLogical ) { int BranchOpcode = TrueBranch == ContMBB ? getBranchNzeroOpcode ( OldOpcode ) : getBranchZeroOpcode ( OldOpcode ) ; insertCondBranchBefore ( I , BranchOpcode , DL ) ; insertInstrEnd ( ContingMBB , <STR_LIT> , DL ) ; insertInstrEnd ( ContingMBB , <STR_LIT> , DL ) ; } else { int BranchOpcode = TrueBranch == ContMBB ? getContinueNzeroOpcode ( OldOpcode ) : getContinueZeroOpcode ( OldOpcode ) ;


<s> MI -> getOperand ( <NUM_LIT> ) . setMBB ( MBB ) ;


<s> dbgs ( ) << <STR_LIT> ; } } if ( FalseMBB ) { dbgs ( ) << <STR_LIT> << FalseMBB -> getNumber ( ) << <STR_LIT> << FalseMBB -> size ( ) << <STR_LIT> << FalseMBB -> pred_size ( ) ; if ( Detail ) { dbgs ( ) << <STR_LIT> ; FalseMBB -> print ( dbgs ( ) ) ; dbgs ( ) << <STR_LIT> ; } } if ( LandMBB ) { dbgs ( ) << <STR_LIT> << LandMBB -> getNumber ( ) << <STR_LIT> << LandMBB -> size ( ) << <STR_LIT> << LandMBB -> pred_size ( ) ; if ( Detail ) { dbgs ( ) << <STR_LIT> ; LandMBB -> print ( dbgs ( ) ) ; dbgs ( ) << <STR_LIT> ; }


<s> SrcMBB = * SrcMBB -> succ_begin ( ) ; if ( SrcMBB == DstMBB ) return SinglePath_InPath ;

if ( ! AllowSideEntry && SrcMBB -> pred_size ( ) > <NUM_LIT> ) return Not_SinglePath ;


<s> <ISA_LIT>CFGStructurizer :: PathToKind <ISA_LIT>CFGStructurizer :: singlePathTo ( MachineBasicBlock * SrcMBB , MachineBasicBlock * DstMBB , bool AllowSideEntry ) const { assert ( DstMBB ) ; if ( SrcMBB == DstMBB ) return SinglePath_InPath ; while ( SrcMBB && SrcMBB -> succ_size ( ) == <NUM_LIT> ) { SrcMBB = * SrcMBB -> succ_begin ( ) ; if ( SrcMBB == DstMBB ) return SinglePath_InPath ; if ( ! AllowSideEntry && SrcMBB -> pred_size ( ) > <NUM_LIT> ) return Not_SinglePath ; } if ( SrcMBB && SrcMBB -> succ_size ( ) == <NUM_LIT> ) return SinglePath_NotInPath ; return Not_SinglePath ;


<s> if ( Pre -> getOpcode ( ) == <STR_LIT> && It -> getOpcode ( ) == <STR_LIT> ) ContInstr . push_back ( Pre ) ; Pre = It ; ++ It ; }

for ( unsigned i = <NUM_LIT> ; i < ContInstr . size ( ) ; ++ i ) ContInstr [ i ] -> eraseFromParent ( ) ;


for ( auto Name : a ) map . insert ( std :: make_pair ( Name , map . size ( ) ) ) ;


<s> const int Size = getPrinterTable ( ) . size ( ) ;

for ( int i = <NUM_LIT> ; i < Size ; ++ i ) {


<s> void llvm :: dumpAmdKernelCode ( const amd_kernel_code_t * C , raw_ostream & OS , const char * tab ) { const int Size = getPrinterTable ( ) . size ( ) ; for ( int i = <NUM_LIT> ; i < Size ; ++ i ) { OS << tab ; printAmdKernelCodeField ( * C , i , OS ) ;


<s> Lexer . Lex ( ) ; if ( Lexer . isNot ( AsmToken :: Integer ) ) { Err << <STR_LIT> ;


<s> Lexer . Lex ( ) ; if ( Lexer . isNot ( AsmToken :: Integer ) ) { Err << <STR_LIT> ; return false ; } return true ;


<s> if ( Idx < <NUM_LIT> ) { Err << <STR_LIT> << ID ; return false ; } auto Parser = getParserTable ( ) [ Idx ] ;

return Parser ? Parser ( C , Lexer , Err ) : false ;


<s> } auto Parser = getParserTable ( ) [ Idx ] ;


C .* ptr |= ( T ) ( ( Lexer . getTok ( ) . getIntVal ( ) << shift ) & Mask ) ;


<s> if ( ! expectEqualInt ( Lexer , Err ) ) return false ;

C .* ptr = ( T ) Lexer . getTok ( ) . getIntVal ( ) ;


<s> void llvm :: printAmdKernelCodeField ( const amd_kernel_code_t & C , int FldIndex , raw_ostream & OS ) { auto Printer = getPrinterTable ( ) [ FldIndex ] ;

if ( Printer ) Printer ( get_amd_kernel_code_t_FieldName ( FldIndex ) , C , OS ) ;


<s> static void printBitField ( StringRef Name , const amd_kernel_code_t & c , raw_ostream & OS ) { const auto Mask = ( static_cast < T > ( <NUM_LIT> ) << width ) - <NUM_LIT> ; printName ( OS , Name ) << ( int ) ( ( c .* ptr >> shift ) & Mask ) ;


<s> static void printBitField ( StringRef Name , const amd_kernel_code_t & c , raw_ostream & OS ) {

const auto Mask = ( static_cast < T > ( <NUM_LIT> ) << width ) - <NUM_LIT> ;


<s> static void printField ( StringRef Name , const amd_kernel_code_t & C , raw_ostream & OS ) {

printName ( OS , Name ) << ( int ) ( C .* ptr ) ;


<s> static void printField ( StringRef Name , const amd_kernel_code_t & C , raw_ostream & OS ) { printName ( OS , Name ) << ( int ) ( C .* ptr ) ;


<s> map . insert ( std :: make_pair ( names [ i ] , i ) ) ; map . insert ( std :: make_pair ( altNames [ i ] , i ) ) ; } return map ;


<s> for ( unsigned i = <NUM_LIT> ; i < names . size ( ) ; ++ i ) { map . insert ( std :: make_pair ( names [ i ] , i ) ) ; map . insert ( std :: make_pair ( altNames [ i ] , i ) ) ; }


<s> static bool expectAbsExpression ( MCAsmParser & MCParser , int64_t & Value , raw_ostream & Err ) { if ( MCParser . getLexer ( ) . isNot ( AsmToken :: Equal ) ) { Err << <STR_LIT> ; return false ; }


<s> Err << <STR_LIT> ; return false ; } MCParser . getLexer ( ) . Lex ( ) ; if ( MCParser . parseAbsoluteExpression ( Value ) ) { Err << <STR_LIT> ; return false ; } return true ;


<s> return map . lookup ( name ) - <NUM_LIT> ;


<s> static int get_amd_kernel_code_t_FieldIndex ( StringRef name ) {

static const auto map = createIndexMap ( get_amd_kernel_code_t_FldNames ( ) , get_amd_kernel_code_t_FldAltNames ( ) ) ;


<s> static StringRef const Table [ ] = { <STR_LIT> , } ; return makeArrayRef ( Table ) ;


return Parser ? Parser ( C , MCParser , Err ) : false ;


<s> return false ; } auto Parser = getParserTable ( ) [ Idx ] ;


<s> C .* ptr &= ( T ) ~ Mask ; C .* ptr |= ( T ) ( ( Value << shift ) & Mask ) ; return true ;


<s> C .* ptr &= ( T ) ~ Mask ; C .* ptr |= ( T ) ( ( Value << shift ) & Mask ) ;


if ( ! expectAbsExpression ( MCParser , Value , Err ) ) return false ;


<s> int64_t Value = <NUM_LIT> ; if ( ! expectAbsExpression ( MCParser , Value , Err ) ) return false ; C .* ptr = ( T ) Value ; return true ;


<s> void llvm :: dumpAmdKernelCode ( const amd_kernel_code_t * C , raw_ostream & OS , const char * tab ) { const int Size = getPrinterTable ( ) . size ( ) ; for ( int i = <NUM_LIT> ; i < Size ; ++ i ) {


<s> const int Size = getPrinterTable ( ) . size ( ) ; for ( int i = <NUM_LIT> ; i < Size ; ++ i ) { OS << tab ; printAmdKernelCodeField ( * C , i , OS ) ; OS << '\n' ;


<s> if ( Lexer . isNot ( AsmToken :: Integer ) ) { Err << <STR_LIT> ; return false ; } return true ;


<s> return false ; } Lexer . Lex ( ) ; if ( Lexer . isNot ( AsmToken :: Integer ) ) { Err << <STR_LIT> ; return false ; }


<s> static ArrayRef < ParseFx > getParserTable ( ) {


<s> static StringRef get_amd_kernel_code_t_FieldName ( int index ) {

return get_amd_kernel_code_t_FldNames ( ) [ index + <NUM_LIT> ] ;


<s> static StringRef const Table [ ] = { <STR_LIT> , } ;


<s> bool llvm :: parseAmdKernelCodeField ( StringRef ID , MCAsmLexer & Lexer , amd_kernel_code_t & C , raw_ostream & Err ) { const int Idx = get_amd_kernel_code_t_FieldIndex ( ID ) ; if ( Idx < <NUM_LIT> ) { Err << <STR_LIT> << ID ; return false ;


<s> bool parseBitField ( amd_kernel_code_t & C , MCAsmLexer & Lexer , raw_ostream & Err ) { if ( ! expectEqualInt ( Lexer , Err ) ) return false ; const uint64_t Mask = ( ( UINT64_C ( <NUM_LIT> ) << width ) - <NUM_LIT> ) << shift ; C .* ptr &= ( T ) ~ Mask ; C .* ptr |= ( T ) ( ( Lexer . getTok ( ) . getIntVal ( ) << shift ) & Mask ) ;


<s> const uint64_t Mask = ( ( UINT64_C ( <NUM_LIT> ) << width ) - <NUM_LIT> ) << shift ; C .* ptr &= ( T ) ~ Mask ; C .* ptr |= ( T ) ( ( Lexer . getTok ( ) . getIntVal ( ) << shift ) & Mask ) ;


<s> C .* ptr = ( T ) Lexer . getTok ( ) . getIntVal ( ) ;


<s> bool parseField ( amd_kernel_code_t & C , MCAsmLexer & Lexer , raw_ostream & Err ) { if ( ! expectEqualInt ( Lexer , Err ) ) return false ;


<s> void llvm :: printAmdKernelCodeField ( const amd_kernel_code_t & C , int FldIndex , raw_ostream & OS ) { auto Printer = getPrinterTable ( ) [ FldIndex ] ;


<s> if ( Printer ) Printer ( get_amd_kernel_code_t_FieldName ( FldIndex ) , C , OS ) ;


<s> Expected < const D * > as ( ) const { if ( descsz < sizeof ( D ) ) { return make_error < StringError > ( <STR_LIT> , object :: object_error :: parse_failed ) ; } return reinterpret_cast < const D * > ( getDesc ( ) . data ( ) ) ;


<s> if ( ( * Sym ) -> getType ( ) != ELF :: STT_FUNC && ( * Sym ) -> getType ( ) != ELF :: STT_<ISA_LIT>_HSA_KERNEL ) return createError ( <STR_LIT> ) ;

return static_cast < const FunctionSym * > ( * Sym ) ;


<s> if ( ! Sym ) return Sym . takeError ( ) ; if ( ( * Sym ) -> getType ( ) != ELF :: STT_FUNC && ( * Sym ) -> getType ( ) != ELF :: STT_<ISA_LIT>_HSA_KERNEL ) return createError ( <STR_LIT> ) ;


<s> Expected < const KernelSym * > KernelSym :: asKernelSym ( const FunctionSym * Sym ) { if ( Sym -> getType ( ) != ELF :: STT_<ISA_LIT>_HSA_KERNEL ) return createError ( <STR_LIT> ) ; return static_cast < const KernelSym * > ( Sym ) ;


<s> conditional_iterator ( BaseIterator BI , BaseIterator E , PredicateTy P ) : conditional_iterator :: iterator_adaptor_base ( BI ) , End ( E ) , Predicate ( P ) { while ( this -> I != End && ! Predicate ( * this -> I ) ) { ++ this -> I ; }


<s> const_varsize_item_iterator ( ArrayRef < uint8_t > Ref_ ) : Ref ( Ref_ ) {


<s> if ( auto E = Obj . takeError ( ) ) return std :: move ( E ) ;

std :: unique_ptr < HSACodeObject > Ret ( new HSACodeObject ( std :: move ( * Obj ) ) ) ;


<s> if ( ! TextIdxOr ) return functions_end ( ) ; auto TextIdx = TextIdxOr . get ( ) ; return function_sym_iterator ( symbol_begin ( ) , symbol_end ( ) , [ this , TextIdx ] ( const SymbolRef & Sym ) -> bool { auto ExpectedFunction = FunctionSym :: asFunctionSym ( getSymbol ( Sym . getRawDataRefImpl ( ) ) ) ; if ( ! ExpectedFunction ) { consumeError ( ExpectedFunction . takeError ( ) ) ; return false ; } auto Function = ExpectedFunction . get ( ) ;

if ( Function -> st_shndx != TextIdx ) return false ;


<s> function_sym_iterator HSACodeObject :: functions_end ( ) const { return function_sym_iterator ( symbol_end ( ) , symbol_end ( ) , [ ] ( const SymbolRef & ) { return true ; } ) ;


<s> function_sym_iterator ( object :: elf_symbol_iterator It , object :: elf_symbol_iterator End , PredicateTy P ) : conditional_iterator < object :: elf_symbol_iterator > ( It , End , P ) {


<s> Expected < uint64_t > FunctionSym :: getAddress ( const HSACodeObject * CodeObject , const object :: ELF64LEObjectFile :: Elf_Shdr * Text ) const {


<s> if ( ! ArrayOr ) return ArrayOr . takeError ( ) ; auto SectionOffsetOr = getSectionOffset ( CodeObject , Text ) ; if ( ! SectionOffsetOr ) return SectionOffsetOr . takeError ( ) ; return reinterpret_cast < const amd_kernel_code_t * > ( ( * ArrayOr ) . data ( ) + * SectionOffsetOr ) ;


<s> return StringRef ( names + vendor_name_size , architecture_name_size - <NUM_LIT> ) ;


<s> StringRef getArchitectureName ( ) const {

return StringRef ( names + vendor_name_size , architecture_name_size - <NUM_LIT> ) ;


<s> auto KernelCodeStartOr = Kernel -> getCodeOffset ( this , * TextOr ) ; if ( ! KernelCodeStartOr ) return KernelCodeStartOr . takeError ( ) ; CodeStart = KernelCodeStartOr . get ( ) ; } else { consumeError ( ExpectedKernel . takeError ( ) ) ; } auto CodeEndI = std :: upper_bound ( FunctionMarkers . begin ( ) , FunctionMarkers . end ( ) , CodeStart ) ; uint64_t CodeEnd = CodeStart ;

if ( CodeEndI != FunctionMarkers . end ( ) ) CodeEnd = * CodeEndI ;


<s> if ( ! SectionOffsetOr ) return SectionOffsetOr . takeError ( ) ; auto KernelCodeTOr = getAmdKernelCodeT ( CodeObject , Text ) ; if ( ! KernelCodeTOr ) return KernelCodeTOr . takeError ( ) ;

return * SectionOffsetOr + ( * KernelCodeTOr ) -> kernel_code_entry_byte_offset ;


<s> StringRef getDesc ( ) const { return StringRef ( getName ( ) . data ( ) + alignTo ( namesz , ALIGN ) , descsz ) ;


assert ( Ref . size ( ) >= sizeof ( Item ) ) ;


<s> StringRef getName ( ) const { return StringRef ( reinterpret_cast < const char * > ( this ) + sizeof ( * this ) , namesz ) ;


<s> Expected < const HSACodeObject :: Elf_Shdr * > HSACodeObject :: getNoteSection ( ) const {

if ( auto IdxOr = getNoteSectionIdx ( ) ) return getELFFile ( ) . getSection ( * IdxOr ) ;


<s> if ( auto IdxOr = getNoteSectionIdx ( ) ) return getELFFile ( ) . getSection ( * IdxOr ) ; return createError ( <STR_LIT> ) ;


<s> Expected < uint32_t > HSACodeObject :: getNoteSectionIdx ( ) const {

return getSectionIdxByName ( <STR_LIT> :: SectionName ) ;


<s> for ( const auto & Sec : * SectionsOr ) { auto SecNameOr = ELF . getSectionName ( Sec ) ; if ( ! SecNameOr ) { return SecNameOr . takeError ( ) ; } else if ( * SecNameOr == Name ) { return Expected < const Elf_Shdr * > ( & Sec ) ; } }


<s> auto SectionsOr = ELF . sections ( ) ; if ( ! SectionsOr ) return SectionsOr . takeError ( ) ; for ( const auto & Sec : * SectionsOr ) { auto SecNameOr = ELF . getSectionName ( Sec ) ; if ( ! SecNameOr ) { return SecNameOr . takeError ( ) ; } else if ( * SecNameOr == Name ) { return Idx ; } ++ Idx ;


<s> if ( ! AddressOr ) return AddressOr . takeError ( ) ;


<s> Expected < uint64_t > FunctionSym :: getSectionOffset ( const HSACodeObject * CodeObject , const object :: ELF64LEObjectFile :: Elf_Shdr * Text ) const { assert ( Text ) ; auto AddressOr = getAddress ( CodeObject , Text ) ; if ( ! AddressOr ) return AddressOr . takeError ( ) ;


<s> size_t getSize ( ) const { return sizeof ( * this ) + alignTo ( namesz , ALIGN ) + alignTo ( descsz , ALIGN ) ;


<s> if ( auto IdxOr = getTextSectionIdx ( ) ) return getELFFile ( ) . getSection ( * IdxOr ) ; return createError ( <STR_LIT> ) ;


<s> auto SecOr = getELFFile ( ) . getSection ( * IdxOr ) ; if ( SecOr || isSectionText ( toDRI ( * SecOr ) ) ) return IdxOr ;


<s> HSACodeObject ( object :: ELF64LEObjectFile && Obj ) : object :: ELF64LEObjectFile ( std :: move ( Obj ) ) {


<s> HSACodeObject ( object :: ELF64LEObjectFile && Obj ) : object :: ELF64LEObjectFile ( std :: move ( Obj ) ) { InitMarkers ( ) ;


<s> void HSACodeObject :: InitMarkers ( ) const { auto TextSecOr = getTextSection ( ) ; if ( ! TextSecOr ) return ; auto TextSec = TextSecOr . get ( ) ; FunctionMarkers . push_back ( TextSec -> sh_size ) ; for ( const auto & Sym : functions ( ) ) { auto ExpectedFunction = FunctionSym :: asFunctionSym ( getSymbol ( Sym . getRawDataRefImpl ( ) ) ) ; if ( ! ExpectedFunction ) {

consumeError ( ExpectedFunction . takeError ( ) ) ;


return reinterpret_cast < const Item * > ( Ref . data ( ) ) ;


<s> iterator_range < HSACodeObject :: note_iterator > HSACodeObject :: notes ( ) const { return make_range ( notes_begin ( ) , notes_end ( ) ) ;


<s> if ( auto ContentsOr = getELFFile ( ) . getSectionContentsAsArray < uint8_t > ( * * NotesOr ) ) return const_varsize_item_iterator < ELFNote > ( * ContentsOr ) ; } return const_varsize_item_iterator < ELFNote > ( ) ;


<s> HSACodeObject :: note_iterator HSACodeObject :: notes_end ( ) const { return const_varsize_item_iterator < ELFNote > ( ) ;


<s> return Ref . size ( ) >= sizeof ( Item ) && Ref . size ( ) >= getItemPadSize ( ) ;


<s> } } else if ( ! TII -> isVOP1 ( OrigOp ) && ! TII -> isVOP2 ( OrigOp ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; break ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << OrigMI ) ; if ( & Use == TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src0 ) ) { if ( auto * DPPInst = createDPPInst ( OrigMI , MovMI , OldOpndVGPR , OldOpndValue , BoundCtrlZero ) ) { DPPMIs . push_back ( DPPInst ) ; Rollback = false ; } } else if ( OrigMI . isCommutable ( ) && & Use == TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src1 ) ) { auto * BB = OrigMI . getParent ( ) ; auto * NewMI = BB -> getParent ( ) -> CloneMachineInstr ( & OrigMI ) ; BB -> insert ( OrigMI , NewMI ) ; if ( TII -> commuteInstruction ( * NewMI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * NewMI ) ; if ( auto * DPPInst = createDPPInst ( * NewMI , MovMI , OldOpndVGPR , OldOpndValue , BoundCtrlZero ) ) { DPPMIs . push_back ( DPPInst ) ; Rollback = false ; } } else LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; NewMI -> eraseFromParent ( ) ; } else LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; if ( Rollback ) break ; OrigMIs . push_back ( & OrigMI ) ;


<s> assert ( OldOpndValue -> isImm ( ) ) ; OldOpndVGPR = foldOldOpnd ( OrigMI , OldOpndVGPR , * OldOpndValue ) ; if ( ! OldOpndVGPR . Reg ) {


<s> initializeGCNDPPCombinePass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . setPreservesCFG ( ) ;

MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> AU . setPreservesCFG ( ) ; MachineFunctionPass :: getAnalysisUsage ( AU ) ;


if ( DPP32 != - <NUM_LIT> ) return DPP32 ;


<s> if ( DPP32 != - <NUM_LIT> ) return DPP32 ; auto E32 = <STR_LIT> ( Op ) ;


<s> if ( ! Def ) return nullptr ; switch ( Def -> getOpcode ( ) ) { default : break ; case <STR_LIT> : return nullptr ; case <STR_LIT> : case <STR_LIT> : { auto & Op1 = Def -> getOperand ( <NUM_LIT> ) ;


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> bool GCNDPPCombine :: hasNoImmOrEqual ( MachineInstr & MI , unsigned OpndName , int64_t Value , int64_t Mask ) const { auto * Imm = TII -> getNamedOperand ( MI , OpndName ) ;


<s> assert ( Imm -> isImm ( ) ) ; return ( Imm -> getImm ( ) & Mask ) == Value ;


<s> assert ( MRI -> isSSA ( ) && <STR_LIT> ) ; bool Changed = false ; for ( auto & MBB : MF ) { for ( auto I = MBB . rbegin ( ) , E = MBB . rend ( ) ; I != E ; ) { auto & MI = * I ++ ; if ( MI . getOpcode ( ) == <STR_LIT> && combineDPPMov ( MI ) ) { Changed = true ; ++ NumDPPMovsCombined ; }


<s> } CombOldVGPR = getRegSubRegPair ( * Src1 ) ; auto MovDst = TII -> getNamedOperand ( MovMI , <STR_LIT> :: vdst ) ; const TargetRegisterClass * RC = MRI -> getRegClass ( MovDst -> getReg ( ) ) ; if ( ! isOfRegClass ( CombOldVGPR , * RC , * MRI ) ) {


<s> if ( IsShrinkable ) { assert ( DPP32 == - <NUM_LIT> ) ; auto E32 = <STR_LIT> ( Op ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( OldOpnd -> getImm ( ) == <NUM_LIT> ) return true ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( static_cast < uint32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < uint32_t > :: max ( ) ) return true ; break ; case <STR_LIT> : case <STR_LIT> : if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < int32_t > :: max ( ) ) return true ; break ; case <STR_LIT> : case <STR_LIT> : if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < int32_t > :: min ( ) ) return true ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( OldOpnd -> getImm ( ) == <NUM_LIT> ) return true ; break ;


<s> if ( ! MRI -> use_nodbg_empty ( SDst -> getReg ( ) ) ) return false ; } const int64_t Mask = ~ ( <STR_LIT> | <STR_LIT> ) ; if ( ! hasNoImmOrEqual ( MI , <STR_LIT> :: src0_modifiers , <NUM_LIT> , Mask ) || ! hasNoImmOrEqual ( MI , <STR_LIT> :: src1_modifiers , <NUM_LIT> , Mask ) || ! hasNoImmOrEqual ( MI , <STR_LIT> :: clamp , <NUM_LIT> ) || ! hasNoImmOrEqual ( MI , <STR_LIT> :: omod , <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ;


<s> if ( ! ST -> hasDPP ( ) || skipFunction ( MF . getFunction ( ) ) ) return false ; MRI = & MF . getRegInfo ( ) ; TII = ST -> getInstrInfo ( ) ; bool Changed = false ; for ( auto & MBB : MF ) { for ( MachineInstr & MI : llvm :: make_early_inc_range ( llvm :: reverse ( MBB ) ) ) { if ( MI . getOpcode ( ) == <STR_LIT> && combineDPPMov ( MI ) ) { Changed = true ; ++ NumDPPMovsCombined ; } else if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) { if ( ST -> has64BitDPP ( ) && combineDPPMov ( MI ) ) { Changed = true ; ++ NumDPPMovsCombined ;


<s> if ( ! <STR_LIT> ( DppCtrl -> getImm ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } } auto * RowMaskOpnd = TII -> getNamedOperand ( MovMI , <STR_LIT> :: row_mask ) ; assert ( RowMaskOpnd && RowMaskOpnd -> isImm ( ) ) ; auto * BankMaskOpnd = TII -> getNamedOperand ( MovMI , <STR_LIT> :: bank_mask ) ; assert ( BankMaskOpnd && BankMaskOpnd -> isImm ( ) ) ; const bool MaskAllLanes = RowMaskOpnd -> getImm ( ) == <NUM_LIT> && BankMaskOpnd -> getImm ( ) == <NUM_LIT> ; auto * BCZOpnd = TII -> getNamedOperand ( MovMI , <STR_LIT> :: bound_ctrl ) ; assert ( BCZOpnd && BCZOpnd -> isImm ( ) ) ; bool BoundCtrlZero = BCZOpnd -> getImm ( ) ; auto * OldOpnd = TII -> getNamedOperand ( MovMI , <STR_LIT> :: old ) ; auto * SrcOpnd = TII -> getNamedOperand ( MovMI , <STR_LIT> :: src0 ) ; assert ( OldOpnd && OldOpnd -> isReg ( ) ) ; assert ( SrcOpnd && SrcOpnd -> isReg ( ) ) ; if ( OldOpnd -> getReg ( ) . isPhysical ( ) || SrcOpnd -> getReg ( ) . isPhysical ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * const OldOpndValue = getOldOpndValue ( * OldOpnd ) ; assert ( ! OldOpndValue || OldOpndValue -> isImm ( ) || OldOpndValue == OldOpnd ) ; bool CombBCZ = false ; if ( MaskAllLanes && BoundCtrlZero ) { CombBCZ = true ; } else { if ( ! OldOpndValue || ! OldOpndValue -> isImm ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( OldOpndValue -> getImm ( ) == <NUM_LIT> ) { if ( MaskAllLanes ) { assert ( ! BoundCtrlZero ) ; CombBCZ = true ; } } else if ( BoundCtrlZero ) { assert ( ! MaskAllLanes ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; if ( ! OldOpndValue ) dbgs ( ) << <STR_LIT> ; else dbgs ( ) << * OldOpndValue ; dbgs ( ) << <STR_LIT> << CombBCZ << '\n' ) ; SmallVector < MachineInstr * , <NUM_LIT> > OrigMIs , DPPMIs ; DenseMap < MachineInstr * , SmallVector < unsigned , <NUM_LIT> >> RegSeqWithOpNos ; auto CombOldVGPR = getRegSubRegPair ( * OldOpnd ) ; if ( CombBCZ && OldOpndValue ) { const TargetRegisterClass * RC = MRI -> getRegClass ( DPPMovReg ) ; CombOldVGPR = RegSubRegPair ( MRI -> createVirtualRegister ( RC ) ) ;

auto UndefInst = BuildMI ( * MovMI . getParent ( ) , MovMI , MovMI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , CombOldVGPR . Reg ) ;


<s> OrigMIs . push_back ( & MovMI ) ; bool Rollback = true ; for ( auto & Use : MRI -> use_nodbg_operands ( DPPMovReg ) ) { Rollback = true ; auto & OrigMI = * Use . getParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << OrigMI ) ; auto OrigOp = OrigMI . getOpcode ( ) ; if ( TII -> isVOP3 ( OrigOp ) ) { if ( ! TII -> hasVALU32BitEncoding ( OrigOp ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; break ; } const int64_t Mask = ~ ( <STR_LIT> | <STR_LIT> ) ; if ( ! hasNoImmOrEqual ( OrigMI , <STR_LIT> :: src0_modifiers , <NUM_LIT> , Mask ) || ! hasNoImmOrEqual ( OrigMI , <STR_LIT> :: src1_modifiers , <NUM_LIT> , Mask ) || ! hasNoImmOrEqual ( OrigMI , <STR_LIT> :: clamp , <NUM_LIT> ) || ! hasNoImmOrEqual ( OrigMI , <STR_LIT> :: omod , <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; break ; } } else if ( ! TII -> isVOP1 ( OrigOp ) && ! TII -> isVOP2 ( OrigOp ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; break ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << OrigMI ) ; if ( & Use == TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src0 ) ) { if ( auto * DPPInst = createDPPInst ( OrigMI , MovMI , CombOldVGPR , OldOpndValue , CombBCZ ) ) { DPPMIs . push_back ( DPPInst ) ; Rollback = false ; } } else if ( OrigMI . isCommutable ( ) && & Use == TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src1 ) ) { auto * BB = OrigMI . getParent ( ) ; auto * NewMI = BB -> getParent ( ) -> CloneMachineInstr ( & OrigMI ) ; BB -> insert ( OrigMI , NewMI ) ;

if ( TII -> commuteInstruction ( * NewMI ) ) {


<s> } if ( ! isIdentityValue ( OrigMI . getOpcode ( ) , OldOpndValue ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return nullptr ; } CombOldVGPR = getRegSubRegPair ( * Src1 ) ;

if ( ! isOfRegClass ( CombOldVGPR , <STR_LIT> , * MRI ) ) {


<s> case <STR_LIT> : if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < int32_t > :: max ( ) ) return true ; break ; case <STR_LIT> : if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < int32_t > :: min ( ) ) return true ; break ; case <STR_LIT> : case <STR_LIT> : if ( OldOpnd -> getImm ( ) == <NUM_LIT> ) return true ; break ; }


<s> case <STR_LIT> : if ( static_cast < uint32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < uint32_t > :: max ( ) ) return true ; break ; case <STR_LIT> : if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < int32_t > :: max ( ) ) return true ; break ; case <STR_LIT> : if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < int32_t > :: min ( ) ) return true ; break ; case <STR_LIT> : case <STR_LIT> :

if ( OldOpnd -> getImm ( ) == <NUM_LIT> ) return true ;


<s> return nullptr ; } auto DPPInst = BuildMI ( * OrigMI . getParent ( ) , OrigMI , OrigMI . getDebugLoc ( ) , TII -> get ( DPPOp ) ) . setMIFlags ( OrigMI . getFlags ( ) ) ; bool Fail = false ; do { auto * Dst = TII -> getNamedOperand ( OrigMI , <STR_LIT> :: vdst ) ; assert ( Dst ) ; DPPInst . add ( * Dst ) ; int NumOperands = <NUM_LIT> ; const int OldIdx = <STR_LIT> ( DPPOp , <STR_LIT> :: old ) ; if ( OldIdx != - <NUM_LIT> ) { assert ( OldIdx == NumOperands ) ; assert ( isOfRegClass ( CombOldVGPR , * MRI -> getRegClass ( TII -> getNamedOperand ( MovMI , <STR_LIT> :: vdst ) -> getReg ( ) ) , * MRI ) ) ; auto * Def = getVRegSubRegDef ( CombOldVGPR , * MRI ) ; DPPInst . addReg ( CombOldVGPR . Reg , Def ? <NUM_LIT> : RegState :: Undef , CombOldVGPR . SubReg ) ; ++ NumOperands ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; Fail = true ; break ; } if ( auto * Mod0 = TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src0_modifiers ) ) { assert ( NumOperands == <STR_LIT> ( DPPOp , <STR_LIT> :: src0_modifiers ) ) ; assert ( <NUM_LIT> == ( Mod0 -> getImm ( ) & ~ ( <STR_LIT> | <STR_LIT> ) ) ) ; DPPInst . addImm ( Mod0 -> getImm ( ) ) ; ++ NumOperands ; } else if ( <STR_LIT> ( DPPOp , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ) { DPPInst . addImm ( <NUM_LIT> ) ; ++ NumOperands ; } auto * Src0 = TII -> getNamedOperand ( MovMI , <STR_LIT> :: src0 ) ; assert ( Src0 ) ; if ( ! TII -> isOperandLegal ( * DPPInst . getInstr ( ) , NumOperands , Src0 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Fail = true ; break ; } DPPInst . add ( * Src0 ) ; DPPInst -> getOperand ( NumOperands ) . setIsKill ( false ) ; ++ NumOperands ; if ( auto * Mod1 = TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src1_modifiers ) ) { assert ( NumOperands == <STR_LIT> ( DPPOp , <STR_LIT> :: src1_modifiers ) ) ; assert ( <NUM_LIT> == ( Mod1 -> getImm ( ) & ~ ( <STR_LIT> | <STR_LIT> ) ) ) ; DPPInst . addImm ( Mod1 -> getImm ( ) ) ; ++ NumOperands ;

} else if ( <STR_LIT> ( DPPOp , <STR_LIT> :: src1_modifiers ) != - <NUM_LIT> ) {


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { auto & Op1 = Def -> getOperand ( <NUM_LIT> ) ; if ( Op1 . isImm ( ) ) return & Op1 ;


<s> if ( ST -> has64BitDPP ( ) && combineDPPMov ( MI ) ) { Changed = true ; ++ NumDPPMovsCombined ; } else { auto Split = TII -> expandMovDPP64 ( MI ) ; for ( auto M : { Split . first , Split . second } ) { if ( M && combineDPPMov ( * M ) ) ++ NumDPPMovsCombined ; } Changed = true ; } } }


<s> TII = ST -> getInstrInfo ( ) ; bool Changed = false ; for ( auto & MBB : MF ) { for ( auto I = MBB . rbegin ( ) , E = MBB . rend ( ) ; I != E ; ) { auto & MI = * I ++ ; if ( MI . getOpcode ( ) == <STR_LIT> && combineDPPMov ( MI ) ) { Changed = true ; ++ NumDPPMovsCombined ; } else if ( MI . getOpcode ( ) == <STR_LIT> ) { if ( ST -> has64BitDPP ( ) && combineDPPMov ( MI ) ) { Changed = true ; ++ NumDPPMovsCombined ;


<s> return false ; } if ( OldOpndValue -> getImm ( ) == <NUM_LIT> ) { if ( MaskAllLanes ) { assert ( ! BoundCtrlZero ) ; CombBCZ = true ; } } else if ( BoundCtrlZero ) { assert ( ! MaskAllLanes ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; if ( ! OldOpndValue ) dbgs ( ) << <STR_LIT> ; else dbgs ( ) << * OldOpndValue ; dbgs ( ) << <STR_LIT> << CombBCZ << '\n' ) ; SmallVector < MachineInstr * , <NUM_LIT> > OrigMIs , DPPMIs ; auto CombOldVGPR = getRegSubRegPair ( * OldOpnd ) ; if ( CombBCZ && OldOpndValue ) { CombOldVGPR = RegSubRegPair ( MRI -> createVirtualRegister ( & <STR_LIT> ) ) ; auto UndefInst = BuildMI ( * MovMI . getParent ( ) , MovMI , MovMI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , CombOldVGPR . Reg ) ; DPPMIs . push_back ( UndefInst . getInstr ( ) ) ; } OrigMIs . push_back ( & MovMI ) ; bool Rollback = true ; for ( auto & Use : MRI -> use_nodbg_operands ( DPPMovReg ) ) { Rollback = true ; auto & OrigMI = * Use . getParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << OrigMI ) ; auto OrigOp = OrigMI . getOpcode ( ) ; if ( TII -> isVOP3 ( OrigOp ) ) { if ( ! TII -> hasVALU32BitEncoding ( OrigOp ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; break ; } const int64_t Mask = ~ ( <STR_LIT> | <STR_LIT> ) ; if ( ! hasNoImmOrEqual ( OrigMI , <STR_LIT> :: src0_modifiers , <NUM_LIT> , Mask ) || ! hasNoImmOrEqual ( OrigMI , <STR_LIT> :: src1_modifiers , <NUM_LIT> , Mask ) || ! hasNoImmOrEqual ( OrigMI , <STR_LIT> :: clamp , <NUM_LIT> ) || ! hasNoImmOrEqual ( OrigMI , <STR_LIT> :: omod , <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> MRI = & MF . getRegInfo ( ) ; TII = ST . getInstrInfo ( ) ; bool Changed = false ; for ( auto & MBB : MF ) { for ( auto I = MBB . rbegin ( ) , E = MBB . rend ( ) ; I != E ; ) {


<s> ++ NumOperands ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; Fail = true ; break ; } if ( auto * Mod0 = TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src0_modifiers ) ) { assert ( NumOperands == <STR_LIT> ( DPPOp , <STR_LIT> :: src0_modifiers ) ) ; assert ( <NUM_LIT> == ( Mod0 -> getImm ( ) & ~ ( <STR_LIT> | <STR_LIT> ) ) ) ; DPPInst . addImm ( Mod0 -> getImm ( ) ) ; ++ NumOperands ; } else if ( <STR_LIT> ( DPPOp , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ) { DPPInst . addImm ( <NUM_LIT> ) ; ++ NumOperands ; } auto * Src0 = TII -> getNamedOperand ( MovMI , <STR_LIT> :: src0 ) ; assert ( Src0 ) ; if ( ! TII -> isOperandLegal ( * DPPInst . getInstr ( ) , NumOperands , Src0 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Fail = true ; break ; } DPPInst . add ( * Src0 ) ; DPPInst -> getOperand ( NumOperands ) . setIsKill ( false ) ; ++ NumOperands ; if ( auto * Mod1 = TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src1_modifiers ) ) { assert ( NumOperands == <STR_LIT> ( DPPOp , <STR_LIT> :: src1_modifiers ) ) ; assert ( <NUM_LIT> == ( Mod1 -> getImm ( ) & ~ ( <STR_LIT> | <STR_LIT> ) ) ) ; DPPInst . addImm ( Mod1 -> getImm ( ) ) ; ++ NumOperands ; } else if ( <STR_LIT> ( DPPOp , <STR_LIT> :: src1_modifiers ) != - <NUM_LIT> ) { DPPInst . addImm ( <NUM_LIT> ) ; ++ NumOperands ; } if ( auto * Src1 = TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src1 ) ) { if ( ! TII -> isOperandLegal ( * DPPInst . getInstr ( ) , NumOperands , Src1 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Fail = true ; break ; } DPPInst . add ( * Src1 ) ; ++ NumOperands ; } if ( auto * Src2 = TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src2 ) ) { if ( ! TII -> getNamedOperand ( * DPPInst . getInstr ( ) , <STR_LIT> :: src2 ) || ! TII -> isOperandLegal ( * DPPInst . getInstr ( ) , NumOperands , Src2 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Fail = true ; break ; } DPPInst . add ( * Src2 ) ; }

DPPInst . add ( * TII -> getNamedOperand ( MovMI , <STR_LIT> :: dpp_ctrl ) ) ;


<s> if ( ! isIdentityValue ( OrigMI . getOpcode ( ) , OldOpndValue ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return nullptr ; } CombOldVGPR = getRegSubRegPair ( * Src1 ) ; if ( ! isOfRegClass ( CombOldVGPR , <STR_LIT> , * MRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return nullptr ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return nullptr ; } if ( ! isIdentityValue ( OrigMI . getOpcode ( ) , OldOpndValue ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return nullptr ; } CombOldVGPR = getRegSubRegPair ( * Src1 ) ; if ( ! isOfRegClass ( CombOldVGPR , <STR_LIT> , * MRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ;


<s> auto DPP32 = <STR_LIT> ( Op ) ; if ( DPP32 == - <NUM_LIT> ) { auto E32 = <STR_LIT> ( Op ) ;


return ( DPP32 == - <NUM_LIT> || TII -> pseudoToMCOpcode ( DPP32 ) == - <NUM_LIT> ) ? - <NUM_LIT> : DPP32 ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( static_cast < uint32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < uint32_t > :: max ( ) ) return true ; break ; case <STR_LIT> : case <STR_LIT> : if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < int32_t > :: max ( ) ) return true ; break ; case <STR_LIT> : case <STR_LIT> :

if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < int32_t > :: min ( ) ) return true ;


<s> break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( static_cast < uint32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < uint32_t > :: max ( ) ) return true ; break ; case <STR_LIT> : case <STR_LIT> : if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < int32_t > :: max ( ) ) return true ; break ; case <STR_LIT> : case <STR_LIT> : if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < int32_t > :: min ( ) ) return true ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( OldOpnd -> getImm ( ) == <NUM_LIT> ) return true ;


<s> assert ( MRI -> isSSA ( ) && <STR_LIT> ) ; bool Changed = false ; for ( auto & MBB : MF ) { for ( auto I = MBB . rbegin ( ) , E = MBB . rend ( ) ; I != E ; ) { auto & MI = * I ++ ; if ( MI . getOpcode ( ) == <STR_LIT> && combineDPPMov ( MI ) ) {


<s> addRegsToSet ( TRI , MI . uses ( ) , ClauseUses ) ;


<s> static void addRegsToSet ( const <ISA_LIT>RegisterInfo & TRI , iterator_range < MachineInstr :: const_mop_iterator > Ops , BitVector & Set ) { for ( const MachineOperand & Op : Ops ) { if ( Op . isReg ( ) ) addRegUnits ( TRI , Set , Op . getReg ( ) . asMCReg ( ) ) ;


<s> static void addRegUnits ( const <ISA_LIT>RegisterInfo & TRI , BitVector & BV , MCRegister Reg ) { for ( MCRegUnitIterator RUI ( Reg , & TRI ) ; RUI . isValid ( ) ; ++ RUI ) BV . set ( * RUI ) ;


<s> } EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; } EmittedInstrs . resize ( getMaxLookAhead ( ) ) ;


<s> static bool breaksSMEMSoftClause ( MachineInstr * MI ) {

return ! <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ;


<s> static bool breaksVMEMSoftClause ( MachineInstr * MI ) {

return ! <ISA_LIT>InstrInfo :: isVMEM ( * MI ) && ! <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ;


<s> auto IsHazardDefFn = [ TII ] ( const MachineInstr & MI ) { return TII -> isVALU ( MI ) ;


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; int DppVgprWaitStates = <NUM_LIT> ; int DppExecWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( const MachineInstr & MI ) { return TII -> isVALU ( MI ) ; } ; for ( const MachineOperand & Use : DPP -> uses ( ) ) { if ( ! Use . isReg ( ) || ! TRI -> isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; int WaitStatesNeededForUse = DppVgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , [ ] ( const MachineInstr & ) { return true ; } , DppVgprWaitStates ) ;

WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ;


<s> auto IsHazardFn = [ ] ( const MachineInstr & I ) { if ( ! <ISA_LIT>InstrInfo :: isVMEM ( I ) && ! <ISA_LIT>InstrInfo :: isFLAT ( I ) ) return false ; return <ISA_LIT>InstrInfo :: isFPAtomic ( I ) ; } ; auto IsExpiredFn = [ ] ( const MachineInstr & MI , int WaitStates ) { if ( WaitStates >= <NUM_LIT> || <ISA_LIT>InstrInfo :: isVALU ( MI ) ) return true ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> auto IsHazardFn = [ TII , GetRegHWReg ] ( const MachineInstr & MI ) { return GetRegHWReg == getHWReg ( TII , MI ) ; } ;

int WaitStatesNeeded = getWaitStatesSinceSetReg ( IsHazardFn , GetRegWaitStates ) ;


<s> int WaitStatesNeeded = <NUM_LIT> ; for ( unsigned I = InlineAsm :: MIOp_FirstOperand , E = IA -> getNumOperands ( ) ; I != E ; ++ I ) { const MachineOperand & Op = IA -> getOperand ( I ) ;

if ( Op . isReg ( ) && Op . isDef ( ) ) {


<s> I != E ; ++ I ) { const MachineOperand & Op = IA -> getOperand ( I ) ; if ( Op . isReg ( ) && Op . isDef ( ) ) { WaitStatesNeeded = std :: max ( WaitStatesNeeded , checkVALUHazardsHelper ( Op , MRI ) ) ; } } return WaitStatesNeeded ;


<s> assert ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) ) ; return ST . hasGFX90AInsts ( ) ? checkMAIHazards90A ( MI ) : checkMAIHazards908 ( MI ) ;


<s> int GCNHazardRecognizer :: checkMAIHazards ( MachineInstr * MI ) { assert ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) ) ;

return ST . hasGFX90AInsts ( ) ? checkMAIHazards90A ( MI ) : checkMAIHazards908 ( MI ) ;


<s> if ( ! IsMFMAFn ( MI ) ) return false ; NeighborMFMALatency = this -> getMFMAPipelineWaitStates ( MI ) ; return true ; } ; const int MaxMFMAPipelineWaitStates = <NUM_LIT> ; int WaitStatesSinceNeighborMFMA = getWaitStatesSince ( IsNeighboringMFMA , MaxMFMAPipelineWaitStates ) ; int NeighborMFMAPaddingNeeded = ( NeighborMFMALatency * MFMAPaddingRatio / <NUM_LIT> ) - WaitStatesSinceNeighborMFMA ;

return std :: max ( <NUM_LIT> , NeighborMFMAPaddingNeeded ) ;


<s> auto IsHazardFn = [ TII ] ( const MachineInstr & I ) { if ( ! <ISA_LIT>InstrInfo :: isMIMG ( I ) ) return false ; const <STR_LIT> * Info = <STR_LIT> ( I . getOpcode ( ) ) ; return Info -> MIMGEncoding == <STR_LIT> && TII -> getInstSizeInBytes ( I ) >= <NUM_LIT> ; } ; return NSAtoVMEMWaitStates - getWaitStatesSince ( IsHazardFn , <NUM_LIT> ) ;


<s> if ( ! ST . hasNSAtoVMEMBug ( ) ) return <NUM_LIT> ; if ( ! <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) && ! <ISA_LIT>InstrInfo :: isMTBUF ( * MI ) ) return <NUM_LIT> ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const auto * Offset = TII -> getNamedOperand ( * MI , <STR_LIT> :: offset ) ; if ( ! Offset || ( Offset -> getImm ( ) & <NUM_LIT> ) == <NUM_LIT> ) return <NUM_LIT> ; auto IsHazardFn = [ TII ] ( const MachineInstr & I ) {

if ( ! <ISA_LIT>InstrInfo :: isMIMG ( I ) ) return false ;


<s> const int ReadM0WaitStates = <NUM_LIT> ; auto IsHazardFn = [ TII ] ( const MachineInstr & MI ) { return TII -> isSALU ( MI ) ; } ;

return ReadM0WaitStates - getWaitStatesSinceDef ( <STR_LIT> , IsHazardFn , ReadM0WaitStates ) ;


<s> auto IsHazardFn = [ TII ] ( const MachineInstr & MI ) { return getHWReg ( TII , MI ) == <STR_LIT> :: ID_TRAPSTS ;


<s> int GCNHazardRecognizer :: checkRFEHazards ( MachineInstr * RFE ) { if ( ! ST . hasRFEHazards ( ) ) return <NUM_LIT> ;

const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ;


<s> int GCNHazardRecognizer :: checkRWLaneHazards ( MachineInstr * RWLane ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const MachineOperand * LaneSelectOp = TII -> getNamedOperand ( * RWLane , <STR_LIT> :: src1 ) ; if ( ! LaneSelectOp -> isReg ( ) || ! TRI -> isSGPRReg ( MRI , LaneSelectOp -> getReg ( ) ) ) return <NUM_LIT> ; Register LaneSelectReg = LaneSelectOp -> getReg ( ) ; auto IsHazardFn = [ TII ] ( const MachineInstr & MI ) { return TII -> isVALU ( MI ) ; } ; const int RWLaneWaitStates = <NUM_LIT> ; int WaitStatesSince = getWaitStatesSinceDef ( LaneSelectReg , IsHazardFn , RWLaneWaitStates ) ; return RWLaneWaitStates - WaitStatesSince ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; unsigned HWReg = getHWReg ( TII , * SetRegInstr ) ; const int SetRegWaitStates = ST . getSetRegWaitStates ( ) ; auto IsHazardFn = [ TII , HWReg ] ( const MachineInstr & MI ) { return HWReg == getHWReg ( TII , MI ) ;


<s> int GCNHazardRecognizer :: checkSetRegHazards ( MachineInstr * SetRegInstr ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; unsigned HWReg = getHWReg ( TII , * SetRegInstr ) ; const int SetRegWaitStates = ST . getSetRegWaitStates ( ) ; auto IsHazardFn = [ TII , HWReg ] ( const MachineInstr & MI ) { return HWReg == getHWReg ( TII , MI ) ; } ;

int WaitStatesNeeded = getWaitStatesSinceSetReg ( IsHazardFn , SetRegWaitStates ) ;


<s> } if ( ClauseDefs . none ( ) ) return <NUM_LIT> ; if ( MEM -> mayStore ( ) ) return <NUM_LIT> ; addClauseInst ( * MEM ) ;

return ClauseDefs . anyCommon ( ClauseUses ) ? <NUM_LIT> : <NUM_LIT> ;


<s> int GCNHazardRecognizer :: checkSoftClauseHazards ( MachineInstr * MEM ) { if ( ! ST . isXNACKEnabled ( ) ) return <NUM_LIT> ; bool IsSMRD = TII . isSMRD ( * MEM ) ; resetClause ( ) ; for ( MachineInstr * MI : EmittedInstrs ) { if ( ! MI ) break ; if ( IsSMRD ? breaksSMEMSoftClause ( MI ) : breaksVMEMSoftClause ( MI ) ) break ; addClauseInst ( * MI ) ; } if ( ClauseDefs . none ( ) ) return <NUM_LIT> ; if ( MEM -> mayStore ( ) ) return <NUM_LIT> ;


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const int VALUWaitStates = ST . hasGFX940Insts ( ) ? <NUM_LIT> : <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; if ( ! TRI -> isVectorRegister ( MRI , Def . getReg ( ) ) ) return WaitStatesNeeded ; Register Reg = Def . getReg ( ) ; auto IsHazardFn = [ this , Reg , TRI ] ( const MachineInstr & MI ) { int DataIdx = createsVALUHazard ( MI ) ; return DataIdx >= <NUM_LIT> && TRI -> regsOverlap ( MI . getOperand ( DataIdx ) . getReg ( ) , Reg ) ; } ; int WaitStatesNeededForDef = VALUWaitStates - getWaitStatesSince ( IsHazardFn , VALUWaitStates ) ;

WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForDef ) ;


<s> int GCNHazardRecognizer :: checkVALUHazardsHelper ( const MachineOperand & Def , const MachineRegisterInfo & MRI ) { const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const int VALUWaitStates = ST . hasGFX940Insts ( ) ? <NUM_LIT> : <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ;

if ( ! TRI -> isVectorRegister ( MRI , Def . getReg ( ) ) ) return WaitStatesNeeded ;


<s> if ( ! Use . isReg ( ) || TRI . isVectorRegister ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; int WaitStatesNeededForUse = VmemSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn , VmemSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; } return WaitStatesNeeded ;


<s> int VDataRCID = - <NUM_LIT> ; if ( VDataIdx != - <NUM_LIT> ) VDataRCID = Desc . OpInfo [ VDataIdx ] . RegClass ; if ( TII -> isMUBUF ( MI ) || TII -> isMTBUF ( MI ) ) { if ( VDataIdx == - <NUM_LIT> ) return - <NUM_LIT> ; const MachineOperand * SOffset = TII -> getNamedOperand ( MI , <STR_LIT> :: soffset ) ; if ( <STR_LIT> ( VDataRCID ) > <NUM_LIT> && ( ! SOffset || ! SOffset -> isReg ( ) ) ) return VDataIdx ; } if ( TII -> isMIMG ( MI ) ) { int SRsrcIdx = <STR_LIT> ( Opcode , <STR_LIT> :: srsrc ) ; assert ( SRsrcIdx != - <NUM_LIT> && <STR_LIT> ( Desc . OpInfo [ SRsrcIdx ] . RegClass ) == <NUM_LIT> ) ; ( void ) SRsrcIdx ; } if ( TII -> isFLAT ( MI ) ) { int DataIdx = <STR_LIT> ( Opcode , <STR_LIT> :: vdata ) ; if ( <STR_LIT> ( Desc . OpInfo [ DataIdx ] . RegClass ) > <NUM_LIT> ) return DataIdx ; } return - <NUM_LIT> ;


<s> if ( TII -> isMIMG ( MI ) ) { int SRsrcIdx = <STR_LIT> ( Opcode , <STR_LIT> :: srsrc ) ; assert ( SRsrcIdx != - <NUM_LIT> && <STR_LIT> ( Desc . OpInfo [ SRsrcIdx ] . RegClass ) == <NUM_LIT> ) ; ( void ) SRsrcIdx ; } if ( TII -> isFLAT ( MI ) ) { int DataIdx = <STR_LIT> ( Opcode , <STR_LIT> :: vdata ) ; if ( <STR_LIT> ( Desc . OpInfo [ DataIdx ] . RegClass ) > <NUM_LIT> ) return DataIdx ; } return - <NUM_LIT> ;


<s> void GCNHazardRecognizer :: EmitInstruction ( MachineInstr * MI ) {


<s> void GCNHazardRecognizer :: EmitNoop ( ) {

EmittedInstrs . push_front ( nullptr ) ;


<s> fixVcmpxPermlaneHazards ( MI ) ; fixSMEMtoVectorWriteHazards ( MI ) ; fixVcmpxExecWARHazard ( MI ) ;


<s> auto InstType = IsHazardInst ( * MI ) ; if ( ! InstType ) return false ; auto IsExpiredFn = [ & IsHazardInst ] ( const MachineInstr & I , int ) { return IsHazardInst ( I ) || ( I . getOpcode ( ) == <STR_LIT> && I . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && ! I . getOperand ( <NUM_LIT> ) . getImm ( ) ) ; } ; auto IsHazardFn = [ InstType , & IsHazardInst ] ( const MachineInstr & I ) { if ( ! I . isBranch ( ) ) return false ; auto IsHazardFn = [ InstType , IsHazardInst ] ( const MachineInstr & I ) { auto InstType2 = IsHazardInst ( I ) ; return InstType2 && InstType != InstType2 ; } ; auto IsExpiredFn = [ InstType , & IsHazardInst ] ( const MachineInstr & I , int ) { auto InstType2 = IsHazardInst ( I ) ; if ( InstType == InstType2 ) return true ; return I . getOpcode ( ) == <STR_LIT> && I . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && ! I . getOperand ( <NUM_LIT> ) . getImm ( ) ; } ; return :: getWaitStatesSince ( IsHazardFn , & I , IsExpiredFn ) != std :: numeric_limits < int > :: max ( ) ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <STR_LIT> IV = <STR_LIT> ( ST . getCPU ( ) ) ; const MachineOperand * SDST = TII -> getNamedOperand ( * MI , SDSTName ) ; if ( ! SDST ) { for ( const auto & MO : MI -> implicit_operands ( ) ) { if ( MO . isDef ( ) && TRI -> isSGPRClass ( TRI -> getPhysRegClass ( MO . getReg ( ) ) ) ) { SDST = & MO ; break ; } } } if ( ! SDST ) return false ; const Register SDSTReg = SDST -> getReg ( ) ; auto IsHazardFn = [ SDSTReg , TRI ] ( const MachineInstr & I ) { return <ISA_LIT>InstrInfo :: isSMRD ( I ) && I . readsRegister ( SDSTReg , TRI ) ; } ; auto IsExpiredFn = [ TII , IV ] ( const MachineInstr & MI , int ) { if ( TII -> isSALU ( MI ) ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> auto IsExpiredFn = [ TII , TRI ] ( const MachineInstr & MI , int ) { if ( <ISA_LIT>InstrInfo :: isVALU ( MI ) ) { if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ) return true ; for ( auto MO : MI . implicit_operands ( ) ) if ( MO . isDef ( ) && TRI -> isSGPRClass ( TRI -> getPhysRegClass ( MO . getReg ( ) ) ) ) return true ; } if ( MI . getOpcode ( ) == <STR_LIT> && ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> ) == <NUM_LIT> ) return true ; return false ; } ;

if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; auto IsExpiredFn = [ TII , TRI ] ( const MachineInstr & MI , int ) { if ( <ISA_LIT>InstrInfo :: isVALU ( MI ) ) { if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ) return true ; for ( auto MO : MI . implicit_operands ( ) ) if ( MO . isDef ( ) && TRI -> isSGPRClass ( TRI -> getPhysRegClass ( MO . getReg ( ) ) ) ) return true ; } if ( MI . getOpcode ( ) == <STR_LIT> && ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> ) == <NUM_LIT> ) return true ; return false ;


<s> auto IsExpiredFn = [ ] ( const MachineInstr & MI , int ) { unsigned Opc = MI . getOpcode ( ) ; return <ISA_LIT>InstrInfo :: isVALU ( MI ) && Opc != <STR_LIT> && Opc != <STR_LIT> && Opc != <STR_LIT> ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ; auto * Src0 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) ; Register Reg = Src0 -> getReg ( ) ; bool IsUndef = Src0 -> isUndef ( ) ;

BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Define | ( IsUndef ? RegState :: Dead : <NUM_LIT> ) ) . addReg ( Reg , IsUndef ? RegState :: Undef : RegState :: Kill ) ;