<s> MachineBasicBlock * LandBlk = * LandMBBPtr ; assert ( ( ! TrueMBB || TrueMBB -> succ_size ( ) <= <NUM_LIT> ) && ( ! FalseMBB || FalseMBB -> succ_size ( ) <= <NUM_LIT> ) ) ; if ( TrueMBB == FalseMBB ) return <NUM_LIT> ; MigrateTrue = needMigrateBlock ( TrueMBB ) ; MigrateFalse = needMigrateBlock ( FalseMBB ) ; if ( ! MigrateTrue && ! MigrateFalse ) return <NUM_LIT> ; if ( ! MigrateTrue && TrueMBB && TrueMBB -> pred_size ( ) > <NUM_LIT> ) MigrateTrue = true ; if ( ! MigrateFalse && FalseMBB && FalseMBB -> pred_size ( ) > <NUM_LIT> ) MigrateFalse = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; showImproveSimpleJumpintoIf ( HeadMBB , TrueMBB , FalseMBB , LandBlk , <NUM_LIT> ) ; ) ; const TargetRegisterClass * I32RC = TRI -> getCFGStructurizerRegClass ( <STR_LIT> ) ; if ( ! MigrateTrue || ! MigrateFalse ) { return <NUM_LIT> ; } int NumNewBlk = <NUM_LIT> ; bool LandBlkHasOtherPred = ( LandBlk -> pred_size ( ) > <NUM_LIT> ) ; MachineBasicBlock :: iterator I = insertInstrBefore ( LandBlk , <STR_LIT> ) ; if ( LandBlkHasOtherPred ) { report_fatal_error ( <STR_LIT> ) ; Register CmpResReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; report_fatal_error ( <STR_LIT>


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) ; dbgs ( ) << <STR_LIT> ; if ( TrueMBB ) { dbgs ( ) << <STR_LIT> << TrueMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; if ( FalseMBB ) { dbgs ( ) << <STR_LIT> << FalseMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; if ( ! LandMBB ) { dbgs ( ) << <STR_LIT> ; } else { dbgs ( ) << <STR_LIT> << LandMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; ) ; int OldOpcode = BranchMI -> getOpcode ( ) ; DebugLoc BranchDL = BranchMI -> getDebugLoc ( ) ; MachineBasicBlock :: iterator I = BranchMI ; insertCondBranchBefore ( I , getBranchNzeroOpcode ( OldOpcode ) , BranchDL ) ; if ( TrueMBB ) { MBB -> splice ( I , TrueMBB , TrueMBB -> begin ( ) , TrueMBB -> end ( ) ) ; MBB -> removeSuccessor ( TrueMBB , true ) ; if ( LandMBB && TrueMBB -> succ_size ( ) != <NUM_LIT> ) TrueMBB -> removeSuccessor ( LandMBB , true ) ; retireBlock ( TrueMBB ) ; MLI -> removeBlock ( TrueMBB ) ; } if ( FalseMBB ) { insertInstrBefore ( I , <STR_LIT> ) ; MBB -> splice ( I , FalseMBB , FalseMBB -> begin ( ) , FalseMBB -> end ( ) ) ; MBB -> removeSuccessor ( FalseMBB , true ) ; if ( LandMBB && FalseMBB -> succ_size ( ) != <NUM_LIT> ) FalseMBB -> removeSuccessor ( LandMBB , true ) ; retireBlock ( FalseMBB ) ; MLI -> removeBlock ( FalseMBB ) ; } insertInstrBefore ( I , <STR_LIT> ) ; BranchMI -> eraseFromParent


<s> assert ( TrueMBB ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) ; dbgs ( ) << <STR_LIT> ; if ( TrueMBB ) { dbgs ( ) << <STR_LIT> << TrueMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; if ( FalseMBB ) { dbgs ( ) << <STR_LIT> << FalseMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; dbgs ( ) << <STR_LIT> ; if ( ! LandMBB ) { dbgs ( ) << <STR_LIT> ; } else { dbgs ( ) << <STR_LIT> << LandMBB -> getNumber ( ) ; } dbgs ( ) << <STR_LIT> ; ) ; int OldOpcode = BranchMI -> getOpcode ( ) ; DebugLoc BranchDL = BranchMI -> getDebugLoc ( ) ; MachineBasicBlock :: iterator I = BranchMI ; insertCondBranchBefore ( I , getBranchNzeroOpcode ( OldOpcode


<s> assert ( ! ExitingMBBs . empty ( ) && <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << ExitingMBBs . size ( ) << <STR_LIT> ; ) ; MBBVector ExitBlks ; LoopRep -> getExitBlocks ( ExitBlks ) ; SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > ExitBlkSet ; for ( unsigned i = <NUM_LIT> , e = ExitBlks . size ( ) ; i < e ; ++ i ) ExitBlkSet . insert ( ExitBlks [ i ] ) ; assert ( ExitBlkSet . size ( ) == <NUM_LIT> ) ; MachineBasicBlock * ExitBlk = * ExitBlks . begin ( ) ; assert ( ExitBlk && <STR_LIT> ) ; MBBVector LatchBlks ; for ( auto * LB : inverse_children < MachineBasicBlock * > ( LoopHeader ) ) if ( LoopRep -> contains ( LB ) ) LatchBlks . push_back ( LB ) ; for ( unsigned i = <NUM_LIT> , e = ExitingMBBs . size ( ) ; i < e ; ++ i ) mergeLoopbreakBlock ( ExitingMBBs [ i ] , ExitBlk ) ; for ( unsigned i = <NUM_LIT> , e = LatchBlks . size ( ) ; i < e ; ++ i ) settleLoopcontBlock ( LatchBlks [ i ] , LoopHeader ) ; int Match = <NUM_LIT> ; do { Match = <NUM_LIT> ; Match += serialPatternMatch ( LoopHeader ) ; Match += ifPatternMatch ( LoopHeader ) ; } while ( Match > <NUM_LIT> ) ; mergeLooplandBlock ( LoopHeader , ExitBlk ) ; MachineLoop * ParentLoop = LoopRep -> getParentLoop ( ) ; if ( ParentLoop ) MLI -> changeLoopFor ( LoopHeader , ParentLoop ) ; else MLI -> removeBlock (


<s> Finish = true ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } else { int NewnumRemainedBlk = countActiveBlock ( OrderedBlks . begin ( ) , OrderedBlks . end ( ) ) ; if ( NewnumRemainedBlk == <NUM_LIT> || NewnumRemainedBlk < NumRemainedBlk ) { MakeProgress = true ; NumRemainedBlk = NewnumRemainedBlk ; } else { MakeProgress = false ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } } } while ( ! Finish && MakeProgress ) ; wrapup ( & * GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ) ; for ( MBBInfoMap :: iterator It = BlockInfoMap . begin ( ) , E = BlockInfoMap . end ( ) ; It != E ; ++ It ) { if ( ( * It ) . second && ( * It ) . second -> IsRetired ) { assert ( ( ( * It ) . first ) -> getNumber ( ) != - <NUM_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << ( ( * It ) . first ) -> getNumber ( ) << <STR_LIT> ; ) ; ( * It ) . first -> eraseFromParent ( ) ; } delete ( * It ) . second ; } BlockInfoMap . clear (


<s> SccBeginMBB = MBB ; SccNumIter = <NUM_LIT> ; SccNumBlk = NumRemainedBlk ; DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( SccBeginMBB ) ; dbgs ( ) << <STR_LIT> ; ) ; } if ( ! isRetiredBlock ( MBB ) ) patternMatch ( MBB ) ; ++ It ; bool ContNextScc = true ; if ( It == E || getSCCNum ( SccBeginMBB ) != getSCCNum ( * It ) ) { ++ SccNumIter ; int sccRemainedNumBlk = countActiveBlock ( SccBeginIter , It ) ; if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk >= SccNumBlk ) { DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter ; dbgs ( ) << <STR_LIT> ; ) ; ContNextScc = true ; } else if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk < SccNumBlk ) { SccNumBlk = sccRemainedNumBlk ; It = SccBeginIter ; ContNextScc = false ; DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter << '\n' ; ) ; } else { ContNextScc = true ; } } else { ContNextScc = false ; } if ( ContNextScc ) SccBeginMBB = nullptr ; } MachineBasicBlock * EntryMBB = & * GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ; if ( EntryMBB -> succ_size ( ) == <NUM_LIT> ) { Finish = true ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } else { int NewnumRemainedBlk = countActiveBlock ( OrderedBlks . begin ( ) , OrderedBlks . end ( ) ) ; if ( NewnumRemainedBlk == <NUM_LIT> || NewnumRemainedBlk < NumRemainedBlk ) { MakeProgress = true ; NumRemainedBlk = NewnumRemainedBlk ; } else { MakeProgress = false ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; } } } while ( ! Finish && MakeProgress ) ; wrapup ( & * GraphTraits < MachineFunction * > :: nodes_begin ( FuncRep ) ) ; for ( MBBInfoMap :: iterator It = BlockInfoMap . begin ( ) , E = BlockInfoMap . end ( ) ; It != E ; ++ It ) { if ( ( * It ) . second && ( * It ) . second -> IsRetired ) { assert ( ( ( * It ) . first ) -> getNumber ( ) != - <NUM_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << ( ( * It ) . first ) -> getNumber ( ) << <STR_LIT> ; ) ; ( * It ) . first -> eraseFromParent ( ) ; } delete ( * It ) . second ; } BlockInfoMap . clear (


<s> if ( I -> getOpcode ( ) == <STR_LIT> ) { switch ( static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . getImm ( ) ) { case OPCODE_IS_ZERO_INT : static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . setImm ( OPCODE_IS_NOT_ZERO_INT ) ; return ; case OPCODE_IS_NOT_ZERO_INT : static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . setImm ( OPCODE_IS_ZERO_INT ) ; return ; case OPCODE_IS_ZERO : static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . setImm (


<s> ) ; SpliceEnd = SrcMBB -> end ( ) ; } else { DEBUG ( dbgs ( ) << <STR_LIT> ; BranchMI -> dump ( ) ; ) ; SpliceEnd = BranchMI ; } DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> size ( ) << <STR_LIT> << SrcMBB -> size ( ) << <STR_LIT> ; ) ; DstMBB -> splice ( I , SrcMBB , SrcMBB


<s> if ( MBB -> succ_size ( ) != <NUM_LIT> ) return ; MachineBasicBlock * MBB1 = * MBB -> succ_begin ( ) ; MachineBasicBlock * MBB2 = * std :: next ( MBB -> succ_begin ( ) ) ; if ( MBB1 != MBB2


<s> MachineBasicBlock * MBB2 = * std :: next ( MBB -> succ_begin ( ) ) ; if ( MBB1 != MBB2 ) return ; MachineInstr * BranchMI = getNormalBlockBranchInstr ( MBB ) ; assert ( BranchMI && isCondBranch ( BranchMI ) ) ; DEBUG ( dbgs ( ) << <STR_LIT> ; BranchMI -> dump ( ) ; ) ; BranchMI -> eraseFromParent ( ) ; SHOWNEWBLK ( MBB1 , <STR_LIT> )


<s> MachineInstr * BranchMI ; while ( ( BranchMI = getLoopendBlockBranchInstr ( MBB


<s> for ( const MachineLoop * L : LoopInfo ) L -> print


<s> ++ NumIter ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << NumIter << <STR_LIT> << NumRemainedBlk << <STR_LIT> ; ) ; SmallVectorImpl < MachineBasicBlock * > :: const_iterator It = OrderedBlks . begin ( ) ; SmallVectorImpl < MachineBasicBlock * > :: const_iterator E = OrderedBlks . end ( ) ; SmallVectorImpl < MachineBasicBlock * > :: const_iterator SccBeginIter = It ; MachineBasicBlock * SccBeginMBB = nullptr ; int SccNumBlk = <NUM_LIT> ; int SccNumIter ; while ( It != E ) { MBB = * It ; if ( ! SccBeginMBB ) { SccBeginIter = It ; SccBeginMBB = MBB ; SccNumIter = <NUM_LIT> ; SccNumBlk = NumRemainedBlk ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( SccBeginMBB ) ; dbgs ( ) << <STR_LIT> ; ) ; } if ( ! isRetiredBlock ( MBB ) ) patternMatch ( MBB ) ; ++ It ; bool ContNextScc = true ; if ( It == E || getSCCNum ( SccBeginMBB ) != getSCCNum ( * It ) ) { ++ SccNumIter ; int sccRemainedNumBlk = countActiveBlock ( SccBeginIter , It ) ; if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk >= SccNumBlk ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter ; dbgs ( ) << <STR_LIT> ; ) ; ContNextScc = true ; } else if ( sccRemainedNumBlk != <NUM_LIT> && sccRemainedNumBlk < SccNumBlk ) { SccNumBlk = sccRemainedNumBlk ; It = SccBeginIter ; ContNextScc = false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << getSCCNum ( MBB ) << <STR_LIT> << SccNumIter << '\n' ; ) ; } else { ContNextScc = true ; } } else { ContNextScc =


<s> int <ISA_LIT>CFGStructurizer :: improveSimpleJumpintoIf ( MachineBasicBlock * HeadMBB , MachineBasicBlock * TrueMBB , MachineBasicBlock * FalseMBB , MachineBasicBlock * * LandMBBPtr ) { bool MigrateTrue = false ; bool MigrateFalse = false ; MachineBasicBlock * LandBlk = * LandMBBPtr ; assert ( ( ! TrueMBB || TrueMBB -> succ_size ( ) <= <NUM_LIT> ) && ( ! FalseMBB || FalseMBB -> succ_size ( ) <= <NUM_LIT> ) ) ; if ( TrueMBB == FalseMBB ) return <NUM_LIT> ; MigrateTrue = needMigrateBlock ( TrueMBB ) ; MigrateFalse = needMigrateBlock ( FalseMBB ) ; if ( ! MigrateTrue && ! MigrateFalse ) return <NUM_LIT> ; if ( ! MigrateTrue && TrueMBB && TrueMBB -> pred_size ( ) > <NUM_LIT> ) MigrateTrue = true ; if ( ! MigrateFalse && FalseMBB && FalseMBB -> pred_size ( ) > <NUM_LIT> ) MigrateFalse = true ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; showImproveSimpleJumpintoIf ( HeadMBB , TrueMBB , FalseMBB , LandBlk , <NUM_LIT> ) ; ) ; const TargetRegisterClass * I32RC = TRI -> getCFGStructurizerRegClass ( <STR_LIT> ) ; if ( ! MigrateTrue || ! MigrateFalse ) { return <NUM_LIT> ; } int NumNewBlk = <NUM_LIT> ; bool LandBlkHasOtherPred = ( LandBlk -> pred_size ( ) > <NUM_LIT> ) ; MachineBasicBlock :: iterator I = insertInstrBefore ( LandBlk , <STR_LIT> ) ; if ( LandBlkHasOtherPred ) { report_fatal_error ( <STR_LIT> ) ; unsigned CmpResReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; report_fatal_error ( <STR_LIT> ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , CmpResReg , DebugLoc ( ) ) ; } unsigned InitReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , InitReg , DebugLoc ( ) ) ; if ( MigrateTrue ) { migrateInstruction ( TrueMBB , LandBlk , I ) ; report_fatal_error ( <STR_LIT> ) ; } insertInstrBefore ( I , <STR_LIT> ) ; if ( MigrateFalse ) { migrateInstruction ( FalseMBB , LandBlk , I ) ; report_fatal_error ( <STR_LIT> ) ; } if ( LandBlkHasOtherPred ) { insertInstrBefore ( I ,


<s> <ISA_LIT>CFGStructurizer ( ) : MachineFunctionPass ( ID ) , TII ( nullptr ) , TRI ( nullptr ) { initialize<ISA_LIT>CFGStructurizerPass ( * PassRegistry :: getPassRegistry (


<s> MachineFunctionPass ( ID ) , TII ( nullptr ) , TRI ( nullptr ) { initialize<ISA_LIT>CFGStructurizerPass ( * PassRegistry ::


<s> for ( MachineBasicBlock :: iterator It = MBB -> begin ( ) , E = MBB -> end ( ) ; It != E ; ++ It ) { MachineInstr * MI = Func -> CloneMachineInstr ( It


<s> MachineBasicBlock * <ISA_LIT>CFGStructurizer :: clone ( MachineBasicBlock * MBB ) { MachineFunction * Func = MBB -> getParent ( ) ; MachineBasicBlock * NewMBB = Func -> CreateMachineBasicBlock ( ) ; Func -> push_back


<s> PredMBB -> addSuccessor ( CloneMBB ) ; cloneSuccessorList ( CloneMBB , MBB ) ; numClonedInstr += MBB -> size ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> << MBB -> size ( ) << <STR_LIT> ; ) ; SHOWNEWBLK ( CloneMBB , <STR_LIT> ) ; return CloneMBB


<s> for ( MachineBasicBlock :: succ_iterator It = SrcMBB -> succ_begin ( ) , iterEnd = SrcMBB -> succ_end ( ) ; It != iterEnd ; ++ It ) DstMBB


<s> int <ISA_LIT>CFGStructurizer :: countActiveBlock ( MBBVector :: const_iterator It , MBBVector :: const_iterator E ) const { int Count =


<s> return new <ISA_LIT>CFGStructurizer ( )


<s> std :: set < MachineBasicBlock * > :: const_iterator It = MBBs . begin ( ) ; std :: set < MachineBasicBlock * > :: const_iterator E = MBBs . end ( ) ; for ( CommonDom = * It ; It != E &&


<s> std :: set < MachineBasicBlock * > :: const_iterator E = MBBs . end ( ) ; for ( CommonDom = * It ; It != E && CommonDom ; ++ It ) { MachineBasicBlock * MBB = * It ; if ( MBB != CommonDom ) CommonDom = findNearestCommonPostDom ( MBB , CommonDom ) ; } DEBUG ( dbgs ( ) << <STR_LIT> ; if ( CommonDom ) dbgs ( ) << <STR_LIT> << CommonDom -> getNumber ( ) << <STR_LIT> ; else dbgs ( ) <<


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addPreserved < MachineFunctionAnalysis > ( ) ; AU . addRequired < MachineFunctionAnalysis > ( ) ; AU . addRequired < MachineDominatorTree


<s> int <ISA_LIT>CFGStructurizer :: getBranchNzeroOpcode ( int OldOpcode ) { switch ( OldOpcode ) { case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT>


<s> case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; default : llvm_unreachable (


<s> MachineBasicBlock :: reverse_iterator It = MBB -> rbegin ( ) ; if ( It != MBB -> rend ( )


<s> int <ISA_LIT>CFGStructurizer :: getContinueNzeroOpcode ( int OldOpcode ) { switch ( OldOpcode ) { case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; default : llvm_unreachable


<s> case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; } ; return - <NUM_LIT>


<s> switch ( OldOpcode ) { case <STR_LIT> : case <STR_LIT> : return <STR_LIT> ; default : llvm_unreachable (


<s> MachineBasicBlock * TrueBranch = getTrueBranch ( MI ) ; MachineBasicBlock :: succ_iterator It = MBB -> succ_begin ( ) ; MachineBasicBlock :: succ_iterator Next = It ; ++


<s> MachineBasicBlock * TrueBranch = getTrueBranch ( MI ) ; MachineBasicBlock :: succ_iterator It = MBB -> succ_begin ( ) ; MachineBasicBlock :: succ_iterator Next =


<s> MachineInstr * instr = & ( * It ) ; if ( instr -> getDebugLoc ( ) ) DL = instr -> getDebugLoc ( ) ; } return


<s> if ( It == LLInfoMap . end ( ) ) return nullptr ; return ( * It


<s> MachineBasicBlock * <ISA_LIT>CFGStructurizer :: getLoopLandInfo ( MachineLoop * LoopRep ) const { LoopLandInfoMap :: const_iterator It = LLInfoMap . find ( LoopRep ) ; if ( It == LLInfoMap . end ( ) ) return nullptr ; return ( * It ) .


<s> MachineInstr * MI = & * It ; if ( MI && ( isCondBranch ( MI ) || isUncondBranch ( MI ) ) ) return MI ; return nullptr


<s> MachineInstr * MI = & * It ; if ( MI && ( isCondBranch ( MI ) ||


<s> const char * getPassName ( ) const override { return <STR_LIT>


<s> if ( It != MBB -> rend ( ) ) { MachineInstr * instr = & ( * It ) ; if ( instr -> getOpcode ( ) == <STR_LIT> ) return instr ; } return


<s> MBBInfoMap :: const_iterator It = BlockInfoMap . find ( MBB ) ; if ( It == BlockInfoMap . end ( ) )


<s> MachineBasicBlock * <ISA_LIT>CFGStructurizer :: getTrueBranch ( MachineInstr * MI ) { return MI -> getOperand ( <NUM_LIT> ) .


<s> int <ISA_LIT>CFGStructurizer :: handleJumpintoIf ( MachineBasicBlock * HeadMBB , MachineBasicBlock * TrueMBB , MachineBasicBlock * FalseMBB ) { int Num = handleJumpintoIfImp ( HeadMBB , TrueMBB , FalseMBB ) ; if ( Num == <NUM_LIT> ) { DEBUG ( dbgs ( )


<s> int <ISA_LIT>CFGStructurizer :: handleJumpintoIfImp ( MachineBasicBlock * HeadMBB , MachineBasicBlock * TrueMBB , MachineBasicBlock * FalseMBB ) { int Num = <NUM_LIT> ; MachineBasicBlock * DownBlk ; DownBlk = TrueMBB ; DEBUG ( dbgs ( ) << <STR_LIT> << HeadMBB -> getNumber ( ) << <STR_LIT> << TrueMBB -> getNumber ( ) << <STR_LIT> << TrueMBB -> succ_size ( ) << <STR_LIT> << FalseMBB -> getNumber ( ) << <STR_LIT> ; ) ; while ( DownBlk ) { DEBUG ( dbgs ( ) << <STR_LIT> << DownBlk -> getNumber ( ) ; ) ; if ( singlePathTo ( FalseMBB , DownBlk ) == SinglePath_InPath ) { DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; Num += cloneOnSideEntryTo ( HeadMBB , TrueMBB , DownBlk ) ; Num += cloneOnSideEntryTo ( HeadMBB , FalseMBB , DownBlk ) ; numClonedBlock += Num ; Num += serialPatternMatch ( * HeadMBB -> succ_begin ( ) ) ; Num += serialPatternMatch ( * std :: next ( HeadMBB -> succ_begin ( ) ) ) ; Num += ifPatternMatch ( HeadMBB ) ; assert ( Num > <NUM_LIT> ) ; break ; } DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; DownBlk = ( DownBlk -> succ_size ( ) == <NUM_LIT> ) ? ( * DownBlk -> succ_begin


<s> void <ISA_LIT>CFGStructurizer :: handleLoopcontBlock ( MachineBasicBlock * ContingMBB , MachineLoop * ContingLoop , MachineBasicBlock * ContMBB , MachineLoop * ContLoop ) { DEBUG ( dbgs ( ) << <STR_LIT> << ContingMBB -> getNumber ( ) << <STR_LIT> << ContMBB ->


<s> dbgs ( ) << <STR_LIT> << getLoopDepth ( ContLoop ) << <STR_LIT> << getLoopDepth ( ContingLoop ) << <STR_LIT> ; ) ; settleLoopcontBlock ( ContingMBB


<s> MachineBasicBlock * LoopHeader = LoopRep -> getHeader ( ) ; return MBB -> isSuccessor


<s> MachineInstr * BranchMI = getNormalBlockBranchInstr ( MBB ) ; if ( ! BranchMI ) return <NUM_LIT> ; assert ( isCondBranch ( BranchMI ) ) ; int NumMatch = <NUM_LIT> ; MachineBasicBlock * TrueMBB = getTrueBranch ( BranchMI ) ; NumMatch += serialPatternMatch ( TrueMBB ) ; NumMatch += ifPatternMatch ( TrueMBB ) ; MachineBasicBlock * FalseMBB = getFalseBranch ( MBB , BranchMI ) ; NumMatch += serialPatternMatch ( FalseMBB ) ; NumMatch += ifPatternMatch ( FalseMBB ) ; MachineBasicBlock * LandBlk ; int Cloned = <NUM_LIT> ; assert ( ! TrueMBB -> succ_empty ( ) || ! FalseMBB -> succ_empty ( ) ) ; if ( TrueMBB -> succ_size ( ) == <NUM_LIT> && FalseMBB -> succ_size ( ) == <NUM_LIT> && * TrueMBB -> succ_begin ( ) == * FalseMBB -> succ_begin ( ) ) { LandBlk = * TrueMBB -> succ_begin ( ) ; } else if ( TrueMBB -> succ_size ( ) == <NUM_LIT> && * TrueMBB -> succ_begin ( ) == FalseMBB ) { LandBlk = FalseMBB ; FalseMBB = nullptr ; } else if ( FalseMBB -> succ_size ( ) == <NUM_LIT> && * FalseMBB -> succ_begin ( ) == TrueMBB ) { std :: swap ( TrueMBB , FalseMBB ) ; reversePredicateSetter ( MBB -> end ( ) ) ; LandBlk = FalseMBB ; FalseMBB = nullptr ; } else if ( FalseMBB -> succ_size ( ) == <NUM_LIT> && isSameloopDetachedContbreak ( TrueMBB ,


<s> bool MigrateTrue = false ; bool MigrateFalse = false ; MachineBasicBlock * LandBlk = * LandMBBPtr ; assert ( ( ! TrueMBB || TrueMBB -> succ_size ( ) <= <NUM_LIT> ) && ( ! FalseMBB || FalseMBB -> succ_size ( ) <= <NUM_LIT> ) ) ; if ( TrueMBB == FalseMBB ) return <NUM_LIT> ; MigrateTrue = needMigrateBlock ( TrueMBB ) ; MigrateFalse = needMigrateBlock ( FalseMBB ) ; if ( ! MigrateTrue && ! MigrateFalse ) return <NUM_LIT> ; if ( ! MigrateTrue && TrueMBB && TrueMBB -> pred_size ( ) > <NUM_LIT> ) MigrateTrue = true ; if ( ! MigrateFalse && FalseMBB && FalseMBB -> pred_size ( ) > <NUM_LIT> ) MigrateFalse = true ; DEBUG ( dbgs ( ) << <STR_LIT> ; showImproveSimpleJumpintoIf ( HeadMBB , TrueMBB , FalseMBB , LandBlk , <NUM_LIT> ) ; ) ; const TargetRegisterClass * I32RC = TRI -> getCFGStructurizerRegClass ( <STR_LIT> ) ; if ( ! MigrateTrue || ! MigrateFalse ) { return <NUM_LIT> ; } int NumNewBlk = <NUM_LIT> ; bool LandBlkHasOtherPred = ( LandBlk -> pred_size ( ) > <NUM_LIT> ) ; MachineBasicBlock :: iterator I = insertInstrBefore ( LandBlk , <STR_LIT> ) ; if ( LandBlkHasOtherPred ) { llvm_unreachable ( <STR_LIT> ) ; unsigned CmpResReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; llvm_unreachable ( <STR_LIT> ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , CmpResReg , DebugLoc ( ) ) ; } unsigned InitReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , InitReg , DebugLoc ( ) ) ; if ( MigrateTrue ) { migrateInstruction ( TrueMBB , LandBlk , I ) ; llvm_unreachable ( <STR_LIT> ) ; } insertInstrBefore ( I , <STR_LIT> ) ; if ( MigrateFalse ) { migrateInstruction ( FalseMBB , LandBlk


<s> ) ; const TargetRegisterClass * I32RC = TRI -> getCFGStructurizerRegClass ( <STR_LIT> ) ; if ( ! MigrateTrue || ! MigrateFalse ) { return <NUM_LIT> ; } int NumNewBlk = <NUM_LIT> ; bool LandBlkHasOtherPred = ( LandBlk -> pred_size ( ) > <NUM_LIT> ) ; MachineBasicBlock :: iterator I = insertInstrBefore ( LandBlk , <STR_LIT> ) ; if ( LandBlkHasOtherPred ) { llvm_unreachable ( <STR_LIT> ) ; unsigned CmpResReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; llvm_unreachable ( <STR_LIT> ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , CmpResReg , DebugLoc ( ) ) ; } unsigned InitReg = HeadMBB -> getParent ( ) -> getRegInfo ( ) . createVirtualRegister ( I32RC ) ; insertCondBranchBefore ( LandBlk , I , <STR_LIT> , InitReg , DebugLoc ( ) ) ; if ( MigrateTrue ) { migrateInstruction ( TrueMBB , LandBlk , I ) ; llvm_unreachable ( <STR_LIT> ) ; } insertInstrBefore ( I , <STR_LIT> ) ; if ( MigrateFalse ) { migrateInstruction ( FalseMBB , LandBlk , I ) ; llvm_unreachable ( <STR_LIT> ) ; } if ( LandBlkHasOtherPred ) { insertInstrBefore ( I , <STR_LIT> ) ; for ( MachineBasicBlock :: pred_iterator PI = LandBlk -> pred_begin ( ) , PE = LandBlk


<s> MachineFunction * MF = blk -> getParent ( ) ; MachineInstr * NewInstr = MF -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DL ) ; blk -> insert ( I , NewInstr ) ; MachineInstrBuilder ( * MF , NewInstr ) . addReg ( RegNum , false ) ; SHOWNEWINSTR (


<s> void <ISA_LIT>CFGStructurizer :: insertCondBranchEnd ( MachineBasicBlock * MBB , int NewOpcode , int RegNum ) { MachineFunction * MF = MBB -> getParent ( ) ; MachineInstr * NewInstr = MF -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DebugLoc ( ) ) ; MBB -> push_back (


<s> void <ISA_LIT>CFGStructurizer :: insertCondBranchEnd ( MachineBasicBlock * MBB , int NewOpcode , int RegNum ) { MachineFunction * MF = MBB -> getParent ( ) ; MachineInstr * NewInstr = MF -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DebugLoc ( ) ) ; MBB -> push_back ( NewInstr ) ; MachineInstrBuilder ( * MF , NewInstr ) . addReg ( RegNum , false


<s> MachineInstr * <ISA_LIT>CFGStructurizer :: insertInstrBefore ( MachineBasicBlock :: iterator I , int NewOpcode ) { MachineInstr * OldMI = &


<s> MachineInstr * <ISA_LIT>CFGStructurizer :: insertInstrBefore ( MachineBasicBlock :: iterator I , int NewOpcode ) { MachineInstr * OldMI = & ( * I ) ; MachineBasicBlock * MBB = OldMI -> getParent ( ) ; MachineInstr * NewMBB = MBB -> getParent ( ) -> CreateMachineInstr ( TII -> get ( NewOpcode ) , DebugLoc ( ) ) ; MBB -> insert ( I , NewMBB ) ; SHOWNEWINSTR ( NewMBB ) ; return NewMBB


<s> MachineLoop * LoopRep = MLI -> getLoopFor ( MBB ) ; while ( LoopRep && LoopRep -> getHeader ( ) == MBB ) { MachineBasicBlock * LoopLand = getLoopLandInfo ( LoopRep ) ; if ( ! LoopLand ) return true ; if ( ! isRetiredBlock ( LoopLand ) ) return true ; LoopRep = LoopRep -> getParentLoop (


<s> bool <ISA_LIT>CFGStructurizer :: isRetiredBlock ( MachineBasicBlock * MBB ) const { MBBInfoMap :: const_iterator It = BlockInfoMap . find ( MBB ) ; if ( It == BlockInfoMap . end (


<s> for ( SmallVectorImpl < MachineBasicBlock * > :: iterator It = ContMBB . begin ( ) , E = ContMBB . end ( ) ; It != E ; ++ It ) { ( * It ) -> removeSuccessor ( LoopHeader ) ; } numLoopcontPatternMatch += NumCont ; return NumCont


<s> assert ( ! ExitingMBBs . empty ( ) && <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << ExitingMBBs . size ( ) << <STR_LIT> ; ) ; MBBVector ExitBlks ; LoopRep -> getExitBlocks ( ExitBlks ) ; SmallPtrSet < MachineBasicBlock * , <NUM_LIT> > ExitBlkSet ; for ( unsigned i = <NUM_LIT> , e = ExitBlks . size ( ) ; i < e ; ++ i ) ExitBlkSet . insert ( ExitBlks [ i ] ) ; assert ( ExitBlkSet . size ( ) == <NUM_LIT> ) ; MachineBasicBlock * ExitBlk = * ExitBlks . begin ( ) ; assert ( ExitBlk &&


<s> DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> getNumber ( ) << <STR_LIT> << SrcMBB -> getNumber ( ) << <STR_LIT> ; ) ; DstMBB -> splice ( DstMBB -> end ( ) , SrcMBB , SrcMBB -> begin ( ) ,


<s> ) ; SpliceEnd = SrcMBB -> end ( ) ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI ) ; SpliceEnd = BranchMI ; } DEBUG ( dbgs ( ) << <STR_LIT> << DstMBB -> size ( ) <<

<STR_LIT> << SrcMBB -> size ( ) << <STR_LIT> ;


<s> bool <ISA_LIT>CFGStructurizer :: needMigrateBlock ( MachineBasicBlock * MBB ) const { unsigned BlockSizeThreshold = <NUM_LIT> ; unsigned CloneInstrThreshold = <NUM_LIT> ; bool MultiplePreds = MBB && ( MBB -> pred_size ( ) > <NUM_LIT> ) ; if ( ! MultiplePreds ) return


<s> bool <ISA_LIT>CFGStructurizer :: needMigrateBlock ( MachineBasicBlock * MBB ) const { unsigned BlockSizeThreshold = <NUM_LIT> ; unsigned CloneInstrThreshold = <NUM_LIT> ; bool MultiplePreds = MBB && ( MBB -> pred_size ( ) > <NUM_LIT> ) ; if ( ! MultiplePreds ) return false ; unsigned BlkSize = MBB -> size (


<s> MachineBasicBlock * <ISA_LIT>CFGStructurizer :: normalizeInfiniteLoopExit ( MachineLoop * LoopRep ) { MachineBasicBlock * LoopHeader = LoopRep -> getHeader ( ) ; MachineBasicBlock * LoopLatch = LoopRep -> getLoopLatch ( ) ; if ( ! LoopHeader || ! LoopLatch ) return nullptr ; MachineInstr * BranchMI = getLoopendBlockBranchInstr ( LoopLatch ) ; if ( ! BranchMI || ! isUncondBranch ( BranchMI ) ) return nullptr ; MachineBasicBlock * DummyExitBlk = FuncRep -> CreateMachineBasicBlock ( ) ; FuncRep -> push_back ( DummyExitBlk ) ; SHOWNEWBLK ( DummyExitBlk , <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI << <STR_LIT> ; ) ; LLVMContext & Ctx = LoopHeader -> getParent ( ) -> getFunction ( ) -> getContext ( ) ; Ctx . emitError ( <STR_LIT> ) ; return


<s> MachineBasicBlock * <ISA_LIT>CFGStructurizer :: normalizeInfiniteLoopExit ( MachineLoop * LoopRep ) { MachineBasicBlock * LoopHeader = LoopRep -> getHeader ( ) ; MachineBasicBlock * LoopLatch = LoopRep -> getLoopLatch ( ) ; if ( ! LoopHeader || ! LoopLatch ) return nullptr ; MachineInstr * BranchMI = getLoopendBlockBranchInstr ( LoopLatch ) ; if ( ! BranchMI || ! isUncondBranch


<s> NumMatch += loopendPatternMatch ( ) ; NumMatch += serialPatternMatch ( MBB ) ; NumMatch += ifPatternMatch ( MBB


<s> bool Changed = false ; DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; orderBlocks ( FuncRep ) ; SmallVector < MachineBasicBlock * , DEFAULT_VEC_SLOTS > RetBlks ; for ( MachineLoopInfo :: iterator It = MLI -> begin ( ) , E = MLI -> end ( ) ; It != E ; ++ It ) { MachineLoop * LoopRep = ( * It ) ; MBBVector ExitingMBBs ; LoopRep -> getExitingBlocks ( ExitingMBBs ) ; if ( ExitingMBBs . size ( ) == <NUM_LIT> ) { MachineBasicBlock * DummyExitBlk = normalizeInfiniteLoopExit ( LoopRep ) ; if ( DummyExitBlk ) RetBlks . push_back ( DummyExitBlk ) ; } } for ( SmallVectorImpl < MachineBasicBlock * > :: const_iterator It = OrderedBlks . begin ( ) , E = OrderedBlks . end ( ) ; It != E ; ++ It ) { MachineBasicBlock * MBB = * It ; removeUnconditionalBranch ( MBB ) ; removeRedundantConditionalBranch ( MBB ) ; if ( isReturnBlock ( MBB ) ) { RetBlks . push_back ( MBB ) ; } assert ( MBB -> succ_size ( ) <= <NUM_LIT> ) ; } if ( RetBlks . size ( ) >= <NUM_LIT> ) { addDummyExitBlock ( RetBlks ) ; Changed =


<s> static void PrintLoopinfo ( const MachineLoopInfo & LoopInfo ) { for ( MachineLoop :: iterator iter = LoopInfo . begin ( ) , iterEnd = LoopInfo . end ( ) ; iter


<s> void printOrderedBlocks ( ) const { size_t i = <NUM_LIT> ; for ( MBBVector :: const_iterator iterBlk = OrderedBlks . begin ( ) , iterBlkEnd = OrderedBlks . end ( ) ; iterBlk != iterBlkEnd ; ++ iterBlk , ++ i ) { dbgs ( ) << <STR_LIT> << ( * iterBlk ) -> getNumber ( ) ; dbgs ( ) << <STR_LIT> << getSCCNum ( * iterBlk ) << <STR_LIT> << ( * iterBlk ) -> size ( ) << <STR_LIT> ; if ( i != <NUM_LIT> && i % <NUM_LIT> == <NUM_LIT> ) { dbgs ( ) << <STR_LIT> ; }


<s> for ( MBBVector :: const_iterator iterBlk = OrderedBlks . begin ( ) , iterBlkEnd = OrderedBlks . end ( ) ; iterBlk != iterBlkEnd ; ++ iterBlk , ++ i ) { dbgs ( ) << <STR_LIT> << ( * iterBlk ) -> getNumber ( ) ; dbgs ( ) << <STR_LIT> << getSCCNum ( * iterBlk ) << <STR_LIT> << ( * iterBlk ) -> size ( ) << <STR_LIT> ; if ( i != <NUM_LIT> && i % <NUM_LIT> == <NUM_LIT> ) { dbgs ( ) << <STR_LIT> ; } else { dbgs ( ) << <STR_LIT>


<s> BlockInformation * & srcBlkInfo = BlockInfoMap [ MBB ] ; if ( ! srcBlkInfo ) srcBlkInfo = new BlockInformation ( ) ; srcBlkInfo -> SccNum =


<s> BlockInformation * & srcBlkInfo = BlockInfoMap [ MBB ] ; if ( ! srcBlkInfo ) srcBlkInfo = new


<s> MachineBasicBlock * MBB2 = * std :: next ( MBB -> succ_begin ( ) ) ; if ( MBB1 != MBB2 ) return ; MachineInstr * BranchMI = getNormalBlockBranchInstr ( MBB ) ; assert ( BranchMI && isCondBranch ( BranchMI ) ) ; DEBUG ( dbgs ( ) <<


<s> while ( MBB -> succ_size ( ) ) MBB -> removeSuccessor (


<s> DEBUG ( dbgs ( ) << <STR_LIT> << * BranchMI ) ; BranchMI -> eraseFromParent (


<s> if ( BranchMI && isCondBranch ( BranchMI ) && getTrueBranch ( BranchMI

) == OldMBB ) setTrueBranch ( BranchMI , NewBlk ) ;


<s> void <ISA_LIT>CFGStructurizer :: replaceInstrUseOfBlockWith ( MachineBasicBlock * SrcMBB , MachineBasicBlock * OldMBB , MachineBasicBlock * NewBlk ) { MachineInstr * BranchMI = getLoopendBlockBranchInstr ( SrcMBB ) ; if ( BranchMI && isCondBranch ( BranchMI ) && getTrueBranch ( BranchMI ) == OldMBB ) setTrueBranch ( BranchMI ,


<s> DEBUG ( dbgs ( ) << <STR_LIT> << MBB -> getNumber ( ) << <STR_LIT> ; ) ; BlockInformation * & SrcBlkInfo = BlockInfoMap [ MBB ] ; if ( ! SrcBlkInfo ) SrcBlkInfo = new BlockInformation ( ) ; SrcBlkInfo -> IsRetired


<s> while ( I -- ) { if ( I -> getOpcode ( ) == <STR_LIT> ) { switch ( static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . getImm ( ) ) { case OPCODE_IS_ZERO_INT : static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . setImm ( OPCODE_IS_NOT_ZERO_INT ) ; return ; case OPCODE_IS_NOT_ZERO_INT : static_cast < MachineInstr * > ( I ) -> getOperand ( <NUM_LIT> ) . setImm ( OPCODE_IS_ZERO_INT ) ; return ; case OPCODE_IS_ZERO


<s> void ReverseVector ( SmallVectorImpl < NodeT * > & Src ) { size_t sz = Src . size ( ) ; for ( size_t i = <NUM_LIT> ; i < sz / <NUM_LIT> ; ++ i ) { NodeT * t = Src [ i ] ; Src [ i ] = Src [ sz - i - <NUM_LIT> ] ; Src [ sz - i - <NUM_LIT> ]


<s> TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ; TRI = & TII -> getRegisterInfo ( ) ; DEBUG ( MF . dump ( ) ; ) ; OrderedBlks . clear ( ) ; Visited . clear ( ) ; FuncRep = &


<s> int <ISA_LIT>CFGStructurizer :: serialPatternMatch ( MachineBasicBlock * MBB ) { if ( MBB -> succ_size ( ) != <NUM_LIT> ) return <NUM_LIT> ; MachineBasicBlock * childBlk = * MBB -> succ_begin ( ) ; if ( childBlk -> pred_size ( ) != <NUM_LIT> ||

isActiveLoophead ( childBlk ) ) return <NUM_LIT> ;


<s> int OldOpcode = MI -> getOpcode ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; bool UseContinueLogical = ( ( & * ContingMBB -> rbegin ( ) ) == MI ) ; if ( ! UseContinueLogical ) { int BranchOpcode = TrueBranch == ContMBB ? getBranchNzeroOpcode ( OldOpcode ) : getBranchZeroOpcode ( OldOpcode ) ; insertCondBranchBefore ( I , BranchOpcode , DL ) ; insertInstrEnd ( ContingMBB , <STR_LIT> , DL ) ; insertInstrEnd ( ContingMBB , <STR_LIT> , DL ) ; } else { int BranchOpcode = TrueBranch == ContMBB ? getContinueNzeroOpcode ( OldOpcode ) : getContinueZeroOpcode ( OldOpcode ) ; insertCondBranchBefore ( I , BranchOpcode , DL


<s> SrcMBB = * SrcMBB -> succ_begin ( ) ; if ( SrcMBB == DstMBB ) return SinglePath_InPath ; if ( ! AllowSideEntry && SrcMBB -> pred_size (


<s> if ( Pre -> getOpcode ( ) == <STR_LIT> && It -> getOpcode ( ) == <STR_LIT> ) ContInstr . push_back ( Pre ) ; Pre = It ; ++ It ; } for ( unsigned i = <NUM_LIT> ; i < ContInstr . size ( ) ;

++ i ) ContInstr [ i ] -> eraseFromParent ( ) ;


<s> StringMap < int > map ; for ( auto Name : a ) map . insert ( std

:: make_pair ( Name , map . size ( ) ) ) ;


<s> const int Size = getPrinterTable ( ) . size ( ) ; for ( int i = <NUM_LIT> ; i < Size ; ++


<s> void llvm :: dumpAmdKernelCode ( const amd_kernel_code_t * C , raw_ostream & OS , const char * tab ) { const int Size = getPrinterTable ( ) . size ( ) ; for ( int i = <NUM_LIT> ; i < Size ; ++ i ) { OS << tab ; printAmdKernelCodeField ( * C , i , OS ) ; OS << '\n'


<s> Lexer . Lex ( ) ; if ( Lexer . isNot ( AsmToken :: Integer ) ) { Err << <STR_LIT> ; return


<s> if ( Idx < <NUM_LIT> ) { Err << <STR_LIT> << ID ; return false ; } auto Parser = getParserTable ( ) [ Idx ] ; return Parser ? Parser ( C , Lexer , Err


<s> } auto Parser = getParserTable ( ) [ Idx ] ; return Parser ? Parser ( C , Lexer , Err ) :


<s> C .* ptr &= ( T ) ~ Mask ; C .* ptr |= ( T ) ( ( Lexer . getTok ( ) . getIntVal ( ) << shift


<s> if ( ! expectEqualInt ( Lexer , Err ) ) return false ; C .* ptr = ( T ) Lexer . getTok (


<s> if ( ! expectEqualInt ( Lexer , Err ) ) return false ; C .* ptr = ( T ) Lexer . getTok ( )


<s> if ( Printer ) Printer ( get_amd_kernel_code_t_FieldName (


<s> void llvm :: printAmdKernelCodeField ( const amd_kernel_code_t & C , int FldIndex , raw_ostream & OS ) { auto Printer = getPrinterTable ( ) [ FldIndex ] ; if ( Printer ) Printer ( get_amd_kernel_code_t_FieldName ( FldIndex ) ,


<s> static void printBitField ( StringRef Name , const amd_kernel_code_t & c , raw_ostream & OS ) { const auto Mask = ( static_cast < T >


<s> static void printField ( StringRef Name , const amd_kernel_code_t & C , raw_ostream & OS ) { printName ( OS , Name ) << (


<s> for ( unsigned i = <NUM_LIT> ; i < names . size ( ) ; ++ i ) { map . insert ( std :: make_pair ( names [ i ] , i ) ) ; map . insert ( std :: make_pair ( altNames [ i ] , i ) ) ; } return map


<s> static bool expectAbsExpression ( MCAsmParser & MCParser , int64_t & Value , raw_ostream & Err ) { if ( MCParser . getLexer ( ) . isNot ( AsmToken :: Equal ) ) { Err << <STR_LIT> ; return false ; } MCParser . getLexer ( ) . Lex


<s> static int get_amd_kernel_code_t_FieldIndex ( StringRef name ) { static const auto map = createIndexMap ( get_amd_kernel_code_t_FldNames ( ) , get_amd_kernel_code_t_FldAltNames (


<s> } auto Parser = getParserTable ( ) [ Idx ] ; return Parser ? Parser ( C ,


<s> return false ; } auto Parser = getParserTable ( ) [ Idx ] ; return Parser ? Parser ( C , MCParser , Err )


<s> C .* ptr &= ( T ) ~ Mask ; C .* ptr |= ( T ) ( ( Value << shift ) & Mask ) ; return true


<s> int64_t Value = <NUM_LIT> ; if ( ! expectAbsExpression ( MCParser , Value , Err ) )


<s> for ( auto Name : a ) map . insert ( std :: make_pair ( Name , map


<s> void llvm :: dumpAmdKernelCode ( const amd_kernel_code_t * C , raw_ostream & OS , const char * tab ) { const int Size = getPrinterTable ( ) . size ( ) ; for ( int i = <NUM_LIT> ; i < Size ; ++ i ) { OS <<


<s> return false ; } Lexer . Lex ( ) ; if ( Lexer . isNot ( AsmToken :: Integer ) ) { Err << <STR_LIT> ; return false ; } return true


<s> static ArrayRef < ParseFx > getParserTable ( ) { static const ParseFx Table [


<s> static const auto map = createIndexMap

( get_amd_kernel_code_t_FldNames ( ) ) ;


<s> static StringRef get_amd_kernel_code_t_FieldName ( int index ) { return get_amd_kernel_code_t_FldNames ( ) [


<s> static StringRef get_amd_kernel_code_t_FieldName ( int index ) { return get_amd_kernel_code_t_FldNames ( ) [ index + <NUM_LIT>


<s> static StringRef const Table [ ] = { <STR_LIT> , } ; return makeArrayRef (


<s> bool parseBitField ( amd_kernel_code_t & C , MCAsmLexer & Lexer , raw_ostream & Err ) { if ( ! expectEqualInt ( Lexer , Err ) ) return false ; const uint64_t Mask = ( ( UINT64_C ( <NUM_LIT> ) << width ) - <NUM_LIT> ) << shift ; C .* ptr &= ( T ) ~ Mask ; C .* ptr |= ( T ) ( ( Lexer . getTok ( ) . getIntVal ( ) << shift ) & Mask ) ; return


<s> const uint64_t Mask = ( ( UINT64_C ( <NUM_LIT> ) << width ) - <NUM_LIT> ) << shift ; C .* ptr &= ( T ) ~ Mask ; C .* ptr |= ( T ) ( ( Lexer . getTok ( ) . getIntVal ( ) << shift ) & Mask ) ; return


<s> C .* ptr = ( T ) Lexer . getTok ( ) . getIntVal ( ) ; return


<s> bool parseField ( amd_kernel_code_t & C , MCAsmLexer & Lexer , raw_ostream & Err ) { if ( ! expectEqualInt ( Lexer , Err ) ) return false ; C .* ptr = ( T ) Lexer


<s> void llvm :: printAmdKernelCodeField ( const amd_kernel_code_t & C , int FldIndex , raw_ostream & OS ) { auto Printer = getPrinterTable ( ) [ FldIndex ] ; if ( Printer ) Printer ( get_amd_kernel_code_t_FieldName ( FldIndex ) ,


<s> printName ( OS , Name ) << ( int ) ( ( c .* ptr >> shift


<s> void printField ( StringRef Name , const amd_kernel_code_t & C , raw_ostream &


<s> printName ( OS , Name ) << ( int ) ( C


<s> raw_ostream & printName ( raw_ostream & OS ,


<s> if ( ( * Sym ) -> getType ( ) != ELF :: STT_FUNC && ( * Sym ) -> getType ( ) != ELF :: STT_<ISA_LIT>_HSA_KERNEL ) return createError ( <STR_LIT> ) ; return static_cast < const FunctionSym *


<s> if ( ! Sym ) return Sym . takeError ( ) ; if ( ( * Sym ) -> getType ( ) != ELF :: STT_FUNC && ( * Sym ) -> getType ( ) != ELF :: STT_<ISA_LIT>_HSA_KERNEL ) return createError ( <STR_LIT> ) ; return static_cast < const FunctionSym * >


<s> if ( auto E = Obj . takeError ( ) ) return std :: move ( E ) ; std :: unique_ptr < HSACodeObject > Ret ( new HSACodeObject ( std :: move ( * Obj )


<s> return make_range ( functions_begin ( )


<s> if ( ! TextIdxOr ) return functions_end ( ) ; auto TextIdx = TextIdxOr . get ( ) ; return function_sym_iterator ( symbol_begin ( ) , symbol_end ( ) , [ this , TextIdx ] ( const SymbolRef & Sym ) -> bool { auto ExpectedFunction = FunctionSym :: asFunctionSym ( getSymbol ( Sym . getRawDataRefImpl ( ) ) ) ; if ( ! ExpectedFunction ) { consumeError ( ExpectedFunction . takeError ( ) ) ; return false ; } auto Function = ExpectedFunction . get ( ) ; if ( Function -> st_shndx != TextIdx )


<s> return function_sym_iterator ( symbol_end ( ) , symbol_end ( ) , [ ] ( const SymbolRef & ) {


<s> Expected < uint64_t > FunctionSym :: getAddress ( const HSACodeObject * CodeObject , const object :: ELF64LEObjectFile :: Elf_Shdr * Text ) const { assert ( Text )


<s> StringRef getArchitectureName ( ) const { return StringRef ( names + vendor_name_size


<s> auto KernelCodeStartOr = Kernel -> getCodeOffset ( this , * TextOr ) ; if ( ! KernelCodeStartOr ) return KernelCodeStartOr . takeError ( ) ; CodeStart = KernelCodeStartOr . get ( ) ; } else { consumeError ( ExpectedKernel . takeError ( ) ) ; } auto CodeEndI = std :: upper_bound ( FunctionMarkers . begin ( ) , FunctionMarkers . end ( ) , CodeStart ) ; uint64_t CodeEnd = CodeStart ; if ( CodeEndI != FunctionMarkers . end ( )


<s> if ( ! SectionOffsetOr ) return SectionOffsetOr . takeError ( ) ; auto KernelCodeTOr = getAmdKernelCodeT ( CodeObject , Text ) ; if ( ! KernelCodeTOr ) return KernelCodeTOr . takeError ( ) ; return * SectionOffsetOr + ( *

KernelCodeTOr ) -> kernel_code_entry_byte_offset ;


<s> size_t getItemPadSize ( ) const { assert ( Ref . size ( ) >= sizeof


<s> return reinterpret_cast < const ELFNote * > ( N . getDesc ( ) . data ( ) + alignTo ( N .


<s> return reinterpret_cast < const ELFNote * > ( N . getDesc ( ) . data ( ) + alignTo ( N


<s> Expected < const HSACodeObject :: Elf_Shdr * > HSACodeObject :: getNoteSection ( ) const { if ( auto IdxOr = getNoteSectionIdx ( ) ) return getELFFile ( ) . getSection


<s> Expected < uint32_t > HSACodeObject :: getNoteSectionIdx ( ) const { return getSectionIdxByName ( <STR_LIT> :: SectionName


<s> for ( const auto & Sec : * SectionsOr ) { auto SecNameOr = ELF . getSectionName ( Sec ) ; if ( ! SecNameOr ) { return SecNameOr . takeError ( ) ; } else if ( * SecNameOr == Name ) { return Expected < const Elf_Shdr * > ( & Sec ) ; } } return createError (


<s> if ( ! AddressOr ) return AddressOr . takeError ( ) ; return * AddressOr - Text ->


<s> Expected < uint64_t > FunctionSym :: getSectionOffset ( const HSACodeObject * CodeObject , const object :: ELF64LEObjectFile :: Elf_Shdr * Text ) const { assert ( Text ) ; auto AddressOr = getAddress ( CodeObject , Text ) ; if ( ! AddressOr ) return AddressOr . takeError ( ) ; return * AddressOr - Text


<s> return sizeof ( * this ) + alignTo ( namesz , ALIGN


<s> if ( auto IdxOr = getTextSectionIdx ( ) ) return getELFFile ( ) . getSection


<s> HSACodeObject ( object :: ELF64LEObjectFile && Obj ) : object :: ELF64LEObjectFile ( std :: move ( Obj ) ) { InitMarkers (


<s> void HSACodeObject :: InitMarkers ( ) const { auto TextSecOr = getTextSection ( ) ; if ( ! TextSecOr ) return ; auto TextSec = TextSecOr . get ( ) ; FunctionMarkers . push_back ( TextSec -> sh_size ) ; for ( const auto & Sym : functions ( ) ) { auto ExpectedFunction = FunctionSym :: asFunctionSym ( getSymbol ( Sym . getRawDataRefImpl ( ) ) ) ; if ( ! ExpectedFunction ) { consumeError ( ExpectedFunction . takeError


<s> const Item * item ( ) const { return reinterpret_cast < const Item * > ( Ref . data (


<s> HSACodeObject :: note_iterator HSACodeObject :: notes_end ( )


<s> assert ( OldOpndValue -> isImm ( ) ) ; OldOpndVGPR = foldOldOpnd ( OrigMI , OldOpndVGPR , * OldOpndValue ) ; if ( ! OldOpndVGPR . Reg ) { LLVM_DEBUG ( dbgs ( ) <<


<s> initializeGCNDPPCombinePass ( * PassRegistry :: getPassRegistry


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . setPreservesCFG ( ) ; MachineFunctionPass :: getAnalysisUsage (


<s> auto DPP32 = <STR_LIT> ( Op ) ; if ( DPP32 != - <NUM_LIT> ) return


<s> if ( DPP32 != - <NUM_LIT> ) return DPP32 ; auto E32 = <STR_LIT> ( Op ) ; return E32 != - <NUM_LIT> ? <STR_LIT> ( E32 ) :


<s> if ( ! Def ) return nullptr ; switch ( Def -> getOpcode ( ) ) { default : break ; case <STR_LIT> : return nullptr ; case <STR_LIT> : case <STR_LIT> : { auto & Op1 = Def -> getOperand ( <NUM_LIT> ) ; if ( Op1 . isImm ( ) ) return


<s> bool GCNDPPCombine :: hasNoImmOrEqual ( MachineInstr & MI , unsigned OpndName , int64_t Value , int64_t Mask ) const { auto * Imm = TII -> getNamedOperand ( MI , OpndName ) ; if ( ! Imm ) return true


<s> } CombOldVGPR = getRegSubRegPair ( * Src1 ) ; auto MovDst = TII -> getNamedOperand ( MovMI , <STR_LIT> :: vdst ) ; const TargetRegisterClass * RC = MRI -> getRegClass ( MovDst -> getReg ( ) ) ; if ( ! isOfRegClass ( CombOldVGPR , * RC , * MRI ) ) { LLVM_DEBUG ( dbgs (


<s> if ( IsShrinkable ) { assert ( DPP32 == - <NUM_LIT> ) ; auto E32 = <STR_LIT> ( Op ) ; DPP32 = ( E32 == - <NUM_LIT> ) ? - <NUM_LIT> :


<s> return MachineFunctionProperties ( ) . set ( MachineFunctionProperties :: Property


<s> if ( ! ST -> hasDPP ( ) || skipFunction ( MF . getFunction ( ) ) ) return false ; MRI = & MF . getRegInfo ( ) ; TII = ST -> getInstrInfo ( ) ; bool Changed = false ; for ( auto & MBB : MF ) { for ( MachineInstr & MI : llvm :: make_early_inc_range ( llvm :: reverse ( MBB ) ) ) { if ( MI . getOpcode ( ) == <STR_LIT> && combineDPPMov ( MI ) ) { Changed = true ; ++ NumDPPMovsCombined ; } else if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) { if ( ST -> has64BitDPP ( ) && combineDPPMov ( MI ) ) { Changed = true ; ++ NumDPPMovsCombined ; } else


<s> if ( ! <STR_LIT> ( DppCtrl -> getImm ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; return false ; } } auto * RowMaskOpnd = TII -> getNamedOperand ( MovMI , <STR_LIT> :: row_mask ) ; assert ( RowMaskOpnd && RowMaskOpnd -> isImm ( ) ) ; auto * BankMaskOpnd = TII -> getNamedOperand ( MovMI , <STR_LIT> :: bank_mask ) ; assert ( BankMaskOpnd && BankMaskOpnd -> isImm ( ) ) ; const bool MaskAllLanes = RowMaskOpnd -> getImm ( ) == <NUM_LIT> && BankMaskOpnd -> getImm ( ) == <NUM_LIT> ; auto * BCZOpnd = TII -> getNamedOperand ( MovMI , <STR_LIT> :: bound_ctrl ) ; assert ( BCZOpnd && BCZOpnd -> isImm ( ) ) ; bool BoundCtrlZero = BCZOpnd -> getImm ( ) ; auto * OldOpnd = TII -> getNamedOperand ( MovMI , <STR_LIT> :: old ) ; auto * SrcOpnd = TII -> getNamedOperand ( MovMI , <STR_LIT> :: src0 ) ; assert ( OldOpnd && OldOpnd -> isReg ( ) ) ; assert ( SrcOpnd && SrcOpnd -> isReg ( ) ) ; if ( OldOpnd -> getReg ( ) . isPhysical ( ) || SrcOpnd -> getReg ( ) . isPhysical ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } auto * const OldOpndValue = getOldOpndValue ( * OldOpnd ) ; assert ( ! OldOpndValue || OldOpndValue -> isImm ( ) || OldOpndValue == OldOpnd ) ; bool CombBCZ = false ; if ( MaskAllLanes && BoundCtrlZero ) { CombBCZ = true ; } else { if ( ! OldOpndValue || ! OldOpndValue -> isImm ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } if ( OldOpndValue -> getImm ( ) == <NUM_LIT> ) { if ( MaskAllLanes ) { assert ( ! BoundCtrlZero ) ; CombBCZ = true ; } } else if ( BoundCtrlZero ) { assert ( ! MaskAllLanes ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return false ; } } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; if ( ! OldOpndValue ) dbgs ( ) << <STR_LIT> ; else dbgs ( ) << * OldOpndValue ; dbgs ( ) << <STR_LIT> << CombBCZ << '\n' ) ; SmallVector < MachineInstr * , <NUM_LIT> > OrigMIs , DPPMIs ; DenseMap < MachineInstr * , SmallVector < unsigned , <NUM_LIT> >> RegSeqWithOpNos ; auto CombOldVGPR = getRegSubRegPair ( * OldOpnd ) ; if ( CombBCZ && OldOpndValue ) { const TargetRegisterClass * RC = MRI -> getRegClass ( DPPMovReg ) ; CombOldVGPR = RegSubRegPair ( MRI -> createVirtualRegister ( RC ) ) ; auto UndefInst = BuildMI ( * MovMI . getParent ( ) , MovMI , MovMI . getDebugLoc ( ) ,

TII -> get ( <STR_LIT> ) , CombOldVGPR . Reg ) ;


<s> OrigMIs . push_back ( & MovMI ) ; bool Rollback = true ; for ( auto & Use : MRI -> use_nodbg_operands ( DPPMovReg ) ) { Rollback = true ; auto & OrigMI = * Use . getParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << OrigMI ) ; auto OrigOp = OrigMI . getOpcode ( ) ; if ( TII -> isVOP3 ( OrigOp ) ) { if ( ! TII -> hasVALU32BitEncoding ( OrigOp ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; break ; } const int64_t Mask = ~ ( <STR_LIT> | <STR_LIT> ) ; if ( ! hasNoImmOrEqual ( OrigMI , <STR_LIT> :: src0_modifiers , <NUM_LIT> , Mask ) || ! hasNoImmOrEqual ( OrigMI , <STR_LIT> :: src1_modifiers , <NUM_LIT> , Mask ) || ! hasNoImmOrEqual ( OrigMI , <STR_LIT> :: clamp , <NUM_LIT> ) || ! hasNoImmOrEqual ( OrigMI , <STR_LIT> :: omod , <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; break ; } } else if ( ! TII -> isVOP1 ( OrigOp ) && ! TII -> isVOP2 ( OrigOp ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; break ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << OrigMI ) ; if ( & Use == TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src0 ) ) { if ( auto * DPPInst = createDPPInst ( OrigMI , MovMI , CombOldVGPR , OldOpndValue , CombBCZ ) ) { DPPMIs . push_back ( DPPInst ) ; Rollback = false ; } } else if ( OrigMI . isCommutable ( ) && & Use == TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src1 ) ) { auto * BB = OrigMI . getParent ( ) ; auto * NewMI = BB -> getParent ( ) -> CloneMachineInstr ( & OrigMI ) ; BB -> insert ( OrigMI , NewMI ) ; if ( TII -> commuteInstruction ( * NewMI


<s> } if ( ! isIdentityValue ( OrigMI . getOpcode ( ) , OldOpndValue ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return nullptr ; } CombOldVGPR = getRegSubRegPair ( * Src1 ) ; if ( ! isOfRegClass ( CombOldVGPR , <STR_LIT> ,


<s> case <STR_LIT> : if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < int32_t > :: max ( ) ) return true ; break ; case <STR_LIT> : if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < int32_t > :: min ( ) ) return true ; break ; case <STR_LIT> : case <STR_LIT> : if ( OldOpnd -> getImm ( ) == <NUM_LIT> ) return true ; break ; } return


<s> case <STR_LIT> : if ( static_cast < uint32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < uint32_t > :: max ( ) ) return true ; break ; case <STR_LIT> : if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < int32_t > :: max ( ) ) return true ; break ; case <STR_LIT> : if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < int32_t > :: min ( ) ) return true ; break ; case <STR_LIT> : case <STR_LIT> : if ( OldOpnd -> getImm ( ) == <NUM_LIT> ) return


<s> return nullptr ; } auto DPPInst = BuildMI ( * OrigMI . getParent ( ) , OrigMI , OrigMI . getDebugLoc ( ) , TII -> get ( DPPOp ) ) . setMIFlags ( OrigMI . getFlags ( ) ) ; bool Fail = false ; do { auto * Dst = TII -> getNamedOperand ( OrigMI , <STR_LIT> :: vdst ) ; assert ( Dst ) ; DPPInst . add ( * Dst ) ; int NumOperands = <NUM_LIT> ; const int OldIdx = <STR_LIT> ( DPPOp , <STR_LIT> :: old ) ; if ( OldIdx != - <NUM_LIT> ) { assert ( OldIdx == NumOperands ) ; assert ( isOfRegClass ( CombOldVGPR , * MRI -> getRegClass ( TII -> getNamedOperand ( MovMI , <STR_LIT> :: vdst ) -> getReg ( ) ) , * MRI ) ) ; auto * Def = getVRegSubRegDef ( CombOldVGPR , * MRI ) ; DPPInst . addReg ( CombOldVGPR . Reg , Def ? <NUM_LIT> : RegState :: Undef , CombOldVGPR . SubReg ) ; ++ NumOperands ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; Fail = true ; break ; } if ( auto * Mod0 = TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src0_modifiers ) ) { assert ( NumOperands == <STR_LIT> ( DPPOp , <STR_LIT> :: src0_modifiers ) ) ; assert ( <NUM_LIT> == ( Mod0 -> getImm ( ) & ~ ( <STR_LIT> | <STR_LIT> ) ) ) ; DPPInst . addImm ( Mod0 -> getImm ( ) ) ; ++ NumOperands ; } else if ( <STR_LIT> ( DPPOp , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ) { DPPInst . addImm ( <NUM_LIT> ) ; ++ NumOperands ; } auto * Src0 = TII -> getNamedOperand ( MovMI , <STR_LIT> :: src0 ) ; assert ( Src0 ) ; if ( ! TII -> isOperandLegal ( * DPPInst . getInstr ( ) , NumOperands , Src0 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Fail = true ; break ; } DPPInst . add ( * Src0 ) ; DPPInst -> getOperand ( NumOperands ) . setIsKill ( false ) ; ++ NumOperands ; if ( auto * Mod1 = TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src1_modifiers ) ) { assert ( NumOperands == <STR_LIT> ( DPPOp , <STR_LIT> :: src1_modifiers ) ) ; assert ( <NUM_LIT> == ( Mod1 -> getImm ( ) & ~ ( <STR_LIT> | <STR_LIT> ) ) ) ; DPPInst . addImm ( Mod1 -> getImm ( ) ) ; ++ NumOperands ; } else if ( <STR_LIT> ( DPPOp , <STR_LIT> :: src1_modifiers


<s> TII = ST -> getInstrInfo ( ) ; bool Changed = false ; for ( auto & MBB : MF ) { for ( auto I = MBB . rbegin ( ) , E = MBB . rend ( ) ; I != E ; ) { auto & MI = * I ++ ; if ( MI . getOpcode ( ) == <STR_LIT> && combineDPPMov ( MI ) ) { Changed = true ; ++ NumDPPMovsCombined ; } else if ( MI . getOpcode ( ) == <STR_LIT> ) { if ( ST -> has64BitDPP ( ) && combineDPPMov ( MI ) ) { Changed = true ; ++ NumDPPMovsCombined ; } else


<s> MRI = & MF . getRegInfo ( ) ; TII = ST . getInstrInfo ( ) ; bool Changed = false ; for ( auto & MBB : MF ) { for ( auto I = MBB . rbegin ( ) , E = MBB . rend ( ) ; I != E ; ) { auto & MI = * I


<s> ++ NumOperands ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ) ; Fail = true ; break ; } if ( auto * Mod0 = TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src0_modifiers ) ) { assert ( NumOperands == <STR_LIT> ( DPPOp , <STR_LIT> :: src0_modifiers ) ) ; assert ( <NUM_LIT> == ( Mod0 -> getImm ( ) & ~ ( <STR_LIT> | <STR_LIT> ) ) ) ; DPPInst . addImm ( Mod0 -> getImm ( ) ) ; ++ NumOperands ; } else if ( <STR_LIT> ( DPPOp , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ) { DPPInst . addImm ( <NUM_LIT> ) ; ++ NumOperands ; } auto * Src0 = TII -> getNamedOperand ( MovMI , <STR_LIT> :: src0 ) ; assert ( Src0 ) ; if ( ! TII -> isOperandLegal ( * DPPInst . getInstr ( ) , NumOperands , Src0 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Fail = true ; break ; } DPPInst . add ( * Src0 ) ; DPPInst -> getOperand ( NumOperands ) . setIsKill ( false ) ; ++ NumOperands ; if ( auto * Mod1 = TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src1_modifiers ) ) { assert ( NumOperands == <STR_LIT> ( DPPOp , <STR_LIT> :: src1_modifiers ) ) ; assert ( <NUM_LIT> == ( Mod1 -> getImm ( ) & ~ ( <STR_LIT> | <STR_LIT> ) ) ) ; DPPInst . addImm ( Mod1 -> getImm ( ) ) ; ++ NumOperands ; } else if ( <STR_LIT> ( DPPOp , <STR_LIT> :: src1_modifiers ) != - <NUM_LIT> ) { DPPInst . addImm ( <NUM_LIT> ) ; ++ NumOperands ; } if ( auto * Src1 = TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src1 ) ) { if ( ! TII -> isOperandLegal ( * DPPInst . getInstr ( ) , NumOperands , Src1 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Fail = true ; break ; } DPPInst . add ( * Src1 ) ; ++ NumOperands ; } if ( auto * Src2 = TII -> getNamedOperand ( OrigMI , <STR_LIT> :: src2 ) ) { if ( ! TII -> getNamedOperand ( * DPPInst . getInstr ( ) , <STR_LIT> :: src2 ) || ! TII -> isOperandLegal ( * DPPInst . getInstr ( ) , NumOperands , Src2 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; Fail = true ; break ; } DPPInst . add ( * Src2 ) ; } DPPInst . add ( * TII -> getNamedOperand ( MovMI , <STR_LIT>


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return nullptr ; } if ( ! isIdentityValue ( OrigMI . getOpcode ( ) , OldOpndValue ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return nullptr ; } CombOldVGPR = getRegSubRegPair ( * Src1 ) ; if ( ! isOfRegClass ( CombOldVGPR , <STR_LIT> , * MRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; return nullptr


<s> auto DPP32 = <STR_LIT> ( Op ) ; if ( DPP32 == - <NUM_LIT> ) { auto E32 = <STR_LIT> ( Op ) ; DPP32 = ( E32 == - <NUM_LIT> ) ? - <NUM_LIT> : <STR_LIT>


<s> DPP32 = ( E32 == - <NUM_LIT> ) ? - <NUM_LIT> : <STR_LIT> ( E32 ) ; } return ( DPP32 == - <NUM_LIT> || TII -> pseudoToMCOpcode ( DPP32 ) == - <NUM_LIT> ) ?


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( static_cast < uint32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < uint32_t > :: max ( ) ) return true ; break ; case <STR_LIT> : case <STR_LIT> : if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std :: numeric_limits < int32_t > :: max ( ) ) return true ; break ; case <STR_LIT> : case <STR_LIT> : if ( static_cast < int32_t > ( OldOpnd -> getImm ( ) ) == std ::

numeric_limits < int32_t > :: min ( ) ) return true ;


<s> assert ( MRI -> isSSA ( ) && <STR_LIT> ) ; bool Changed = false ; for ( auto & MBB : MF ) { for ( auto I = MBB . rbegin ( ) , E = MBB . rend ( ) ; I != E ; ) { auto & MI = * I ++ ; if ( MI . getOpcode ( ) == <STR_LIT> && combineDPPMov ( MI ) ) { Changed = true


<s> if ( Op . isReg ( ) ) addRegUnits ( TRI , Set , Op . getReg


<s> static void addRegUnits ( const <ISA_LIT>RegisterInfo & TRI , BitVector


<s> } EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; } EmittedInstrs . resize ( getMaxLookAhead ( ) ) ; CurrCycleInstr =


<s> return ! <ISA_LIT>InstrInfo :: isSMRD ( *


<s> static bool breaksSMEMSoftClause ( MachineInstr * MI ) { return ! <ISA_LIT>InstrInfo :: isSMRD


<s> static bool breaksVMEMSoftClause ( MachineInstr *


<s> static bool breaksVMEMSoftClause ( MachineInstr * MI ) { return ! <ISA_LIT>InstrInfo :: isVMEM ( * MI ) &&

! <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ;


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; int DppVgprWaitStates = <NUM_LIT> ; int DppExecWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( const MachineInstr & MI ) { return TII -> isVALU ( MI ) ; } ; for ( const MachineOperand & Use : DPP -> uses ( ) ) { if ( ! Use . isReg ( ) || ! TRI -> isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; int WaitStatesNeededForUse = DppVgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , [ ] ( const MachineInstr & ) { return true ; } , DppVgprWaitStates ) ; WaitStatesNeeded = std :: max

( WaitStatesNeeded , WaitStatesNeededForUse ) ;


<s> auto IsHazardFn = [ ] ( const MachineInstr & I ) { if ( ! <ISA_LIT>InstrInfo :: isVMEM ( I ) && ! <ISA_LIT>InstrInfo :: isFLAT ( I ) ) return false ; return <ISA_LIT>InstrInfo :: isFPAtomic ( I ) ; } ; auto IsExpiredFn = [ ] ( const MachineInstr & MI , int WaitStates ) { if ( WaitStates >= <NUM_LIT> || <ISA_LIT>InstrInfo :: isVALU ( MI ) ) return true ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case


<s> auto IsHazardFn = [ TII , GetRegHWReg ] ( const MachineInstr & MI ) { return GetRegHWReg == getHWReg ( TII , MI ) ; } ; int WaitStatesNeeded = getWaitStatesSinceSetReg ( IsHazardFn ,


<s> int WaitStatesNeeded = <NUM_LIT> ; for ( unsigned I = InlineAsm :: MIOp_FirstOperand , E = IA -> getNumOperands ( ) ; I != E ; ++ I ) { const MachineOperand & Op = IA -> getOperand ( I ) ; if ( Op . isReg ( ) && Op . isDef (


<s> int GCNHazardRecognizer :: checkMAIHazards ( MachineInstr * MI ) { assert ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) ) ; return ST . hasGFX90AInsts ( ) ? checkMAIHazards90A ( MI ) : checkMAIHazards908 ( MI


<s> if ( ! IsMFMAFn ( MI ) ) return false ; NeighborMFMALatency = this -> getMFMAPipelineWaitStates ( MI ) ; return true ; } ; const int MaxMFMAPipelineWaitStates = <NUM_LIT> ; int WaitStatesSinceNeighborMFMA = getWaitStatesSince ( IsNeighboringMFMA , MaxMFMAPipelineWaitStates ) ; int NeighborMFMAPaddingNeeded = ( NeighborMFMALatency * MFMAPaddingRatio / <NUM_LIT> ) - WaitStatesSinceNeighborMFMA ; return std :: max ( <NUM_LIT> , NeighborMFMAPaddingNeeded


<s> if ( ! ST . hasNSAtoVMEMBug ( ) ) return <NUM_LIT> ; if ( ! <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) && ! <ISA_LIT>InstrInfo :: isMTBUF ( * MI ) ) return <NUM_LIT> ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const auto * Offset = TII -> getNamedOperand ( * MI , <STR_LIT> :: offset ) ; if ( ! Offset || ( Offset -> getImm ( ) & <NUM_LIT> ) == <NUM_LIT> ) return <NUM_LIT> ; auto IsHazardFn = [ TII ] ( const MachineInstr & I ) { if ( ! <ISA_LIT>InstrInfo :: isMIMG ( I ) ) return


<s> const int ReadM0WaitStates = <NUM_LIT> ; auto IsHazardFn = [ TII ] ( const MachineInstr & MI ) { return TII -> isSALU ( MI ) ; } ; return ReadM0WaitStates - getWaitStatesSinceDef ( <STR_LIT> , IsHazardFn , ReadM0WaitStates


<s> int GCNHazardRecognizer :: checkRFEHazards ( MachineInstr * RFE ) { if ( ! ST . hasRFEHazards ( ) ) return <NUM_LIT> ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo (


<s> int GCNHazardRecognizer :: checkSetRegHazards ( MachineInstr * SetRegInstr ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; unsigned HWReg = getHWReg ( TII , * SetRegInstr ) ; const int SetRegWaitStates = ST . getSetRegWaitStates ( ) ; auto IsHazardFn = [ TII , HWReg ] ( const MachineInstr & MI ) { return HWReg == getHWReg ( TII , MI ) ; } ; int WaitStatesNeeded = getWaitStatesSinceSetReg ( IsHazardFn , SetRegWaitStates


<s> } if ( ClauseDefs . none ( ) ) return <NUM_LIT> ; if ( MEM -> mayStore ( ) ) return <NUM_LIT> ; addClauseInst ( * MEM ) ; return ClauseDefs . anyCommon ( ClauseUses ) ? <NUM_LIT> :


<s> int GCNHazardRecognizer :: checkSoftClauseHazards ( MachineInstr * MEM ) { if ( ! ST . isXNACKEnabled ( ) ) return <NUM_LIT> ; bool IsSMRD = TII . isSMRD ( * MEM ) ; resetClause ( ) ; for ( MachineInstr * MI : EmittedInstrs ) { if ( ! MI ) break ; if ( IsSMRD ? breaksSMEMSoftClause ( MI ) : breaksVMEMSoftClause ( MI ) ) break ; addClauseInst ( * MI ) ; } if ( ClauseDefs . none ( ) ) return <NUM_LIT> ; if ( MEM -> mayStore ( ) ) return <NUM_LIT> ; addClauseInst ( *


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const int VALUWaitStates = ST . hasGFX940Insts ( ) ? <NUM_LIT> : <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; if ( ! TRI -> isVectorRegister ( MRI , Def . getReg ( ) ) ) return WaitStatesNeeded ; Register Reg = Def . getReg ( ) ; auto IsHazardFn = [ this , Reg , TRI ] ( const MachineInstr & MI ) { int DataIdx = createsVALUHazard ( MI ) ; return DataIdx >= <NUM_LIT> && TRI -> regsOverlap ( MI . getOperand ( DataIdx ) . getReg ( ) , Reg ) ; } ; int WaitStatesNeededForDef = VALUWaitStates - getWaitStatesSince ( IsHazardFn , VALUWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded


<s> int GCNHazardRecognizer :: checkVALUHazardsHelper ( const MachineOperand & Def , const MachineRegisterInfo & MRI ) { const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const int VALUWaitStates = ST . hasGFX940Insts ( ) ? <NUM_LIT> : <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; if ( ! TRI -> isVectorRegister ( MRI , Def

. getReg ( ) ) ) return WaitStatesNeeded ;


<s> void GCNHazardRecognizer :: EmitInstruction ( MachineInstr * MI ) { CurrCycleInstr =


<s> void GCNHazardRecognizer :: EmitNoop ( ) { EmittedInstrs . push_front ( nullptr


<s> fixVcmpxPermlaneHazards ( MI ) ; fixSMEMtoVectorWriteHazards ( MI ) ; fixVcmpxExecWARHazard ( MI ) ; fixLdsBranchVmemWARHazard ( MI


<s> auto InstType = IsHazardInst ( * MI ) ; if ( ! InstType ) return false ; auto IsExpiredFn = [ & IsHazardInst ] ( const MachineInstr & I , int ) { return IsHazardInst ( I ) || ( I . getOpcode ( ) == <STR_LIT> && I . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && ! I . getOperand ( <NUM_LIT> ) . getImm ( ) ) ; } ; auto IsHazardFn = [ InstType , & IsHazardInst ] ( const MachineInstr & I ) { if ( ! I . isBranch ( ) ) return false ; auto IsHazardFn = [ InstType , IsHazardInst ] ( const MachineInstr & I ) { auto InstType2 = IsHazardInst ( I ) ; return InstType2 && InstType != InstType2 ; } ; auto IsExpiredFn = [ InstType , & IsHazardInst ] ( const MachineInstr & I , int ) { auto InstType2 = IsHazardInst ( I ) ; if ( InstType == InstType2 ) return true ; return I . getOpcode ( ) == <STR_LIT> && I . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && ! I . getOperand ( <NUM_LIT> ) . getImm ( ) ; } ; return :: getWaitStatesSince ( IsHazardFn , & I , IsExpiredFn ) != std :: numeric_limits < int > :: max ( ) ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ; const <ISA_LIT>InstrInfo * TII = ST


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <STR_LIT> IV = <STR_LIT> ( ST . getCPU ( ) ) ; const MachineOperand * SDST = TII -> getNamedOperand ( * MI , SDSTName ) ; if ( ! SDST ) { for ( const auto & MO : MI -> implicit_operands ( ) ) { if ( MO . isDef ( ) && TRI -> isSGPRClass ( TRI -> getPhysRegClass ( MO . getReg ( ) ) ) ) { SDST = & MO ; break ; } } } if ( ! SDST ) return false ; const Register SDSTReg = SDST -> getReg ( ) ; auto IsHazardFn = [ SDSTReg , TRI ] ( const MachineInstr & I ) { return <ISA_LIT>InstrInfo :: isSMRD ( I ) && I . readsRegister ( SDSTReg , TRI ) ; } ; auto IsExpiredFn = [ TII , IV ] ( const MachineInstr & MI , int ) { if ( TII -> isSALU ( MI ) ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT>


<s> auto IsExpiredFn = [ TII , TRI ] ( const MachineInstr & MI , int ) { if ( <ISA_LIT>InstrInfo :: isVALU ( MI ) ) { if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ) return true ; for ( auto MO : MI . implicit_operands ( ) ) if ( MO . isDef ( ) && TRI -> isSGPRClass ( TRI -> getPhysRegClass ( MO . getReg ( ) ) ) ) return true ; } if ( MI . getOpcode ( ) == <STR_LIT> && ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> ) == <NUM_LIT> ) return true ; return false ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits <


<s> auto IsExpiredFn = [ ] ( const MachineInstr & MI , int ) { unsigned Opc = MI . getOpcode ( ) ; return <ISA_LIT>InstrInfo :: isVALU ( MI ) && Opc != <STR_LIT> && Opc != <STR_LIT> && Opc != <STR_LIT> ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ; auto * Src0 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) ; Register Reg = Src0 -> getReg ( ) ; bool IsUndef = Src0 -> isUndef ( ) ; BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( Reg ,

RegState :: Define | ( IsUndef ? RegState :: Dead : <NUM_LIT> ) ) . addReg ( Reg , IsUndef ? RegState :: Undef : RegState :: Kill ) ;


<s> auto IsHazardFn = [ TII ] ( const MachineInstr & MI ) { return TII -> isVOPC ( MI ) ; } ; auto IsExpiredFn = [ ] ( const MachineInstr & MI , int ) { unsigned Opc = MI . getOpcode ( ) ; return <ISA_LIT>InstrInfo :: isVALU ( MI ) && Opc != <STR_LIT> && Opc != <STR_LIT> && Opc != <STR_LIT> ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ; auto * Src0 = TII -> getNamedOperand (


<s> bool GCNHazardRecognizer :: fixVMEMtoScalarWriteHazards ( MachineInstr * MI ) { if ( ! ST . hasVMEMtoScalarWriteHazard ( ) ) return false ; if ( ! <ISA_LIT>InstrInfo :: isSALU ( * MI ) && ! <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) return false ; if ( MI -> getNumDefs ( ) == <NUM_LIT> ) return false ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; auto IsHazardFn = [ TRI , MI ] ( const MachineInstr & I ) { if ( ! <ISA_LIT>InstrInfo :: isVMEM ( I ) && ! <ISA_LIT>InstrInfo :: isDS ( I ) && ! <ISA_LIT>InstrInfo :: isFLAT ( I ) ) return false ; for ( const MachineOperand & Def : MI -> defs ( ) ) { const MachineOperand * Op = I . findRegisterUseOperand ( Def . getReg ( ) , false , TRI ) ; if ( ! Op ) continue ; return true ; } return false ; } ; auto IsExpiredFn = [ ] ( const MachineInstr & MI , int ) { return <ISA_LIT>InstrInfo :: isVALU ( MI ) || ( MI . getOpcode ( ) == <STR_LIT> && ! MI . getOperand ( <NUM_LIT> ) . getImm ( ) ) || ( MI . getOpcode ( ) == <STR_LIT> && MI . getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc ( )

, TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ;


<s> const MachineOperand * RegOp = TII -> getNamedOperand ( RegInstr , <STR_LIT> :: simm16 ) ; return RegOp -> getImm ( ) & <STR_LIT>


<s> const MCSchedClassDesc * SC = TSchedModel . resolveSchedClass ( & MI ) ; assert ( TSchedModel . getWriteProcResBegin ( SC ) != TSchedModel . getWriteProcResEnd ( SC ) ) ; return TSchedModel . getWriteProcResBegin ( SC )


<s> unsigned GCNHazardRecognizer :: getMFMAPipelineWaitStates ( const MachineInstr & MI ) const { const MCSchedClassDesc * SC = TSchedModel . resolveSchedClass ( & MI ) ; assert ( TSchedModel . getWriteProcResBegin ( SC ) != TSchedModel


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; auto IsHazardFn = [ IsHazardDef , TRI , Reg ] ( const


<s> int GCNHazardRecognizer :: getWaitStatesSinceSetReg ( IsHazardFn IsHazard , int Limit ) { auto IsHazardFn = [ IsHazard ] ( const MachineInstr & MI ) { return isSSetReg ( MI . getOpcode ( )


<s> static void insertNoopsInBundle ( MachineInstr * MI , const <ISA_LIT>InstrInfo & TII , unsigned Quantity ) { while ( Quantity > <NUM_LIT> ) { unsigned Arg = std :: min


<s> static bool isRFE ( unsigned Opcode ) { return Opcode ==


<s> static bool isRWLane ( unsigned Opcode


<s> return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return false ; default : if ( TII . isDS ( MI . getOpcode ( ) ) ) { int GDS = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: gds ) ; if ( MI . getOperand ( GDS ) . getImm


<s> static bool isSGetReg ( unsigned Opcode ) { return Opcode == <STR_LIT>


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : return false


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return


<s> static bool isSSetReg ( unsigned Opcode ) { switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case


<s> if ( ! <ISA_LIT>InstrInfo :: isMAI ( MI ) || isDGEMM ( Opcode ) || Opcode ==

<STR_LIT> || Opcode == <STR_LIT> ) return false ;


<s> if ( Arg . getAsInteger ( <NUM_LIT> , Value ) ) return O . error ( <STR_LIT> + Arg + <STR_LIT> ) ; if ( Value > <NUM_LIT> ) return O . error ( <STR_LIT> + Arg + <STR_LIT> ) ; return


<s> bool parse ( cl :: Option & O , StringRef ArgName , StringRef Arg , unsigned & Value ) { if ( Arg . getAsInteger ( <NUM_LIT> , Value ) ) return O . error ( <STR_LIT> + Arg + <STR_LIT> ) ; if ( Value > <NUM_LIT> ) return O . error ( <STR_LIT> + Arg +


<s> unsigned GCNHazardRecognizer :: PreEmitNoops ( MachineInstr * MI ) { IsHazardRecognizerMode = true ; CurrCycleInstr = MI ; unsigned W = PreEmitNoopsCommon (


<s> if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) || <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) || <ISA_LIT>InstrInfo :: isDS ( * MI ) || <ISA_LIT>InstrInfo :: isEXP ( * MI ) ) && checkMAIVALUHazards ( MI ) > <NUM_LIT> ) WaitStates = std :: max ( WaitStates , checkMAIVALUHazards ( MI ) ) ; if ( MI -> isInlineAsm ( ) ) return std :: max ( WaitStates , checkInlineAsmHazards ( MI ) ) ; if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards ( MI ) ) ; if ( ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) ) || ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) ) || ( ST . hasReadM0LdsDmaHazard ( ) && isLdsDma ( * MI ) ) || ( ST . hasReadM0LdsDirectHazard ( ) && MI -> readsRegister ( <STR_LIT> ) ) ) return std :: max ( WaitStates , checkReadM0Hazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) ) return std :: max ( WaitStates , checkMAIHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) || <ISA_LIT>InstrInfo :: isDS ( * MI ) ) return std :: max

( WaitStates , checkMAILdStHazards ( MI ) ) ;


<s> int WaitStates = <NUM_LIT> ; if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) ) return std :: max ( WaitStates , checkSMRDHazards ( MI ) ) ; if ( ST . hasNSAtoVMEMBug ( ) ) WaitStates = std :: max ( WaitStates , checkNSAtoVMEMHazard ( MI ) ) ; WaitStates = std :: max ( WaitStates , checkFPAtomicToDenormModeHazard ( MI ) ) ; if ( ST . hasNoDataDepHazard ( ) ) return WaitStates ; if ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVMEMHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates =

std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ;


<s> CurrCycleInstr = & * MI ; unsigned WaitStates = PreEmitNoopsCommon ( CurrCycleInstr ) ; if ( IsHazardRecognizerMode ) { fixHazards ( CurrCycleInstr ) ; insertNoopsInBundle ( CurrCycleInstr , TII , WaitStates ) ; } for ( unsigned i = <NUM_LIT> , e = std :: min ( WaitStates , MaxLookAhead - <NUM_LIT> ) ; i < e ; ++ i ) EmittedInstrs . push_front ( nullptr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; EmittedInstrs . resize ( MaxLookAhead ) ; } CurrCycleInstr = nullptr


<s> MachineBasicBlock :: instr_iterator E = CurrCycleInstr -> getParent ( ) -> instr_end ( ) ; for ( ; MI != E && MI -> isInsideBundle ( ) ; ++ MI ) { CurrCycleInstr = & * MI ; unsigned WaitStates = PreEmitNoopsCommon ( CurrCycleInstr ) ; if ( IsHazardRecognizerMode ) { fixHazards ( CurrCycleInstr ) ; insertNoopsInBundle ( CurrCycleInstr , TII , WaitStates ) ; } for ( unsigned i = <NUM_LIT> , e = std :: min ( WaitStates , MaxLookAhead - <NUM_LIT> ) ; i < e ; ++ i ) EmittedInstrs . push_front ( nullptr ) ; EmittedInstrs . push_front


<s> void GCNHazardRecognizer :: RecedeCycle ( ) { llvm_unreachable ( <STR_LIT>


<s> void GCNHazardRecognizer :: Reset ( ) { EmittedInstrs . clear (


<s> ClauseUses . reset ( ) ; ClauseDefs . reset


<s> void resetClause ( ) { ClauseUses . reset ( ) ; ClauseDefs . reset


<s> for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; } EmittedInstrs . resize ( getMaxLookAhead ( ) ) ; CurrCycleInstr = nullptr


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const int DivFMasWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ; int WaitStatesNeeded = getWaitStatesSinceDef ( <STR_LIT> , IsHazardDefFn ) ; return DivFMasWaitStates -


<s> int GCNHazardRecognizer :: checkDivFMasHazards ( MachineInstr * DivFMas ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const int DivFMasWaitStates = <NUM_LIT>


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; int DppVgprWaitStates = <NUM_LIT> ; int DppExecWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; auto IsHazardDefFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ; for ( const MachineOperand & Use :


<s> int GCNHazardRecognizer :: checkGetRegHazards ( MachineInstr * GetRegInstr ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; unsigned GetRegHWReg = getHWReg ( TII , * GetRegInstr


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const int SMovRelWaitStates = <NUM_LIT> ; auto IsHazardFn = [ TII ] ( MachineInstr * MI ) { return TII -> isSALU (


<s> int GCNHazardRecognizer :: checkRFEHazards ( MachineInstr * RFE ) { if ( ST . getGeneration ( ) < <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return <NUM_LIT> ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const int RFEWaitStates =


<s> const MachineOperand * LaneSelectOp = TII -> getNamedOperand ( * RWLane , <STR_LIT> :: src1 ) ; if ( ! LaneSelectOp -> isReg ( ) || ! TRI -> isSGPRReg ( MRI , LaneSelectOp -> getReg ( ) ) ) return <NUM_LIT> ; unsigned LaneSelectReg = LaneSelectOp -> getReg ( ) ; auto IsHazardFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU ( * MI ) ; } ; const int RWLaneWaitStates


<s> unsigned HWReg = getHWReg ( TII , * SetRegInstr ) ; const int SetRegWaitStates = ST . getGeneration ( ) <= <ISA_LIT>Subtarget :: SEA_ISLANDS ? <NUM_LIT> : <NUM_LIT> ; auto IsHazardFn = [ TII , HWReg ] (


<s> int GCNHazardRecognizer :: checkSMRDHazards ( MachineInstr * SMRD ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; int WaitStatesNeeded = <NUM_LIT> ; WaitStatesNeeded = checkSoftClauseHazards ( SMRD ) ; if ( ST . getGeneration ( ) != <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return WaitStatesNeeded ; int SmrdSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isVALU ( * MI ) ; } ; auto IsBufferHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isSALU ( * MI ) ; } ; bool IsBufferSMRD = TII . isBufferSMRD ( * SMRD ) ; for ( const MachineOperand & Use : SMRD -> uses (


<s> int WaitStatesNeeded = <NUM_LIT> ; WaitStatesNeeded = checkSoftClauseHazards ( SMRD ) ; if ( ST . getGeneration ( ) != <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return WaitStatesNeeded ; int SmrdSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isVALU ( * MI ) ; } ; auto IsBufferHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isSALU ( * MI ) ; } ; bool IsBufferSMRD = TII . isBufferSMRD ( * SMRD ) ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) { if ( ! Use . isReg ( ) ) continue ; int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const int VALUWaitStates = <NUM_LIT> ; int WaitStatesNeeded = <NUM_LIT> ; if ( ! TRI -> isVGPR ( MRI , Def . getReg ( ) ) ) return WaitStatesNeeded ; unsigned Reg = Def . getReg ( ) ; auto IsHazardFn = [ this , Reg , TRI ] ( MachineInstr * MI ) { int DataIdx = createsVALUHazard ( * MI ) ; return DataIdx >= <NUM_LIT> && TRI -> regsOverlap ( MI -> getOperand ( DataIdx ) .


<s> if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkAnyInstHazards ( MI ) > <NUM_LIT> )


<s> if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI )


<s> if ( IsHazard ( MI ) ) return WaitStates ; unsigned Opcode = MI -> getOpcode ( ) ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT>


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; auto IsHazardFn = [ IsHazardDef , TRI , Reg ] ( MachineInstr * MI ) { return IsHazardDef ( MI ) && MI -> modifiesRegister ( Reg , TRI


<s> if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( MI -> isInlineAsm ( ) ) return std :: max ( WaitStates , checkInlineAsmHazards ( MI ) ) ; if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max

( WaitStates , checkSetRegHazards ( MI ) ) ;


<s> for ( const MachineOperand & Use : MI -> uses ( ) ) { if ( ! Use . isReg ( ) || TRI -> isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; auto IsHazardFn = [ ] ( MachineInstr * MI ) { return MI -> getOpcode (


<s> auto IsHazardFn = [ ] ( MachineInstr * I ) { if ( ! <ISA_LIT>InstrInfo :: isVMEM ( * I ) && ! <ISA_LIT>InstrInfo :: isFLAT ( * I ) ) return false ; return <ISA_LIT>InstrInfo :: isFPAtomic ( * I ) ; } ; auto IsExpiredFn = [ ] ( MachineInstr * MI , int WaitStates ) { if ( WaitStates >= <NUM_LIT> || <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) return true ; switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; default : break ; } return false


<s> int NSAtoVMEMWaitStates = <NUM_LIT> ; if ( ! ST . hasNSAtoVMEMBug ( ) ) return <NUM_LIT> ; if ( ! <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) && ! <ISA_LIT>InstrInfo :: isMTBUF ( * MI ) ) return <NUM_LIT> ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const auto * Offset = TII -> getNamedOperand ( * MI , <STR_LIT> :: offset ) ; if ( ! Offset || ( Offset -> getImm ( ) & <NUM_LIT> ) == <NUM_LIT> ) return <NUM_LIT> ; auto IsHazardFn = [ TII ] ( MachineInstr * I ) { if ( ! <ISA_LIT>InstrInfo :: isMIMG ( * I ) ) return false ; const <STR_LIT> * Info = <STR_LIT> ( I ->


<s> auto IsHazardFn = [ TII ] ( MachineInstr * MI ) { return getHWReg ( TII , * MI ) == <STR_LIT> :: ID_TRAPSTS ; } ; int WaitStatesNeeded = getWaitStatesSinceSetReg ( IsHazardFn , RFEWaitStates ) ; return RFEWaitStates


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const MachineOperand * LaneSelectOp = TII -> getNamedOperand ( * RWLane , <STR_LIT> :: src1 ) ; if ( ! LaneSelectOp -> isReg ( ) || ! TRI -> isSGPRReg ( MRI , LaneSelectOp -> getReg ( ) ) ) return <NUM_LIT> ; Register LaneSelectReg = LaneSelectOp -> getReg ( ) ; auto IsHazardFn = [ TII ] ( MachineInstr * MI ) { return TII -> isVALU (


<s> int GCNHazardRecognizer :: checkSetRegHazards ( MachineInstr * SetRegInstr ) { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; unsigned HWReg = getHWReg ( TII , * SetRegInstr ) ; const int SetRegWaitStates = ST . getSetRegWaitStates ( ) ; auto IsHazardFn = [ TII , HWReg ] ( MachineInstr * MI ) { return HWReg == getHWReg ( TII , * MI ) ; } ; int WaitStatesNeeded = getWaitStatesSinceSetReg (


<s> auto IsBufferHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isSALU ( * MI ) ; } ; bool IsBufferSMRD = TII . isBufferSMRD ( * SMRD ) ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) { if ( ! Use . isReg ( ) ) continue ; int WaitStatesNeededForUse = SmrdSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn , SmrdSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; if ( IsBufferSMRD )


<s> Register Reg = Def . getReg ( ) ; auto IsHazardFn = [ this , Reg , TRI ] ( MachineInstr * MI ) { int DataIdx = createsVALUHazard ( * MI ) ; return DataIdx >= <NUM_LIT> && TRI -> regsOverlap ( MI -> getOperand ( DataIdx ) . getReg ( ) , Reg ) ; } ; int WaitStatesNeededForDef = VALUWaitStates - getWaitStatesSince ( IsHazardFn , VALUWaitStates ) ; WaitStatesNeeded = std :: max (

WaitStatesNeeded , WaitStatesNeededForDef ) ;


<s> if ( ! ST . hasVMEMReadSGPRVALUDefHazard ( ) ) return <NUM_LIT> ; int WaitStatesNeeded = checkSoftClauseHazards ( VMEM ) ; const int VmemSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isVALU ( * MI ) ; } ; for ( const MachineOperand & Use : VMEM -> uses ( ) ) { if ( ! Use . isReg ( ) || TRI . isVGPR ( MF .

getRegInfo ( ) , Use . getReg ( ) ) ) continue ;


<s> for ( const MachineOperand & Use : VMEM -> uses ( ) ) { if ( ! Use . isReg ( ) || TRI . isVGPR ( MF . getRegInfo ( ) , Use . getReg ( ) ) ) continue ; int WaitStatesNeededForUse = VmemSgprWaitStates - getWaitStatesSinceDef ( Use . getReg ( ) , IsHazardDefFn , VmemSgprWaitStates ) ; WaitStatesNeeded = std :: max ( WaitStatesNeeded , WaitStatesNeededForUse ) ; } return


<s> if ( ! MI -> modifiesRegister ( <STR_LIT> , TRI ) ) return false ; auto IsHazardFn = [ TRI ] ( MachineInstr * I ) { if ( <ISA_LIT>InstrInfo :: isVALU ( * I ) ) return false ; return I -> readsRegister ( <STR_LIT> , TRI ) ; } ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; auto IsExpiredFn = [ TII , TRI ] ( MachineInstr * MI , int ) { if ( ! MI ) return false ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) { if ( TII -> getNamedOperand ( * MI , <STR_LIT> :: sdst ) ) return true ; for ( auto MO : MI -> implicit_operands ( ) ) if ( MO . isDef ( ) && TRI -> isSGPRClass ( TRI -> getPhysRegClass ( MO . getReg ( ) ) ) ) return true ; } if ( MI -> getOpcode ( ) == <STR_LIT> && ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) & <NUM_LIT> ) == <NUM_LIT> ) return true ; return


<s> return <ISA_LIT>InstrInfo :: isVALU ( * MI ) && Opc != <STR_LIT> && Opc != <STR_LIT> && Opc != <STR_LIT> ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ; auto * Src0 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) ; Register Reg = Src0 -> getReg ( ) ; bool IsUndef = Src0 -> isUndef ( ) ; BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Define | ( IsUndef ? RegState :: Dead : <NUM_LIT> ) ) . addReg ( Reg , IsUndef ? RegState :: Undef : RegState :: Kill ) ; return true


<s> auto IsExpiredFn = [ ] ( MachineInstr * MI , int ) { if ( ! MI ) return false ; unsigned Opc = MI -> getOpcode ( ) ; return <ISA_LIT>InstrInfo :: isVALU ( * MI ) && Opc != <STR_LIT> && Opc != <STR_LIT> && Opc != <STR_LIT> ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ; auto * Src0 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) ; Register Reg = Src0 -> getReg ( ) ; bool IsUndef = Src0 -> isUndef


<s> auto IsExpiredFn = [ ] ( MachineInstr * MI , int ) { return MI && ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) || ( MI -> getOpcode ( ) == <STR_LIT> && ! MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) ) ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) ; return true


<s> IsHazardRecognizerMode ( false ) , CurrCycleInstr ( nullptr ) , MF ( MF ) , ST ( MF . getSubtarget < GCNSubtarget > ( ) ) , TII ( * ST . getInstrInfo ( ) ) , TRI ( TII . getRegisterInfo ( ) ) , ClauseUses ( TRI . getNumRegUnits ( ) ) , ClauseDefs ( TRI . getNumRegUnits ( ) ) { MaxLookAhead = MF . getRegInfo ( ) . isPhysRegUsed ( <STR_LIT>


<s> if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( MI -> mayLoad ( ) || MI -> mayStore ( ) ) && checkMAILdStHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkAnyInstHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; return NoHazard


<s> if ( isDivFMas ( MI -> getOpcode ( ) ) && checkDivFMasHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRWLane ( MI -> getOpcode ( ) ) && checkRWLaneHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSGetReg ( MI -> getOpcode ( ) ) && checkGetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isSSetReg ( MI -> getOpcode ( ) ) && checkSetRegHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( ( MI -> mayLoad ( ) || MI -> mayStore ( ) ) && checkMAILdStHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return NoopHazard ; if ( checkAnyInstHazards ( MI ) > <NUM_LIT>


<s> static int getWaitStatesSince ( GCNHazardRecognizer :: IsHazardFn IsHazard , MachineBasicBlock * MBB , MachineBasicBlock :: reverse_instr_iterator I , int WaitStates , IsExpiredFn IsExpired , DenseSet < const MachineBasicBlock * > & Visited ) { for ( auto E = MBB -> instr_rend ( ) ; I != E ; ++ I ) { if ( I -> isBundle ( ) ) continue ; if ( IsHazard ( & * I ) ) return WaitStates ; if ( I -> isInlineAsm ( ) || I -> isImplicitDef ( ) || I -> isDebugInstr ( ) ) continue ; WaitStates += <ISA_LIT>InstrInfo :: getNumWaitStates ( * I ) ; if ( IsExpired ( & * I , WaitStates ) ) return std :: numeric_limits < int > :: max ( ) ; } int MinWaitStates = WaitStates ; bool Found = false ; for ( MachineBasicBlock * Pred : MBB -> predecessors ( ) ) { if ( ! Visited . insert ( Pred ) . second ) continue ; int W = getWaitStatesSince ( IsHazard , Pred , Pred -> instr_rbegin

( ) , WaitStates , IsExpired , Visited ) ;


<s> BuildMI ( * MI -> getParent ( ) , MI , MI -> getDebugLoc ( ) , TII . get

( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ;


<s> unsigned Opcode = MI . getOpcode ( ) ; return Opcode == <STR_LIT> || Opcode ==


<s> void GCNHazardRecognizer :: processBundle ( ) { MachineBasicBlock :: instr_iterator MI = std :: next ( CurrCycleInstr -> getIterator ( ) ) ; MachineBasicBlock :: instr_iterator E = CurrCycleInstr -> getParent ( ) -> instr_end ( ) ; for ( ; MI != E && MI -> isInsideBundle ( ) ; ++ MI ) { CurrCycleInstr = & * MI ; unsigned WaitStates = PreEmitNoopsCommon ( CurrCycleInstr


<s> void GCNHazardRecognizer :: processBundle ( ) { MachineBasicBlock :: instr_iterator MI = std :: next ( CurrCycleInstr -> getIterator ( ) ) ; MachineBasicBlock :: instr_iterator E = CurrCycleInstr -> getParent ( ) -> instr_end ( ) ; for ( ; MI != E && MI -> isInsideBundle ( ) ; ++ MI ) { CurrCycleInstr = & * MI ; unsigned WaitStates = PreEmitNoopsCommon ( CurrCycleInstr ) ; if ( IsHazardRecognizerMode ) fixHazards ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> ; i < WaitStates ; ++ i ) insertNoopInBundle ( CurrCycleInstr , TII ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( WaitStates , MaxLookAhead - <NUM_LIT> ) ; i < e ; ++ i ) EmittedInstrs . push_front ( nullptr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; EmittedInstrs . resize ( MaxLookAhead


<s> else if ( CurrCycleInstr -> isDebugInstr ( ) ) return ; unsigned NumWaitStates = TII . getNumWaitStates ( * CurrCycleInstr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; } EmittedInstrs . resize ( getMaxLookAhead


<s> int GCNHazardRecognizer :: checkSMRDHazards ( MachineInstr * SMRD ) { const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; int WaitStatesNeeded = <NUM_LIT> ; WaitStatesNeeded = checkSoftClauseHazards ( SMRD ) ; if ( ST . getGeneration ( ) != <ISA_LIT>Subtarget :: SOUTHERN_ISLANDS ) return WaitStatesNeeded ; int SmrdSgprWaitStates = <NUM_LIT> ; auto IsHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isVALU ( * MI ) ; } ; auto IsBufferHazardDefFn = [ this ] ( MachineInstr * MI ) { return TII . isSALU ( * MI ) ; } ; bool IsBufferSMRD = TII . isBufferSMRD ( * SMRD ) ; for ( const MachineOperand & Use : SMRD -> uses ( ) ) { if ( ! Use . isReg


<s> GCNHazardRecognizer :: GCNHazardRecognizer ( const MachineFunction & MF ) : CurrCycleInstr ( nullptr ) , MF ( MF ) , ST ( MF . getSubtarget < GCNSubtarget > ( ) ) , TII ( * ST . getInstrInfo ( ) ) , TRI (

TII . getRegisterInfo ( ) ) , ClauseUses ( TRI . getNumRegUnits ( ) ) , ClauseDefs ( TRI . getNumRegUnits ( ) ) {


<s> return ; } if ( CurrCycleInstr -> isImplicitDef ( ) || CurrCycleInstr -> isDebugInstr ( ) || CurrCycleInstr -> isKill ( ) ) { CurrCycleInstr = nullptr ; return ; } if ( CurrCycleInstr -> isBundle ( ) ) { processBundle ( ) ; return ; } unsigned NumWaitStates = TII . getNumWaitStates ( * CurrCycleInstr ) ; EmittedInstrs . push_front ( CurrCycleInstr ) ; for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; } EmittedInstrs . resize ( getMaxLookAhead ( ) ) ; CurrCycleInstr =


<s> for ( unsigned i = <NUM_LIT> , e = std :: min ( NumWaitStates , getMaxLookAhead ( ) ) ; i < e ; ++ i ) { EmittedInstrs . push_front ( nullptr ) ; } EmittedInstrs . resize ( getMaxLookAhead ( ) ) ; CurrCycleInstr =


<s> int FPAtomicToDenormModeWaitStates = <NUM_LIT> ; if ( MI -> getOpcode ( ) != <STR_LIT> ) return <NUM_LIT> ; auto IsHazardFn = [ ] ( MachineInstr * I ) { if ( ! <ISA_LIT>InstrInfo :: isVMEM ( * I ) && ! <ISA_LIT>InstrInfo :: isFLAT ( * I ) ) return false ; return <ISA_LIT>InstrInfo :: isFPAtomic ( * I ) ; } ; auto IsExpiredFn = [ ] ( MachineInstr * MI , int WaitStates ) { if ( WaitStates >= <NUM_LIT> || <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) return true ; switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : case


<s> if ( ! Op ) continue ; return true ; } return false ; } ; auto IsExpiredFn = [ ] ( MachineInstr * MI , int ) { return MI && ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) || ( MI -> getOpcode ( ) == <STR_LIT> && ! MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) || ( MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) ) ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits


<s> } ; auto IsExpiredFn = [ ] ( MachineInstr * MI , int ) { return MI && ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) || ( MI -> getOpcode ( ) == <STR_LIT> && ! MI -> getOperand ( <NUM_LIT> ) . getImm ( ) ) || ( MI -> getOpcode ( ) == <STR_LIT> && MI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) ) ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > :: max ( ) ) return false ; const <ISA_LIT>InstrInfo * TII = ST .


<s> if ( isRFE ( MI -> getOpcode ( ) ) && checkRFEHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ST . hasReadM0MovRelInterpHazard ( ) && ( TII . isVINTRP ( * MI ) || isSMovRel ( MI -> getOpcode ( ) ) ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ST . hasReadM0SendMsgHazard ( ) && isSendMsgTraceDataOrGDS ( TII , * MI ) && checkReadM0Hazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( <ISA_LIT>InstrInfo :: isMAI ( * MI ) && checkMAIHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) || <ISA_LIT>InstrInfo :: isDS ( * MI ) ) && checkMAILdStHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( MI -> isInlineAsm ( ) && checkInlineAsmHazards ( MI ) > <NUM_LIT> ) return HazardType ; return NoHazard


<s> auto HazardType = IsHazardRecognizerMode ? NoopHazard : Hazard ; if ( MI -> isBundle ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isSMRD ( * MI ) && checkSMRDHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ( <ISA_LIT>InstrInfo :: isVMEM ( * MI ) || <ISA_LIT>InstrInfo :: isFLAT ( * MI ) ) && checkVMEMHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( ST . hasNSAtoVMEMBug ( ) && checkNSAtoVMEMHazard ( MI ) > <NUM_LIT> ) return HazardType ; if ( checkFPAtomicToDenormModeHazard ( MI ) > <NUM_LIT> ) return HazardType ; if ( ST . hasNoDataDepHazard ( ) ) return NoHazard ; if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) && checkVALUHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) && checkDPPHazards ( MI ) > <NUM_LIT> ) return HazardType ; if ( isDivFMas ( MI -> getOpcode ( ) )

&& checkDivFMasHazards ( MI ) > <NUM_LIT> ) return HazardType ;


<s> if ( IsExpired ( & * I , WaitStates ) ) return std :: numeric_limits < int > :: max ( ) ; } int MinWaitStates = WaitStates ; bool Found = false ; for ( MachineBasicBlock * Pred : MBB -> predecessors ( ) ) { if ( ! Visited . insert ( Pred ) . second ) continue ; int W = getWaitStatesSince ( IsHazard , Pred , Pred -> instr_rbegin ( ) , WaitStates , IsExpired , Visited ) ; if ( W == std :: numeric_limits < int > :: max ( ) ) continue ; MinWaitStates = Found ? std :: min ( MinWaitStates , W ) : W ; if ( IsExpired ( nullptr , MinWaitStates )


<s> if ( <ISA_LIT>InstrInfo :: isVALU ( * MI ) ) WaitStates = std :: max ( WaitStates , checkVALUHazards ( MI ) ) ; if ( <ISA_LIT>InstrInfo :: isDPP ( * MI ) ) WaitStates = std :: max ( WaitStates , checkDPPHazards ( MI ) ) ; if ( isDivFMas ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkDivFMasHazards ( MI ) ) ; if ( isRWLane ( MI -> getOpcode ( ) ) ) WaitStates = std :: max ( WaitStates , checkRWLaneHazards ( MI ) ) ; if ( MI -> isInlineAsm ( ) ) return std :: max ( WaitStates , checkInlineAsmHazards ( MI ) ) ; if ( isSGetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkGetRegHazards ( MI ) ) ; if ( isSSetReg ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkSetRegHazards ( MI ) ) ; if ( isRFE ( MI -> getOpcode ( ) ) ) return std :: max ( WaitStates , checkRFEHazards (


<s> } ; auto IsExpiredFn = [ InstType , & IsHazardInst ] ( const MachineInstr & I , int ) { auto InstType2 = IsHazardInst ( I ) ; if ( InstType == InstType2 ) return true ; return I . getOpcode ( ) == <STR_LIT> && I . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> && ! I . getOperand ( <NUM_LIT> ) . getImm ( ) ; } ; return :: getWaitStatesSince ( IsHazardFn , & I , IsExpiredFn ) != std :: numeric_limits < int > :: max ( ) ; } ; if ( :: getWaitStatesSince ( IsHazardFn , MI , IsExpiredFn ) == std :: numeric_limits < int > ::


<s> case <STR_LIT> : case <STR_LIT> : SDSTName = <STR_LIT> :: vdst ; break ; default : SDSTName = <STR_LIT> :: sdst ; break ; } const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <STR_LIT> IV = <STR_LIT> ( ST . getCPU ( ) ) ; const MachineOperand * SDST = TII -> getNamedOperand ( * MI , SDSTName ) ; if ( ! SDST ) { for ( const auto & MO : MI -> implicit_operands ( ) ) { if ( MO . isDef ( ) && TRI -> isSGPRClass ( TRI -> getPhysRegClass ( MO . getReg ( ) ) ) ) { SDST = & MO ; break ; } } } if ( ! SDST ) return false ; const unsigned SDSTReg = SDST -> getReg ( ) ; auto IsHazardFn = [ SDSTReg , TRI ] ( MachineInstr * I ) { return <ISA_LIT>InstrInfo :: isSMRD ( * I ) && I -> readsRegister ( SDSTReg , TRI ) ; } ; auto IsExpiredFn = [ TII , IV ] ( MachineInstr * MI , int ) { if ( MI ) { if ( TII -> isSALU ( * MI ) ) { switch ( MI -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return false ; case <STR_LIT> : return ( MI -> getOperand ( <NUM_LIT> ) . getImm ( ) == <NUM_LIT> ) && (

MI -> getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) ;