<s> DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } else if ( ReadsCopyFromExec && ! SaveExecInst ) { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0 ; } else llvm_unreachable ( <STR_LIT> ) ; CopyFromExecInst -> eraseFromParent ( ) ; auto InsPt = SaveExecInst -> getIterator ( ) ; const DebugLoc & DL = SaveExecInst -> getDebugLoc ( ) ; BuildMI ( MBB , InsPt , DL , TII -> get ( getSaveExecOp ( SaveExecInst -> getOpcode ( ) ) ) , CopyFromExec ) . addReg ( OtherOp -> getReg ( ) ) ; SaveExecInst -> eraseFromParent ( ) ; CopyToExecInst -> eraseFromParent ( ) ; for ( MachineInstr * OtherInst : OtherUseInsts ) {

OtherInst -> substituteRegister ( CopyToExec , <STR_LIT> , <STR_LIT> , * TRI ) ;


<s> static MachineBasicBlock :: reverse_iterator findExecCopy ( const <ISA_LIT>InstrInfo & TII , const GCNSubtarget & ST , MachineBasicBlock & MBB , MachineBasicBlock :: reverse_iterator I , unsigned CopyToExec ) { const unsigned InstLimit = <NUM_LIT> ; auto E = MBB . rend ( ) ; for ( unsigned N = <NUM_LIT> ; N <= InstLimit && I != E ; ++ I , ++ N ) {

unsigned CopyFromExec = isCopyFromExec ( * I , ST ) ;


<s> static unsigned isCopyToExec ( const MachineInstr & MI , const GCNSubtarget & ST ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && Dst . getReg ( ) == ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) && MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; break ; } case <STR_LIT> : case <STR_LIT> : llvm_unreachable ( <STR_LIT> ) ; } return <STR_LIT> ;


<s> const MachineOperand & Src1 = MI . getOperand ( <NUM_LIT> ) ; if ( Src1 . isReg ( ) && Src1 . getReg ( ) == <STR_LIT> ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const MachineOperand & Src2 = MI . getOperand ( <NUM_LIT> ) ; if ( Src2 . isReg ( ) && Src2 . getReg ( ) == <STR_LIT> ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Src1 = MI . getOperand ( <NUM_LIT> ) ;


<s> SmallVector < MachineInstr * , <NUM_LIT> > OtherUseInsts ; for ( MachineBasicBlock :: iterator J = std :: next ( CopyFromExecInst -> getIterator ( ) ) , JE = I -> getIterator ( ) ; J != JE ; ++ J ) { if ( SaveExecInst && J -> readsRegister ( Exec , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; SaveExecInst = nullptr ; break ; } bool ReadsCopyFromExec = J -> readsRegister ( CopyFromExec , TRI ) ; if ( J -> modifiesRegister ( CopyToExec , TRI ) ) { if ( SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( CopyToExec , TRI ) << '\n' ) ; SaveExecInst = nullptr ; break ; } unsigned SaveExecOp = getSaveExecOp ( J -> getOpcode ( ) ) ; if ( SaveExecOp == <STR_LIT> ) break ; if ( ReadsCopyFromExec ) { SaveExecInst = & * J ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } else if ( ReadsCopyFromExec && ! SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ;


<s> if ( ReadsCopyFromExec ) { SaveExecInst = & * J ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } else if ( ReadsCopyFromExec && ! SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0 ; } else llvm_unreachable ( <STR_LIT> ) ; CopyFromExecInst -> eraseFromParent ( ) ; auto InsPt = SaveExecInst -> getIterator ( ) ; const DebugLoc & DL = SaveExecInst -> getDebugLoc ( ) ; BuildMI ( MBB , InsPt , DL , TII -> get ( getSaveExecOp ( SaveExecInst -> getOpcode ( ) ) ) , CopyFromExec ) . addReg ( OtherOp -> getReg ( ) ) ; SaveExecInst -> eraseFromParent ( ) ; CopyToExecInst -> eraseFromParent ( ) ; for ( MachineInstr * OtherInst : OtherUseInsts ) { OtherInst -> substituteRegister ( CopyToExec , Exec , <STR_LIT> , * TRI ) ; } }


<s> if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0 ; } else llvm_unreachable ( <STR_LIT> ) ; CopyFromExecInst -> eraseFromParent ( ) ; auto InsPt = SaveExecInst -> getIterator ( ) ; const DebugLoc & DL = SaveExecInst -> getDebugLoc ( ) ; BuildMI ( MBB , InsPt , DL , TII -> get ( getSaveExecOp ( SaveExecInst -> getOpcode ( ) ) ) , CopyFromExec ) . addReg ( OtherOp -> getReg ( ) ) ; SaveExecInst -> eraseFromParent ( ) ; CopyToExecInst -> eraseFromParent ( ) ; for ( MachineInstr * OtherInst : OtherUseInsts ) { OtherInst -> substituteRegister ( CopyToExec , <STR_LIT> , <STR_LIT> , * TRI ) ; }


<s> } unsigned CopyFromExec = CopyFromExecInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * SaveExecInst = nullptr ; SmallVector < MachineInstr * , <NUM_LIT> > OtherUseInsts ; for ( MachineBasicBlock :: iterator J = std :: next ( CopyFromExecInst -> getIterator ( ) ) , JE = I -> getIterator ( ) ; J != JE ; ++ J ) { if ( SaveExecInst && J -> readsRegister ( <STR_LIT> , TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; SaveExecInst = nullptr ; break ; } bool ReadsCopyFromExec = J -> readsRegister ( CopyFromExec , TRI ) ; if ( J -> modifiesRegister ( CopyToExec , TRI ) ) { if ( SaveExecInst ) { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( CopyToExec , TRI ) << '\n' ) ; SaveExecInst = nullptr ; break ; } unsigned SaveExecOp = getSaveExecOp ( J -> getOpcode ( ) ) ; if ( SaveExecOp == <STR_LIT> ) break ; if ( ReadsCopyFromExec ) { SaveExecInst = & * J ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } else if ( ReadsCopyFromExec && ! SaveExecInst ) { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0 ; } else llvm_unreachable ( <STR_LIT> ) ; CopyFromExecInst -> eraseFromParent ( ) ; auto InsPt = SaveExecInst -> getIterator ( ) ;

const DebugLoc & DL = SaveExecInst -> getDebugLoc ( ) ;


<s> J != JE ; ++ J ) { if ( SaveExecInst && J -> readsRegister ( <STR_LIT> , TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; SaveExecInst = nullptr ; break ; } bool ReadsCopyFromExec = J -> readsRegister ( CopyFromExec , TRI ) ; if ( J -> modifiesRegister ( CopyToExec , TRI ) ) { if ( SaveExecInst ) { DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( CopyToExec , TRI ) << '\n' ) ; SaveExecInst = nullptr ; break ; } unsigned SaveExecOp = getSaveExecOp ( J -> getOpcode ( ) ) ; if ( SaveExecOp == <STR_LIT> ) break ; if ( ReadsCopyFromExec ) { SaveExecInst = & * J ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; continue ; } else { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } } else if ( ReadsCopyFromExec && ! SaveExecInst ) { DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; break ; } if ( SaveExecInst && J -> readsRegister ( CopyToExec , TRI ) ) { assert ( SaveExecInst != & * J ) ; OtherUseInsts . push_back ( & * J ) ; } } if ( ! SaveExecInst ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExecInst << '\n' ) ; MachineOperand & Src0 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand & Src1 = SaveExecInst -> getOperand ( <NUM_LIT> ) ; MachineOperand * OtherOp = nullptr ; if ( Src0 . isReg ( ) && Src0 . getReg ( ) == CopyFromExec ) { OtherOp = & Src1 ; } else if ( Src1 . isReg ( ) && Src1 . getReg ( ) == CopyFromExec ) { if ( ! SaveExecInst -> isCommutable ( ) ) break ; OtherOp = & Src0 ; } else llvm_unreachable ( <STR_LIT> ) ; CopyFromExecInst -> eraseFromParent ( ) ; auto InsPt = SaveExecInst -> getIterator ( ) ; const DebugLoc & DL = SaveExecInst -> getDebugLoc ( ) ; BuildMI ( MBB , InsPt , DL , TII -> get ( getSaveExecOp ( SaveExecInst -> getOpcode ( ) ) ) , CopyFromExec ) . addReg ( OtherOp -> getReg ( ) ) ; SaveExecInst -> eraseFromParent ( ) ; CopyToExecInst -> eraseFromParent ( ) ; for ( MachineInstr * OtherInst : OtherUseInsts ) {

OtherInst -> substituteRegister ( CopyToExec , <STR_LIT> , <STR_LIT> , * TRI , true ) ;


<s> if ( I != E ) { if ( I -> isUnconditionalBranch ( ) || I -> getOpcode ( ) == <STR_LIT> ) ++ I ; else if ( I -> isBranch ( ) ) continue ; } while ( I != E ) { if ( I -> isDebugInstr ( ) ) { I = std :: next ( I ) ; continue ; } if ( I -> mayStore ( ) || I -> isBarrier ( ) || I -> isCall ( ) || I -> hasUnmodeledSideEffects ( ) || I -> hasOrderedMemoryRef ( ) ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I << '\n' ) ; for ( auto & Op : I -> operands ( ) ) { if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } auto Next = std :: next ( I ) ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; I -> eraseFromParent ( ) ; I = Next ; Changed = true ; } if ( I != E ) continue ; for ( auto * Pred : CurBB -> predecessors ( ) ) { if ( Pred -> succ_size ( ) == <NUM_LIT> ) Blocks . push_back ( Pred ) ; } } continue ; } unsigned ScanThreshold = <NUM_LIT> ; for ( auto I = MBB . rbegin ( ) , E = MBB . rend ( ) ; I != E && ScanThreshold -- ; ++ I ) { if ( ! ( I -> isFullCopy ( ) && I -> getOperand ( <NUM_LIT> ) . getReg ( ) == Register ( ExecReg ) ) ) continue ; Register SavedExec = I -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( SavedExec . isVirtual ( ) && MRI -> hasOneNonDBGUse ( SavedExec ) ) { MachineInstr * SingleExecUser = & * MRI -> use_instr_nodbg_begin ( SavedExec ) ; int Idx = SingleExecUser -> findRegisterUseOperandIdx ( SavedExec ) ; assert ( Idx != - <NUM_LIT> ) ; if ( SingleExecUser -> getParent ( ) == I -> getParent ( ) && ! SingleExecUser -> getOperand ( Idx ) . isImplicit ( ) && TII -> isOperandLegal ( * SingleExecUser , Idx , & I -> getOperand ( <NUM_LIT> ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I << '\n' ) ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; I -> eraseFromParent ( ) ; MRI -> replaceRegWith ( SavedExec , ExecReg ) ; LIS -> removeInterval ( SavedExec ) ; Changed = true ; } } break ;


<s> for ( MCRegUnitIterator UI ( Reg , & TRI ) ; UI . isValid ( ) ; ++ UI ) { if ( isDefBetween ( LIS -> getRegUnit ( * UI ) , AndIdx , SelIdx ) ) return true ;


<s> static bool isFullExecCopy ( const MachineInstr & MI , const GCNSubtarget & ST ) { unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ;

if ( MI . isFullCopy ( ) && MI . getOperand ( <NUM_LIT> ) . getReg ( ) == Exec ) return true ;


<s> if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } auto Next = std :: next ( I ) ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; I -> eraseFromParent ( ) ; I = Next ; Changed = true ; } if ( I != E ) continue ; for ( auto * Pred : CurBB -> predecessors ( ) ) { if ( Pred -> succ_size ( ) == <NUM_LIT> ) Blocks . push_back ( Pred ) ; } } continue ; } unsigned ScanThreshold = <NUM_LIT> ; for ( auto I = MBB . rbegin ( ) , E = MBB . rend ( ) ; I != E && ScanThreshold -- ; ++ I ) { if ( ! isFullExecCopy ( * I , ST ) ) continue ; Register SavedExec = I -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( SavedExec . isVirtual ( ) && MRI . hasOneNonDBGUse ( SavedExec ) && MRI . use_instr_nodbg_begin ( SavedExec ) -> getParent ( ) == I -> getParent ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I << '\n' ) ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; I -> eraseFromParent ( ) ; MRI . replaceRegWith ( SavedExec , Exec ) ; LIS -> removeInterval ( SavedExec ) ; Changed = true ; }


<s> auto * Cmp = TRI -> findReachingDef ( CmpReg , CmpSubReg , * And , MRI , LIS ) ; if ( ! Cmp || ! ( Cmp -> getOpcode ( ) == <STR_LIT> || Cmp -> getOpcode ( ) == <STR_LIT> ) || Cmp -> getParent ( ) != And -> getParent ( ) ) return <STR_LIT> ; MachineOperand * Op1 = TII -> getNamedOperand ( * Cmp , <STR_LIT> :: src0 ) ; MachineOperand * Op2 = TII -> getNamedOperand ( * Cmp , <STR_LIT> :: src1 ) ; if ( Op1 -> isImm ( ) && Op2 -> isReg ( ) ) std :: swap ( Op1 , Op2 ) ; if ( ! Op1 -> isReg ( ) || ! Op2 -> isImm ( ) || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; Register SelReg = Op1 -> getReg ( ) ; auto * Sel = TRI -> findReachingDef ( SelReg , Op1 -> getSubReg ( ) , * Cmp , MRI , LIS ) ; if ( ! Sel || Sel -> getOpcode ( ) != <STR_LIT> ) return <STR_LIT> ; if ( TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src0_modifiers ) || TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src1_modifiers ) ) return <STR_LIT> ; Op1 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src0 ) ; Op2 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src1 ) ; MachineOperand * CC = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src2 ) ; if ( ! Op1 -> isImm ( ) || ! Op2 -> isImm ( ) || ! CC -> isReg ( ) || Op1 -> getImm ( ) != <NUM_LIT> || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; Register CCReg = CC -> getReg ( ) ; if ( isDefBetween ( * TRI , LIS , CCReg , * Sel , * And ) ) return <STR_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Sel << '\t' << * Cmp << '\t' << * And ) ;

LIS -> RemoveMachineInstrFromMaps ( * And ) ;


<s> RecalcRegs . insert ( <STR_LIT> ) ; Changed = true ; } if ( MBB . succ_empty ( ) ) { if ( MBB . empty ( ) ) continue ; MachineInstr & Term = MBB . back ( ) ; if ( Term . getOpcode ( ) != <STR_LIT> || Term . getNumOperands ( ) != <NUM_LIT> ) continue ; SmallVector < MachineBasicBlock * , <NUM_LIT> > Blocks ( { & MBB } ) ; while ( ! Blocks . empty ( ) ) { auto CurBB = Blocks . pop_back_val ( ) ; auto I = CurBB -> rbegin ( ) , E = CurBB -> rend ( ) ; if ( I != E ) { if ( I -> isUnconditionalBranch ( ) || I -> getOpcode ( ) == <STR_LIT> ) ++ I ; else if ( I -> isBranch ( ) ) continue ; } while ( I != E ) { if ( I -> isDebugInstr ( ) ) { I = std :: next ( I ) ; continue ; } if ( I -> mayStore ( ) || I -> isBarrier ( ) || I -> isCall ( ) || I -> hasUnmodeledSideEffects ( ) || I -> hasOrderedMemoryRef ( ) ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I << '\n' ) ; for ( auto & Op : I -> operands ( ) ) { if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } auto Next = std :: next ( I ) ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; I -> eraseFromParent ( ) ; I = Next ; Changed = true ; } if ( I != E ) continue ; for ( auto * Pred : CurBB -> predecessors ( ) ) { if ( Pred -> succ_size ( ) == <NUM_LIT> ) Blocks . push_back ( Pred ) ;


<s> if ( I != E ) continue ; for ( auto * Pred : CurBB -> predecessors ( ) ) { if ( Pred -> succ_size ( ) == <NUM_LIT> ) Blocks . push_back ( Pred ) ; } } continue ; } unsigned ScanThreshold = <NUM_LIT> ; for ( auto I = MBB . rbegin ( ) , E = MBB . rend ( ) ; I != E && ScanThreshold -- ; ++ I ) { if ( ! isFullExecCopy ( * I , ST ) ) continue ; Register SavedExec = I -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( SavedExec . isVirtual ( ) && MRI . hasOneNonDBGUse ( SavedExec ) && MRI . use_instr_nodbg_begin ( SavedExec ) -> getParent ( ) == I -> getParent ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I << '\n' ) ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; I -> eraseFromParent ( ) ; MRI . replaceRegWith ( SavedExec , Exec ) ; LIS -> removeInterval ( SavedExec ) ; Changed = true ; } break ; } } if ( Changed ) { for ( auto Reg : RecalcRegs ) { if ( Reg . isVirtual ( ) ) { LIS -> removeInterval ( Reg ) ; if ( ! MRI . reg_empty ( Reg ) ) LIS -> createAndComputeVirtRegInterval ( Reg ) ; } else { LIS -> removeAllRegUnitsForPhysReg ( Reg ) ; } } } return Changed ;


<s> MachineOperand * Op2 = TII -> getNamedOperand ( * Cmp , <STR_LIT> :: src1 ) ; if ( Op1 -> isImm ( ) && Op2 -> isReg ( ) ) std :: swap ( Op1 , Op2 ) ; if ( ! Op1 -> isReg ( ) || ! Op2 -> isImm ( ) || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; Register SelReg = Op1 -> getReg ( ) ; auto * Sel = TRI -> findReachingDef ( SelReg , Op1 -> getSubReg ( ) , * Cmp , MRI , LIS ) ; if ( ! Sel || Sel -> getOpcode ( ) != <STR_LIT> ) return <STR_LIT> ; if ( TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src0_modifiers ) || TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src1_modifiers ) ) return <STR_LIT> ; Op1 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src0 ) ; Op2 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src1 ) ; MachineOperand * CC = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src2 ) ; if ( ! Op1 -> isImm ( ) || ! Op2 -> isImm ( ) || ! CC -> isReg ( ) || Op1 -> getImm ( ) != <NUM_LIT> || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Sel << '\t' << * Cmp << '\t' << * And ) ; Register CCReg = CC -> getReg ( ) ; LIS -> RemoveMachineInstrFromMaps ( * And ) ; MachineInstr * Andn2 = BuildMI ( MBB , * And , And -> getDebugLoc ( ) , TII -> get ( Andn2Opc ) , And -> getOperand ( <NUM_LIT> ) . getReg ( ) ) . addReg ( ExecReg ) . addReg ( CCReg , getUndefRegState ( CC -> isUndef ( ) ) , CC -> getSubReg ( ) ) ; And -> eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * Andn2 ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Andn2 << '\n' ) ; if ( ( Register :: isVirtualRegister ( CmpReg ) && MRI . use_nodbg_empty ( CmpReg ) ) || ( CmpReg == CondReg && std :: none_of ( std :: next ( Cmp -> getIterator ( ) ) , Andn2 -> getIterator ( ) , [ & ] ( const MachineInstr & MI ) { return MI . readsRegister ( CondReg , TRI ) ; } ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Cmp << '\n' ) ;

LIS -> RemoveMachineInstrFromMaps ( * Cmp ) ;


<s> if ( SavedExec == <STR_LIT> ) return nullptr ; auto SaveExecInst = MRI . getUniqueVRegDef ( SavedExec ) ; if ( ! SaveExecInst || ! isFullExecCopy ( * SaveExecInst , ST ) ) return nullptr ;


<s> auto Op = TII . getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Op -> isReg ( ) && Op -> getReg ( ) != Exec ) return Op -> getReg ( ) ; Op = TII . getNamedOperand ( MI , <STR_LIT> :: src0 ) ; if ( Op -> isReg ( ) && Op -> getReg ( ) != Exec ) return Op -> getReg ( ) ;


<s> static unsigned getOrNonExecReg ( const MachineInstr & MI , const <ISA_LIT>InstrInfo & TII , const GCNSubtarget & ST ) { unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; auto Op = TII . getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Op -> isReg ( ) && Op -> getReg ( ) != Exec ) return Op -> getReg ( ) ; Op = TII . getNamedOperand ( MI , <STR_LIT> :: src0 ) ; if ( Op -> isReg ( ) && Op -> getReg ( ) != Exec ) return Op -> getReg ( ) ;


<s> static bool isEndCF ( const MachineInstr & MI , const <ISA_LIT>RegisterInfo * TRI , const GCNSubtarget & ST ) { if ( ST . isWave32 ( ) ) { return MI . getOpcode ( ) == <STR_LIT> && MI . modifiesRegister ( <STR_LIT> , TRI ) ; }


<s> if ( ST . isWave32 ( ) ) { return MI . getOpcode ( ) == <STR_LIT> && MI . modifiesRegister ( <STR_LIT> , TRI ) ; } return MI . getOpcode ( ) == <STR_LIT> && MI . modifiesRegister ( <STR_LIT> , TRI ) ;


<s> if ( MI . isCopy ( ) && MI . getOperand ( <NUM_LIT> ) . getReg ( ) == Exec ) { assert ( MI . isFullCopy ( ) ) ; return true ;


<s> if ( I != E ) { if ( I -> isUnconditionalBranch ( ) || I -> getOpcode ( ) == <STR_LIT> ) ++ I ; else if ( I -> isBranch ( ) ) continue ; } while ( I != E ) { if ( I -> isDebugInstr ( ) ) { I = std :: next ( I ) ; continue ; } if ( I -> mayStore ( ) || I -> isBarrier ( ) || I -> isCall ( ) || I -> hasUnmodeledSideEffects ( ) || I -> hasOrderedMemoryRef ( ) ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I << '\n' ) ; for ( auto & Op : I -> operands ( ) ) { if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } auto Next = std :: next ( I ) ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; I -> eraseFromParent ( ) ; I = Next ; Changed = true ; } if ( I != E ) continue ; for ( auto * Pred : CurBB -> predecessors ( ) ) { if ( Pred -> succ_size ( ) == <NUM_LIT> ) Blocks . push_back ( Pred ) ; } } continue ; } auto E = MBB . end ( ) ; auto Lead = skipDebugInstructionsForward ( MBB . begin ( ) , E ) ; if ( MBB . succ_size ( ) != <NUM_LIT> || Lead == E || ! isEndCF ( * Lead , TRI , ST ) ) continue ;


<s> for ( ; I != E ; ++ I ) { if ( TII -> mayReadEXEC ( * MRI , * I ) ) break ; }


<s> MachineBasicBlock :: iterator <ISA_LIT>OptimizeExecMaskingPreRA :: skipIgnoreExecInsts ( MachineBasicBlock :: iterator I , MachineBasicBlock :: iterator E ) const { for ( ; I != E ; ++ I ) {

if ( TII -> mayReadEXEC ( * MRI , * I ) ) break ;


<s> if ( It != MBB -> end ( ) || MBB -> succ_size ( ) != <NUM_LIT> ) break ;

MachineBasicBlock * Succ = * MBB -> succ_begin ( ) ;


<s> if ( skipFunction ( MF . getFunction ( ) ) ) return false ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; LiveIntervals * LIS = & getAnalysis < LiveIntervals > ( ) ; DenseSet < unsigned > RecalcRegs ( { <STR_LIT> , <STR_LIT> } ) ; bool Changed = false ; for ( MachineBasicBlock & MBB : MF ) { if ( MBB . succ_empty ( ) ) { if ( MBB . empty ( ) || MBB . back ( ) . getOpcode ( ) != <STR_LIT> ) continue ; SmallVector < MachineBasicBlock * , <NUM_LIT> > Blocks ( { & MBB } ) ; while ( ! Blocks . empty ( ) ) { auto CurBB = Blocks . pop_back_val ( ) ; auto I = CurBB -> rbegin ( ) , E = CurBB -> rend ( ) ; if ( I != E ) { if ( I -> isUnconditionalBranch ( ) || I -> getOpcode ( ) == <STR_LIT> ) ++ I ; else if ( I -> isBranch ( ) ) continue ; } while ( I != E ) { if ( I -> isDebugInstr ( ) ) { I = std :: next ( I ) ; continue ; } if ( I -> mayStore ( ) || I -> isBarrier ( ) || I -> isCall ( ) || I -> hasUnmodeledSideEffects ( ) || I -> hasOrderedMemoryRef ( ) ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I << '\n' ) ; for ( auto & Op : I -> operands ( ) ) { if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } auto Next = std :: next ( I ) ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; I -> eraseFromParent ( ) ; I = Next ; Changed = true ; } if ( I != E ) continue ; for ( auto * Pred : CurBB -> predecessors ( ) ) { if ( Pred -> succ_size ( ) == <NUM_LIT> ) Blocks . push_back ( Pred ) ;


<s> if ( SavedExec == <STR_LIT> ) return nullptr ; auto SaveExecInst = MRI . getUniqueVRegDef ( SavedExec ) ;

if ( ! SaveExecInst || ! isFullExecCopy ( * SaveExecInst ) ) return nullptr ;


<s> Op = TII . getNamedOperand ( MI , <STR_LIT> :: src0 ) ; if ( Op -> isReg ( ) && Op -> getReg ( ) != <STR_LIT> ) return Op -> getReg ( ) ;


<s> Op = TII . getNamedOperand ( MI , <STR_LIT> :: src0 ) ; if ( Op -> isReg ( ) && Op -> getReg ( ) != <STR_LIT> ) return Op -> getReg ( ) ; return <STR_LIT> ;


<s> for ( auto & Op : I -> operands ( ) ) { if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } auto Next = std :: next ( I ) ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; I -> eraseFromParent ( ) ; I = Next ; Changed = true ; } if ( I != E ) continue ; for ( auto * Pred : CurBB -> predecessors ( ) ) { if ( Pred -> succ_size ( ) == <NUM_LIT> ) Blocks . push_back ( Pred ) ; } } continue ; } auto Lead = MBB . begin ( ) , E = MBB . end ( ) ; if ( MBB . succ_size ( ) != <NUM_LIT> || Lead == E || ! isEndCF ( * Lead , TRI ) ) continue ; const MachineBasicBlock * Succ = * MBB . succ_begin ( ) ; if ( ! MBB . isLayoutSuccessor ( Succ ) ) continue ; auto I = std :: next ( Lead ) ; for ( ; I != E ; ++ I ) if ( ! TII -> isSALU ( * I ) || I -> readsRegister ( <STR_LIT> , TRI ) ) break ; if ( I != E ) continue ; const auto NextLead = Succ -> begin ( ) ; if ( NextLead == Succ -> end ( ) || ! isEndCF ( * NextLead , TRI ) || ! getOrExecSource ( * NextLead , * TII , MRI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Lead << '\n' ) ; auto SaveExec = getOrExecSource ( * Lead , * TII , MRI ) ;

unsigned SaveExecReg = getOrNonExecReg ( * Lead , * TII ) ;


<s> MachineOperand * Op2 = TII -> getNamedOperand ( * Cmp , <STR_LIT> :: src1 ) ; if ( Op1 -> isImm ( ) && Op2 -> isReg ( ) ) std :: swap ( Op1 , Op2 ) ; if ( ! Op1 -> isReg ( ) || ! Op2 -> isImm ( ) || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; unsigned SelReg = Op1 -> getReg ( ) ; auto * Sel = TRI -> findReachingDef ( SelReg , Op1 -> getSubReg ( ) , * Cmp , MRI , LIS ) ; if ( ! Sel || Sel -> getOpcode ( ) != <STR_LIT> ) return <STR_LIT> ; Op1 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src0 ) ; Op2 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src1 ) ; MachineOperand * CC = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src2 ) ; if ( ! Op1 -> isImm ( ) || ! Op2 -> isImm ( ) || ! CC -> isReg ( ) || Op1 -> getImm ( ) != <NUM_LIT> || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Sel << '\t' << * Cmp << '\t' << * And ) ; unsigned CCReg = CC -> getReg ( ) ; LIS -> RemoveMachineInstrFromMaps ( * And ) ; MachineInstr * Andn2 = BuildMI ( MBB , * And , And -> getDebugLoc ( ) , TII -> get ( Andn2Opc ) , And -> getOperand ( <NUM_LIT> ) . getReg ( ) ) . addReg ( ExecReg ) . addReg ( CCReg , CC -> getSubReg ( ) ) ; And -> eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * Andn2 ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Andn2 << '\n' ) ; if ( ( TargetRegisterInfo :: isVirtualRegister ( CmpReg ) && MRI . use_nodbg_empty ( CmpReg ) ) || ( CmpReg == CondReg && std :: none_of ( std :: next ( Cmp -> getIterator ( ) ) , Andn2 -> getIterator ( ) , [ & ] ( const MachineInstr & MI ) { return MI . readsRegister ( CondReg , TRI ) ; } ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Cmp << '\n' ) ; LIS -> RemoveMachineInstrFromMaps ( * Cmp ) ; Cmp -> eraseFromParent ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SelReg ) && MRI . use_nodbg_empty ( SelReg ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Sel << '\n' ) ; LIS -> RemoveMachineInstrFromMaps ( * Sel ) ;


<s> DenseSet < unsigned > RecalcRegs ( { <STR_LIT> , <STR_LIT> } ) ; bool Changed = false ; for ( MachineBasicBlock & MBB : MF ) { if ( unsigned Reg = optimizeVcndVcmpPair ( MBB , ST , MRI , LIS ) ) { RecalcRegs . insert ( Reg ) ; RecalcRegs . insert ( <STR_LIT> ) ; RecalcRegs . insert ( <STR_LIT> ) ; RecalcRegs . insert ( <STR_LIT> ) ; Changed = true ; } if ( MBB . succ_empty ( ) ) { if ( MBB . empty ( ) ) continue ; MachineInstr & Term = MBB . back ( ) ; if ( Term . getOpcode ( ) != <STR_LIT> || Term . getNumOperands ( ) != <NUM_LIT> ) continue ; SmallVector < MachineBasicBlock * , <NUM_LIT> > Blocks ( { & MBB } ) ; while ( ! Blocks . empty ( ) ) { auto CurBB = Blocks . pop_back_val ( ) ; auto I = CurBB -> rbegin ( ) , E = CurBB -> rend ( ) ; if ( I != E ) { if ( I -> isUnconditionalBranch ( ) || I -> getOpcode ( ) == <STR_LIT> ) ++ I ; else if ( I -> isBranch ( ) ) continue ; } while ( I != E ) { if ( I -> isDebugInstr ( ) ) { I = std :: next ( I ) ; continue ; } if ( I -> mayStore ( ) || I -> isBarrier ( ) || I -> isCall ( ) || I -> hasUnmodeledSideEffects ( ) || I -> hasOrderedMemoryRef ( ) ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I << '\n' ) ; for ( auto & Op : I -> operands ( ) ) { if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } auto Next = std :: next ( I ) ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; I -> eraseFromParent ( ) ; I = Next ; Changed = true ; } if ( I != E ) continue ; for ( auto * Pred : CurBB -> predecessors ( ) ) { if ( Pred -> succ_size ( ) == <NUM_LIT> ) Blocks . push_back ( Pred ) ; } } continue ; } auto Lead = MBB . begin ( ) , E = MBB . end ( ) ; if ( MBB . succ_size ( ) != <NUM_LIT> || Lead == E || ! isEndCF ( * Lead , TRI ) ) continue ; const MachineBasicBlock * Succ = * MBB . succ_begin ( ) ; if ( ! MBB . isLayoutSuccessor ( Succ ) ) continue ;


<s> if ( MI . isCopy ( ) && MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) { assert ( MI . isFullCopy ( ) ) ; return true ; }


<s> if ( TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src0_modifiers ) || TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src1_modifiers ) ) return <STR_LIT> ; Op1 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src0 ) ; Op2 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src1 ) ; MachineOperand * CC = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src2 ) ; if ( ! Op1 -> isImm ( ) || ! Op2 -> isImm ( ) || ! CC -> isReg ( ) || Op1 -> getImm ( ) != <NUM_LIT> || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Sel << '\t' << * Cmp << '\t' << * And ) ; unsigned CCReg = CC -> getReg ( ) ; LIS -> RemoveMachineInstrFromMaps ( * And ) ; MachineInstr * Andn2 = BuildMI ( MBB , * And , And -> getDebugLoc ( ) , TII -> get ( Andn2Opc ) , And -> getOperand ( <NUM_LIT> ) . getReg ( ) ) . addReg ( ExecReg ) . addReg ( CCReg , <NUM_LIT> , CC -> getSubReg ( ) ) ; And -> eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * Andn2 ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Andn2 << '\n' ) ;

if ( ( TargetRegisterInfo :: isVirtualRegister ( CmpReg ) && MRI . use_nodbg_empty ( CmpReg ) ) || ( CmpReg == CondReg && std :: none_of ( std :: next ( Cmp -> getIterator ( ) ) , Andn2 -> getIterator ( ) , [ & ] ( const MachineInstr & MI ) {


<s> static bool isFullExecCopy ( const MachineInstr & MI ) { return MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ;


<s> Op1 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src0 ) ; Op2 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src1 ) ; MachineOperand * CC = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src2 ) ; if ( ! Op1 -> isImm ( ) || ! Op2 -> isImm ( ) || ! CC -> isReg ( ) || Op1 -> getImm ( ) != <NUM_LIT> || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Sel << '\t' << * Cmp << '\t' << * And ) ; unsigned CCReg = CC -> getReg ( ) ; LIS -> RemoveMachineInstrFromMaps ( * And ) ; MachineInstr * Andn2 = BuildMI ( MBB , * And , And -> getDebugLoc ( ) , TII -> get ( Andn2Opc ) , And -> getOperand ( <NUM_LIT> ) . getReg ( ) ) . addReg ( ExecReg ) . addReg ( CCReg , CC -> getSubReg ( ) ) ; And -> eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * Andn2 ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Andn2 << '\n' ) ; if ( ( TargetRegisterInfo :: isVirtualRegister ( CmpReg ) && MRI . use_nodbg_empty ( CmpReg ) ) || ( CmpReg == CondReg && std :: none_of ( std :: next ( Cmp -> getIterator ( ) ) , Andn2 -> getIterator ( ) , [ & ] ( const MachineInstr & MI ) { return MI . readsRegister ( CondReg , TRI ) ; } ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Cmp << '\n' ) ; LIS -> RemoveMachineInstrFromMaps ( * Cmp ) ; Cmp -> eraseFromParent ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( SelReg ) && MRI . use_nodbg_empty ( SelReg ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Sel << '\n' ) ; LIS -> RemoveMachineInstrFromMaps ( * Sel ) ; Sel -> eraseFromParent ( ) ; }


<s> if ( ! And || And -> getOpcode ( ) != AndOpc || ! And -> getOperand ( <NUM_LIT> ) . isReg ( ) || ! And -> getOperand ( <NUM_LIT> ) . isReg ( ) ) return <STR_LIT> ; MachineOperand * AndCC = & And -> getOperand ( <NUM_LIT> ) ; unsigned CmpReg = AndCC -> getReg ( ) ; unsigned CmpSubReg = AndCC -> getSubReg ( ) ; if ( CmpReg == ExecReg ) { AndCC = & And -> getOperand ( <NUM_LIT> ) ; CmpReg = AndCC -> getReg ( ) ; CmpSubReg = AndCC -> getSubReg ( ) ; } else if ( And -> getOperand ( <NUM_LIT> ) . getReg ( ) != ExecReg ) { return <STR_LIT> ; } auto * Cmp = TRI -> findReachingDef ( CmpReg , CmpSubReg , * And , MRI , LIS ) ; if ( ! Cmp || ! ( Cmp -> getOpcode ( ) == <STR_LIT> || Cmp -> getOpcode ( ) == <STR_LIT> ) || Cmp -> getParent ( ) != And -> getParent ( ) ) return <STR_LIT> ; MachineOperand * Op1 = TII -> getNamedOperand ( * Cmp , <STR_LIT> :: src0 ) ; MachineOperand * Op2 = TII -> getNamedOperand ( * Cmp , <STR_LIT> :: src1 ) ; if ( Op1 -> isImm ( ) && Op2 -> isReg ( ) ) std :: swap ( Op1 , Op2 ) ; if ( ! Op1 -> isReg ( ) || ! Op2 -> isImm ( ) || Op2 -> getImm ( ) != <NUM_LIT> ) return <STR_LIT> ; unsigned SelReg = Op1 -> getReg ( ) ; auto * Sel = TRI -> findReachingDef ( SelReg , Op1 -> getSubReg ( ) , * Cmp , MRI , LIS ) ; if ( ! Sel || Sel -> getOpcode ( ) != <STR_LIT> ) return <STR_LIT> ; if ( TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src0_modifiers ) || TII -> hasModifiersSet ( * Sel , <STR_LIT> :: src1_modifiers ) ) return <STR_LIT> ;

Op1 = TII -> getNamedOperand ( * Sel , <STR_LIT> :: src0 ) ;


<s> MachineInstr & Term = MBB . back ( ) ; if ( Term . getOpcode ( ) != <STR_LIT> || Term . getNumOperands ( ) != <NUM_LIT> ) continue ; SmallVector < MachineBasicBlock * , <NUM_LIT> > Blocks ( { & MBB } ) ; while ( ! Blocks . empty ( ) ) { auto CurBB = Blocks . pop_back_val ( ) ; auto I = CurBB -> rbegin ( ) , E = CurBB -> rend ( ) ; if ( I != E ) { if ( I -> isUnconditionalBranch ( ) || I -> getOpcode ( ) == <STR_LIT> ) ++ I ; else if ( I -> isBranch ( ) ) continue ; } while ( I != E ) { if ( I -> isDebugInstr ( ) ) { I = std :: next ( I ) ; continue ; } if ( I -> mayStore ( ) || I -> isBarrier ( ) || I -> isCall ( ) || I -> hasUnmodeledSideEffects ( ) || I -> hasOrderedMemoryRef ( ) ) break ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * I << '\n' ) ; for ( auto & Op : I -> operands ( ) ) { if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } auto Next = std :: next ( I ) ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; I -> eraseFromParent ( ) ; I = Next ; Changed = true ; } if ( I != E ) continue ; for ( auto * Pred : CurBB -> predecessors ( ) ) { if ( Pred -> succ_size ( ) == <NUM_LIT> ) Blocks . push_back ( Pred ) ; } } continue ;


<s> const MachineBasicBlock * Succ = * MBB . succ_begin ( ) ; if ( ! MBB . isLayoutSuccessor ( Succ ) ) continue ; auto I = std :: next ( Lead ) ; for ( ; I != E ; ++ I ) if ( ! TII -> isSALU ( * I ) || I -> readsRegister ( <STR_LIT> , TRI ) ) break ; if ( I != E ) continue ; const auto NextLead = Succ -> begin ( ) ; if ( NextLead == Succ -> end ( ) || ! isEndCF ( * NextLead , TRI ) || ! getOrExecSource ( * NextLead , * TII , MRI ) ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * Lead << '\n' ) ; auto SaveExec = getOrExecSource ( * Lead , * TII , MRI ) ; unsigned SaveExecReg = getOrNonExecReg ( * Lead , * TII ) ; for ( auto & Op : Lead -> operands ( ) ) { if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } LIS -> RemoveMachineInstrFromMaps ( * Lead ) ; Lead -> eraseFromParent ( ) ; if ( SaveExecReg ) { LIS -> removeInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( SaveExecReg ) ; } Changed = true ; if ( ! SaveExec || ! SaveExec -> isFullCopy ( ) ) continue ; unsigned SavedExec = SaveExec -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool SafeToReplace = true ; for ( auto & U : MRI . use_nodbg_instructions ( SavedExec ) ) { if ( U . getParent ( ) != SaveExec -> getParent ( ) ) { SafeToReplace = false ; break ; } DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExec << '\n' ) ; } if ( SafeToReplace ) { LIS -> RemoveMachineInstrFromMaps ( * SaveExec ) ; SaveExec -> eraseFromParent ( ) ; MRI . replaceRegWith ( SavedExec , <STR_LIT> ) ; LIS -> removeInterval ( SavedExec ) ; } }


<s> auto Lead = MBB . begin ( ) , E = MBB . end ( ) ; if ( MBB . succ_size ( ) != <NUM_LIT> || Lead == E || ! isEndCF ( * Lead , TRI ) ) continue ; const MachineBasicBlock * Succ = * MBB . succ_begin ( ) ; if ( ! MBB . isLayoutSuccessor ( Succ ) ) continue ; auto I = std :: next ( Lead ) ; for ( ; I != E ; ++ I ) if ( ! TII -> isSALU ( * I ) || I -> readsRegister ( <STR_LIT> , TRI ) ) break ; if ( I != E ) continue ; const auto NextLead = Succ -> begin ( ) ; if ( NextLead == Succ -> end ( ) || ! isEndCF ( * NextLead , TRI ) || ! getOrExecSource ( * NextLead , * TII , MRI ) ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << * Lead << '\n' ) ; auto SaveExec = getOrExecSource ( * Lead , * TII , MRI ) ; unsigned SaveExecReg = getOrNonExecReg ( * Lead , * TII ) ; for ( auto & Op : Lead -> operands ( ) ) { if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } LIS -> RemoveMachineInstrFromMaps ( * Lead ) ; Lead -> eraseFromParent ( ) ; if ( SaveExecReg ) { LIS -> removeInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( SaveExecReg ) ; } Changed = true ; if ( ! SaveExec || ! SaveExec -> isFullCopy ( ) ) continue ; unsigned SavedExec = SaveExec -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool SafeToReplace = true ; for ( auto & U : MRI . use_nodbg_instructions ( SavedExec ) ) { if ( U . getParent ( ) != SaveExec -> getParent ( ) ) { SafeToReplace = false ; break ; } DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExec << '\n' ) ; } if ( SafeToReplace ) {

LIS -> RemoveMachineInstrFromMaps ( * SaveExec ) ;


<s> } } continue ; } auto Lead = MBB . begin ( ) , E = MBB . end ( ) ; if ( MBB . succ_size ( ) != <NUM_LIT> || Lead == E || ! isEndCF ( * Lead , TRI ) ) continue ; const MachineBasicBlock * Succ = * MBB . succ_begin ( ) ; if ( ! MBB . isLayoutSuccessor ( Succ ) ) continue ; auto I = std :: next ( Lead ) ; for ( ; I != E ; ++ I ) if ( ! TII -> isSALU ( * I ) || I -> readsRegister ( <STR_LIT> , TRI ) ) break ; if ( I != E ) continue ; const auto NextLead = Succ -> begin ( ) ; if ( NextLead == Succ -> end ( ) || ! isEndCF ( * NextLead , TRI ) || ! getOrExecSource ( * NextLead , * TII , MRI ) ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Lead << '\n' ) ; auto SaveExec = getOrExecSource ( * Lead , * TII , MRI ) ; unsigned SaveExecReg = getOrNonExecReg ( * Lead , * TII ) ; for ( auto & Op : Lead -> operands ( ) ) { if ( Op . isReg ( ) ) RecalcRegs . insert ( Op . getReg ( ) ) ; } LIS -> RemoveMachineInstrFromMaps ( * Lead ) ; Lead -> eraseFromParent ( ) ; if ( SaveExecReg ) { LIS -> removeInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( SaveExecReg ) ; } Changed = true ; if ( ! SaveExec || ! SaveExec -> isFullCopy ( ) ) continue ; unsigned SavedExec = SaveExec -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool SafeToReplace = true ; for ( auto & U : MRI . use_nodbg_instructions ( SavedExec ) ) { if ( U . getParent ( ) != SaveExec -> getParent ( ) ) { SafeToReplace = false ; break ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * SaveExec << '\n' ) ; } if ( SafeToReplace ) { LIS -> RemoveMachineInstrFromMaps ( * SaveExec ) ; SaveExec -> eraseFromParent ( ) ; MRI . replaceRegWith ( SavedExec , <STR_LIT> ) ; LIS -> removeInterval ( SavedExec ) ; } } if ( Changed ) { for ( auto Reg : RecalcRegs ) { if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) ) { LIS -> removeInterval ( Reg ) ; if ( ! MRI . reg_empty ( Reg ) ) LIS -> createAndComputeVirtRegInterval ( Reg ) ; } else { for ( MCRegUnitIterator U ( Reg , TRI ) ; U . isValid ( ) ; ++ U ) LIS -> removeRegUnit ( * U ) ; }


return new <ISA_LIT>OptimizeExecMaskingPreRA ( ) ;


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> StringRef getPassName ( ) const override {


<s> I -- ; } if ( ! AndExecMI ) return false ; SlotIndex StartIdx = LIS -> getInstructionIndex ( SaveExecMI ) ; SlotIndex EndIdx = LIS -> getInstructionIndex ( * AndExecMI ) ; for ( MCRegUnitIterator UI ( ExecReg , TRI ) ; UI . isValid ( ) ; ++ UI ) { LiveRange & RegUnit = LIS -> getRegUnit ( * UI ) ; if ( RegUnit . find ( StartIdx ) != std :: prev ( RegUnit . find ( EndIdx ) ) ) return false ; } LIS -> removeInterval ( SavedExecReg ) ; LIS -> removeInterval ( DstReg ) ; SaveExecMI . getOperand ( <NUM_LIT> ) . setReg ( DstReg ) ; LIS -> RemoveMachineInstrFromMaps ( * AndExecMI ) ; AndExecMI -> eraseFromParent ( ) ; LIS -> createAndComputeVirtRegInterval ( DstReg ) ;


<s> for ( unsigned N = <NUM_LIT> ; N <= InstLimit && I != E ; ++ I , ++ N ) { Register CopyFromExec = isCopyFromExec ( * I , ST ) ;

if ( CopyFromExec . isValid ( ) ) return I ;


<s> for ( unsigned N = <NUM_LIT> ; N <= InstLimit && I != E ; ++ I , ++ N ) { Register CopyFromExec = isCopyFromExec ( * I , ST ) ; if ( CopyFromExec . isValid ( ) ) return I ;


<s> unsigned CurrentIteration = <NUM_LIT> ; for ( ++ A ; CurrentIteration < MaxInstructions && A != E ; ++ A ) { if ( Pred ( & * A ) ) return & * A ; for ( MCRegister Reg : NonModifiableRegs ) { if ( A -> modifiesRegister ( Reg , TRI ) ) return nullptr ;


<s> static MachineBasicBlock :: reverse_iterator fixTerminators ( const <ISA_LIT>InstrInfo & TII , MachineBasicBlock & MBB ) { MachineBasicBlock :: reverse_iterator I = MBB . rbegin ( ) , E = MBB . rend ( ) ; bool Seen = false ; MachineBasicBlock :: reverse_iterator FirstNonTerm = I ; for ( ; I != E ; ++ I ) { if ( ! I -> isTerminator ( ) ) return Seen ? FirstNonTerm : I ; if ( removeTerminatorBit ( TII , * I ) ) { if ( ! Seen ) { FirstNonTerm = I ; Seen = true ; } }


<s> return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; default : return <STR_LIT> ;


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isReg ( ) && Src . getReg ( ) == ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> static Register isCopyFromExec ( const MachineInstr & MI , const GCNSubtarget & ST ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isReg ( ) && Src . getReg ( ) == ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && Dst . getReg ( ) == ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) && MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; break ; } case <STR_LIT> : case <STR_LIT> :


<s> case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && Dst . getReg ( ) == ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) && MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; break ; } case <STR_LIT> : case <STR_LIT> : llvm_unreachable ( <STR_LIT> ) ; }


<s> static bool isLiveOut ( const MachineBasicBlock & MBB , unsigned Reg ) {

for ( MachineBasicBlock * Succ : MBB . successors ( ) ) {


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand & Src1 = MI . getOperand ( <NUM_LIT> ) ; if ( Src1 . isReg ( ) && Src1 . getReg ( ) == <STR_LIT> ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const MachineOperand & Src2 = MI . getOperand ( <NUM_LIT> ) ; if ( Src2 . isReg ( ) && Src2 . getReg ( ) == <STR_LIT> ) return MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> LivePhysRegs LR ( * TRI ) ; LR . addLiveOuts ( * Stop . getParent ( ) ) ; for ( auto A = Stop . getParent ( ) -> rbegin ( ) ; A != Stop . getParent ( ) -> rend ( ) && A != Stop ; ++ A ) { LR . stepBackward ( * A ) ; } return ! LR . available ( MRI , Reg ) ;


<s> LivePhysRegs LR ( * TRI ) ; LR . addLiveOuts ( * Stop . getParent ( ) ) ;

for ( auto A = Stop . getParent ( ) -> rbegin ( ) ;


<s> Register MoveDest = SaveExecInstr . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock :: instr_iterator InsertPosIt = SaveExecInstr . getIterator ( ) ; if ( ! SaveExecInstr . uses ( ) . empty ( ) ) { bool isSGPR32 = TRI -> getRegSizeInBits ( MoveDest , MRI ) == <NUM_LIT> ; unsigned MovOpcode = isSGPR32 ? <STR_LIT> : <STR_LIT> ; BuildMI ( * SaveExecInstr . getParent ( ) , InsertPosIt , SaveExecInstr . getDebugLoc ( ) , TII -> get ( MovOpcode ) , MoveDest ) . addReg ( Exec ) ; } auto Builder = BuildMI ( * VCmp . getParent ( ) , std :: next ( InsertPosIt ) , VCmp . getDebugLoc ( ) , TII -> get ( NewOpcode ) ) ; auto TryAddImmediateValueFromNamedOperand = [ & ] ( unsigned OperandName ) -> void {

if ( auto * Mod = TII -> getNamedOperand ( VCmp , OperandName ) ) Builder . addImm ( Mod -> getImm ( ) ) ;


<s> bool Changed = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: reverse_iterator I = fixTerminators ( * TII , MBB ) ; MachineBasicBlock :: reverse_iterator E = MBB . rend ( ) ; if ( I == E ) continue ; Register CopyToExec ; unsigned SearchCount = <NUM_LIT> ; const unsigned SearchLimit = <NUM_LIT> ; while ( I != E && SearchCount ++ < SearchLimit ) { CopyToExec = isCopyToExec ( * I , ST ) ; if ( CopyToExec ) break ; ++ I ; } if ( ! CopyToExec ) continue ; auto CopyToExecInst = & * I ; auto CopyFromExecInst = findExecCopy ( * TII , ST , MBB , I , CopyToExec ) ; if ( CopyFromExecInst == E ) { auto PrepareExecInst = std :: next ( I ) ; if ( PrepareExecInst == E ) continue ; if ( CopyToExecInst -> getOperand ( <NUM_LIT> ) . isKill ( ) && isLogicalOpOnExec ( * PrepareExecInst ) == CopyToExec ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PrepareExecInst ) ; PrepareExecInst -> getOperand ( <NUM_LIT> ) . setReg ( Exec ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * PrepareExecInst << '\n' ) ; CopyToExecInst -> eraseFromParent ( ) ; Changed = true ; } continue ; } if ( isLiveOut ( MBB , CopyToExec ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; continue ; } Register CopyFromExec = CopyFromExecInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * SaveExecInst = nullptr ; SmallVector < MachineInstr * , <NUM_LIT> > OtherUseInsts ; for ( MachineBasicBlock :: iterator J = std :: next ( CopyFromExecInst -> getIterator ( ) ) , JE = I -> getIterator ( ) ; J != JE ; ++ J ) { if ( SaveExecInst && J -> readsRegister ( Exec , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * J << '\n' ) ; SaveExecInst = nullptr ; break ; } bool ReadsCopyFromExec = J -> readsRegister ( CopyFromExec , TRI ) ; if ( J -> modifiesRegister ( CopyToExec , TRI ) ) { if ( SaveExecInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( CopyToExec , TRI ) << '\n' ) ; SaveExecInst = nullptr ; break ; } unsigned SaveExecOp = getSaveExecOp ( J -> getOpcode ( ) ) ; if ( SaveExecOp == <STR_LIT> ) break ; if ( ReadsCopyFromExec ) {


<s> void <ISA_LIT>OptimizeVGPRLiveRange :: collectWaterfallCandidateRegisters ( MachineBasicBlock * Loop , SmallSetVector < Register , <NUM_LIT> > & CandidateRegs ) const { for ( auto & MI : Loop -> instrs ( ) ) { if ( MI . isDebugInstr ( ) ) continue ; for ( auto & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . getReg ( ) || MO . isDef ( ) ) continue ; Register MOReg = MO . getReg ( ) ; if ( MOReg . isPhysical ( ) || ! TRI -> isVectorRegister ( * MRI , MOReg ) ) continue ; if ( MO . readsReg ( ) ) { const MachineBasicBlock * DefMBB = MRI -> getVRegDef ( MOReg ) -> getParent ( ) ; if ( DefMBB != Loop && ! CandidateRegs . contains ( MOReg ) ) { LiveVariables :: VarInfo & OldVarInfo = LV -> getVarInfo ( MOReg ) ; bool IsUsed = false ; for ( auto * Succ : Loop -> successors ( ) ) { if ( Succ != Loop && OldVarInfo . isLiveIn ( * Succ , MOReg , * MRI ) ) { IsUsed = true ; break ; } } if ( ! IsUsed ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( MOReg , TRI , <NUM_LIT> , MRI ) << '\n' ) ; CandidateRegs . insert ( MOReg ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( MOReg , TRI , <NUM_LIT> , MRI ) << '\n' ) ; } } } } }


<s> for ( auto & O : make_early_inc_range ( MRI -> use_operands ( Reg ) ) ) { auto * UseMI = O . getParent ( ) ; auto * UseBlock = UseMI -> getParent ( ) ; if ( UseBlock == Loop ) O . setReg ( NewReg ) ; } MachineInstrBuilder PHI = BuildMI ( * Loop , Loop -> getFirstNonPHI ( ) , DebugLoc ( ) , TII -> get ( TargetOpcode :: PHI ) , NewReg ) ; for ( auto * Pred : Loop -> predecessors ( ) ) {

if ( Pred == Loop ) PHI . addReg ( UndefReg , RegState :: Undef ) . addMBB ( Pred ) ;


<s>
MDT
=
&
getAnalysis
<
MachineDominatorTree
>
(
)
;
Loops
=
&
getAnalysis
<
MachineLoopInfo
>
(
)
;
LV
=
&
getAnalysis
<
LiveVariables
>
(
)
;
MRI
=
&
MF
.
getRegInfo
(
)
;
if
(
skipFunction
(
MF
.
getFunction
(
)
)
)
return
false
;
bool
MadeChange
=
false
;
for
(
MachineBasicBlock
&
MBB
:
MF
)
{
for
(
auto
&
MI
:
MBB
.
terminators
(
)
)
{
if
(
MI
.
getOpcode
(
)
==
<STR_LIT>
)
{
MachineBasicBlock
*
IfTarget
=
MI
.
getOperand
(
<NUM_LIT>
)
.
getMBB
(
)
;
auto
*
Endif
=
getElseTarget
(
IfTarget
)
;
if
(
!
Endif
)
continue
;
SmallSetVector
<
MachineBasicBlock
*
,
<NUM_LIT>
>
ElseBlocks
;
SmallVector
<
Register
>
CandidateRegs
;
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
printMBBReference
(
MBB
)
<<
' '
<<
printMBBReference
(
*
IfTarget
)
<<
' '
<<
printMBBReference
(
*
Endif
)
<<
'\n'
)
;
collectElseRegionBlocks
(
IfTarget
,
Endif
,
ElseBlocks
)
;
collectCandidateRegisters
(
&
MBB
,
IfTarget
,
Endif
,
ElseBlocks
,
CandidateRegs
)
;
MadeChange
|=
!
CandidateRegs
.
empty
(
)
;
for
(
auto
Reg
:
CandidateRegs
)
optimizeLiveRange
(
Reg
,
&
MBB
,
IfTarget
,
Endif
,
ElseBlocks
)
;
}
else
if
(
MI
.
getOpcode
(
)
==
<STR_LIT>
)
{
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
printMBBReference
(
MBB
)
<<
'\n'
)
;
SmallSetVector
<
Register
,
<NUM_LIT>
>
CandidateRegs
;
collectWaterfallCandidateRegisters
(
&
MBB
,
CandidateRegs
)
;
MadeChange
|=
!
CandidateRegs
.
empty
(
)
;
for
(
auto
Reg
:
CandidateRegs
)
optimizeWaterfallLiveRange
(
Reg
,
&
MBB
)
;
}
}
}


<s>
for
(
auto
&
MI
:
MBB
.
terminators
(
)
)
{
if
(
MI
.
getOpcode
(
)
==
<STR_LIT>
)
{
MachineBasicBlock
*
IfTarget
=
MI
.
getOperand
(
<NUM_LIT>
)
.
getMBB
(
)
;
auto
*
Endif
=
getElseTarget
(
IfTarget
)
;
if
(
!
Endif
)
continue
;
SmallSetVector
<
MachineBasicBlock
*
,
<NUM_LIT>
>
ElseBlocks
;
SmallVector
<
Register
>
CandidateRegs
;
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
printMBBReference
(
MBB
)
<<
' '
<<
printMBBReference
(
*
IfTarget
)
<<
' '
<<
printMBBReference
(
*
Endif
)
<<
'\n'
)
;
collectElseRegionBlocks
(
IfTarget
,
Endif
,
ElseBlocks
)
;
collectCandidateRegisters
(
&
MBB
,
IfTarget
,
Endif
,
ElseBlocks
,
CandidateRegs
)
;
MadeChange
|=
!
CandidateRegs
.
empty
(
)
;
for
(
auto
Reg
:
CandidateRegs
)
optimizeLiveRange
(
Reg
,
&
MBB
,
IfTarget
,
Endif
,
ElseBlocks
)
;
}
else
if
(
MI
.
getOpcode
(
)
==
<STR_LIT>
)
{

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( MBB ) << '\n' ) ;


<s> } for ( auto I = MRI -> use_nodbg_begin ( Reg ) , E = MRI -> use_nodbg_end ( ) ; I != E ; ++ I ) { auto * UseMI = I -> getParent ( ) ; if ( UseMI -> isPHI ( ) && I -> readsReg ( ) ) { if ( Visited . contains ( UseMI -> getParent ( ) ) ) PHIIncoming . insert ( UseMI -> getOperand ( I . getOperandNo ( ) + <NUM_LIT> ) . getMBB ( ) ) ; } } Visited . clear ( ) ; for ( MachineBasicBlock * MBB : depth_first_ext ( ThenEntry , Visited ) ) { if ( MBB == Flow ) break ; SmallVector < MachineInstr * > Uses ; findNonPHIUsesInBlock ( Reg , MBB , Uses ) ; if ( Uses . size ( ) == <NUM_LIT> ) {

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MBB ) << '\n' ) ;


<s> if ( UseMI -> isPHI ( ) && I -> readsReg ( ) ) { if ( Visited . contains ( UseMI -> getParent ( ) ) ) PHIIncoming . insert ( UseMI -> getOperand ( I . getOperandNo ( ) + <NUM_LIT> ) . getMBB ( ) ) ; } } Visited . clear ( ) ; for ( MachineBasicBlock * MBB : depth_first_ext ( ThenEntry , Visited ) ) { if ( MBB == Flow ) break ; SmallVector < MachineInstr * > Uses ; findNonPHIUsesInBlock ( Reg , MBB , Uses ) ; if ( Uses . size ( ) == <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MBB ) << '\n' ) ; LV -> HandleVirtRegUse ( Reg , MBB , * ( * Uses . begin ( ) ) ) ; } else if ( Uses . size ( ) > <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << Uses . size ( ) << <STR_LIT> << printMBBReference ( * MBB ) << '\n' ) ; for ( MachineInstr & MI : * MBB ) { if ( llvm :: is_contained ( Uses , & MI ) ) LV -> HandleVirtRegUse ( Reg , MBB , MI ) ;


<s> if ( MOReg . isPhysical ( ) || ! TRI -> isVectorRegister ( * MRI , MOReg ) ) continue ; if ( MO . readsReg ( ) ) { MachineBasicBlock * DefMBB = MRI -> getVRegDef ( MOReg ) -> getParent ( ) ; if ( ! Blocks . contains ( DefMBB ) && ! CandidateRegs . contains ( MOReg ) ) { LiveVariables :: VarInfo & OldVarInfo = LV -> getVarInfo ( MOReg ) ; bool IsUsed = false ; for ( auto * Succ : LoopEnd -> successors ( ) ) { if ( ! Blocks . contains ( Succ ) && OldVarInfo . isLiveIn ( * Succ , MOReg , * MRI ) ) { IsUsed = true ; break ; } } if ( ! IsUsed ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( MOReg , TRI , <NUM_LIT> , MRI ) << '\n' ) ; CandidateRegs . insert ( MOReg ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( MOReg , TRI , <NUM_LIT> , MRI ) << '\n' ) ; }


<s> if ( ( VI . AliveBlocks . test ( If -> getNumber ( ) ) || DefMBB == If ) && Loops -> getLoopFor ( DefMBB ) == Loops -> getLoopFor ( If ) ) KillsInElse . insert ( Reg ) ; } } auto IsLiveThroughThen = [ & ] ( Register Reg ) { for ( auto I = MRI -> use_nodbg_begin ( Reg ) , E = MRI -> use_nodbg_end ( ) ; I != E ; ++ I ) { if ( ! I -> readsReg ( ) ) continue ; auto * UseMI = I -> getParent ( ) ; auto * UseMBB = UseMI -> getParent ( ) ; if ( UseMBB == Flow || UseMBB == Endif ) { if ( ! UseMI -> isPHI ( ) ) return true ; auto * IncomingMBB = UseMI -> getOperand ( I . getOperandNo ( ) + <NUM_LIT> ) . getMBB ( ) ; if ( ( UseMBB == Flow && IncomingMBB != If ) || ( UseMBB == Endif && IncomingMBB == Flow ) ) return true ; } } return false ; } ; for ( auto Reg : KillsInElse ) {

if ( ! IsLiveThroughThen ( Reg ) ) CandidateRegs . push_back ( Reg ) ;


<s> if ( ! VI . isLiveIn ( * Endif , MOReg , * MRI ) ) { KillsInElse . insert ( MOReg ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( MOReg , TRI ) << <STR_LIT> ) ; } } } } } } for ( auto & MI : Endif -> phis ( ) ) { for ( unsigned Idx = <NUM_LIT> ; Idx < MI . getNumOperands ( ) ; Idx += <NUM_LIT> ) { auto & MO = MI . getOperand ( Idx ) ; auto * Pred = MI . getOperand ( Idx + <NUM_LIT> ) . getMBB ( ) ; if ( Pred == Flow ) continue ; assert ( ElseBlocks . contains ( Pred ) && <STR_LIT> ) ; if ( ! MO . isReg ( ) || ! MO . getReg ( ) || MO . isUndef ( ) ) continue ; Register Reg = MO . getReg ( ) ; if ( Reg . isPhysical ( ) || ! TRI -> isVectorRegister ( * MRI , Reg ) ) continue ; LiveVariables :: VarInfo & VI = LV -> getVarInfo ( Reg ) ; if ( VI . isLiveIn ( * Endif , Reg , * MRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( Reg , TRI ) << <STR_LIT> ) ; continue ; } const MachineBasicBlock * DefMBB = MRI -> getVRegDef ( Reg ) -> getParent ( ) ; if ( ( VI . AliveBlocks . test ( If -> getNumber ( ) ) || DefMBB == If ) && Loops -> getLoopFor ( DefMBB ) == Loops -> getLoopFor ( If ) ) KillsInElse . insert ( Reg ) ; } } auto IsLiveThroughThen = [ & ] ( Register Reg ) { for ( auto I = MRI -> use_nodbg_begin ( Reg ) , E = MRI -> use_nodbg_end ( ) ; I != E ; ++ I ) { if ( ! I -> readsReg ( ) ) continue ; auto * UseMI = I -> getParent ( ) ; auto * UseMBB = UseMI -> getParent ( ) ; if ( UseMBB == Flow || UseMBB == Endif ) { if ( ! UseMI -> isPHI ( ) ) return true ;

auto * IncomingMBB = UseMI -> getOperand ( I . getOperandNo ( ) + <NUM_LIT> ) . getMBB ( ) ;


<s> unsigned Cur = <NUM_LIT> ; while ( MBB ) { for ( auto * Pred : MBB -> predecessors ( ) ) { if ( Pred != Flow && ! Blocks . contains ( Pred ) ) Blocks . insert ( Pred ) ; } if ( Cur < Blocks . size ( ) ) MBB = Blocks [ Cur ++ ] ; else MBB = nullptr ; }


<s> for ( auto * Pred : MBB -> predecessors ( ) ) { if ( Pred != Flow && ! Blocks . contains ( Pred ) ) Blocks . insert ( Pred ) ; } if ( Cur < Blocks . size ( ) ) MBB = Blocks [ Cur ++ ] ; else MBB = nullptr ; }


<s> void <ISA_LIT>OptimizeVGPRLiveRange :: collectWaterfallCandidateRegisters ( MachineBasicBlock * LoopHeader , MachineBasicBlock * LoopEnd , SmallSetVector < Register , <NUM_LIT> > & CandidateRegs , SmallSetVector < MachineBasicBlock * , <NUM_LIT> > & Blocks , SmallVectorImpl < MachineInstr * > & Instructions ) const { auto * MBB = LoopHeader ; for ( ; ; ) { Blocks . insert ( MBB ) ; for ( auto & MI : * MBB ) { if ( MI . isDebugInstr ( ) ) continue ; Instructions . push_back ( & MI ) ; } if ( MBB == LoopEnd ) break ; assert ( MBB -> pred_size ( ) == <NUM_LIT> || ( MBB == LoopHeader && MBB -> pred_size ( ) == <NUM_LIT> ) ) ; assert ( MBB -> succ_size ( ) == <NUM_LIT> ) ; MBB = * MBB -> succ_begin ( ) ; } for ( auto * I : Instructions ) { auto & MI = * I ; for ( auto & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . getReg ( ) || MO . isDef ( ) ) continue ; Register MOReg = MO . getReg ( ) ; if ( MOReg . isPhysical ( ) || ! TRI -> isVectorRegister ( * MRI , MOReg ) ) continue ; if ( MO . readsReg ( ) ) { MachineBasicBlock * DefMBB = MRI -> getVRegDef ( MOReg ) -> getParent ( ) ; if ( ! Blocks . contains ( DefMBB ) && ! CandidateRegs . contains ( MOReg ) ) { LiveVariables :: VarInfo & OldVarInfo = LV -> getVarInfo ( MOReg ) ; bool IsUsed = false ; for ( auto * Succ : LoopEnd -> successors ( ) ) { if ( ! Blocks . contains ( Succ ) && OldVarInfo . isLiveIn ( * Succ , MOReg , * MRI ) ) { IsUsed = true ; break ; } } if ( ! IsUsed ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( MOReg , TRI , <NUM_LIT> , MRI ) << '\n' ) ; CandidateRegs . insert ( MOReg ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printReg ( MOReg , TRI , <NUM_LIT> , MRI ) << '\n' ) ; } } }


<s> if ( MBB == LoopEnd ) break ; assert ( MBB -> pred_size ( ) == <NUM_LIT> || ( MBB == LoopHeader && MBB -> pred_size ( ) == <NUM_LIT> ) ) ; assert ( MBB -> succ_size ( ) == <NUM_LIT> ) ; MBB = * MBB -> succ_begin ( ) ; } for ( auto * I : Instructions ) { auto & MI = * I ; for ( auto & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) || ! MO . getReg ( ) || MO . isDef ( ) ) continue ; Register MOReg = MO . getReg ( ) ; if ( MOReg . isPhysical ( ) || ! TRI -> isVectorRegister ( * MRI , MOReg ) ) continue ; if ( MO . readsReg ( ) ) { MachineBasicBlock * DefMBB = MRI -> getVRegDef ( MOReg ) -> getParent ( ) ; if ( ! Blocks . contains ( DefMBB ) && ! CandidateRegs . contains ( MOReg ) ) { LiveVariables :: VarInfo & OldVarInfo = LV -> getVarInfo ( MOReg ) ; bool IsUsed = false ;

for ( auto * Succ : LoopEnd -> successors ( ) ) {


<s> void <ISA_LIT>OptimizeVGPRLiveRange :: findNonPHIUsesInBlock ( Register Reg , MachineBasicBlock * MBB , SmallVectorImpl < MachineInstr * > & Uses ) const { for ( auto & UseMI : MRI -> use_nodbg_instructions ( Reg ) ) { if ( UseMI . getParent ( ) == MBB && ! UseMI . isPHI ( ) ) Uses . push_back ( & UseMI ) ; }


<s> AU . addRequired < MachineDominatorTree > ( ) ; AU . addRequired < MachineLoopInfo > ( ) ; AU . addPreserved < LiveVariables > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ;

AU . addPreserved < MachineLoopInfo > ( ) ;


<s> AU . addPreserved < LiveVariables > ( ) ; AU . addPreserved < MachineDominatorTree > ( ) ;


<s> return MachineFunctionProperties ( ) . set ( MachineFunctionProperties :: Property :: NoPHIs ) ;


<s> MachineFunctionProperties getClearedProperties ( ) const override { return MachineFunctionProperties ( ) . set ( MachineFunctionProperties :: Property :: NoPHIs ) ;


<s> if ( BR . getOpcode ( ) == <STR_LIT> ) return BR . getOperand ( <NUM_LIT> ) . getMBB ( ) ;


<s> for ( auto & BR : MBB -> terminators ( ) ) { if ( BR . getOpcode ( ) == <STR_LIT> ) return BR . getOperand ( <NUM_LIT> ) . getMBB ( ) ;


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> return MachineFunctionProperties ( ) . set ( MachineFunctionProperties :: Property :: IsSSA ) ;


<s> const auto * RC = MRI -> getRegClass ( Reg ) ; Register NewReg = MRI -> createVirtualRegister ( RC ) ; Register UndefReg = MRI -> createVirtualRegister ( RC ) ; MachineInstrBuilder PHI = BuildMI ( * Flow , Flow -> getFirstNonPHI ( ) , DebugLoc ( ) , TII -> get ( TargetOpcode :: PHI ) , NewReg ) ; for ( auto * Pred : Flow -> predecessors ( ) ) { if ( Pred == If ) PHI . addReg ( Reg ) . addMBB ( Pred ) ; else PHI . addReg ( UndefReg , RegState :: Undef ) . addMBB ( Pred ) ; } for ( auto & O : make_early_inc_range ( MRI -> use_operands ( Reg ) ) ) { auto * UseMI = O . getParent ( ) ; auto * UseBlock = UseMI -> getParent ( ) ; if ( UseBlock == Endif ) {

assert ( UseMI -> isPHI ( ) && <STR_LIT> ) ;


<s> for ( auto * Pred : LoopHeader -> predecessors ( ) ) { if ( Blocks . contains ( Pred ) ) PHI . addReg ( UndefReg , RegState :: Undef ) . addMBB ( Pred ) ; else PHI . addReg ( Reg ) . addMBB ( Pred ) ; } LiveVariables :: VarInfo & NewVarInfo = LV -> getVarInfo ( NewReg ) ; LiveVariables :: VarInfo & OldVarInfo = LV -> getVarInfo ( Reg ) ; MachineInstr * Kill = nullptr ; for ( auto * MI : reverse ( Instructions ) ) { if ( MI -> readsRegister ( NewReg , TRI ) ) { MI -> addRegisterKilled ( NewReg , TRI ) ; NewVarInfo . Kills . push_back ( MI ) ; Kill = MI ; break ; } } assert ( Kill && <STR_LIT> ) ; MachineBasicBlock * KillBlock = Kill -> getParent ( ) ; bool PostKillBlock = false ; for ( auto * Block : Blocks ) { auto BBNum = Block -> getNumber ( ) ; OldVarInfo . AliveBlocks . reset ( BBNum ) ; PostKillBlock |= ( Block == KillBlock ) ; if ( PostKillBlock ) { NewVarInfo . AliveBlocks . reset ( BBNum ) ; } else if ( Block != LoopHeader ) { NewVarInfo . AliveBlocks . set ( BBNum ) ;


<s>
SmallSetVector
<
MachineBasicBlock
*
,
<NUM_LIT>
>
ElseBlocks
;
SmallVector
<
Register
>
CandidateRegs
;
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
printMBBReference
(
MBB
)
<<
' '
<<
printMBBReference
(
*
IfTarget
)
<<
' '
<<
printMBBReference
(
*
Endif
)
<<
'\n'
)
;
collectElseRegionBlocks
(
IfTarget
,
Endif
,
ElseBlocks
)
;
collectCandidateRegisters
(
&
MBB
,
IfTarget
,
Endif
,
ElseBlocks
,
CandidateRegs
)
;
MadeChange
|=
!
CandidateRegs
.
empty
(
)
;
for
(
auto
Reg
:
CandidateRegs
)
optimizeLiveRange
(
Reg
,
&
MBB
,
IfTarget
,
Endif
,
ElseBlocks
)
;
}
else
if
(
MI
.
getOpcode
(
)
==
<STR_LIT>
)
{
auto
*
LoopHeader
=
MI
.
getOperand
(
<NUM_LIT>
)
.
getMBB
(
)
;
auto
*
LoopEnd
=
&
MBB
;
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
printMBBReference
(
*
LoopHeader
)
<<
'\n'
)
;
SmallSetVector
<
Register
,
<NUM_LIT>
>
CandidateRegs
;
SmallVector
<
MachineInstr
*
,
<NUM_LIT>
>
Instructions
;
SmallSetVector
<
MachineBasicBlock
*
,
<NUM_LIT>
>
Blocks
;
collectWaterfallCandidateRegisters
(
LoopHeader
,
LoopEnd
,
CandidateRegs
,
Blocks
,
Instructions
)
;
MadeChange
|=
!
CandidateRegs
.
empty
(
)
;
for
(
auto
Reg
:
CandidateRegs
)
optimizeWaterfallLiveRange
(
Reg
,
LoopHeader
,
Blocks
,
Instructions
)
;
}
}
}


<s> <ISA_LIT>OptimizeVGPRLiveRange ( ) : MachineFunctionPass ( ID ) {


<s> NewVarInfo . AliveBlocks . set ( BBNum ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MBB ) << '\n' ) ; OldVarInfo . AliveBlocks . reset ( BBNum ) ; } } auto I = OldVarInfo . Kills . begin ( ) ; while ( I != OldVarInfo . Kills . end ( ) ) { if ( ElseBlocks . contains ( ( * I ) -> getParent ( ) ) ) { NewVarInfo . Kills . push_back ( * I ) ; I = OldVarInfo . Kills . erase ( I ) ;


<s> void <ISA_LIT>OptimizeVGPRLiveRange :: updateLiveRangeInElseRegion ( Register Reg , Register NewReg , MachineBasicBlock * Flow , MachineBasicBlock * Endif , SmallSetVector < MachineBasicBlock * , <NUM_LIT> > & ElseBlocks ) const { LiveVariables :: VarInfo & NewVarInfo = LV -> getVarInfo ( NewReg ) ; LiveVariables :: VarInfo & OldVarInfo = LV -> getVarInfo ( Reg ) ; for ( auto * MBB : ElseBlocks ) { unsigned BBNum = MBB -> getNumber ( ) ; if ( OldVarInfo . AliveBlocks . test ( BBNum ) ) { NewVarInfo . AliveBlocks . set ( BBNum ) ;


<s> uint64_t SDWASrcOperand :: getSrcMods ( const <ISA_LIT>InstrInfo * TII , const MachineOperand * SrcOp ) const { uint64_t Mods = <NUM_LIT> ; const auto * MI = SrcOp -> getParent ( ) ; if ( TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) == SrcOp ) { if ( auto * Mod = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0_modifiers ) ) { Mods = Mod -> getImm ( ) ; } } else if ( TII -> getNamedOperand ( * MI , <STR_LIT> :: src1 ) == SrcOp ) { if ( auto * Mod = TII -> getNamedOperand ( * MI , <STR_LIT> :: src1_modifiers ) ) { Mods = Mod -> getImm ( ) ; } } if ( Abs || Neg ) { assert ( ! Sext && <STR_LIT> ) ; Mods |= Abs ? <STR_LIT> : <NUM_LIT> ; Mods ^= Neg ? <STR_LIT> : <NUM_LIT> ; } else if ( Sext ) { Mods |= <STR_LIT> ; }


<s> if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) Opc = <STR_LIT> ( Opc ) ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) return false ; if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ;

if ( SDst && ( SDst -> getReg ( ) != <STR_LIT> && SDst -> getReg ( ) != <STR_LIT> ) ) return false ;


<s> void <ISA_LIT>PeepholeSDWA :: pseudoOpConvertToVOP2 ( MachineInstr & MI , const GCNSubtarget & ST ) const { int Opc = MI . getOpcode ( ) ; assert ( ( Opc == <STR_LIT> || Opc == <STR_LIT> ) && <STR_LIT> ) ; if ( ! TII -> canShrink ( MI , * MRI ) ) return ; Opc = <STR_LIT> ( Opc ) ; const MachineOperand * Sdst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( ! Sdst ) return ; MachineOperand * NextOp = findSingleRegUse ( Sdst , MRI ) ; if ( ! NextOp ) return ; MachineInstr & MISucc = * NextOp -> getParent ( ) ; if ( ! TII -> canShrink ( MISucc , * MRI ) ) return ; int SuccOpc = <STR_LIT> ( MISucc . getOpcode ( ) ) ; MachineOperand * CarryIn = TII -> getNamedOperand ( MISucc , <STR_LIT> :: src2 ) ; if ( ! CarryIn ) return ; MachineOperand * CarryOut = TII -> getNamedOperand ( MISucc , <STR_LIT> :: sdst ) ; if ( ! CarryOut ) return ; if ( ! MRI -> hasOneUse ( CarryIn -> getReg ( ) ) || ! MRI -> use_empty ( CarryOut -> getReg ( ) ) ) return ; MachineBasicBlock & MBB = * MI . getParent ( ) ; auto Liveness = MBB . computeRegisterLiveness ( TRI , <STR_LIT> , MI , <NUM_LIT> ) ; if ( Liveness != MachineBasicBlock :: LQR_Dead ) return ; for ( auto I = std :: next ( MI . getIterator ( ) ) , E = MISucc . getIterator ( ) ; I != E ; ++ I ) { if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return ; }

auto NewMI = BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) ) ;


<s> if ( ! MRI -> hasOneUse ( CarryIn -> getReg ( ) ) || ! MRI -> use_empty ( CarryOut -> getReg ( ) ) ) return ; MachineBasicBlock & MBB = * MI . getParent ( ) ; auto Liveness = MBB . computeRegisterLiveness ( TRI , <STR_LIT> , MI , <NUM_LIT> ) ; if ( Liveness != MachineBasicBlock :: LQR_Dead ) return ; for ( auto I = std :: next ( MI . getIterator ( ) ) , E = MISucc . getIterator ( ) ; I != E ; ++ I ) { if ( I -> modifiesRegister ( <STR_LIT> , TRI ) ) return ; } auto NewMI = BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) ) ; NewMI . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) ; NewMI . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ) ; NewMI . add ( * TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ) ; MI . eraseFromParent ( ) ; auto NewInst = BuildMI ( MBB , MISucc , MISucc . getDebugLoc ( ) , TII -> get ( SuccOpc ) ) ; NewInst . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: vdst ) ) ; NewInst . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src0 ) ) ; NewInst . add ( * TII -> getNamedOperand ( MISucc , <STR_LIT> :: src1 ) ) ;


<s> } SDWAOperands . clear ( ) ; matchSDWAOperands ( MBB ) ; for ( const auto & OperandPair : SDWAOperands ) { const auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI && isConvertibleToSDWA ( * PotentialMI , ST ) ) { PotentialMatches [ PotentialMI ] . push_back ( Operand . get ( ) ) ; } } for ( auto & PotentialPair : PotentialMatches ) { MachineInstr & PotentialMI = * PotentialPair . first ; convertToSDWA ( PotentialMI , PotentialPair . second ) ; } PotentialMatches . clear ( ) ;


<s> MachineOperand * Clamp = TII -> getNamedOperand ( MI , <STR_LIT> :: clamp ) ; if ( Clamp ) { SDWAInst . add ( * Clamp ) ; } else { SDWAInst . addImm ( <NUM_LIT> ) ; } if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: omod ) != - <NUM_LIT> ) { MachineOperand * OMod = TII -> getNamedOperand ( MI , <STR_LIT> :: omod ) ; if ( OMod ) { SDWAInst . add ( * OMod ) ; } else { SDWAInst . addImm ( <NUM_LIT> ) ; } } if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_sel ) != - <NUM_LIT> ) { SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; } if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_unused ) != - <NUM_LIT> ) { SDWAInst . addImm ( <STR_LIT> :: DstUnused :: UNUSED_PAD ) ; } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_sel ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_sel ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; } bool Converted = false ; for ( auto & Operand : SDWAOperands ) { if ( PotentialMatches . count ( Operand -> getParentInst ( ) ) == <NUM_LIT> ) Converted |= Operand -> convertToSDWA ( * SDWAInst , TII ) ; } if ( Converted ) { ConvertedInstructions . push_back ( SDWAInst ) ; } else { SDWAInst -> eraseFromParent ( ) ; return false ;


<s> assert ( Dst && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: sdst ) != - <NUM_LIT> ) ; SDWAInst . add ( * Dst ) ; } else { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: sdst ) != - <NUM_LIT> ) ; SDWAInst . addReg ( <STR_LIT> , RegState :: Define ) ; } MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; assert ( Src0 && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ) ; if ( auto * Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ) SDWAInst . addImm ( Mod -> getImm ( ) ) ; else SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src0 ) ; MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_modifiers ) != - <NUM_LIT> ) ; if ( auto * Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ) SDWAInst . addImm ( Mod -> getImm ( ) ) ; else SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src1 ) ; } if ( SDWAOpcode == <STR_LIT> || SDWAOpcode == <STR_LIT> ) { MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; assert ( Src2 ) ; SDWAInst . add ( * Src2 ) ; } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: clamp ) != - <NUM_LIT> ) ; MachineOperand * Clamp = TII -> getNamedOperand ( MI , <STR_LIT> :: clamp ) ; if ( Clamp ) { SDWAInst . add ( * Clamp ) ; } else { SDWAInst . addImm ( <NUM_LIT> ) ; } if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: omod ) != - <NUM_LIT> ) { MachineOperand * OMod = TII -> getNamedOperand ( MI , <STR_LIT> :: omod ) ; if ( OMod ) { SDWAInst . add ( * OMod ) ; } else { SDWAInst . addImm ( <NUM_LIT> ) ; } }

if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_sel ) != - <NUM_LIT> ) {


<s> int Opc = MI . getOpcode ( ) ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) Opc = <STR_LIT> ( Opc ) ; if ( Opc == - <NUM_LIT> || <STR_LIT> ( Opc ) == - <NUM_LIT> ) return false ; if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) {

const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ;


<s> if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ; }

if ( ! ST . hasSDWAOutModsVOPC ( ) && ( TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) ) return false ;


<s> auto SDWADst = make_unique < SDWADstOperand > ( Dst , Src1 , * Imm == <NUM_LIT> ? WORD_1 : BYTE_3 , UNUSED_PAD ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWADst << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWADst ) ; ++ NumSDWAPatternsFound ; } else { auto SDWASrc = make_unique < SDWASrcOperand > ( Src1 , Dst , * Imm == <NUM_LIT> ? WORD_1 : BYTE_3 , false , false , Opcode != <STR_LIT> && Opcode != <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWASrc << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWASrc ) ; ++ NumSDWAPatternsFound ; } break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; auto Imm = foldToImm ( * Src0 ) ; if ( ! Imm || * Imm != <NUM_LIT> ) break ; MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; MachineOperand * Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ; if ( TRI -> isPhysicalRegister ( Src1 -> getReg ( ) ) || TRI -> isPhysicalRegister ( Dst -> getReg ( ) ) ) break ; if ( Opcode == <STR_LIT> || Opcode == <STR_LIT> ) { auto SDWADst = make_unique < SDWADstOperand > ( Dst , Src1 , BYTE_1 , UNUSED_PAD ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWADst << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWADst ) ; ++ NumSDWAPatternsFound ; } else { auto SDWASrc = make_unique < SDWASrcOperand > ( Src1 , Dst , BYTE_1 , false , false , Opcode != <STR_LIT> && Opcode != <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWASrc << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWASrc ) ; ++ NumSDWAPatternsFound ; } break ; } case <STR_LIT> : case <STR_LIT> : {


<s> } else if ( ( Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ) ) { assert ( Dst && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: sdst ) != - <NUM_LIT> ) ; SDWAInst . add ( * Dst ) ; } else { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: sdst ) != - <NUM_LIT> ) ; SDWAInst . addReg ( <STR_LIT> , RegState :: Define ) ; } MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; assert ( Src0 && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ) ; if ( auto * Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ) SDWAInst . addImm ( Mod -> getImm ( ) ) ; else SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src0 ) ; MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_modifiers ) != - <NUM_LIT> ) ; if ( auto * Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ) SDWAInst . addImm ( Mod -> getImm ( ) ) ; else SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src1 ) ; } if ( SDWAOpcode == <STR_LIT> || SDWAOpcode == <STR_LIT> ) { MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; assert ( Src2 ) ; SDWAInst . add ( * Src2 ) ; } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: clamp ) != - <NUM_LIT> ) ; MachineOperand * Clamp = TII -> getNamedOperand ( MI , <STR_LIT> :: clamp ) ; if ( Clamp ) { SDWAInst . add ( * Clamp ) ; } else { SDWAInst . addImm ( <NUM_LIT> ) ; } if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: omod ) != - <NUM_LIT> ) { MachineOperand * OMod = TII -> getNamedOperand ( MI , <STR_LIT> :: omod ) ;


<s> const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ; } if ( ! ST . hasSDWAOutModsVOPC ( ) && ( TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) ) return false ; } else if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) || ! TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) { return false ; }

if ( ! ST . hasSDWAMac ( ) && ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ) return false ;


<s> if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ; } if ( ! ST . hasSDWAOutModsVOPC ( ) && ( TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) ) return false ;

} else if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) || ! TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) {


<s> DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * Operand << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( Operand ) ; ++ NumSDWAPatternsFound ;


<s> if ( auto Operand = matchSDWAOperand ( MI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * Operand << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( Operand ) ; ++ NumSDWAPatternsFound ;


<s> TRI = ST . getRegisterInfo ( ) ; TII = ST . getInstrInfo ( ) ; bool Ret = false ; for ( MachineBasicBlock & MBB : MF ) { bool Changed = false ; do { matchSDWAOperands ( MBB ) ; for ( const auto & OperandPair : SDWAOperands ) { const auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI && isConvertibleToSDWA ( * PotentialMI , ST ) ) { PotentialMatches [ PotentialMI ] . push_back ( Operand . get ( ) ) ; } } for ( auto & PotentialPair : PotentialMatches ) { MachineInstr & PotentialMI = * PotentialPair . first ; convertToSDWA ( PotentialMI , PotentialPair . second ) ; } PotentialMatches . clear ( ) ;


<s> Mods = Mod -> getImm ( ) ; } } else if ( TII -> getNamedOperand ( * MI , <STR_LIT> :: src1 ) == SrcOp ) { if ( auto * Mod = TII -> getNamedOperand ( * MI , <STR_LIT> :: src1_modifiers ) ) { Mods = Mod -> getImm ( ) ; } } if ( Abs || Neg ) { assert ( ! Sext && <STR_LIT> ) ; Mods |= Abs ? <STR_LIT> : <NUM_LIT> ; Mods ^= Neg ? <STR_LIT> : <NUM_LIT> ; } else if ( Sext ) {


<s> if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) Opc = <STR_LIT> ( Opc ) ; if ( Opc == - <NUM_LIT> || <STR_LIT> ( Opc ) == - <NUM_LIT> ) return false ; if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ;

if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ;


<s> if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ; } if ( ! ST . hasSDWAClampVOPC ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) ) return false ; } else if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ) { return false ; } if ( ! ST . hasSDWAMac ( ) && ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ) return false ;


<s> if ( SuperReg . getReg ( ) != SubReg . getReg ( ) ) return false ; LaneBitmask SuperMask = TRI -> getSubRegIndexLaneMask ( SuperReg . getSubReg ( ) ) ; LaneBitmask SubMask = TRI -> getSubRegIndexLaneMask ( SubReg . getSubReg ( ) ) ;


<s> static bool isSubregOf ( const MachineOperand & SubReg , const MachineOperand & SuperReg , const TargetRegisterInfo * TRI ) { if ( ! SuperReg . isReg ( ) || ! SubReg . isReg ( ) ) return false ; if ( isSameReg ( SuperReg , SubReg ) ) return true ; if ( SuperReg . getReg ( ) != SubReg . getReg ( ) ) return false ; LaneBitmask SuperMask = TRI -> getSubRegIndexLaneMask ( SuperReg . getSubReg ( ) ) ; LaneBitmask SubMask = TRI -> getSubRegIndexLaneMask ( SubReg . getSubReg ( ) ) ; SuperMask |= ~ SubMask ;


<s> auto Copy = BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , VGPR ) ; if ( Op . isImm ( ) ) Copy . addImm ( Op . getImm ( ) ) ; else if ( Op . isReg ( ) ) Copy . addReg ( Op . getReg ( ) , Op . isKill ( ) ? RegState :: Kill : <NUM_LIT> , Op . getSubReg ( ) ) ; Op . ChangeToRegister ( VGPR , false ) ;


<s> MachineInstr * ParentMI = getParentInst ( ) ; MachineOperand * Replaced = getReplacedOperand ( ) ; assert ( Replaced -> isReg ( ) ) ; for ( MachineOperand & PotentialMO : MRI -> def_operands ( Replaced -> getReg ( ) ) ) { if ( ! isSubregOf ( * Replaced , PotentialMO , MRI -> getTargetRegisterInfo ( ) ) ) continue ; if ( ! isSameReg ( * Replaced , PotentialMO ) ) return nullptr ; for ( MachineOperand & UseMO : MRI -> use_operands ( PotentialMO . getReg ( ) ) ) { if ( isSubregOf ( UseMO , PotentialMO , MRI -> getTargetRegisterInfo ( ) ) && UseMO . getParent ( ) != ParentMI ) { return nullptr ; }


<s> if ( ! isSubregOf ( * Replaced , PotentialMO , MRI -> getTargetRegisterInfo ( ) ) ) continue ; if ( ! isSameReg ( * Replaced , PotentialMO ) ) return nullptr ; for ( MachineOperand & UseMO : MRI -> use_operands ( PotentialMO . getReg ( ) ) ) { if ( isSubregOf ( UseMO , PotentialMO , MRI -> getTargetRegisterInfo ( ) ) && UseMO . getParent ( ) != ParentMI ) {


<s> const auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI && isConvertibleToSDWA ( * PotentialMI , ST ) ) { PotentialMatches [ PotentialMI ] . push_back ( Operand . get ( ) ) ; } } for ( auto & PotentialPair : PotentialMatches ) { MachineInstr & PotentialMI = * PotentialPair . first ; convertToSDWA ( PotentialMI , PotentialPair . second ) ; }


<s> if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_sel ) != - <NUM_LIT> ) { MachineOperand * DstSel = TII -> getNamedOperand ( MI , <STR_LIT> :: dst_sel ) ; if ( DstSel ) { SDWAInst . add ( * DstSel ) ; } else { SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; } } if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_unused ) != - <NUM_LIT> ) { MachineOperand * DstUnused = TII -> getNamedOperand ( MI , <STR_LIT> :: dst_unused ) ; if ( DstUnused ) { SDWAInst . add ( * DstUnused ) ; } else { SDWAInst . addImm ( <STR_LIT> :: DstUnused :: UNUSED_PAD ) ; } } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_sel ) != - <NUM_LIT> ) ; MachineOperand * Src0Sel = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_sel ) ; if ( Src0Sel ) { SDWAInst . add ( * Src0Sel ) ; } else { SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; } if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_sel ) != - <NUM_LIT> ) ; MachineOperand * Src1Sel = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_sel ) ; if ( Src1Sel ) { SDWAInst . add ( * Src1Sel ) ; } else { SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; } } auto DstUnused = TII -> getNamedOperand ( MI , <STR_LIT> :: dst_unused ) ; if ( DstUnused && DstUnused -> getImm ( ) == <STR_LIT> :: DstUnused :: UNUSED_PRESERVE ) { assert ( Dst && Dst -> isTied ( ) ) ; assert ( Opcode == static_cast < unsigned int > ( SDWAOpcode ) ) ; auto PreserveDstIdx = <STR_LIT> ( SDWAOpcode , <STR_LIT> :: vdst ) ; assert ( PreserveDstIdx != - <NUM_LIT> ) ; auto TiedIdx = MI . findTiedOperandIdx ( PreserveDstIdx ) ; auto Tied = MI . getOperand ( TiedIdx ) ; SDWAInst . add ( Tied ) ; SDWAInst -> tieOperands ( PreserveDstIdx , SDWAInst -> getNumOperands ( ) - <NUM_LIT> ) ; } bool Converted = false ; for ( auto & Operand : SDWAOperands ) { DEBUG ( dbgs ( ) << * SDWAInst << <STR_LIT> << * Operand ) ; if ( PotentialMatches . count ( Operand -> getParentInst ( ) ) == <NUM_LIT> ) Converted |= Operand -> convertToSDWA ( * SDWAInst , TII ) ; }


<s> int SDWAOpcode = <STR_LIT> ( MI . getOpcode ( ) ) ; assert ( SDWAOpcode != - <NUM_LIT> ) ; const MCInstrDesc & SDWADesc = TII -> get ( SDWAOpcode ) ; MachineInstrBuilder SDWAInst = BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , SDWADesc ) ; MachineOperand * Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ; if ( Dst ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: vdst ) != - <NUM_LIT> ) ; SDWAInst . add ( * Dst ) ; } else { assert ( TII -> isVOPC ( MI ) ) ; } MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; assert ( Src0 && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src0 ) ; MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_modifiers ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src1 ) ; } else { assert ( TII -> isVOP1 ( MI ) ) ; } if ( SDWAOpcode == <STR_LIT> || SDWAOpcode == <STR_LIT> ) { MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; assert ( Src2 ) ; SDWAInst . add ( * Src2 ) ; } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: clamp ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <NUM_LIT> ) ; if ( Dst ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_sel ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_unused ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; SDWAInst . addImm ( <STR_LIT> :: DstUnused :: UNUSED_PAD ) ; } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_sel ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <STR_LIT> :: SdwaSel :: DWORD ) ; if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_sel ) != - <NUM_LIT> ) ;


<s> MachineInstrBuilder SDWAInst = BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , SDWADesc ) ; MachineOperand * Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ; if ( Dst ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: vdst ) != - <NUM_LIT> ) ; SDWAInst . add ( * Dst ) ; } else { assert ( TII -> isVOPC ( MI ) ) ; } MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; assert ( Src0 && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src0 ) ; MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_modifiers ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src1 ) ; } else { assert ( TII -> isVOP1 ( MI ) ) ; } if ( SDWAOpcode == <STR_LIT> || SDWAOpcode == <STR_LIT> ) { MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; assert ( Src2 ) ; SDWAInst . add ( * Src2 ) ; } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: clamp ) != - <NUM_LIT> ) ; SDWAInst . addImm ( <NUM_LIT> ) ; if ( Dst ) {

assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_sel ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: dst_unused ) != - <NUM_LIT> ) ;


<s> uint64_t Mods = <NUM_LIT> ; if ( Abs || Neg ) { assert ( ! Sext && <STR_LIT> ) ; Mods |= Abs ? <STR_LIT> : <NUM_LIT> ; Mods |= Neg ? <STR_LIT> : <NUM_LIT> ; } else if ( Sext ) {


<s> Mods |= Neg ? <STR_LIT> : <NUM_LIT> ; } else if ( Sext ) { Mods |= <STR_LIT> ;


<s> else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = WORD_0 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = DWORD ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = BYTE_1 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = BYTE_2 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = WORD_1 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = BYTE_3 ; else break ; MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; MachineOperand * Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ; if ( TRI -> isPhysicalRegister ( Src0 -> getReg ( ) ) || TRI -> isPhysicalRegister ( Dst -> getReg ( ) ) ) break ; auto SDWASrc = make_unique < SDWASrcOperand > ( Src0 , Dst , SrcSel , false , false , Opcode == <STR_LIT> ? false : true ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWASrc << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWASrc ) ; ++ NumSDWAPatternsFound ; break ; } case <STR_LIT> : { MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; auto Imm = foldToImm ( * Src0 ) ; if ( ! Imm ) break ; if ( * Imm != <NUM_LIT> && * Imm != <NUM_LIT> ) break ; MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; MachineOperand * Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ; if ( TRI -> isPhysicalRegister ( Src1 -> getReg ( ) ) || TRI -> isPhysicalRegister ( Dst -> getReg ( ) ) ) break ;

auto SDWASrc = make_unique < SDWASrcOperand > ( Src1 , Dst , * Imm == <NUM_LIT> ? WORD_0 : BYTE_0 ) ;


<s> auto SDWADst = make_unique < SDWADstOperand > ( Dst , Src1 , BYTE_1 , UNUSED_PAD ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWADst << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWADst ) ; ++ NumSDWAPatternsFound ; } else { auto SDWASrc = make_unique < SDWASrcOperand > ( Src1 , Dst , BYTE_1 , false , false , Opcode == <STR_LIT> ? false : true ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWASrc << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWASrc ) ; ++ NumSDWAPatternsFound ; } break ; } case <STR_LIT> : case <STR_LIT> : { MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; auto Offset = foldToImm ( * Src1 ) ; if ( ! Offset ) break ; MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; auto Width = foldToImm ( * Src2 ) ; if ( ! Width ) break ; SdwaSel SrcSel = DWORD ; if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = BYTE_0 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = WORD_0 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = DWORD ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = BYTE_1 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = BYTE_2 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = WORD_1 ; else if ( * Offset == <NUM_LIT> && * Width == <NUM_LIT> ) SrcSel = BYTE_3 ; else break ; MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; MachineOperand * Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ; if ( TRI -> isPhysicalRegister ( Src0 -> getReg ( ) ) || TRI -> isPhysicalRegister ( Dst -> getReg ( ) ) ) break ; auto SDWASrc = make_unique < SDWASrcOperand > ( Src0 , Dst , SrcSel , false , false , Opcode == <STR_LIT> ? false : true ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * SDWASrc << '\n' ) ; SDWAOperands [ & MI ] = std :: move ( SDWASrc ) ; ++ NumSDWAPatternsFound ; break ; } case <STR_LIT> : {


<s> TRI = ST . getRegisterInfo ( ) ; TII = ST . getInstrInfo ( ) ; std :: unordered_map < MachineInstr * , SDWAOperandsVector > PotentialMatches ; matchSDWAOperands ( MF ) ; for ( auto & OperandPair : SDWAOperands ) { auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI ) {

PotentialMatches [ PotentialMI ] . push_back ( std :: move ( Operand ) ) ;


<s> void <ISA_LIT>PeepholeSDWA :: matchSDWAOperands ( MachineFunction & MF ) { for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : MBB ) {

if ( auto Operand = matchSDWAOperand ( MI ) ) {


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ! ST . hasSDWA ( ) || skipFunction ( MF . getFunction ( ) ) ) return false ; MRI = & MF . getRegInfo ( ) ; TRI = ST . getRegisterInfo ( ) ; TII = ST . getInstrInfo ( ) ; bool Changed = false ; bool Ret = false ; do { matchSDWAOperands ( MF ) ; for ( const auto & OperandPair : SDWAOperands ) { const auto & Operand = OperandPair . second ; MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI && isConvertibleToSDWA ( * PotentialMI , ST ) ) {

PotentialMatches [ PotentialMI ] . push_back ( Operand . get ( ) ) ;


<s> bool <ISA_LIT>PeepholeSDWA :: isConvertibleToSDWA ( MachineInstr & MI , const GCNSubtarget & ST ) const { unsigned Opc = MI . getOpcode ( ) ; if ( TII -> isSDWA ( Opc ) ) return true ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) Opc = <STR_LIT> ( Opc ) ; if ( <STR_LIT> ( Opc ) == - <NUM_LIT> ) return false ; if ( ! ST . hasSDWAOmod ( ) && TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return false ; if ( TII -> isVOPC ( Opc ) ) { if ( ! ST . hasSDWASdst ( ) ) { const MachineOperand * SDst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ; if ( SDst && SDst -> getReg ( ) != <STR_LIT> ) return false ; } if ( ! ST . hasSDWAOutModsVOPC ( ) && ( TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) ) return false ; } else if ( TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) || ! TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ) { return false ; } if ( ! ST . hasSDWAMac ( ) && ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) ) return false ; if ( TII -> pseudoToMCOpcode ( Opc ) == - <NUM_LIT> ) return false ;


<s> const MCInstrDesc & Desc = TII -> get ( MI . getOpcode ( ) ) ; unsigned ConstantBusCount = <NUM_LIT> ; for ( MachineOperand & Op : MI . explicit_uses ( ) ) { if ( ! Op . isImm ( ) && ! ( Op . isReg ( ) && ! TRI -> isVGPR ( * MRI , Op . getReg ( ) ) ) ) continue ; unsigned I = MI . getOperandNo ( & Op ) ; if ( Desc . OpInfo [ I ] . RegClass == - <NUM_LIT> || ! TRI -> hasVGPRs ( TRI -> getRegClass ( Desc . OpInfo [ I ] . RegClass ) ) ) continue ;

if ( ST . hasSDWAScalar ( ) && ConstantBusCount == <NUM_LIT> && Op . isReg ( ) && TRI -> isSGPRReg ( * MRI , Op . getReg ( ) ) ) {


<s> if ( Op . isReg ( ) ) { for ( const MachineOperand & Def : MRI -> def_operands ( Op . getReg ( ) ) ) { if ( ! isSameReg ( Op , Def ) ) continue ;

const MachineInstr * DefInst = Def . getParent ( ) ;


<s> const MachineInstr * DefInst = Def . getParent ( ) ; if ( ! TII -> isFoldableCopy ( * DefInst ) || ! isSameBB ( Op . getParent ( ) , DefInst ) ) return None ; const MachineOperand & Copied = DefInst -> getOperand ( <NUM_LIT> ) ; if ( ! Copied . isImm ( ) ) return None ; return Copied . getImm ( ) ;


<s> MachineInstr * SDWADstOperand :: potentialToConvert ( const <ISA_LIT>InstrInfo * TII ) { MachineRegisterInfo * MRI = getMRI ( ) ; MachineInstr * ParentMI = getParentInst ( ) ; MachineOperand * Replaced = getReplacedOperand ( ) ; assert ( Replaced -> isReg ( ) ) ; for ( MachineOperand & PotentialMO : MRI -> def_operands ( Replaced -> getReg ( ) ) ) { if ( ! isSubregOf ( * Replaced , PotentialMO , MRI -> getTargetRegisterInfo ( ) ) ) continue ; if ( ! isSameBB ( getParentInst ( ) , PotentialMO . getParent ( ) ) || ! isSameReg ( * Replaced , PotentialMO ) ) return nullptr ; for ( MachineOperand & UseMO : MRI -> use_operands ( PotentialMO . getReg ( ) ) ) { if ( isSubregOf ( UseMO , PotentialMO , MRI -> getTargetRegisterInfo ( ) ) && UseMO . getParent ( ) != ParentMI ) {


<s> MachineInstr * PotentialMI = Operand -> potentialToConvert ( TII ) ; if ( PotentialMI ) { PotentialMatches [ PotentialMI ] . push_back ( std :: move ( Operand ) ) ; } } for ( auto & PotentialPair : PotentialMatches ) { MachineInstr & PotentialMI = * PotentialPair . first ; convertToSDWA ( PotentialMI , PotentialPair . second ) ;


<s> SDWAOpcode = <STR_LIT> ( Opcode ) ; if ( SDWAOpcode == - <NUM_LIT> ) SDWAOpcode = <STR_LIT> ( <STR_LIT> ( Opcode ) ) ; } assert ( SDWAOpcode != - <NUM_LIT> ) ; const MCInstrDesc & SDWADesc = TII -> get ( SDWAOpcode ) ; MachineInstrBuilder SDWAInst = BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , SDWADesc ) ; MachineOperand * Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst ) ; if ( Dst ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: vdst ) != - <NUM_LIT> ) ; SDWAInst . add ( * Dst ) ; } else if ( ( Dst = TII -> getNamedOperand ( MI , <STR_LIT> :: sdst ) ) ) { assert ( Dst && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: sdst ) != - <NUM_LIT> ) ; SDWAInst . add ( * Dst ) ; } else { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: sdst ) != - <NUM_LIT> ) ; SDWAInst . addReg ( <STR_LIT> , RegState :: Define ) ; } MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; assert ( Src0 && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src0_modifiers ) != - <NUM_LIT> ) ; if ( auto * Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) ) SDWAInst . addImm ( Mod -> getImm ( ) ) ; else SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src0 ) ; MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 ) { assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1 ) != - <NUM_LIT> && <STR_LIT> ( SDWAOpcode , <STR_LIT> :: src1_modifiers ) != - <NUM_LIT> ) ; if ( auto * Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ) SDWAInst . addImm ( Mod -> getImm ( ) ) ; else SDWAInst . addImm ( <NUM_LIT> ) ; SDWAInst . add ( * Src1 ) ; } if ( SDWAOpcode == <STR_LIT> || SDWAOpcode == <STR_LIT> ) { MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; assert ( Src2 ) ; SDWAInst . add ( * Src2 ) ; } assert ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: clamp ) != - <NUM_LIT> ) ; MachineOperand * Clamp = TII -> getNamedOperand ( MI , <STR_LIT> :: clamp ) ; if ( Clamp ) { SDWAInst . add ( * Clamp ) ; } else { SDWAInst . addImm ( <NUM_LIT> ) ; }

if ( <STR_LIT> ( SDWAOpcode , <STR_LIT> :: omod ) != - <NUM_LIT> ) {