<s> const MachineRegisterInfo & MRI = MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( ! MI . getOperand ( i ) . isReg ( ) || ! TargetRegisterInfo :: isVirtualRegister ( MI . getOperand ( i ) . getReg ( ) ) ) continue ;

if ( TRI -> hasVGPRs ( MRI . getRegClass ( MI . getOperand ( i ) . getReg ( ) ) ) ) return true ;


<s> static bool isSGPRToVGPRCopy ( const TargetRegisterClass * SrcRC , const TargetRegisterClass * DstRC , const <ISA_LIT>RegisterInfo & TRI ) { return TRI . isSGPRClass ( SrcRC ) && TRI . hasVGPRs ( DstRC ) ;


<s> static bool isVGPRToSGPRCopy ( const TargetRegisterClass * SrcRC , const TargetRegisterClass * DstRC , const <ISA_LIT>RegisterInfo & TRI ) {

return TRI . isSGPRClass ( DstRC ) && TRI . hasVGPRs ( SrcRC ) ;


<s> unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool SGPRBranch = false ; bool HasBreakDef = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; if ( TRI -> hasVGPRs ( MRI . getRegClass ( Reg ) ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefInstr = MRI . getUniqueVRegDef ( Reg ) ; assert ( DefInstr ) ; switch ( DefInstr -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : HasBreakDef = true ; break ; } } if ( ! SGPRBranch && ! HasBreakDef ) TII -> moveToVALU ( MI ) ; break ; } case <STR_LIT> : { if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; break ; } case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( TRI -> isSGPRClass ( DstRC ) && ( TRI -> hasVGPRs ( Src0RC ) || TRI -> hasVGPRs ( Src1RC ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } } }


<s> switch ( MI . getOpcode ( ) ) { default : continue ; case <STR_LIT> : { if ( ! TargetRegisterInfo :: isVirtualRegister ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( MI , * TRI , MRI ) ; if ( isVGPRToSGPRCopy ( SrcRC , DstRC , * TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool SGPRBranch = false ; bool HasBreakDef = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; if ( TRI -> hasVGPRs ( MRI . getRegClass ( Reg ) ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefInstr = MRI . getUniqueVRegDef ( Reg ) ; assert ( DefInstr ) ; switch ( DefInstr -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : HasBreakDef = true ; break ; } } if ( ! SGPRBranch && ! HasBreakDef ) TII -> moveToVALU ( MI ) ; break ; }


<s> <ISA_LIT>FixSGPRCopies ( ) : MachineFunctionPass ( ID ) {


<s> const TargetRegisterClass * SrcRC = MRI . getRegClass ( SrcReg ) ; assert ( TRI -> isSGPRClass ( SrcRC ) && <STR_LIT> ) ; SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ; const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentVGPRClass ( SrcRC ) ; unsigned TmpReg = MRI . createVirtualRegister ( NewSrcRC ) ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . add ( MI . getOperand ( I ) ) ; MI . getOperand ( I ) . setReg ( TmpReg ) ; } CopyUse . eraseFromParent ( ) ; return true ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override {

AU . addRequired < MachineDominatorTree > ( ) ;


<s> AU . addPreserved < MachineDominatorTree > ( ) ; AU . setPreservesCFG ( ) ; MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> StringRef getPassName ( ) const override {


<s> for ( MachineBasicBlock :: const_iterator I = MBB . getFirstTerminator ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { if ( I -> modifiesRegister ( <STR_LIT> , & TRI ) ) return true ; }


<s> static bool isReachable ( const MachineInstr * From , const MachineInstr * To , const MachineBasicBlock * CutOff , MachineDominatorTree & MDT ) { if ( MDT . dominates ( From , To ) ) return true ;

const MachineBasicBlock * MBBFrom = From -> getParent ( ) ;


<s> for ( unsigned i = <NUM_LIT> ; i < PHI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned Reg = PHI . getOperand ( i ) . getReg ( ) ; if ( Visited . count ( Reg ) ) continue ; Visited . insert ( Reg ) ; MachineInstr * DefInstr = MRI . getVRegDef ( Reg ) ; switch ( DefInstr -> getOpcode ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> :


<s> return searchPredecessors ( MBB , nullptr , [ TRI ] ( MachineBasicBlock * MBB ) {

return hasTerminatorThatModifiesExec ( * MBB , * TRI ) ; } ) ;


<s> case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( phiHasVGPROperands ( MI , MRI , TRI , TII ) || ! phiHasBreakDef ( MI , MRI , Visited ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; break ; } case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ;

Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> SmallVector < MachineBasicBlock * , <NUM_LIT> > Worklist ( MBB -> pred_begin ( ) , MBB -> pred_end ( ) ) ; while ( ! Worklist . empty ( ) ) {

MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ;


<s> unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( DstReg ) ) ) return false ; if ( ! MRI . hasOneUse ( DstReg ) ) return false ; MachineInstr & CopyUse = * MRI . use_instr_begin ( DstReg ) ; if ( ! CopyUse . isCopy ( ) ) return false ; if ( TargetRegisterInfo :: isPhysicalRegister ( CopyUse . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) return false ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( CopyUse , * TRI , MRI ) ; if ( ! isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) return false ; if ( tryChangeVGPRtoSGPRinCopy ( CopyUse , TRI , TII ) ) return true ; unsigned SubReg = CopyUse . getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( SubReg != <STR_LIT> ) return false ; MRI . setRegClass ( DstReg , DstRC ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( CopyUse . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { unsigned SrcReg = MI . getOperand ( I ) . getReg ( ) ; unsigned SrcSubReg = MI . getOperand ( I ) . getSubReg ( ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( SrcReg ) ; assert ( TRI -> isSGPRClass ( SrcRC ) && <STR_LIT> ) ;

SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ;


<s> for ( auto I2 = std :: next ( I1 ) ; I2 != E ; ) { MachineInstr * MI2 = * I2 ; auto intereferes = [ & ] ( MachineBasicBlock :: iterator From , MachineBasicBlock :: iterator To ) -> bool { assert ( MDT . dominates ( & * To , & * From ) ) ; auto interferes = [ & MDT , From , To ] ( MachineInstr * & Clobber ) -> bool { const MachineBasicBlock * MBBFrom = From -> getParent ( ) ; const MachineBasicBlock * MBBTo = To -> getParent ( ) ; bool MayClobberFrom = isReachable ( Clobber , & * From , MBBTo , MDT ) ; bool MayClobberTo = isReachable ( Clobber , & * To , MBBTo , MDT ) ; if ( ! MayClobberFrom && ! MayClobberTo ) return false ; if ( ( MayClobberFrom && ! MayClobberTo ) || ( ! MayClobberFrom && MayClobberTo ) ) return true ; return ! ( ( MBBFrom == MBBTo && MDT . dominates ( Clobber , & * From ) && MDT . dominates ( Clobber , & * To ) ) || MDT . properlyDominates ( Clobber -> getParent ( ) , MBBTo ) ) ; } ; return ( llvm :: any_of ( Clobbers , interferes ) ) || ( llvm :: any_of ( Inits , [ & ] ( InitListMap :: value_type & C ) { return C . first != Init . first && llvm :: any_of ( C . second , interferes ) ; } ) ) ; } ; if ( MDT . dominates ( MI1 , MI2 ) ) { if ( ! intereferes ( MI2 , MI1 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MI2 -> getParent ( ) ) << <STR_LIT> << * MI2 ) ; MI2 -> eraseFromParent ( ) ; Defs . erase ( I2 ++ ) ; Changed = true ; continue ; } } else if ( MDT . dominates ( MI2 , MI1 ) ) { if ( ! intereferes ( MI1 , MI2 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MI1 -> getParent ( ) ) << <STR_LIT> << * MI1 ) ; MI1 -> eraseFromParent ( ) ; Defs . erase ( I1 ++ ) ; Changed = true ; break ; } } else { auto * MBB = MDT . findNearestCommonDominator ( MI1 -> getParent ( ) , MI2 -> getParent ( ) ) ; if ( ! MBB ) { ++ I2 ;


<s> for ( auto I2 = std :: next ( I1 ) ; I2 != E ; ) { MachineInstr * MI2 = * I2 ; auto intereferes = [ & ] ( MachineBasicBlock :: iterator From , MachineBasicBlock :: iterator To ) -> bool { assert ( MDT . dominates ( & * To , & * From ) ) ; auto interferes = [ & MDT , From , To ] ( MachineInstr * & Clobber ) -> bool { const MachineBasicBlock * MBBFrom = From -> getParent ( ) ; const MachineBasicBlock * MBBTo = To -> getParent ( ) ; bool MayClobberFrom = isReachable ( Clobber , & * From , MBBTo , MDT ) ; bool MayClobberTo = isReachable ( Clobber , & * To , MBBTo , MDT ) ; if ( ! MayClobberFrom && ! MayClobberTo ) return false ; if ( ( MayClobberFrom && ! MayClobberTo ) || ( ! MayClobberFrom && MayClobberTo ) ) return true ; return ! ( ( MBBFrom == MBBTo && MDT . dominates ( Clobber , & * From ) && MDT . dominates ( Clobber , & * To ) ) || MDT . properlyDominates ( Clobber -> getParent ( ) , MBBTo ) ) ; } ; return ( llvm :: any_of ( Clobbers , interferes ) ) || ( llvm :: any_of ( Inits , [ & ] ( InitListMap :: value_type & C ) { return C . first != Init . first && llvm :: any_of ( C . second , interferes ) ; } ) ) ; } ; if ( MDT . dominates ( MI1 , MI2 ) ) { if ( ! intereferes ( MI2 , MI1 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MI2 -> getParent ( ) ) << <STR_LIT> << * MI2 ) ; MI2 -> eraseFromParent ( ) ; Defs . erase ( I2 ++ ) ; Changed = true ; continue ; } } else if ( MDT . dominates ( MI2 , MI1 ) ) { if ( ! intereferes ( MI1 , MI2 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MI1 -> getParent ( ) ) << <STR_LIT> << * MI1 ) ; MI1 -> eraseFromParent ( ) ; Defs . erase ( I1 ++ ) ; Changed = true ; break ; } } else { auto * MBB = MDT . findNearestCommonDominator ( MI1 -> getParent ( ) , MI2 -> getParent ( ) ) ; if ( ! MBB ) { ++ I2 ; continue ; } MachineBasicBlock :: iterator I = MBB -> getFirstNonPHI ( ) ; if ( ! intereferes ( MI1 , I ) && ! intereferes ( MI2 , I ) ) {

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << printMBBReference ( * MI1 -> getParent ( ) ) << <STR_LIT> << * MI1 << <STR_LIT> << printMBBReference ( * MI2 -> getParent ( ) ) << <STR_LIT> << printMBBReference ( * I -> getParent ( ) ) << <STR_LIT> << * MI2 ) ;


<s> static bool isSGPRToVGPRCopy ( const TargetRegisterClass * SrcRC , const TargetRegisterClass * DstRC , const <ISA_LIT>RegisterInfo & TRI ) {

return DstRC != & <STR_LIT> && TRI . isSGPRClass ( SrcRC ) && TRI . hasVGPRs ( DstRC ) ;


<s> return SrcRC != & <STR_LIT> && TRI . isSGPRClass ( DstRC ) && TRI . hasVGPRs ( SrcRC ) ;


<s> break ; } TII -> moveToVALU ( MI , MDT ) ; } else if ( isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) { tryChangeVGPRtoSGPRinCopy ( MI , TRI , TII ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool HasVGPROperand = phiHasVGPROperands ( MI , MRI , TRI , TII ) ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> && ! HasVGPROperand ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( HasVGPROperand || ! phiHasBreakDef ( MI , MRI , Visited ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI , MDT ) ; } break ; } case <STR_LIT> : if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI , MDT ) ; break ; case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( TRI -> isSGPRClass ( DstRC ) && ( TRI -> hasVGPRs ( Src0RC ) || TRI -> hasVGPRs ( Src1RC ) ) ) {


<s> MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; if ( ! Visited . insert ( MBB ) . second ) continue ; if ( MBB == CutOff ) continue ; if ( Predicate ( MBB ) ) return true ; Worklist . append ( MBB -> pred_begin ( ) , MBB -> pred_end ( ) ) ;


<s> const auto * UseMI = MO . getParent ( ) ; if ( UseMI == & MI ) continue ; if ( MO . isDef ( ) || UseMI -> getParent ( ) != MI . getParent ( ) || UseMI -> getOpcode ( ) <= TargetOpcode :: GENERIC_OP_END || ! TII -> isOperandLegal ( * UseMI , UseMI -> getOperandNo ( & MO ) , & Src ) ) return false ; } MRI . setRegClass ( DstReg , TRI -> getEquivalentSGPRClass ( MRI . getRegClass ( DstReg ) ) ) ;


<s> if ( MO . isDef ( ) || UseMI -> getParent ( ) != MI . getParent ( ) || UseMI -> getOpcode ( ) <= TargetOpcode :: GENERIC_OP_END || ! TII -> isOperandLegal ( * UseMI , UseMI -> getOperandNo ( & MO ) , & Src ) ) return false ; }

MRI . setRegClass ( DstReg , TRI -> getEquivalentSGPRClass ( MRI . getRegClass ( DstReg ) ) ) ;


<s> unsigned DstReg = Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned SrcReg = Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * SrcRC = TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ? MRI . getRegClass ( SrcReg ) : TRI . getPhysRegClass ( SrcReg ) ; const TargetRegisterClass * DstRC = TargetRegisterInfo :: isVirtualRegister ( DstReg ) ? MRI . getRegClass ( DstReg ) :


<s> bool <ISA_LIT>FixSGPRCopies :: isSGPRToVGPRCopy ( const TargetRegisterClass * SrcRC , const TargetRegisterClass * DstRC , const <ISA_LIT>RegisterInfo & TRI ) const {

return TRI . isSGPRClass ( SrcRC ) && TRI . hasVGPRs ( DstRC ) ;


<s> return TRI . isSGPRClass ( SrcRC ) && TRI . hasVGPRs ( DstRC ) ;


<s> return TRI . isSGPRClass ( DstRC ) && TRI . hasVGPRs ( SrcRC ) ;


<s> bool <ISA_LIT>FixSGPRCopies :: isVGPRToSGPRCopy ( const TargetRegisterClass * SrcRC , const TargetRegisterClass * DstRC , const <ISA_LIT>RegisterInfo & TRI ) const { return TRI . isSGPRClass ( DstRC ) && TRI . hasVGPRs ( SrcRC ) ;


<s> MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { default : continue ; case <STR_LIT> : { if ( ! TargetRegisterInfo :: isVirtualRegister ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( MI , * TRI , MRI ) ; if ( isVGPRToSGPRCopy ( SrcRC , DstRC , * TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool SGPRBranch = false ; bool HasBreakDef = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; if ( TRI -> hasVGPRs ( MRI . getRegClass ( Reg ) ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefInstr = MRI . getUniqueVRegDef ( Reg ) ; assert ( DefInstr ) ; switch ( DefInstr -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : HasBreakDef = true ; break ; } } if ( ! SGPRBranch && ! HasBreakDef ) TII -> moveToVALU ( MI ) ; break ; } case <STR_LIT> : { if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) continue ; DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; break ; } case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( TRI -> isSGPRClass ( DstRC ) && ( TRI -> hasVGPRs ( Src0RC ) || TRI -> hasVGPRs ( Src1RC ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; }


<s> for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { default : continue ; case <STR_LIT> : { if ( ! TargetRegisterInfo :: isVirtualRegister ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( MI , * TRI , MRI ) ; if ( isVGPRToSGPRCopy ( SrcRC , DstRC , * TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool SGPRBranch = false ; bool HasBreakDef = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; if ( TRI -> hasVGPRs ( MRI . getRegClass ( Reg ) ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefInstr = MRI . getUniqueVRegDef ( Reg ) ; assert ( DefInstr ) ; switch ( DefInstr -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> :


<s> bool IsAGPR = TRI -> isAGPRClass ( DstRC ) ; for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { Register SrcReg = MI . getOperand ( I ) . getReg ( ) ; unsigned SrcSubReg = MI . getOperand ( I ) . getSubReg ( ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( SrcReg ) ; assert ( TRI -> isSGPRClass ( SrcRC ) && <STR_LIT> ) ; SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ; const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentVGPRClass ( SrcRC ) ; Register TmpReg = MRI . createVirtualRegister ( NewSrcRC ) ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . add ( MI . getOperand ( I ) ) ; if ( IsAGPR ) { const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentAGPRClass ( SrcRC ) ; Register TmpAReg = MRI . createVirtualRegister ( NewSrcRC ) ; unsigned Opc = NewSrcRC == & <STR_LIT> ? <STR_LIT> : <STR_LIT> ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) , TmpAReg ) . addReg ( TmpReg , RegState :: Kill ) ;


<s> assert ( MI . isRegSequence ( ) ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( DstReg ) ) ) return false ; if ( ! MRI . hasOneUse ( DstReg ) ) return false ; MachineInstr & CopyUse = * MRI . use_instr_begin ( DstReg ) ; if ( ! CopyUse . isCopy ( ) ) return false ; if ( CopyUse . getOperand ( <NUM_LIT> ) . getReg ( ) . isPhysical ( ) ) return false ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( CopyUse , * TRI , MRI ) ; if ( ! isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) return false ; if ( tryChangeVGPRtoSGPRinCopy ( CopyUse , TRI , TII ) ) return true ; unsigned SubReg = CopyUse . getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( SubReg != <STR_LIT> ) return false ; MRI . setRegClass ( DstReg , DstRC ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( CopyUse . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; bool IsAGPR = TRI -> isAGPRClass ( DstRC ) ; for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { Register SrcReg = MI . getOperand ( I ) . getReg ( ) ; unsigned SrcSubReg = MI . getOperand ( I ) . getSubReg ( ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( SrcReg ) ; assert ( TRI -> isSGPRClass ( SrcRC ) && <STR_LIT> ) ; SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ; const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentVGPRClass ( SrcRC ) ; Register TmpReg = MRI . createVirtualRegister ( NewSrcRC ) ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . add ( MI . getOperand ( I ) ) ; if ( IsAGPR ) { const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentAGPRClass ( SrcRC ) ; Register TmpAReg = MRI . createVirtualRegister ( NewSrcRC ) ; unsigned Opc = NewSrcRC == & <STR_LIT> ? <STR_LIT> : <STR_LIT> ;

BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) , TmpAReg ) . addReg ( TmpReg , RegState :: Kill ) ;


<s> Register SrcReg = Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * SrcRC = SrcReg . isVirtual ( ) ? MRI . getRegClass ( SrcReg ) : TRI . getPhysRegClass ( SrcReg ) ;

const TargetRegisterClass * DstRC = DstReg . isVirtual ( ) ? MRI . getRegClass ( DstReg ) : TRI . getPhysRegClass ( DstReg ) ;


<s> while ( I != MBB -> end ( ) && TII -> isBasicBlockPrologue ( * I ) ) ++ I ;


<s> for ( const MachineOperand & MO : MI . operands ( ) ) {

if ( ! MO . isReg ( ) || ! MO . getReg ( ) . isVirtual ( ) ) continue ;


<s> if ( MDT . dominates ( From , To ) ) return true ; const MachineBasicBlock * MBBFrom = From -> getParent ( ) ;

const MachineBasicBlock * MBBTo = To -> getParent ( ) ;


<s> const MachineBasicBlock * MBBFrom = From -> getParent ( ) ; const MachineBasicBlock * MBBTo = To -> getParent ( ) ; return searchPredecessors ( MBBTo , CutOff , [ MBBFrom ] ( const MachineBasicBlock * MBB ) { return MBB == MBBFrom ; } ) ;


<s> static bool isSGPRToVGPRCopy ( const TargetRegisterClass * SrcRC , const TargetRegisterClass * DstRC , const <ISA_LIT>RegisterInfo & TRI ) {

return DstRC != & <STR_LIT> && TRI . isSGPRClass ( SrcRC ) && TRI . hasVectorRegisters ( DstRC ) ;


<s> static bool isVGPRToSGPRCopy ( const TargetRegisterClass * SrcRC , const TargetRegisterClass * DstRC , const <ISA_LIT>RegisterInfo & TRI ) {

return SrcRC != & <STR_LIT> && TRI . isSGPRClass ( DstRC ) && TRI . hasVectorRegisters ( SrcRC ) ;


<s> } } Register PHIRes = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC0 = MRI -> getRegClass ( PHIRes ) ; if ( AllAGPRUses && numVGPRUses && ! TRI -> isAGPRClass ( RC0 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MRI -> setRegClass ( PHIRes , TRI -> getEquivalentAGPRClass ( RC0 ) ) ; for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { MachineInstr * DefMI = MRI -> getVRegDef ( MI . getOperand ( I ) . getReg ( ) ) ; if ( DefMI && DefMI -> isPHI ( ) ) PHIOperands . insert ( DefMI ) ; } } bool hasVGPRInput = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { Register InputReg = MI . getOperand ( i ) . getReg ( ) ; MachineInstr * Def = MRI -> getVRegDef ( InputReg ) ; if ( TRI -> isVectorRegister ( * MRI , InputReg ) ) { if ( Def -> isCopy ( ) ) { Register SrcReg = Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC = TRI -> getRegClassForReg ( * MRI , SrcReg ) ; if ( TRI -> isSGPRClass ( RC ) ) continue ; } hasVGPRInput = true ; break ; } else if ( Def -> isCopy ( ) && TRI -> isVectorRegister ( * MRI , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Register SrcReg = Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * SrcDef = MRI -> getVRegDef ( SrcReg ) ; unsigned SMovOp ; int64_t Imm ; if ( ! isSafeToFoldImmIntoCopy ( Def , SrcDef , TII , SMovOp , Imm ) ) { hasVGPRInput = true ; break ; } else { MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; Def -> getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; Def -> addImplicitDefUseOperands ( * MF ) ; Def -> setDesc ( TII -> get ( SMovOp ) ) ; } } } if ( ( ! TRI -> isVectorRegister ( * MRI , PHIRes ) && RC0 != & <STR_LIT> ) && ( hasVGPRInput || numVGPRUses > <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; CreatedBB = TII -> moveToVALU ( MI ) ; }


<s> Register PHIRes = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC0 = MRI -> getRegClass ( PHIRes ) ; if ( AllAGPRUses && numVGPRUses && ! TRI -> isAGPRClass ( RC0 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MRI -> setRegClass ( PHIRes , TRI -> getEquivalentAGPRClass ( RC0 ) ) ; for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { MachineInstr * DefMI = MRI -> getVRegDef ( MI . getOperand ( I ) . getReg ( ) ) ; if ( DefMI && DefMI -> isPHI ( ) ) PHIOperands . insert ( DefMI ) ; } } bool hasVGPRInput = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { Register InputReg = MI . getOperand ( i ) . getReg ( ) ; MachineInstr * Def = MRI -> getVRegDef ( InputReg ) ; if ( TRI -> isVectorRegister ( * MRI , InputReg ) ) { if ( Def -> isCopy ( ) ) { Register SrcReg = Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC = TRI -> getRegClassForReg ( * MRI , SrcReg ) ; if ( TRI -> isSGPRClass ( RC ) ) continue ; } hasVGPRInput = true ; break ; } else if ( Def -> isCopy ( ) && TRI -> isVectorRegister ( * MRI , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Register SrcReg = Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * SrcDef = MRI -> getVRegDef ( SrcReg ) ; unsigned SMovOp ; int64_t Imm ; if ( ! isSafeToFoldImmIntoCopy ( Def , SrcDef , TII , SMovOp , Imm ) ) { hasVGPRInput = true ; break ; } else { MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; Def -> getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; Def -> addImplicitDefUseOperands ( * MF ) ; Def -> setDesc ( TII -> get ( SMovOp ) ) ; } } } if ( ( ! TRI -> isVectorRegister ( * MRI , PHIRes ) && RC0 != & <STR_LIT> ) && ( hasVGPRInput || numVGPRUses > <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; CreatedBB = TII -> moveToVALU ( MI ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> legalizeOperands ( MI , MDT ) ; } while ( ! PHIOperands . empty ( ) ) {

processPHINode ( * PHIOperands . pop_back_val ( ) ) ;


<s> MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } MachineBasicBlock * NewBB = TII -> moveToVALU ( MI , MDT ) ; if ( NewBB && NewBB != MBB ) { MBB = NewBB ; E = MBB -> end ( ) ; BI = MachineFunction :: iterator ( MBB ) ; BE = MF . end ( ) ; } assert ( ( ! NewBB || NewBB == I -> getParent ( ) ) && <STR_LIT> ) ; } else if ( isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) { tryChangeVGPRtoSGPRinCopy ( MI , TRI , TII ) ; } break ; } case <STR_LIT> : { MachineBasicBlock * NewBB = processPHINode ( MI ) ; if ( NewBB && NewBB != MBB ) { MBB = NewBB ; E = MBB -> end ( ) ; BI = MachineFunction :: iterator ( MBB ) ; BE = MF . end ( ) ; } assert ( ( ! NewBB || NewBB == I -> getParent ( ) ) && <STR_LIT> ) ; break ; } case <STR_LIT> : { if ( TRI -> hasVectorRegisters ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVectorOperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , * MRI ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MachineBasicBlock * NewBB = TII -> moveToVALU ( MI , MDT ) ; if ( NewBB && NewBB != MBB ) { MBB = NewBB ; E = MBB -> end ( ) ; BI = MachineFunction :: iterator ( MBB ) ; BE = MF . end ( ) ; } assert ( ( ! NewBB || NewBB == I -> getParent ( ) ) && <STR_LIT> ) ; break ; } case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI -> getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI -> getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI -> getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( TRI -> isSGPRClass ( DstRC ) && ( TRI -> hasVectorRegisters ( Src0RC ) || TRI -> hasVectorRegisters ( Src1RC ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MachineBasicBlock * NewBB = TII -> moveToVALU ( MI , MDT ) ; if ( NewBB && NewBB != MBB ) { MBB = NewBB ; E = MBB -> end ( ) ; BI = MachineFunction :: iterator ( MBB ) ; BE = MF . end ( ) ; } assert ( ( ! NewBB || NewBB == I -> getParent ( ) ) && <STR_LIT> ) ; }


<s> DenseSet < const MachineBasicBlock * > Visited ; SmallVector < MachineBasicBlock * , <NUM_LIT> > Worklist ( MBB -> predecessors ( ) ) ; while ( ! Worklist . empty ( ) ) { MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; if ( ! Visited . insert ( MBB ) . second ) continue ; if ( MBB == CutOff ) continue ; if ( Predicate ( MBB ) ) return true ; Worklist . append ( MBB -> pred_begin ( ) , MBB -> pred_end ( ) ) ; } return false ;


<s> if ( ! Visited . insert ( MBB ) . second ) continue ; if ( MBB == CutOff ) continue ; if ( Predicate ( MBB ) ) return true ; Worklist . append ( MBB -> pred_begin ( ) , MBB -> pred_end ( ) ) ; } return false ;


<s> auto & Src = MI . getOperand ( <NUM_LIT> ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; Register SrcReg = Src . getReg ( ) ; if ( ! SrcReg . isVirtual ( ) || ! DstReg . isVirtual ( ) ) return false ;

for ( const auto & MO : MRI . reg_nodbg_operands ( DstReg ) ) {


<s> if ( ! SrcReg . isVirtual ( ) || ! DstReg . isVirtual ( ) ) return false ; for ( const auto & MO : MRI . reg_nodbg_operands ( DstReg ) ) { const auto * UseMI = MO . getParent ( ) ; if ( UseMI == & MI ) continue ; if ( MO . isDef ( ) || UseMI -> getParent ( ) != MI . getParent ( ) || UseMI -> getOpcode ( ) <= TargetOpcode :: GENERIC_OP_END ) return false ;

unsigned OpIdx = UseMI -> getOperandNo ( & MO ) ;


<s> bool MayClobberFrom = isReachable ( Clobber , & * From , MBBTo , MDT ) ; bool MayClobberTo = isReachable ( Clobber , & * To , MBBTo , MDT ) ; if ( ! MayClobberFrom && ! MayClobberTo ) return false ; if ( ( MayClobberFrom && ! MayClobberTo ) || ( ! MayClobberFrom && MayClobberTo ) ) return true ; return ! ( ( MBBFrom == MBBTo && MDT . dominates ( Clobber , & * From ) && MDT . dominates ( Clobber , & * To ) ) || MDT . properlyDominates ( Clobber -> getParent ( ) , MBBTo ) ) ; } ; return ( llvm :: any_of ( Clobbers , interferes ) ) || ( llvm :: any_of ( Inits , [ & ] ( InitListMap :: value_type & C ) { return C . first != Init . first && llvm :: any_of ( C . second , interferes ) ; } ) ) ; } ; if ( MDT . dominates ( MI1 , MI2 ) ) { if ( ! intereferes ( MI2 , MI1 ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI2 -> getParent ( ) -> getNumber ( ) << <STR_LIT> << * MI2 ) ; MI2 -> eraseFromParent ( ) ; Defs . erase ( I2 ++ ) ; Changed = true ; continue ; } } else if ( MDT . dominates ( MI2 , MI1 ) ) { if ( ! intereferes ( MI1 , MI2 ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI1 -> getParent ( ) -> getNumber ( ) << <STR_LIT> << * MI1 ) ; MI1 -> eraseFromParent ( ) ; Defs . erase ( I1 ++ ) ; Changed = true ; break ; } } else { auto * MBB = MDT . findNearestCommonDominator ( MI1 -> getParent ( ) , MI2 -> getParent ( ) ) ; if ( ! MBB ) { ++ I2 ; continue ; } MachineBasicBlock :: iterator I = MBB -> getFirstNonPHI ( ) ; if ( ! intereferes ( MI1 , I ) && ! intereferes ( MI2 , I ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI1 -> getParent ( ) -> getNumber ( ) << <STR_LIT> << * MI1 << <STR_LIT> << MI2 -> getParent ( ) -> getNumber ( ) << <STR_LIT> << I -> getParent ( ) -> getNumber ( ) << <STR_LIT> << * MI2 ) ; I -> getParent ( ) -> splice ( I , MI2 -> getParent ( ) , MI2 ) ; MI1 -> eraseFromParent ( ) ; Defs . erase ( I1 ++ ) ; Changed = true ; break ; } } ++ I2 ; } ++ I1 ; } }

if ( Changed ) MRI . clearKillFlags ( Reg ) ;


<s> if ( ! MayClobberFrom && ! MayClobberTo ) return false ; if ( ( MayClobberFrom && ! MayClobberTo ) || ( ! MayClobberFrom && MayClobberTo ) ) return true ; return ! ( ( MBBFrom == MBBTo && MDT . dominates ( Clobber , & * From ) && MDT . dominates ( Clobber , & * To ) ) || MDT . properlyDominates ( Clobber -> getParent ( ) , MBBTo ) ) ; } ; return ( llvm :: any_of ( Clobbers , interferes ) ) || ( llvm :: any_of ( Inits , [ & ] ( InitListMap :: value_type & C ) { return C . first != Init . first && llvm :: any_of ( C . second , interferes ) ; } ) ) ; } ; if ( MDT . dominates ( MI1 , MI2 ) ) { if ( ! intereferes ( MI2 , MI1 ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI2 -> getParent ( ) -> getNumber ( ) << <STR_LIT> << * MI2 ) ; MI2 -> eraseFromParent ( ) ; Defs . erase ( I2 ++ ) ; Changed = true ; continue ; } } else if ( MDT . dominates ( MI2 , MI1 ) ) { if ( ! intereferes ( MI1 , MI2 ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI1 -> getParent ( ) -> getNumber ( ) << <STR_LIT> << * MI1 ) ; MI1 -> eraseFromParent ( ) ; Defs . erase ( I1 ++ ) ; Changed = true ; break ; }


<s> MachineInstr * DefMI = MRI . getVRegDef ( SrcReg ) ; unsigned SMovOp ; int64_t Imm ; if ( isSafeToFoldImmIntoCopy ( & MI , DefMI , TII , SMovOp , Imm ) ) { MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI ) ; } else if ( isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) { tryChangeVGPRtoSGPRinCopy ( MI , TRI , TII ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool HasVGPROperand = phiHasVGPROperands ( MI , MRI , TRI , TII ) ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> && ! HasVGPROperand ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( HasVGPROperand || ! phiHasBreakDef ( MI , MRI , Visited ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; break ; case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( TRI -> isSGPRClass ( DstRC ) && ( TRI -> hasVGPRs ( Src0RC ) || TRI -> hasVGPRs ( Src1RC ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } } }


<s> unsigned SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefMI = MRI . getVRegDef ( SrcReg ) ; unsigned SMovOp ; int64_t Imm ; if ( isSafeToFoldImmIntoCopy ( & MI , DefMI , TII , SMovOp , Imm ) ) { MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI ) ; } else if ( isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) { tryChangeVGPRtoSGPRinCopy ( MI , TRI , TII ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool HasVGPROperand = phiHasVGPROperands ( MI , MRI , TRI , TII ) ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> && ! HasVGPROperand ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( HasVGPROperand || ! phiHasBreakDef ( MI , MRI , Visited ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; break ; case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( TRI -> isSGPRClass ( DstRC ) && ( TRI -> hasVGPRs ( Src0RC ) || TRI -> hasVGPRs ( Src1RC ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } }


<s> for ( unsigned i = <NUM_LIT> ; i < PHI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned Reg = PHI . getOperand ( i ) . getReg ( ) ; if ( Visited . count ( Reg ) ) continue ; Visited . insert ( Reg ) ; MachineInstr * DefInstr = MRI . getUniqueVRegDef ( Reg ) ; assert ( DefInstr ) ; switch ( DefInstr -> getOpcode ( ) ) { default : break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return true ; case <STR_LIT> : if ( phiHasBreakDef ( * DefInstr , MRI , Visited ) ) return true ; } }


<s> if ( Visited . count ( Reg ) ) continue ; Visited . insert ( Reg ) ; MachineInstr * DefInstr = MRI . getUniqueVRegDef ( Reg ) ; assert ( DefInstr ) ; switch ( DefInstr -> getOpcode ( ) ) {


<s> int64_t Imm ; if ( isSafeToFoldImmIntoCopy ( & MI , DefMI , TII , SMovOp , Imm ) ) { MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( phiHasVGPROperands ( MI , MRI , TRI , TII ) || ! phiHasBreakDef ( MI , MRI , Visited ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; break ; } case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ;

Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { unsigned SrcReg = MI . getOperand ( I ) . getReg ( ) ; unsigned SrcSubReg = MI . getOperand ( I ) . getSubReg ( ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( SrcReg ) ; assert ( TRI -> isSGPRClass ( SrcRC ) && <STR_LIT> ) ; SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ; const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentVGPRClass ( SrcRC ) ; unsigned TmpReg = MRI . createVirtualRegister ( NewSrcRC ) ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . addOperand ( MI . getOperand ( I ) ) ; MI . getOperand ( I ) . setReg ( TmpReg ) ; }


<s> break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( phiHasVGPROperands ( MI , MRI , TRI , TII ) || ! phiHasBreakDef ( MI , MRI , Visited ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; break ; } case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( TRI -> isSGPRClass ( DstRC ) && ( TRI -> hasVGPRs ( Src0RC ) || TRI -> hasVGPRs ( Src1RC ) ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ;


<s> SmallSet < unsigned , <NUM_LIT> > Visited ; if ( phiHasVGPROperands ( MI , MRI , TRI , TII ) || ! phiHasBreakDef ( MI , MRI , Visited ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; break ; } case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ;

if ( TRI -> isSGPRClass ( DstRC ) && ( TRI -> hasVGPRs ( Src0RC ) || TRI -> hasVGPRs ( Src1RC ) ) ) {


<s> assert ( TRI -> isSGPRClass ( SrcRC ) && <STR_LIT> ) ; SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ; const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentVGPRClass ( SrcRC ) ; Register TmpReg = MRI . createVirtualRegister ( NewSrcRC ) ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . add ( MI . getOperand ( I ) ) ; if ( IsAGPR ) { const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentAGPRClass ( SrcRC ) ; Register TmpAReg = MRI . createVirtualRegister ( NewSrcRC ) ; unsigned Opc = NewSrcRC == & <STR_LIT> ? <STR_LIT> : <STR_LIT> ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) , TmpAReg ) . addReg ( TmpReg , RegState :: Kill ) ; TmpReg = TmpAReg ; } MI . getOperand ( I ) . setReg ( TmpReg ) ; } CopyUse . eraseFromParent ( ) ;


<s> assert ( MI . isRegSequence ( ) ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( DstReg ) ) ) return false ; if ( ! MRI . hasOneUse ( DstReg ) ) return false ; MachineInstr & CopyUse = * MRI . use_instr_begin ( DstReg ) ; if ( ! CopyUse . isCopy ( ) ) return false ; if ( Register :: isPhysicalRegister ( CopyUse . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) return false ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( CopyUse , * TRI , MRI ) ; if ( ! isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) return false ; if ( tryChangeVGPRtoSGPRinCopy ( CopyUse , TRI , TII ) ) return true ; unsigned SubReg = CopyUse . getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( SubReg != <STR_LIT> ) return false ; MRI . setRegClass ( DstReg , DstRC ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( CopyUse . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; bool IsAGPR = TRI -> hasAGPRs ( DstRC ) ; for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { Register SrcReg = MI . getOperand ( I ) . getReg ( ) ; unsigned SrcSubReg = MI . getOperand ( I ) . getSubReg ( ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( SrcReg ) ; assert ( TRI -> isSGPRClass ( SrcRC ) && <STR_LIT> ) ; SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ; const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentVGPRClass ( SrcRC ) ; Register TmpReg = MRI . createVirtualRegister ( NewSrcRC ) ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . add ( MI . getOperand ( I ) ) ; if ( IsAGPR ) { const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentAGPRClass ( SrcRC ) ; Register TmpAReg = MRI . createVirtualRegister ( NewSrcRC ) ; unsigned Opc = NewSrcRC == & <STR_LIT> ? <STR_LIT> : <STR_LIT> ;


<s> const TargetRegisterClass * SrcRC = Register :: isVirtualRegister ( SrcReg ) ? MRI . getRegClass ( SrcReg ) : TRI . getPhysRegClass ( SrcReg ) ; const TargetRegisterClass * DstRC = Register :: isVirtualRegister ( DstReg ) ? MRI . getRegClass ( DstReg ) : TRI . getPhysRegClass ( DstReg ) ; return std :: make_pair ( SrcRC , DstRC ) ;


<s> Register SrcReg = Copy . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * SrcRC = Register :: isVirtualRegister ( SrcReg ) ? MRI . getRegClass ( SrcReg ) : TRI . getPhysRegClass ( SrcReg ) ;

const TargetRegisterClass * DstRC = Register :: isVirtualRegister ( DstReg ) ? MRI . getRegClass ( DstReg ) : TRI . getPhysRegClass ( DstReg ) ;


<s> const MachineRegisterInfo & MRI = MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( ! MI . getOperand ( i ) . isReg ( ) || ! Register :: isVirtualRegister ( MI . getOperand ( i ) . getReg ( ) ) ) continue ; if ( TRI -> hasVectorRegisters ( MRI . getRegClass ( MI . getOperand ( i ) . getReg ( ) ) ) ) return true ; }


<s> if ( Visited . count ( Reg ) ) continue ; Visited . insert ( Reg ) ; MachineInstr * DefInstr = MRI . getVRegDef ( Reg ) ; switch ( DefInstr -> getOpcode ( ) ) { default : break ; case <STR_LIT> : return true ; case <STR_LIT> : if ( phiHasBreakDef ( * DefInstr , MRI , Visited ) ) return true ; }


<s> default : continue ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( MI , * TRI , MRI ) ; if ( ! Register :: isVirtualRegister ( DstReg ) ) { if ( DstReg == <STR_LIT> && TRI -> hasVectorRegisters ( SrcRC ) ) { Register TmpReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( MBB , MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( TmpReg ) ; } continue ; } if ( isVGPRToSGPRCopy ( SrcRC , DstRC , * TRI ) ) { Register SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! Register :: isVirtualRegister ( SrcReg ) ) { TII -> moveToVALU ( MI , MDT ) ; break ; } MachineInstr * DefMI = MRI . getVRegDef ( SrcReg ) ; unsigned SMovOp ; int64_t Imm ; if ( isSafeToFoldImmIntoCopy ( & MI , DefMI , TII , SMovOp , Imm ) ) { MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI , MDT ) ; } else if ( isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) { tryChangeVGPRtoSGPRinCopy ( MI , TRI , TII ) ; } break ; } case <STR_LIT> : { Register Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool HasVGPROperand = phiHasVGPROperands ( MI , MRI , TRI , TII ) ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> && ! HasVGPROperand ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ;


<s> break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( HasVGPROperand || ! phiHasBreakDef ( MI , MRI , Visited ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI , MDT ) ; } break ; } case <STR_LIT> : if ( TRI -> hasVectorRegisters ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVectorOperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI , MDT ) ; break ; case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( TRI -> isSGPRClass ( DstRC ) && ( TRI -> hasVectorRegisters ( Src0RC ) || TRI -> hasVectorRegisters ( Src1RC ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI , MDT ) ; } break ; } } } } if ( MF . getTarget ( ) . getOptLevel ( ) > CodeGenOpt :: None && EnableM0Merge ) hoistAndMergeSGPRInits ( <STR_LIT> , MRI , TRI , * MDT , TII ) ;


<s> if ( UseMI == & MI ) continue ; if ( MO . isDef ( ) || UseMI -> getParent ( ) != MI . getParent ( ) || UseMI -> getOpcode ( ) <= TargetOpcode :: GENERIC_OP_END || ! TII -> isOperandLegal ( * UseMI , UseMI -> getOperandNo ( & MO ) , & Src ) ) return false ; } MRI . setRegClass ( DstReg , TRI -> getEquivalentSGPRClass ( MRI . getRegClass ( DstReg ) ) ) ;


<s> if ( UseMI == & MI ) continue ; if ( MO . isDef ( ) || UseMI -> getParent ( ) != MI . getParent ( ) || UseMI -> getOpcode ( ) <= TargetOpcode :: GENERIC_OP_END || ! TII -> isOperandLegal ( * UseMI , UseMI -> getOperandNo ( & MO ) , & Src ) ) return false ; } MRI . setRegClass ( DstReg , TRI -> getEquivalentSGPRClass ( MRI . getRegClass ( DstReg ) ) ) ; return true ;


<s> SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ; const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentVGPRClass ( SrcRC ) ; Register TmpReg = MRI . createVirtualRegister ( NewSrcRC ) ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . add ( MI . getOperand ( I ) ) ; if ( IsAGPR ) { const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentAGPRClass ( SrcRC ) ; Register TmpAReg = MRI . createVirtualRegister ( NewSrcRC ) ; unsigned Opc = NewSrcRC == & <STR_LIT> ? <STR_LIT> : <STR_LIT> ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( Opc ) , TmpAReg ) . addReg ( TmpReg , RegState :: Kill ) ; TmpReg = TmpAReg ; } MI . getOperand ( I ) . setReg ( TmpReg ) ; } CopyUse . eraseFromParent ( ) ;


<s> const MachineRegisterInfo & MRI = MI . getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; for ( unsigned i = <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( ! MI . getOperand ( i ) . isReg ( ) || ! MI . getOperand ( i ) . getReg ( ) . isVirtual ( ) ) continue ; if ( TRI -> hasVectorRegisters ( MRI . getRegClass ( MI . getOperand ( i ) . getReg ( ) ) ) ) return true ; } return false ;


<s> } if ( UseMI -> isPHI ( ) ) { const TargetRegisterClass * UseRC = MRI -> getRegClass ( Use . getReg ( ) ) ; if ( ! TRI -> isSGPRReg ( * MRI , Use . getReg ( ) ) && UseRC != & <STR_LIT> ) numVGPRUses ++ ; continue ; } const TargetRegisterClass * OpRC = TII -> getOpRegClass ( * UseMI , UseMI -> getOperandNo ( & Use ) ) ; if ( ! TRI -> isSGPRClass ( OpRC ) && OpRC != & <STR_LIT> && OpRC != & <STR_LIT> ) { numVGPRUses ++ ; } } } Register PHIRes = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC0 = MRI -> getRegClass ( PHIRes ) ; if ( AllAGPRUses && numVGPRUses && ! TRI -> hasAGPRs ( RC0 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MRI -> setRegClass ( PHIRes , TRI -> getEquivalentAGPRClass ( RC0 ) ) ; for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { MachineInstr * DefMI = MRI -> getVRegDef ( MI . getOperand ( I ) . getReg ( ) ) ; if ( DefMI && DefMI -> isPHI ( ) ) PHIOperands . insert ( DefMI ) ; } } bool hasVGPRInput = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { Register InputReg = MI . getOperand ( i ) . getReg ( ) ; MachineInstr * Def = MRI -> getVRegDef ( InputReg ) ; if ( TRI -> isVectorRegister ( * MRI , InputReg ) ) { if ( Def -> isCopy ( ) ) { Register SrcReg = Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ;

const TargetRegisterClass * RC = TRI -> getRegClassForReg ( * MRI , SrcReg ) ;


<s> if ( UseMI -> isCopy ( ) && UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) . isPhysical ( ) && ! TRI -> isSGPRReg ( * MRI , UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { numVGPRUses ++ ; } if ( Visited . insert ( UseMI ) . second ) worklist . insert ( UseMI ) ; continue ; } if ( UseMI -> isPHI ( ) ) { const TargetRegisterClass * UseRC = MRI -> getRegClass ( Use . getReg ( ) ) ; if ( ! TRI -> isSGPRReg ( * MRI , Use . getReg ( ) ) && UseRC != & <STR_LIT> ) numVGPRUses ++ ; continue ; } const TargetRegisterClass * OpRC = TII -> getOpRegClass ( * UseMI , UseMI -> getOperandNo ( & Use ) ) ; if ( ! TRI -> isSGPRClass ( OpRC ) && OpRC != & <STR_LIT> && OpRC != & <STR_LIT> ) { numVGPRUses ++ ; } } } Register PHIRes = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ;

const TargetRegisterClass * RC0 = MRI -> getRegClass ( PHIRes ) ;


<s> if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( DstReg ) ) ) return false ; if ( ! MRI . hasOneUse ( DstReg ) ) return false ; MachineInstr & CopyUse = * MRI . use_instr_begin ( DstReg ) ; if ( ! CopyUse . isCopy ( ) ) return false ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( CopyUse , * TRI , MRI ) ; if ( ! isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) return false ; unsigned SubReg = CopyUse . getOperand ( <NUM_LIT> ) . getSubReg ( ) ;

if ( SubReg != <STR_LIT> ) return false ;


<s> if ( ! MI . getOperand ( i ) . isReg ( ) || ! TargetRegisterInfo :: isVirtualRegister ( MI . getOperand ( i ) . getReg ( ) ) ) continue ; if ( TRI -> hasVectorRegisters ( MRI . getRegClass ( MI . getOperand ( i ) . getReg ( ) ) ) ) return true ; }


<s> bool Changed = false ; for ( auto & MI : MRI . def_instructions ( Reg ) ) { MachineOperand * Imm = nullptr ; for ( auto & MO : MI . operands ( ) ) { if ( ( MO . isReg ( ) && ( ( MO . isDef ( ) && MO . getReg ( ) != Reg ) || ! MO . isDef ( ) ) ) || ( ! MO . isImm ( ) && ! MO . isReg ( ) ) || ( MO . isImm ( ) && Imm ) ) { Imm = nullptr ; break ; } else if ( MO . isImm ( ) ) Imm = & MO ; } if ( Imm ) Inits [ Imm -> getImm ( ) ] . push_front ( & MI ) ; else Clobbers . push_back ( & MI ) ; } for ( auto & Init : Inits ) { auto & Defs = Init . second ; for ( auto I1 = Defs . begin ( ) , E = Defs . end ( ) ; I1 != E ; ) { MachineInstr * MI1 = * I1 ; for ( auto I2 = std :: next ( I1 ) ; I2 != E ; ) { MachineInstr * MI2 = * I2 ; auto interferes = [ & ] ( MachineBasicBlock :: iterator From , MachineBasicBlock :: iterator To ) -> bool { assert ( MDT . dominates ( & * To , & * From ) ) ; auto interferes = [ & MDT , From , To ] ( MachineInstr * & Clobber ) -> bool { const MachineBasicBlock * MBBFrom = From -> getParent ( ) ; const MachineBasicBlock * MBBTo = To -> getParent ( ) ; bool MayClobberFrom = isReachable ( Clobber , & * From , MBBTo , MDT ) ; bool MayClobberTo = isReachable ( Clobber , & * To , MBBTo , MDT ) ; if ( ! MayClobberFrom && ! MayClobberTo ) return false ; if ( ( MayClobberFrom && ! MayClobberTo ) || ( ! MayClobberFrom && MayClobberTo ) ) return true ; return ! ( ( MBBFrom == MBBTo && MDT . dominates ( Clobber , & * From ) && MDT . dominates ( Clobber , & * To ) ) || MDT . properlyDominates ( Clobber -> getParent ( ) , MBBTo ) ) ;


<s> MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI , MDT ) ; } else if ( isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) { tryChangeVGPRtoSGPRinCopy ( MI , TRI , TII ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool HasVGPROperand = phiHasVGPROperands ( MI , MRI , TRI , TII ) ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> && ! HasVGPROperand ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( HasVGPROperand || ! phiHasBreakDef ( MI , MRI , Visited ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ;


<s> std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( MI , * TRI , MRI ) ; if ( isVGPRToSGPRCopy ( SrcRC , DstRC , * TRI ) ) { unsigned SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ) { TII -> moveToVALU ( MI , MDT ) ; break ; } MachineInstr * DefMI = MRI . getVRegDef ( SrcReg ) ; unsigned SMovOp ; int64_t Imm ; if ( isSafeToFoldImmIntoCopy ( & MI , DefMI , TII , SMovOp , Imm ) ) { MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI , MDT ) ; } else if ( isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) { tryChangeVGPRtoSGPRinCopy ( MI , TRI , TII ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool HasVGPROperand = phiHasVGPROperands ( MI , MRI , TRI , TII ) ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> && ! HasVGPROperand ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ;


<s> bool AllAGPRUses = true ; SetVector < const MachineInstr * > worklist ; SmallSet < const MachineInstr * , <NUM_LIT> > Visited ; SetVector < MachineInstr * > PHIOperands ; worklist . insert ( & MI ) ; Visited . insert ( & MI ) ; while ( ! worklist . empty ( ) ) { const MachineInstr * Instr = worklist . pop_back_val ( ) ; unsigned Reg = Instr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( const auto & Use : MRI -> use_operands ( Reg ) ) { const MachineInstr * UseMI = Use . getParent ( ) ; AllAGPRUses &= ( UseMI -> isCopy ( ) && TRI -> isAGPR ( * MRI , UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) || TRI -> isAGPR ( * MRI , Use . getReg ( ) ) ; if ( UseMI -> isCopy ( ) || UseMI -> isRegSequence ( ) ) { if ( UseMI -> isCopy ( ) && UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) . isPhysical ( ) && ! TRI -> isSGPRReg ( * MRI , UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { numVGPRUses ++ ; }

if ( Visited . insert ( UseMI ) . second ) worklist . insert ( UseMI ) ;


<s> worklist . insert ( & MI ) ; Visited . insert ( & MI ) ; while ( ! worklist . empty ( ) ) { const MachineInstr * Instr = worklist . pop_back_val ( ) ; unsigned Reg = Instr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( const auto & Use : MRI -> use_operands ( Reg ) ) { const MachineInstr * UseMI = Use . getParent ( ) ; AllAGPRUses &= ( UseMI -> isCopy ( ) && TRI -> isAGPR ( * MRI , UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) || TRI -> isAGPR ( * MRI , Use . getReg ( ) ) ; if ( UseMI -> isCopy ( ) || UseMI -> isRegSequence ( ) ) { if ( UseMI -> isCopy ( ) && UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) . isPhysical ( ) && ! TRI -> isSGPRReg ( * MRI , UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { numVGPRUses ++ ; } if ( Visited . insert ( UseMI ) . second ) worklist . insert ( UseMI ) ; continue ; } if ( UseMI -> isPHI ( ) ) { const TargetRegisterClass * UseRC = MRI -> getRegClass ( Use . getReg ( ) ) ; if ( ! TRI -> isSGPRReg ( * MRI , Use . getReg ( ) ) && UseRC != & <STR_LIT> ) numVGPRUses ++ ; continue ; } const TargetRegisterClass * OpRC = TII -> getOpRegClass ( * UseMI , UseMI -> getOperandNo ( & Use ) ) ; if ( ! TRI -> isSGPRClass ( OpRC ) && OpRC != & <STR_LIT> && OpRC != & <STR_LIT> ) { numVGPRUses ++ ;


<s> SetVector < const MachineInstr * > worklist ; SmallSet < const MachineInstr * , <NUM_LIT> > Visited ; worklist . insert ( & MI ) ; Visited . insert ( & MI ) ; while ( ! worklist . empty ( ) ) { const MachineInstr * Instr = worklist . pop_back_val ( ) ; unsigned Reg = Instr -> getOperand ( <NUM_LIT> ) . getReg ( ) ; for ( const auto & Use : MRI -> use_operands ( Reg ) ) { const MachineInstr * UseMI = Use . getParent ( ) ; if ( UseMI -> isCopy ( ) || UseMI -> isRegSequence ( ) ) { if ( UseMI -> isCopy ( ) && UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) . isPhysical ( ) && ! TRI -> isSGPRReg ( * MRI , UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { numVGPRUses ++ ; } if ( Visited . insert ( UseMI ) . second ) worklist . insert ( UseMI ) ; continue ; } if ( UseMI -> isPHI ( ) ) { const TargetRegisterClass * UseRC = MRI -> getRegClass ( Use . getReg ( ) ) ; if ( ! TRI -> isSGPRReg ( * MRI , Use . getReg ( ) ) && UseRC != & <STR_LIT> ) numVGPRUses ++ ; continue ; } const TargetRegisterClass * OpRC = TII -> getOpRegClass ( * UseMI , UseMI -> getOperandNo ( & Use ) ) ; if ( ! TRI -> isSGPRClass ( OpRC ) && OpRC != & <STR_LIT> && OpRC != & <STR_LIT> ) { numVGPRUses ++ ; } } } bool hasVGPRInput = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned InputReg = MI . getOperand ( i ) . getReg ( ) ; MachineInstr * Def = MRI -> getVRegDef ( InputReg ) ; if ( TRI -> isVGPR ( * MRI , InputReg ) ) { if ( Def -> isCopy ( ) ) { unsigned SrcReg = Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC = TRI -> getRegClassForReg ( * MRI , SrcReg ) ;

if ( TRI -> isSGPRClass ( RC ) ) continue ;


<s> MachineInstr * Def = MRI -> getVRegDef ( InputReg ) ; if ( TRI -> isVectorRegister ( * MRI , InputReg ) ) { if ( Def -> isCopy ( ) ) { Register SrcReg = Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * RC = TRI -> getRegClassForReg ( * MRI , SrcReg ) ; if ( TRI -> isSGPRClass ( RC ) ) continue ; } hasVGPRInput = true ; break ; } else if ( Def -> isCopy ( ) && TRI -> isVectorRegister ( * MRI , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Register SrcReg = Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * SrcDef = MRI -> getVRegDef ( SrcReg ) ; unsigned SMovOp ; int64_t Imm ; if ( ! isSafeToFoldImmIntoCopy ( Def , SrcDef , TII , SMovOp , Imm ) ) { hasVGPRInput = true ; break ; } else { MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; Def -> getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; Def -> addImplicitDefUseOperands ( * MF ) ; Def -> setDesc ( TII -> get ( SMovOp ) ) ; } } } if ( ( ! TRI -> isVectorRegister ( * MRI , PHIRes ) && RC0 != & <STR_LIT> ) && ( hasVGPRInput || numVGPRUses > <NUM_LIT> ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } else { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> legalizeOperands ( MI , MDT ) ;


<s> DstRC = MRI -> getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI -> getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI -> getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( TRI -> isSGPRClass ( DstRC ) && ( TRI -> hasVectorRegisters ( Src0RC ) || TRI -> hasVectorRegisters ( Src1RC ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI , MDT ) ; } break ; } case <STR_LIT> : { if ( ST . getConstantBusLimit ( MI . getOpcode ( ) ) != <NUM_LIT> ) break ; int Src0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: src0 ) ; int Src1Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: src1 ) ; MachineOperand & Src0 = MI . getOperand ( Src0Idx ) ; MachineOperand & Src1 = MI . getOperand ( Src1Idx ) ; if ( ( Src0 . isReg ( ) && TRI -> isSGPRReg ( * MRI , Src0 . getReg ( ) ) && Src0 . getReg ( ) != <STR_LIT> ) && ( Src1 . isReg ( ) && TRI -> isSGPRReg ( * MRI , Src1 . getReg ( ) ) && Src1 . getReg ( ) != <STR_LIT> ) ) { bool Resolved = false ; for ( MachineOperand * MO : { & Src0 , & Src1 } ) { if ( MO -> getReg ( ) . isVirtual ( ) ) { MachineInstr * DefMI = MRI -> getVRegDef ( MO -> getReg ( ) ) ; if ( DefMI && TII -> isFoldableCopy ( * DefMI ) ) { const MachineOperand & Def = DefMI -> getOperand ( <NUM_LIT> ) ; if ( Def . isReg ( ) && MO -> getReg ( ) == Def . getReg ( ) && MO -> getSubReg ( ) == Def . getSubReg ( ) ) { const MachineOperand & Copied = DefMI -> getOperand ( <NUM_LIT> ) ; if ( Copied . isImm ( ) && TII -> isInlineConstant ( APInt ( <NUM_LIT> , Copied . getImm ( ) , true ) ) ) { MO -> ChangeToImmediate ( Copied . getImm ( ) ) ; Resolved = true ; break ; } } } } } if ( ! Resolved ) { BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . add ( Src1 ) ; Src1 . ChangeToRegister ( <STR_LIT> , false ) ; } } break ; } } } } if ( MF . getTarget ( ) . getOptLevel ( ) > CodeGenOpt :: None && EnableM0Merge ) hoistAndMergeSGPRInits ( <STR_LIT> , * MRI , TRI , * MDT , TII ) ; return true ;


<s> unsigned SubReg = CopyUse . getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( SubReg != <STR_LIT> ) return false ; MRI . setRegClass ( DstReg , DstRC ) ; MI . getOperand ( <NUM_LIT> ) . setReg ( CopyUse . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; bool IsAGPR = TRI -> hasAGPRs ( DstRC ) ; for ( unsigned I = <NUM_LIT> , N = MI . getNumOperands ( ) ; I != N ; I += <NUM_LIT> ) { Register SrcReg = MI . getOperand ( I ) . getReg ( ) ; unsigned SrcSubReg = MI . getOperand ( I ) . getSubReg ( ) ; const TargetRegisterClass * SrcRC = MRI . getRegClass ( SrcReg ) ; assert ( TRI -> isSGPRClass ( SrcRC ) && <STR_LIT> ) ; SrcRC = TRI -> getSubRegClass ( SrcRC , SrcSubReg ) ; const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentVGPRClass ( SrcRC ) ; Register TmpReg = MRI . createVirtualRegister ( NewSrcRC ) ; BuildMI ( * MI . getParent ( ) , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , TmpReg ) . add ( MI . getOperand ( I ) ) ; if ( IsAGPR ) { const TargetRegisterClass * NewSrcRC = TRI -> getEquivalentAGPRClass ( SrcRC ) ; Register TmpAReg = MRI . createVirtualRegister ( NewSrcRC ) ;

unsigned Opc = NewSrcRC == & <STR_LIT> ? <STR_LIT> : <STR_LIT> ;


<s> for ( const auto & MO : MRI . reg_nodbg_operands ( DstReg ) ) { const auto * UseMI = MO . getParent ( ) ; if ( UseMI == & MI ) continue ; if ( MO . isDef ( ) || UseMI -> getParent ( ) != MI . getParent ( ) || UseMI -> getOpcode ( ) <= TargetOpcode :: GENERIC_OP_END || ! TII -> isOperandLegal ( * UseMI , UseMI -> getOperandNo ( & MO ) , & Src ) ) return false ; } MRI . setRegClass ( DstReg , TRI -> getEquivalentSGPRClass ( MRI . getRegClass ( DstReg ) ) ) ; return true ;


<s> break ; } MachineInstr * DefMI = MRI . getVRegDef ( SrcReg ) ; unsigned SMovOp ; int64_t Imm ; if ( isSafeToFoldImmIntoCopy ( & MI , DefMI , TII , SMovOp , Imm ) ) { MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI ) ; } else if ( isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) { tryChangeVGPRtoSGPRinCopy ( MI , TRI , TII ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( phiHasVGPROperands ( MI , MRI , TRI , TII ) || ! phiHasBreakDef ( MI , MRI , Visited ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; }


<s> const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( MI , * TRI , MRI ) ; if ( isVGPRToSGPRCopy ( SrcRC , DstRC , * TRI ) ) { unsigned SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefMI = MRI . getVRegDef ( SrcReg ) ; unsigned SMovOp ; int64_t Imm ; if ( isSafeToFoldImmIntoCopy ( & MI , DefMI , TII , SMovOp , Imm ) ) { MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI ) ; } else if ( isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) { tryChangeVGPRtoSGPRinCopy ( MI , TRI , TII ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } }

SmallSet < unsigned , <NUM_LIT> > Visited ;


<s> } MachineInstr * DefMI = MRI . getVRegDef ( SrcReg ) ; unsigned SMovOp ; int64_t Imm ; if ( isSafeToFoldImmIntoCopy ( & MI , DefMI , TII , SMovOp , Imm ) ) { MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI , MDT ) ; } else if ( isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) { tryChangeVGPRtoSGPRinCopy ( MI , TRI , TII ) ; } break ; } case <STR_LIT> : { Register Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool HasVGPROperand = phiHasVGPROperands ( MI , MRI , TRI , TII ) ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> && ! HasVGPROperand ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( HasVGPROperand || ! phiHasBreakDef ( MI , MRI , Visited ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI , MDT ) ; } break ;


<s> return TRI -> getSubRegClass ( MRI . getRegClass ( Reg ) , SubReg ) ; } return inferRegClassFromDef ( TRI , MRI , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) , Def -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ;


<s> RC = TRI -> getSubRegClass ( RC , SubReg ) ; for ( MachineRegisterInfo :: use_instr_iterator I = MRI . use_instr_begin ( Reg ) , E = MRI . use_instr_end ( ) ; I != E ; ++ I ) { switch ( I -> getOpcode ( ) ) { case <STR_LIT> : RC = TRI -> getCommonSubClass ( RC , inferRegClassFromUses ( TRI , MRI , I -> getOperand ( <NUM_LIT> ) . getReg ( ) , I -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ) ;


<s> const TargetRegisterClass * RC = TargetRegisterInfo :: isVirtualRegister ( Reg ) ? MRI . getRegClass ( Reg ) : TRI -> getPhysRegClass ( Reg ) ; RC = TRI -> getSubRegClass ( RC , SubReg ) ; for ( MachineRegisterInfo :: use_instr_iterator I = MRI . use_instr_begin ( Reg ) , E = MRI . use_instr_end ( ) ; I != E ; ++ I ) { switch ( I -> getOpcode ( ) ) { case <STR_LIT> : RC = TRI -> getCommonSubClass ( RC , inferRegClassFromUses ( TRI , MRI , I -> getOperand ( <NUM_LIT> ) . getReg ( ) , I -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ) ) ; break ; }


<s> if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) || MRI . getRegClass ( SrcReg ) == & <STR_LIT> ) return false ; SrcRC = TRI -> getSubRegClass ( MRI . getRegClass ( SrcReg ) , SrcSubReg ) ; return TRI -> isSGPRClass ( DstRC ) && TRI -> hasVGPRs ( SrcRC ) ;


<s> const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( MI , * TRI , MRI ) ; if ( isVGPRToSGPRCopy ( SrcRC , DstRC , * TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool SGPRBranch = false ; bool HasBreakDef = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; if ( TRI -> hasVGPRs ( MRI . getRegClass ( Reg ) ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefInstr = MRI . getUniqueVRegDef ( Reg ) ; assert ( DefInstr ) ; switch ( DefInstr -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : HasBreakDef = true ; break ; } } if ( ! SGPRBranch && ! HasBreakDef ) TII -> moveToVALU ( MI ) ; break ; } case <STR_LIT> : { if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; }


<s> MachineDomTreeNode * sentinel = MPDT -> getNode ( & * B ) -> getIDom ( ) ; while ( runner && runner != sentinel ) {

PDF [ runner -> getBlock ( ) ] . insert ( & * B ) ;


<s> AU . addPreserved < MachineDominatorTree > ( ) ; AU . addRequired < MachinePostDominatorTree > ( ) ; AU . addPreserved < MachinePostDominatorTree > ( ) ;


<s> InitListMap Inits ; SmallVector < MachineInstr * , <NUM_LIT> > Clobbers ; bool Changed = false ; for ( auto & MI : MRI . def_instructions ( Reg ) ) { MachineOperand * Imm = nullptr ; for ( auto & MO : MI . operands ( ) ) { if ( ( MO . isReg ( ) && ( ( MO . isDef ( ) && MO . getReg ( ) != Reg ) || ! MO . isDef ( ) ) ) || ( ! MO . isImm ( ) && ! MO . isReg ( ) ) || ( MO . isImm ( ) && Imm ) ) { Imm = nullptr ; break ; } else if ( MO . isImm ( ) ) Imm = & MO ; } if ( Imm ) Inits [ Imm -> getImm ( ) ] . push_front ( & MI ) ; else Clobbers . push_back ( & MI ) ; } for ( auto & Init : Inits ) { auto & Defs = Init . second ; for ( auto I1 = Defs . begin ( ) , E = Defs . end ( ) ; I1 != E ; ) { MachineInstr * MI1 = * I1 ; for ( auto I2 = std :: next ( I1 ) ; I2 != E ; ) { MachineInstr * MI2 = * I2 ; auto intereferes = [ & ] ( MachineBasicBlock :: iterator From , MachineBasicBlock :: iterator To ) -> bool { assert ( MDT . dominates ( & * To , & * From ) ) ; auto interferes = [ & MDT , From , To ] ( MachineInstr * & Clobber ) -> bool {

const MachineBasicBlock * MBBFrom = From -> getParent ( ) ;


<s> if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefMI = MRI . getVRegDef ( SrcReg ) ; unsigned SMovOp ; int64_t Imm ; if ( isSafeToFoldImmIntoCopy ( & MI , DefMI , TII , SMovOp , Imm ) ) { MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI ) ; } else if ( isSGPRToVGPRCopy ( SrcRC , DstRC , * TRI ) ) { tryChangeVGPRtoSGPRinCopy ( MI , TRI , TII ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool HasVGPROperand = phiHasVGPROperands ( MI , MRI , TRI , TII ) ; if ( ! HasVGPROperand ) { bool Uniform = true ; MachineBasicBlock * Join = MI . getParent ( ) ; for ( auto & O : MI . explicit_operands ( ) ) { if ( O . isMBB ( ) ) { MachineBasicBlock * Source = O . getMBB ( ) ; SetVector < MachineBasicBlock * > & SourcePDF = PDF [ Source ] ; SetVector < MachineBasicBlock * > & JoinPDF = PDF [ Join ] ; SetVector < MachineBasicBlock * > CDList ; for ( auto & I : SourcePDF ) { if ( ! JoinPDF . count ( I ) || MDT -> dominates ( Join , I ) ) { if ( hasTerminatorThatModifiesExec ( * I , * TRI ) ) Uniform = false ; } } } } if ( Uniform ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( HasVGPROperand || ! phiHasBreakDef ( MI , MRI , Visited ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; break ; case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ;

Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> bool <ISA_LIT>FixSGPRCopies :: runOnMachineFunction ( MachineFunction & MF ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; MPDT = & getAnalysis < MachinePostDominatorTree > ( ) ; PDF . clear ( ) ; computePDF ( & MF ) ; DEBUG ( printPDF ( ) ) ; SmallVector < MachineInstr * , <NUM_LIT> > Worklist ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; ++ I ) { MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { default : continue ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( ! TargetRegisterInfo :: isVirtualRegister ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; const TargetRegisterClass * SrcRC , * DstRC ; std :: tie ( SrcRC , DstRC ) = getCopyRegClasses ( MI , * TRI , MRI ) ; if ( isVGPRToSGPRCopy ( SrcRC , DstRC , * TRI ) ) { unsigned SrcReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefMI = MRI . getVRegDef ( SrcReg ) ; unsigned SMovOp ;


<s> static bool predsHasDivergentTerminator ( MachineBasicBlock * MBB , const TargetRegisterInfo * TRI ) { DenseSet < MachineBasicBlock * > Visited ; SmallVector < MachineBasicBlock * , <NUM_LIT> > Worklist ( MBB -> pred_begin ( ) , MBB -> pred_end ( ) ) ; while ( ! Worklist . empty ( ) ) { MachineBasicBlock * mbb = Worklist . back ( ) ; Worklist . pop_back ( ) ; if ( ! Visited . insert ( mbb ) . second ) continue ;

if ( hasTerminatorThatModifiesExec ( * mbb , * TRI ) ) return true ;


<s> static bool predsHasDivergentTerminator ( MachineBasicBlock * MBB , const TargetRegisterInfo * TRI ) { DenseSet < MachineBasicBlock * > Visited ; SmallVector < MachineBasicBlock * , <NUM_LIT> > Worklist ( MBB -> pred_begin ( ) , MBB -> pred_end ( ) ) ; while ( ! Worklist . empty ( ) ) { MachineBasicBlock * mbb = Worklist . back ( ) ; Worklist . pop_back ( ) ; if ( ! Visited . insert ( mbb ) . second ) continue ; if ( hasTerminatorThatModifiesExec ( * mbb , * TRI ) ) return true ; Worklist . insert ( Worklist . end ( ) , mbb -> pred_begin ( ) , mbb -> pred_end ( ) ) ;


<s> if ( isVGPRToSGPRCopy ( SrcRC , DstRC , * TRI ) ) { MachineInstr * DefMI = MRI . getVRegDef ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; unsigned SMovOp ; int64_t Imm ; if ( isSafeToFoldImmIntoCopy ( & MI , DefMI , TII , SMovOp , Imm ) ) { MI . getOperand ( <NUM_LIT> ) . ChangeToImmediate ( Imm ) ; MI . addImplicitDefUseOperands ( MF ) ; MI . setDesc ( TII -> get ( SMovOp ) ) ; break ; } TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( phiHasVGPROperands ( MI , MRI , TRI , TII ) || ! phiHasBreakDef ( MI , MRI , Visited ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI ) ; } break ; } case <STR_LIT> : { if ( TRI -> hasVGPRs ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVGPROperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ;


<s> if ( MI . getNumExplicitOperands ( ) == <NUM_LIT> && ! HasVGPROperand ) { MachineBasicBlock * MBB0 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock * MBB1 = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; if ( ! predsHasDivergentTerminator ( MBB0 , TRI ) && ! predsHasDivergentTerminator ( MBB1 , TRI ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << '\n' ) ; break ; } } SmallSet < unsigned , <NUM_LIT> > Visited ; if ( HasVGPROperand || ! phiHasBreakDef ( MI , MRI , Visited ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI , MDT ) ; } break ; } case <STR_LIT> : if ( TRI -> hasVectorRegisters ( TII -> getOpRegClass ( MI , <NUM_LIT> ) ) || ! hasVectorOperands ( MI , TRI ) ) { foldVGPRCopyIntoRegSequence ( MI , TRI , TII , MRI ) ; continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI , MDT ) ; break ; case <STR_LIT> : { const TargetRegisterClass * DstRC , * Src0RC , * Src1RC ; DstRC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src0RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; Src1RC = MRI . getRegClass ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( TRI -> isSGPRClass ( DstRC ) && ( TRI -> hasVectorRegisters ( Src0RC ) || TRI -> hasVectorRegisters ( Src1RC ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; TII -> moveToVALU ( MI , MDT ) ; } break ; }


<s> } break ; } case <STR_LIT> : { DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isSGPRClass ( MRI . getRegClass ( Reg ) ) ) break ; bool SGPRBranch = false ; bool HasBreakDef = false ; for ( unsigned i = <NUM_LIT> ; i < MI . getNumOperands ( ) ; i += <NUM_LIT> ) { unsigned Reg = MI . getOperand ( i ) . getReg ( ) ; if ( TRI -> hasVGPRs ( MRI . getRegClass ( Reg ) ) ) { TII -> moveToVALU ( MI ) ; break ; } MachineInstr * DefInstr = MRI . getUniqueVRegDef ( Reg ) ; assert ( DefInstr ) ; switch ( DefInstr -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : HasBreakDef = true ; break ; } }

if ( ! SGPRBranch && ! HasBreakDef ) TII -> moveToVALU ( MI ) ;


<s> <ISA_LIT>FixSGPRCopies ( TargetMachine & tm ) : MachineFunctionPass ( ID ) {


<s> static bool hasOnlyScalarBr ( const MachineBasicBlock * MBB , const <ISA_LIT>InstrInfo * TII ) { for ( MachineBasicBlock :: const_iterator I = MBB -> getFirstTerminator ( ) , E = MBB -> end ( ) ; I != E ; ++ I ) { if ( ! TII -> isSOPP ( * I ) ) return false ; } return true ;


<s> DEBUG ( dbgs ( ) << PrintReg ( Reg , TRI , <NUM_LIT> ) << <STR_LIT> ) ; continue ; } if ( LiveInToA && LiveInToB ) { DEBUG ( dbgs ( ) << PrintReg ( Reg , TRI , <NUM_LIT> ) << <STR_LIT> ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Reg , TRI , <NUM_LIT> ) << <STR_LIT> << SuccA -> getNumber ( ) << <STR_LIT> << SuccB -> getNumber ( ) << <STR_LIT> << NCD -> getNumber ( ) << '\n' ) ; assert ( TargetRegisterInfo :: isVirtualRegister ( Reg ) && <STR_LIT> ) ; MachineInstr * NCDSGPRUse = BuildMI ( * NCD , NCD -> getFirstNonPHI ( ) , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Implicit ) ; MadeChange = true ; LV -> HandleVirtRegUse ( Reg , NCD , NCDSGPRUse ) ; DEBUG ( NCDSGPRUse -> dump ( ) ) ; } } return MadeChange ;


<s> AU . addRequired < MachinePostDominatorTree > ( ) ; AU . setPreservesCFG ( ) ; AU . addPreserved < LiveIntervals > ( ) ; MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> AU . addRequired < MachinePostDominatorTree > ( ) ; AU . setPreservesCFG ( ) ;

AU . addPreserved < LiveIntervals > ( ) ;


<s> for ( std :: pair < unsigned , LiveRange * > RegLR : SGPRLiveRanges ) { unsigned Reg = RegLR . first ; LiveRange * LR = RegLR . second ; bool LiveInToA = LIS -> isLiveInToMBB ( * LR , SuccA ) ; bool LiveInToB = LIS -> isLiveInToMBB ( * LR , SuccB ) ; if ( ! LiveInToA && ! LiveInToB ) { DEBUG ( dbgs ( ) << PrintReg ( Reg , TRI , <NUM_LIT> ) << <STR_LIT> ) ; continue ; } if ( LiveInToA && LiveInToB ) { DEBUG ( dbgs ( ) << PrintReg ( Reg , TRI , <NUM_LIT> ) << <STR_LIT> ) ; continue ; } DEBUG ( dbgs ( ) << <STR_LIT> << PrintReg ( Reg , TRI , <NUM_LIT> ) << <STR_LIT> << * LR << <STR_LIT> << SuccA -> getNumber ( ) << <STR_LIT> << SuccB -> getNumber ( ) << <STR_LIT> << NCD -> getNumber ( ) << '\n' ) ; assert ( TargetRegisterInfo :: isVirtualRegister ( Reg ) && <STR_LIT> ) ; MachineInstr * NCDSGPRUse = BuildMI ( * NCD , NCD -> getFirstNonPHI ( ) , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Implicit ) ; MadeChange = true ;

SlotIndex <ISA_LIT> = LIS -> InsertMachineInstrInMaps ( NCDSGPRUse ) ;


<s> return new <ISA_LIT>FixSGPRLiveRanges ( ) ;


<s> AU . addRequired < LiveVariables > ( ) ; AU . addPreserved < LiveVariables > ( ) ; AU . addRequired < MachinePostDominatorTree > ( ) ; AU . addPreserved < MachinePostDominatorTree > ( ) ; AU . setPreservesCFG ( ) ;

MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> const char * getPassName ( ) const override { return <STR_LIT> ;


<s> const char * getPassName ( ) const override {


<s> for ( const MachineOperand & MO : MI . defs ( ) ) { unsigned Def = MO . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( Def ) ) { if ( TRI -> isSGPRClass ( MRI . getRegClass ( Def ) ) ) { if ( LV -> isLiveOut ( Def , * MBB ) ) SGPRLiveRanges . push_back ( Def ) ; } } } } if ( MBB -> succ_size ( ) < <NUM_LIT> ) continue ; assert ( MBB -> succ_size ( ) == <NUM_LIT> ) ; MachineBasicBlock * SuccA = * MBB -> succ_begin ( ) ; MachineBasicBlock * SuccB = * ( ++ MBB -> succ_begin ( ) ) ; MachineBasicBlock * NCD = PDT -> findNearestCommonDominator ( SuccA , SuccB ) ; if ( ! NCD ) continue ; MachineBasicBlock :: iterator NCDTerm = NCD -> getFirstTerminator ( ) ; if ( NCDTerm != NCD -> end ( ) && NCDTerm -> getOpcode ( ) == <STR_LIT> ) { assert ( NCD -> succ_size ( ) == <NUM_LIT> ) ; NCD = PDT -> findNearestCommonDominator ( * NCD -> succ_begin ( ) , * ( ++ NCD -> succ_begin ( ) ) ) ;


<s> MachineOperand * WOp = Worklist . pop_back_val ( ) ; if ( ! WOp -> isReg ( ) || ! Register :: isVirtualRegister ( WOp -> getReg ( ) ) ) continue ; MachineInstr * DefInst = MRI . getUniqueVRegDef ( WOp -> getReg ( ) ) ; switch ( DefInst -> getOpcode ( ) ) { default : continue ; case <STR_LIT> : Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : if ( DefInst -> getNumOperands ( ) != <NUM_LIT> ) continue ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : if ( DefInst -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <STR_LIT> ) continue ;

BaseReg = DefInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ;


<s> if ( DefInst -> getNumOperands ( ) != <NUM_LIT> ) continue ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : if ( DefInst -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <STR_LIT> ) continue ; BaseReg = DefInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( DefInst -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <STR_LIT> ) continue ; IndexReg = DefInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * MI = MRI . getUniqueVRegDef ( IndexReg ) ; if ( ! MI || ! MI -> isCopy ( ) ) continue ; const TargetRegisterClass * IdxRC , * BaseRC ;

IdxRC = MRI . getRegClass ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> MachineInstr * NewGlob = nullptr ; NewGlob = BuildMI ( MBB , I , MI . getDebugLoc ( ) , TII -> get ( NewOpcd ) ) ; if ( HasVdst ) NewGlob -> addOperand ( MF , MI . getOperand ( <NUM_LIT> ) ) ; NewGlob -> addOperand ( MF , MachineOperand :: CreateReg ( IndexReg , false ) ) ; if ( VData ) NewGlob -> addOperand ( MF , * VData ) ; NewGlob -> addOperand ( MF , MachineOperand :: CreateReg ( BaseReg , false ) ) ; NewGlob -> addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) ) ; MachineOperand * Glc = TII -> getNamedOperand ( MI , <STR_LIT> :: glc ) ; if ( Glc ) NewGlob -> addOperand ( MF , * Glc ) ; MachineOperand * DLC = TII -> getNamedOperand ( MI , <STR_LIT> :: dlc ) ; if ( DLC ) NewGlob -> addOperand ( MF , * DLC ) ; NewGlob -> addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: slc ) ) ; MachineOperand * VDstInOp = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst_in ) ; if ( VDstInOp ) NewGlob -> addOperand ( MF , * VDstInOp ) ; NewGlob -> copyImplicitOps ( MF , MI ) ;


<s> if ( skipFunction ( MF . getFunction ( ) ) ) return false ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; bool FuncModified = false ; for ( MachineBasicBlock & MBB : MF ) { FuncModified |= fixupGlobalSaddr ( MBB , MF , MRI , ST , TII , TRI ) ;


<s> break ; case <STR_LIT> : if ( DefInst -> getNumOperands ( ) != <NUM_LIT> ) continue ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : if ( DefInst -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <STR_LIT> ) continue ; BaseReg = DefInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( DefInst -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <STR_LIT> ) continue ; IndexReg = DefInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * MI = MRI . getUniqueVRegDef ( IndexReg ) ; if ( ! MI || ! MI -> isCopy ( ) ) continue ; const TargetRegisterClass * IdxRC , * BaseRC ; IdxRC = MRI . getRegClass ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( <STR_LIT> ( IdxRC -> getID ( ) ) != <NUM_LIT> ) continue ; IndexReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MI = MRI . getUniqueVRegDef ( BaseReg ) ; if ( ! MI || ! MI -> isCopy ( ) ) continue ; BaseReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; BaseRC = MRI . getRegClass ( BaseReg ) ; if ( <STR_LIT> ( BaseRC -> getID ( ) ) != <NUM_LIT> ) continue ; if ( ! TRI -> isSGPRReg ( MRI , BaseReg ) ) return false ;

if ( ! TRI -> hasVGPRs ( MRI . getRegClass ( IndexReg ) ) ) return false ;


<s> Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : if ( DefInst -> getNumOperands ( ) != <NUM_LIT> ) continue ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; Worklist . push_back ( & DefInst -> getOperand ( <NUM_LIT> ) ) ; break ; case <STR_LIT> : if ( DefInst -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <STR_LIT> ) continue ; BaseReg = DefInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( DefInst -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != <STR_LIT> ) continue ; IndexReg = DefInst -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * MI = MRI . getUniqueVRegDef ( IndexReg ) ; if ( ! MI || ! MI -> isCopy ( ) ) continue ; const TargetRegisterClass * IdxRC , * BaseRC ; IdxRC = MRI . getRegClass ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( <STR_LIT> ( IdxRC -> getID ( ) ) != <NUM_LIT> ) continue ; IndexReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; MI = MRI . getUniqueVRegDef ( BaseReg ) ; if ( ! MI || ! MI -> isCopy ( ) ) continue ; BaseReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; BaseRC = MRI . getRegClass ( BaseReg ) ;

if ( <STR_LIT> ( BaseRC -> getID ( ) ) != <NUM_LIT> ) continue ;


<s> NewGlob = BuildMI ( MBB , I , MI . getDebugLoc ( ) , TII -> get ( NewOpcd ) ) ; if ( HasVdst ) NewGlob -> addOperand ( MF , MI . getOperand ( <NUM_LIT> ) ) ; NewGlob -> addOperand ( MF , MachineOperand :: CreateReg ( IndexReg , false ) ) ; if ( VData ) NewGlob -> addOperand ( MF , * VData ) ; NewGlob -> addOperand ( MF , MachineOperand :: CreateReg ( BaseReg , false ) ) ; NewGlob -> addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) ) ; MachineOperand * Glc = TII -> getNamedOperand ( MI , <STR_LIT> :: glc ) ; if ( Glc ) NewGlob -> addOperand ( MF , * Glc ) ; NewGlob -> addOperand ( * TII -> getNamedOperand ( MI , <STR_LIT> :: slc ) ) ; MachineOperand * VDstInOp = TII -> getNamedOperand ( MI , <STR_LIT> :: vdst_in ) ; if ( VDstInOp ) NewGlob -> addOperand ( MF , * VDstInOp ) ;

NewGlob -> copyImplicitOps ( MF , MI ) ;


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; bool FuncModified = false ; for ( MachineBasicBlock & MBB : MF ) { FuncModified |= fixupGlobalSaddr ( MBB , MF , MRI , ST , TII , TRI ) ; } return FuncModified ;


<s> <ISA_LIT>FixupVectorISel ( ) : MachineFunctionPass ( ID ) { initialize<ISA_LIT>FixupVectorISelPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> for ( MachineInstr & MI : MBB ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : if ( TII -> isVGPRCopy ( MI ) && ! MI . readsRegister ( <STR_LIT> , TRI ) ) { MI . addOperand ( MF , MachineOperand :: CreateReg ( <STR_LIT> , false , true ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; Changed = true ; }


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; bool Changed = false ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : MBB ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : if ( TII -> isVGPRCopy ( MI ) && ! MI . readsRegister ( <STR_LIT> , TRI ) ) { MI . addOperand ( MF , MachineOperand :: CreateReg ( <STR_LIT> , false , true ) ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; Changed = true ; } break ; default : break ; } }


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; bool Changed = false ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : MBB ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : if ( TII -> isVGPRCopy ( MI ) && ! MI . readsRegister ( <STR_LIT> , TRI ) ) { MI . addOperand ( MF , MachineOperand :: CreateReg ( <STR_LIT> , false , true ) ) ; DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; Changed = true ; } break ; default : break ; }


<s> StringRef getPassName ( ) const override {


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; bool Changed = false ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : MBB ) { switch ( MI . getOpcode ( ) ) { case <STR_LIT> : if ( TII -> isVGPRCopy ( MI ) && ! MI . readsRegister ( <STR_LIT> , TRI ) ) { MI . addOperand ( MF , MachineOperand :: CreateReg ( <STR_LIT> , false , true ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; Changed = true ;


<s> initialize<ISA_LIT>FixVGPRCopiesPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> AU . addRequiredID ( MachineLoopInfoID ) ; AU . addPreserved < SlotIndexes > ( ) ; AU . addPreserved < LiveIntervals > ( ) ;

AU . addPreservedID ( LiveVariablesID ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequiredID ( MachineDominatorsID ) ; AU . addRequiredID ( MachineLoopInfoID ) ; AU . addPreserved < SlotIndexes > ( ) ; AU . addPreserved < LiveIntervals > ( ) ; AU . addPreservedID ( LiveVariablesID ) ;

AU . addPreservedID ( MachineLoopInfoID ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * WWM ) ; ContainsWWM = true ; break ; } } if ( ! ContainsWWM ) return false ; unsigned Reg = DefOpnd -> getReg ( ) ; for ( auto Pred : Loop -> getHeader ( ) -> predecessors ( ) ) { if ( ! Loop -> contains ( Pred ) ) { auto ImplicitDef = BuildMI ( * Pred , Pred -> getFirstTerminator ( ) , DebugLoc ( ) , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , Reg ) ; LLVM_DEBUG ( dbgs ( ) << * ImplicitDef ) ; ( void ) ImplicitDef ; } } DefOpnd -> getParent ( ) -> addOperand ( MachineOperand :: CreateReg ( Reg , false , true ) ) ; LLVM_DEBUG ( dbgs ( ) << * DefOpnd -> getParent ( ) ) ;


<s> } } if ( ! ContainsWWM ) return false ; unsigned Reg = DefOpnd -> getReg ( ) ; for ( auto Pred : Loop -> getHeader ( ) -> predecessors ( ) ) { if ( ! Loop -> contains ( Pred ) ) { auto ImplicitDef = BuildMI ( * Pred , Pred -> getFirstTerminator ( ) , DebugLoc ( ) , TII -> get ( TargetOpcode :: IMPLICIT_DEF ) , Reg ) ; LLVM_DEBUG ( dbgs ( ) << * ImplicitDef ) ;


<s> bool ContainsWWM = false ; for ( auto WWM : WWMs ) { if ( Loop -> contains ( WWM -> getParent ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * WWM ) ; ContainsWWM = true ; break ; } } if ( ! ContainsWWM ) return false ; unsigned Reg = DefOpnd -> getReg ( ) ; for ( auto & Use : MRI -> use_operands ( Reg ) ) Use . setIsKill ( false ) ; SmallVector < MachineInstr * , <NUM_LIT> > Defs ; for ( auto & Def : MRI -> def_instructions ( Reg ) ) Defs . push_back ( & Def ) ; for ( auto Def : Defs ) { if ( DefOpnd -> getParent ( ) == Def ) continue ;

Def -> addOperand ( MachineOperand :: CreateReg ( Reg , false , true ) ) ;


<s> } if ( ! ContainsWWM ) return false ; unsigned Reg = DefOpnd -> getReg ( ) ; for ( auto & Use : MRI -> use_operands ( Reg ) ) Use . setIsKill ( false ) ; SmallVector < MachineInstr * , <NUM_LIT> > Defs ; for ( auto & Def : MRI -> def_instructions ( Reg ) ) Defs . push_back ( & Def ) ; for ( auto Def : Defs ) { if ( DefOpnd -> getParent ( ) == Def ) continue ; Def -> addOperand ( MachineOperand :: CreateReg ( Reg , false , true ) ) ; LLVM_DEBUG ( dbgs ( ) << * Def ) ; } return true ;


<s> if ( ! ContainsWWM ) return false ; MachineInstr * OtherDef = nullptr ; for ( auto & MI : MRI -> def_instructions ( Reg ) ) { if ( & MI != DefOpnd -> getParent ( ) ) OtherDef = & MI ; } OtherDef -> addOperand ( MachineOperand :: CreateReg ( Reg , false , true ) ) ; LLVM_DEBUG ( dbgs ( ) << * OtherDef ) ; return true ;