<s> TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; DomTree = & getAnalysis < MachineDominatorTree > ( ) ; LoopInfo = & getAnalysis < MachineLoopInfo > ( ) ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) WWMs . push_back ( & MI ) ; else { for ( MachineOperand & DefOpnd : MI . defs ( ) ) { if ( DefOpnd . isReg ( ) ) { unsigned Reg = DefOpnd . getReg ( ) ; if ( TRI -> isVGPR ( * MRI , Reg ) ) processDef ( DefOpnd ) ; } } } } } if ( ! WWMs . empty ( ) ) { for ( auto ThenDef : ThenDefs ) Modified |= processThenDef ( ThenDef ) ; for ( auto LoopExitDef : LoopExitDefs ) Modified |= processLoopExitDef ( LoopExitDef . first , LoopExitDef . second ) ; for ( auto LoopPhiDef : LoopPhiDefs ) Modified |= processLoopPhiDef ( LoopPhiDef . first , LoopPhiDef . second ) ; } WWMs . clear ( ) ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; for ( MachineBasicBlock & MBB : MF ) {


<s> void <ISA_LIT>FixWWMLiveness :: addDefs ( const MachineInstr & MI , SparseBitVector < > & Regs ) { for ( const MachineOperand & Op : MI . defs ( ) ) { if ( Op . isReg ( ) ) { unsigned Reg = Op . getReg ( ) ; if ( TRI -> isVGPR ( * MRI , Reg ) ) Regs . set ( Reg ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>FixWWMLivenessPass ( ) {

return new <ISA_LIT>FixWWMLiveness ( ) ;


<s> AU . addPreservedID ( LiveVariablesID ) ; AU . addPreservedID ( MachineLoopInfoID ) ; AU . addPreservedID ( MachineDominatorsID ) ; AU . setPreservesCFG ( ) ;

MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> bool Modified = false ; LIS = getAnalysisIfAvailable < LiveIntervals > ( ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) { Modified |= runOnWWMInstruction ( MI ) ; } } } return Modified ;


<s> } SparseBitVector < > Reachable ; for ( auto II = ++ MachineBasicBlock :: reverse_iterator ( WWM ) , IE = MBB -> rend ( ) ; II != IE ; ++ II ) { addDefs ( * II , Reachable ) ; } for ( idf_iterator < MachineBasicBlock * > I = ++ idf_begin ( MBB ) , E = idf_end ( MBB ) ; I != E ; ++ I ) { for ( const MachineInstr & MI : * * I ) { addDefs ( MI , Reachable ) ; } } LiveOut &= Reachable ; bool Modified = false ; for ( unsigned Reg : LiveOut ) { WWM . addOperand ( MachineOperand :: CreateReg ( Reg , false , true ) ) ; if ( LIS ) { LIS -> removeInterval ( Reg ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ;


<s> initialize<ISA_LIT>FixWWMLivenessPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> foldOperand ( OpToFold , UseMI , NonInlineUseOpNo , FoldList , CopiesToReplace ) ; } } else { for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; ++ Use ) { MachineInstr * UseMI = Use -> getParent ( ) ; foldOperand ( OpToFold , UseMI , Use . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } } MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; for ( MachineInstr * Copy : CopiesToReplace ) Copy -> addImplicitDefUseOperands ( * MF ) ; for ( FoldCandidate & Fold : FoldList ) { if ( updateOperand ( Fold , * TRI ) ) { if ( Fold . isReg ( ) ) { assert ( Fold . OpToFold && Fold . OpToFold -> isReg ( ) ) ; MRI -> clearKillFlags ( Fold . OpToFold -> getReg ( ) ) ;


<s> UseMI -> setDesc ( TII -> get ( MovOp ) ) ; CopiesToReplace . push_back ( UseMI ) ; } else { const MCInstrDesc & UseDesc = UseMI -> getDesc ( ) ; if ( UseDesc . isVariadic ( ) || UseDesc . OpInfo [ UseOpIdx ] . RegClass == - <NUM_LIT> ) return ; } if ( ! FoldingImm ) { tryAddToFoldList ( FoldList , UseMI , UseOpIdx , & OpToFold , TII ) ; return ; } const MCInstrDesc & FoldDesc = OpToFold . getParent ( ) -> getDesc ( ) ; const TargetRegisterClass * FoldRC = TRI -> getRegClass ( FoldDesc . OpInfo [ <NUM_LIT> ] . RegClass ) ; if ( UseOp . getSubReg ( ) && <STR_LIT> ( FoldRC -> getID ( ) ) == <NUM_LIT> ) { unsigned UseReg = UseOp . getReg ( ) ; const TargetRegisterClass * UseRC = TargetRegisterInfo :: isVirtualRegister ( UseReg ) ? MRI -> getRegClass ( UseReg ) : TRI -> getPhysRegClass ( UseReg ) ; if ( <STR_LIT> ( UseRC -> getID ( ) ) != <NUM_LIT> ) return ; APInt Imm ( <NUM_LIT> , OpToFold . getImm ( ) ) ; if ( UseOp . getSubReg ( ) == <STR_LIT> ) { Imm = Imm . getLoBits ( <NUM_LIT> ) ;


<s> if ( Op . getSubReg ( ) != <STR_LIT> ) return & Op ; MachineInstr * Def = MRI . getVRegDef ( Op . getReg ( ) ) ;

if ( Def && Def -> isMoveImmediate ( ) ) {


<s> static bool isInlineConstantIfFolded ( const <ISA_LIT>InstrInfo * TII , const MachineInstr & UseMI , unsigned OpNo , const MachineOperand & OpToFold ) { if ( TII -> isInlineConstant ( UseMI , OpNo , OpToFold ) ) return true ; unsigned Opc = UseMI . getOpcode ( ) ; switch ( Opc ) { case <STR_LIT> : case <STR_LIT> : { int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( static_cast < int > ( OpNo ) == Src2Idx ) { bool IsF32 = Opc == <STR_LIT> ; const MCInstrDesc & MadDesc = TII -> get ( IsF32 ? <STR_LIT> : <STR_LIT> ) ; return TII -> isInlineConstant ( OpToFold , MadDesc . OpInfo [ OpNo ] . OperandType ) ; } return false ; }


<s> bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( OpToFold . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( OpToFold . getReg ( ) ) ) continue ; MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( Dst . getReg ( ) ) ) continue ; foldInstOperand ( MI , OpToFold ) ; }


<s> } if ( Opc == <STR_LIT> && OpToFold -> isImm ( ) ) { MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; FoldList . push_back ( FoldCandidate ( MI , OpNo , OpToFold ) ) ; return true ; } if ( isUseMIInFoldList ( FoldList , MI ) ) return false ; unsigned CommuteIdx0 = TargetInstrInfo :: CommuteAnyOperandIndex ; unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( * MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) OpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) OpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , OpNo , OpToFold ) ) { TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ; return false ; } FoldList . push_back ( FoldCandidate ( MI , OpNo , OpToFold , true ) ) ; return true ; } FoldList . push_back ( FoldCandidate ( MI , OpNo , OpToFold ) ) ; return true ;


<s> unsigned CommuteIdx0 = TargetInstrInfo :: CommuteAnyOperandIndex ; unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( * MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) OpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) OpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ;

if ( ! TII -> isOperandLegal ( * MI , OpNo , OpToFold ) ) {


<s> } int Src1Idx = <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ; if ( Src1Idx == - <NUM_LIT> ) return false ; int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; MachineOperand * Src0 = getImmOrMaterializedImm ( MRI , MI -> getOperand ( Src0Idx ) ) ; MachineOperand * Src1 = getImmOrMaterializedImm ( MRI , MI -> getOperand ( Src1Idx ) ) ; if ( ! Src0 -> isImm ( ) && ! Src1 -> isImm ( ) ) return false ; if ( Src0 -> isImm ( ) && Src1 -> isImm ( ) ) { int32_t NewImm ; if ( ! evalBinaryInstruction ( Opc , NewImm , Src0 -> getImm ( ) , Src1 -> getImm ( ) ) ) return false ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; bool IsSGPR = TRI . isSGPRReg ( MRI , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MI -> getOperand ( Src0Idx ) . ChangeToImmediate ( NewImm ) ; MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( IsSGPR ) ) ) ; return true ; } if ( ! MI -> isCommutable ( ) ) return false ; if ( Src0 -> isImm ( ) && ! Src1 -> isImm ( ) ) { std :: swap ( Src0 , Src1 ) ;


<s> if ( ! evalBinaryInstruction ( Opc , NewImm , Src0 -> getImm ( ) , Src1 -> getImm ( ) ) ) return false ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; bool IsSGPR = TRI . isSGPRReg ( MRI , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MI -> getOperand ( Src0Idx ) . ChangeToImmediate ( NewImm ) ; MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( IsSGPR ) ) ) ; return true ; } if ( ! MI -> isCommutable ( ) ) return false ; if ( Src0 -> isImm ( ) && ! Src1 -> isImm ( ) ) { std :: swap ( Src0 , Src1 ) ; std :: swap ( Src0Idx , Src1Idx ) ; } int32_t Src1Val = static_cast < int32_t > ( Src1 -> getImm ( ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) { if ( Src1Val == <NUM_LIT> ) { MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( <STR_LIT> ) ) ; } else if ( Src1Val == - <NUM_LIT> ) { MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( Opc == <STR_LIT> ) ) ) ; } else return false ; return true ; } if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { if ( Src1Val == <NUM_LIT> ) { MI -> RemoveOperand ( Src0Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( Opc == <STR_LIT> ) ) ) ; } else if ( Src1Val == - <NUM_LIT> ) { MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( <STR_LIT> ) ) ; stripExtraCopyOperands ( * MI ) ; } else return false ; return true ; } if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { if ( Src1Val == <NUM_LIT> ) { MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( <STR_LIT> ) ) ; return true ; } }


<s> MachineInstr * Def = MRI -> getVRegDef ( ClampSrc -> getReg ( ) ) ; if ( TII -> getClampMask ( * Def ) != TII -> getClampMask ( MI ) ) return false ; MachineOperand * DefClamp = TII -> getNamedOperand ( * Def , <STR_LIT> :: clamp ) ; if ( ! DefClamp ) return false ; DEBUG ( dbgs ( ) << <STR_LIT> << * DefClamp << <STR_LIT> << * Def << '\n' ) ; DefClamp -> setImm ( <NUM_LIT> ) ;

MRI -> replaceRegWith ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> bool <ISA_LIT>FoldOperands :: tryFoldClamp ( MachineInstr & MI ) { const MachineOperand * ClampSrc = isClamp ( MI ) ; if ( ! ClampSrc || ! hasOneNonDBGUseInst ( * MRI , ClampSrc -> getReg ( ) ) ) return false ; MachineInstr * Def = MRI -> getVRegDef ( ClampSrc -> getReg ( ) ) ; if ( TII -> getClampMask ( * Def ) != TII -> getClampMask ( MI ) ) return false ; MachineOperand * DefClamp = TII -> getNamedOperand ( * Def , <STR_LIT> :: clamp ) ; if ( ! DefClamp ) return false ;

DEBUG ( dbgs ( ) << <STR_LIT> << * DefClamp << <STR_LIT> << * Def << '\n' ) ;


<s> const MachineOperand * Src0 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src1 ) ; if ( Src1 -> isIdenticalTo ( * Src0 ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * MI << <STR_LIT> ) ; int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( Src2Idx != - <NUM_LIT> ) MI -> RemoveOperand ( Src2Idx ) ; MI -> RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ) ; mutateCopyOp ( * MI , TII -> get ( Src0 -> isReg ( ) ? ( unsigned ) <STR_LIT> : getMovOpc ( false ) ) ) ; DEBUG ( dbgs ( ) << * MI << '\n' ) ;


<s> const MachineOperand * RegOp ; int OMod ; std :: tie ( RegOp , OMod ) = isOMod ( MI ) ; if ( OMod == <STR_LIT> || ! RegOp -> isReg ( ) || RegOp -> getSubReg ( ) != <STR_LIT> || ! hasOneNonDBGUseInst ( * MRI , RegOp -> getReg ( ) ) ) return false ;

MachineInstr * Def = MRI -> getVRegDef ( RegOp -> getReg ( ) ) ;


<s> MachineOperand * New = Fold . OpToFold ; if ( TargetRegisterInfo :: isVirtualRegister ( Old . getReg ( ) ) && TargetRegisterInfo :: isVirtualRegister ( New -> getReg ( ) ) ) { Old . substVirtReg ( New -> getReg ( ) , New -> getSubReg ( ) , TRI ) ;

Old . setIsUndef ( New -> isUndef ( ) ) ;


<s> Old . ChangeToImmediate ( Fold . ImmToFold ) ; return true ; } if ( Fold . isFI ( ) ) { Old . ChangeToFrameIndex ( Fold . FrameIndexToFold ) ; return true ; } MachineOperand * New = Fold . OpToFold ;

if ( TargetRegisterInfo :: isVirtualRegister ( Old . getReg ( ) ) && TargetRegisterInfo :: isVirtualRegister ( New -> getReg ( ) ) ) {


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ( Commuted ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << OpNo << <STR_LIT> << * MI ) ; FoldList . emplace_back ( MI , OpNo , FoldOp , Commuted , ShrinkOp ) ;


<s> } else if ( FoldOp -> isFI ( ) ) { FrameIndexToFold = FoldOp -> getIndex ( ) ; } else {

assert ( FoldOp -> isReg ( ) || FoldOp -> isGlobal ( ) ) ;


<s> FoldCandidate ( MachineInstr * MI , unsigned OpNo , MachineOperand * FoldOp , bool Commuted_ = false , int ShrinkOp = - <NUM_LIT> ) : UseMI ( MI ) , OpToFold ( nullptr ) , ShrinkOpcode ( ShrinkOp ) , UseOpNo ( OpNo ) , Kind ( FoldOp -> getType ( ) ) , Commuted ( Commuted_ ) { if ( FoldOp -> isImm ( ) ) { ImmToFold = FoldOp -> getImm ( ) ; } else if ( FoldOp -> isFI ( ) ) { FrameIndexToFold = FoldOp -> getIndex ( ) ; } else { assert ( FoldOp -> isReg ( ) || FoldOp -> isGlobal ( ) ) ; OpToFold = FoldOp ; }


<s> static bool frameIndexMayFold ( const <ISA_LIT>InstrInfo * TII , const MachineInstr & UseMI , int OpNo , const MachineOperand & OpToFold ) { if ( ! OpToFold . isFI ( ) ) return false ; if ( TII -> isMUBUF ( UseMI ) ) return OpNo == <STR_LIT> ( UseMI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; if ( ! TII -> isFLATScratch ( UseMI ) ) return false ; int <ISA_LIT>dx = <STR_LIT> ( UseMI . getOpcode ( ) , <STR_LIT> :: saddr ) ; if ( OpNo == <ISA_LIT>dx ) return true ; int VIdx = <STR_LIT> ( UseMI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; return OpNo == VIdx && <ISA_LIT>dx == - <NUM_LIT> ;


<s> if ( TII -> isMUBUF ( UseMI ) ) return OpNo == <STR_LIT> ( UseMI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; if ( ! TII -> isFLATScratch ( UseMI ) ) return false ; int <ISA_LIT>dx = <STR_LIT> ( UseMI . getOpcode ( ) , <STR_LIT> :: saddr ) ;

if ( OpNo == <ISA_LIT>dx ) return true ;


<s> MachineInstr * Def = MRI . getVRegDef ( Op . getReg ( ) ) ; if ( Def && Def -> isMoveImmediate ( ) ) { MachineOperand & ImmSrc = Def -> getOperand ( <NUM_LIT> ) ; if ( ImmSrc . isImm ( ) ) return & ImmSrc ; } } return & Op ;


<s> if ( Op . isReg ( ) ) { if ( Op . getSubReg ( ) != <STR_LIT> || ! Op . getReg ( ) . isVirtual ( ) ) return & Op ; MachineInstr * Def = MRI . getVRegDef ( Op . getReg ( ) ) ; if ( Def && Def -> isMoveImmediate ( ) ) { MachineOperand & ImmSrc = Def -> getOperand ( <NUM_LIT> ) ; if ( ImmSrc . isImm ( ) ) return & ImmSrc ; } } return & Op ;


<s> return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; default : return <STR_LIT> ; } } case <STR_LIT> : { switch ( static_cast < uint16_t > ( Val ) ) { case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ;


<s> case <STR_LIT> : { switch ( Val ) { case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; default : return <STR_LIT> ; } } case <STR_LIT> : { switch ( static_cast < uint32_t > ( Val ) ) { case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ;


<s> static bool getRegSeqInit ( SmallVectorImpl < std :: pair < MachineOperand * , unsigned >> & Defs , Register UseReg , uint8_t OpTy , const <ISA_LIT>InstrInfo * TII , const MachineRegisterInfo & MRI ) { MachineInstr * Def = MRI . getVRegDef ( UseReg ) ; if ( ! Def || ! Def -> isRegSequence ( ) ) return false ; for ( unsigned I = <NUM_LIT> , E = Def -> getNumExplicitOperands ( ) ; I < E ; I += <NUM_LIT> ) { MachineOperand * Sub = & Def -> getOperand ( I ) ; assert ( Sub -> isReg ( ) ) ; for ( MachineInstr * SubDef = MRI . getVRegDef ( Sub -> getReg ( ) ) ;

SubDef && Sub -> isReg ( ) && Sub -> getReg ( ) . isVirtual ( ) && ! Sub -> getSubReg ( ) && TII -> isFoldableCopy ( * SubDef ) ;


<s> assert ( Sub -> isReg ( ) ) ; for ( MachineInstr * SubDef = MRI . getVRegDef ( Sub -> getReg ( ) ) ; SubDef && Sub -> isReg ( ) && Sub -> getReg ( ) . isVirtual ( ) && ! Sub -> getSubReg ( ) && TII -> isFoldableCopy ( * SubDef ) ; SubDef = MRI . getVRegDef ( Sub -> getReg ( ) ) ) { MachineOperand * Op = & SubDef -> getOperand ( <NUM_LIT> ) ; if ( Op -> isImm ( ) ) { if ( TII -> isInlineConstant ( * Op , OpTy ) ) Sub = Op ; break ;


<s> int getShrinkOpcode ( ) const { return ShrinkOpcode ;


<s> const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( ! Src0 -> isReg ( ) || ! Src1 -> isReg ( ) || Src0 -> getReg ( ) != Src1 -> getReg ( ) || Src0 -> getSubReg ( ) != Src1 -> getSubReg ( ) || Src0 -> getSubReg ( ) != <STR_LIT> ) return nullptr ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return nullptr ;

unsigned Src0Mods = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) -> getImm ( ) ;


<s> if ( ! Src0 -> isReg ( ) || ! Src1 -> isReg ( ) || Src0 -> getReg ( ) != Src1 -> getReg ( ) || Src0 -> getSubReg ( ) != Src1 -> getSubReg ( ) || Src0 -> getSubReg ( ) != <STR_LIT> ) return nullptr ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return nullptr ; unsigned Src0Mods = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) -> getImm ( ) ; unsigned Src1Mods = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) -> getImm ( ) ; unsigned UnsetMods = ( Op == <STR_LIT> ) ? <STR_LIT> : <NUM_LIT> ; if ( Src0Mods != UnsetMods && Src1Mods != UnsetMods ) return nullptr ; return Src0 ; } default :


<s> return Kind == MachineOperand :: MO_GlobalAddress ;


<s> static bool isInlineConstantIfFolded ( const <ISA_LIT>InstrInfo * TII , const MachineInstr & UseMI , unsigned OpNo , const MachineOperand & OpToFold ) { if ( TII -> isInlineConstant ( UseMI , OpNo , OpToFold ) ) return true ; unsigned Opc = UseMI . getOpcode ( ) ; unsigned NewOpc = macToMad ( Opc ) ; if ( NewOpc != <STR_LIT> ) { int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( static_cast < int > ( OpNo ) == Src2Idx ) { const MCInstrDesc & MadDesc = TII -> get ( NewOpc ) ; return TII -> isInlineConstant ( OpToFold , MadDesc . OpInfo [ OpNo ] . OperandType ) ;


<s> RegOp = Src0 ; } else return std :: make_pair ( nullptr , <STR_LIT> ) ; int OMod = getOModValue ( Op , ImmOp -> getImm ( ) ) ; if ( OMod == <STR_LIT> || TII -> hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) ) return std :: make_pair ( nullptr , <STR_LIT> ) ; return std :: make_pair ( RegOp , OMod ) ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( ( Op == <STR_LIT> && MFI -> getMode ( ) . FP32OutputDenormals ) || ( ( Op == <STR_LIT> || Op == <STR_LIT> ) && MFI -> getMode ( ) . FP64FP16OutputDenormals ) ) return std :: make_pair ( nullptr , <STR_LIT> ) ; const MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src0 -> isReg ( ) && Src1 -> isReg ( ) && Src0 -> getReg ( ) == Src1 -> getReg ( ) && Src0 -> getSubReg ( ) == Src1 -> getSubReg ( ) && ! TII -> hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) && ! TII -> hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) && ! TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) && ! TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return std :: make_pair ( Src0 , <STR_LIT> ) ; return std :: make_pair ( nullptr , <STR_LIT> ) ; } default : return std :: make_pair ( nullptr , <STR_LIT> ) ; }


<s> case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ;


<s> if ( ! TII -> isFoldableCopy ( MI ) ) { if ( CurrentKnownM0Val && MI . modifiesRegister ( <STR_LIT> , TRI ) ) CurrentKnownM0Val = nullptr ; if ( IsIEEEMode || ( ! HasNSZ && ! MI . getFlag ( MachineInstr :: FmNsz ) ) || ! tryFoldOMod ( MI ) ) tryFoldClamp ( MI ) ; continue ; } if ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) { MachineOperand & NewM0Val = MI . getOperand ( <NUM_LIT> ) ; if ( CurrentKnownM0Val && CurrentKnownM0Val -> isIdenticalTo ( NewM0Val ) ) { MI . eraseFromParent ( ) ; continue ; } CurrentKnownM0Val = ( NewM0Val . isReg ( ) && NewM0Val . getReg ( ) . isPhysical ( ) ) ? nullptr : & NewM0Val ; continue ; } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) || OpToFold . isGlobal ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( OpToFold . isReg ( ) && ! OpToFold . getReg ( ) . isVirtual ( ) ) continue ;

if ( ! MI . getOperand ( <NUM_LIT> ) . getReg ( ) . isVirtual ( ) ) continue ;


<s> } if ( isUseMIInFoldList ( FoldList , MI ) ) return false ; unsigned CommuteOpNo = OpNo ; unsigned CommuteIdx0 = TargetInstrInfo :: CommuteAnyOperandIndex ; unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( * MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) CommuteOpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) CommuteOpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , CommuteOpNo , OpToFold ) ) { if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) && ( OpToFold -> isImm ( ) || OpToFold -> isFI ( ) || OpToFold -> isGlobal ( ) ) ) { MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0 ; MachineOperand & OtherOp = MI -> getOperand ( OtherIdx ) ; if ( ! OtherOp . isReg ( ) || ! TII -> getRegisterInfo ( ) . isVGPR ( MRI , OtherOp . getReg ( ) ) ) return false ; assert ( MI -> getOperand ( <NUM_LIT> ) . isDef ( ) ) ; unsigned MaybeCommutedOpc = MI -> getOpcode ( ) ; int Op32 = <STR_LIT> ( MaybeCommutedOpc ) ; appendFoldCandidate ( FoldList , MI , CommuteOpNo , OpToFold , true , Op32 ) ; return true ; } TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ; return false ; } appendFoldCandidate ( FoldList , MI , CommuteOpNo , OpToFold , true ) ; return true ; } if ( TII -> isSALU ( MI -> getOpcode ( ) ) ) { const MCInstrDesc & InstDesc = MI -> getDesc ( ) ; const MCOperandInfo & OpInfo = InstDesc . OpInfo [ OpNo ] ; const <ISA_LIT>RegisterInfo & SRI = TII -> getRegisterInfo ( ) ; if ( TII -> isLiteralConstantLike ( * OpToFold , OpInfo ) ) { if ( ! SRI . opCanUseInlineConstant ( OpInfo . OperandType ) || ! TII -> isInlineConstant ( * OpToFold , OpInfo ) ) { for ( unsigned i = <NUM_LIT> , e = InstDesc . getNumOperands ( ) ; i != e ; ++ i ) { auto & Op = MI -> getOperand ( i ) ; if ( OpNo != i && TII -> isLiteralConstantLike ( Op , OpInfo ) ) { return false ; } } } } } appendFoldCandidate ( FoldList , MI , OpNo , OpToFold ) ;


<s> int32_t Src1Val = static_cast < int32_t > ( Src1 -> getImm ( ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) { if ( Src1Val == <NUM_LIT> ) { MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( <STR_LIT> ) ) ; } else if ( Src1Val == - <NUM_LIT> ) { MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( Opc == <STR_LIT> ) ) ) ; } else return false ; return true ; } if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { if ( Src1Val == <NUM_LIT> ) { MI -> RemoveOperand ( Src0Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( Opc == <STR_LIT> ) ) ) ; } else if ( Src1Val == - <NUM_LIT> ) { MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( <STR_LIT> ) ) ; stripExtraCopyOperands ( * MI ) ; } else return false ; return true ; }


<s> DefClamp -> setImm ( <NUM_LIT> ) ; MRI -> replaceRegWith ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MI . eraseFromParent ( ) ;

if ( TII -> convertToThreeAddress ( * Def , nullptr , nullptr ) ) Def -> eraseFromParent ( ) ;


<s> MRI -> replaceRegWith ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MI . eraseFromParent ( ) ; if ( TII -> convertToThreeAddress ( * Def , nullptr , nullptr ) ) Def -> eraseFromParent ( ) ;


<s> auto * Src1Imm = getImmOrMaterializedImm ( * MRI , * Src1 ) ; if ( ! Src1Imm -> isIdenticalTo ( * Src0Imm ) ) return false ; } int Src1ModIdx = <STR_LIT> ( Opc , <STR_LIT> :: src1_modifiers ) ; int Src0ModIdx = <STR_LIT> ( Opc , <STR_LIT> :: src0_modifiers ) ; if ( ( Src1ModIdx != - <NUM_LIT> && MI . getOperand ( Src1ModIdx ) . getImm ( ) != <NUM_LIT> ) || ( Src0ModIdx != - <NUM_LIT> && MI . getOperand ( Src0ModIdx ) . getImm ( ) != <NUM_LIT> ) ) return false ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> ) ;


<s> } int Src1ModIdx = <STR_LIT> ( Opc , <STR_LIT> :: src1_modifiers ) ; int Src0ModIdx = <STR_LIT> ( Opc , <STR_LIT> :: src0_modifiers ) ; if ( ( Src1ModIdx != - <NUM_LIT> && MI . getOperand ( Src1ModIdx ) . getImm ( ) != <NUM_LIT> ) || ( Src0ModIdx != - <NUM_LIT> && MI . getOperand ( Src0ModIdx ) . getImm ( ) != <NUM_LIT> ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> ) ; auto & NewDesc = TII -> get ( Src0 -> isReg ( ) ? ( unsigned ) <STR_LIT> : getMovOpc ( false ) ) ; int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( Src2Idx != - <NUM_LIT> ) MI . RemoveOperand ( Src2Idx ) ; MI . RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ) ; if ( Src1ModIdx != - <NUM_LIT> ) MI . RemoveOperand ( Src1ModIdx ) ; if ( Src0ModIdx != - <NUM_LIT> ) MI . RemoveOperand ( Src0ModIdx ) ; mutateCopyOp ( MI , NewDesc ) ; LLVM_DEBUG ( dbgs ( ) << MI ) ; return true ;


<s> PHI . getOperand ( <NUM_LIT> ) . setReg ( CopyIn ) ; PHI . getOperand ( <NUM_LIT> ) . setReg ( NewReg ) ; MachineBasicBlock * MBB = PHI . getParent ( ) ; BuildMI ( * MBB , MBB -> getFirstNonPHI ( ) , Copy -> getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , PhiOut ) . addReg ( NewReg , RegState :: Kill ) ; Copy -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << PHI ) ; return true ;


<s> MachineOperand & Def = MI . getOperand ( <NUM_LIT> ) ; if ( ! Def . isDef ( ) ) return false ; Register DefReg = Def . getReg ( ) ; if ( DefReg . isPhysical ( ) || ! TRI -> isVGPR ( * MRI , DefReg ) ) return false ; SmallVector < const MachineInstr * , <NUM_LIT> > Users ; SmallVector < Register , <NUM_LIT> > MoveRegs ; for ( const MachineInstr & I : MRI -> use_nodbg_instructions ( DefReg ) ) { Users . push_back ( & I ) ; } if ( Users . empty ( ) ) return false ; while ( ! Users . empty ( ) ) { const MachineInstr * I = Users . pop_back_val ( ) ; if ( ! I -> isCopy ( ) && ! I -> isRegSequence ( ) ) return false ; Register DstReg = I -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TRI -> isAGPR ( * MRI , DstReg ) ) continue ; MoveRegs . push_back ( DstReg ) ; for ( const MachineInstr & U : MRI -> use_nodbg_instructions ( DstReg ) ) { Users . push_back ( & U ) ; } } const TargetRegisterClass * RC = MRI -> getRegClass ( DefReg ) ; MRI -> setRegClass ( DefReg , TRI -> getEquivalentAGPRClass ( RC ) ) ; if ( ! TII -> isOperandLegal ( MI , <NUM_LIT> , & Def ) ) { MRI -> setRegClass ( DefReg , RC ) ; return false ; } while ( ! MoveRegs . empty ( ) ) {

Register Reg = MoveRegs . pop_back_val ( ) ;


<s> bool <ISA_LIT>FoldOperands :: tryFoldLoad ( MachineInstr & MI ) { assert ( MI . mayLoad ( ) ) ; if ( ! ST -> hasGFX90AInsts ( ) || MI . getNumExplicitDefs ( ) != <NUM_LIT> ) return false ; MachineOperand & Def = MI . getOperand ( <NUM_LIT> ) ; if ( ! Def . isDef ( ) ) return false ; Register DefReg = Def . getReg ( ) ; if ( DefReg . isPhysical ( ) || ! TRI -> isVGPR ( * MRI , DefReg ) ) return false ; SmallVector < const MachineInstr * , <NUM_LIT> > Users ; SmallVector < Register , <NUM_LIT> > MoveRegs ;

for ( const MachineInstr & I : MRI -> use_nodbg_instructions ( DefReg ) ) {


<s> std :: tie ( RegOp , OMod ) = isOMod ( MI ) ; if ( OMod == <STR_LIT> || ! RegOp -> isReg ( ) || RegOp -> getSubReg ( ) != <STR_LIT> || ! MRI -> hasOneNonDBGUser ( RegOp -> getReg ( ) ) ) return false ; MachineInstr * Def = MRI -> getVRegDef ( RegOp -> getReg ( ) ) ; MachineOperand * DefOMod = TII -> getNamedOperand ( * Def , <STR_LIT> :: omod ) ; if ( ! DefOMod || DefOMod -> getImm ( ) != <STR_LIT> ) return false ; if ( TII -> hasModifiersSet ( * Def , <STR_LIT> :: clamp ) ) return false ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * Def ) ;


<s> MachineOperand * DefOMod = TII -> getNamedOperand ( * Def , <STR_LIT> :: omod ) ; if ( ! DefOMod || DefOMod -> getImm ( ) != <STR_LIT> ) return false ; if ( TII -> hasModifiersSet ( * Def , <STR_LIT> :: clamp ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * Def ) ; DefOMod -> setImm ( OMod ) ; MRI -> replaceRegWith ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> case <STR_LIT> : LLVM_FALLTHROUGH ; case <STR_LIT> : LLVM_FALLTHROUGH ; case <STR_LIT> : LLVM_FALLTHROUGH ; case <STR_LIT> : LLVM_FALLTHROUGH ; case <STR_LIT> : break ; default : return false ; } const auto * NewDstRC = TRI -> getEquivalentAGPRClass ( MRI -> getRegClass ( Reg ) ) ; auto Dst = MRI -> createVirtualRegister ( NewDstRC ) ; auto RS = BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Dst ) ; for ( unsigned I = <NUM_LIT> ; I < Defs . size ( ) ; ++ I ) { MachineOperand * Def = Defs [ I ] . first ; Def -> setIsKill ( false ) ; if ( TRI -> isAGPR ( * MRI , Def -> getReg ( ) ) ) { RS . add ( * Def ) ; } else { MachineInstr * SubDef = MRI -> getVRegDef ( Def -> getReg ( ) ) ; SubDef -> getOperand ( <NUM_LIT> ) . setIsKill ( false ) ; RS . addReg ( SubDef -> getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , Def -> getSubReg ( ) ) ; } RS . addImm ( Defs [ I ] . second ) ;


<s> Register Src1 = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * SrcDef = MRI -> getVRegDef ( Src1 ) ; if ( ST -> zeroesHigh16BitsOfDest ( SrcDef -> getOpcode ( ) ) ) { Register Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MRI -> replaceRegWith ( Dst , SrcDef -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MI . eraseFromParent ( ) ; return true ; } return false ;


<s> if ( ! OpInfo || UseOpIdx >= Desc . getNumOperands ( ) ) return false ; uint8_t OpTy = OpInfo [ UseOpIdx ] . OperandType ; if ( ( OpTy < <STR_LIT> || OpTy > <STR_LIT> ) && ( OpTy < <STR_LIT> || OpTy > <STR_LIT> ) ) return false ; if ( OpToFold . isImm ( ) && TII -> isInlineConstant ( OpToFold , OpTy ) && TII -> isOperandLegal ( * UseMI , UseOpIdx , & OpToFold ) ) { UseMI -> getOperand ( UseOpIdx ) . ChangeToImmediate ( OpToFold . getImm ( ) ) ; return true ; } if ( ! OpToFold . isReg ( ) ) return false ; Register UseReg = OpToFold . getReg ( ) ; if ( ! UseReg . isVirtual ( ) ) return false ; if ( isUseMIInFoldList ( FoldList , UseMI ) ) return false ; MachineRegisterInfo & MRI = UseMI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; MachineInstr * Def = MRI . getVRegDef ( UseReg ) ; MachineOperand & UseOp = UseMI -> getOperand ( UseOpIdx ) ; if ( ! UseOp . getSubReg ( ) && Def && TII -> isFoldableCopy ( * Def ) ) { MachineOperand & DefOp = Def -> getOperand ( <NUM_LIT> ) ; if ( DefOp . isImm ( ) && TII -> isInlineConstant ( DefOp , OpTy ) && TII -> isOperandLegal ( * UseMI , UseOpIdx , & DefOp ) ) { UseMI -> getOperand ( UseOpIdx ) . ChangeToImmediate ( DefOp . getImm ( ) ) ; return true ; } } SmallVector < std :: pair < MachineOperand * , unsigned > , <NUM_LIT> > Defs ; if ( ! getRegSeqInit ( Defs , UseReg , OpTy , TII , MRI ) ) return false ; int32_t Imm ; for ( unsigned I = <NUM_LIT> , E = Defs . size ( ) ; I != E ; ++ I ) { const MachineOperand * Op = Defs [ I ] . first ; if ( ! Op -> isImm ( ) ) return false ;


<s> SmallVector < MachineRegisterInfo :: use_iterator , <NUM_LIT> > UsesToProcess ; for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; ++ Use ) { UsesToProcess . push_back ( Use ) ; } for ( auto U : UsesToProcess ) { MachineInstr * UseMI = U -> getParent ( ) ; foldOperand ( OpToFold , UseMI , U . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } } MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; for ( MachineInstr * Copy : CopiesToReplace ) Copy -> addImplicitDefUseOperands ( * MF ) ; for ( FoldCandidate & Fold : FoldList ) { if ( Fold . isReg ( ) && Register :: isVirtualRegister ( Fold . OpToFold -> getReg ( ) ) ) { Register Reg = Fold . OpToFold -> getReg ( ) ; MachineInstr * DefMI = Fold . OpToFold -> getParent ( ) ; if ( DefMI -> readsRegister ( <STR_LIT> , TRI ) && execMayBeModifiedBeforeUse ( * MRI , Reg , * DefMI , * Fold . UseMI ) ) continue ; } if ( updateOperand ( Fold , * TII , * TRI , * ST ) ) { if ( Fold . isReg ( ) ) { assert ( Fold . OpToFold && Fold . OpToFold -> isReg ( ) ) ; MRI -> clearKillFlags ( Fold . OpToFold -> getReg ( ) ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << static_cast < int > ( Fold . UseOpNo ) << <STR_LIT> << * Fold . UseMI << '\n' ) ; tryFoldInst ( TII , Fold . UseMI ) ; } else if ( Fold . isCommuted ( ) ) { TII -> commuteInstruction ( * Fold . UseMI , false ) ;


<s> static bool frameIndexMayFold ( const <ISA_LIT>InstrInfo * TII , const MachineInstr & UseMI , int OpNo , const MachineOperand & OpToFold ) { return OpToFold . isFI ( ) && ( TII -> isMUBUF ( UseMI ) || TII -> isFLATScratch ( UseMI ) ) && OpNo == <STR_LIT> ( UseMI . getOpcode ( ) , <STR_LIT> :: vaddr ) ;


<s> return OpToFold . isFI ( ) && ( TII -> isMUBUF ( UseMI ) || TII -> isFLATScratch ( UseMI ) ) && OpNo == <STR_LIT> ( UseMI . getOpcode ( ) , <STR_LIT> :: vaddr ) ;


<s> if ( Op . isReg ( ) ) { if ( Op . getSubReg ( ) != <STR_LIT> || ! Register :: isVirtualRegister ( Op . getReg ( ) ) ) return & Op ; MachineInstr * Def = MRI . getVRegDef ( Op . getReg ( ) ) ; if ( Def && Def -> isMoveImmediate ( ) ) { MachineOperand & ImmSrc = Def -> getOperand ( <NUM_LIT> ) ;

if ( ImmSrc . isImm ( ) ) return & ImmSrc ;


<s> case <STR_LIT> : { if ( ! TII -> getNamedOperand ( MI , <STR_LIT> :: clamp ) -> getImm ( ) ) return nullptr ; const MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( ! Src0 -> isReg ( ) || ! Src1 -> isReg ( ) || Src0 -> getReg ( ) != Src1 -> getReg ( ) || Src0 -> getSubReg ( ) != Src1 -> getSubReg ( ) || Src0 -> getSubReg ( ) != <STR_LIT> ) return nullptr ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return nullptr ; unsigned Src0Mods = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) -> getImm ( ) ; unsigned Src1Mods = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) -> getImm ( ) ; unsigned UnsetMods = ( Op == <STR_LIT> ) ? <STR_LIT> : <NUM_LIT> ;

if ( Src0Mods != UnsetMods && Src1Mods != UnsetMods ) return nullptr ;


<s> unsigned Opc = UseMI . getOpcode ( ) ; switch ( Opc ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( static_cast < int > ( OpNo ) == Src2Idx ) { bool IsFMA = Opc == <STR_LIT> || Opc == <STR_LIT> ;

bool IsF32 = Opc == <STR_LIT> || Opc == <STR_LIT> ;


<s> bool IsIEEEMode = MFI -> getMode ( ) . IEEE ; bool HasNSZ = MFI -> hasNoSignedZerosFPMath ( ) ; for ( MachineBasicBlock * MBB : depth_first ( & MF ) ) { MachineBasicBlock :: iterator I , Next ; MachineOperand * CurrentKnownM0Val = nullptr ; for ( I = MBB -> begin ( ) ; I != MBB -> end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; tryFoldInst ( TII , & MI ) ; if ( ! TII -> isFoldableCopy ( MI ) ) { if ( IsIEEEMode || ( ! HasNSZ && ! MI . getFlag ( MachineInstr :: FmNsz ) ) || ! tryFoldOMod ( MI ) ) tryFoldClamp ( MI ) ; if ( CurrentKnownM0Val && MI . modifiesRegister ( <STR_LIT> , TRI ) ) CurrentKnownM0Val = nullptr ; continue ; } if ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) { MachineOperand & NewM0Val = MI . getOperand ( <NUM_LIT> ) ; if ( CurrentKnownM0Val && CurrentKnownM0Val -> isIdenticalTo ( NewM0Val ) ) { MI . eraseFromParent ( ) ; continue ; } CurrentKnownM0Val = ( NewM0Val . isReg ( ) && NewM0Val . getReg ( ) . isPhysical ( ) ) ? nullptr : & NewM0Val ; continue ; } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) || OpToFold . isGlobal ( ) ;

if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ;


<s> if ( Opc == <STR_LIT> && OpToFold -> isImm ( ) ) { MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; FoldList . push_back ( FoldCandidate ( MI , OpNo , OpToFold ) ) ; return true ; } if ( isUseMIInFoldList ( FoldList , MI ) ) return false ; unsigned CommuteOpNo = OpNo ; unsigned CommuteIdx0 = TargetInstrInfo :: CommuteAnyOperandIndex ; unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( * MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) CommuteOpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) CommuteOpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , CommuteOpNo , OpToFold ) ) { if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) && ( OpToFold -> isImm ( ) || OpToFold -> isFI ( ) || OpToFold -> isGlobal ( ) ) ) { MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0 ; MachineOperand & OtherOp = MI -> getOperand ( OtherIdx ) ; if ( ! OtherOp . isReg ( ) || ! TII -> getRegisterInfo ( ) . isVGPR ( MRI , OtherOp . getReg ( ) ) ) return false ; assert ( MI -> getOperand ( <NUM_LIT> ) . isDef ( ) ) ;

unsigned MaybeCommutedOpc = MI -> getOpcode ( ) ;


<s> int Src1ModIdx = <STR_LIT> ( Opc , <STR_LIT> :: src1_modifiers ) ; int Src0ModIdx = <STR_LIT> ( Opc , <STR_LIT> :: src0_modifiers ) ; if ( Src1 -> isIdenticalTo ( * Src0 ) && ( Src1ModIdx == - <NUM_LIT> || ! MI -> getOperand ( Src1ModIdx ) . getImm ( ) ) && ( Src0ModIdx == - <NUM_LIT> || ! MI -> getOperand ( Src0ModIdx ) . getImm ( ) ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI << <STR_LIT> ) ; auto & NewDesc = TII -> get ( Src0 -> isReg ( ) ? ( unsigned ) <STR_LIT> : getMovOpc ( false ) ) ; int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( Src2Idx != - <NUM_LIT> ) MI -> RemoveOperand ( Src2Idx ) ;

MI -> RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ) ;


<s> for ( FoldCandidate & Fold : FoldList ) if ( Fold . UseMI == MI && Fold . UseOpNo == OpNo ) return ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << ( Commuted ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << OpNo << <STR_LIT> << * MI << '\n' ) ;

FoldList . push_back ( FoldCandidate ( MI , OpNo , FoldOp , Commuted , ShrinkOp ) ) ;


<s> int NonInlineUseOpNo = - <NUM_LIT> ; MachineRegisterInfo :: use_iterator NextUse ; for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; Use = NextUse ) { NextUse = std :: next ( Use ) ; MachineInstr * UseMI = Use -> getParent ( ) ; unsigned OpNo = Use . getOperandNo ( ) ; if ( OpToFold . isImm ( ) && tryConstantFoldOp ( * MRI , TII , UseMI , & OpToFold ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * UseMI << '\n' ) ; NextUse = MRI -> use_begin ( Dst . getReg ( ) ) ; FoldList . clear ( ) ; continue ; } if ( isInlineConstantIfFolded ( TII , * UseMI , OpNo , OpToFold ) ) { foldOperand ( OpToFold , UseMI , OpNo , FoldList , CopiesToReplace ) ; } else if ( frameIndexMayFold ( TII , * UseMI , OpNo , OpToFold ) ) { foldOperand ( OpToFold , UseMI , OpNo , FoldList , CopiesToReplace ) ; } else { if ( ++ NumLiteralUses == <NUM_LIT> ) { NonInlineUse = & * Use ; NonInlineUseOpNo = OpNo ; } } }


<s> if ( ! Def || ! Def -> isRegSequence ( ) ) return false ; for ( unsigned I = <NUM_LIT> , E = Def -> getNumExplicitOperands ( ) ; I < E ; I += <NUM_LIT> ) { MachineOperand * Sub = & Def -> getOperand ( I ) ; assert ( Sub -> isReg ( ) ) ;

for ( MachineInstr * SubDef = MRI . getUniqueVRegDef ( Sub -> getReg ( ) ) ;


<s> case <STR_LIT> : { if ( ( Op == <STR_LIT> && MFI -> getMode ( ) . FP32OutputDenormals ) || ( Op == <STR_LIT> && MFI -> getMode ( ) . FP64FP16OutputDenormals ) ) return std :: make_pair ( nullptr , <STR_LIT> ) ; const MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ;

if ( Src0 -> isReg ( ) && Src1 -> isReg ( ) && Src0 -> getReg ( ) == Src1 -> getReg ( ) && Src0 -> getSubReg ( ) == Src1 -> getSubReg ( ) && ! TII -> hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) && ! TII -> hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) && ! TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) && ! TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return std :: make_pair ( Src0 , <STR_LIT> ) ;


<s> if ( skipFunction ( MF . getFunction ( ) ) ) return false ; MRI = & MF . getRegInfo ( ) ; ST = & MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; bool IsIEEEMode = MFI -> getMode ( ) . IEEE ; bool HasNSZ = MFI -> hasNoSignedZerosFPMath ( ) ; for ( MachineBasicBlock * MBB : depth_first ( & MF ) ) { MachineBasicBlock :: iterator I , Next ; MachineOperand * CurrentKnownM0Val = nullptr ; for ( I = MBB -> begin ( ) ; I != MBB -> end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; tryFoldInst ( TII , & MI ) ; if ( ! TII -> isFoldableCopy ( MI ) ) { if ( CurrentKnownM0Val && MI . modifiesRegister ( <STR_LIT> , TRI ) ) CurrentKnownM0Val = nullptr ; if ( IsIEEEMode || ( ! HasNSZ && ! MI . getFlag ( MachineInstr :: FmNsz ) ) || ! tryFoldOMod ( MI ) ) tryFoldClamp ( MI ) ; continue ; } if ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) { MachineOperand & NewM0Val = MI . getOperand ( <NUM_LIT> ) ; if ( CurrentKnownM0Val && CurrentKnownM0Val -> isIdenticalTo ( NewM0Val ) ) { MI . eraseFromParent ( ) ; continue ; }

CurrentKnownM0Val = ( NewM0Val . isReg ( ) && NewM0Val . getReg ( ) . isPhysical ( ) ) ? nullptr : & NewM0Val ;


<s> unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( * MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) CommuteOpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) CommuteOpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , CommuteOpNo , OpToFold ) ) { if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) && ( OpToFold -> isImm ( ) || OpToFold -> isFI ( ) || OpToFold -> isGlobal ( ) ) ) { MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0 ; MachineOperand & OtherOp = MI -> getOperand ( OtherIdx ) ; if ( ! OtherOp . isReg ( ) || ! TII -> getRegisterInfo ( ) . isVGPR ( MRI , OtherOp . getReg ( ) ) ) return false ; assert ( MI -> getOperand ( <NUM_LIT> ) . isDef ( ) ) ; unsigned MaybeCommutedOpc = MI -> getOpcode ( ) ; int Op32 = <STR_LIT> ( MaybeCommutedOpc ) ;

appendFoldCandidate ( FoldList , MI , CommuteOpNo , OpToFold , true , Op32 ) ;


<s> int Src1Idx = <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ; if ( Src1Idx == - <NUM_LIT> ) return false ; int Src0Idx = <STR_LIT> ( Opc , <STR_LIT> :: src0 ) ; MachineOperand * Src0 = getImmOrMaterializedImm ( MRI , MI -> getOperand ( Src0Idx ) ) ; MachineOperand * Src1 = getImmOrMaterializedImm ( MRI , MI -> getOperand ( Src1Idx ) ) ; if ( ! Src0 -> isImm ( ) && ! Src1 -> isImm ( ) ) return false ; if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { if ( Src0 -> isImm ( ) && Src0 -> getImm ( ) == <NUM_LIT> ) { bool UseCopy = TII -> getNamedOperand ( * MI , <STR_LIT> :: src2 ) -> isReg ( ) ; MI -> RemoveOperand ( Src1Idx ) ; MI -> RemoveOperand ( Src0Idx ) ; MI -> setDesc ( TII -> get ( UseCopy ? <STR_LIT> : <STR_LIT> ) ) ; return true ; } } if ( Src0 -> isImm ( ) && Src1 -> isImm ( ) ) { int32_t NewImm ; if ( ! evalBinaryInstruction ( Opc , NewImm , Src0 -> getImm ( ) , Src1 -> getImm ( ) ) ) return false ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; bool IsSGPR = TRI . isSGPRReg ( MRI , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MI -> getOperand ( Src0Idx ) . ChangeToImmediate ( NewImm ) ; MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( IsSGPR ) ) ) ; return true ; } if ( ! MI -> isCommutable ( ) ) return false ; if ( Src0 -> isImm ( ) && ! Src1 -> isImm ( ) ) {


<s> MachineRegisterInfo & MRI = UseMI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; SmallVector < std :: pair < MachineOperand * , unsigned > , <NUM_LIT> > Defs ; if ( ! getRegSeqInit ( Defs , UseReg , OpTy , TII , MRI ) ) return false ; int32_t Imm ; for ( unsigned I = <NUM_LIT> , E = Defs . size ( ) ; I != E ; ++ I ) { const MachineOperand * Op = Defs [ I ] . first ; if ( ! Op -> isImm ( ) ) return false ; auto SubImm = Op -> getImm ( ) ; if ( ! I ) { Imm = SubImm ; if ( ! TII -> isInlineConstant ( * Op , OpTy ) || ! TII -> isOperandLegal ( * UseMI , UseOpIdx , Op ) ) return false ; continue ; } if ( Imm != SubImm ) return false ; } appendFoldCandidate ( FoldList , UseMI , UseOpIdx , Defs [ <NUM_LIT> ] . first ) ; return true ;


<s> if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( OpToFold . isReg ( ) && ! OpToFold . getReg ( ) . isVirtual ( ) ) continue ; if ( ! MI . getOperand ( <NUM_LIT> ) . getReg ( ) . isVirtual ( ) ) continue ; foldInstOperand ( MI , OpToFold ) ; auto * InstToErase = & MI ; while ( MRI -> use_nodbg_empty ( InstToErase -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { auto & SrcOp = InstToErase -> getOperand ( <NUM_LIT> ) ; auto SrcReg = SrcOp . isReg ( ) ? SrcOp . getReg ( ) : Register ( ) ; InstToErase -> eraseFromParent ( ) ; InstToErase = nullptr ; if ( ! SrcReg || SrcReg . isPhysical ( ) ) break ; InstToErase = MRI -> getVRegDef ( SrcReg ) ; if ( ! InstToErase || ! TII -> isFoldableCopy ( * InstToErase ) ) break ; } if ( InstToErase && InstToErase -> isRegSequence ( ) && MRI -> use_nodbg_empty ( InstToErase -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) InstToErase -> eraseFromParent ( ) ;


<s> MachineInstr * UseMI = Op -> getParent ( ) ; while ( UseMI -> isCopy ( ) && ! Op -> getSubReg ( ) ) { Reg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isVGPR ( * MRI , Reg ) || ! MRI -> hasOneNonDBGUse ( Reg ) ) return false ; Op = & * MRI -> use_nodbg_begin ( Reg ) ; UseMI = Op -> getParent ( ) ; } if ( Op -> getSubReg ( ) ) return false ; unsigned OpIdx = Op - & UseMI -> getOperand ( <NUM_LIT> ) ; const MCInstrDesc & InstDesc = UseMI -> getDesc ( ) ; if ( ! TRI -> isVectorSuperClass ( TRI -> getRegClass ( InstDesc . OpInfo [ OpIdx ] . RegClass ) ) ) return false ; const auto * NewDstRC = TRI -> getEquivalentAGPRClass ( MRI -> getRegClass ( Reg ) ) ; auto Dst = MRI -> createVirtualRegister ( NewDstRC ) ; auto RS = BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Dst ) ; for ( unsigned I = <NUM_LIT> ; I < Defs . size ( ) ; ++ I ) { MachineOperand * Def = Defs [ I ] . first ;


<s> MachineOperand * Op = & * MRI -> use_nodbg_begin ( Reg ) ; MachineInstr * UseMI = Op -> getParent ( ) ; while ( UseMI -> isCopy ( ) && ! Op -> getSubReg ( ) ) { Reg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isVGPR ( * MRI , Reg ) || ! MRI -> hasOneNonDBGUse ( Reg ) ) return false ; Op = & * MRI -> use_nodbg_begin ( Reg ) ; UseMI = Op -> getParent ( ) ; } if ( Op -> getSubReg ( ) ) return false ; unsigned OpIdx = Op - & UseMI -> getOperand ( <NUM_LIT> ) ; const MCInstrDesc & InstDesc = UseMI -> getDesc ( ) ; if ( ! TRI -> isVectorSuperClass ( TRI -> getRegClass ( InstDesc . OpInfo [ OpIdx ] . RegClass ) ) ) return false ; const auto * NewDstRC = TRI -> getEquivalentAGPRClass ( MRI -> getRegClass ( Reg ) ) ; auto Dst = MRI -> createVirtualRegister ( NewDstRC ) ; auto RS = BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Dst ) ; for ( unsigned I = <NUM_LIT> ; I < Defs . size ( ) ; ++ I ) { MachineOperand * Def = Defs [ I ] . first ; Def -> setIsKill ( false ) ; if ( TRI -> isAGPR ( * MRI , Def -> getReg ( ) ) ) { RS . add ( * Def ) ; } else {

MachineInstr * SubDef = MRI -> getVRegDef ( Def -> getReg ( ) ) ;


<s> unsigned OpIdx = Op - & UseMI -> getOperand ( <NUM_LIT> ) ; const MCInstrDesc & InstDesc = UseMI -> getDesc ( ) ; if ( ! TRI -> isVectorSuperClass ( TRI -> getRegClass ( InstDesc . OpInfo [ OpIdx ] . RegClass ) ) ) return false ; const auto * NewDstRC = TRI -> getEquivalentAGPRClass ( MRI -> getRegClass ( Reg ) ) ; auto Dst = MRI -> createVirtualRegister ( NewDstRC ) ; auto RS = BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Dst ) ; for ( unsigned I = <NUM_LIT> ; I < Defs . size ( ) ; ++ I ) { MachineOperand * Def = Defs [ I ] . first ; Def -> setIsKill ( false ) ; if ( TRI -> isAGPR ( * MRI , Def -> getReg ( ) ) ) { RS . add ( * Def ) ; } else { MachineInstr * SubDef = MRI -> getVRegDef ( Def -> getReg ( ) ) ; SubDef -> getOperand ( <NUM_LIT> ) . setIsKill ( false ) ; RS . addReg ( SubDef -> getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , Def -> getSubReg ( ) ) ; } RS . addImm ( Defs [ I ] . second ) ; } Op -> setReg ( Dst ) ; if ( ! TII -> isOperandLegal ( * UseMI , OpIdx , Op ) ) { Op -> setReg ( Reg ) ; RS -> eraseFromParent ( ) ; return false ; }

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * RS << <STR_LIT> << * UseMI ) ;


<s> if ( OpToFold . isImm ( ) && tryConstantFoldOp ( * MRI , TII , UseMI , & OpToFold ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * UseMI << '\n' ) ; NextUse = MRI -> use_begin ( Dst . getReg ( ) ) ; FoldList . clear ( ) ; continue ; } if ( isInlineConstantIfFolded ( TII , * UseMI , OpNo , OpToFold ) ) { foldOperand ( OpToFold , UseMI , OpNo , FoldList , CopiesToReplace ) ; } else { if ( ++ NumLiteralUses == <NUM_LIT> ) { NonInlineUse = & * Use ; NonInlineUseOpNo = OpNo ; } } } if ( NumLiteralUses == <NUM_LIT> ) { MachineInstr * UseMI = NonInlineUse -> getParent ( ) ; foldOperand ( OpToFold , UseMI , NonInlineUseOpNo , FoldList , CopiesToReplace ) ; } } else { for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; ++ Use ) { MachineInstr * UseMI = Use -> getParent ( ) ; foldOperand ( OpToFold , UseMI , Use . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } } MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; for ( MachineInstr * Copy : CopiesToReplace ) Copy -> addImplicitDefUseOperands ( * MF ) ; for ( FoldCandidate & Fold : FoldList ) { if ( updateOperand ( Fold , * TRI ) ) {


<s> if ( ! TII -> getNamedOperand ( MI , <STR_LIT> :: clamp ) -> getImm ( ) ) return nullptr ; const MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( ! Src0 -> isReg ( ) || ! Src1 -> isReg ( ) || Src0 -> getSubReg ( ) != Src1 -> getSubReg ( ) || Src0 -> getSubReg ( ) != <STR_LIT> ) return nullptr ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return nullptr ; unsigned Src0Mods = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) -> getImm ( ) ; unsigned Src1Mods = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) -> getImm ( ) ; unsigned UnsetMods = ( Op == <STR_LIT> ) ? <STR_LIT> : <NUM_LIT> ; if ( Src0Mods != UnsetMods && Src1Mods != UnsetMods ) return nullptr ; return Src0 ;


<s> if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return nullptr ; unsigned Src0Mods = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) -> getImm ( ) ; unsigned Src1Mods = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) -> getImm ( ) ; unsigned UnsetMods = ( Op == <STR_LIT> ) ? <STR_LIT> : <NUM_LIT> ;


<s> MRI = & MF . getRegInfo ( ) ; ST = & MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = ST -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; bool IsIEEEMode = ST -> enableIEEEBit ( MF ) || ! MFI -> hasNoSignedZerosFPMath ( ) ; for ( MachineBasicBlock * MBB : depth_first ( & MF ) ) { MachineBasicBlock :: iterator I , Next ; for ( I = MBB -> begin ( ) ; I != MBB -> end ( ) ; I = Next ) { Next = std :: next ( I ) ;


<s> FoldCandidate ( MachineInstr * MI , unsigned OpNo , MachineOperand * FoldOp , bool Commuted_ = false , int ShrinkOp = - <NUM_LIT> ) : UseMI ( MI ) , OpToFold ( nullptr ) , ShrinkOpcode ( ShrinkOp ) , UseOpNo ( OpNo ) , Kind ( FoldOp -> getType ( ) ) , Commuted ( Commuted_ ) { if ( FoldOp -> isImm ( ) ) { ImmToFold = FoldOp -> getImm ( ) ; } else if ( FoldOp -> isFI ( ) ) { FrameIndexToFold = FoldOp -> getIndex ( ) ; } else { assert ( FoldOp -> isReg ( ) ) ; OpToFold = FoldOp ;


<s> void <ISA_LIT>FoldOperands :: foldInstOperand ( MachineInstr & MI , MachineOperand & OpToFold ) const { SmallVector < MachineInstr * , <NUM_LIT> > CopiesToReplace ; SmallVector < FoldCandidate , <NUM_LIT> > FoldList ; MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) ; if ( FoldingImm ) { unsigned NumLiteralUses = <NUM_LIT> ; MachineOperand * NonInlineUse = nullptr ; int NonInlineUseOpNo = - <NUM_LIT> ; MachineRegisterInfo :: use_iterator NextUse ; for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; Use = NextUse ) { NextUse = std :: next ( Use ) ; MachineInstr * UseMI = Use -> getParent ( ) ; unsigned OpNo = Use . getOperandNo ( ) ; if ( OpToFold . isImm ( ) && tryConstantFoldOp ( * MRI , TII , UseMI , & OpToFold ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * UseMI << '\n' ) ; NextUse = MRI -> use_begin ( Dst . getReg ( ) ) ; FoldList . clear ( ) ; continue ; } if ( isInlineConstantIfFolded ( TII , * UseMI , OpNo , OpToFold ) ) { foldOperand ( OpToFold , UseMI , OpNo , FoldList , CopiesToReplace ) ; } else { if ( ++ NumLiteralUses == <NUM_LIT> ) { NonInlineUse = & * Use ;


<s> const MCInstrDesc & UseDesc = UseMI -> getDesc ( ) ; if ( UseDesc . isVariadic ( ) || UseOp . isImplicit ( ) || UseDesc . OpInfo [ UseOpIdx ] . RegClass == - <NUM_LIT> ) return ; } if ( ! FoldingImm ) { tryAddToFoldList ( FoldList , UseMI , UseOpIdx , & OpToFold , TII ) ; return ; } const MCInstrDesc & FoldDesc = OpToFold . getParent ( ) -> getDesc ( ) ; const TargetRegisterClass * FoldRC = TRI -> getRegClass ( FoldDesc . OpInfo [ <NUM_LIT> ] . RegClass ) ; if ( UseOp . getSubReg ( ) && <STR_LIT> ( FoldRC -> getID ( ) ) == <NUM_LIT> ) { unsigned UseReg = UseOp . getReg ( ) ; const TargetRegisterClass * UseRC = TargetRegisterInfo :: isVirtualRegister ( UseReg ) ? MRI -> getRegClass ( UseReg ) : TRI -> getPhysRegClass ( UseReg ) ; if ( <STR_LIT> ( UseRC -> getID ( ) ) != <NUM_LIT> ) return ; APInt Imm ( <NUM_LIT> , OpToFold . getImm ( ) ) ; if ( UseOp . getSubReg ( ) == <STR_LIT> ) { Imm = Imm . getLoBits ( <NUM_LIT> ) ; } else { assert ( UseOp . getSubReg ( ) == <STR_LIT> ) ; Imm = Imm . getHiBits ( <NUM_LIT> ) ; } MachineOperand ImmOp = MachineOperand :: CreateImm ( Imm . getSExtValue ( ) ) ; tryAddToFoldList ( FoldList , UseMI , UseOpIdx , & ImmOp , TII ) ;


<s> const MachineOperand & UseOp = UseMI -> getOperand ( UseOpIdx ) ; if ( ! isUseSafeToFold ( TII , * UseMI , UseOp ) ) return ; if ( UseOp . isReg ( ) && OpToFold . isReg ( ) ) { if ( UseOp . isImplicit ( ) || UseOp . getSubReg ( ) != <STR_LIT> ) return ; if ( UseOp . isTied ( ) && OpToFold . getSubReg ( ) != <STR_LIT> ) return ; } if ( UseMI -> isRegSequence ( ) ) { unsigned RegSeqDstReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned RegSeqDstSubReg = UseMI -> getOperand ( UseOpIdx + <NUM_LIT> ) . getImm ( ) ; for ( MachineRegisterInfo :: use_iterator RSUse = MRI -> use_begin ( RegSeqDstReg ) , RSE = MRI -> use_end ( ) ; RSUse != RSE ; ++ RSUse ) { MachineInstr * RSUseMI = RSUse -> getParent ( ) ; if ( RSUse -> getSubReg ( ) != RegSeqDstSubReg ) continue ; foldOperand ( OpToFold , RSUseMI , RSUse . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } return ; } bool FoldingImm = OpToFold . isImm ( ) ; if ( FoldingImm && UseMI -> isCopy ( ) ) { unsigned DestReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * DestRC = TargetRegisterInfo :: isVirtualRegister ( DestReg ) ? MRI -> getRegClass ( DestReg ) : TRI -> getPhysRegClass ( DestReg ) ; unsigned SrcReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( DestReg ) && TargetRegisterInfo :: isVirtualRegister ( SrcReg ) ) { const TargetRegisterClass * SrcRC = MRI -> getRegClass ( SrcReg ) ; if ( TRI -> isSGPRClass ( SrcRC ) && TRI -> hasVGPRs ( DestRC ) ) { MachineRegisterInfo :: use_iterator NextUse ; SmallVector < FoldCandidate , <NUM_LIT> > CopyUses ; for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( DestReg ) , E = MRI -> use_end ( ) ; Use != E ; Use = NextUse ) { NextUse = std :: next ( Use ) ; FoldCandidate FC = FoldCandidate ( Use -> getParent ( ) , Use . getOperandNo ( ) , & UseMI -> getOperand ( <NUM_LIT> ) ) ; CopyUses . push_back ( FC ) ; } for ( auto & F : CopyUses ) { foldOperand ( * F . OpToFold , F . UseMI , F . UseOpNo , FoldList , CopiesToReplace ) ; }


<s> MRI = & MF . getRegInfo ( ) ; ST = & MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; bool IsIEEEMode = ST -> enableIEEEBit ( MF ) ; bool HasNSZ = MFI -> hasNoSignedZerosFPMath ( ) ; for ( MachineBasicBlock * MBB : depth_first ( & MF ) ) { MachineBasicBlock :: iterator I , Next ; for ( I = MBB -> begin ( ) ; I != MBB -> end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; tryFoldInst ( TII , & MI ) ; if ( ! TII -> isFoldableCopy ( MI ) ) { if ( IsIEEEMode || ( ! HasNSZ && ! MI . getFlag ( MachineInstr :: FmNsz ) ) || ! tryFoldOMod ( MI ) ) tryFoldClamp ( MI ) ; continue ; } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( OpToFold . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( OpToFold . getReg ( ) ) ) continue ;

MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ;


<s> bool FoldAsMAD = tryAddToFoldList ( FoldList , MI , OpNo , OpToFold , TII ) ; if ( FoldAsMAD ) { MI -> untieRegOperand ( OpNo ) ; return true ; } MI -> setDesc ( TII -> get ( Opc ) ) ; } if ( Opc == <STR_LIT> && OpToFold -> isImm ( ) ) { MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; FoldList . push_back ( FoldCandidate ( MI , OpNo , OpToFold ) ) ; return true ; } if ( isUseMIInFoldList ( FoldList , MI ) ) return false ; unsigned CommuteOpNo = OpNo ; unsigned CommuteIdx0 = TargetInstrInfo :: CommuteAnyOperandIndex ; unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( * MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) CommuteOpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) CommuteOpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ;

if ( ! TII -> isOperandLegal ( * MI , CommuteOpNo , OpToFold ) ) {


<s> if ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) { unsigned Opcode = MI -> getOpcode ( ) ; int OpNo = MI -> getOperandNo ( & Old ) ; int ModIdx = - <NUM_LIT> ; if ( OpNo == <STR_LIT> ( Opcode , <STR_LIT> :: src0 ) ) ModIdx = <STR_LIT> :: src0_modifiers ; else if ( OpNo == <STR_LIT> ( Opcode , <STR_LIT> :: src1 ) ) ModIdx = <STR_LIT> :: src1_modifiers ; else if ( OpNo == <STR_LIT> ( Opcode , <STR_LIT> :: src2 ) ) ModIdx = <STR_LIT> :: src2_modifiers ; assert ( ModIdx != - <NUM_LIT> ) ; ModIdx = <STR_LIT> ( Opcode , ModIdx ) ; MachineOperand & Mod = MI -> getOperand ( ModIdx ) ; unsigned Val = Mod . getImm ( ) ; if ( ( Val & <STR_LIT> ) || ! ( Val & <STR_LIT> ) ) return false ; if ( ! isUInt < <NUM_LIT> > ( Fold . ImmToFold ) ) { if ( ! ( Fold . ImmToFold & <NUM_LIT> ) ) { Mod . setImm ( Mod . getImm ( ) | <STR_LIT> ) ; Mod . setImm ( Mod . getImm ( ) & ~ <STR_LIT> ) ; Old . ChangeToImmediate ( ( Fold . ImmToFold >> <NUM_LIT> ) & <NUM_LIT> ) ; return true ; } Mod . setImm ( Mod . getImm ( ) & ~ <STR_LIT> ) ;


<s> assert ( Dst0 . isDef ( ) && Dst1 . isDef ( ) ) ; bool HaveNonDbgCarryUse = ! MRI . use_nodbg_empty ( Dst1 . getReg ( ) ) ; const TargetRegisterClass * Dst0RC = MRI . getRegClass ( Dst0 . getReg ( ) ) ; unsigned NewReg0 = MRI . createVirtualRegister ( Dst0RC ) ; const TargetRegisterClass * Dst1RC = MRI . getRegClass ( Dst1 . getReg ( ) ) ; unsigned NewReg1 = MRI . createVirtualRegister ( Dst1RC ) ; MachineInstr * Inst32 = TII . buildShrunkInst ( * MI , Op32 ) ; if ( HaveNonDbgCarryUse ) { BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII . get ( <STR_LIT> ) , Dst1 . getReg ( ) ) . addReg ( <STR_LIT> , RegState :: Kill ) ; } Dst0 . setReg ( NewReg0 ) ; Dst1 . setReg ( NewReg1 ) ; if ( Fold . isCommuted ( ) ) TII . commuteInstruction ( * Inst32 , false ) ; return true ; } Old . ChangeToImmediate ( Fold . ImmToFold ) ; return true ; } assert ( ! Fold . needsShrink ( ) && <STR_LIT> ) ; if ( Fold . isFI ( ) ) { Old . ChangeToFrameIndex ( Fold . FrameIndexToFold ) ; return true ;


<s> if ( UseOp . isReg ( ) && ( ( UseOp . getSubReg ( ) && OpToFold . isReg ( ) ) || UseOp . isImplicit ( ) ) ) { return ; } bool FoldingImm = OpToFold . isImm ( ) ; APInt Imm ; if ( FoldingImm ) { unsigned UseReg = UseOp . getReg ( ) ; const TargetRegisterClass * UseRC = TargetRegisterInfo :: isVirtualRegister ( UseReg ) ? MRI . getRegClass ( UseReg ) : TRI . getPhysRegClass ( UseReg ) ; Imm = APInt ( <NUM_LIT> , OpToFold . getImm ( ) ) ; const MCInstrDesc & FoldDesc = TII -> get ( OpToFold . getParent ( ) -> getOpcode ( ) ) ; const TargetRegisterClass * FoldRC = TRI . getRegClass ( FoldDesc . OpInfo [ <NUM_LIT> ] . RegClass ) ; if ( FoldRC -> getSize ( ) == <NUM_LIT> && UseOp . getSubReg ( ) ) { if ( UseRC -> getSize ( ) != <NUM_LIT> ) return ; if ( UseOp . getSubReg ( ) == <STR_LIT> ) { Imm = Imm . getLoBits ( <NUM_LIT> ) ; } else { assert ( UseOp . getSubReg ( ) == <STR_LIT> ) ; Imm = Imm . getHiBits ( <NUM_LIT> ) ; } } if ( UseMI -> getOpcode ( ) == <STR_LIT> ) { unsigned DestReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * DestRC = TargetRegisterInfo :: isVirtualRegister ( DestReg ) ? MRI . getRegClass ( DestReg ) : TRI . getPhysRegClass ( DestReg ) ; unsigned MovOp = TII -> getMovOpcode ( DestRC ) ; if ( MovOp == <STR_LIT> ) return ; UseMI -> setDesc ( TII -> get ( MovOp ) ) ; CopiesToReplace . push_back ( UseMI ) ; }


<s> const char * getPassName ( ) const override { return <STR_LIT> ;


<s> const char * getPassName ( ) const override {


<s> switch ( Opcode ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> static bool isSafeToFold ( unsigned Opcode ) { switch ( Opcode ) {


<s> static bool isUseMIInFoldList ( const std :: vector < FoldCandidate > & FoldList , const MachineInstr * MI ) {


<s> static bool isUseMIInFoldList ( const std :: vector < FoldCandidate > & FoldList , const MachineInstr * MI ) { for ( auto Candidate : FoldList ) { if ( Candidate . UseMI == MI ) return true ; }


<s> } MI -> setDesc ( TII -> get ( Opc ) ) ; } if ( isUseMIInFoldList ( FoldList , MI ) ) return false ; unsigned CommuteIdx0 = TargetInstrInfo :: CommuteAnyOperandIndex ; unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) OpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) OpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( MI , OpNo , OpToFold ) ) return false ; } FoldList . push_back ( FoldCandidate ( MI , OpNo , OpToFold ) ) ; return true ;


<s> MachineInstr * MI = Fold . UseMI ; MachineOperand & Old = MI -> getOperand ( Fold . UseOpNo ) ; assert ( Old . isReg ( ) ) ; if ( Fold . isImm ( ) ) { Old . ChangeToImmediate ( Fold . ImmToFold ) ; return true ; } MachineOperand * New = Fold . OpToFold ; if ( TargetRegisterInfo :: isVirtualRegister ( Old . getReg ( ) ) && TargetRegisterInfo :: isVirtualRegister ( New -> getReg ( ) ) ) { Old . substVirtReg ( New -> getReg ( ) , New -> getSubReg ( ) , TRI ) ; return true ; } return false ;


<s> MachineInstr & MI = * I ; if ( ! isSafeToFold ( MI . getOpcode ( ) ) ) continue ; unsigned OpSize = TII -> getOpSize ( MI , <NUM_LIT> ) ; MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( FoldingImm && ! TII -> isInlineConstant ( OpToFold , OpSize ) && ! MRI . hasOneUse ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; if ( OpToFold . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( OpToFold . getReg ( ) ) ) continue ; MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( Dst . getReg ( ) ) ) continue ; SmallVector < MachineInstr * , <NUM_LIT> > CopiesToReplace ; std :: vector < FoldCandidate > FoldList ; for ( MachineRegisterInfo :: use_iterator Use = MRI . use_begin ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) , E = MRI . use_end ( ) ; Use != E ; ++ Use ) { MachineInstr * UseMI = Use -> getParent ( ) ; foldOperand ( OpToFold , UseMI , Use . getOperandNo ( ) , FoldList , CopiesToReplace , TII , TRI , MRI ) ; } for ( MachineInstr * Copy : CopiesToReplace ) Copy -> addImplicitDefUseOperands ( MF ) ; for ( FoldCandidate & Fold : FoldList ) { if ( updateOperand ( Fold , TRI ) ) {


<s> if ( ! TII -> isFoldableCopy ( MI ) ) { if ( IsIEEEMode || ( ! HasNSZ && ! MI . getFlag ( MachineInstr :: FmNsz ) ) || ! tryFoldOMod ( MI ) ) tryFoldClamp ( MI ) ; continue ; } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) || OpToFold . isGlobal ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( OpToFold . isReg ( ) && ! Register :: isVirtualRegister ( OpToFold . getReg ( ) ) ) continue ; MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && ! Register :: isVirtualRegister ( Dst . getReg ( ) ) ) continue ; foldInstOperand ( MI , OpToFold ) ;


<s> MachineInstr & MI = * I ; tryFoldInst ( TII , & MI ) ; if ( ! TII -> isFoldableCopy ( MI ) ) { if ( IsIEEEMode || ( ! HasNSZ && ! MI . getFlag ( MachineInstr :: FmNsz ) ) || ! tryFoldOMod ( MI ) ) tryFoldClamp ( MI ) ; continue ; } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) || OpToFold . isGlobal ( ) ;


<s> if ( DefMI -> readsRegister ( <STR_LIT> , TRI ) && execMayBeModifiedBeforeUse ( * MRI , Reg , * DefMI , * Fold . UseMI ) ) continue ; } if ( updateOperand ( Fold , * TII , * TRI , * ST ) ) { if ( Fold . isReg ( ) ) { assert ( Fold . OpToFold && Fold . OpToFold -> isReg ( ) ) ; MRI -> clearKillFlags ( Fold . OpToFold -> getReg ( ) ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << static_cast < int > ( Fold . UseOpNo ) << <STR_LIT> << * Fold . UseMI ) ; } else if ( Fold . isCommuted ( ) ) { TII -> commuteInstruction ( * Fold . UseMI , false ) ; }


<s> for ( auto & Use : MRI -> use_nodbg_operands ( Dst . getReg ( ) ) ) UsesToProcess . push_back ( & Use ) ; for ( auto U : UsesToProcess ) { MachineInstr * UseMI = U -> getParent ( ) ; foldOperand ( OpToFold , UseMI , UseMI -> getOperandNo ( U ) , FoldList , CopiesToReplace ) ; } if ( CopiesToReplace . empty ( ) && FoldList . empty ( ) ) return Changed ; MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; for ( MachineInstr * Copy : CopiesToReplace ) Copy -> addImplicitDefUseOperands ( * MF ) ; for ( FoldCandidate & Fold : FoldList ) { assert ( ! Fold . isReg ( ) || Fold . OpToFold ) ; if ( Fold . isReg ( ) && Fold . OpToFold -> getReg ( ) . isVirtual ( ) ) { Register Reg = Fold . OpToFold -> getReg ( ) ; MachineInstr * DefMI = Fold . OpToFold -> getParent ( ) ; if ( DefMI -> readsRegister ( <STR_LIT> , TRI ) && execMayBeModifiedBeforeUse ( * MRI , Reg , * DefMI , * Fold . UseMI ) ) continue ; } if ( updateOperand ( Fold , * TII , * TRI , * ST ) ) {


<s> if ( UseMO . isUndef ( ) || TII -> isSDWA ( MI ) ) return false ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return ! MI . hasRegisterImplicitUseOperand ( <STR_LIT> ) ;


<s> bool IsIEEEMode = MFI -> getMode ( ) . IEEE ; bool HasNSZ = MFI -> hasNoSignedZerosFPMath ( ) ; bool Changed = false ; for ( MachineBasicBlock * MBB : depth_first ( & MF ) ) { MachineOperand * CurrentKnownM0Val = nullptr ; for ( auto & MI : make_early_inc_range ( * MBB ) ) { Changed |= tryFoldCndMask ( MI ) ; if ( tryFoldZeroHighBits ( MI ) ) { Changed = true ; continue ; } if ( MI . isRegSequence ( ) && tryFoldRegSequence ( MI ) ) { Changed = true ; continue ; } if ( MI . isPHI ( ) && tryFoldLCSSAPhi ( MI ) ) { Changed = true ; continue ; } if ( MI . mayLoad ( ) && tryFoldLoad ( MI ) ) { Changed = true ; continue ; } if ( ! TII -> isFoldableCopy ( MI ) ) { if ( CurrentKnownM0Val && MI . modifiesRegister ( <STR_LIT> , TRI ) ) CurrentKnownM0Val = nullptr ; if ( IsIEEEMode || ( ! HasNSZ && ! MI . getFlag ( MachineInstr :: FmNsz ) ) || ! tryFoldOMod ( MI ) ) Changed |= tryFoldClamp ( MI ) ; continue ; } if ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) { MachineOperand & NewM0Val = MI . getOperand ( <NUM_LIT> ) ; if ( CurrentKnownM0Val && CurrentKnownM0Val -> isIdenticalTo ( NewM0Val ) ) { MI . eraseFromParent ( ) ; Changed = true ; continue ; } CurrentKnownM0Val = ( NewM0Val . isReg ( ) && NewM0Val . getReg ( ) . isPhysical ( ) ) ? nullptr : & NewM0Val ; continue ;


<s> for ( unsigned I = MI . getNumOperands ( ) - <NUM_LIT> ; I >= NumOps ; -- I ) MI . removeOperand ( I ) ;


<s> if ( ! evalBinaryInstruction ( Opc , NewImm , Src0 -> getImm ( ) , Src1 -> getImm ( ) ) ) return false ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; bool IsSGPR = TRI . isSGPRReg ( MRI , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MI -> getOperand ( Src0Idx ) . ChangeToImmediate ( NewImm ) ; MI -> removeOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( IsSGPR ) ) ) ; return true ; } if ( ! MI -> isCommutable ( ) ) return false ; if ( Src0 -> isImm ( ) && ! Src1 -> isImm ( ) ) { std :: swap ( Src0 , Src1 ) ; std :: swap ( Src0Idx , Src1Idx ) ; } int32_t Src1Val = static_cast < int32_t > ( Src1 -> getImm ( ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) { if ( Src1Val == <NUM_LIT> ) { MI -> removeOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( <STR_LIT> ) ) ; } else if ( Src1Val == - <NUM_LIT> ) { MI -> removeOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( Opc == <STR_LIT> ) ) ) ; } else return false ; return true ; } if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { if ( Src1Val == <NUM_LIT> ) { MI -> removeOperand ( Src0Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( Opc == <STR_LIT> ) ) ) ; } else if ( Src1Val == - <NUM_LIT> ) { MI -> removeOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( <STR_LIT> ) ) ; stripExtraCopyOperands ( * MI ) ; } else return false ; return true ; } if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { if ( Src1Val == <NUM_LIT> ) { MI -> removeOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( <STR_LIT> ) ) ;


<s> } int Src1ModIdx = <STR_LIT> ( Opc , <STR_LIT> :: src1_modifiers ) ; int Src0ModIdx = <STR_LIT> ( Opc , <STR_LIT> :: src0_modifiers ) ; if ( ( Src1ModIdx != - <NUM_LIT> && MI . getOperand ( Src1ModIdx ) . getImm ( ) != <NUM_LIT> ) || ( Src0ModIdx != - <NUM_LIT> && MI . getOperand ( Src0ModIdx ) . getImm ( ) != <NUM_LIT> ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> ) ; auto & NewDesc = TII -> get ( Src0 -> isReg ( ) ? ( unsigned ) <STR_LIT> : getMovOpc ( false ) ) ; int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( Src2Idx != - <NUM_LIT> ) MI . removeOperand ( Src2Idx ) ; MI . removeOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ) ; if ( Src1ModIdx != - <NUM_LIT> ) MI . removeOperand ( Src1ModIdx ) ; if ( Src0ModIdx != - <NUM_LIT> ) MI . removeOperand ( Src0ModIdx ) ; mutateCopyOp ( MI , NewDesc ) ;


<s> while ( UseMI -> isCopy ( ) && ! Op -> getSubReg ( ) ) { Reg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( ! TRI -> isVGPR ( * MRI , Reg ) || ! MRI -> hasOneNonDBGUse ( Reg ) ) return false ; Op = & * MRI -> use_nodbg_begin ( Reg ) ; UseMI = Op -> getParent ( ) ; } if ( Op -> getSubReg ( ) ) return false ; unsigned OpIdx = Op - & UseMI -> getOperand ( <NUM_LIT> ) ; const MCInstrDesc & InstDesc = UseMI -> getDesc ( ) ; const TargetRegisterClass * OpRC = TII -> getRegClass ( InstDesc , OpIdx , TRI , * MI . getMF ( ) ) ; if ( ! OpRC || ! TRI -> isVectorSuperClass ( OpRC ) ) return false ; const auto * NewDstRC = TRI -> getEquivalentAGPRClass ( MRI -> getRegClass ( Reg ) ) ; auto Dst = MRI -> createVirtualRegister ( NewDstRC ) ; auto RS = BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Dst ) ; for ( unsigned I = <NUM_LIT> ; I < Defs . size ( ) ; ++ I ) { MachineOperand * Def = Defs [ I ] . first ; Def -> setIsKill ( false ) ; if ( TRI -> isAGPR ( * MRI , Def -> getReg ( ) ) ) { RS . add ( * Def ) ; } else { MachineInstr * SubDef = MRI -> getVRegDef ( Def -> getReg ( ) ) ; SubDef -> getOperand ( <NUM_LIT> ) . setIsKill ( false ) ; RS . addReg ( SubDef -> getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , Def -> getSubReg ( ) ) ; } RS . addImm ( Defs [ I ] . second ) ; } Op -> setReg ( Dst ) ; if ( ! TII -> isOperandLegal ( * UseMI , OpIdx , Op ) ) { Op -> setReg ( Reg ) ; RS -> eraseFromParent ( ) ; return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * RS << <STR_LIT> << * UseMI ) ; if ( MRI -> use_nodbg_empty ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) MI . eraseFromParent ( ) ; return true ;


<s> Old . ChangeToImmediate ( Fold . ImmToFold & <NUM_LIT> ) ; return true ; } break ; default : break ; } } } } if ( ( Fold . isImm ( ) || Fold . isFI ( ) || Fold . isGlobal ( ) ) && Fold . needsShrink ( ) ) { MachineBasicBlock * MBB = MI -> getParent ( ) ; auto Liveness = MBB -> computeRegisterLiveness ( & TRI , <STR_LIT> , MI , <NUM_LIT> ) ; if ( Liveness != MachineBasicBlock :: LQR_Dead ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> ) ; return false ; } MachineRegisterInfo & MRI = MBB -> getParent ( ) -> getRegInfo ( ) ; int Op32 = Fold . getShrinkOpcode ( ) ; MachineOperand & Dst0 = MI -> getOperand ( <NUM_LIT> ) ; MachineOperand & Dst1 = MI -> getOperand ( <NUM_LIT> ) ; assert ( Dst0 . isDef ( ) && Dst1 . isDef ( ) ) ; bool HaveNonDbgCarryUse = ! MRI . use_nodbg_empty ( Dst1 . getReg ( ) ) ; const TargetRegisterClass * Dst0RC = MRI . getRegClass ( Dst0 . getReg ( ) ) ; Register NewReg0 = MRI . createVirtualRegister ( Dst0RC ) ; MachineInstr * Inst32 = TII . buildShrunkInst ( * MI , Op32 ) ; if ( HaveNonDbgCarryUse ) { BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII . get ( <STR_LIT> ) , Dst1 . getReg ( ) ) . addReg ( <STR_LIT> , RegState :: Kill ) ; } Dst0 . setReg ( NewReg0 ) ; for ( unsigned I = MI -> getNumOperands ( ) - <NUM_LIT> ; I > <NUM_LIT> ; -- I ) MI -> removeOperand ( I ) ; MI -> setDesc ( TII . get ( <STR_LIT> ) ) ; if ( Fold . isCommuted ( ) ) TII . commuteInstruction ( * Inst32 , false ) ; return true ; } assert ( ! Fold . needsShrink ( ) && <STR_LIT> ) ; if ( Fold . isImm ( ) ) { if ( Old . isTied ( ) ) { int NewMFMAOpc = <STR_LIT> ( MI -> getOpcode ( ) ) ; if ( NewMFMAOpc == - <NUM_LIT> ) return false ; MI -> setDesc ( TII . get ( NewMFMAOpc ) ) ; MI -> untieRegOperand ( <NUM_LIT> ) ; } Old . ChangeToImmediate ( Fold . ImmToFold ) ; return true ; } if ( Fold . isGlobal ( ) ) { Old . ChangeToGA ( Fold . OpToFold -> getGlobal ( ) , Fold . OpToFold -> getOffset ( ) , Fold . OpToFold -> getTargetFlags ( ) ) ; return true ; } if ( Fold . isFI ( ) ) { Old . ChangeToFrameIndex ( Fold . FrameIndexToFold ) ;


<s> SmallVector < MachineRegisterInfo :: use_iterator , <NUM_LIT> > UsesToProcess ; for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; ++ Use ) { UsesToProcess . push_back ( Use ) ; } for ( auto U : UsesToProcess ) { MachineInstr * UseMI = U -> getParent ( ) ; foldOperand ( OpToFold , UseMI , U . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } } MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; for ( MachineInstr * Copy : CopiesToReplace ) Copy -> addImplicitDefUseOperands ( * MF ) ; for ( FoldCandidate & Fold : FoldList ) { if ( updateOperand ( Fold , * TII , * TRI , * ST ) ) { if ( Fold . isReg ( ) ) { assert ( Fold . OpToFold && Fold . OpToFold -> isReg ( ) ) ; MRI -> clearKillFlags ( Fold . OpToFold -> getReg ( ) ) ;


<s> FoldCandidate FC = FoldCandidate ( Use -> getParent ( ) , Use . getOperandNo ( ) , & UseMI -> getOperand ( <NUM_LIT> ) ) ; CopyUses . push_back ( FC ) ; } for ( auto & F : CopyUses ) { foldOperand ( * F . OpToFold , F . UseMI , F . UseOpNo , FoldList , CopiesToReplace ) ; } } } unsigned MovOp = TII -> getMovOpcode ( DestRC ) ; if ( MovOp == <STR_LIT> ) return ; UseMI -> setDesc ( TII -> get ( MovOp ) ) ; CopiesToReplace . push_back ( UseMI ) ; } else { if ( UseMI -> isCopy ( ) && OpToFold . isReg ( ) && TargetRegisterInfo :: isVirtualRegister ( UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) && TRI -> isVGPR ( * MRI , UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) && TRI -> isVGPR ( * MRI , UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) && ! UseMI -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ) { UseMI -> getOperand ( <NUM_LIT> ) . setReg ( OpToFold . getReg ( ) ) ; UseMI -> getOperand ( <NUM_LIT> ) . setSubReg ( OpToFold . getSubReg ( ) ) ; UseMI -> getOperand ( <NUM_LIT> ) . setIsKill ( false ) ; CopiesToReplace . push_back ( UseMI ) ; OpToFold . setIsKill ( false ) ; return ; } unsigned UseOpc = UseMI -> getOpcode ( ) ; if ( UseOpc == <STR_LIT> || ( UseOpc == <STR_LIT> && ( int ) UseOpIdx == <STR_LIT> ( UseOpc , <STR_LIT> :: src0 ) ) ) { if ( FoldingImmLike ) { if ( execMayBeModifiedBeforeUse ( * MRI , UseMI -> getOperand ( UseOpIdx ) . getReg ( ) , * OpToFold . getParent ( ) , UseMI ) ) return ; UseMI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; UseMI -> getOperand ( <NUM_LIT> ) . setSubReg ( <NUM_LIT> ) ; if ( OpToFold . isImm ( ) ) UseMI -> getOperand ( <NUM_LIT> ) . ChangeToImmediate ( OpToFold . getImm ( ) ) ; else UseMI -> getOperand ( <NUM_LIT> ) . ChangeToFrameIndex ( OpToFold . getIndex ( ) ) ; UseMI -> RemoveOperand ( <NUM_LIT> ) ; return ; }

if ( OpToFold . isReg ( ) && TRI -> isSGPRReg ( * MRI , OpToFold . getReg ( ) ) ) {


<s> } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) || OpToFold . isGlobal ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( OpToFold . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( OpToFold . getReg ( ) ) ) continue ; MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ;

if ( Dst . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( Dst . getReg ( ) ) ) continue ;


<s> MachineBasicBlock :: iterator I , Next ; for ( I = MBB -> begin ( ) ; I != MBB -> end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; tryFoldInst ( TII , & MI ) ; if ( ! TII -> isFoldableCopy ( MI ) ) { if ( IsIEEEMode || ( ! HasNSZ && ! MI . getFlag ( MachineInstr :: FmNsz ) ) || ! tryFoldOMod ( MI ) ) tryFoldClamp ( MI ) ; continue ; } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) || OpToFold . isGlobal ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( OpToFold . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( OpToFold . getReg ( ) ) ) continue ; MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( Dst . getReg ( ) ) ) continue ;


<s> if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) CommuteOpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) CommuteOpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , CommuteOpNo , OpToFold ) ) { if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) && ( OpToFold -> isImm ( ) || OpToFold -> isFI ( ) || OpToFold -> isGlobal ( ) ) ) { MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0 ; MachineOperand & OtherOp = MI -> getOperand ( OtherIdx ) ; if ( ! OtherOp . isReg ( ) || ! TII -> getRegisterInfo ( ) . isVGPR ( MRI , OtherOp . getReg ( ) ) ) return false ; assert ( MI -> getOperand ( <NUM_LIT> ) . isDef ( ) ) ; unsigned MaybeCommutedOpc = MI -> getOpcode ( ) ; int Op32 = <STR_LIT> ( MaybeCommutedOpc ) ; FoldList . push_back ( FoldCandidate ( MI , CommuteOpNo , OpToFold , true , Op32 ) ) ; return true ; } TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ; return false ; } FoldList . push_back ( FoldCandidate ( MI , CommuteOpNo , OpToFold , true ) ) ;


<s> if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , CommuteOpNo , OpToFold ) ) { if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) && ( OpToFold -> isImm ( ) || OpToFold -> isFI ( ) || OpToFold -> isGlobal ( ) ) ) { MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0 ; MachineOperand & OtherOp = MI -> getOperand ( OtherIdx ) ; if ( ! OtherOp . isReg ( ) || ! TII -> getRegisterInfo ( ) . isVGPR ( MRI , OtherOp . getReg ( ) ) ) return false ; assert ( MI -> getOperand ( <NUM_LIT> ) . isDef ( ) ) ; unsigned MaybeCommutedOpc = MI -> getOpcode ( ) ; int Op32 = <STR_LIT> ( MaybeCommutedOpc ) ; FoldList . push_back ( FoldCandidate ( MI , CommuteOpNo , OpToFold , true , Op32 ) ) ; return true ; } TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ; return false ; } FoldList . push_back ( FoldCandidate ( MI , CommuteOpNo , OpToFold , true ) ) ; return true ; }

FoldList . push_back ( FoldCandidate ( MI , OpNo , OpToFold ) ) ;


<s> switch ( TII . get ( Opcode ) . OpInfo [ OpNo ] . OperandType ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( ! isUInt < <NUM_LIT> > ( Fold . ImmToFold ) ) { if ( ! ( Fold . ImmToFold & <NUM_LIT> ) ) { Mod . setImm ( Mod . getImm ( ) | <STR_LIT> ) ; Mod . setImm ( Mod . getImm ( ) & ~ <STR_LIT> ) ; Old . ChangeToImmediate ( ( Fold . ImmToFold >> <NUM_LIT> ) & <NUM_LIT> ) ; return true ; } Mod . setImm ( Mod . getImm ( ) & ~ <STR_LIT> ) ; Old . ChangeToImmediate ( Fold . ImmToFold & <NUM_LIT> ) ; return true ; } break ; default : break ; } } } if ( ( Fold . isImm ( ) || Fold . isFI ( ) || Fold . isGlobal ( ) ) && Fold . needsShrink ( ) ) { MachineBasicBlock * MBB = MI -> getParent ( ) ; auto Liveness = MBB -> computeRegisterLiveness ( & TRI , <STR_LIT> , MI ) ; if ( Liveness != MachineBasicBlock :: LQR_Dead ) return false ; MachineRegisterInfo & MRI = MBB -> getParent ( ) -> getRegInfo ( ) ; int Op32 = Fold . getShrinkOpcode ( ) ; MachineOperand & Dst0 = MI -> getOperand ( <NUM_LIT> ) ; MachineOperand & Dst1 = MI -> getOperand ( <NUM_LIT> ) ; assert ( Dst0 . isDef ( ) && Dst1 . isDef ( ) ) ; bool HaveNonDbgCarryUse = ! MRI . use_nodbg_empty ( Dst1 . getReg ( ) ) ; const TargetRegisterClass * Dst0RC = MRI . getRegClass ( Dst0 . getReg ( ) ) ; unsigned NewReg0 = MRI . createVirtualRegister ( Dst0RC ) ; MachineInstr * Inst32 = TII . buildShrunkInst ( * MI , Op32 ) ; if ( HaveNonDbgCarryUse ) { BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII . get ( <STR_LIT> ) , Dst1 . getReg ( ) ) . addReg ( <STR_LIT> , RegState :: Kill ) ; } Dst0 . setReg ( NewReg0 ) ; for ( unsigned I = MI -> getNumOperands ( ) - <NUM_LIT> ; I > <NUM_LIT> ; -- I ) MI -> RemoveOperand ( I ) ; MI -> setDesc ( TII . get ( <STR_LIT> ) ) ; if ( Fold . isCommuted ( ) ) TII . commuteInstruction ( * Inst32 , false ) ; return true ; } assert ( ! Fold . needsShrink ( ) && <STR_LIT> ) ; if ( Fold . isImm ( ) ) { Old . ChangeToImmediate ( Fold . ImmToFold ) ; return true ;


<s> if ( FoldOp -> isImm ( ) ) { ImmToFold = FoldOp -> getImm ( ) ; } else if ( FoldOp -> isFI ( ) ) { FrameIndexToFold = FoldOp -> getIndex ( ) ;


<s> MachineOperand * NonInlineUse = nullptr ; int NonInlineUseOpNo = - <NUM_LIT> ; MachineRegisterInfo :: use_iterator NextUse , NextInstUse ; for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; Use = NextUse ) { NextUse = std :: next ( Use ) ; MachineInstr * UseMI = Use -> getParent ( ) ; unsigned OpNo = Use . getOperandNo ( ) ; if ( OpToFold . isImm ( ) && tryConstantFoldOp ( * MRI , TII , UseMI , & OpToFold ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * UseMI << '\n' ) ; NextUse = MRI -> use_begin ( Dst . getReg ( ) ) ; continue ; } if ( isInlineConstantIfFolded ( TII , * UseMI , OpNo , OpToFold ) ) { foldOperand ( OpToFold , UseMI , OpNo , FoldList , CopiesToReplace ) ; } else { if ( ++ NumLiteralUses == <NUM_LIT> ) { NonInlineUse = & * Use ; NonInlineUseOpNo = OpNo ; } } } if ( NumLiteralUses == <NUM_LIT> ) {

MachineInstr * UseMI = NonInlineUse -> getParent ( ) ;


<s> if ( UseMI -> isRegSequence ( ) ) { unsigned RegSeqDstReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned RegSeqDstSubReg = UseMI -> getOperand ( UseOpIdx + <NUM_LIT> ) . getImm ( ) ; for ( MachineRegisterInfo :: use_iterator RSUse = MRI -> use_begin ( RegSeqDstReg ) , RSE = MRI -> use_end ( ) ; RSUse != RSE ; ++ RSUse ) { MachineInstr * RSUseMI = RSUse -> getParent ( ) ; if ( RSUse -> getSubReg ( ) != RegSeqDstSubReg ) continue ; foldOperand ( OpToFold , RSUseMI , RSUse . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } return ; } bool FoldingImm = OpToFold . isImm ( ) ; if ( FoldingImm && UseMI -> isCopy ( ) ) { unsigned DestReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * DestRC = TargetRegisterInfo :: isVirtualRegister ( DestReg ) ? MRI -> getRegClass ( DestReg ) : TRI -> getPhysRegClass ( DestReg ) ; unsigned MovOp = TII -> getMovOpcode ( DestRC ) ; if ( MovOp == <STR_LIT> ) return ; UseMI -> setDesc ( TII -> get ( MovOp ) ) ; CopiesToReplace . push_back ( UseMI ) ; } else { const MCInstrDesc & UseDesc = UseMI -> getDesc ( ) ; if ( UseDesc . isVariadic ( ) || UseDesc . OpInfo [ UseOpIdx ] . RegClass == - <NUM_LIT> ) return ; } if ( ! FoldingImm ) { tryAddToFoldList ( FoldList , UseMI , UseOpIdx , & OpToFold , TII ) ; return ; } const MCInstrDesc & FoldDesc = OpToFold . getParent ( ) -> getDesc ( ) ; const TargetRegisterClass * FoldRC = TRI -> getRegClass ( FoldDesc . OpInfo [ <NUM_LIT> ] . RegClass ) ; APInt Imm ( TII -> operandBitWidth ( FoldDesc . OpInfo [ <NUM_LIT> ] . OperandType ) , OpToFold . getImm ( ) ) ; if ( UseOp . getSubReg ( ) && <STR_LIT> ( FoldRC -> getID ( ) ) == <NUM_LIT> ) { unsigned UseReg = UseOp . getReg ( ) ; const TargetRegisterClass * UseRC = TargetRegisterInfo :: isVirtualRegister ( UseReg ) ? MRI -> getRegClass ( UseReg ) : TRI -> getPhysRegClass ( UseReg ) ;

assert ( Imm . getBitWidth ( ) == <NUM_LIT> ) ;


<s> if ( Op . isReg ( ) ) { if ( Op . getSubReg ( ) != <STR_LIT> ) return & Op ; MachineInstr * Def = MRI . getVRegDef ( Op . getReg ( ) ) ;


<s> const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( ! Src0 -> isReg ( ) || Src0 -> getSubReg ( ) != Src1 -> getSubReg ( ) || Src0 -> getSubReg ( ) != <STR_LIT> ) return nullptr ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return nullptr ; return Src0 ;


<s> case <STR_LIT> : { int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( static_cast < int > ( OpNo ) == Src2Idx ) { bool IsF32 = Opc == <STR_LIT> ; const MCInstrDesc & MadDesc = TII -> get ( IsF32 ? <STR_LIT> : <STR_LIT> ) ; return TII -> isInlineConstant ( OpToFold , MadDesc . OpInfo [ OpNo ] . OperandType ) ; }


<s> if ( static_cast < int > ( OpNo ) == Src2Idx ) { bool IsF32 = Opc == <STR_LIT> ; const MCInstrDesc & MadDesc = TII -> get ( IsF32 ? <STR_LIT> : <STR_LIT> ) ; return TII -> isInlineConstant ( OpToFold , MadDesc . OpInfo [ OpNo ] . OperandType ) ; } }


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { unsigned NumOps = MI . getDesc ( ) . getNumOperands ( ) + MI . getDesc ( ) . getNumImplicitUses ( ) ; return MI . getNumOperands ( ) == NumOps ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> static bool isUseSafeToFold ( const MachineInstr & MI , const MachineOperand & UseMO ) {


<s> TII = ST -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; if ( ! isSafeToFold ( MI ) ) { tryFoldClamp ( MI ) ; continue ; } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( OpToFold . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( OpToFold . getReg ( ) ) ) continue ;


<s> if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) OpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) OpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , OpNo , OpToFold ) ) return false ; } FoldList . push_back ( FoldCandidate ( MI , OpNo , OpToFold ) ) ;


<s> if ( ! ClampSrc || ! hasOneNonDBGUseInst ( * MRI , ClampSrc -> getReg ( ) ) ) return false ; MachineInstr * Def = MRI -> getVRegDef ( ClampSrc -> getReg ( ) ) ; if ( ! TII -> hasFPClamp ( * Def ) ) return false ; MachineOperand * DefClamp = TII -> getNamedOperand ( * Def , <STR_LIT> :: clamp ) ; if ( ! DefClamp ) return false ; DEBUG ( dbgs ( ) << <STR_LIT> << * DefClamp << <STR_LIT> << * Def << '\n' ) ; DefClamp -> setImm ( <NUM_LIT> ) ; MRI -> replaceRegWith ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MI . eraseFromParent ( ) ; return true ;


<s> return true ; } if ( Fold . isFI ( ) ) { Old . ChangeToFrameIndex ( Fold . FrameIndexToFold ) ; return true ; } MachineOperand * New = Fold . OpToFold ; if ( TargetRegisterInfo :: isVirtualRegister ( Old . getReg ( ) ) && TargetRegisterInfo :: isVirtualRegister ( New -> getReg ( ) ) ) { Old . substVirtReg ( New -> getReg ( ) , New -> getSubReg ( ) , TRI ) ; return true ; }


<s> MachineRegisterInfo :: use_nodbg_iterator NextUse ; for ( MachineRegisterInfo :: use_nodbg_iterator Use = MRI -> use_nodbg_begin ( Dst . getReg ( ) ) , E = MRI -> use_nodbg_end ( ) ; Use != E ; Use = NextUse ) { NextUse = std :: next ( Use ) ; MachineInstr * UseMI = Use -> getParent ( ) ; unsigned OpNo = Use . getOperandNo ( ) ; if ( OpToFold . isImm ( ) && tryConstantFoldOp ( * MRI , TII , UseMI , & OpToFold ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * UseMI << '\n' ) ; NextUse = MRI -> use_nodbg_begin ( Dst . getReg ( ) ) ; FoldList . clear ( ) ; continue ; } if ( isInlineConstantIfFolded ( TII , * UseMI , OpNo , OpToFold ) ) { foldOperand ( OpToFold , UseMI , OpNo , FoldList , CopiesToReplace ) ; } else if ( frameIndexMayFold ( TII , * UseMI , OpNo , OpToFold ) ) { foldOperand ( OpToFold , UseMI , OpNo , FoldList , CopiesToReplace ) ; } else { if ( ++ NumLiteralUses == <NUM_LIT> ) { NonInlineUse = & * Use ; NonInlineUseOpNo = OpNo ; } } } if ( NumLiteralUses == <NUM_LIT> ) { MachineInstr * UseMI = NonInlineUse -> getParent ( ) ; foldOperand ( OpToFold , UseMI , NonInlineUseOpNo , FoldList , CopiesToReplace ) ; } } else { SmallVector < MachineRegisterInfo :: use_nodbg_iterator , <NUM_LIT> > UsesToProcess ; for ( MachineRegisterInfo :: use_nodbg_iterator Use = MRI -> use_nodbg_begin ( Dst . getReg ( ) ) , E = MRI -> use_nodbg_end ( ) ; Use != E ; ++ Use ) { UsesToProcess . push_back ( Use ) ; } for ( auto U : UsesToProcess ) { MachineInstr * UseMI = U -> getParent ( ) ; foldOperand ( OpToFold , UseMI , U . getOperandNo ( ) , FoldList , CopiesToReplace ) ;


<s> } } else { SmallVector < MachineRegisterInfo :: use_nodbg_iterator , <NUM_LIT> > UsesToProcess ; for ( MachineRegisterInfo :: use_nodbg_iterator Use = MRI -> use_nodbg_begin ( Dst . getReg ( ) ) , E = MRI -> use_nodbg_end ( ) ; Use != E ; ++ Use ) { UsesToProcess . push_back ( Use ) ; } for ( auto U : UsesToProcess ) { MachineInstr * UseMI = U -> getParent ( ) ; foldOperand ( OpToFold , UseMI , U . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } } MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; for ( MachineInstr * Copy : CopiesToReplace ) Copy -> addImplicitDefUseOperands ( * MF ) ; SmallPtrSet < MachineInstr * , <NUM_LIT> > Folded ; for ( FoldCandidate & Fold : FoldList ) { assert ( ! Fold . isReg ( ) || Fold . OpToFold ) ; if ( Folded . count ( Fold . UseMI ) ) continue ; if ( Fold . isReg ( ) && Fold . OpToFold -> getReg ( ) . isVirtual ( ) ) { Register Reg = Fold . OpToFold -> getReg ( ) ; MachineInstr * DefMI = Fold . OpToFold -> getParent ( ) ; if ( DefMI -> readsRegister ( <STR_LIT> , TRI ) && execMayBeModifiedBeforeUse ( * MRI , Reg , * DefMI , * Fold . UseMI ) ) continue ; } if ( updateOperand ( Fold , * TII , * TRI , * ST ) ) { if ( Fold . isReg ( ) ) { assert ( Fold . OpToFold && Fold . OpToFold -> isReg ( ) ) ; MRI -> clearKillFlags ( Fold . OpToFold -> getReg ( ) ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << static_cast < int > ( Fold . UseOpNo ) << <STR_LIT> << * Fold . UseMI << '\n' ) ; if ( tryFoldInst ( TII , Fold . UseMI ) ) Folded . insert ( Fold . UseMI ) ; } else if ( Fold . isCommuted ( ) ) { TII -> commuteInstruction ( * Fold . UseMI , false ) ; } }


<s> return OpToFold . isFI ( ) && TII -> isMUBUF ( UseMI ) && OpNo == <STR_LIT> ( UseMI . getOpcode ( ) , <STR_LIT> :: vaddr ) ;


<s> Next = std :: next ( I ) ; MachineInstr & MI = * I ; tryFoldInst ( TII , & MI ) ; if ( ! TII -> isFoldableCopy ( MI ) ) { if ( CurrentKnownM0Val && MI . modifiesRegister ( <STR_LIT> , TRI ) ) CurrentKnownM0Val = nullptr ; if ( IsIEEEMode || ( ! HasNSZ && ! MI . getFlag ( MachineInstr :: FmNsz ) ) || ! tryFoldOMod ( MI ) ) tryFoldClamp ( MI ) ; continue ; } if ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) { MachineOperand & NewM0Val = MI . getOperand ( <NUM_LIT> ) ; if ( CurrentKnownM0Val && CurrentKnownM0Val -> isIdenticalTo ( NewM0Val ) ) { MI . eraseFromParent ( ) ; continue ; } CurrentKnownM0Val = ( NewM0Val . isReg ( ) && NewM0Val . getReg ( ) . isPhysical ( ) ) ? nullptr : & NewM0Val ; continue ;