<s> bool FoldAsMAD = tryAddToFoldList ( FoldList , MI , OpNo , OpToFold , TII ) ; if ( FoldAsMAD ) { MI -> untieRegOperand ( OpNo ) ; return true ; } MI -> setDesc ( TII -> get ( Opc ) ) ; } if ( OpToFold -> isImm ( ) ) { unsigned ImmOpc = <NUM_LIT> ; if ( Opc == <STR_LIT> ) ImmOpc = <STR_LIT> ; else if ( Opc == <STR_LIT> ) ImmOpc = <STR_LIT> ; if ( ImmOpc ) { MI -> setDesc ( TII -> get ( ImmOpc ) ) ; appendFoldCandidate ( FoldList , MI , OpNo , OpToFold ) ; return true ; } } if ( isUseMIInFoldList ( FoldList , MI ) ) return false ; unsigned CommuteOpNo = OpNo ; unsigned CommuteIdx0 = TargetInstrInfo :: CommuteAnyOperandIndex ; unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( * MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) CommuteOpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) CommuteOpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , CommuteOpNo , OpToFold ) ) { if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) && ( OpToFold -> isImm ( ) || OpToFold -> isFI ( ) || OpToFold -> isGlobal ( ) ) ) { MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0 ; MachineOperand & OtherOp = MI -> getOperand ( OtherIdx ) ; if ( ! OtherOp . isReg ( ) || ! TII -> getRegisterInfo ( ) . isVGPR ( MRI , OtherOp . getReg ( ) ) ) return false ; assert ( MI -> getOperand ( <NUM_LIT> ) . isDef ( ) ) ; unsigned MaybeCommutedOpc = MI -> getOpcode ( ) ;

int Op32 = <STR_LIT> ( MaybeCommutedOpc ) ;


<s> const MCInstrDesc & Desc = UseMI -> getDesc ( ) ; const MCOperandInfo * OpInfo = Desc . OpInfo ; if ( ! OpInfo || UseOpIdx >= Desc . getNumOperands ( ) ) return false ; uint8_t OpTy = OpInfo [ UseOpIdx ] . OperandType ; if ( OpTy < <STR_LIT> || OpTy > <STR_LIT> ) return false ; if ( OpToFold . isImm ( ) && TII -> isInlineConstant ( OpToFold , OpTy ) && TII -> isOperandLegal ( * UseMI , UseOpIdx , & OpToFold ) ) { UseMI -> getOperand ( UseOpIdx ) . ChangeToImmediate ( OpToFold . getImm ( ) ) ; return true ; } if ( ! OpToFold . isReg ( ) ) return false ;

Register UseReg = OpToFold . getReg ( ) ;


<s> int NonInlineUseOpNo = - <NUM_LIT> ; MachineRegisterInfo :: use_iterator NextUse ; for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; Use = NextUse ) { NextUse = std :: next ( Use ) ; MachineInstr * UseMI = Use -> getParent ( ) ; unsigned OpNo = Use . getOperandNo ( ) ; if ( OpToFold . isImm ( ) && tryConstantFoldOp ( * MRI , TII , UseMI , & OpToFold ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * UseMI << '\n' ) ; NextUse = MRI -> use_begin ( Dst . getReg ( ) ) ; FoldList . clear ( ) ; continue ; } if ( isInlineConstantIfFolded ( TII , * UseMI , OpNo , OpToFold ) ) { foldOperand ( OpToFold , UseMI , OpNo , FoldList , CopiesToReplace ) ; } else { if ( ++ NumLiteralUses == <NUM_LIT> ) { NonInlineUse = & * Use ;


<s> unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( * MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) CommuteOpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) CommuteOpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , CommuteOpNo , OpToFold ) ) { if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) && ( OpToFold -> isImm ( ) || OpToFold -> isFI ( ) ) ) { MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0 ; MachineOperand & OtherOp = MI -> getOperand ( OtherIdx ) ; if ( ! OtherOp . isReg ( ) || ! TII -> getRegisterInfo ( ) . isVGPR ( MRI , OtherOp . getReg ( ) ) ) return false ; assert ( MI -> getOperand ( <NUM_LIT> ) . isDef ( ) ) ; unsigned MaybeCommutedOpc = MI -> getOpcode ( ) ; int Op32 = <STR_LIT> ( MaybeCommutedOpc ) ; FoldList . push_back ( FoldCandidate ( MI , CommuteOpNo , OpToFold , true , Op32 ) ) ;


<s> static unsigned macToMad ( unsigned Opc ) { switch ( Opc ) { case <STR_LIT> : return <STR_LIT> ;


<s> else if ( CommuteIdx1 == OpNo ) CommuteOpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , CommuteOpNo , OpToFold ) ) { if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) && ( OpToFold -> isImm ( ) || OpToFold -> isFI ( ) || OpToFold -> isGlobal ( ) ) ) { MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0 ; MachineOperand & OtherOp = MI -> getOperand ( OtherIdx ) ; if ( ! OtherOp . isReg ( ) || ! TII -> getRegisterInfo ( ) . isVGPR ( MRI , OtherOp . getReg ( ) ) ) return false ; assert ( MI -> getOperand ( <NUM_LIT> ) . isDef ( ) ) ; unsigned MaybeCommutedOpc = MI -> getOpcode ( ) ; int Op32 = <STR_LIT> ( MaybeCommutedOpc ) ; appendFoldCandidate ( FoldList , MI , CommuteOpNo , OpToFold , true , Op32 ) ; return true ; } TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ; return false ; } appendFoldCandidate ( FoldList , MI , CommuteOpNo , OpToFold , true ) ; return true ; } if ( TII -> isSALU ( MI -> getOpcode ( ) ) ) { const MCInstrDesc & InstDesc = MI -> getDesc ( ) ; const MCOperandInfo & OpInfo = InstDesc . OpInfo [ OpNo ] ; const <ISA_LIT>RegisterInfo & SRI = TII -> getRegisterInfo ( ) ; if ( OpToFold -> isImm ( ) ) { if ( ! SRI . opCanUseInlineConstant ( OpInfo . OperandType ) || ! TII -> isInlineConstant ( * OpToFold , OpInfo ) ) { for ( unsigned i = <NUM_LIT> , e = InstDesc . getNumOperands ( ) ; i != e ; ++ i ) { auto & Op = MI -> getOperand ( i ) ; if ( OpNo != i && TII -> isLiteralConstantLike ( Op , OpInfo ) ) { return false ; }


<s> UseMI -> getOperand ( UseOpIdx ) . ChangeToImmediate ( OpToFold . getImm ( ) ) ; return true ; } if ( ! OpToFold . isReg ( ) ) return false ; Register UseReg = OpToFold . getReg ( ) ; if ( ! UseReg . isVirtual ( ) ) return false ; if ( llvm :: any_of ( FoldList , [ UseMI ] ( const FoldCandidate & FC ) { return FC . UseMI == UseMI ; } ) ) return false ; MachineRegisterInfo & MRI = UseMI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; SmallVector < std :: pair < MachineOperand * , unsigned > , <NUM_LIT> > Defs ; if ( ! getRegSeqInit ( Defs , UseReg , OpTy , TII , MRI ) ) return false ; int32_t Imm ; for ( unsigned I = <NUM_LIT> , E = Defs . size ( ) ; I != E ; ++ I ) { const MachineOperand * Op = Defs [ I ] . first ;


<s> static bool tryToFoldACImm ( const <ISA_LIT>InstrInfo * TII , const MachineOperand & OpToFold , MachineInstr * UseMI , unsigned UseOpIdx , SmallVectorImpl < FoldCandidate > & FoldList ) { const MCInstrDesc & Desc = UseMI -> getDesc ( ) ; const MCOperandInfo * OpInfo = Desc . OpInfo ; if ( ! OpInfo || UseOpIdx >= Desc . getNumOperands ( ) ) return false ; uint8_t OpTy = OpInfo [ UseOpIdx ] . OperandType ; if ( OpTy < <STR_LIT> || OpTy > <STR_LIT> ) return false ; if ( OpToFold . isImm ( ) && TII -> isInlineConstant ( OpToFold , OpTy ) && TII -> isOperandLegal ( * UseMI , UseOpIdx , & OpToFold ) ) { UseMI -> getOperand ( UseOpIdx ) . ChangeToImmediate ( OpToFold . getImm ( ) ) ; return true ; } if ( ! OpToFold . isReg ( ) ) return false ; Register UseReg = OpToFold . getReg ( ) ; if ( ! UseReg . isVirtual ( ) ) return false ; if ( llvm :: any_of ( FoldList , [ UseMI ] ( const FoldCandidate & FC ) { return FC . UseMI == UseMI ; } ) ) return false ; MachineRegisterInfo & MRI = UseMI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; SmallVector < std :: pair < MachineOperand * , unsigned > , <NUM_LIT> > Defs ; if ( ! getRegSeqInit ( Defs , UseReg , OpTy , TII , MRI ) ) return false ; int32_t Imm ; for ( unsigned I = <NUM_LIT> , E = Defs . size ( ) ; I != E ; ++ I ) { const MachineOperand * Op = Defs [ I ] . first ; if ( ! Op -> isImm ( ) ) return false ; auto SubImm = Op -> getImm ( ) ; if ( ! I ) {


<s> } if ( Opc == <STR_LIT> && OpToFold -> isImm ( ) ) { MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; appendFoldCandidate ( FoldList , MI , OpNo , OpToFold ) ; return true ; } if ( isUseMIInFoldList ( FoldList , MI ) ) return false ; unsigned CommuteOpNo = OpNo ; unsigned CommuteIdx0 = TargetInstrInfo :: CommuteAnyOperandIndex ; unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( * MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) CommuteOpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) CommuteOpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , CommuteOpNo , OpToFold ) ) { if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) && ( OpToFold -> isImm ( ) || OpToFold -> isFI ( ) || OpToFold -> isGlobal ( ) ) ) { MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0 ; MachineOperand & OtherOp = MI -> getOperand ( OtherIdx ) ; if ( ! OtherOp . isReg ( ) || ! TII -> getRegisterInfo ( ) . isVGPR ( MRI , OtherOp . getReg ( ) ) ) return false ; assert ( MI -> getOperand ( <NUM_LIT> ) . isDef ( ) ) ; unsigned MaybeCommutedOpc = MI -> getOpcode ( ) ; int Op32 = <STR_LIT> ( MaybeCommutedOpc ) ; appendFoldCandidate ( FoldList , MI , CommuteOpNo , OpToFold , true , Op32 ) ; return true ; } TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ; return false ; } appendFoldCandidate ( FoldList , MI , CommuteOpNo , OpToFold , true ) ; return true ; } if ( TII -> isSALU ( MI -> getOpcode ( ) ) ) {

const MCInstrDesc & InstDesc = MI -> getDesc ( ) ;


<s> if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) CommuteOpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) CommuteOpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , CommuteOpNo , OpToFold ) ) { if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) && ( OpToFold -> isImm ( ) || OpToFold -> isFI ( ) || OpToFold -> isGlobal ( ) ) ) { MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0 ; MachineOperand & OtherOp = MI -> getOperand ( OtherIdx ) ; if ( ! OtherOp . isReg ( ) || ! TII -> getRegisterInfo ( ) . isVGPR ( MRI , OtherOp . getReg ( ) ) ) return false ; assert ( MI -> getOperand ( <NUM_LIT> ) . isDef ( ) ) ; unsigned MaybeCommutedOpc = MI -> getOpcode ( ) ; int Op32 = <STR_LIT> ( MaybeCommutedOpc ) ; appendFoldCandidate ( FoldList , MI , CommuteOpNo , OpToFold , true , Op32 ) ; return true ; } TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ; return false ; } appendFoldCandidate ( FoldList , MI , CommuteOpNo , OpToFold , true ) ;


<s> if ( ++ NumLiteralUses == <NUM_LIT> ) { NonInlineUse = & * Use ; NonInlineUseOpNo = OpNo ; } } } if ( NumLiteralUses == <NUM_LIT> ) { MachineInstr * UseMI = NonInlineUse -> getParent ( ) ; foldOperand ( OpToFold , UseMI , NonInlineUseOpNo , FoldList , CopiesToReplace ) ; } } else { SmallVector < MachineRegisterInfo :: use_nodbg_iterator , <NUM_LIT> > UsesToProcess ; for ( MachineRegisterInfo :: use_nodbg_iterator Use = MRI -> use_nodbg_begin ( Dst . getReg ( ) ) , E = MRI -> use_nodbg_end ( ) ; Use != E ; ++ Use ) { UsesToProcess . push_back ( Use ) ; } for ( auto U : UsesToProcess ) { MachineInstr * UseMI = U -> getParent ( ) ; foldOperand ( OpToFold , UseMI , U . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } } MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; for ( MachineInstr * Copy : CopiesToReplace ) Copy -> addImplicitDefUseOperands ( * MF ) ; for ( FoldCandidate & Fold : FoldList ) { assert ( ! Fold . isReg ( ) || Fold . OpToFold ) ; if ( Fold . isReg ( ) && Fold . OpToFold -> getReg ( ) . isVirtual ( ) ) { Register Reg = Fold . OpToFold -> getReg ( ) ; MachineInstr * DefMI = Fold . OpToFold -> getParent ( ) ; if ( DefMI -> readsRegister ( <STR_LIT> , TRI ) && execMayBeModifiedBeforeUse ( * MRI , Reg , * DefMI , * Fold . UseMI ) ) continue ; } if ( updateOperand ( Fold , * TII , * TRI , * ST ) ) { if ( Fold . isReg ( ) ) { assert ( Fold . OpToFold && Fold . OpToFold -> isReg ( ) ) ; MRI -> clearKillFlags ( Fold . OpToFold -> getReg ( ) ) ; }

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << static_cast < int > ( Fold . UseOpNo ) << <STR_LIT> << * Fold . UseMI << '\n' ) ;


<s> SmallVector < MachineOperand * , <NUM_LIT> > UsesToProcess ; for ( auto & Use : MRI -> use_nodbg_operands ( Dst . getReg ( ) ) ) UsesToProcess . push_back ( & Use ) ; for ( auto U : UsesToProcess ) { MachineInstr * UseMI = U -> getParent ( ) ; foldOperand ( OpToFold , UseMI , UseMI -> getOperandNo ( U ) , FoldList , CopiesToReplace ) ; } } if ( CopiesToReplace . empty ( ) && FoldList . empty ( ) ) return Changed ; MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; for ( MachineInstr * Copy : CopiesToReplace ) Copy -> addImplicitDefUseOperands ( * MF ) ; for ( FoldCandidate & Fold : FoldList ) { assert ( ! Fold . isReg ( ) || Fold . OpToFold ) ; if ( Fold . isReg ( ) && Fold . OpToFold -> getReg ( ) . isVirtual ( ) ) { Register Reg = Fold . OpToFold -> getReg ( ) ; MachineInstr * DefMI = Fold . OpToFold -> getParent ( ) ; if ( DefMI -> readsRegister ( <STR_LIT> , TRI ) && execMayBeModifiedBeforeUse ( * MRI , Reg , * DefMI , * Fold . UseMI ) ) continue ;


<s> bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) || OpToFold . isGlobal ( ) ; if ( FoldingImm ) { unsigned NumLiteralUses = <NUM_LIT> ; MachineOperand * NonInlineUse = nullptr ; int NonInlineUseOpNo = - <NUM_LIT> ; for ( auto & Use : make_early_inc_range ( MRI -> use_nodbg_operands ( Dst . getReg ( ) ) ) ) { MachineInstr * UseMI = Use . getParent ( ) ; unsigned OpNo = UseMI -> getOperandNo ( & Use ) ; if ( isInlineConstantIfFolded ( TII , * UseMI , OpNo , OpToFold ) ) { foldOperand ( OpToFold , UseMI , OpNo , FoldList , CopiesToReplace ) ; } else if ( frameIndexMayFold ( TII , * UseMI , OpNo , OpToFold ) ) { foldOperand ( OpToFold , UseMI , OpNo , FoldList , CopiesToReplace ) ; } else { if ( ++ NumLiteralUses == <NUM_LIT> ) { NonInlineUse = & Use ; NonInlineUseOpNo = OpNo ; } } } if ( NumLiteralUses == <NUM_LIT> ) { MachineInstr * UseMI = NonInlineUse -> getParent ( ) ; foldOperand ( OpToFold , UseMI , NonInlineUseOpNo , FoldList , CopiesToReplace ) ; } } else { SmallVector < MachineOperand * , <NUM_LIT> > UsesToProcess ; for ( auto & Use : MRI -> use_nodbg_operands ( Dst . getReg ( ) ) ) UsesToProcess . push_back ( & Use ) ; for ( auto U : UsesToProcess ) { MachineInstr * UseMI = U -> getParent ( ) ; foldOperand ( OpToFold , UseMI , UseMI -> getOperandNo ( U ) , FoldList , CopiesToReplace ) ; } } if ( CopiesToReplace . empty ( ) && FoldList . empty ( ) ) return Changed ; MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; for ( MachineInstr * Copy : CopiesToReplace ) Copy -> addImplicitDefUseOperands ( * MF ) ; for ( FoldCandidate & Fold : FoldList ) { assert ( ! Fold . isReg ( ) || Fold . OpToFold ) ; if ( Fold . isReg ( ) && Fold . OpToFold -> getReg ( ) . isVirtual ( ) ) { Register Reg = Fold . OpToFold -> getReg ( ) ; MachineInstr * DefMI = Fold . OpToFold -> getParent ( ) ; if ( DefMI -> readsRegister ( <STR_LIT> , TRI ) && execMayBeModifiedBeforeUse ( * MRI , Reg , * DefMI , * Fold . UseMI ) ) continue ; }

if ( updateOperand ( Fold , * TII , * TRI , * ST ) ) {


<s> if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( FoldingImm && ! TII -> isInlineConstant ( OpToFold , OpSize ) && ! MRI . hasOneUse ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; if ( OpToFold . isReg ( ) && ( ! TargetRegisterInfo :: isVirtualRegister ( OpToFold . getReg ( ) ) || OpToFold . getSubReg ( ) ) ) continue ; SmallVector < MachineInstr * , <NUM_LIT> > CopiesToReplace ; std :: vector < FoldCandidate > FoldList ; for ( MachineRegisterInfo :: use_iterator Use = MRI . use_begin ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) , E = MRI . use_end ( ) ; Use != E ; ++ Use ) { MachineInstr * UseMI = Use -> getParent ( ) ; foldOperand ( OpToFold , UseMI , Use . getOperandNo ( ) , FoldList , CopiesToReplace , TII , TRI , MRI ) ; } for ( MachineInstr * Copy : CopiesToReplace ) Copy -> addImplicitDefUseOperands ( MF ) ; for ( FoldCandidate & Fold : FoldList ) { if ( updateOperand ( Fold , TRI ) ) { if ( ! Fold . isImm ( ) ) {

assert ( Fold . OpToFold && Fold . OpToFold -> isReg ( ) ) ;


<s> if ( ! ClampSrc || ! MRI -> hasOneNonDBGUser ( ClampSrc -> getReg ( ) ) ) return false ; MachineInstr * Def = MRI -> getVRegDef ( ClampSrc -> getReg ( ) ) ;

if ( TII -> getClampMask ( * Def ) != TII -> getClampMask ( MI ) ) return false ;


<s> std :: tie ( RegOp , OMod ) = isOMod ( MI ) ; if ( OMod == <STR_LIT> || ! RegOp -> isReg ( ) || RegOp -> getSubReg ( ) != <STR_LIT> || ! MRI -> hasOneNonDBGUser ( RegOp -> getReg ( ) ) ) return false ; MachineInstr * Def = MRI -> getVRegDef ( RegOp -> getReg ( ) ) ; MachineOperand * DefOMod = TII -> getNamedOperand ( * Def , <STR_LIT> :: omod ) ; if ( ! DefOMod || DefOMod -> getImm ( ) != <STR_LIT> ) return false ; if ( TII -> hasModifiersSet ( * Def , <STR_LIT> :: clamp ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * Def ) ; DefOMod -> setImm ( OMod ) ; MRI -> replaceRegWith ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MI . eraseFromParent ( ) ;


<s> unsigned Op = MI . getOpcode ( ) ; switch ( Op ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( ! TII -> getNamedOperand ( MI , <STR_LIT> :: clamp ) -> getImm ( ) ) return nullptr ; const MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ;

const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ;


<s> case <STR_LIT> : { if ( ! TII -> getNamedOperand ( MI , <STR_LIT> :: clamp ) -> getImm ( ) ) return nullptr ; const MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( ! Src0 -> isReg ( ) || ! Src1 -> isReg ( ) || Src0 -> getSubReg ( ) != Src1 -> getSubReg ( ) || Src0 -> getSubReg ( ) != <STR_LIT> ) return nullptr ;

if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return nullptr ;


<s> for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; if ( ! isSafeToFold ( MI . getOpcode ( ) ) ) continue ; unsigned OpSize = TII -> getOpSize ( MI , <NUM_LIT> ) ; MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( FoldingImm && ! TII -> isInlineConstant ( OpToFold , OpSize ) && ! MRI . hasOneUse ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; if ( OpToFold . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( OpToFold . getReg ( ) ) ) continue ; MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( Dst . getReg ( ) ) ) continue ;

SmallVector < MachineInstr * , <NUM_LIT> > CopiesToReplace ;


<s> } } else { for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; ++ Use ) { MachineInstr * UseMI = Use -> getParent ( ) ; foldOperand ( OpToFold , UseMI , Use . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } } MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; for ( MachineInstr * Copy : CopiesToReplace ) Copy -> addImplicitDefUseOperands ( * MF ) ; for ( FoldCandidate & Fold : FoldList ) { if ( updateOperand ( Fold , * TRI ) ) { if ( Fold . isReg ( ) ) { assert ( Fold . OpToFold && Fold . OpToFold -> isReg ( ) ) ; MRI -> clearKillFlags ( Fold . OpToFold -> getReg ( ) ) ; } DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << static_cast < int > ( Fold . UseOpNo ) << <STR_LIT> << * Fold . UseMI << '\n' ) ; tryFoldInst ( TII , Fold . UseMI ) ; } else if ( Fold . isCommuted ( ) ) { TII -> commuteInstruction ( * Fold . UseMI , false ) ; }


<s> bool HasNSZ = MFI -> hasNoSignedZerosFPMath ( ) ; for ( MachineBasicBlock * MBB : depth_first ( & MF ) ) { MachineOperand * CurrentKnownM0Val = nullptr ; for ( auto & MI : make_early_inc_range ( * MBB ) ) { tryFoldCndMask ( MI ) ; if ( MI . isRegSequence ( ) && tryFoldRegSequence ( MI ) ) continue ; if ( MI . isPHI ( ) && tryFoldLCSSAPhi ( MI ) ) continue ; if ( MI . mayLoad ( ) && tryFoldLoad ( MI ) ) continue ; if ( ! TII -> isFoldableCopy ( MI ) ) { if ( CurrentKnownM0Val && MI . modifiesRegister ( <STR_LIT> , TRI ) ) CurrentKnownM0Val = nullptr ; if ( IsIEEEMode || ( ! HasNSZ && ! MI . getFlag ( MachineInstr :: FmNsz ) ) || ! tryFoldOMod ( MI ) ) tryFoldClamp ( MI ) ; continue ; } if ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) == <STR_LIT> ) { MachineOperand & NewM0Val = MI . getOperand ( <NUM_LIT> ) ;

if ( CurrentKnownM0Val && CurrentKnownM0Val -> isIdenticalTo ( NewM0Val ) ) {


<s> if ( CurrentKnownM0Val && CurrentKnownM0Val -> isIdenticalTo ( NewM0Val ) ) { MI . eraseFromParent ( ) ; continue ; } CurrentKnownM0Val = ( NewM0Val . isReg ( ) && NewM0Val . getReg ( ) . isPhysical ( ) ) ? nullptr : & NewM0Val ; continue ; } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) || OpToFold . isGlobal ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( OpToFold . isReg ( ) && ! OpToFold . getReg ( ) . isVirtual ( ) ) continue ; if ( ! MI . getOperand ( <NUM_LIT> ) . getReg ( ) . isVirtual ( ) ) continue ; foldInstOperand ( MI , OpToFold ) ; auto * InstToErase = & MI ; while ( MRI -> use_nodbg_empty ( InstToErase -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ) {

auto & SrcOp = InstToErase -> getOperand ( <NUM_LIT> ) ;


<s> unsigned Op = MI . getOpcode ( ) ; switch ( Op ) { case <STR_LIT> : case <STR_LIT> : { if ( ( Op == <STR_LIT> && MFI -> getMode ( ) . FP32Denormals ) || ( Op == <STR_LIT> && MFI -> getMode ( ) . FP64FP16Denormals ) ) return std :: make_pair ( nullptr , <STR_LIT> ) ; const MachineOperand * RegOp = nullptr ; const MachineOperand * ImmOp = nullptr ; const MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src0 -> isImm ( ) ) { ImmOp = Src0 ; RegOp = Src1 ; } else if ( Src1 -> isImm ( ) ) { ImmOp = Src1 ; RegOp = Src0 ; } else return std :: make_pair ( nullptr , <STR_LIT> ) ;

int OMod = getOModValue ( Op , ImmOp -> getImm ( ) ) ;


<s> ImmOp = Src1 ; RegOp = Src0 ; } else return std :: make_pair ( nullptr , <STR_LIT> ) ; int OMod = getOModValue ( Op , ImmOp -> getImm ( ) ) ; if ( OMod == <STR_LIT> || TII -> hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) ) return std :: make_pair ( nullptr , <STR_LIT> ) ; return std :: make_pair ( RegOp , OMod ) ; } case <STR_LIT> : case <STR_LIT> : { if ( ( Op == <STR_LIT> && MFI -> getMode ( ) . FP32Denormals ) || ( Op == <STR_LIT> && MFI -> getMode ( ) . FP64FP16Denormals ) ) return std :: make_pair ( nullptr , <STR_LIT> ) ;

const MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ;


<s> continue ; } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( OpToFold . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( OpToFold . getReg ( ) ) ) continue ; MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( Dst . getReg ( ) ) ) continue ;


<s> if ( ! TII -> isFoldableCopy ( MI ) ) { if ( IsIEEEMode || ! tryFoldOMod ( MI ) ) tryFoldClamp ( MI ) ; continue ; } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( OpToFold . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( OpToFold . getReg ( ) ) ) continue ; MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( Dst . getReg ( ) ) ) continue ; foldInstOperand ( MI , OpToFold ) ; } }


<s> MachineInstr & MI = * I ; if ( ! isSafeToFold ( MI . getOpcode ( ) ) ) continue ; unsigned OpSize = TII -> getOpSize ( MI , <NUM_LIT> ) ; MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( FoldingImm && ! TII -> isInlineConstant ( OpToFold , OpSize ) && ! MRI . hasOneUse ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; if ( OpToFold . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( OpToFold . getReg ( ) ) ) continue ; MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( Dst . getReg ( ) ) ) continue ; SmallVector < MachineInstr * , <NUM_LIT> > CopiesToReplace ; std :: vector < FoldCandidate > FoldList ; for ( MachineRegisterInfo :: use_iterator Use = MRI . use_begin ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) , E = MRI . use_end ( ) ;


<s> for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; if ( ! isSafeToFold ( MI . getOpcode ( ) ) ) continue ; unsigned OpSize = TII -> getOpSize ( MI , <NUM_LIT> ) ; MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( FoldingImm && ! TII -> isInlineConstant ( OpToFold , OpSize ) && ! MRI . hasOneUse ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; if ( OpToFold . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( OpToFold . getReg ( ) ) ) continue ; MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; if ( Dst . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( Dst . getReg ( ) ) ) continue ; SmallVector < MachineInstr * , <NUM_LIT> > CopiesToReplace ; std :: vector < FoldCandidate > FoldList ; for ( MachineRegisterInfo :: use_iterator Use = MRI . use_begin ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) , E = MRI . use_end ( ) ; Use != E ; ++ Use ) { MachineInstr * UseMI = Use -> getParent ( ) ; foldOperand ( OpToFold , UseMI , Use . getOperandNo ( ) , FoldList , CopiesToReplace , TII , TRI , MRI ) ; } for ( MachineInstr * Copy : CopiesToReplace ) Copy -> addImplicitDefUseOperands ( MF ) ; for ( FoldCandidate & Fold : FoldList ) { if ( updateOperand ( Fold , TRI ) ) { if ( ! Fold . isImm ( ) ) { assert ( Fold . OpToFold && Fold . OpToFold -> isReg ( ) ) ; MRI . clearKillFlags ( Fold . OpToFold -> getReg ( ) ) ; }

DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << Fold . UseOpNo << <STR_LIT> << * Fold . UseMI << '\n' ) ;


<s> bool FoldAsMAD = tryAddToFoldList ( FoldList , MI , OpNo , OpToFold , TII ) ; if ( FoldAsMAD ) { MI -> untieRegOperand ( OpNo ) ; return true ; } MI -> setDesc ( TII -> get ( Opc ) ) ; } if ( isUseMIInFoldList ( FoldList , MI ) ) return false ; unsigned CommuteIdx0 = TargetInstrInfo :: CommuteAnyOperandIndex ; unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( * MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) {

if ( CommuteIdx0 == OpNo ) OpNo = CommuteIdx1 ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < MachineDominatorTree > ( ) ; AU . setPreservesCFG ( ) ;

MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override { AU . addRequired < MachineDominatorTree > ( ) ;


<s> if ( isUseMIInFoldList ( FoldList , MI ) ) return false ; unsigned CommuteIdx0 ; unsigned CommuteIdx1 ; bool CanCommute = TII -> findCommutedOpIndices ( MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) OpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) OpNo = CommuteIdx0 ; } if ( ! CanCommute || ! TII -> commuteInstruction ( MI ) ) return false ; if ( ! TII -> isOperandLegal ( MI , OpNo , OpToFold ) ) return false ;


<s> int NonInlineUseOpNo = - <NUM_LIT> ; MachineRegisterInfo :: use_iterator NextUse , NextInstUse ; for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; Use = NextUse ) { NextUse = std :: next ( Use ) ; MachineInstr * UseMI = Use -> getParent ( ) ; unsigned OpNo = Use . getOperandNo ( ) ; if ( OpToFold . isImm ( ) && tryConstantFoldOp ( * MRI , TII , UseMI , & OpToFold ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * UseMI << '\n' ) ; NextUse = MRI -> use_begin ( Dst . getReg ( ) ) ; continue ; } if ( isInlineConstantIfFolded ( TII , * UseMI , OpNo , OpToFold ) ) { foldOperand ( OpToFold , UseMI , OpNo , FoldList , CopiesToReplace ) ; } else { if ( ++ NumLiteralUses == <NUM_LIT> ) { NonInlineUse = & * Use ; NonInlineUseOpNo = OpNo ; } } } if ( NumLiteralUses == <NUM_LIT> ) { MachineInstr * UseMI = NonInlineUse -> getParent ( ) ; foldOperand ( OpToFold , UseMI , NonInlineUseOpNo , FoldList , CopiesToReplace ) ; } } else { for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; ++ Use ) { MachineInstr * UseMI = Use -> getParent ( ) ; foldOperand ( OpToFold , UseMI , Use . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } }

MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ;


<s> SmallVector < MachineInstr * , <NUM_LIT> > CopiesToReplace ; SmallVector < FoldCandidate , <NUM_LIT> > FoldList ; MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) ; if ( FoldingImm ) { unsigned NumLiteralUses = <NUM_LIT> ; MachineOperand * NonInlineUse = nullptr ; int NonInlineUseOpNo = - <NUM_LIT> ; MachineRegisterInfo :: use_iterator NextUse , NextInstUse ; for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; Use = NextUse ) { NextUse = std :: next ( Use ) ; MachineInstr * UseMI = Use -> getParent ( ) ; unsigned OpNo = Use . getOperandNo ( ) ; if ( OpToFold . isImm ( ) && tryConstantFoldOp ( * MRI , TII , UseMI , & OpToFold ) ) { DEBUG ( dbgs ( ) << <STR_LIT> << * UseMI << '\n' ) ; NextUse = MRI -> use_begin ( Dst . getReg ( ) ) ; continue ; } if ( isInlineConstantIfFolded ( TII , * UseMI , OpNo , OpToFold ) ) { foldOperand ( OpToFold , UseMI , OpNo , FoldList , CopiesToReplace ) ; } else { if ( ++ NumLiteralUses == <NUM_LIT> ) { NonInlineUse = & * Use ; NonInlineUseOpNo = OpNo ; }


<s> if ( ! isUseSafeToFold ( * UseMI , UseOp ) ) return ; if ( UseOp . isReg ( ) && OpToFold . isReg ( ) ) { if ( UseOp . isImplicit ( ) || UseOp . getSubReg ( ) != <STR_LIT> ) return ; if ( UseOp . isTied ( ) && OpToFold . getSubReg ( ) != <STR_LIT> ) return ; } if ( UseMI -> isRegSequence ( ) ) { unsigned RegSeqDstReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned RegSeqDstSubReg = UseMI -> getOperand ( UseOpIdx + <NUM_LIT> ) . getImm ( ) ; for ( MachineRegisterInfo :: use_iterator RSUse = MRI -> use_begin ( RegSeqDstReg ) , RSE = MRI -> use_end ( ) ; RSUse != RSE ; ++ RSUse ) { MachineInstr * RSUseMI = RSUse -> getParent ( ) ; if ( RSUse -> getSubReg ( ) != RegSeqDstSubReg ) continue ; foldOperand ( OpToFold , RSUseMI , RSUse . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } return ; } bool FoldingImm = OpToFold . isImm ( ) ; if ( FoldingImm && UseMI -> isCopy ( ) ) { unsigned DestReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * DestRC = TargetRegisterInfo :: isVirtualRegister ( DestReg ) ? MRI -> getRegClass ( DestReg ) : TRI -> getPhysRegClass ( DestReg ) ; unsigned MovOp = TII -> getMovOpcode ( DestRC ) ; if ( MovOp == <STR_LIT> ) return ; UseMI -> setDesc ( TII -> get ( MovOp ) ) ; CopiesToReplace . push_back ( UseMI ) ;


<s> unsigned RegSeqDstSubReg = UseMI -> getOperand ( UseOpIdx + <NUM_LIT> ) . getImm ( ) ; for ( MachineRegisterInfo :: use_iterator RSUse = MRI -> use_begin ( RegSeqDstReg ) , RSE = MRI -> use_end ( ) ; RSUse != RSE ; ++ RSUse ) { MachineInstr * RSUseMI = RSUse -> getParent ( ) ; if ( RSUse -> getSubReg ( ) != RegSeqDstSubReg ) continue ; foldOperand ( OpToFold , RSUseMI , RSUse . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } return ; } bool FoldingImm = OpToFold . isImm ( ) ; if ( FoldingImm && UseMI -> isCopy ( ) ) { unsigned DestReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * DestRC = TargetRegisterInfo :: isVirtualRegister ( DestReg ) ? MRI -> getRegClass ( DestReg ) : TRI -> getPhysRegClass ( DestReg ) ; unsigned MovOp = TII -> getMovOpcode ( DestRC ) ; if ( MovOp == <STR_LIT> ) return ; UseMI -> setDesc ( TII -> get ( MovOp ) ) ;


<s> bool <ISA_LIT>FoldOperands :: runOnMachineFunction ( MachineFunction & MF ) { if ( skipFunction ( * MF . getFunction ( ) ) ) return false ; MRI = & MF . getRegInfo ( ) ; ST = & MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; TII = ST -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; bool IsIEEEMode = ST -> enableIEEEBit ( MF ) || ! MFI -> hasNoSignedZerosFPMath ( ) ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; tryFoldInst ( TII , & MI ) ; if ( ! TII -> isFoldableCopy ( MI ) ) { if ( IsIEEEMode || ! tryFoldOMod ( MI ) ) tryFoldClamp ( MI ) ; continue ; } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ;

bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) ;


<s> TRI = & TII -> getRegisterInfo ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; bool IsIEEEMode = ST -> enableIEEEBit ( MF ) || ! MFI -> hasNoSignedZerosFPMath ( ) ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; tryFoldInst ( TII , & MI ) ; if ( ! TII -> isFoldableCopy ( MI ) ) { if ( IsIEEEMode || ! tryFoldOMod ( MI ) ) tryFoldClamp ( MI ) ; continue ; } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ;

if ( OpToFold . isReg ( ) && ! TargetRegisterInfo :: isVirtualRegister ( OpToFold . getReg ( ) ) ) continue ;


<s> void <ISA_LIT>FoldOperands :: foldInstOperand ( MachineInstr & MI , MachineOperand & OpToFold ) const { SmallVector < MachineInstr * , <NUM_LIT> > CopiesToReplace ; SmallVector < FoldCandidate , <NUM_LIT> > FoldList ; MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) ; if ( FoldingImm ) { unsigned NumLiteralUses = <NUM_LIT> ; MachineOperand * NonInlineUse = nullptr ; int NonInlineUseOpNo = - <NUM_LIT> ; MachineRegisterInfo :: use_iterator NextUse ; for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; Use = NextUse ) { NextUse = std :: next ( Use ) ; MachineInstr * UseMI = Use -> getParent ( ) ; unsigned OpNo = Use . getOperandNo ( ) ; if ( OpToFold . isImm ( ) && tryConstantFoldOp ( * MRI , TII , UseMI , & OpToFold ) ) {

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * UseMI << '\n' ) ;


<s> for ( auto U : UsesToProcess ) { MachineInstr * UseMI = U -> getParent ( ) ; foldOperand ( OpToFold , UseMI , U . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } } MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; for ( MachineInstr * Copy : CopiesToReplace ) Copy -> addImplicitDefUseOperands ( * MF ) ; for ( FoldCandidate & Fold : FoldList ) { if ( updateOperand ( Fold , * TII , * TRI , * ST ) ) { if ( Fold . isReg ( ) ) { assert ( Fold . OpToFold && Fold . OpToFold -> isReg ( ) ) ; MRI -> clearKillFlags ( Fold . OpToFold -> getReg ( ) ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << static_cast < int > ( Fold . UseOpNo ) << <STR_LIT> << * Fold . UseMI << '\n' ) ; tryFoldInst ( TII , Fold . UseMI ) ; } else if ( Fold . isCommuted ( ) ) {

TII -> commuteInstruction ( * Fold . UseMI , false ) ;


<s> Next = std :: next ( I ) ; MachineInstr & MI = * I ; tryFoldInst ( TII , & MI ) ; if ( ! TII -> isFoldableCopy ( MI ) ) { if ( IsIEEEMode || ( ! HasNSZ && ! MI . getFlag ( MachineInstr :: FmNsz ) ) || ! tryFoldOMod ( MI ) ) tryFoldClamp ( MI ) ; continue ; } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ;


<s> bool <ISA_LIT>FoldOperands :: runOnMachineFunction ( MachineFunction & MF ) { if ( skipFunction ( MF . getFunction ( ) ) ) return false ; MRI = & MF . getRegInfo ( ) ; ST = & MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST -> getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; bool IsIEEEMode = MFI -> getMode ( ) . IEEE ; bool HasNSZ = MFI -> hasNoSignedZerosFPMath ( ) ; for ( MachineBasicBlock * MBB : depth_first ( & MF ) ) { MachineBasicBlock :: iterator I , Next ; for ( I = MBB -> begin ( ) ; I != MBB -> end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ;


<s> break ; } } } if ( ( Fold . isImm ( ) || Fold . isFI ( ) ) && Fold . needsShrink ( ) ) { MachineBasicBlock * MBB = MI -> getParent ( ) ; auto Liveness = MBB -> computeRegisterLiveness ( & TRI , <STR_LIT> , MI ) ; if ( Liveness != MachineBasicBlock :: LQR_Dead ) return false ; MachineRegisterInfo & MRI = MBB -> getParent ( ) -> getRegInfo ( ) ; int Op32 = Fold . getShrinkOpcode ( ) ; MachineOperand & Dst0 = MI -> getOperand ( <NUM_LIT> ) ; MachineOperand & Dst1 = MI -> getOperand ( <NUM_LIT> ) ; assert ( Dst0 . isDef ( ) && Dst1 . isDef ( ) ) ; bool HaveNonDbgCarryUse = ! MRI . use_nodbg_empty ( Dst1 . getReg ( ) ) ; const TargetRegisterClass * Dst0RC = MRI . getRegClass ( Dst0 . getReg ( ) ) ; unsigned NewReg0 = MRI . createVirtualRegister ( Dst0RC ) ; MachineInstr * Inst32 = TII . buildShrunkInst ( * MI , Op32 ) ; if ( HaveNonDbgCarryUse ) { BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII . get ( <STR_LIT> ) , Dst1 . getReg ( ) ) . addReg ( <STR_LIT> , RegState :: Kill ) ; } Dst0 . setReg ( NewReg0 ) ; for ( unsigned I = MI -> getNumOperands ( ) - <NUM_LIT> ; I > <NUM_LIT> ; -- I ) MI -> RemoveOperand ( I ) ; MI -> setDesc ( TII . get ( <STR_LIT> ) ) ; if ( Fold . isCommuted ( ) ) TII . commuteInstruction ( * Inst32 , false ) ; return true ; } assert ( ! Fold . needsShrink ( ) && <STR_LIT> ) ; if ( Fold . isImm ( ) ) { Old . ChangeToImmediate ( Fold . ImmToFold ) ; return true ; } if ( Fold . isFI ( ) ) { Old . ChangeToFrameIndex ( Fold . FrameIndexToFold ) ;


<s> } if ( UseMI -> getOpcode ( ) == <STR_LIT> ) { unsigned DestReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * DestRC = TargetRegisterInfo :: isVirtualRegister ( DestReg ) ? MRI . getRegClass ( DestReg ) : TRI . getPhysRegClass ( DestReg ) ; unsigned MovOp = TII -> getMovOpcode ( DestRC ) ; if ( MovOp == <STR_LIT> ) return ; UseMI -> setDesc ( TII -> get ( MovOp ) ) ; CopiesToReplace . push_back ( UseMI ) ; } } if ( UseMI -> getOpcode ( ) == <STR_LIT> ) { unsigned RegSeqDstReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned RegSeqDstSubReg = UseMI -> getOperand ( UseOpIdx + <NUM_LIT> ) . getImm ( ) ; for ( MachineRegisterInfo :: use_iterator RSUse = MRI . use_begin ( RegSeqDstReg ) , RSE = MRI . use_end ( ) ; RSUse != RSE ; ++ RSUse ) { MachineInstr * RSUseMI = RSUse -> getParent ( ) ; if ( RSUse -> getSubReg ( ) != RegSeqDstSubReg ) continue ; foldOperand ( OpToFold , RSUseMI , RSUse . getOperandNo ( ) , FoldList , CopiesToReplace , TII , TRI , MRI ) ; } return ; } const MCInstrDesc & UseDesc = UseMI -> getDesc ( ) ; if ( UseDesc . isVariadic ( ) || UseDesc . OpInfo [ UseOpIdx ] . RegClass == - <NUM_LIT> ) return ; if ( FoldingImm ) {

MachineOperand ImmOp = MachineOperand :: CreateImm ( Imm . getSExtValue ( ) ) ;


<s> const TargetRegisterClass * FoldRC = TRI . getRegClass ( FoldDesc . OpInfo [ <NUM_LIT> ] . RegClass ) ; if ( FoldRC -> getSize ( ) == <NUM_LIT> && UseOp . getSubReg ( ) ) { if ( UseRC -> getSize ( ) != <NUM_LIT> ) return ; if ( UseOp . getSubReg ( ) == <STR_LIT> ) { Imm = Imm . getLoBits ( <NUM_LIT> ) ; } else { assert ( UseOp . getSubReg ( ) == <STR_LIT> ) ; Imm = Imm . getHiBits ( <NUM_LIT> ) ; } } if ( UseMI -> getOpcode ( ) == <STR_LIT> ) { unsigned DestReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * DestRC = TargetRegisterInfo :: isVirtualRegister ( DestReg ) ? MRI . getRegClass ( DestReg ) : TRI . getPhysRegClass ( DestReg ) ; unsigned MovOp = TII -> getMovOpcode ( DestRC ) ; if ( MovOp == <STR_LIT> ) return ; UseMI -> setDesc ( TII -> get ( MovOp ) ) ; CopiesToReplace . push_back ( UseMI ) ; }


<s> } if ( UseMI -> getOpcode ( ) == <STR_LIT> ) { unsigned DestReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * DestRC = TargetRegisterInfo :: isVirtualRegister ( DestReg ) ? MRI . getRegClass ( DestReg ) : TRI . getPhysRegClass ( DestReg ) ; unsigned MovOp = TII -> getMovOpcode ( DestRC ) ; if ( MovOp == <STR_LIT> ) return ; UseMI -> setDesc ( TII -> get ( MovOp ) ) ; } } const MCInstrDesc & UseDesc = UseMI -> getDesc ( ) ; if ( UseDesc . isVariadic ( ) || UseDesc . OpInfo [ UseOpIdx ] . RegClass == - <NUM_LIT> ) return ; if ( FoldingImm ) { MachineOperand ImmOp = MachineOperand :: CreateImm ( Imm . getSExtValue ( ) ) ; tryAddToFoldList ( FoldList , UseMI , UseOpIdx , & ImmOp , TII ) ; return ; } tryAddToFoldList ( FoldList , UseMI , UseOpIdx , & OpToFold , TII ) ; return ;


<s> unsigned UseReg = UseOp . getReg ( ) ; const TargetRegisterClass * UseRC = TargetRegisterInfo :: isVirtualRegister ( UseReg ) ? MRI . getRegClass ( UseReg ) : TRI . getPhysRegClass ( UseReg ) ; Imm = APInt ( <NUM_LIT> , OpToFold . getImm ( ) ) ; const MCInstrDesc & FoldDesc = TII -> get ( OpToFold . getParent ( ) -> getOpcode ( ) ) ; const TargetRegisterClass * FoldRC = TRI . getRegClass ( FoldDesc . OpInfo [ <NUM_LIT> ] . RegClass ) ; if ( FoldRC -> getSize ( ) == <NUM_LIT> && UseOp . getSubReg ( ) ) { if ( UseRC -> getSize ( ) != <NUM_LIT> ) return ; if ( UseOp . getSubReg ( ) == <STR_LIT> ) { Imm = Imm . getLoBits ( <NUM_LIT> ) ; } else { assert ( UseOp . getSubReg ( ) == <STR_LIT> ) ; Imm = Imm . getHiBits ( <NUM_LIT> ) ; } } if ( UseMI -> getOpcode ( ) == <STR_LIT> ) { unsigned DestReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * DestRC = TargetRegisterInfo :: isVirtualRegister ( DestReg ) ? MRI . getRegClass ( DestReg ) : TRI . getPhysRegClass ( DestReg ) ; unsigned MovOp = TII -> getMovOpcode ( DestRC ) ; if ( MovOp == <STR_LIT> ) return ; UseMI -> setDesc ( TII -> get ( MovOp ) ) ; } } const MCInstrDesc & UseDesc = UseMI -> getDesc ( ) ; if ( UseDesc . isVariadic ( ) || UseDesc . OpInfo [ UseOpIdx ] . RegClass == - <NUM_LIT> ) return ; if ( FoldingImm ) { MachineOperand ImmOp = MachineOperand :: CreateImm ( Imm . getSExtValue ( ) ) ; tryAddToFoldList ( FoldList , UseMI , UseOpIdx , & ImmOp , TII ) ; return ; } tryAddToFoldList ( FoldList , UseMI , UseOpIdx , & OpToFold , TII ) ;


<s> bool FoldingImm = OpToFold . isImm ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ; if ( FoldingImm && ! TII -> isInlineConstant ( OpToFold , OpSize ) && ! MRI . hasOneUse ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) continue ; if ( OpToFold . isReg ( ) && ( ! TargetRegisterInfo :: isVirtualRegister ( OpToFold . getReg ( ) ) || OpToFold . getSubReg ( ) ) ) continue ; std :: vector < FoldCandidate > FoldList ; for ( MachineRegisterInfo :: use_iterator Use = MRI . use_begin ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) , E = MRI . use_end ( ) ; Use != E ; ++ Use ) {

MachineInstr * UseMI = Use -> getParent ( ) ;


<s> const MachineOperand * ClampSrc = isClamp ( MI ) ; if ( ! ClampSrc || ! MRI -> hasOneNonDBGUser ( ClampSrc -> getReg ( ) ) ) return false ; MachineInstr * Def = MRI -> getVRegDef ( ClampSrc -> getReg ( ) ) ; if ( TII -> getClampMask ( * Def ) != TII -> getClampMask ( MI ) ) return false ; MachineOperand * DefClamp = TII -> getNamedOperand ( * Def , <STR_LIT> :: clamp ) ;


<s> const MachineOperand * RegOp ; int OMod ; std :: tie ( RegOp , OMod ) = isOMod ( MI ) ; if ( OMod == <STR_LIT> || ! RegOp -> isReg ( ) || RegOp -> getSubReg ( ) != <STR_LIT> || ! MRI -> hasOneNonDBGUser ( RegOp -> getReg ( ) ) ) return false ; MachineInstr * Def = MRI -> getVRegDef ( RegOp -> getReg ( ) ) ; MachineOperand * DefOMod = TII -> getNamedOperand ( * Def , <STR_LIT> :: omod ) ; if ( ! DefOMod || DefOMod -> getImm ( ) != <STR_LIT> ) return false ;

if ( TII -> hasModifiersSet ( * Def , <STR_LIT> :: clamp ) ) return false ;


<s> int OMod ; std :: tie ( RegOp , OMod ) = isOMod ( MI ) ; if ( OMod == <STR_LIT> || ! RegOp -> isReg ( ) || RegOp -> getSubReg ( ) != <STR_LIT> || ! MRI -> hasOneNonDBGUser ( RegOp -> getReg ( ) ) ) return false ; MachineInstr * Def = MRI -> getVRegDef ( RegOp -> getReg ( ) ) ; MachineOperand * DefOMod = TII -> getNamedOperand ( * Def , <STR_LIT> :: omod ) ; if ( ! DefOMod || DefOMod -> getImm ( ) != <STR_LIT> ) return false ;


<s> if ( ! Def || ! Def -> isRegSequence ( ) ) return false ; int64_t Imm ; MachineOperand * Op ; for ( unsigned I = <NUM_LIT> , E = Def -> getNumExplicitOperands ( ) ; I < E ; I += <NUM_LIT> ) { const MachineOperand & Sub = Def -> getOperand ( I ) ; if ( ! Sub . isReg ( ) || Sub . getSubReg ( ) ) return false ; MachineInstr * SubDef = MRI . getUniqueVRegDef ( Sub . getReg ( ) ) ; while ( SubDef && ! SubDef -> isMoveImmediate ( ) && ! SubDef -> getOperand ( <NUM_LIT> ) . isImm ( ) && TII -> isFoldableCopy ( * SubDef ) ) SubDef = MRI . getUniqueVRegDef ( SubDef -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( ! SubDef || ! SubDef -> isMoveImmediate ( ) || ! SubDef -> getOperand ( <NUM_LIT> ) . isImm ( ) ) return false ; Op = & SubDef -> getOperand ( <NUM_LIT> ) ; auto SubImm = Op -> getImm ( ) ; if ( I == <NUM_LIT> ) { if ( ! TII -> isInlineConstant ( SubDef -> getOperand ( <NUM_LIT> ) , OpTy ) ) return false ; Imm = SubImm ; continue ;


<s> if ( ! Def || ! Def -> isRegSequence ( ) ) return false ; int64_t Imm ; MachineOperand * Op ; for ( unsigned I = <NUM_LIT> , E = Def -> getNumExplicitOperands ( ) ; I < E ; I += <NUM_LIT> ) { const MachineOperand & Sub = Def -> getOperand ( I ) ; if ( ! Sub . isReg ( ) || Sub . getSubReg ( ) ) return false ; MachineInstr * SubDef = MRI . getUniqueVRegDef ( Sub . getReg ( ) ) ; while ( SubDef && ! SubDef -> isMoveImmediate ( ) && ! SubDef -> getOperand ( <NUM_LIT> ) . isImm ( ) && TII -> isFoldableCopy ( * SubDef ) ) SubDef = MRI . getUniqueVRegDef ( SubDef -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; if ( ! SubDef || ! SubDef -> isMoveImmediate ( ) || ! SubDef -> getOperand ( <NUM_LIT> ) . isImm ( ) ) return false ; Op = & SubDef -> getOperand ( <NUM_LIT> ) ; auto SubImm = Op -> getImm ( ) ; if ( I == <NUM_LIT> ) { if ( ! TII -> isInlineConstant ( SubDef -> getOperand ( <NUM_LIT> ) , OpTy ) ) return false ; Imm = SubImm ; continue ; }


<s> } Mod . setImm ( Mod . getImm ( ) & ~ <STR_LIT> ) ; Old . ChangeToImmediate ( Fold . ImmToFold & <NUM_LIT> ) ; return true ; } break ; default : break ; } } } if ( ( Fold . isImm ( ) || Fold . isFI ( ) || Fold . isGlobal ( ) ) && Fold . needsShrink ( ) ) { MachineBasicBlock * MBB = MI -> getParent ( ) ; auto Liveness = MBB -> computeRegisterLiveness ( & TRI , <STR_LIT> , MI ) ; if ( Liveness != MachineBasicBlock :: LQR_Dead ) return false ; MachineRegisterInfo & MRI = MBB -> getParent ( ) -> getRegInfo ( ) ; int Op32 = Fold . getShrinkOpcode ( ) ; MachineOperand & Dst0 = MI -> getOperand ( <NUM_LIT> ) ; MachineOperand & Dst1 = MI -> getOperand ( <NUM_LIT> ) ; assert ( Dst0 . isDef ( ) && Dst1 . isDef ( ) ) ; bool HaveNonDbgCarryUse = ! MRI . use_nodbg_empty ( Dst1 . getReg ( ) ) ; const TargetRegisterClass * Dst0RC = MRI . getRegClass ( Dst0 . getReg ( ) ) ; unsigned NewReg0 = MRI . createVirtualRegister ( Dst0RC ) ; MachineInstr * Inst32 = TII . buildShrunkInst ( * MI , Op32 ) ; if ( HaveNonDbgCarryUse ) { BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII . get ( <STR_LIT> ) , Dst1 . getReg ( ) ) . addReg ( <STR_LIT> , RegState :: Kill ) ; } Dst0 . setReg ( NewReg0 ) ; for ( unsigned I = MI -> getNumOperands ( ) - <NUM_LIT> ; I > <NUM_LIT> ; -- I ) MI -> RemoveOperand ( I ) ; MI -> setDesc ( TII . get ( <STR_LIT> ) ) ; if ( Fold . isCommuted ( ) ) TII . commuteInstruction ( * Inst32 , false ) ; return true ; } assert ( ! Fold . needsShrink ( ) && <STR_LIT> ) ; if ( Fold . isImm ( ) ) { Old . ChangeToImmediate ( Fold . ImmToFold ) ; return true ; } if ( Fold . isGlobal ( ) ) { Old . ChangeToGA ( Fold . OpToFold -> getGlobal ( ) , Fold . OpToFold -> getOffset ( ) , Fold . OpToFold -> getTargetFlags ( ) ) ; return true ; } if ( Fold . isFI ( ) ) { Old . ChangeToFrameIndex ( Fold . FrameIndexToFold ) ; return true ; } MachineOperand * New = Fold . OpToFold ; Old . substVirtReg ( New -> getReg ( ) , New -> getSubReg ( ) , TRI ) ;

Old . setIsUndef ( New -> isUndef ( ) ) ;


<s> } break ; default : break ; } } } if ( ( Fold . isImm ( ) || Fold . isFI ( ) ) && Fold . needsShrink ( ) ) { MachineBasicBlock * MBB = MI -> getParent ( ) ; auto Liveness = MBB -> computeRegisterLiveness ( & TRI , <STR_LIT> , MI ) ; if ( Liveness != MachineBasicBlock :: LQR_Dead ) return false ; MachineRegisterInfo & MRI = MBB -> getParent ( ) -> getRegInfo ( ) ; int Op32 = Fold . getShrinkOpcode ( ) ; MachineOperand & Dst0 = MI -> getOperand ( <NUM_LIT> ) ; MachineOperand & Dst1 = MI -> getOperand ( <NUM_LIT> ) ; assert ( Dst0 . isDef ( ) && Dst1 . isDef ( ) ) ; bool HaveNonDbgCarryUse = ! MRI . use_nodbg_empty ( Dst1 . getReg ( ) ) ; const TargetRegisterClass * Dst0RC = MRI . getRegClass ( Dst0 . getReg ( ) ) ; unsigned NewReg0 = MRI . createVirtualRegister ( Dst0RC ) ; MachineInstr * Inst32 = TII . buildShrunkInst ( * MI , Op32 ) ; if ( HaveNonDbgCarryUse ) { BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII . get ( <STR_LIT> ) , Dst1 . getReg ( ) ) . addReg ( <STR_LIT> , RegState :: Kill ) ; } Dst0 . setReg ( NewReg0 ) ;

for ( unsigned I = MI -> getNumOperands ( ) - <NUM_LIT> ; I > <NUM_LIT> ; -- I ) MI -> RemoveOperand ( I ) ;


<s> if ( Fold . isImm ( ) ) { if ( MI -> getDesc ( ) . TSFlags & <STR_LIT> && ! ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) && <STR_LIT> ( Fold . ImmToFold , ST . hasInv2PiInlineImm ( ) ) ) { unsigned Opcode = MI -> getOpcode ( ) ; int OpNo = MI -> getOperandNo ( & Old ) ; int ModIdx = - <NUM_LIT> ; if ( OpNo == <STR_LIT> ( Opcode , <STR_LIT> :: src0 ) ) ModIdx = <STR_LIT> :: src0_modifiers ; else if ( OpNo == <STR_LIT> ( Opcode , <STR_LIT> :: src1 ) ) ModIdx = <STR_LIT> :: src1_modifiers ; else if ( OpNo == <STR_LIT> ( Opcode , <STR_LIT> :: src2 ) ) ModIdx = <STR_LIT> :: src2_modifiers ; assert ( ModIdx != - <NUM_LIT> ) ; ModIdx = <STR_LIT> ( Opcode , ModIdx ) ; MachineOperand & Mod = MI -> getOperand ( ModIdx ) ; unsigned Val = Mod . getImm ( ) ; if ( ! ( Val & <STR_LIT> ) && ( Val & <STR_LIT> ) ) { switch ( TII . get ( Opcode ) . OpInfo [ OpNo ] . OperandType ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( ! isUInt < <NUM_LIT> > ( Fold . ImmToFold ) ) { if ( ! ( Fold . ImmToFold & <NUM_LIT> ) ) { Mod . setImm ( Mod . getImm ( ) | <STR_LIT> ) ; Mod . setImm ( Mod . getImm ( ) & ~ <STR_LIT> ) ; Old . ChangeToImmediate ( ( Fold . ImmToFold >> <NUM_LIT> ) & <NUM_LIT> ) ; return true ; } Mod . setImm ( Mod . getImm ( ) & ~ <STR_LIT> ) ; Old . ChangeToImmediate ( Fold . ImmToFold & <NUM_LIT> ) ; return true ; } break ; default : break ; } } } } if ( ( Fold . isImm ( ) || Fold . isFI ( ) || Fold . isGlobal ( ) ) && Fold . needsShrink ( ) ) { MachineBasicBlock * MBB = MI -> getParent ( ) ; auto Liveness = MBB -> computeRegisterLiveness ( & TRI , <STR_LIT> , MI , <NUM_LIT> ) ; if ( Liveness != MachineBasicBlock :: LQR_Dead ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> ) ; return false ; } MachineRegisterInfo & MRI = MBB -> getParent ( ) -> getRegInfo ( ) ; int Op32 = Fold . getShrinkOpcode ( ) ; MachineOperand & Dst0 = MI -> getOperand ( <NUM_LIT> ) ; MachineOperand & Dst1 = MI -> getOperand ( <NUM_LIT> ) ; assert ( Dst0 . isDef ( ) && Dst1 . isDef ( ) ) ; bool HaveNonDbgCarryUse = ! MRI . use_nodbg_empty ( Dst1 . getReg ( ) ) ; const TargetRegisterClass * Dst0RC = MRI . getRegClass ( Dst0 . getReg ( ) ) ; Register NewReg0 = MRI . createVirtualRegister ( Dst0RC ) ;

MachineInstr * Inst32 = TII . buildShrunkInst ( * MI , Op32 ) ;


<s> switch ( Opc ) { case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; } return <STR_LIT> ;


<s> case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> :


<s> if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) && OpToFold -> isImm ( ) ) { MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0 ; MachineOperand & OtherOp = MI -> getOperand ( OtherIdx ) ; if ( ! OtherOp . isReg ( ) || ! TII -> getRegisterInfo ( ) . isVGPR ( MRI , OtherOp . getReg ( ) ) ) return false ; assert ( MI -> getOperand ( <NUM_LIT> ) . isDef ( ) ) ; unsigned MaybeCommutedOpc = MI -> getOpcode ( ) ; int Op32 = <STR_LIT> ( MaybeCommutedOpc ) ; FoldList . push_back ( FoldCandidate ( MI , CommuteOpNo , OpToFold , true , Op32 ) ) ; return true ; } TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ; return false ; } FoldList . push_back ( FoldCandidate ( MI , CommuteOpNo , OpToFold , true ) ) ; return true ; }

FoldList . push_back ( FoldCandidate ( MI , OpNo , OpToFold ) ) ;


<s> } if ( isUseMIInFoldList ( FoldList , MI ) ) return false ; unsigned CommuteOpNo = OpNo ; unsigned CommuteIdx0 = TargetInstrInfo :: CommuteAnyOperandIndex ; unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( * MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) CommuteOpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) CommuteOpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , CommuteOpNo , OpToFold ) ) { if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) && OpToFold -> isImm ( ) ) { MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0 ; MachineOperand & OtherOp = MI -> getOperand ( OtherIdx ) ; if ( ! OtherOp . isReg ( ) || ! TII -> getRegisterInfo ( ) . isVGPR ( MRI , OtherOp . getReg ( ) ) ) return false ; assert ( MI -> getOperand ( <NUM_LIT> ) . isDef ( ) ) ; unsigned MaybeCommutedOpc = MI -> getOpcode ( ) ; int Op32 = <STR_LIT> ( MaybeCommutedOpc ) ; FoldList . push_back ( FoldCandidate ( MI , CommuteOpNo , OpToFold , true , Op32 ) ) ; return true ; } TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ; return false ; } FoldList . push_back ( FoldCandidate ( MI , CommuteOpNo , OpToFold , true ) ) ; return true ; } FoldList . push_back ( FoldCandidate ( MI , OpNo , OpToFold ) ) ;


<s> if ( skipFunction ( * MF . getFunction ( ) ) ) return false ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; MRI = & MF . getRegInfo ( ) ; TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock & MBB = * BI ;

MachineBasicBlock :: iterator I , Next ;


<s> } MI -> setDesc ( TII -> get ( Opc ) ) ; } if ( Opc == <STR_LIT> && OpToFold -> isImm ( ) ) { MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; appendFoldCandidate ( FoldList , MI , OpNo , OpToFold ) ; return true ; } if ( isUseMIInFoldList ( FoldList , MI ) ) return false ; unsigned CommuteOpNo = OpNo ; unsigned CommuteIdx0 = TargetInstrInfo :: CommuteAnyOperandIndex ; unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( * MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) CommuteOpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) CommuteOpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , CommuteOpNo , OpToFold ) ) { if ( ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) && ( OpToFold -> isImm ( ) || OpToFold -> isFI ( ) || OpToFold -> isGlobal ( ) ) ) { MachineRegisterInfo & MRI = MI -> getParent ( ) -> getParent ( ) -> getRegInfo ( ) ; unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0 ; MachineOperand & OtherOp = MI -> getOperand ( OtherIdx ) ; if ( ! OtherOp . isReg ( ) || ! TII -> getRegisterInfo ( ) . isVGPR ( MRI , OtherOp . getReg ( ) ) ) return false ; assert ( MI -> getOperand ( <NUM_LIT> ) . isDef ( ) ) ; unsigned MaybeCommutedOpc = MI -> getOpcode ( ) ; int Op32 = <STR_LIT> ( MaybeCommutedOpc ) ; appendFoldCandidate ( FoldList , MI , CommuteOpNo , OpToFold , true , Op32 ) ; return true ; } TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ; return false ; }

appendFoldCandidate ( FoldList , MI , CommuteOpNo , OpToFold , true ) ;


<s> return new <ISA_LIT>FoldOperands ( ) ;


<s> return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Result = LHS ^ RHS ; return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Result = LHS << ( RHS & <NUM_LIT> ) ; return true ; case <STR_LIT> : case <STR_LIT> : Result = RHS << ( LHS & <NUM_LIT> ) ; return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Result = LHS >> ( RHS & <NUM_LIT> ) ; return true ; case <STR_LIT> : case <STR_LIT> : Result = RHS >> ( LHS & <NUM_LIT> ) ; return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Result = static_cast < int32_t > ( LHS ) >> ( RHS & <NUM_LIT> ) ; return true ;


<s> ImmToFold = FoldOp -> getImm ( ) ; } else if ( FoldOp -> isFI ( ) ) { FrameIndexToFold = FoldOp -> getIndex ( ) ; } else {


<s> if ( isInlineConstantIfFolded ( TII , * UseMI , OpNo , OpToFold ) ) { foldOperand ( OpToFold , UseMI , OpNo , FoldList , CopiesToReplace ) ; } else { if ( ++ NumLiteralUses == <NUM_LIT> ) { NonInlineUse = & * Use ; NonInlineUseOpNo = OpNo ; } } } if ( NumLiteralUses == <NUM_LIT> ) { MachineInstr * UseMI = NonInlineUse -> getParent ( ) ; foldOperand ( OpToFold , UseMI , NonInlineUseOpNo , FoldList , CopiesToReplace ) ; } } else { for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; ++ Use ) { MachineInstr * UseMI = Use -> getParent ( ) ; foldOperand ( OpToFold , UseMI , Use . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } }


<s> for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; Use = NextUse ) { NextUse = std :: next ( Use ) ; MachineInstr * UseMI = Use -> getParent ( ) ; unsigned OpNo = Use . getOperandNo ( ) ; if ( OpToFold . isImm ( ) && tryConstantFoldOp ( * MRI , TII , UseMI , & OpToFold ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * UseMI << '\n' ) ; NextUse = MRI -> use_begin ( Dst . getReg ( ) ) ; FoldList . clear ( ) ; continue ; } if ( isInlineConstantIfFolded ( TII , * UseMI , OpNo , OpToFold ) ) { foldOperand ( OpToFold , UseMI , OpNo , FoldList , CopiesToReplace ) ; } else { if ( ++ NumLiteralUses == <NUM_LIT> ) { NonInlineUse = & * Use ; NonInlineUseOpNo = OpNo ; } } } if ( NumLiteralUses == <NUM_LIT> ) { MachineInstr * UseMI = NonInlineUse -> getParent ( ) ; foldOperand ( OpToFold , UseMI , NonInlineUseOpNo , FoldList , CopiesToReplace ) ; } } else { for ( MachineRegisterInfo :: use_iterator Use = MRI -> use_begin ( Dst . getReg ( ) ) , E = MRI -> use_end ( ) ; Use != E ; ++ Use ) { MachineInstr * UseMI = Use -> getParent ( ) ; foldOperand ( OpToFold , UseMI , Use . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } } MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ;

for ( MachineInstr * Copy : CopiesToReplace ) Copy -> addImplicitDefUseOperands ( * MF ) ;


<s> unsigned RegSeqDstReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned RegSeqDstSubReg = UseMI -> getOperand ( UseOpIdx + <NUM_LIT> ) . getImm ( ) ; for ( MachineRegisterInfo :: use_iterator RSUse = MRI -> use_begin ( RegSeqDstReg ) , RSE = MRI -> use_end ( ) ; RSUse != RSE ; ++ RSUse ) { MachineInstr * RSUseMI = RSUse -> getParent ( ) ; if ( RSUse -> getSubReg ( ) != RegSeqDstSubReg ) continue ; foldOperand ( OpToFold , RSUseMI , RSUse . getOperandNo ( ) , FoldList , CopiesToReplace ) ; } return ; } bool FoldingImm = OpToFold . isImm ( ) ; if ( FoldingImm && UseMI -> isCopy ( ) ) { unsigned DestReg = UseMI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; const TargetRegisterClass * DestRC = TargetRegisterInfo :: isVirtualRegister ( DestReg ) ? MRI -> getRegClass ( DestReg ) : TRI -> getPhysRegClass ( DestReg ) ; unsigned MovOp = TII -> getMovOpcode ( DestRC ) ; if ( MovOp == <STR_LIT> ) return ; UseMI -> setDesc ( TII -> get ( MovOp ) ) ; CopiesToReplace . push_back ( UseMI ) ; } else { const MCInstrDesc & UseDesc = UseMI -> getDesc ( ) ; if ( UseDesc . isVariadic ( ) || UseOp . isImplicit ( ) || UseDesc . OpInfo [ UseOpIdx ] . RegClass == - <NUM_LIT> ) return ; } if ( ! FoldingImm ) { tryAddToFoldList ( FoldList , UseMI , UseOpIdx , & OpToFold , TII ) ;


<s> static MachineOperand * getImmOrMaterializedImm ( MachineRegisterInfo & MRI , MachineOperand & Op ) { if ( Op . isReg ( ) ) { if ( Op . getSubReg ( ) != <STR_LIT> || ! TargetRegisterInfo :: isVirtualRegister ( Op . getReg ( ) ) ) return & Op ; MachineInstr * Def = MRI . getVRegDef ( Op . getReg ( ) ) ; if ( Def && Def -> isMoveImmediate ( ) ) { MachineOperand & ImmSrc = Def -> getOperand ( <NUM_LIT> ) ; if ( ImmSrc . isImm ( ) ) return & ImmSrc ; } } return & Op ;


<s> return IsScalar ? <STR_LIT> : <STR_LIT> ;


<s> static int getOModValue ( unsigned Opc , int64_t Val ) { switch ( Opc ) { case <STR_LIT> : { switch ( static_cast < uint32_t > ( Val ) ) { case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ; default : return <STR_LIT> ; } } case <STR_LIT> : { switch ( static_cast < uint16_t > ( Val ) ) { case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ;


<s> I != E ; ++ I ) { if ( ++ Count > <NUM_LIT> ) return false ; } return true ;


<s> static bool hasOneNonDBGUseInst ( const MachineRegisterInfo & MRI , unsigned Reg ) { int Count = <NUM_LIT> ; for ( auto I = MRI . use_instr_nodbg_begin ( Reg ) , E = MRI . use_instr_nodbg_end ( ) ; I != E ; ++ I ) { if ( ++ Count > <NUM_LIT> ) return false ; }


<s> unsigned Op = MI . getOpcode ( ) ; switch ( Op ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { if ( ! TII -> getNamedOperand ( MI , <STR_LIT> :: clamp ) -> getImm ( ) ) return nullptr ; const MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( ! Src0 -> isReg ( ) || ! Src1 -> isReg ( ) || Src0 -> getReg ( ) != Src1 -> getReg ( ) || Src0 -> getSubReg ( ) != Src1 -> getSubReg ( ) || Src0 -> getSubReg ( ) != <STR_LIT> ) return nullptr ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) ) return nullptr ; unsigned Src0Mods = TII -> getNamedOperand ( MI , <STR_LIT> :: src0_modifiers ) -> getImm ( ) ;

unsigned Src1Mods = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) -> getImm ( ) ;


return Kind == MachineOperand :: MO_FrameIndex ;


<s> return Kind == MachineOperand :: MO_FrameIndex ;


<s> bool IsFMA = Opc == <STR_LIT> ; bool IsF32 = Opc == <STR_LIT> ; unsigned Opc = IsFMA ? <STR_LIT> : ( IsF32 ? <STR_LIT> : <STR_LIT> ) ; const MCInstrDesc & MadDesc = TII -> get ( Opc ) ; return TII -> isInlineConstant ( OpToFold , MadDesc . OpInfo [ OpNo ] . OperandType ) ; } return false ; } default : return false ;


<s> case <STR_LIT> : case <STR_LIT> : { int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( static_cast < int > ( OpNo ) == Src2Idx ) { bool IsFMA = Opc == <STR_LIT> ; bool IsF32 = Opc == <STR_LIT> ; unsigned Opc = IsFMA ? <STR_LIT> : ( IsF32 ? <STR_LIT> : <STR_LIT> ) ; const MCInstrDesc & MadDesc = TII -> get ( Opc ) ; return TII -> isInlineConstant ( OpToFold , MadDesc . OpInfo [ OpNo ] . OperandType ) ; }


<s> std :: pair < const MachineOperand * , int > <ISA_LIT>FoldOperands :: isOMod ( const MachineInstr & MI ) const { unsigned Op = MI . getOpcode ( ) ; switch ( Op ) { case <STR_LIT> : case <STR_LIT> : { if ( ( Op == <STR_LIT> && ST -> hasFP32Denormals ( ) ) || ( Op == <STR_LIT> && ST -> hasFP16Denormals ( ) ) ) return std :: make_pair ( nullptr , <STR_LIT> ) ; const MachineOperand * RegOp = nullptr ; const MachineOperand * ImmOp = nullptr ; const MachineOperand * Src0 = TII -> getNamedOperand ( MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src0 -> isImm ( ) ) { ImmOp = Src0 ; RegOp = Src1 ; } else if ( Src1 -> isImm ( ) ) { ImmOp = Src1 ; RegOp = Src0 ; } else return std :: make_pair ( nullptr , <STR_LIT> ) ; int OMod = getOModValue ( Op , ImmOp -> getImm ( ) ) ; if ( OMod == <STR_LIT> || TII -> hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: clamp ) ) return std :: make_pair ( nullptr , <STR_LIT> ) ; return std :: make_pair ( RegOp , OMod ) ; } case <STR_LIT> :


<s> bool isReg ( ) const { return Kind == MachineOperand :: MO_Register ;


<s> static bool isUseMIInFoldList ( ArrayRef < FoldCandidate > FoldList , const MachineInstr * MI ) {


<s> static bool isUseMIInFoldList ( ArrayRef < FoldCandidate > FoldList , const MachineInstr * MI ) { for ( auto Candidate : FoldList ) {

if ( Candidate . UseMI == MI ) return true ;


<s> return ! UseMO . isUndef ( ) && ! TII -> isSDWA ( MI ) ;


<s> static bool isUseSafeToFold ( const <ISA_LIT>InstrInfo * TII , const MachineInstr & MI , const MachineOperand & UseMO ) { return ! UseMO . isUndef ( ) && ! TII -> isSDWA ( MI ) ;


<s> static void mutateCopyOp ( MachineInstr & MI , const MCInstrDesc & NewDesc ) {


<s> TRI = & TII -> getRegisterInfo ( ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; bool IsIEEEMode = ST -> enableIEEEBit ( MF ) || ! MFI -> hasNoSignedZerosFPMath ( ) ; for ( MachineBasicBlock * MBB : depth_first ( & MF ) ) { MachineBasicBlock :: iterator I , Next ; for ( I = MBB -> begin ( ) ; I != MBB -> end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; tryFoldInst ( TII , & MI ) ; if ( ! TII -> isFoldableCopy ( MI ) ) { if ( IsIEEEMode || ! tryFoldOMod ( MI ) ) tryFoldClamp ( MI ) ; continue ; } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ;


<s> MachineBasicBlock :: iterator I , Next ; for ( I = MBB -> begin ( ) ; I != MBB -> end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; tryFoldInst ( TII , & MI ) ; if ( ! TII -> isFoldableCopy ( MI ) ) { if ( IsIEEEMode || ! tryFoldOMod ( MI ) ) tryFoldClamp ( MI ) ; continue ; } MachineOperand & OpToFold = MI . getOperand ( <NUM_LIT> ) ; bool FoldingImm = OpToFold . isImm ( ) || OpToFold . isFI ( ) ; if ( ! FoldingImm && ! OpToFold . isReg ( ) ) continue ;


<s> initialize<ISA_LIT>FoldOperandsPass ( * PassRegistry :: getPassRegistry ( ) ) ;


<s> const MCInstrDesc & Desc = MI . getDesc ( ) ;

unsigned NumOps = Desc . getNumOperands ( ) + Desc . getNumImplicitUses ( ) + Desc . getNumImplicitDefs ( ) ;


<s> unsigned NumOps = Desc . getNumOperands ( ) + Desc . getNumImplicitUses ( ) + Desc . getNumImplicitDefs ( ) ; for ( unsigned I = MI . getNumOperands ( ) - <NUM_LIT> ; I >= NumOps ; -- I ) MI . RemoveOperand ( I ) ;


<s> bool FoldAsMAD = tryAddToFoldList ( FoldList , MI , OpNo , OpToFold , TII ) ; if ( FoldAsMAD ) { MI -> untieRegOperand ( OpNo ) ; return true ; } MI -> setDesc ( TII -> get ( Opc ) ) ; } if ( Opc == <STR_LIT> && OpToFold -> isImm ( ) ) { MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; FoldList . push_back ( FoldCandidate ( MI , OpNo , OpToFold ) ) ; return true ; } if ( isUseMIInFoldList ( FoldList , MI ) ) return false ; unsigned CommuteIdx0 = TargetInstrInfo :: CommuteAnyOperandIndex ; unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( * MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) OpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) OpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , OpNo , OpToFold ) ) { TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ; return false ; }

FoldList . push_back ( FoldCandidate ( MI , OpNo , OpToFold , true ) ) ;


<s> if ( FoldAsMAD ) { MI -> untieRegOperand ( OpNo ) ; return true ; } MI -> setDesc ( TII -> get ( Opc ) ) ; } if ( Opc == <STR_LIT> && OpToFold -> isImm ( ) ) { MI -> setDesc ( TII -> get ( <STR_LIT> ) ) ; FoldList . push_back ( FoldCandidate ( MI , OpNo , OpToFold ) ) ; return true ; } if ( isUseMIInFoldList ( FoldList , MI ) ) return false ; unsigned CommuteIdx0 = TargetInstrInfo :: CommuteAnyOperandIndex ; unsigned CommuteIdx1 = TargetInstrInfo :: CommuteAnyOperandIndex ; bool CanCommute = TII -> findCommutedOpIndices ( * MI , CommuteIdx0 , CommuteIdx1 ) ; if ( CanCommute ) { if ( CommuteIdx0 == OpNo ) OpNo = CommuteIdx1 ; else if ( CommuteIdx1 == OpNo ) OpNo = CommuteIdx0 ; } if ( CanCommute && ( ! MI -> getOperand ( CommuteIdx0 ) . isReg ( ) || ! MI -> getOperand ( CommuteIdx1 ) . isReg ( ) ) ) return false ; if ( ! CanCommute || ! TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ) return false ; if ( ! TII -> isOperandLegal ( * MI , OpNo , OpToFold ) ) {

TII -> commuteInstruction ( * MI , false , CommuteIdx0 , CommuteIdx1 ) ;


<s> MI -> setDesc ( TII -> get ( UseCopy ? <STR_LIT> : <STR_LIT> ) ) ; return true ; } } if ( Src0 -> isImm ( ) && Src1 -> isImm ( ) ) { int32_t NewImm ; if ( ! evalBinaryInstruction ( Opc , NewImm , Src0 -> getImm ( ) , Src1 -> getImm ( ) ) ) return false ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; bool IsSGPR = TRI . isSGPRReg ( MRI , MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MI -> getOperand ( Src0Idx ) . ChangeToImmediate ( NewImm ) ; MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( IsSGPR ) ) ) ; return true ; } if ( ! MI -> isCommutable ( ) ) return false ; if ( Src0 -> isImm ( ) && ! Src1 -> isImm ( ) ) { std :: swap ( Src0 , Src1 ) ; std :: swap ( Src0Idx , Src1Idx ) ; } int32_t Src1Val = static_cast < int32_t > ( Src1 -> getImm ( ) ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) { if ( Src1Val == <NUM_LIT> ) { MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( <STR_LIT> ) ) ; } else if ( Src1Val == - <NUM_LIT> ) { MI -> RemoveOperand ( Src1Idx ) ; mutateCopyOp ( * MI , TII -> get ( getMovOpc ( Opc == <STR_LIT> ) ) ) ; } else return false ; return true ; } if ( MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> || MI -> getOpcode ( ) == <STR_LIT> ) { if ( Src1Val == <NUM_LIT> ) { MI -> RemoveOperand ( Src0Idx ) ;


<s> LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * DefClamp << <STR_LIT> << * Def << '\n' ) ; DefClamp -> setImm ( <NUM_LIT> ) ; MRI -> replaceRegWith ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ; MI . eraseFromParent ( ) ;


<s> if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) { const MachineOperand * Src0 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src1 ) ; if ( Src1 -> isIdenticalTo ( * Src0 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI << <STR_LIT> ) ; int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( Src2Idx != - <NUM_LIT> ) MI -> RemoveOperand ( Src2Idx ) ; MI -> RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ) ; mutateCopyOp ( * MI , TII -> get ( Src0 -> isReg ( ) ? ( unsigned ) <STR_LIT> : getMovOpc ( false ) ) ) ; LLVM_DEBUG ( dbgs ( ) << * MI << '\n' ) ; return true ; }


<s> unsigned Opc = MI -> getOpcode ( ) ; if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) { const MachineOperand * Src0 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src0 ) ; const MachineOperand * Src1 = TII -> getNamedOperand ( * MI , <STR_LIT> :: src1 ) ; if ( Src1 -> isIdenticalTo ( * Src0 ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * MI << <STR_LIT> ) ; int Src2Idx = <STR_LIT> ( Opc , <STR_LIT> :: src2 ) ; if ( Src2Idx != - <NUM_LIT> ) MI -> RemoveOperand ( Src2Idx ) ; MI -> RemoveOperand ( <STR_LIT> ( Opc , <STR_LIT> :: src1 ) ) ;

mutateCopyOp ( * MI , TII -> get ( Src0 -> isReg ( ) ? ( unsigned ) <STR_LIT> : getMovOpc ( false ) ) ) ;


<s> if ( TII -> hasModifiersSet ( * Def , <STR_LIT> :: clamp ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI << <STR_LIT> << * Def << '\n' ) ; DefOMod -> setImm ( OMod ) ;

MRI -> replaceRegWith ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , Def -> getOperand ( <NUM_LIT> ) . getReg ( ) ) ;


<s> static bool updateOperand ( FoldCandidate & Fold , const TargetRegisterInfo & TRI ) { MachineInstr * MI = Fold . UseMI ; MachineOperand & Old = MI -> getOperand ( Fold . UseOpNo ) ; assert ( Old . isReg ( ) ) ; if ( Fold . isImm ( ) ) { if ( MI -> getDesc ( ) . TSFlags & <STR_LIT> ) { unsigned Opcode = MI -> getOpcode ( ) ; int OpNo = MI -> getOperandNo ( & Old ) ; int ModIdx = - <NUM_LIT> ; if ( OpNo == <STR_LIT> ( Opcode , <STR_LIT> :: src0 ) ) ModIdx = <STR_LIT> :: src0_modifiers ; else if ( OpNo == <STR_LIT> ( Opcode , <STR_LIT> :: src1 ) ) ModIdx = <STR_LIT> :: src1_modifiers ; else if ( OpNo == <STR_LIT> ( Opcode , <STR_LIT> :: src2 ) ) ModIdx = <STR_LIT> :: src2_modifiers ; assert ( ModIdx != - <NUM_LIT> ) ; ModIdx = <STR_LIT> ( Opcode , ModIdx ) ; MachineOperand & Mod = MI -> getOperand ( ModIdx ) ; unsigned Val = Mod . getImm ( ) ; if ( ( Val & <STR_LIT> ) || ! ( Val & <STR_LIT> ) ) return false ;

if ( ! isUInt < <NUM_LIT> > ( Fold . ImmToFold ) ) {


<s> bool IsVMEM = isVMEMClauseInst ( MI ) ; if ( ! isValidClauseInst ( MI , IsVMEM ) ) continue ; RegUse Defs , Uses ; GCNDownwardRPTracker RPT ( * LIS ) ; RPT . reset ( MI ) ; if ( ! processRegUses ( MI , Defs , Uses , RPT ) ) continue ; unsigned Length = <NUM_LIT> ; for ( ; Next != E && Length < MaxClause ; ++ Next ) { if ( ! isValidClauseInst ( * Next , IsVMEM ) ) break ; if ( ! processRegUses ( * Next , Defs , Uses , RPT ) ) break ; ++ Length ; } if ( Length < <NUM_LIT> ) continue ; Changed = true ; MFI -> limitOccupancy ( LastRecordedOccupancy ) ; auto B = BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( TargetOpcode :: BUNDLE ) ) ; Ind -> insertMachineInstrInMaps ( * B ) ; for ( auto BI = I ; BI != Next ; ++ BI ) { BI -> bundleWithPred ( ) ; Ind -> removeSingleMachineInstrFromMaps ( * BI ) ; for ( MachineOperand & MO : BI -> defs ( ) ) if ( MO . readsReg ( ) ) MO . setIsInternalRead ( true ) ; } for ( auto && R : Defs ) { forAllLanes ( R . first , R . second . second , [ & R , & B ] ( unsigned SubReg ) { unsigned S = R . second . first | RegState :: EarlyClobber ; if ( ! SubReg ) S &= ~ ( RegState :: Undef | RegState :: Dead ) ; B . addDef ( R . first , S , SubReg ) ; } ) ; }


<s> if ( LaneMask . all ( ) || TargetRegisterInfo :: isPhysicalRegister ( Reg ) || LaneMask == MRI -> getMaxLaneMaskForVReg ( Reg ) ) { Func ( <NUM_LIT> ) ; return ; } const TargetRegisterClass * RC = MRI -> getRegClass ( Reg ) ; unsigned E = TRI -> getNumSubRegIndices ( ) ; SmallVector < unsigned , <STR_LIT> > CoveringSubregs ; for ( unsigned Idx = <NUM_LIT> ; Idx < E ; ++ Idx ) { if ( TRI -> getSubClassWithSubReg ( RC , Idx ) != RC ) continue ; LaneBitmask SubRegMask = TRI -> getSubRegIndexLaneMask ( Idx ) ; if ( SubRegMask == LaneMask ) { Func ( Idx ) ; return ; } if ( ( SubRegMask & ~ LaneMask ) . any ( ) || ( SubRegMask & LaneMask ) . none ( ) ) continue ; CoveringSubregs . push_back ( Idx ) ; } llvm :: sort ( CoveringSubregs , [ this ] ( unsigned A , unsigned B ) { LaneBitmask MaskA = TRI -> getSubRegIndexLaneMask ( A ) ; LaneBitmask MaskB = TRI -> getSubRegIndexLaneMask ( B ) ; unsigned NA = MaskA . getNumLanes ( ) ; unsigned NB = MaskB . getNumLanes ( ) ; if ( NA != NB ) return NA > NB ; return MaskA . getHighestLane ( ) > MaskB . getHighestLane ( ) ;


<s> if ( LaneMask . all ( ) || TargetRegisterInfo :: isPhysicalRegister ( Reg ) || LaneMask == MRI -> getMaxLaneMaskForVReg ( Reg ) ) { Func ( <NUM_LIT> ) ; return ; } const TargetRegisterClass * RC = MRI -> getRegClass ( Reg ) ; unsigned E = TRI -> getNumSubRegIndices ( ) ; SmallVector < unsigned , <STR_LIT> > CoveringSubregs ; for ( unsigned Idx = <NUM_LIT> ; Idx < E ; ++ Idx ) { if ( TRI -> getSubClassWithSubReg ( RC , Idx ) != RC ) continue ; LaneBitmask SubRegMask = TRI -> getSubRegIndexLaneMask ( Idx ) ; if ( SubRegMask == LaneMask ) { Func ( Idx ) ; return ; } if ( ( SubRegMask & ~ LaneMask ) . any ( ) || ( SubRegMask & LaneMask ) . none ( ) ) continue ; CoveringSubregs . push_back ( Idx ) ; } llvm :: sort ( CoveringSubregs , [ this ] ( unsigned A , unsigned B ) { LaneBitmask MaskA = TRI -> getSubRegIndexLaneMask ( A ) ; LaneBitmask MaskB = TRI -> getSubRegIndexLaneMask ( B ) ; unsigned NA = MaskA . getNumLanes ( ) ; unsigned NB = MaskB . getNumLanes ( ) ; if ( NA != NB ) return NA > NB ; return MaskA . getHighestLane ( ) > MaskB . getHighestLane ( ) ; } ) ;

for ( unsigned Idx : CoveringSubregs ) {


<s> for ( const MachineOperand & MO : MI . operands ( ) ) { if ( MO . isFI ( ) ) return false ; if ( ! MO . isReg ( ) ) continue ; Register Reg = MO . getReg ( ) ; if ( MO . isTied ( ) ) return false ; RegUse & Map = MO . isDef ( ) ? Uses : Defs ; auto Conflict = Map . find ( Reg ) ;

if ( Conflict == Map . end ( ) ) continue ;


<s> auto Conflict = Map . find ( Reg ) ; if ( Conflict == Map . end ( ) ) continue ; if ( Reg . isPhysical ( ) ) return false ; LaneBitmask Mask = TRI -> getSubRegIndexLaneMask ( MO . getSubReg ( ) ) ; if ( ( Conflict -> second . second & Mask ) . any ( ) ) return false ;


<s> void <ISA_LIT>FormMemoryClauses :: collectRegUses ( const MachineInstr & MI , RegUse & Defs , RegUse & Uses ) const { for ( const MachineOperand & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) ) continue ; Register Reg = MO . getReg ( ) ; if ( ! Reg ) continue ;

LaneBitmask Mask = Reg . isVirtual ( ) ? TRI -> getSubRegIndexLaneMask ( MO . getSubReg ( ) ) : LaneBitmask :: getAll ( ) ;


<s> auto Loc = Map . find ( Reg ) ; unsigned State = getMopState ( MO ) ; if ( Loc == Map . end ( ) ) { Map [ Reg ] = std :: make_pair ( State , Mask ) ; } else { Loc -> second . first |= State ; Loc -> second . second |= Mask ; }


<s> Func ( <NUM_LIT> ) ; return ; } const TargetRegisterClass * RC = MRI -> getRegClass ( Reg ) ; unsigned E = TRI -> getNumSubRegIndices ( ) ; SmallVector < unsigned , <STR_LIT> > CoveringSubregs ; for ( unsigned Idx = <NUM_LIT> ; Idx < E ; ++ Idx ) { if ( TRI -> getSubClassWithSubReg ( RC , Idx ) != RC ) continue ; LaneBitmask SubRegMask = TRI -> getSubRegIndexLaneMask ( Idx ) ; if ( SubRegMask == LaneMask ) { Func ( Idx ) ; return ; } if ( ( SubRegMask & ~ LaneMask ) . any ( ) || ( SubRegMask & LaneMask ) . none ( ) ) continue ; CoveringSubregs . push_back ( Idx ) ; } llvm :: sort ( CoveringSubregs , [ this ] ( unsigned A , unsigned B ) { LaneBitmask MaskA = TRI -> getSubRegIndexLaneMask ( A ) ; LaneBitmask MaskB = TRI -> getSubRegIndexLaneMask ( B ) ; unsigned NA = MaskA . getNumLanes ( ) ; unsigned NB = MaskB . getNumLanes ( ) ; if ( NA != NB ) return NA > NB ;

return MaskA . getHighestLane ( ) > MaskB . getHighestLane ( ) ;


<s> SmallVector < unsigned , <STR_LIT> > CoveringSubregs ; for ( unsigned Idx = <NUM_LIT> ; Idx < E ; ++ Idx ) { if ( TRI -> getSubClassWithSubReg ( RC , Idx ) != RC ) continue ; LaneBitmask SubRegMask = TRI -> getSubRegIndexLaneMask ( Idx ) ; if ( SubRegMask == LaneMask ) { Func ( Idx ) ; return ; } if ( ( SubRegMask & ~ LaneMask ) . any ( ) || ( SubRegMask & LaneMask ) . none ( ) ) continue ; CoveringSubregs . push_back ( Idx ) ; } llvm :: sort ( CoveringSubregs , [ this ] ( unsigned A , unsigned B ) { LaneBitmask MaskA = TRI -> getSubRegIndexLaneMask ( A ) ; LaneBitmask MaskB = TRI -> getSubRegIndexLaneMask ( B ) ; unsigned NA = MaskA . getNumLanes ( ) ; unsigned NB = MaskB . getNumLanes ( ) ; if ( NA != NB ) return NA > NB ; return MaskA . getHighestLane ( ) > MaskB . getHighestLane ( ) ; } ) ; for ( unsigned Idx : CoveringSubregs ) { LaneBitmask SubRegMask = TRI -> getSubRegIndexLaneMask ( Idx ) ;


<s> if ( MO . isKill ( ) ) S |= RegState :: Kill ; if ( MO . isEarlyClobber ( ) ) S |= RegState :: EarlyClobber ; if ( MO . getReg ( ) . isPhysical ( ) && MO . isRenamable ( ) ) S |= RegState :: Renamable ; return S ;


<s> for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: instr_iterator Next ; for ( auto I = MBB . instr_begin ( ) , E = MBB . instr_end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; bool IsVMEM = isVMEMClauseInst ( MI ) ; if ( ! isValidClauseInst ( MI , IsVMEM ) ) continue ; RegUse Defs , Uses ; GCNDownwardRPTracker RPT ( * LIS ) ; RPT . reset ( MI ) ; if ( ! processRegUses ( MI , Defs , Uses , RPT ) ) continue ; unsigned Length = <NUM_LIT> ; for ( ; Next != E && Length < FuncMaxClause ; ++ Next ) { if ( ! isValidClauseInst ( * Next , IsVMEM ) ) break ; if ( ! processRegUses ( * Next , Defs , Uses , RPT ) ) break ; ++ Length ; } if ( Length < <NUM_LIT> ) continue ; Changed = true ; MFI -> limitOccupancy ( LastRecordedOccupancy ) ; auto B = BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( TargetOpcode :: BUNDLE ) ) ; Ind -> insertMachineInstrInMaps ( * B ) ;

for ( auto BI = I ; BI != Next ; ++ BI ) {


<s> Register Reg = MO . getReg ( ) ; if ( MO . isTied ( ) ) return false ; RegUse & Map = MO . isDef ( ) ? Uses : Defs ; auto Conflict = Map . find ( Reg ) ; if ( Conflict == Map . end ( ) ) continue ;

if ( Register :: isPhysicalRegister ( Reg ) ) return false ;


<s> if ( MO . isTied ( ) ) return false ; RegUse & Map = MO . isDef ( ) ? Uses : Defs ; auto Conflict = Map . find ( Reg ) ; if ( Conflict == Map . end ( ) ) continue ;


<s> void <ISA_LIT>FormMemoryClauses :: collectRegUses ( const MachineInstr & MI , RegUse & Defs , RegUse & Uses ) const { for ( const MachineOperand & MO : MI . operands ( ) ) { if ( ! MO . isReg ( ) ) continue ; Register Reg = MO . getReg ( ) ; if ( ! Reg ) continue ; LaneBitmask Mask = Register :: isVirtualRegister ( Reg ) ? TRI -> getSubRegIndexLaneMask ( MO . getSubReg ( ) ) : LaneBitmask :: getAll ( ) ; RegUse & Map = MO . isDef ( ) ? Defs : Uses ; auto Loc = Map . find ( Reg ) ; unsigned State = getMopState ( MO ) ; if ( Loc == Map . end ( ) ) { Map [ Reg ] = std :: make_pair ( State , Mask ) ; } else {


<s> void <ISA_LIT>FormMemoryClauses :: forAllLanes ( unsigned Reg , LaneBitmask LaneMask , Callable Func ) const { if ( LaneMask . all ( ) || Register :: isPhysicalRegister ( Reg ) || LaneMask == MRI -> getMaxLaneMaskForVReg ( Reg ) ) { Func ( <NUM_LIT> ) ; return ; } const TargetRegisterClass * RC = MRI -> getRegClass ( Reg ) ; unsigned E = TRI -> getNumSubRegIndices ( ) ; SmallVector < unsigned , <STR_LIT> > CoveringSubregs ; for ( unsigned Idx = <NUM_LIT> ; Idx < E ; ++ Idx ) { if ( TRI -> getSubClassWithSubReg ( RC , Idx ) != RC ) continue ; LaneBitmask SubRegMask = TRI -> getSubRegIndexLaneMask ( Idx ) ; if ( SubRegMask == LaneMask ) {


<s> static unsigned getMopState ( const MachineOperand & MO ) { unsigned S = <NUM_LIT> ; if ( MO . isImplicit ( ) ) S |= RegState :: Implicit ; if ( MO . isDead ( ) ) S |= RegState :: Dead ; if ( MO . isUndef ( ) ) S |= RegState :: Undef ;

if ( MO . isKill ( ) ) S |= RegState :: Kill ;


<s> if ( MO . isEarlyClobber ( ) ) S |= RegState :: EarlyClobber ;

if ( Register :: isPhysicalRegister ( MO . getReg ( ) ) && MO . isRenamable ( ) ) S |= RegState :: Renamable ;


<s> if ( IsVMEMClause && ! isVMEMClauseInst ( MI ) ) return false ; if ( ! IsVMEMClause && ! isSMEMClauseInst ( MI ) ) return false ; for ( const MachineOperand & ResMO : MI . defs ( ) ) { Register ResReg = ResMO . getReg ( ) ; for ( const MachineOperand & MO : MI . uses ( ) ) { if ( ! MO . isReg ( ) || MO . isDef ( ) ) continue ; if ( MO . getReg ( ) == ResReg ) return false ; }


<s> if ( ! isValidClauseInst ( MI , IsVMEM ) ) continue ; RegUse Defs , Uses ; GCNDownwardRPTracker RPT ( * LIS ) ; RPT . reset ( MI ) ; if ( ! processRegUses ( MI , Defs , Uses , RPT ) ) continue ; unsigned Length = <NUM_LIT> ; for ( ; Next != E && Length < FuncMaxClause ; ++ Next ) { if ( ! isValidClauseInst ( * Next , IsVMEM ) ) break ; if ( ! processRegUses ( * Next , Defs , Uses , RPT ) ) break ; ++ Length ; } if ( Length < <NUM_LIT> ) continue ; Changed = true ; MFI -> limitOccupancy ( LastRecordedOccupancy ) ; auto B = BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( TargetOpcode :: BUNDLE ) ) ; Ind -> insertMachineInstrInMaps ( * B ) ; for ( auto BI = I ; BI != Next ; ++ BI ) { BI -> bundleWithPred ( ) ; Ind -> removeSingleMachineInstrFromMaps ( * BI ) ; for ( MachineOperand & MO : BI -> defs ( ) ) if ( MO . readsReg ( ) ) MO . setIsInternalRead ( true ) ; } for ( auto && R : Defs ) { forAllLanes ( R . first , R . second . second , [ & R , & B ] ( unsigned SubReg ) { unsigned S = R . second . first | RegState :: EarlyClobber ; if ( ! SubReg ) S &= ~ ( RegState :: Undef | RegState :: Dead ) ; B . addDef ( R . first , S , SubReg ) ; } ) ; } for ( auto && R : Uses ) { forAllLanes ( R . first , R . second . second , [ & R , & B ] ( unsigned SubReg ) { B . addUse ( R . first , R . second . first & ~ RegState :: Kill , SubReg ) ; } ) ; } for ( auto && R : Defs ) { unsigned Reg = R . first ; Uses . erase ( Reg ) ; if ( Register :: isPhysicalRegister ( Reg ) ) continue ; LIS -> removeInterval ( Reg ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; } for ( auto && R : Uses ) { unsigned Reg = R . first ;


<s> if ( ! isValidClauseInst ( * Next , IsVMEM ) ) break ; if ( ! processRegUses ( * Next , Defs , Uses , RPT ) ) break ; ++ Length ; } if ( Length < <NUM_LIT> ) continue ; Changed = true ; MFI -> limitOccupancy ( LastRecordedOccupancy ) ; auto B = BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( TargetOpcode :: BUNDLE ) ) ; Ind -> insertMachineInstrInMaps ( * B ) ; for ( auto BI = I ; BI != Next ; ++ BI ) { BI -> bundleWithPred ( ) ; Ind -> removeSingleMachineInstrFromMaps ( * BI ) ; for ( MachineOperand & MO : BI -> defs ( ) ) if ( MO . readsReg ( ) ) MO . setIsInternalRead ( true ) ; } for ( auto && R : Defs ) { forAllLanes ( R . first , R . second . second , [ & R , & B ] ( unsigned SubReg ) { unsigned S = R . second . first | RegState :: EarlyClobber ; if ( ! SubReg ) S &= ~ ( RegState :: Undef | RegState :: Dead ) ; B . addDef ( R . first , S , SubReg ) ; } ) ; } for ( auto && R : Uses ) { forAllLanes ( R . first , R . second . second , [ & R , & B ] ( unsigned SubReg ) { B . addUse ( R . first , R . second . first & ~ RegState :: Kill , SubReg ) ; } ) ; } for ( auto && R : Defs ) { unsigned Reg = R . first ; Uses . erase ( Reg ) ; if ( Register :: isPhysicalRegister ( Reg ) ) continue ; LIS -> removeInterval ( Reg ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; } for ( auto && R : Uses ) { unsigned Reg = R . first ;


<s> if ( <STR_LIT> ( MI . getOpcode ( ) ) != - <NUM_LIT> || <STR_LIT> ( MI . getOpcode ( ) ) != - <NUM_LIT> ) return false ; if ( IsVMEMClause && ! isVMEMClauseInst ( MI ) ) return false ; if ( ! IsVMEMClause && ! isSMEMClauseInst ( MI ) ) return false ; for ( const MachineOperand & ResMO : MI . defs ( ) ) { unsigned ResReg = ResMO . getReg ( ) ; for ( const MachineOperand & MO : MI . uses ( ) ) { if ( ! MO . isReg ( ) || MO . isDef ( ) ) continue ; if ( MO . getReg ( ) == ResReg ) return false ; } break ;


<s> unsigned ResReg = ResMO . getReg ( ) ; for ( const MachineOperand & MO : MI . uses ( ) ) { if ( ! MO . isReg ( ) || MO . isDef ( ) ) continue ; if ( MO . getReg ( ) == ResReg ) return false ; }


<s> MFI -> limitOccupancy ( LastRecordedOccupancy ) ; auto B = BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( TargetOpcode :: BUNDLE ) ) ; Ind -> insertMachineInstrInMaps ( * B ) ; for ( auto BI = I ; BI != Next ; ++ BI ) { BI -> bundleWithPred ( ) ; Ind -> removeSingleMachineInstrFromMaps ( * BI ) ; for ( MachineOperand & MO : BI -> defs ( ) ) if ( MO . readsReg ( ) ) MO . setIsInternalRead ( true ) ; } for ( auto && R : Defs ) { forAllLanes ( R . first , R . second . second , [ & R , & B ] ( unsigned SubReg ) { unsigned S = R . second . first | RegState :: EarlyClobber ; if ( ! SubReg ) S &= ~ ( RegState :: Undef | RegState :: Dead ) ; B . addDef ( R . first , S , SubReg ) ; } ) ; } for ( auto && R : Uses ) { forAllLanes ( R . first , R . second . second , [ & R , & B ] ( unsigned SubReg ) { B . addUse ( R . first , R . second . first & ~ RegState :: Kill , SubReg ) ; } ) ; } for ( auto && R : Defs ) {


<s> if ( ! MO . isReg ( ) ) continue ; Register Reg = MO . getReg ( ) ; if ( MO . isTied ( ) ) return false ; const RegUse & Map = MO . isDef ( ) ? Uses : Defs ; auto Conflict = Map . find ( Reg ) ; if ( Conflict == Map . end ( ) ) continue ; if ( Reg . isPhysical ( ) ) return false ; LaneBitmask Mask = TRI -> getSubRegIndexLaneMask ( MO . getSubReg ( ) ) ;

if ( ( Conflict -> second . second & Mask ) . any ( ) ) return false ;


<s> bool <ISA_LIT>FormMemoryClauses :: canBundle ( const MachineInstr & MI , const RegUse & Defs , const RegUse & Uses ) const { for ( const MachineOperand & MO : MI . operands ( ) ) { if ( MO . isFI ( ) ) return false ; if ( ! MO . isReg ( ) ) continue ; Register Reg = MO . getReg ( ) ; if ( MO . isTied ( ) ) return false ; const RegUse & Map = MO . isDef ( ) ? Uses : Defs ; auto Conflict = Map . find ( Reg ) ;


<s> RPT . advanceToNext ( ) ; GCNRegPressure MaxPressure = RPT . moveMaxPressure ( ) ; unsigned Occupancy = MaxPressure . getOccupancy ( * ST ) ; if ( Occupancy >= MFI -> getMinAllowedOccupancy ( ) && MaxPressure . getVGPRNum ( ST -> hasGFX90AInsts ( ) ) <= MaxVGPRs / <NUM_LIT> && MaxPressure . getSGPRNum ( ) <= MaxSGPRs / <NUM_LIT> ) { LastRecordedOccupancy = Occupancy ;


<s> bool <ISA_LIT>FormMemoryClauses :: checkPressure ( const MachineInstr & MI , GCNDownwardRPTracker & RPT ) { RPT . advanceToNext ( ) ; GCNRegPressure MaxPressure = RPT . moveMaxPressure ( ) ; unsigned Occupancy = MaxPressure . getOccupancy ( * ST ) ; if ( Occupancy >= MFI -> getMinAllowedOccupancy ( ) && MaxPressure . getVGPRNum ( ST -> hasGFX90AInsts ( ) ) <= MaxVGPRs / <NUM_LIT> && MaxPressure . getSGPRNum ( ) <= MaxSGPRs / <NUM_LIT> ) { LastRecordedOccupancy = Occupancy ; return true ; } return false ;


<s> return MachineFunctionProperties ( ) . set ( MachineFunctionProperties :: Property :: IsSSA ) ;


<s> if ( ! IsVMEMClause && ! isSMEMClauseInst ( MI ) ) return false ; for ( const MachineOperand & ResMO : MI . defs ( ) ) { Register ResReg = ResMO . getReg ( ) ; for ( const MachineOperand & MO : MI . uses ( ) ) { if ( ! MO . isReg ( ) || MO . isDef ( ) ) continue ; if ( MO . getReg ( ) == ResReg ) return false ; } break ;


<s> if ( MI . isBundled ( ) ) return false ; if ( ! MI . mayLoad ( ) || MI . mayStore ( ) ) return false ; if ( <ISA_LIT>InstrInfo :: isAtomic ( MI ) ) return false ; if ( IsVMEMClause && ! isVMEMClauseInst ( MI ) ) return false ;

if ( ! IsVMEMClause && ! isSMEMClauseInst ( MI ) ) return false ;


<s> RPT . advanceBeforeNext ( ) ; } const GCNRPTracker :: LiveRegSet LiveRegsCopy ( RPT . getLiveRegs ( ) ) ; RegUse Defs , Uses ; if ( ! processRegUses ( MI , Defs , Uses , RPT ) ) { RPT . reset ( MI , & LiveRegsCopy ) ; continue ; } MachineBasicBlock :: iterator LastClauseInst = Next ; unsigned Length = <NUM_LIT> ; for ( ; Next != E && Length < FuncMaxClause ; ++ Next ) { if ( Next -> isMetaInstruction ( ) ) continue ; if ( ! isValidClauseInst ( * Next , IsVMEM ) ) break ; if ( ! processRegUses ( * Next , Defs , Uses , RPT ) ) break ; LastClauseInst = Next ; ++ Length ; } if ( Length < <NUM_LIT> ) { RPT . reset ( MI , & LiveRegsCopy ) ; continue ; } Changed = true ; MFI -> limitOccupancy ( LastRecordedOccupancy ) ; assert ( ! LastClauseInst -> isMetaInstruction ( ) ) ; SlotIndex ClauseLiveInIdx = LIS -> getInstructionIndex ( MI ) ; SlotIndex ClauseLiveOutIdx = LIS -> getInstructionIndex ( * LastClauseInst ) . getNextIndex ( ) ; MachineInstrBuilder Kill ; for ( auto && R : Uses ) { Register Reg = R . first ; if ( Reg . isPhysical ( ) ) continue ; SmallVector < std :: tuple < unsigned , unsigned >> KillOps ; const LiveInterval & LI = LIS -> getInterval ( R . first ) ; if ( ! LI . hasSubRanges ( ) ) { if ( ! LI . liveAt ( ClauseLiveOutIdx ) ) { KillOps . emplace_back ( R . second . first | RegState :: Kill , <STR_LIT> ) ; } } else { LaneBitmask KilledMask ; for ( const LiveInterval :: SubRange & SR : LI . subranges ( ) ) { if ( SR . liveAt ( ClauseLiveInIdx ) && ! SR . liveAt ( ClauseLiveOutIdx ) ) KilledMask |= SR . LaneMask ; }


<s> } if ( Length < <NUM_LIT> ) { RPT . reset ( MI , & LiveRegsCopy ) ; continue ; } Changed = true ; MFI -> limitOccupancy ( LastRecordedOccupancy ) ; auto B = BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( TargetOpcode :: BUNDLE ) ) ; Ind -> insertMachineInstrInMaps ( * B ) ; RPT . reset ( * B , & LiveRegsCopy ) ; for ( auto BI = I ; BI != Next ; ++ BI ) { BI -> bundleWithPred ( ) ; Ind -> removeSingleMachineInstrFromMaps ( * BI ) ; for ( MachineOperand & MO : BI -> defs ( ) ) if ( MO . readsReg ( ) ) MO . setIsInternalRead ( true ) ; } for ( auto && R : Defs ) { forAllLanes ( R . first , R . second . second , [ & R , & B ] ( unsigned SubReg ) { unsigned S = R . second . first | RegState :: EarlyClobber ; if ( ! SubReg ) S &= ~ ( RegState :: Undef | RegState :: Dead ) ; B . addDef ( R . first , S , SubReg ) ; } ) ; } for ( auto && R : Uses ) { forAllLanes ( R . first , R . second . second , [ & R , & B ] ( unsigned SubReg ) { B . addUse ( R . first , R . second . first & ~ RegState :: Kill , SubReg ) ; } ) ; } for ( auto && R : Defs ) { Register Reg = R . first ; Uses . erase ( Reg ) ; if ( Reg . isPhysical ( ) ) continue ; LIS -> removeInterval ( Reg ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; } for ( auto && R : Uses ) { Register Reg = R . first ; if ( Reg . isPhysical ( ) ) continue ; LIS -> removeInterval ( Reg ) ; LIS -> createAndComputeVirtRegInterval ( Reg ) ; } } }


<s> bool <ISA_LIT>FormMemoryClauses :: canBundle ( const MachineInstr & MI , RegUse & Defs , RegUse & Uses ) const { for ( const MachineOperand & MO : MI . operands ( ) ) { if ( MO . isFI ( ) ) return false ; if ( ! MO . isReg ( ) ) continue ; unsigned Reg = MO . getReg ( ) ; if ( MO . isTied ( ) ) return false ;

RegUse & Map = MO . isDef ( ) ? Uses : Defs ;


<s> if ( Occupancy >= MFI -> getMinAllowedOccupancy ( ) && MaxPressure . getVGPRNum ( ) <= MaxVGPRs && MaxPressure . getSGPRNum ( ) <= MaxSGPRs ) {


<s> bool <ISA_LIT>FormMemoryClauses :: checkPressure ( const MachineInstr & MI , GCNDownwardRPTracker & RPT ) { RPT . advanceToNext ( ) ; GCNRegPressure MaxPressure = RPT . moveMaxPressure ( ) ; unsigned Occupancy = MaxPressure . getOccupancy ( * ST ) ; if ( Occupancy >= MFI -> getMinAllowedOccupancy ( ) && MaxPressure . getVGPRNum ( ) <= MaxVGPRs && MaxPressure . getSGPRNum ( ) <= MaxSGPRs ) { LastRecordedOccupancy = Occupancy ; return true ; } return false ;


<s> unsigned State = getMopState ( MO ) ; if ( Loc == Map . end ( ) ) { Map [ Reg ] = std :: make_pair ( State , Mask ) ; } else { Loc -> second . first |= State ; Loc -> second . second |= Mask ;


<s> FunctionPass * llvm :: create<ISA_LIT>FormMemoryClausesPass ( ) {

return new <ISA_LIT>FormMemoryClauses ( ) ;


<s> FunctionPass * llvm :: create<ISA_LIT>FormMemoryClausesPass ( ) { return new <ISA_LIT>FormMemoryClauses ( ) ;