<s> if ( Info0 -> BitsPerComp != <NUM_LIT> ) return false ; if ( getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , STI ) == <NUM_LIT> ) return false ; } unsigned EltOffset0 = CI . Offset / CI . EltSize ; unsigned EltOffset1 = Paired . Offset / CI . EltSize ; CI . UseST64 = false ; CI . BaseOff = <NUM_LIT> ; if ( ( CI . InstClass != DS_READ ) && ( CI . InstClass != DS_WRITE ) ) { return ( EltOffset0 + CI . Width == EltOffset1 || EltOffset1 + Paired . Width == EltOffset0 ) && CI . GLC == Paired . GLC && CI . DLC == Paired . DLC && ( CI . InstClass == S_BUFFER_LOAD_IMM || CI . SLC == Paired . SLC ) ; } if ( ( EltOffset0 % <NUM_LIT> == <NUM_LIT> ) && ( EltOffset1 % <NUM_LIT> ) == <NUM_LIT> && isUInt < <NUM_LIT> > ( EltOffset0 / <NUM_LIT> ) && isUInt < <NUM_LIT> > ( EltOffset1 / <NUM_LIT> ) ) { if ( Modify ) { CI . Offset = EltOffset0 / <NUM_LIT> ; Paired . Offset = EltOffset1 / <NUM_LIT> ; CI . UseST64 = true ; } return true ; } if ( isUInt < <NUM_LIT> > ( EltOffset0 ) && isUInt < <NUM_LIT> > ( EltOffset1 ) ) { if ( Modify ) { CI . Offset = EltOffset0 ; Paired . Offset = EltOffset1 ; } return true ; } unsigned OffsetDiff = std :: abs ( ( int ) EltOffset1 - ( int ) EltOffset0 ) ; CI . BaseOff = std :: min ( CI . Offset , Paired . Offset ) ; if ( ( OffsetDiff % <NUM_LIT> == <NUM_LIT> ) && isUInt < <NUM_LIT> > ( OffsetDiff / <NUM_LIT> ) ) { if ( Modify ) { CI . Offset = ( EltOffset0 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; Paired . Offset = ( EltOffset1 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ;


<s> break ; } case S_BUFFER_LOAD_IMM : { MachineBasicBlock :: iterator NewMI = mergeSBufferLoadImmPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeBufferLoadPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_STORE : { MachineBasicBlock :: iterator NewMI = mergeBufferStorePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case MIMG : { MachineBasicBlock :: iterator NewMI = mergeImagePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case TBUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeTBufferLoadPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ;


<s> MRI = & MF . getRegInfo ( ) ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; assert ( MRI -> isSSA ( ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : MF ) { std :: list < std :: list < CombineInfo > > MergeableInsts ; Modified |= collectMergeableInsts ( MBB , MergeableInsts ) ; do { OptimizeAgain = false ;

Modified |= optimizeBlock ( MergeableInsts ) ;


<s> Base -> setIsKill ( false ) ; TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> setImm ( NewOffset ) ;


<s> void <ISA_LIT>LoadStoreOptimizer :: updateBaseAndOffset ( MachineInstr & MI , unsigned NewBase , int32_t NewOffset ) const { auto Base = TII -> getNamedOperand ( MI , <STR_LIT> :: vaddr ) ; Base -> setReg ( NewBase ) ; Base -> setIsKill ( false ) ; TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> setImm ( NewOffset ) ;


<s> const auto * LWEOp = TII . getNamedOperand ( * CI . I , <STR_LIT> :: lwe ) ; if ( ( TFEOp && TFEOp -> getImm ( ) ) || ( LWEOp && LWEOp -> getImm ( ) ) ) return false ; unsigned OperandsToMatch [ ] = { <STR_LIT> :: glc , <STR_LIT> :: slc , <STR_LIT> :: d16 , <STR_LIT> :: unorm , <STR_LIT> :: da , <STR_LIT> :: r128 } ; for ( auto op : OperandsToMatch ) { int Idx = <STR_LIT> ( CI . I -> getOpcode ( ) , op ) ; if ( <STR_LIT> ( CI . Paired -> getOpcode ( ) , op ) != Idx ) return false ; if ( Idx != - <NUM_LIT> && CI . I -> getOperand ( Idx ) . getImm ( ) != CI . Paired -> getOperand ( Idx ) . getImm ( ) ) return false ; } unsigned MaxMask = std :: max ( CI . DMask0 , CI . DMask1 ) ;

unsigned MinMask = std :: min ( CI . DMask0 , CI . DMask1 ) ;


<s> if ( Swizzled != - <NUM_LIT> && CI . I -> getOperand ( Swizzled ) . getImm ( ) ) return false ; ++ MBBI ; DenseSet < unsigned > RegDefsToMove ; DenseSet < unsigned > PhysRegUsesToMove ; addDefsUsesToList ( * CI . I , RegDefsToMove , PhysRegUsesToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { if ( ( getInstClass ( MBBI -> getOpcode ( ) , * TII ) != InstClass ) || ( getInstSubclass ( MBBI -> getOpcode ( ) , * TII ) != InstSubclass ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) { return false ; } if ( MBBI -> mayLoadOrStore ( ) && ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , AA ) ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = CI . hasSameBaseAddress ( * MBBI ) ;


<s> CI . InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = CI . hasSameBaseAddress ( * MBBI ) ; if ( Match ) { CI . setPaired ( MBBI , * TII ) ; bool canBeCombined = CI . InstClass == MIMG ? dmasksCanBeCombined ( CI , * TII ) : widthsFit ( * STM , CI ) && offsetsCanBeCombined ( CI ) ;

if ( canBeCombined && canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , AA ) ) return true ;


<s> return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } if ( TII . isMIMG ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) == - <NUM_LIT> ) return UNKNOWN ; if ( TII . get ( Opc ) . mayStore ( ) || ! TII . get ( Opc ) . mayLoad ( ) || TII . isGather4 ( Opc ) ) return UNKNOWN ; return MIMG ; } return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return S_BUFFER_LOAD_IMM ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return DS_READ ; case <STR_LIT> : case <STR_LIT> :


<s> return BUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } if ( TII . isMIMG ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) == - <NUM_LIT> ) return UNKNOWN ; if ( TII . get ( Opc ) . mayStore ( ) || ! TII . get ( Opc ) . mayLoad ( ) || TII . isGather4 ( Opc ) ) return UNKNOWN ; return MIMG ; } return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return S_BUFFER_LOAD_IMM ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> if ( TII . isMUBUF ( Opc ) ) return <STR_LIT> ( Opc ) ; if ( TII . isMIMG ( Opc ) ) { const <STR_LIT> * Info = <STR_LIT> ( Opc ) ; assert ( Info ) ;


<s> default : assert ( CI . InstClass == BUFFER_LOAD || CI . InstClass == BUFFER_STORE ) ; return <STR_LIT> ( <STR_LIT> ( CI . I -> getOpcode ( ) ) , Width ) ; case UNKNOWN : llvm_unreachable ( <STR_LIT> ) ; case S_BUFFER_LOAD_IMM : switch ( Width ) { default : return <NUM_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> :


<s> unsigned <ISA_LIT>LoadStoreOptimizer :: getNewOpcode ( const CombineInfo & CI ) { const unsigned Width = CI . Width0 + CI . Width1 ; switch ( CI . InstClass ) { default : assert ( CI . InstClass == BUFFER_LOAD || CI . InstClass == BUFFER_STORE ) ; return <STR_LIT> ( <STR_LIT> ( CI . I -> getOpcode ( ) ) , Width ) ; case UNKNOWN : llvm_unreachable ( <STR_LIT> ) ; case S_BUFFER_LOAD_IMM : switch ( Width ) { default : return <NUM_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT> ;


<s> if ( TII . isMUBUF ( Opc ) ) { return <STR_LIT> ( Opc ) ; } if ( TII . isMIMG ( MI ) ) { uint64_t DMaskImm = TII . getNamedOperand ( MI , <STR_LIT> :: dmask ) -> getImm ( ) ; return countPopulation ( DMaskImm ) ;


<s> return <STR_LIT> ( Opc ) ; } if ( TII . isMIMG ( MI ) ) { uint64_t DMaskImm = TII . getNamedOperand ( MI , <STR_LIT> :: dmask ) -> getImm ( ) ; return countPopulation ( DMaskImm ) ; } switch ( Opc ) { case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> :


<s> std :: pair < unsigned , unsigned > <ISA_LIT>LoadStoreOptimizer :: getSubRegIdxs ( const CombineInfo & CI ) { if ( CI . Width0 == <NUM_LIT> || CI . Width0 == <NUM_LIT> || CI . Width0 + CI . Width1 > <NUM_LIT> ) return std :: make_pair ( <NUM_LIT> , <NUM_LIT> ) ; bool ReverseOrder ; if ( CI . InstClass == MIMG ) { assert ( ( countPopulation ( CI . DMask0 | CI . DMask1 ) == CI . Width0 + CI . Width1 ) && <STR_LIT> ) ; ReverseOrder = CI . DMask0 > CI . DMask1 ; } else ReverseOrder = CI . Offset0 > CI . Offset1 ; static const unsigned Idxs [ <NUM_LIT> ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> , <NUM_LIT> } , { <STR_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } , } ; unsigned Idx0 ; unsigned Idx1 ; assert ( CI . Width0 >= <NUM_LIT> && CI . Width0 <= <NUM_LIT> ) ; assert ( CI . Width1 >= <NUM_LIT> && CI . Width1 <= <NUM_LIT> ) ; if ( ReverseOrder ) { Idx1 = Idxs [ <NUM_LIT> ] [ CI . Width1 - <NUM_LIT> ] ; Idx0 = Idxs [ CI . Width1 ] [ CI . Width0 - <NUM_LIT> ] ;


<s> assert ( ( countPopulation ( CI . DMask0 | CI . DMask1 ) == CI . Width0 + CI . Width1 ) && <STR_LIT> ) ; ReverseOrder = CI . DMask0 > CI . DMask1 ; } else ReverseOrder = CI . Offset0 > CI . Offset1 ; static const unsigned Idxs [ <NUM_LIT> ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> , <NUM_LIT> } , { <STR_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } , } ; unsigned Idx0 ; unsigned Idx1 ; assert ( CI . Width0 >= <NUM_LIT> && CI . Width0 <= <NUM_LIT> ) ; assert ( CI . Width1 >= <NUM_LIT> && CI . Width1 <= <NUM_LIT> ) ; if ( ReverseOrder ) { Idx1 = Idxs [ <NUM_LIT> ] [ CI . Width1 - <NUM_LIT> ] ;

Idx0 = Idxs [ CI . Width1 ] [ CI . Width0 - <NUM_LIT> ] ;


<s> case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; } } else { switch ( CI . Width0 + CI . Width1 ) { default : return nullptr ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; } }


<s> unsigned DMaskIdx = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: dmask ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) ; for ( unsigned I = <NUM_LIT> , E = ( * CI . I ) . getNumOperands ( ) ; I != E ; ++ I ) { if ( I == DMaskIdx ) MIB . addImm ( MergedDMask ) ; else MIB . add ( ( * CI . I ) . getOperand ( I ) ) ; } assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ;

MachineInstr * New = MIB . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ;


<s> const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; Register BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Read2 << '\n' ) ;


<s> DebugLoc DL = CI . I -> getDebugLoc ( ) ; Register BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , CI . Paired , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; moveInstsAfter ( Write2 , CI . InstsToMove ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Write2 << '\n' ) ; return Write2 ;


<s> MachineInstr * LoHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub0 ) . addReg ( CarryReg , RegState :: Define ) . addReg ( Addr . Base . LoReg , <NUM_LIT> , Addr . Base . LoSubReg ) . add ( OffsetLo ) . addImm ( <NUM_LIT> ) ; ( void ) LoHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; LoHalf -> dump ( ) ; ) ; MachineInstr * HiHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub1 ) . addReg ( DeadCarryReg , RegState :: Define | RegState :: Dead ) . addReg ( Addr . Base . HiReg , <NUM_LIT> , Addr . Base . HiSubReg ) . add ( OffsetHi ) . addReg ( CarryReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; ( void ) HiHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; HiHalf -> dump ( ) ; ) ;

Register FullDestReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ;


<s> MachineBasicBlock :: iterator <ISA_LIT>LoadStoreOptimizer :: mergeTBufferStorePair ( CombineInfo & CI , CombineInfo & Paired , const SmallVectorImpl < MachineInstr * > & InstsToMove ) { MachineBasicBlock * MBB = CI . I -> getParent ( ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI , Paired ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , * STM ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ;

const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ;


<s> assert ( MRI -> isSSA ( ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Modified = false ; SmallPtrSet < MachineInstr * , <NUM_LIT> > AnchorList ; MemInfoMap Visited ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator SectionEnd ; for ( MachineBasicBlock :: iterator I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = SectionEnd ) { bool CollectModified ; std :: list < std :: list < CombineInfo >> MergeableInsts ; std :: tie ( SectionEnd , CollectModified ) = collectMergeableInsts ( I , E , Visited , AnchorList , MergeableInsts ) ; Modified |= CollectModified ; do { OptimizeAgain = false ; Modified |= optimizeBlock ( MergeableInsts ) ; } while ( OptimizeAgain ) ;


<s> continue ; } if ( STM -> hasSBufferLoadStoreAtomicDwordxN ( ) && ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ) { CI . InstClass = S_BUFFER_LOAD_IMM ; CI . EltSize = <STR_LIT> ( * STM , <NUM_LIT> ) ; CI . IsX2 = Opc == <STR_LIT> ; if ( findMatchingInst ( CI ) ) { Modified = true ; I = mergeSBufferLoadImmPair ( CI ) ; if ( ! CI . IsX2 ) CreatedX2 ++ ; } else { ++ I ; } continue ; } if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) { if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) CI . InstClass = BUFFER_LOAD_OFFEN ; else CI . InstClass = BUFFER_LOAD_OFFSET ; CI . EltSize = <NUM_LIT> ; CI . IsX2 = Opc == <STR_LIT> || Opc == <STR_LIT> ; if ( findMatchingInst ( CI ) ) { Modified = true ; I = mergeBufferLoadPair ( CI ) ; if ( ! CI . IsX2 ) CreatedX2 ++ ; } else { ++ I ; } continue ; } bool StoreIsX2 , IsOffen ; if ( promoteBufferStoreOpcode ( * I , StoreIsX2 , IsOffen ) ) { CI . InstClass = IsOffen ? BUFFER_STORE_OFFEN : BUFFER_STORE_OFFSET ; CI . EltSize = <NUM_LIT> ;


<s> assert ( MRI -> isSSA ( ) && <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Modified = false ; CreatedX2 = <NUM_LIT> ; for ( MachineBasicBlock & MBB : MF ) Modified |= optimizeBlock ( MBB ) ; if ( CreatedX2 >= <NUM_LIT> ) { for ( MachineBasicBlock & MBB : MF ) Modified |= optimizeBlock ( MBB ) ; }


<s> MRI = & MF . getRegInfo ( ) ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; assert ( MRI -> isSSA ( ) && <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Modified = false ; CreatedX2 = <NUM_LIT> ; for ( MachineBasicBlock & MBB : MF ) Modified |= optimizeBlock ( MBB ) ; if ( CreatedX2 >= <NUM_LIT> ) {


<s> switch ( Opc ) { default : if ( TII . isMUBUF ( Opc ) ) { switch ( <STR_LIT> ( Opc ) ) { default : return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } if ( TII . isMIMG ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) == - <NUM_LIT> && <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) == - <NUM_LIT> ) return UNKNOWN ; if ( <STR_LIT> ( Opc ) -> BVH ) return UNKNOWN ; if ( TII . get ( Opc ) . mayStore ( ) || ! TII . get ( Opc ) . mayLoad ( ) || TII . isGather4 ( Opc ) ) return UNKNOWN ; return MIMG ; } if ( TII . isMTBUF ( Opc ) ) { switch ( <STR_LIT> ( Opc ) ) { default : return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_STORE ; } } return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return S_BUFFER_LOAD_IMM ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> TII = static_cast < const <ISA_LIT>InstrInfo * > ( STM . getInstrInfo ( ) ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; assert ( ! MRI -> isSSA ( ) ) ; bool Modified = false ;


<s> MachineOperand OffsetHi = createRegOrImm ( static_cast < int32_t > ( Addr . Offset >> <NUM_LIT> ) , MI ) ; const auto * CarryRC = TRI -> getRegClass ( <STR_LIT> ) ; Register CarryReg = MRI -> createVirtualRegister ( CarryRC ) ; Register DeadCarryReg = MRI -> createVirtualRegister ( CarryRC ) ; Register DestSub0 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; Register DestSub1 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * LoHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub0 ) . addReg ( CarryReg , RegState :: Define ) . addReg ( Addr . Base . LoReg , <NUM_LIT> , Addr . Base . LoSubReg ) . add ( OffsetLo ) . addImm ( <NUM_LIT> ) ; ( void ) LoHalf ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; LoHalf -> dump ( ) ; ) ;


<s> Register Reg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * Mov = BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Reg ) . addImm ( Val ) ;


<s> addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = true ; for ( unsigned i = <NUM_LIT> ; i < NumAddresses ; i ++ ) { const MachineOperand & AddrRegNext = MBBI -> getOperand ( AddrIdx [ i ] ) ; if ( AddrReg [ i ] -> isImm ( ) || AddrRegNext . isImm ( ) ) { if ( AddrReg [ i ] -> isImm ( ) != AddrRegNext . isImm ( ) || AddrReg [ i ] -> getImm ( ) != AddrRegNext . getImm ( ) ) { Match = false ; break ; } continue ; } if ( AddrReg [ i ] -> getReg ( ) != AddrRegNext . getReg ( ) || AddrReg [ i ] -> getSubReg ( ) != AddrRegNext . getSubReg ( ) ) { Match = false ; break ; } } if ( Match ) { int OffsetIdx = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: offset ) ; CI . Offset0 = CI . I -> getOperand ( OffsetIdx ) . getImm ( ) ; CI . Width0 = getOpcodeWidth ( * CI . I ) ; CI . Offset1 = MBBI -> getOperand ( OffsetIdx ) . getImm ( ) ; CI . Width1 = getOpcodeWidth ( * MBBI ) ; CI . Paired = MBBI ; if ( ( CI . InstClass == DS_READ ) || ( CI . InstClass == DS_WRITE ) ) { CI . Offset0 &= <NUM_LIT> ; CI . Offset1 &= <NUM_LIT> ; } else { CI . GLC0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: glc ) -> getImm ( ) ; CI . GLC1 = TII -> getNamedOperand ( * MBBI , <STR_LIT> :: glc ) -> getImm ( ) ; if ( CI . InstClass != S_BUFFER_LOAD_IMM ) { CI . SLC0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: slc ) -> getImm ( ) ; CI . SLC1 = TII -> getNamedOperand ( * MBBI , <STR_LIT> :: slc ) -> getImm ( ) ; } CI . DLC0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: dlc ) -> getImm ( ) ;

CI . DLC1 = TII -> getNamedOperand ( * MBBI , <STR_LIT> :: dlc ) -> getImm ( ) ;


<s> if ( baseOpcode == - <NUM_LIT> ) { return UNKNOWN ; } switch ( baseOpcode ) { default : return UNKNOWN ; case <STR_LIT> : return BUFFER_LOAD_OFFEN ; case <STR_LIT> : return BUFFER_LOAD_OFFSET ; case <STR_LIT> : return BUFFER_STORE_OFFEN ; case <STR_LIT> : return BUFFER_STORE_OFFSET ; case <STR_LIT> : return BUFFER_LOAD_OFFEN_exact ; case <STR_LIT> : return BUFFER_LOAD_OFFSET_exact ; case <STR_LIT> : return BUFFER_STORE_OFFEN_exact ; case <STR_LIT> : return BUFFER_STORE_OFFSET_exact ; } } switch ( Opc ) { default : return UNKNOWN ;


<s> default : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ;


<s> } switch ( Opc ) { default : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return SBASE ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return ADDR ; }


<s> MachineBasicBlock * MBB = CI . I -> getParent ( ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ;

MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC0 ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ;


<s> const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC0 ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ;


<s> const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset0 , CI . Offset1 ) ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC0 ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; moveInstsAfter ( MIB , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ;


<s> MachineBasicBlock :: iterator <ISA_LIT>LoadStoreOptimizer :: mergeBufferStorePair ( CombineInfo & CI ) { MachineBasicBlock * MBB = CI . I -> getParent ( ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ;

const auto * Src1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdata ) ;


<s> const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; Register BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ;

moveInstsAfter ( Copy1 , CI . InstsToMove ) ;


<s> MachineInstr & MINext = * MBBI ; if ( MINext . getOpcode ( ) != MI . getOpcode ( ) || TII -> getNamedOperand ( MINext , <STR_LIT> :: offset ) -> getImm ( ) ) continue ; const MachineOperand & BaseNext = * TII -> getNamedOperand ( MINext , <STR_LIT> :: vaddr ) ; MemAddress MAddrNext ; if ( Visited . find ( & MINext ) == Visited . end ( ) ) { processBaseWithConstOffset ( BaseNext , MAddrNext ) ; Visited [ & MINext ] = MAddrNext ; } else MAddrNext = Visited [ & MINext ] ; if ( MAddrNext . Base . LoReg != MAddr . Base . LoReg || MAddrNext . Base . HiReg != MAddr . Base . HiReg || MAddrNext . Base . LoSubReg != MAddr . Base . LoSubReg || MAddrNext . Base . HiSubReg != MAddr . Base . HiSubReg ) continue ; InstsWCommonBase . push_back ( std :: make_pair ( & MINext , MAddrNext . Offset ) ) ; int64_t Dist = MAddr . Offset - MAddrNext . Offset ; TargetLoweringBase :: AddrMode AM ; AM . HasBaseReg = true ; AM . BaseOffs = Dist ; if ( TLI -> isLegalGlobalAddressingMode ( AM ) && ( uint32_t ) std :: abs ( Dist ) > MaxDist ) { MaxDist = std :: abs ( Dist ) ; AnchorAddr = MAddrNext ; AnchorInst = & MINext ; } } if ( AnchorInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; AnchorInst -> dump ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << AnchorAddr . Offset << <STR_LIT> ) ; unsigned Base = computeBase ( MI , AnchorAddr ) ; updateBaseAndOffset ( MI , Base , MAddr . Offset - AnchorAddr . Offset ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; ) ;


<s> SlotIndex PairedIndex = LIS -> getInstructionIndex ( * Paired ) ; LiveRange & M0Range = LIS -> getRegUnit ( * MCRegUnitIterator ( <STR_LIT> , TRI ) ) ; LiveRange :: Segment * M0Segment = M0Range . getSegmentContaining ( PairedIndex ) ; bool UpdateM0Range = M0Segment -> end == PairedIndex . getRegSlot ( ) ; MachineInstrBuilder Write2 = BuildMI ( * MBB , I , DL , Write2Desc ) . addOperand ( * Addr ) . addOperand ( * Data0 ) . addOperand ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . addMemOperand ( * I -> memoperands_begin ( ) ) . addMemOperand ( * Paired -> memoperands_begin ( ) ) ; unsigned OrigRegs [ ] = { Data0 -> getReg ( ) , Data1 -> getReg ( ) , Addr -> getReg ( ) } ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; LIS -> RemoveMachineInstrFromMaps ( * Paired ) ; I -> eraseFromParent ( ) ; Paired -> eraseFromParent ( ) ; LIS -> repairIntervalsInRange ( MBB , Write2 , Write2 , OrigRegs ) ; if ( UpdateM0Range ) { SlotIndex Write2Index = LIS -> getInstructionIndex ( * Write2 ) ; M0Segment -> end = Write2Index . getRegSlot ( ) ; } DEBUG ( dbgs ( ) << <STR_LIT> << * Write2 << '\n' ) ;


<s> BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) ) ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . setMemRefs ( CI . I -> mergeMemRefsWith ( * CI . Paired ) ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ;

DEBUG ( dbgs ( ) << <STR_LIT> << * Read2 << '\n' ) ;


<s> std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) ) ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . setMemRefs ( CI . I -> mergeMemRefsWith ( * CI . Paired ) ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ;

BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ;


<s> const MachineOperand * Data0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: data0 ) ; const MachineOperand * Data1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: data0 ) ; unsigned NewOffset0 = CI . Offset0 ; unsigned NewOffset1 = CI . Offset1 ; unsigned Opc = CI . UseST64 ? write2ST64Opcode ( CI . EltSize ) : write2Opcode ( CI . EltSize ) ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( Data0 , Data1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Write2Desc = TII -> get ( Opc ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ;


<s> MachineOperand OffsetHi = createRegOrImm ( static_cast < int32_t > ( Addr . Offset >> <NUM_LIT> ) , MI ) ; unsigned CarryReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned DeadCarryReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned DestSub0 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned DestSub1 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * LoHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub0 ) . addReg ( CarryReg , RegState :: Define ) . addReg ( Addr . Base . LoReg , <NUM_LIT> , Addr . Base . LoSubReg ) . add ( OffsetLo ) . addImm ( <NUM_LIT> ) ; ( void ) LoHalf ;


<s> const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ;

MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ;


<s> const unsigned Opcode = getNewOpcode ( CI ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; unsigned SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode ) ;

if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ;


<s> BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . DLC0 ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ;

const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: sdst ) ;


<s> const unsigned Opc = CI . I -> getOpcode ( ) ; const InstClassEnum InstClass = getInstClass ( Opc , * TII ) ; if ( InstClass == UNKNOWN ) { return false ; } const unsigned InstSubclass = getInstSubclass ( Opc , * TII ) ; int Swizzled = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: swz ) ; if ( Swizzled != - <NUM_LIT> && CI . I -> getOperand ( Swizzled ) . getImm ( ) ) return false ; ++ MBBI ; DenseSet < unsigned > RegDefsToMove ; DenseSet < unsigned > PhysRegUsesToMove ; addDefsUsesToList ( * CI . I , RegDefsToMove , PhysRegUsesToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { if ( ( getInstClass ( MBBI -> getOpcode ( ) , * TII ) != InstClass ) || ( getInstSubclass ( MBBI -> getOpcode ( ) , * TII ) != InstSubclass ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) { return false ; } if ( MBBI -> mayLoadOrStore ( ) && ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , AA ) ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; int Swizzled = <STR_LIT> ( MBBI -> getOpcode ( ) , <STR_LIT> :: swz ) ; if ( Swizzled != - <NUM_LIT> && MBBI -> getOperand ( Swizzled ) . getImm ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = CI . hasSameBaseAddress ( * MBBI ) ; if ( Match ) { CI . setPaired ( MBBI , * TII ) ; bool canBeCombined = CI . InstClass == MIMG ? dmasksCanBeCombined ( CI , * TII ) : widthsFit ( * STM , CI ) && offsetsCanBeCombined ( CI , * STI ) ;


<s> if ( ( getInstClass ( MBBI -> getOpcode ( ) , * TII ) != InstClass ) || ( getInstSubclass ( MBBI -> getOpcode ( ) , * TII ) != InstSubclass ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) { return false ; } if ( MBBI -> mayLoadOrStore ( ) && ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , AA ) ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; int Swizzled = <STR_LIT> ( MBBI -> getOpcode ( ) , <STR_LIT> :: swz ) ; if ( Swizzled != - <NUM_LIT> && MBBI -> getOperand ( Swizzled ) . getImm ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = CI . hasSameBaseAddress ( * MBBI ) ; if ( Match ) {


<s> unsigned <ISA_LIT>LoadStoreOptimizer :: getNewOpcode ( const CombineInfo & CI ) { const unsigned Width = CI . Width0 + CI . Width1 ; switch ( CI . InstClass ) { default : assert ( CI . InstClass == BUFFER_LOAD || CI . InstClass == BUFFER_STORE ) ;

return <STR_LIT> ( <STR_LIT> ( CI . I -> getOpcode ( ) ) , Width ) ;


<s> } else ReverseOrder = CI . Offset0 > CI . Offset1 ; static const unsigned Idxs [ <NUM_LIT> ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> , <NUM_LIT> } , { <STR_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } , } ; unsigned Idx0 ; unsigned Idx1 ; assert ( CI . Width0 >= <NUM_LIT> && CI . Width0 <= <NUM_LIT> ) ; assert ( CI . Width1 >= <NUM_LIT> && CI . Width1 <= <NUM_LIT> ) ; if ( ReverseOrder ) { Idx1 = Idxs [ <NUM_LIT> ] [ CI . Width1 - <NUM_LIT> ] ; Idx0 = Idxs [ CI . Width1 ] [ CI . Width0 - <NUM_LIT> ] ; } else { Idx0 = Idxs [ <NUM_LIT> ] [ CI . Width0 - <NUM_LIT> ] ;

Idx1 = Idxs [ CI . Width0 ] [ CI . Width1 - <NUM_LIT> ] ;


<s> DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format0 , CI . Width0 + CI . Width1 , * STI ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( JoinedFormat ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC0 ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ;

const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ;


<s> if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format0 , CI . Width0 + CI . Width1 , * STI ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset0 , CI . Offset1 ) ) . addImm ( JoinedFormat ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC0 ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; moveInstsAfter ( MIB , CI . InstsToMove ) ;


<s> const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format0 , CI . Width0 + CI . Width1 , * STI ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset0 , CI . Offset1 ) ) . addImm ( JoinedFormat ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC0 ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; moveInstsAfter ( MIB , CI . InstsToMove ) ; CI . I -> eraseFromParent ( ) ;


<s> <ISA_LIT>LoadStoreOptimizer ( ) : MachineFunctionPass ( ID ) {


<s> if ( MBBI -> mayLoadOrStore ( ) && ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , InstsToMove , AA ) ) ) { InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , InstsToMove ) ; continue ; } if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , InstsToMove ) ) continue ; if ( & * MBBI == & * Paired . I ) { if ( canMoveInstsAcrossMemOp ( * MBBI , InstsToMove , AA ) ) { if ( CI . InstClass != MIMG ) offsetsCanBeCombined ( CI , * STM , Paired , true ) ; return true ;


<s> unsigned Idx0 ; unsigned Idx1 ; static const unsigned Idxs [ <NUM_LIT> ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , } ; assert ( CI . Width >= <NUM_LIT> && CI . Width <= <NUM_LIT> ) ; assert ( Paired . Width >= <NUM_LIT> && Paired . Width <= <NUM_LIT> ) ; if ( ReverseOrder ) { Idx1 = Idxs [ <NUM_LIT> ] [ Paired . Width - <NUM_LIT> ] ; Idx0 = Idxs [ Paired . Width ] [ CI . Width - <NUM_LIT> ] ;


<s> if ( CI . InstClass == MIMG ) { assert ( ( countPopulation ( CI . DMask | Paired . DMask ) == CI . Width + Paired . Width ) && <STR_LIT> ) ; ReverseOrder = CI . DMask > Paired . DMask ; } else { ReverseOrder = CI . Offset > Paired . Offset ; } unsigned Idx0 ; unsigned Idx1 ; static const unsigned Idxs [ <NUM_LIT> ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , } ; assert ( CI . Width >= <NUM_LIT> && CI . Width <= <NUM_LIT> ) ; assert ( Paired . Width >= <NUM_LIT> && Paired . Width <= <NUM_LIT> ) ; if ( ReverseOrder ) { Idx1 = Idxs [ <NUM_LIT> ] [ Paired . Width - <NUM_LIT> ] ; Idx0 = Idxs [ Paired . Width ] [ CI . Width - <NUM_LIT> ] ; } else { Idx0 = Idxs [ <NUM_LIT> ] [ CI . Width - <NUM_LIT> ] ; Idx1 = Idxs [ CI . Width ] [ Paired . Width - <NUM_LIT> ] ;


<s> } case BUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeBufferLoadPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * this ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_STORE : { MachineBasicBlock :: iterator NewMI = mergeBufferStorePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * this ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case MIMG : { MachineBasicBlock :: iterator NewMI = mergeImagePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * this ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case TBUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeTBufferLoadPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * this ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case TBUFFER_STORE : { MachineBasicBlock :: iterator NewMI = mergeTBufferStorePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * this ) ;


<s> const MachineOperand & AddrReg0 = I -> getOperand ( AddrIdx ) ; const MachineOperand & AddrReg1 = MBBI -> getOperand ( AddrIdx ) ; if ( AddrReg0 . getReg ( ) == AddrReg1 . getReg ( ) && AddrReg0 . getSubReg ( ) == AddrReg1 . getSubReg ( ) ) { int OffsetIdx = <STR_LIT> ( I -> getOpcode ( ) , <STR_LIT> :: offset ) ; unsigned Offset0 = I -> getOperand ( OffsetIdx ) . getImm ( ) & <NUM_LIT> ;

unsigned Offset1 = MBBI -> getOperand ( OffsetIdx ) . getImm ( ) & <NUM_LIT> ;


<s> bool <ISA_LIT>LoadStoreOptimizer :: runOnMachineFunction ( MachineFunction & MF ) { if ( skipFunction ( * MF . getFunction ( ) ) ) return false ; const <ISA_LIT>Subtarget & STM = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ! STM . loadStoreOptEnabled ( ) ) return false ; TII = STM . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ;


<s> if ( Regs & SOFFSET ) { AddrOpName [ NumAddresses ++ ] = <STR_LIT> :: soffset ; } if ( Regs & VADDR ) { AddrOpName [ NumAddresses ++ ] = <STR_LIT> :: vaddr ; } for ( unsigned i = <NUM_LIT> ; i < NumAddresses ; i ++ ) { AddrIdx [ i ] = <STR_LIT> ( CI . I -> getOpcode ( ) , AddrOpName [ i ] ) ; AddrReg [ i ] = & CI . I -> getOperand ( AddrIdx [ i ] ) ; if ( AddrReg [ i ] -> isReg ( ) && ( TargetRegisterInfo :: isPhysicalRegister ( AddrReg [ i ] -> getReg ( ) ) || MRI -> hasOneNonDBGUse ( AddrReg [ i ] -> getReg ( ) ) ) ) return false ; } ++ MBBI ; DenseSet < unsigned > RegDefsToMove ; DenseSet < unsigned > PhysRegUsesToMove ; addDefsUsesToList ( * CI . I , RegDefsToMove , PhysRegUsesToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { const bool IsDS = ( InstClass == DS_READ ) || ( InstClass == DS_WRITE ) ; if ( ( getInstClass ( MBBI -> getOpcode ( ) ) != InstClass ) || ( IsDS && ( MBBI -> getOpcode ( ) != Opc ) ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) { return false ; } if ( MBBI -> mayLoadOrStore ( ) && ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , TII , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , TII , AA ) ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = true ; for ( unsigned i = <NUM_LIT> ; i < NumAddresses ; i ++ ) { const MachineOperand & AddrRegNext = MBBI -> getOperand ( AddrIdx [ i ] ) ; if ( AddrReg [ i ] -> isImm ( ) || AddrRegNext . isImm ( ) ) { if ( AddrReg [ i ] -> isImm ( ) != AddrRegNext . isImm ( ) || AddrReg [ i ] -> getImm ( ) != AddrRegNext . getImm ( ) ) {


<s> bool <ISA_LIT>LoadStoreOptimizer :: widthsFit ( const CombineInfo & CI ) { const unsigned Width = ( CI . Width0 + CI . Width1 ) ; switch ( CI . InstClass ) { default :


<s> assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ;


<s> unsigned NewOffset0 = CI . Offset0 ; unsigned NewOffset1 = CI . Offset1 ; unsigned Opc = CI . UseST64 ? read2ST64Opcode ( CI . EltSize ) : read2Opcode ( CI . EltSize ) ; unsigned SubRegIdx0 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ;


<s> unsigned NewOffset1 = CI . Offset1 ; unsigned Opc = CI . UseST64 ? write2ST64Opcode ( CI . EltSize ) : write2Opcode ( CI . EltSize ) ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( Data0 , Data1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Write2Desc = TII -> get ( Opc ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , CI . Paired , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; moveInstsAfter ( Write2 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Write2 << '\n' ) ;


<s> CombineInfo & CI = * I ; switch ( CI . InstClass ) { default : break ; case DS_READ : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeRead2Pair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; } break ; case DS_WRITE : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeWrite2Pair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; } break ; case S_BUFFER_LOAD_IMM : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeSBufferLoadImmPair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width0 + CI . Width1 ) < <NUM_LIT> ; } break ; case BUFFER_LOAD : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeBufferLoadPair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width0 + CI . Width1 ) < <NUM_LIT> ; } break ; case BUFFER_STORE : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeBufferStorePair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width0 + CI . Width1 ) < <NUM_LIT> ; } break ; case MIMG : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeImagePair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width0 + CI . Width1 ) < <NUM_LIT> ; } break ; case TBUFFER_LOAD : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeTBufferLoadPair ( CI ) ;


<s> std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = I -> getDebugLoc ( ) ; MachineInstrBuilder Read2 = BuildMI ( * MBB , Paired , DL , Read2Desc , DestReg ) . add ( * AddrReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . addMemOperand ( * I -> memoperands_begin ( ) ) . addMemOperand ( * Paired -> memoperands_begin ( ) ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( I ) ; I -> eraseFromParent ( ) ;


<s> if ( UseST64 ) { NewOffset0 /= <NUM_LIT> ; NewOffset1 /= <NUM_LIT> ; Opc = ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; } if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( Data0 , Data1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Write2Desc = TII -> get ( Opc ) ; DebugLoc DL = I -> getDebugLoc ( ) ; MachineInstrBuilder Write2 = BuildMI ( * MBB , Paired , DL , Write2Desc ) . add ( * Addr ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . addMemOperand ( * I -> memoperands_begin ( ) ) . addMemOperand ( * Paired -> memoperands_begin ( ) ) ; moveInstsAfter ( Write2 , InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( I ) ; I -> eraseFromParent ( ) ;


<s> for ( const MachineOperand & Def : MI . operands ( ) ) {

if ( Def . isReg ( ) && Def . isDef ( ) ) Defs . insert ( Def . getReg ( ) ) ;


<s> if ( Def . isReg ( ) && Def . isDef ( ) ) Defs . insert ( Def . getReg ( ) ) ; }


<s> if ( <STR_LIT> ( Paired . I -> getOpcode ( ) , op ) != Idx ) return false ; if ( Idx != - <NUM_LIT> && CI . I -> getOperand ( Idx ) . getImm ( ) != Paired . I -> getOperand ( Idx ) . getImm ( ) ) return false ; } unsigned MaxMask = std :: max ( CI . DMask , Paired . DMask ) ;

unsigned MinMask = std :: min ( CI . DMask , Paired . DMask ) ;


<s> const auto * LWEOp = TII . getNamedOperand ( * CI . I , <STR_LIT> :: lwe ) ; if ( ( TFEOp && TFEOp -> getImm ( ) ) || ( LWEOp && LWEOp -> getImm ( ) ) ) return false ; unsigned OperandsToMatch [ ] = { <STR_LIT> :: glc , <STR_LIT> :: slc , <STR_LIT> :: d16 , <STR_LIT> :: unorm , <STR_LIT> :: da , <STR_LIT> :: r128 } ; for ( auto op : OperandsToMatch ) { int Idx = <STR_LIT> ( CI . I -> getOpcode ( ) , op ) ; if ( <STR_LIT> ( Paired . I -> getOpcode ( ) , op ) != Idx ) return false ; if ( Idx != - <NUM_LIT> && CI . I -> getOperand ( Idx ) . getImm ( ) != Paired . I -> getOperand ( Idx ) . getImm ( ) ) return false ; } unsigned MaxMask = std :: max ( CI . DMask , Paired . DMask ) ; unsigned MinMask = std :: min ( CI . DMask , Paired . DMask ) ; unsigned AllowedBitsForMin = llvm :: countTrailingZeros ( MaxMask ) ;

if ( ( <NUM_LIT> << AllowedBitsForMin ) <= MinMask ) return false ;


<s> int Swizzled = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: swz ) ; if ( Swizzled != - <NUM_LIT> && CI . I -> getOperand ( Swizzled ) . getImm ( ) ) return false ; ++ MBBI ; DenseSet < unsigned > RegDefsToMove ; DenseSet < unsigned > PhysRegUsesToMove ; addDefsUsesToList ( * CI . I , RegDefsToMove , PhysRegUsesToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { if ( ( getInstClass ( MBBI -> getOpcode ( ) , * TII ) != InstClass ) || ( getInstSubclass ( MBBI -> getOpcode ( ) , * TII ) != InstSubclass ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) { return false ; } if ( MBBI -> mayLoadOrStore ( ) && ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , AA ) ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; int Swizzled = <STR_LIT> ( MBBI -> getOpcode ( ) , <STR_LIT> :: swz ) ; if ( Swizzled != - <NUM_LIT> && MBBI -> getOperand ( Swizzled ) . getImm ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = CI . hasSameBaseAddress ( * MBBI ) ;


<s> assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ;


<s> if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; CI . I -> eraseFromParent ( ) ; Paired . I -> eraseFromParent ( ) ; return New ;


<s> MachineBasicBlock :: iterator <ISA_LIT>LoadStoreOptimizer :: mergeBufferStorePair ( CombineInfo & CI , CombineInfo & Paired ) { MachineBasicBlock * MBB = CI . I -> getParent ( ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI , Paired ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset , Paired . Offset ) ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ;

moveInstsAfter ( MIB , CI . InstsToMove ) ;


<s> const unsigned Opcode = getNewOpcode ( CI , Paired ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ;

MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset , Paired . Offset ) ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ;


<s> } assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; unsigned SubRegIdx0 , SubRegIdx1 ; std :: tie ( SubRegIdx0 , SubRegIdx1 ) = getSubRegIdxs ( CI , Paired ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ;

moveInstsAfter ( Copy1 , CI . InstsToMove ) ;


<s> assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; Register BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , Paired . I , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , Paired . I , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * Paired . I } ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; CI . I -> eraseFromParent ( ) ; Paired . I -> eraseFromParent ( ) ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Read2 << '\n' ) ;


<s> const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; CI . I -> eraseFromParent ( ) ;


<s> const unsigned Opcode = getNewOpcode ( CI , Paired ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset , Paired . Offset ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ;

MachineInstr * New = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC ) . addImm ( CI . DLC ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ;


<s> MachineBasicBlock :: iterator <ISA_LIT>LoadStoreOptimizer :: mergeTBufferLoadPair ( CombineInfo & CI , CombineInfo & Paired ) { MachineBasicBlock * MBB = CI . I -> getParent ( ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI , Paired ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset , Paired . Offset ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , * STI ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( JoinedFormat ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ;


<s> const unsigned Opcode = getNewOpcode ( CI , Paired ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset , Paired . Offset ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , * STI ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( JoinedFormat ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ;

const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ;


<s> Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , * STI ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ;

const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ;


<s> const MCInstrDesc & Write2Desc = TII -> get ( Opc ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; Register BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , Paired . I , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , Paired . I , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * Paired . I } ) ; moveInstsAfter ( Write2 , CI . InstsToMove ) ; CI . I -> eraseFromParent ( ) ;


<s> if ( Info0 -> BitsPerComp != <NUM_LIT> ) return false ; if ( getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , STI ) == <NUM_LIT> ) return false ; } unsigned EltOffset0 = CI . Offset / CI . EltSize ; unsigned EltOffset1 = Paired . Offset / CI . EltSize ; CI . UseST64 = false ; CI . BaseOff = <NUM_LIT> ; if ( ( CI . InstClass != DS_READ ) && ( CI . InstClass != DS_WRITE ) ) { return ( EltOffset0 + CI . Width == EltOffset1 || EltOffset1 + Paired . Width == EltOffset0 ) && CI . GLC == Paired . GLC && CI . DLC == Paired . DLC && ( CI . InstClass == S_BUFFER_LOAD_IMM || CI . SLC == Paired . SLC ) ; } if ( ( EltOffset0 % <NUM_LIT> == <NUM_LIT> ) && ( EltOffset1 % <NUM_LIT> ) == <NUM_LIT> && isUInt < <NUM_LIT> > ( EltOffset0 / <NUM_LIT> ) && isUInt < <NUM_LIT> > ( EltOffset1 / <NUM_LIT> ) ) { CI . Offset = EltOffset0 / <NUM_LIT> ; Paired . Offset = EltOffset1 / <NUM_LIT> ; CI . UseST64 = true ; return true ; } if ( isUInt < <NUM_LIT> > ( EltOffset0 ) && isUInt < <NUM_LIT> > ( EltOffset1 ) ) { CI . Offset = EltOffset0 ; Paired . Offset = EltOffset1 ; return true ; } unsigned OffsetDiff = std :: abs ( ( int ) EltOffset1 - ( int ) EltOffset0 ) ; CI . BaseOff = std :: min ( CI . Offset , Paired . Offset ) ; if ( ( OffsetDiff % <NUM_LIT> == <NUM_LIT> ) && isUInt < <NUM_LIT> > ( OffsetDiff / <NUM_LIT> ) ) { CI . Offset = ( EltOffset0 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; Paired . Offset = ( EltOffset1 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; CI . UseST64 = true ;


<s> CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeBufferLoadPair ( CI , Paired ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_STORE : { MachineBasicBlock :: iterator NewMI = mergeBufferStorePair ( CI , Paired ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case MIMG : { MachineBasicBlock :: iterator NewMI = mergeImagePair ( CI , Paired ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case TBUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeTBufferLoadPair ( CI , Paired ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case TBUFFER_STORE : {

MachineBasicBlock :: iterator NewMI = mergeTBufferStorePair ( CI , Paired ) ;


<s> if ( Op . isReg ( ) ) { if ( Op . isDef ( ) ) RegDefs . insert ( Op . getReg ( ) ) ; else if ( Op . readsReg ( ) && TargetRegisterInfo :: isPhysicalRegister ( Op . getReg ( ) ) ) PhysRegUses . insert ( Op . getReg ( ) ) ; } }


<s> else if ( Op . readsReg ( ) && TargetRegisterInfo :: isPhysicalRegister ( Op . getReg ( ) ) ) PhysRegUses . insert ( Op . getReg ( ) ) ; }


<s> for ( MachineOperand & Use : MI . operands ( ) ) {

if ( Use . isReg ( ) && ( ( Use . readsReg ( ) && RegDefs . count ( Use . getReg ( ) ) ) || ( Use . isDef ( ) && RegDefs . count ( Use . getReg ( ) ) ) || ( Use . isDef ( ) && TargetRegisterInfo :: isPhysicalRegister ( Use . getReg ( ) ) && PhysRegUses . count ( Use . getReg ( ) ) ) ) ) {


<s> static bool addToListsIfDependent ( MachineInstr & MI , DenseSet < unsigned > & RegDefs , DenseSet < unsigned > & PhysRegUses , SmallVectorImpl < MachineInstr * > & Insts ) { for ( MachineOperand & Use : MI . operands ( ) ) { if ( Use . isReg ( ) && ( ( Use . readsReg ( ) && RegDefs . count ( Use . getReg ( ) ) ) || ( Use . isDef ( ) && RegDefs . count ( Use . getReg ( ) ) ) || ( Use . isDef ( ) && TargetRegisterInfo :: isPhysicalRegister ( Use . getReg ( ) ) && PhysRegUses . count ( Use . getReg ( ) ) ) ) ) { Insts . push_back ( & MI ) ; addDefsUsesToList ( MI , RegDefs , PhysRegUses ) ;


<s> static bool canMoveInstsAcrossMemOp ( MachineInstr & MemOp , ArrayRef < MachineInstr * > InstsToMove , AliasAnalysis * AA ) { assert ( MemOp . mayLoadOrStore ( ) ) ; for ( MachineInstr * InstToMove : InstsToMove ) { if ( ! InstToMove -> mayLoadOrStore ( ) ) continue ; if ( ! memAccessesCanBeReordered ( MemOp , * InstToMove , AA ) ) return false ; }


<s> unsigned MinOffset = std :: min ( A -> getOffset ( ) , B -> getOffset ( ) ) ; unsigned Size = A -> getSize ( ) + B -> getSize ( ) ; return MF . getMachineMemOperand ( A , MinOffset , Size ) ;


<s> MachineInstr * HiHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub1 ) . addReg ( DeadCarryReg , RegState :: Define | RegState :: Dead ) . addReg ( Addr . Base . HiReg , <NUM_LIT> , Addr . Base . HiSubReg ) . add ( OffsetHi ) . addReg ( CarryReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; ( void ) HiHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; HiHalf -> dump ( ) ; ) ; unsigned FullDestReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ;

MachineInstr * FullBase = BuildMI ( * MBB , MBBI , DL , TII -> get ( TargetOpcode :: REG_SEQUENCE ) , FullDestReg ) . addReg ( DestSub0 ) . addImm ( <STR_LIT> ) . addReg ( DestSub1 ) . addImm ( <STR_LIT> ) ;


<s> unsigned DestSub1 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * LoHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub0 ) . addReg ( CarryReg , RegState :: Define ) . addReg ( Addr . Base . LoReg , <NUM_LIT> , Addr . Base . LoSubReg ) . add ( OffsetLo ) . addImm ( <NUM_LIT> ) ; ( void ) LoHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; LoHalf -> dump ( ) ; ) ; MachineInstr * HiHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub1 ) . addReg ( DeadCarryReg , RegState :: Define | RegState :: Dead ) . addReg ( Addr . Base . HiReg , <NUM_LIT> , Addr . Base . HiSubReg ) . add ( OffsetHi ) . addReg ( CarryReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; ( void ) HiHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; HiHalf -> dump ( ) ; ) ; unsigned FullDestReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * FullBase = BuildMI ( * MBB , MBBI , DL , TII -> get ( TargetOpcode :: REG_SEQUENCE ) , FullDestReg ) . addReg ( DestSub0 ) . addImm ( <STR_LIT> ) . addReg ( DestSub1 ) . addImm ( <STR_LIT> ) ; ( void ) FullBase ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; FullBase -> dump ( ) ; dbgs ( ) << <STR_LIT> ; ) ; return FullDestReg ;


<s> unsigned Reg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * Mov = BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Reg ) . addImm ( Val ) ; ( void ) Mov ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Mov -> dump ( ) ) ;

return MachineOperand :: CreateReg ( Reg , false ) ;


<s> MachineOperand <ISA_LIT>LoadStoreOptimizer :: createRegOrImm ( int32_t Val , MachineInstr & MI ) { APInt V ( <NUM_LIT> , Val , true ) ;

if ( TII -> isInlineConstant ( V ) ) return MachineOperand :: CreateImm ( Val ) ;


<s> MachineInstr * Def = MRI -> getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Def || Def -> getOpcode ( ) != <STR_LIT> || ! Def -> getOperand ( <NUM_LIT> ) . isImm ( ) ) return None ;

return Def -> getOperand ( <NUM_LIT> ) . getImm ( ) ;


<s> MachineInstr * Def = MRI -> getUniqueVRegDef ( Op . getReg ( ) ) ;

if ( ! Def || Def -> getOpcode ( ) != <STR_LIT> || ! Def -> getOperand ( <NUM_LIT> ) . isImm ( ) ) return None ;


<s> switch ( baseOpcode ) { default : return UNKNOWN ; case <STR_LIT> : return BUFFER_LOAD_OFFEN ; case <STR_LIT> : return BUFFER_LOAD_OFFSET ; case <STR_LIT> : return BUFFER_STORE_OFFEN ; case <STR_LIT> : return BUFFER_STORE_OFFSET ; case <STR_LIT> : return BUFFER_LOAD_OFFEN_exact ; case <STR_LIT> : return BUFFER_LOAD_OFFSET_exact ; case <STR_LIT> : return BUFFER_STORE_OFFEN_exact ; case <STR_LIT> : return BUFFER_STORE_OFFSET_exact ; } } switch ( Opc ) { default : return UNKNOWN ;


<s> default : return <STR_LIT> ( CI . InstClass , Width ) ; case UNKNOWN : llvm_unreachable ( <STR_LIT> ) ; case S_BUFFER_LOAD_IMM :


<s> unsigned <ISA_LIT>LoadStoreOptimizer :: getNewOpcode ( const CombineInfo & CI ) { const unsigned Width = CI . Width0 + CI . Width1 ; switch ( CI . InstClass ) { default : return <STR_LIT> ( CI . InstClass , Width ) ; case UNKNOWN :


<s> default : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> :


<s> const unsigned Opc = MI . getOpcode ( ) ; if ( TII -> isMUBUF ( MI ) ) { return <STR_LIT> ( Opc ) ; } switch ( Opc ) { default : return <NUM_LIT> ; case <STR_LIT> :


<s> StringRef getPassName ( ) const override { return <STR_LIT> ;


<s> result |= SOFFSET ; } return result ; } switch ( Opc ) { default : return <NUM_LIT> ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return SBASE ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return ADDR ; }


<s> const TargetRegisterClass * <ISA_LIT>LoadStoreOptimizer :: getTargetRegisterClass ( const CombineInfo & CI ) { if ( CI . InstClass == S_BUFFER_LOAD_IMM ) { switch ( CI . Width0 + CI . Width1 ) { default : return nullptr ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ;


<s> static bool memAccessesCanBeReordered ( MachineBasicBlock :: iterator A , MachineBasicBlock :: iterator B , AliasAnalysis * AA ) {

return ! ( A -> mayStore ( ) || B -> mayStore ( ) ) || ! A -> mayAlias ( AA , * B , true ) ;


<s> DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ;


<s> const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset0 , CI . Offset1 ) ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC0 ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ;


<s> BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ;

const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ;


<s> const auto * AddrReg = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: addr ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdst ) ; unsigned NewOffset0 = CI . Offset0 ; unsigned NewOffset1 = CI . Offset1 ; unsigned Opc = CI . UseST64 ? read2ST64Opcode ( CI . EltSize ) : read2Opcode ( CI . EltSize ) ; unsigned SubRegIdx0 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ;

const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ;


<s> unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . DLC0 ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ;


<s> unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . DLC0 ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ;

MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ;


<s> const MachineOperand * AddrReg = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: addr ) ; const MachineOperand * Data0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: data0 ) ; const MachineOperand * Data1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: data0 ) ; unsigned NewOffset0 = CI . Offset0 ; unsigned NewOffset1 = CI . Offset1 ; unsigned Opc = CI . UseST64 ? write2ST64Opcode ( CI . EltSize ) : write2Opcode ( CI . EltSize ) ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( Data0 , Data1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Write2Desc = TII -> get ( Opc ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , CI . Paired , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; moveInstsAfter ( Write2 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Write2 << '\n' ) ; return Next ;


<s> BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , CI . Paired , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; moveInstsAfter ( Write2 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ;

LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Write2 << '\n' ) ;


<s> MachineBasicBlock * MBB = I -> getParent ( ) ; ++ I ; for ( MachineInstr * MI : InstsToMove ) { MI -> removeFromParent ( ) ;


<s> CI . Offset0 = EltOffset0 / <NUM_LIT> ; CI . Offset1 = EltOffset1 / <NUM_LIT> ; CI . UseST64 = true ; return true ; } if ( isUInt < <NUM_LIT> > ( EltOffset0 ) && isUInt < <NUM_LIT> > ( EltOffset1 ) ) { CI . Offset0 = EltOffset0 ; CI . Offset1 = EltOffset1 ; return true ; } unsigned OffsetDiff = std :: abs ( ( int ) EltOffset1 - ( int ) EltOffset0 ) ; CI . BaseOff = std :: min ( CI . Offset0 , CI . Offset1 ) ; if ( ( OffsetDiff % <NUM_LIT> == <NUM_LIT> ) && isUInt < <NUM_LIT> > ( OffsetDiff / <NUM_LIT> ) ) { CI . Offset0 = ( EltOffset0 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; CI . Offset1 = ( EltOffset1 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; CI . UseST64 = true ;