<s> DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned Opcode ; if ( CI . InstClass == BUFFER_LOAD_OFFEN ) { Opcode = CI . IsX2 ? <STR_LIT> : <STR_LIT> ; } else { Opcode = CI . IsX2 ? <STR_LIT> : <STR_LIT> ; } const TargetRegisterClass * SuperRC = CI . IsX2 ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) ; if ( CI . InstClass == BUFFER_LOAD_OFFEN ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . setMemRefs ( CI . I -> mergeMemRefsWith ( * CI . Paired ) ) ; unsigned SubRegIdx0 = CI . IsX2 ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = CI . IsX2 ? <STR_LIT>


<s> auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; if ( CI . InstClass == BUFFER_STORE_OFFEN ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset0 , CI . Offset1 ) ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . setMemRefs ( CI . I -> mergeMemRefsWith ( * CI . Paired ) ) ; moveInstsAfter ( MIB , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std ::


<s> unsigned SubRegIdx1 = CI . IsX2 ? <STR_LIT> : <STR_LIT> ; if ( CI . Offset0 > CI . Offset1 ) std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; const TargetRegisterClass * SuperRC = CI . IsX2 ? & <STR_LIT> : & <STR_LIT> ; unsigned SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; if ( CI . InstClass == BUFFER_STORE_OFFEN ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset0 , CI . Offset1 )

) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . setMemRefs ( CI . I -> mergeMemRefsWith ( * CI . Paired ) ) ;


<s> unsigned NewOffset1 = CI . Offset1 ; unsigned Opc = CI . UseST64 ? read2ST64Opcode ( CI . EltSize ) : read2Opcode ( CI . EltSize ) ; unsigned SubRegIdx0 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState


<s> unsigned NewOffset0 = CI . Offset0 ; unsigned NewOffset1 = CI . Offset1 ; unsigned Opc = CI . UseST64 ? read2ST64Opcode ( CI . EltSize ) : read2Opcode ( CI . EltSize ) ; unsigned SubRegIdx0 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I ->


<s> const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent (


<s> std :: swap ( Data0 , Data1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Write2Desc = TII -> get ( Opc ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = Addr -> getReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; unsigned AddOpc = STM -> hasAddNoCarry ( ) ? <STR_LIT> : <STR_LIT> ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( AddOpc ) , BaseReg ) . addImm ( CI . BaseOff ) . addReg ( Addr -> getReg ( ) ) ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , CI . Paired , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . setMemRefs ( CI . I -> mergeMemRefsWith ( * CI . Paired ) ) ; moveInstsAfter ( Write2 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ; DEBUG ( dbgs ( ) << <STR_LIT>


<s> Register DestSub1 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * LoHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub0 ) . addReg ( CarryReg , RegState :: Define ) . addReg ( Addr . Base . LoReg , <NUM_LIT> , Addr . Base . LoSubReg ) . add ( OffsetLo ) . addImm ( <NUM_LIT> ) ; ( void ) LoHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; LoHalf -> dump ( ) ; ) ; MachineInstr * HiHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub1 ) . addReg ( DeadCarryReg , RegState :: Define | RegState :: Dead ) . addReg ( Addr . Base . HiReg , <NUM_LIT> , Addr . Base . HiSubReg ) . add ( OffsetHi ) . addReg ( CarryReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; ( void )


<s> const InstClassEnum InstClass = getInstClass ( Opc , * TII ) ; if ( InstClass == UNKNOWN ) { return false ; } const unsigned InstSubclass = getInstSubclass ( Opc , * TII ) ; int Swizzled = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: swz ) ; if ( Swizzled != - <NUM_LIT> && CI . I -> getOperand ( Swizzled ) . getImm ( ) ) return false ; ++ MBBI ; DenseSet < unsigned > RegDefsToMove ; DenseSet < unsigned > PhysRegUsesToMove ; addDefsUsesToList ( * CI . I , RegDefsToMove , PhysRegUsesToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { if ( ( getInstClass ( MBBI -> getOpcode ( ) , * TII ) != InstClass ) || ( getInstSubclass ( MBBI -> getOpcode ( ) , * TII ) != InstSubclass ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) { return false ; } if ( MBBI -> mayLoadOrStore ( ) && ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , AA ) ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = CI . hasSameBaseAddress ( * MBBI ) ; if ( Match ) { CI . setPaired ( MBBI , * TII ) ; if ( widthsFit ( * STM , CI ) && offsetsCanBeCombined ( CI ) ) if ( canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , AA ) ) return true ; } if ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , AA ) || ! canMoveInstsAcrossMemOp ( *

MBBI , CI . InstsToMove , AA ) ) break ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return S_BUFFER_LOAD_IMM ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return


<s> unsigned <ISA_LIT>LoadStoreOptimizer :: getNewOpcode ( const CombineInfo & CI ) { const unsigned Width = CI . Width0 + CI . Width1 ; switch ( CI . InstClass ) { default : assert ( CI . InstClass == BUFFER_LOAD || CI . InstClass == BUFFER_STORE ) ; return <STR_LIT> ( <STR_LIT> ( CI . I -> getOpcode ( ) ) , Width ) ; case


<s> assert ( CI . Width1 >= <NUM_LIT> && CI . Width1 <= <NUM_LIT> ) ; if ( ReverseOrder ) { Idx1 = Idxs [ <NUM_LIT> ] [ CI . Width1 - <NUM_LIT> ] ; Idx0 = Idxs [ CI . Width1 ] [ CI . Width0 - <NUM_LIT> ] ; } else { Idx0 = Idxs [ <NUM_LIT> ] [ CI . Width0 - <NUM_LIT> ] ; Idx1 = Idxs [ CI . Width0 ] [ CI . Width1 - <NUM_LIT> ] ; } return std :: make_pair (


<s> for ( unsigned i = <NUM_LIT> ; i < NumAddresses ; ++ i ) { const MachineOperand * AddrOp = AddrReg [ i ] ; if ( AddrOp -> isImm ( ) ) continue ; if ( ! AddrOp -> isReg ( ) )


<s> const MachineOperand * AddrOp = AddrReg [ i ] ; if ( AddrOp -> isImm ( ) ) continue ; if ( ! AddrOp -> isReg ( ) ) return false ; if ( Register :: isPhysicalRegister ( AddrOp -> getReg (


<s> const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC0 ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand (


<s> const unsigned Opcode = getNewOpcode ( CI ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand


<s> TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Read2 << '\n' ) ; return Read2


<s> const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MachineInstr * New = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . DLC0 ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( *


<s> } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Write2Desc = TII -> get ( Opc ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; Register BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , CI . Paired , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; moveInstsAfter ( Write2 , CI . InstsToMove ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT>


<s> if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( Data0 , Data1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Write2Desc = TII -> get ( Opc ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; Register BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill


<s> bool Modified = false ; for ( std :: list < CombineInfo > & MergeList : MergeableInsts ) { if ( MergeList . size ( ) < <NUM_LIT> ) continue ; bool OptimizeListAgain = false ; if ( ! optimizeInstsWithSameBaseAddr ( MergeList , OptimizeListAgain ) ) { MergeList . clear ( ) ; continue ; } if ( ! OptimizeListAgain ) MergeList . clear ( ) ; OptimizeAgain |= OptimizeListAgain ; Modified = true ; } return Modified


<s> MachineBasicBlock :: iterator NewMI = mergeWrite2Pair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; } break ; case S_BUFFER_LOAD_IMM : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeSBufferLoadImmPair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width0 + CI . Width1 ) < <NUM_LIT> ; } break ; case BUFFER_LOAD : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeBufferLoadPair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width0 + CI . Width1 ) < <NUM_LIT> ; } break ; case BUFFER_STORE : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeBufferStorePair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width0 + CI . Width1 ) < <NUM_LIT> ; } break ; } CI . InstsToMove . clear ( ) ; } return


<s> MachineInstr * BaseHiDef = MRI -> getUniqueVRegDef ( BaseHi . getReg ( ) ) ; if ( ! BaseLoDef || BaseLoDef -> getOpcode ( ) != <STR_LIT> || ! BaseHiDef || BaseHiDef -> getOpcode ( ) != <STR_LIT> ) return ; const auto * Src0 = TII -> getNamedOperand ( * BaseLoDef , <STR_LIT> :: src0 ) ; const auto * Src1 = TII -> getNamedOperand ( * BaseLoDef , <STR_LIT> :: src1 ) ; auto Offset0P = extractConstOffset ( * Src0 ) ; if ( Offset0P ) BaseLo = * Src1 ; else { if ( ! ( Offset0P = extractConstOffset ( * Src1 ) ) ) return ; BaseLo = * Src0 ; } Src0 = TII -> getNamedOperand ( * BaseHiDef , <STR_LIT> :: src0 ) ; Src1 = TII -> getNamedOperand ( * BaseHiDef , <STR_LIT> :: src1 ) ; if ( Src0 -> isImm ( ) ) std :: swap ( Src0 , Src1 ) ; if ( ! Src1 -> isImm ( ) ) return ; uint64_t Offset1 = Src1 -> getImm (


<s> if ( MI . mayLoad ( ) && TII -> getNamedOperand ( MI , <STR_LIT> :: vdata ) != NULL ) return false ; if ( AnchorList . count ( & MI ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ) ; if ( TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> getImm ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; return false ; } MachineOperand & Base = * TII -> getNamedOperand ( MI , <STR_LIT> :: vaddr ) ; MemAddress MAddr ; if ( Visited . find ( & MI ) == Visited . end ( ) ) { processBaseWithConstOffset ( Base , MAddr ) ; Visited [ & MI ] = MAddr ; } else MAddr = Visited [ & MI ] ; if ( MAddr . Offset == <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ; ) ; return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MAddr . Base . HiReg << <STR_LIT> << MAddr . Base . LoReg << <STR_LIT> << MAddr . Offset << <STR_LIT> ; ) ; MachineInstr * AnchorInst = nullptr ; MemAddress AnchorAddr ; uint32_t MaxDist = std :: numeric_limits < uint32_t > :: min ( ) ; SmallVector < std :: pair < MachineInstr * , int64_t > , <NUM_LIT> > InstsWCommonBase ; MachineBasicBlock * MBB = MI . getParent ( ) ; MachineBasicBlock :: iterator E = MBB -> end ( ) ; MachineBasicBlock :: iterator MBBI = MI . getIterator ( ) ; ++ MBBI ; const <ISA_LIT>TargetLowering * TLI = static_cast < const <ISA_LIT>TargetLowering * > ( STM -> getTargetLowering ( ) ) ; for ( ; MBBI != E ; ++ MBBI ) { MachineInstr & MINext = * MBBI ; if ( MINext . getOpcode ( ) != MI . getOpcode ( ) || TII -> getNamedOperand ( MINext , <STR_LIT> :: offset ) -> getImm ( ) ) continue ; const MachineOperand & BaseNext = * TII -> getNamedOperand ( MINext , <STR_LIT> :: vaddr ) ; MemAddress MAddrNext ; if ( Visited . find ( & MINext ) == Visited . end ( ) ) { processBaseWithConstOffset ( BaseNext , MAddrNext ) ; Visited [ & MINext ] = MAddrNext ; } else MAddrNext = Visited [ & MINext ] ; if ( MAddrNext . Base . LoReg != MAddr . Base . LoReg || MAddrNext . Base . HiReg != MAddr . Base . HiReg || MAddrNext . Base . LoSubReg != MAddr . Base . LoSubReg || MAddrNext . Base . HiSubReg != MAddr . Base . HiSubReg ) continue ; InstsWCommonBase . push_back ( std :: make_pair ( & MINext , MAddrNext . Offset ) ) ; int64_t Dist = MAddr .


<s> TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo ( ) ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; assert ( MRI -> isSSA ( ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Modified = false


<s> static void addDefsUsesToList ( const MachineInstr & MI , DenseSet < Register > & RegDefs , DenseSet < Register > & RegUses ) { for ( const auto & Op : MI . operands ( ) ) { if ( ! Op . isReg ( )


<s> void <ISA_LIT>LoadStoreOptimizer :: addInstToMergeableList ( const CombineInfo & CI , std :: list < std :: list < CombineInfo > > & MergeableInsts ) const { for ( std :: list < CombineInfo > & AddrList : MergeableInsts


<s> break ; } const InstClassEnum InstClass = getInstClass ( MI . getOpcode ( ) , * TII ) ; if ( InstClass == UNKNOWN ) continue ; int Swizzled = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: swz ) ; if ( Swizzled != - <NUM_LIT> && MI . getOperand ( Swizzled ) . getImm ( ) ) continue ; CombineInfo CI ; CI . setMI ( MI , * this ) ; CI . Order = Order ++ ; if ( ! CI . hasMergeableAddress ( * MRI ) ) continue ; if ( CI . InstClass == DS_WRITE && CI . IsAGPR ) { continue ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; addInstToMergeableList ( CI , MergeableInsts ) ; } for ( std :: list < std :: list < CombineInfo >> :: iterator I = MergeableInsts . begin ( ) , E = MergeableInsts . end ( ) ; I != E ; ) { std :: list < CombineInfo > & MergeList = * I ; if ( MergeList . size ( ) <= <NUM_LIT> ) { I = MergeableInsts . erase ( I ) ; continue ; } MergeList . sort ( [ ] ( const CombineInfo & A


<s> MachineMemOperand * <ISA_LIT>LoadStoreOptimizer :: combineKnownAdjacentMMOs ( const CombineInfo & CI , const CombineInfo & Paired ) { const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; unsigned Size = MMOa -> getSize ( ) + MMOb -> getSize ( ) ; if ( Paired < CI ) std :: swap ( MMOa , MMOb ) ; MachinePointerInfo PtrInfo ( MMOa -> getPointerInfo ( ) ) ; if ( MMOb -> getAddrSpace ( ) == <ISA_LIT>AS :: FLAT_ADDRESS ) PtrInfo . AddrSpace = <ISA_LIT>AS :: FLAT_ADDRESS ; MachineFunction * MF = CI . I -> getMF ( ) ; return MF -> getMachineMemOperand ( MMOa , PtrInfo


<s> unsigned Size = MMOa -> getSize ( ) + MMOb -> getSize ( ) ; if ( Paired < CI ) std :: swap ( MMOa , MMOb ) ; MachinePointerInfo PtrInfo ( MMOa -> getPointerInfo ( ) ) ; if ( MMOb -> getAddrSpace ( ) == <ISA_LIT>AS :: FLAT_ADDRESS ) PtrInfo . AddrSpace = <ISA_LIT>AS :: FLAT_ADDRESS ; MachineFunction * MF = CI . I -> getMF ( ) ; return MF -> getMachineMemOperand ( MMOa , PtrInfo


<s> if ( ( CI . InstClass == FLAT_LOAD || CI . InstClass == FLAT_STORE ) && <ISA_LIT>InstrInfo :: isFLATGlobal ( * CI . I ) && <ISA_LIT>InstrInfo :: isFLATGlobal ( * Paired . I ) ) return ( CI . InstClass == FLAT_STORE ) ? GLOBAL_STORE : GLOBAL_LOAD ; return CI .


<s> assert ( CI . InstClass == Paired . InstClass ) ; if ( ( CI . InstClass == FLAT_LOAD || CI . InstClass == FLAT_STORE ) && <ISA_LIT>InstrInfo :: isFLATGlobal ( * CI . I ) && <ISA_LIT>InstrInfo :: isFLATGlobal ( * Paired . I ) ) return

( CI . InstClass == FLAT_STORE ) ? GLOBAL_STORE : GLOBAL_LOAD ;


<s> switch ( <STR_LIT> ( Opc ) ) { default : return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } if ( TII . isMIMG ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) == - <NUM_LIT> && <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) == - <NUM_LIT> ) return UNKNOWN ; if ( <STR_LIT> ( Opc ) -> BVH ) return UNKNOWN ; if ( TII . get ( Opc ) . mayStore ( ) || ! TII . get ( Opc ) . mayLoad ( ) || TII . isGather4 ( Opc ) ) return UNKNOWN ; return MIMG ; } if ( TII . isMTBUF ( Opc ) ) { switch ( <STR_LIT> ( Opc ) ) { default : return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_STORE ; } } return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return S_BUFFER_LOAD_IMM ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return DS_READ ; case <STR_LIT> : case


<s> if ( VAddr0Idx >= <NUM_LIT> ) { int SRsrcIdx = <STR_LIT> ( Opc , <STR_LIT> :: srsrc ) ; Result . NumVAddrs = SRsrcIdx - VAddr0Idx ; } else { Result . VAddr = true ; } Result . SRsrc = true ; const <STR_LIT> * Info = <STR_LIT> ( Opc ) ; if ( Info && <STR_LIT> ( Info -> BaseOpcode ) -> Sampler ) Result . SSamp = true ; return Result ; } if ( TII . isMTBUF ( Opc ) ) { if ( <STR_LIT> ( Opc ) ) Result . VAddr = true ; if ( <STR_LIT> ( Opc ) ) Result . SRsrc = true ; if ( <STR_LIT> ( Opc ) ) Result . SOffset = true ; return Result ; } switch ( Opc ) { default : return Result ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : Result . SBase = true ; return Result ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT>


<s> assert ( CI . Width >= <NUM_LIT> && CI . Width <= <NUM_LIT> ) ; assert ( Paired . Width >= <NUM_LIT> && Paired . Width <= <NUM_LIT> ) ; if ( Paired < CI ) { Idx1 = Idxs [ <NUM_LIT> ] [ Paired . Width - <NUM_LIT> ] ; Idx0 = Idxs [ Paired . Width ] [ CI . Width - <NUM_LIT> ] ; } else { Idx0 = Idxs [ <NUM_LIT> ] [


<s> Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset , Paired . Offset ) ; auto MIB = BuildMI ( * MBB , InsertBefore , DL , TII -> get ( Opcode ) , DestReg ) ; AddressRegs Regs = getRegs ( Opcode , * TII ) ; if ( Regs . VAddr ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . CPol ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( CI , Paired ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I


<s> Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , InsertBefore , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , InsertBefore , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; AddressRegs Regs = getRegs ( Opcode , * TII ) ; if ( Regs . VAddr ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand


<s> Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; auto MIB = BuildMI ( * MBB , InsertBefore , DL , TII -> get ( Opcode ) , DestReg ) ; if ( auto * SAddr = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: saddr ) ) MIB . add ( * SAddr ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) . addImm ( std :: min ( CI . Offset , Paired . Offset ) ) . addImm ( CI . CPol ) . addMemOperand ( combineKnownAdjacentMMOs ( CI , Paired ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I


<s> MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) . addImm ( std :: min ( CI . Offset , Paired . Offset ) ) . addImm ( CI . CPol ) . addMemOperand ( combineKnownAdjacentMMOs ( CI , Paired ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdst ) ; BuildMI ( * MBB , InsertBefore , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; BuildMI ( * MBB , InsertBefore , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; CI . I -> eraseFromParent ( ) ; Paired . I -> eraseFromParent ( ) ; return New


<s> Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedDMask = CI . DMask | Paired . DMask ; unsigned DMaskIdx = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: dmask ) ; auto MIB = BuildMI ( * MBB , InsertBefore , DL , TII -> get ( Opcode ) , DestReg ) ; for ( unsigned I = <NUM_LIT> , E = ( * CI . I ) . getNumOperands ( ) ; I != E ; ++ I ) { if ( I == DMaskIdx ) MIB . addImm ( MergedDMask ) ; else MIB . add ( ( * CI . I ) . getOperand ( I ) ) ; } assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand


<s> unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , InsertBefore , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , InsertBefore , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , InsertBefore , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * Paired . I } ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , InsertBefore , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; BuildMI ( * MBB , InsertBefore , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; CI . I -> eraseFromParent ( ) ; Paired . I -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Read2 << '\n' ) ; return


<s> const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , InsertBefore , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; BuildMI ( * MBB , InsertBefore , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; CI . I -> eraseFromParent


<s> const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset , Paired . Offset ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; MachineInstr * New = BuildMI ( * MBB , InsertBefore , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . CPol ) . addMemOperand ( combineKnownAdjacentMMOs ( CI , Paired ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , InsertBefore , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg ,


<s> MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( JoinedFormat ) . addImm ( CI . CPol ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( CI , Paired ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get


<s> const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , InsertBefore , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , InsertBefore , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; AddressRegs Regs = getRegs ( Opcode , * TII ) ; if ( Regs . VAddr ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , * STM ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired .


<s> BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , InsertBefore , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , InsertBefore , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * Paired . I } ) ; CI . I -> eraseFromParent ( ) ; Paired . I -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Write2 << '\n' ) ; return Write2


<s> if ( Info0 -> BitsPerComp != Info1 -> BitsPerComp || Info0 -> NumFormat != Info1 -> NumFormat ) return false ; if ( Info0 -> BitsPerComp != <NUM_LIT> ) return false ; if ( getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , STI ) == <NUM_LIT> ) return false ; } uint32_t EltOffset0 = CI . Offset / CI . EltSize ; uint32_t EltOffset1 = Paired . Offset / CI . EltSize ; CI . UseST64 = false ; CI . BaseOff = <NUM_LIT> ; if ( ( CI . InstClass != DS_READ ) && ( CI . InstClass != DS_WRITE ) ) { return ( EltOffset0 + CI . Width == EltOffset1 || EltOffset1 + Paired . Width == EltOffset0 ) && CI . CPol == Paired . CPol ; } if ( ( EltOffset0 % <NUM_LIT> == <NUM_LIT> ) && ( EltOffset1 % <NUM_LIT> ) == <NUM_LIT> && isUInt < <NUM_LIT> > ( EltOffset0 / <NUM_LIT> ) && isUInt < <NUM_LIT> > ( EltOffset1 / <NUM_LIT> ) ) { if ( Modify ) { CI . Offset = EltOffset0 / <NUM_LIT> ; Paired . Offset = EltOffset1 / <NUM_LIT> ; CI . UseST64 = true ; } return true ; } if ( isUInt < <NUM_LIT> > ( EltOffset0 ) && isUInt < <NUM_LIT> > ( EltOffset1 ) ) { if ( Modify ) { CI . Offset = EltOffset0 ; Paired . Offset = EltOffset1 ; } return true ; } uint32_t Min = std :: min ( EltOffset0 , EltOffset1 ) ; uint32_t Max = std :: max ( EltOffset0 , EltOffset1 ) ; const uint32_t Mask = maskTrailingOnes < uint32_t > ( <NUM_LIT> ) * <NUM_LIT> ; if ( ( ( Max - Min ) & ~ Mask ) == <NUM_LIT> ) { if ( Modify


<s> bool <ISA_LIT>LoadStoreOptimizer :: promoteConstantOffsetToImm ( MachineInstr & MI , MemInfoMap & Visited , SmallPtrSet < MachineInstr * , <NUM_LIT> > & AnchorList ) const { if ( ! ( MI . mayLoad ( ) ^ MI . mayStore ( ) ) ) return false ; if ( <STR_LIT> ( MI . getOpcode ( ) ) < <NUM_LIT> ) return false ; if ( MI . mayLoad ( ) && TII -> getNamedOperand ( MI , <STR_LIT> :: vdata ) != nullptr ) return false ; if ( AnchorList . count ( & MI ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ) ; if ( TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) -> getImm ( ) ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; ) ; return false ; } MachineOperand & Base = * TII -> getNamedOperand ( MI , <STR_LIT> :: vaddr ) ; MemAddress MAddr ; if ( Visited . find ( & MI ) == Visited . end ( ) ) { processBaseWithConstOffset ( Base , MAddr ) ; Visited [ & MI ] = MAddr ; } else MAddr = Visited [ & MI ] ; if ( MAddr . Offset == <NUM_LIT> ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <STR_LIT> ; ) ; return false ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MAddr . Base . HiReg << <STR_LIT> << MAddr . Base . LoReg << <STR_LIT> << MAddr . Offset << <STR_LIT> ; ) ; MachineInstr * AnchorInst = nullptr ; MemAddress AnchorAddr ; uint32_t MaxDist = std :: numeric_limits < uint32_t > :: min ( ) ; SmallVector < std :: pair < MachineInstr * , int64_t > , <NUM_LIT> > InstsWCommonBase ; MachineBasicBlock * MBB = MI . getParent ( ) ; MachineBasicBlock :: iterator E = MBB -> end ( ) ; MachineBasicBlock :: iterator MBBI = MI . getIterator ( ) ; ++ MBBI ; const <ISA_LIT>TargetLowering * TLI = static_cast < const <ISA_LIT>TargetLowering * > ( STM -> getTargetLowering ( ) ) ; for ( ; MBBI != E ; ++ MBBI ) { MachineInstr & MINext = * MBBI ; if ( MINext . getOpcode ( ) != MI . getOpcode ( ) || TII -> getNamedOperand ( MINext , <STR_LIT> :: offset ) -> getImm ( ) ) continue ; const MachineOperand & BaseNext = * TII -> getNamedOperand ( MINext , <STR_LIT> :: vaddr ) ; MemAddress MAddrNext ; if ( Visited . find ( & MINext ) == Visited . end ( ) ) { processBaseWithConstOffset ( BaseNext , MAddrNext ) ; Visited [ & MINext ] = MAddrNext ; } else MAddrNext = Visited [ & MINext ] ; if ( MAddrNext . Base . LoReg != MAddr . Base . LoReg || MAddrNext . Base . HiReg != MAddr . Base . HiReg || MAddrNext . Base . LoSubReg != MAddr . Base . LoSubReg

|| MAddrNext . Base . HiSubReg != MAddr . Base . HiSubReg ) continue ;


<s> for ( const MachineOperand & Op : MI . operands ( ) ) { if ( Op . isReg ( ) ) { if ( Op . isDef ( ) ) RegDefs . insert ( Op . getReg ( ) ) ; else if ( Op . readsReg ( ) && Register :: isPhysicalRegister ( Op .

getReg ( ) ) ) PhysRegUses . insert ( Op . getReg ( ) ) ;


<s> const auto * CarryRC = TRI -> getRegClass ( <STR_LIT> ) ; Register CarryReg = MRI -> createVirtualRegister ( CarryRC ) ; Register DeadCarryReg = MRI -> createVirtualRegister ( CarryRC ) ; Register DestSub0 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; Register DestSub1 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * LoHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub0 ) . addReg ( CarryReg , RegState :: Define ) . addReg ( Addr . Base . LoReg , <NUM_LIT> , Addr . Base . LoSubReg ) . add ( OffsetLo ) . addImm ( <NUM_LIT> ) ; ( void ) LoHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; LoHalf -> dump ( ) ; ) ; MachineInstr * HiHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub1 ) . addReg ( DeadCarryReg , RegState :: Define | RegState :: Dead ) . addReg ( Addr . Base . HiReg , <NUM_LIT> , Addr . Base . HiSubReg ) . add ( OffsetHi ) . addReg ( CarryReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; ( void ) HiHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; HiHalf -> dump ( ) ; ) ; Register FullDestReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * FullBase = BuildMI ( * MBB , MBBI , DL , TII -> get ( TargetOpcode :: REG_SEQUENCE ) , FullDestReg ) . addReg ( DestSub0 ) . addImm ( <STR_LIT> ) . addReg ( DestSub1 ) . addImm ( <STR_LIT> ) ; ( void


<s> const auto * LWEOp = TII . getNamedOperand ( * CI . I , <STR_LIT> :: lwe ) ; if ( ( TFEOp && TFEOp -> getImm ( ) ) || ( LWEOp && LWEOp -> getImm ( ) ) ) return false ; unsigned OperandsToMatch [ ] = { <STR_LIT> :: glc , <STR_LIT> :: slc , <STR_LIT> :: d16 , <STR_LIT> :: unorm , <STR_LIT> :: da , <STR_LIT> :: r128 , <STR_LIT> :: a16 , <STR_LIT> :: dlc } ; for ( auto op : OperandsToMatch ) { int Idx = <STR_LIT> ( CI . I


<s> return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } if ( TII . isMIMG ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) == - <NUM_LIT> && <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) == - <NUM_LIT> ) return UNKNOWN ; if ( TII . get ( Opc ) . mayStore ( ) || ! TII . get ( Opc ) . mayLoad ( ) || TII . isGather4 ( Opc ) ) return UNKNOWN ; return MIMG ; } if ( TII . isMTBUF ( Opc ) ) { switch ( <STR_LIT> ( Opc ) ) { default : return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_STORE ; } } return UNKNOWN ; case <STR_LIT>


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } if ( TII . isMIMG ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) == - <NUM_LIT> && <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) == - <NUM_LIT> ) return UNKNOWN ; if ( TII . get ( Opc ) . mayStore ( ) || ! TII . get ( Opc ) . mayLoad ( ) || TII . isGather4 ( Opc ) ) return UNKNOWN ; return MIMG ; } if ( TII . isMTBUF ( Opc ) ) { switch ( <STR_LIT> ( Opc ) ) { default : return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_STORE ; } } return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return S_BUFFER_LOAD_IMM ; case <STR_LIT> : case <STR_LIT>


<s> if ( TII . isMUBUF ( Opc ) ) { return <STR_LIT> ( Opc ) ; } if ( TII . isMIMG ( MI ) ) { uint64_t DMaskImm = TII . getNamedOperand ( MI , <STR_LIT> :: dmask ) -> getImm ( ) ; return countPopulation ( DMaskImm


<s> return countPopulation ( DMaskImm ) ; } if ( TII . isMTBUF ( Opc ) ) { return <STR_LIT> ( Opc ) ; } switch ( Opc ) { case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT>


<s> case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; } } else { switch ( CI . Width + Paired . Width ) { default : return nullptr ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT>


<s> if ( Regs . VAddr ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , InstsToMove ) ; CI . I -> eraseFromParent (


<s> const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; AddressRegs Regs = getRegs ( Opcode , * TII ) ; if ( Regs . VAddr ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin (


<s> const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdst ) ; unsigned NewOffset0 = CI . Offset ; unsigned NewOffset1 = Paired . Offset ; unsigned Opc = CI . UseST64 ? read2ST64Opcode ( CI . EltSize ) : read2Opcode ( CI . EltSize ) ; unsigned SubRegIdx0 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; Register BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , Paired . I , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , Paired . I , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * Paired . I } ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 ,


<s> MachineInstr * New = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC ) . addImm ( CI . DLC ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 )

. addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ;


<s> MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( JoinedFormat ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , InstsToMove )


<s> if ( Info0 -> BitsPerComp != <NUM_LIT> ) return false ; if ( getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , STI ) == <NUM_LIT> ) return false ; } unsigned EltOffset0 = CI . Offset / CI . EltSize ; unsigned EltOffset1 = Paired . Offset / CI . EltSize ; CI . UseST64 = false ; CI . BaseOff = <NUM_LIT> ; if ( ( CI . InstClass != DS_READ ) && ( CI . InstClass != DS_WRITE ) ) { return ( EltOffset0 + CI . Width == EltOffset1 || EltOffset1 + Paired . Width == EltOffset0 ) && CI . GLC == Paired . GLC && CI . DLC == Paired . DLC && ( CI . InstClass == S_BUFFER_LOAD_IMM || CI . SLC == Paired . SLC ) ; } if ( ( EltOffset0 % <NUM_LIT> == <NUM_LIT> ) && ( EltOffset1 % <NUM_LIT> ) == <NUM_LIT> && isUInt < <NUM_LIT> > ( EltOffset0 / <NUM_LIT> ) && isUInt < <NUM_LIT> > ( EltOffset1 / <NUM_LIT> ) ) { if ( Modify


<s> <STR_LIT> ; } else { Opcode = CI . IsX2 ? <STR_LIT> : <STR_LIT> ; } const TargetRegisterClass * SuperRC = CI . IsX2 ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) ; if ( CI . InstClass == BUFFER_LOAD_OFFEN ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; unsigned SubRegIdx0 = CI . IsX2


<s> const TargetRegisterClass * SuperRC = CI . IsX2 ? & <STR_LIT> : & <STR_LIT> ; unsigned SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; if ( CI . InstClass == BUFFER_STORE_OFFEN ) MIB . add ( * TII

-> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ;


<s> MachineBasicBlock * MBB = CI . I -> getParent ( ) ; const auto * AddrReg = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: addr ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdst ) ; unsigned NewOffset0 = CI . Offset0 ; unsigned NewOffset1 = CI . Offset1 ; unsigned Opc = CI . UseST64 ? read2ST64Opcode ( CI . EltSize ) : read2Opcode ( CI . EltSize ) ; unsigned SubRegIdx0 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseRegFlags = <NUM_LIT>


<s> const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdst ) ; unsigned NewOffset0 = CI . Offset0 ; unsigned NewOffset1 = CI . Offset1 ; unsigned Opc = CI . UseST64 ? read2ST64Opcode ( CI . EltSize ) : read2Opcode ( CI . EltSize ) ; unsigned SubRegIdx0 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill


<s> const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent


<s> <STR_LIT> ; const TargetRegisterClass * SuperRC = CI . IsX2 ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; unsigned SubRegIdx0 = CI . IsX2 ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = CI . IsX2 ? <STR_LIT> : <STR_LIT> ; if ( CI . Offset0 > CI . Offset1 ) std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> ::


<s> if ( <STR_LIT> ( Paired . I -> getOpcode ( ) , op ) != Idx ) return false ; if ( Idx != - <NUM_LIT> && CI . I -> getOperand ( Idx ) . getImm ( ) != Paired . I -> getOperand ( Idx ) . getImm ( ) ) return false ; } unsigned MaxMask = std :: max ( CI . DMask , Paired . DMask ) ; unsigned MinMask = std :: min ( CI . DMask , Paired . DMask ) ; unsigned AllowedBitsForMin = llvm ::


<s> if ( Idx != - <NUM_LIT> && CI . I -> getOperand ( Idx ) . getImm ( ) != Paired . I -> getOperand ( Idx ) . getImm ( ) ) return false ; } unsigned MaxMask = std :: max ( CI . DMask , Paired . DMask ) ; unsigned MinMask = std :: min ( CI . DMask , Paired . DMask ) ; unsigned AllowedBitsForMin = llvm :: countTrailingZeros ( MaxMask ) ; if ( ( <NUM_LIT> << AllowedBitsForMin ) <= MinMask ) return false ; return


<s> static unsigned getBufferFormatWithCompCount ( unsigned OldFormat , unsigned ComponentCount , const MCSubtargetInfo & STI ) { if ( ComponentCount > <NUM_LIT> ) return <NUM_LIT> ; const llvm :: <STR_LIT> * OldFormatInfo = llvm :: <STR_LIT> ( OldFormat , STI ) ; if ( ! OldFormatInfo ) return


<s> case <STR_LIT> : case <STR_LIT> : return BUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } if ( TII . isMIMG ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) == - <NUM_LIT> ) return UNKNOWN ; if ( TII . get ( Opc ) . mayStore ( ) || ! TII . get ( Opc ) . mayLoad ( ) || TII . isGather4 ( Opc ) ) return UNKNOWN ; return MIMG ; } if ( TII . isMTBUF ( Opc ) ) { switch ( <STR_LIT> ( Opc ) ) { default : return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_STORE ; } } return UNKNOWN ; case <STR_LIT> : case <STR_LIT>


<s> case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; } } else { switch ( CI . Width +


<s> const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset , Paired . Offset ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add

( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ;


<s> if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , InstsToMove ) ; CI . I -> eraseFromParent ( ) ; Paired . I -> eraseFromParent ( )


<s> auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset , Paired . Offset ) ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; moveInstsAfter ( MIB , InstsToMove


<s> MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( JoinedFormat ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , InstsToMove ) ; CI . I -> eraseFromParent ( ) ; Paired . I -> eraseFromParent ( )


<s> const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , * STI ) ; assert ( CI . I -> hasOneMemOperand ( )

&& Paired . I -> hasOneMemOperand ( ) ) ;


<s> CI . BaseOff = <NUM_LIT> ; if ( ( CI . InstClass != DS_READ ) && ( CI . InstClass != DS_WRITE ) ) { return ( EltOffset0 + CI . Width == EltOffset1 || EltOffset1 + Paired . Width == EltOffset0 ) && CI . GLC == Paired . GLC && CI . DLC == Paired . DLC && ( CI . InstClass == S_BUFFER_LOAD_IMM || CI . SLC == Paired . SLC ) ; } if ( ( EltOffset0 % <NUM_LIT> == <NUM_LIT> ) && ( EltOffset1 % <NUM_LIT> ) == <NUM_LIT> && isUInt < <NUM_LIT> > ( EltOffset0 / <NUM_LIT> ) && isUInt < <NUM_LIT> > ( EltOffset1 / <NUM_LIT> ) ) { if ( Modify ) { CI . Offset = EltOffset0 / <NUM_LIT> ; Paired . Offset = EltOffset1 / <NUM_LIT> ; CI . UseST64 = true ; } return true ; } if ( isUInt < <NUM_LIT> > ( EltOffset0 ) && isUInt < <NUM_LIT> > ( EltOffset1 ) ) { if ( Modify ) { CI . Offset = EltOffset0 ; Paired . Offset = EltOffset1 ; } return true ; } unsigned OffsetDiff = std :: abs ( ( int ) EltOffset1 - ( int ) EltOffset0 ) ; CI . BaseOff = std :: min ( CI . Offset , Paired . Offset ) ; if ( ( OffsetDiff % <NUM_LIT> == <NUM_LIT> ) && isUInt < <NUM_LIT> > ( OffsetDiff / <NUM_LIT> ) ) { if ( Modify ) { CI . Offset = ( EltOffset0 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; Paired . Offset = ( EltOffset1 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; CI . UseST64 = true ; } return true ; } if ( isUInt < <NUM_LIT> > ( OffsetDiff ) ) { if ( Modify


<s> if ( Info0 -> BitsPerComp != <NUM_LIT> ) return false ; if ( getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , STI ) == <NUM_LIT> ) return false ; } unsigned EltOffset0 = CI . Offset / CI . EltSize ; unsigned EltOffset1 = Paired . Offset / CI . EltSize ; CI . UseST64 = false ; CI . BaseOff = <NUM_LIT> ; if ( ( CI . InstClass != DS_READ ) && ( CI . InstClass != DS_WRITE ) ) { return ( EltOffset0 + CI . Width == EltOffset1 || EltOffset1 + Paired . Width == EltOffset0 ) && CI . GLC == Paired . GLC && CI . DLC == Paired . DLC && ( CI . InstClass == S_BUFFER_LOAD_IMM || CI . SLC == Paired . SLC ) ; } if ( ( EltOffset0 % <NUM_LIT> == <NUM_LIT> ) && ( EltOffset1 % <NUM_LIT> ) == <NUM_LIT> && isUInt < <NUM_LIT> > ( EltOffset0 / <NUM_LIT> ) && isUInt < <NUM_LIT> > ( EltOffset1 / <NUM_LIT> ) ) { if ( Modify ) { CI . Offset = EltOffset0 / <NUM_LIT> ; Paired . Offset = EltOffset1 / <NUM_LIT> ; CI . UseST64 = true ; } return true ; } if ( isUInt < <NUM_LIT> > ( EltOffset0 ) && isUInt < <NUM_LIT> > ( EltOffset1 ) ) { if ( Modify ) { CI . Offset = EltOffset0 ; Paired . Offset = EltOffset1 ; } return true ; } unsigned OffsetDiff = std :: abs ( ( int ) EltOffset1 - ( int ) EltOffset0 ) ; CI . BaseOff = std :: min ( CI . Offset , Paired . Offset ) ; if ( ( OffsetDiff % <NUM_LIT> == <NUM_LIT> ) && isUInt < <NUM_LIT> > ( OffsetDiff / <NUM_LIT> ) ) { if ( Modify ) { CI . Offset = ( EltOffset0 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; Paired . Offset = ( EltOffset1 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; CI . UseST64


<s> break ; } case S_BUFFER_LOAD_IMM : { MachineBasicBlock :: iterator NewMI = mergeSBufferLoadImmPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeBufferLoadPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_STORE : { MachineBasicBlock :: iterator NewMI = mergeBufferStorePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case MIMG : { MachineBasicBlock :: iterator NewMI = mergeImagePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case TBUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeTBufferLoadPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width )


<s> MRI = & MF . getRegInfo ( ) ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; assert ( MRI -> isSSA ( ) && <STR_LIT> ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : MF ) { std :: list < std :: list < CombineInfo > > MergeableInsts ; Modified |= collectMergeableInsts ( MBB , MergeableInsts ) ; do { OptimizeAgain = false ; Modified |= optimizeBlock (


<s> const auto * LWEOp = TII . getNamedOperand ( * CI . I , <STR_LIT> :: lwe ) ; if ( ( TFEOp && TFEOp -> getImm ( ) ) || ( LWEOp && LWEOp -> getImm ( ) ) ) return false ; unsigned OperandsToMatch [ ] = { <STR_LIT> :: glc , <STR_LIT> :: slc , <STR_LIT> :: d16 , <STR_LIT> :: unorm , <STR_LIT> :: da , <STR_LIT> :: r128 } ; for ( auto op : OperandsToMatch ) { int Idx = <STR_LIT> ( CI . I -> getOpcode ( ) , op ) ; if ( <STR_LIT> ( CI . Paired -> getOpcode ( ) , op ) != Idx ) return false ; if ( Idx != - <NUM_LIT> && CI . I -> getOperand ( Idx ) . getImm ( ) != CI . Paired -> getOperand ( Idx ) . getImm ( ) ) return false ; } unsigned MaxMask = std :: max ( CI . DMask0 , CI . DMask1 ) ; unsigned MinMask = std :: min ( CI . DMask0


<s> if ( Swizzled != - <NUM_LIT> && CI . I -> getOperand ( Swizzled ) . getImm ( ) ) return false ; ++ MBBI ; DenseSet < unsigned > RegDefsToMove ; DenseSet < unsigned > PhysRegUsesToMove ; addDefsUsesToList ( * CI . I , RegDefsToMove , PhysRegUsesToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { if ( ( getInstClass ( MBBI -> getOpcode ( ) , * TII ) != InstClass ) || ( getInstSubclass ( MBBI -> getOpcode ( ) , * TII ) != InstSubclass ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) { return false ; } if ( MBBI -> mayLoadOrStore ( ) && ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , AA ) ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = CI . hasSameBaseAddress ( * MBBI ) ; if ( Match


<s> CI . InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = CI . hasSameBaseAddress ( * MBBI ) ; if ( Match ) { CI . setPaired ( MBBI , * TII ) ; bool canBeCombined = CI . InstClass == MIMG ? dmasksCanBeCombined ( CI , * TII ) : widthsFit ( * STM , CI ) && offsetsCanBeCombined ( CI ) ; if ( canBeCombined && canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , AA


<s> return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } if ( TII . isMIMG ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) == - <NUM_LIT> ) return UNKNOWN ; if ( TII . get ( Opc ) . mayStore ( ) || ! TII . get ( Opc ) . mayLoad ( ) || TII . isGather4 ( Opc ) ) return UNKNOWN ; return MIMG ; } return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return S_BUFFER_LOAD_IMM ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return DS_READ ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT>


<s> return BUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } if ( TII . isMIMG ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) == - <NUM_LIT> ) return UNKNOWN ; if ( TII . get ( Opc ) . mayStore ( ) || ! TII . get ( Opc ) . mayLoad ( ) || TII . isGather4 ( Opc ) ) return UNKNOWN ; return MIMG ; } return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return S_BUFFER_LOAD_IMM ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return


<s> if ( TII . isMUBUF ( Opc ) ) return <STR_LIT> ( Opc ) ; if ( TII . isMIMG ( Opc ) ) { const <STR_LIT> * Info = <STR_LIT> ( Opc ) ; assert ( Info ) ; return Info


<s> default : assert ( CI . InstClass == BUFFER_LOAD || CI . InstClass == BUFFER_STORE ) ; return <STR_LIT> ( <STR_LIT> ( CI . I -> getOpcode ( ) ) , Width ) ; case UNKNOWN : llvm_unreachable ( <STR_LIT> ) ; case S_BUFFER_LOAD_IMM : switch ( Width ) { default : return <NUM_LIT> ; case <NUM_LIT> : return <STR_LIT> ; case <NUM_LIT> : return <STR_LIT>


<s> return <STR_LIT> ( Opc ) ; } if ( TII . isMIMG ( MI ) ) { uint64_t DMaskImm = TII . getNamedOperand ( MI , <STR_LIT> :: dmask ) -> getImm ( ) ; return countPopulation ( DMaskImm ) ; } switch ( Opc ) { case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT>


<s> std :: pair < unsigned , unsigned > <ISA_LIT>LoadStoreOptimizer :: getSubRegIdxs ( const CombineInfo & CI ) { if ( CI . Width0 == <NUM_LIT> || CI . Width0 == <NUM_LIT> || CI . Width0 + CI . Width1 > <NUM_LIT> ) return std :: make_pair ( <NUM_LIT> , <NUM_LIT> ) ; bool ReverseOrder ; if ( CI . InstClass == MIMG ) { assert ( ( countPopulation ( CI . DMask0 | CI . DMask1 ) == CI . Width0 + CI . Width1 ) && <STR_LIT> ) ; ReverseOrder = CI . DMask0 > CI . DMask1 ; } else ReverseOrder = CI . Offset0 > CI . Offset1 ; static const unsigned Idxs [ <NUM_LIT> ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> , <NUM_LIT> } , { <STR_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } , } ; unsigned Idx0 ; unsigned Idx1 ; assert ( CI . Width0 >= <NUM_LIT> && CI . Width0 <= <NUM_LIT> ) ; assert ( CI . Width1 >= <NUM_LIT> && CI . Width1 <= <NUM_LIT> ) ; if ( ReverseOrder ) { Idx1 = Idxs [ <NUM_LIT> ] [ CI . Width1 - <NUM_LIT> ] ; Idx0 = Idxs [ CI . Width1 ] [ CI . Width0 - <NUM_LIT> ] ; } else


<s> assert ( ( countPopulation ( CI . DMask0 | CI . DMask1 ) == CI . Width0 + CI . Width1 ) && <STR_LIT> ) ; ReverseOrder = CI . DMask0 > CI . DMask1 ; } else ReverseOrder = CI . Offset0 > CI . Offset1 ; static const unsigned Idxs [ <NUM_LIT> ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> , <NUM_LIT> } , { <STR_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } , } ; unsigned Idx0 ; unsigned Idx1 ; assert ( CI . Width0 >= <NUM_LIT> && CI . Width0 <= <NUM_LIT> ) ; assert ( CI . Width1 >= <NUM_LIT> && CI . Width1 <= <NUM_LIT> ) ; if ( ReverseOrder ) { Idx1 = Idxs [ <NUM_LIT> ] [ CI . Width1 - <NUM_LIT> ] ; Idx0 = Idxs [ CI . Width1 ] [ CI . Width0


<s> unsigned DMaskIdx = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: dmask ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) ; for ( unsigned I = <NUM_LIT> , E = ( * CI . I ) . getNumOperands ( ) ; I != E ; ++ I ) { if ( I == DMaskIdx ) MIB . addImm ( MergedDMask ) ; else MIB . add ( ( * CI . I ) . getOperand ( I ) ) ; } assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MachineInstr * New = MIB . addMemOperand ( combineKnownAdjacentMMOs ( * MBB ->


<s> const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; Register BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Read2 << '\n' ) ; return Read2


<s> MachineInstr * LoHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub0 ) . addReg ( CarryReg , RegState :: Define ) . addReg ( Addr . Base . LoReg , <NUM_LIT> , Addr . Base . LoSubReg ) . add ( OffsetLo ) . addImm ( <NUM_LIT> ) ; ( void ) LoHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; LoHalf -> dump ( ) ; ) ; MachineInstr * HiHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub1 ) . addReg ( DeadCarryReg , RegState :: Define | RegState :: Dead ) . addReg ( Addr . Base . HiReg , <NUM_LIT> , Addr . Base . HiSubReg ) . add ( OffsetHi ) . addReg ( CarryReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; ( void ) HiHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; HiHalf -> dump ( ) ; ) ; Register FullDestReg = MRI -> createVirtualRegister ( & <STR_LIT>


<s> MachineBasicBlock :: iterator <ISA_LIT>LoadStoreOptimizer :: mergeTBufferStorePair ( CombineInfo & CI , CombineInfo & Paired , const SmallVectorImpl < MachineInstr * > & InstsToMove ) { MachineBasicBlock * MBB = CI . I -> getParent ( ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI , Paired ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , * STM ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin


<s> continue ; } if ( STM -> hasSBufferLoadStoreAtomicDwordxN ( ) && ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ) { CI . InstClass = S_BUFFER_LOAD_IMM ; CI . EltSize = <STR_LIT> ( * STM , <NUM_LIT> ) ; CI . IsX2 = Opc == <STR_LIT> ; if ( findMatchingInst ( CI ) ) { Modified = true ; I = mergeSBufferLoadImmPair ( CI ) ; if ( ! CI . IsX2 ) CreatedX2 ++ ; } else { ++ I ; } continue ; } if ( Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> || Opc == <STR_LIT> ) { if ( Opc == <STR_LIT> || Opc == <STR_LIT> ) CI . InstClass = BUFFER_LOAD_OFFEN ; else CI . InstClass = BUFFER_LOAD_OFFSET ; CI . EltSize = <NUM_LIT> ; CI . IsX2 = Opc == <STR_LIT> || Opc == <STR_LIT> ; if ( findMatchingInst ( CI ) ) { Modified = true ; I = mergeBufferLoadPair ( CI ) ; if ( ! CI . IsX2 ) CreatedX2 ++ ; } else { ++ I ; } continue ; } bool StoreIsX2 , IsOffen ; if ( promoteBufferStoreOpcode ( * I , StoreIsX2 , IsOffen ) ) { CI . InstClass = IsOffen ? BUFFER_STORE_OFFEN : BUFFER_STORE_OFFSET ; CI . EltSize = <NUM_LIT> ; CI . IsX2


<s> assert ( MRI -> isSSA ( ) && <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Modified = false ; CreatedX2 = <NUM_LIT> ; for ( MachineBasicBlock & MBB : MF ) Modified |= optimizeBlock ( MBB ) ; if ( CreatedX2 >= <NUM_LIT> ) { for ( MachineBasicBlock & MBB : MF ) Modified |= optimizeBlock ( MBB ) ; } return Modified


<s> MRI = & MF . getRegInfo ( ) ; AA = & getAnalysis < AAResultsWrapperPass > ( ) . getAAResults ( ) ; assert ( MRI -> isSSA ( ) && <STR_LIT> ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; bool Modified = false ; CreatedX2 = <NUM_LIT> ; for ( MachineBasicBlock & MBB : MF ) Modified |= optimizeBlock ( MBB ) ; if ( CreatedX2 >= <NUM_LIT> ) { for ( MachineBasicBlock & MBB : MF ) Modified


<s> switch ( Opc ) { default : if ( TII . isMUBUF ( Opc ) ) { switch ( <STR_LIT> ( Opc ) ) { default : return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return BUFFER_STORE ; } } if ( TII . isMIMG ( Opc ) ) { if ( <STR_LIT> ( Opc , <STR_LIT> :: vaddr ) == - <NUM_LIT> && <STR_LIT> ( Opc , <STR_LIT> :: vaddr0 ) == - <NUM_LIT> ) return UNKNOWN ; if ( <STR_LIT> ( Opc ) -> BVH ) return UNKNOWN ; if ( TII . get ( Opc ) . mayStore ( ) || ! TII . get ( Opc ) . mayLoad ( ) || TII . isGather4 ( Opc ) ) return UNKNOWN ; return MIMG ; } if ( TII . isMTBUF ( Opc ) ) { switch ( <STR_LIT> ( Opc ) ) { default : return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_LOAD ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return TBUFFER_STORE ; } } return UNKNOWN ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return S_BUFFER_LOAD_IMM ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return DS_READ


<s> TII = static_cast < const <ISA_LIT>InstrInfo * > ( STM . getInstrInfo ( ) ) ; MRI = & MF . getRegInfo ( ) ; LIS = & getAnalysis < LiveIntervals > ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> ) ; assert ( ! MRI -> isSSA ( ) ) ; bool Modified = false ; for ( MachineBasicBlock & MBB : MF ) Modified |= optimizeBlock ( MBB


<s> MachineOperand OffsetHi = createRegOrImm ( static_cast < int32_t > ( Addr . Offset >> <NUM_LIT> ) , MI ) ; const auto * CarryRC = TRI -> getRegClass ( <STR_LIT> ) ; Register CarryReg = MRI -> createVirtualRegister ( CarryRC ) ; Register DeadCarryReg = MRI -> createVirtualRegister ( CarryRC ) ; Register DestSub0 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; Register DestSub1 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * LoHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub0 ) . addReg ( CarryReg , RegState :: Define ) . addReg ( Addr . Base . LoReg , <NUM_LIT> , Addr . Base . LoSubReg ) . add ( OffsetLo ) . addImm ( <NUM_LIT> ) ; ( void ) LoHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; LoHalf -> dump (


<s> Register Reg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * Mov = BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Reg ) . addImm ( Val ) ; ( void )


<s> addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = true ; for ( unsigned i = <NUM_LIT> ; i < NumAddresses ; i ++ ) { const MachineOperand & AddrRegNext = MBBI -> getOperand ( AddrIdx [ i ] ) ; if ( AddrReg [ i ] -> isImm ( ) || AddrRegNext . isImm ( ) ) { if ( AddrReg [ i ] -> isImm ( ) != AddrRegNext . isImm ( ) || AddrReg [ i ] -> getImm ( ) != AddrRegNext . getImm ( ) ) { Match = false ; break ; } continue ; } if ( AddrReg [ i ] -> getReg ( ) != AddrRegNext . getReg ( ) || AddrReg [ i ] -> getSubReg ( ) != AddrRegNext . getSubReg ( ) ) { Match = false ; break ; } } if ( Match ) { int OffsetIdx = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: offset ) ; CI . Offset0 = CI . I -> getOperand ( OffsetIdx ) . getImm ( ) ; CI . Width0 = getOpcodeWidth ( * CI . I ) ; CI . Offset1 = MBBI -> getOperand ( OffsetIdx ) . getImm ( ) ; CI . Width1 = getOpcodeWidth ( * MBBI ) ; CI . Paired = MBBI ; if ( ( CI . InstClass == DS_READ ) || ( CI . InstClass == DS_WRITE ) ) { CI . Offset0 &= <NUM_LIT> ; CI . Offset1 &= <NUM_LIT> ; } else { CI . GLC0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: glc ) -> getImm ( ) ; CI . GLC1 = TII -> getNamedOperand ( * MBBI , <STR_LIT> :: glc ) -> getImm ( ) ; if ( CI . InstClass != S_BUFFER_LOAD_IMM ) { CI . SLC0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: slc ) -> getImm ( ) ; CI . SLC1 = TII -> getNamedOperand ( * MBBI , <STR_LIT> :: slc ) -> getImm ( ) ; } CI . DLC0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: dlc ) -> getImm ( ) ; CI . DLC1 = TII -> getNamedOperand ( * MBBI , <STR_LIT>


<s> if ( baseOpcode == - <NUM_LIT> ) { return UNKNOWN ; } switch ( baseOpcode ) { default : return UNKNOWN ; case <STR_LIT> : return BUFFER_LOAD_OFFEN ; case <STR_LIT> : return BUFFER_LOAD_OFFSET ; case <STR_LIT> : return BUFFER_STORE_OFFEN ; case <STR_LIT> : return BUFFER_STORE_OFFSET ; case <STR_LIT> : return BUFFER_LOAD_OFFEN_exact ; case <STR_LIT> : return BUFFER_LOAD_OFFSET_exact ; case <STR_LIT> : return BUFFER_STORE_OFFEN_exact ; case <STR_LIT> : return BUFFER_STORE_OFFSET_exact ; } } switch ( Opc ) { default : return UNKNOWN ; case <STR_LIT>


<s> MachineBasicBlock * MBB = CI . I -> getParent ( ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI

. SLC0 ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC0 ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ;


<s> const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC0 ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent (


<s> const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset0 , CI . Offset1 ) ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC0 ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; moveInstsAfter ( MIB , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ; return Next


<s> MachineBasicBlock :: iterator <ISA_LIT>LoadStoreOptimizer :: mergeBufferStorePair ( CombineInfo & CI ) { MachineBasicBlock * MBB = CI . I -> getParent ( ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand (


<s> const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; Register BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 ,


<s> MachineInstr & MINext = * MBBI ; if ( MINext . getOpcode ( ) != MI . getOpcode ( ) || TII -> getNamedOperand ( MINext , <STR_LIT> :: offset ) -> getImm ( ) ) continue ; const MachineOperand & BaseNext = * TII -> getNamedOperand ( MINext , <STR_LIT> :: vaddr ) ; MemAddress MAddrNext ; if ( Visited . find ( & MINext ) == Visited . end ( ) ) { processBaseWithConstOffset ( BaseNext , MAddrNext ) ; Visited [ & MINext ] = MAddrNext ; } else MAddrNext = Visited [ & MINext ] ; if ( MAddrNext . Base . LoReg != MAddr . Base . LoReg || MAddrNext . Base . HiReg != MAddr . Base . HiReg || MAddrNext . Base . LoSubReg != MAddr . Base . LoSubReg || MAddrNext . Base . HiSubReg != MAddr . Base . HiSubReg ) continue ; InstsWCommonBase . push_back ( std :: make_pair ( & MINext , MAddrNext . Offset ) ) ; int64_t Dist = MAddr . Offset - MAddrNext . Offset ; TargetLoweringBase :: AddrMode AM ; AM . HasBaseReg = true ; AM . BaseOffs = Dist ; if ( TLI -> isLegalGlobalAddressingMode ( AM ) && ( uint32_t ) std :: abs ( Dist ) > MaxDist ) { MaxDist = std :: abs ( Dist ) ; AnchorAddr = MAddrNext ; AnchorInst = & MINext ; } } if ( AnchorInst ) { LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; AnchorInst -> dump ( ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << AnchorAddr . Offset << <STR_LIT> ) ; unsigned Base = computeBase ( MI , AnchorAddr ) ; updateBaseAndOffset ( MI , Base , MAddr . Offset - AnchorAddr . Offset ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; MI . dump ( ) ; ) ; for ( auto P : InstsWCommonBase )


<s> SlotIndex PairedIndex = LIS -> getInstructionIndex ( * Paired ) ; LiveRange & M0Range = LIS -> getRegUnit ( * MCRegUnitIterator ( <STR_LIT> , TRI ) ) ; LiveRange :: Segment * M0Segment = M0Range . getSegmentContaining ( PairedIndex ) ; bool UpdateM0Range = M0Segment -> end == PairedIndex . getRegSlot ( ) ; MachineInstrBuilder Write2 = BuildMI ( * MBB , I , DL , Write2Desc ) . addOperand ( * Addr ) . addOperand ( * Data0 ) . addOperand ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . addMemOperand ( * I -> memoperands_begin ( ) ) . addMemOperand ( * Paired -> memoperands_begin ( ) ) ; unsigned OrigRegs [ ] = { Data0 -> getReg ( ) , Data1 -> getReg ( ) , Addr -> getReg ( ) } ; LIS -> RemoveMachineInstrFromMaps ( * I ) ; LIS -> RemoveMachineInstrFromMaps ( * Paired ) ; I -> eraseFromParent ( ) ; Paired -> eraseFromParent ( ) ; LIS -> repairIntervalsInRange ( MBB , Write2 , Write2 , OrigRegs ) ; if ( UpdateM0Range ) { SlotIndex Write2Index = LIS -> getInstructionIndex ( * Write2 ) ; M0Segment -> end = Write2Index . getRegSlot ( ) ; } DEBUG ( dbgs ( ) << <STR_LIT> << * Write2 << '\n' ) ; return Write2 . getInstr


<s> BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) ) ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . setMemRefs ( CI . I -> mergeMemRefsWith ( * CI . Paired ) ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ; DEBUG ( dbgs ( ) << <STR_LIT> << * Read2 << '\n'


<s> std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) ) ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . setMemRefs ( CI . I -> mergeMemRefsWith ( * CI . Paired ) ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg (


<s> const MachineOperand * Data0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: data0 ) ; const MachineOperand * Data1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: data0 ) ; unsigned NewOffset0 = CI . Offset0 ; unsigned NewOffset1 = CI . Offset1 ; unsigned Opc = CI . UseST64 ? write2ST64Opcode ( CI . EltSize ) : write2Opcode ( CI . EltSize ) ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( Data0 , Data1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Write2Desc = TII -> get ( Opc ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState ::