<s> MachineOperand OffsetHi = createRegOrImm ( static_cast < int32_t > ( Addr . Offset >> <NUM_LIT> ) , MI ) ; unsigned CarryReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned DeadCarryReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned DestSub0 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; unsigned DestSub1 = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * LoHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub0 ) . addReg ( CarryReg , RegState :: Define ) . addReg ( Addr . Base . LoReg , <NUM_LIT> , Addr . Base . LoSubReg ) . add ( OffsetLo ) . addImm ( <NUM_LIT> ) ; ( void ) LoHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; LoHalf


<s> const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next


<s> const unsigned Opcode = getNewOpcode ( CI ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; unsigned SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand


<s> BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . DLC0 ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired ,


<s> const unsigned Opc = CI . I -> getOpcode ( ) ; const InstClassEnum InstClass = getInstClass ( Opc , * TII ) ; if ( InstClass == UNKNOWN ) { return false ; } const unsigned InstSubclass = getInstSubclass ( Opc , * TII ) ; int Swizzled = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: swz ) ; if ( Swizzled != - <NUM_LIT> && CI . I -> getOperand ( Swizzled ) . getImm ( ) ) return false ; ++ MBBI ; DenseSet < unsigned > RegDefsToMove ; DenseSet < unsigned > PhysRegUsesToMove ; addDefsUsesToList ( * CI . I , RegDefsToMove , PhysRegUsesToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { if ( ( getInstClass ( MBBI -> getOpcode ( ) , * TII ) != InstClass ) || ( getInstSubclass ( MBBI -> getOpcode ( ) , * TII ) != InstSubclass ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) { return false ; } if ( MBBI -> mayLoadOrStore ( ) && ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , AA ) ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; int Swizzled = <STR_LIT> ( MBBI -> getOpcode ( ) , <STR_LIT> :: swz ) ; if ( Swizzled != - <NUM_LIT> && MBBI -> getOperand ( Swizzled ) . getImm ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = CI . hasSameBaseAddress ( * MBBI ) ; if ( Match ) { CI . setPaired ( MBBI , * TII ) ; bool canBeCombined = CI . InstClass == MIMG ? dmasksCanBeCombined ( CI , * TII ) : widthsFit ( * STM , CI ) && offsetsCanBeCombined ( CI , * STI ) ; if ( canBeCombined && canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , AA


<s> if ( ( getInstClass ( MBBI -> getOpcode ( ) , * TII ) != InstClass ) || ( getInstSubclass ( MBBI -> getOpcode ( ) , * TII ) != InstSubclass ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) { return false ; } if ( MBBI -> mayLoadOrStore ( ) && ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , AA ) ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; int Swizzled = <STR_LIT> ( MBBI -> getOpcode ( ) , <STR_LIT> :: swz ) ; if ( Swizzled != - <NUM_LIT> && MBBI -> getOperand ( Swizzled ) . getImm ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = CI . hasSameBaseAddress ( * MBBI ) ; if ( Match ) { CI . setPaired ( MBBI ,


<s> unsigned <ISA_LIT>LoadStoreOptimizer :: getNewOpcode ( const CombineInfo & CI ) { const unsigned Width = CI . Width0 + CI . Width1 ; switch ( CI . InstClass ) { default : assert ( CI . InstClass == BUFFER_LOAD || CI . InstClass == BUFFER_STORE ) ; return <STR_LIT> ( <STR_LIT> ( CI . I -> getOpcode ( )


<s> } else ReverseOrder = CI . Offset0 > CI . Offset1 ; static const unsigned Idxs [ <NUM_LIT> ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <NUM_LIT> } , { <STR_LIT> , <STR_LIT> , <NUM_LIT> , <NUM_LIT> } , { <STR_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } , } ; unsigned Idx0 ; unsigned Idx1 ; assert ( CI . Width0 >= <NUM_LIT> && CI . Width0 <= <NUM_LIT> ) ; assert ( CI . Width1 >= <NUM_LIT> && CI . Width1 <= <NUM_LIT> ) ; if ( ReverseOrder ) { Idx1 = Idxs [ <NUM_LIT> ] [ CI . Width1 - <NUM_LIT> ] ; Idx0 = Idxs [ CI . Width1 ] [ CI . Width0 - <NUM_LIT> ] ; } else { Idx0 = Idxs [ <NUM_LIT> ] [ CI . Width0 - <NUM_LIT> ] ; Idx1 = Idxs [ CI . Width0 ]


<s> DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format0 , CI . Width0 + CI . Width1 , * STI ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( JoinedFormat ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC0 ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT>


<s> if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format0 , CI . Width0 + CI . Width1 , * STI ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset0 , CI . Offset1 ) ) . addImm ( JoinedFormat ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC0 ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; moveInstsAfter ( MIB , CI . InstsToMove ) ; CI . I -> eraseFromParent (


<s> const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format0 , CI . Width0 + CI . Width1 , * STI ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset0 , CI . Offset1 ) ) . addImm ( JoinedFormat ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC0 ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; moveInstsAfter ( MIB , CI . InstsToMove ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent


<s> unsigned Idx0 ; unsigned Idx1 ; static const unsigned Idxs [ <NUM_LIT> ] [ <NUM_LIT> ] = { { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } , } ; assert ( CI . Width >= <NUM_LIT> && CI . Width <= <NUM_LIT> ) ; assert ( Paired . Width >= <NUM_LIT> && Paired . Width <= <NUM_LIT> ) ; if ( ReverseOrder ) { Idx1 = Idxs [ <NUM_LIT> ] [ Paired . Width - <NUM_LIT> ] ; Idx0 = Idxs [ Paired . Width ] [ CI . Width - <NUM_LIT> ] ; }


<s> } case BUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeBufferLoadPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * this ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_STORE : { MachineBasicBlock :: iterator NewMI = mergeBufferStorePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * this ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case MIMG : { MachineBasicBlock :: iterator NewMI = mergeImagePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * this ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case TBUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeTBufferLoadPair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * this ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case TBUFFER_STORE : { MachineBasicBlock :: iterator NewMI = mergeTBufferStorePair ( CI , Paired , InstsToMove ) ; CI . setMI ( NewMI , * this ) ; OptimizeListAgain |= ( CI . Width + Paired .


<s> const MachineOperand & AddrReg0 = I -> getOperand ( AddrIdx ) ; const MachineOperand & AddrReg1 = MBBI -> getOperand ( AddrIdx ) ; if ( AddrReg0 . getReg ( ) == AddrReg1 . getReg ( ) && AddrReg0 . getSubReg ( ) == AddrReg1 . getSubReg ( ) ) { int OffsetIdx = <STR_LIT> ( I -> getOpcode ( ) , <STR_LIT> :: offset ) ; unsigned Offset0 = I -> getOperand ( OffsetIdx ) . getImm ( ) & <NUM_LIT> ; unsigned Offset1 = MBBI -> getOperand ( OffsetIdx ) . getImm ( ) &


<s> bool <ISA_LIT>LoadStoreOptimizer :: runOnMachineFunction ( MachineFunction & MF ) { if ( skipFunction ( * MF . getFunction ( ) ) ) return false ; const <ISA_LIT>Subtarget & STM = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ! STM . loadStoreOptEnabled ( ) ) return false ; TII = STM . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MRI = & MF . getRegInfo (


<s> if ( Regs & SOFFSET ) { AddrOpName [ NumAddresses ++ ] = <STR_LIT> :: soffset ; } if ( Regs & VADDR ) { AddrOpName [ NumAddresses ++ ] = <STR_LIT> :: vaddr ; } for ( unsigned i = <NUM_LIT> ; i < NumAddresses ; i ++ ) { AddrIdx [ i ] = <STR_LIT> ( CI . I -> getOpcode ( ) , AddrOpName [ i ] ) ; AddrReg [ i ] = & CI . I -> getOperand ( AddrIdx [ i ] ) ; if ( AddrReg [ i ] -> isReg ( ) && ( TargetRegisterInfo :: isPhysicalRegister ( AddrReg [ i ] -> getReg ( ) ) || MRI -> hasOneNonDBGUse ( AddrReg [ i ] -> getReg ( ) ) ) ) return false ; } ++ MBBI ; DenseSet < unsigned > RegDefsToMove ; DenseSet < unsigned > PhysRegUsesToMove ; addDefsUsesToList ( * CI . I , RegDefsToMove , PhysRegUsesToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { const bool IsDS = ( InstClass == DS_READ ) || ( InstClass == DS_WRITE ) ; if ( ( getInstClass ( MBBI -> getOpcode ( ) ) != InstClass ) || ( IsDS && ( MBBI -> getOpcode ( ) != Opc ) ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) { return false ; } if ( MBBI -> mayLoadOrStore ( ) && ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , TII , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , TII , AA ) ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = true ; for ( unsigned i = <NUM_LIT> ; i < NumAddresses ; i ++ ) { const MachineOperand & AddrRegNext = MBBI -> getOperand ( AddrIdx [ i ] ) ; if ( AddrReg [ i ] -> isImm ( ) || AddrRegNext . isImm ( ) ) { if ( AddrReg [ i ] -> isImm ( ) != AddrRegNext . isImm ( ) || AddrReg [ i ] -> getImm ( ) != AddrRegNext . getImm ( ) ) { Match =


<s> bool <ISA_LIT>LoadStoreOptimizer :: widthsFit ( const CombineInfo & CI ) { const unsigned Width = ( CI . Width0 + CI . Width1 ) ; switch ( CI . InstClass ) { default : return Width <= <NUM_LIT>


<s> assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , CI . Paired , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI .


<s> unsigned NewOffset0 = CI . Offset0 ; unsigned NewOffset1 = CI . Offset1 ; unsigned Opc = CI . UseST64 ? read2ST64Opcode ( CI . EltSize ) : read2Opcode ( CI . EltSize ) ; unsigned SubRegIdx0 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState ::


<s> unsigned NewOffset1 = CI . Offset1 ; unsigned Opc = CI . UseST64 ? write2ST64Opcode ( CI . EltSize ) : write2Opcode ( CI . EltSize ) ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( Data0 , Data1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Write2Desc = TII -> get ( Opc ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; unsigned BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { unsigned ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , CI . Paired , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; moveInstsAfter ( Write2 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> <<


<s> CombineInfo & CI = * I ; switch ( CI . InstClass ) { default : break ; case DS_READ : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeRead2Pair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; } break ; case DS_WRITE : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeWrite2Pair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; } break ; case S_BUFFER_LOAD_IMM : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeSBufferLoadImmPair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width0 + CI . Width1 ) < <NUM_LIT> ; } break ; case BUFFER_LOAD : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeBufferLoadPair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width0 + CI . Width1 ) < <NUM_LIT> ; } break ; case BUFFER_STORE : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeBufferStorePair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width0 + CI . Width1 ) < <NUM_LIT> ; } break ; case MIMG : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeImagePair ( CI ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width0 + CI . Width1 ) < <NUM_LIT> ; } break ; case TBUFFER_LOAD : if ( findMatchingInst ( CI ) ) { Modified = true ; removeCombinedInst ( MergeList , * CI . Paired ) ; MachineBasicBlock :: iterator NewMI = mergeTBufferLoadPair ( CI ) ; CI . setMI ( NewMI , * TII , *


<s> std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = I -> getDebugLoc ( ) ; MachineInstrBuilder Read2 = BuildMI ( * MBB , Paired , DL , Read2Desc , DestReg ) . add ( * AddrReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . addMemOperand ( * I -> memoperands_begin ( ) ) . addMemOperand ( * Paired -> memoperands_begin ( ) ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( I ) ; I -> eraseFromParent ( ) ; Paired -> eraseFromParent


<s> if ( UseST64 ) { NewOffset0 /= <NUM_LIT> ; NewOffset1 /= <NUM_LIT> ; Opc = ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; } if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( Data0 , Data1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Write2Desc = TII -> get ( Opc ) ; DebugLoc DL = I -> getDebugLoc ( ) ; MachineInstrBuilder Write2 = BuildMI ( * MBB , Paired , DL , Write2Desc ) . add ( * Addr ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . addMemOperand ( * I -> memoperands_begin ( ) ) . addMemOperand ( * Paired -> memoperands_begin ( ) ) ; moveInstsAfter ( Write2 , InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( I ) ; I -> eraseFromParent ( ) ; Paired -> eraseFromParent (


<s> for ( const MachineOperand & Def : MI . operands ( ) ) { if ( Def . isReg ( ) && Def . isDef (

) ) Defs . insert ( Def . getReg ( ) ) ;


<s> if ( <STR_LIT> ( Paired . I -> getOpcode ( ) , op ) != Idx ) return false ; if ( Idx != - <NUM_LIT> && CI . I -> getOperand ( Idx ) . getImm ( ) != Paired . I -> getOperand ( Idx ) . getImm ( ) ) return false ; } unsigned MaxMask = std :: max ( CI . DMask , Paired . DMask ) ; unsigned MinMask = std :: min ( CI .


<s> const auto * LWEOp = TII . getNamedOperand ( * CI . I , <STR_LIT> :: lwe ) ; if ( ( TFEOp && TFEOp -> getImm ( ) ) || ( LWEOp && LWEOp -> getImm ( ) ) ) return false ; unsigned OperandsToMatch [ ] = { <STR_LIT> :: glc , <STR_LIT> :: slc , <STR_LIT> :: d16 , <STR_LIT> :: unorm , <STR_LIT> :: da , <STR_LIT> :: r128 } ; for ( auto op : OperandsToMatch ) { int Idx = <STR_LIT> ( CI . I -> getOpcode ( ) , op ) ; if ( <STR_LIT> ( Paired . I -> getOpcode ( ) , op ) != Idx ) return false ; if ( Idx != - <NUM_LIT> && CI . I -> getOperand ( Idx ) . getImm ( ) != Paired . I -> getOperand ( Idx ) . getImm ( ) ) return false ; } unsigned MaxMask = std :: max ( CI . DMask , Paired . DMask ) ; unsigned MinMask = std :: min ( CI . DMask , Paired . DMask ) ; unsigned AllowedBitsForMin = llvm :: countTrailingZeros ( MaxMask ) ; if ( ( <NUM_LIT> << AllowedBitsForMin ) <= MinMask )


<s> int Swizzled = <STR_LIT> ( CI . I -> getOpcode ( ) , <STR_LIT> :: swz ) ; if ( Swizzled != - <NUM_LIT> && CI . I -> getOperand ( Swizzled ) . getImm ( ) ) return false ; ++ MBBI ; DenseSet < unsigned > RegDefsToMove ; DenseSet < unsigned > PhysRegUsesToMove ; addDefsUsesToList ( * CI . I , RegDefsToMove , PhysRegUsesToMove ) ; for ( ; MBBI != E ; ++ MBBI ) { if ( ( getInstClass ( MBBI -> getOpcode ( ) , * TII ) != InstClass ) || ( getInstSubclass ( MBBI -> getOpcode ( ) , * TII ) != InstSubclass ) ) { if ( MBBI -> hasUnmodeledSideEffects ( ) ) { return false ; } if ( MBBI -> mayLoadOrStore ( ) && ( ! memAccessesCanBeReordered ( * CI . I , * MBBI , AA ) || ! canMoveInstsAcrossMemOp ( * MBBI , CI . InstsToMove , AA ) ) ) { CI . InstsToMove . push_back ( & * MBBI ) ; addDefsUsesToList ( * MBBI , RegDefsToMove , PhysRegUsesToMove ) ; continue ; } addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ; continue ; } if ( MBBI -> hasOrderedMemoryRef ( ) ) return false ; int Swizzled = <STR_LIT> ( MBBI -> getOpcode ( ) , <STR_LIT> :: swz ) ; if ( Swizzled != - <NUM_LIT> && MBBI -> getOperand ( Swizzled ) . getImm ( ) ) return false ; if ( addToListsIfDependent ( * MBBI , RegDefsToMove , PhysRegUsesToMove , CI . InstsToMove ) ) continue ; bool Match = CI . hasSameBaseAddress ( * MBBI ) ; if ( Match )


<s> assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI


<s> MachineBasicBlock :: iterator <ISA_LIT>LoadStoreOptimizer :: mergeBufferStorePair ( CombineInfo & CI , CombineInfo & Paired ) { MachineBasicBlock * MBB = CI . I -> getParent ( ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI , Paired ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset , Paired . Offset ) ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; moveInstsAfter ( MIB , CI . InstsToMove


<s> const unsigned Opcode = getNewOpcode ( CI , Paired ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset , Paired . Offset ) ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm (

<NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ;


<s> } assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; unsigned SubRegIdx0 , SubRegIdx1 ; std :: tie ( SubRegIdx0 , SubRegIdx1 ) = getSubRegIdxs ( CI , Paired ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI .


<s> assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize == <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; Register BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , Paired . I , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Read2 = BuildMI ( * MBB , Paired . I , DL , Read2Desc , DestReg ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * Paired . I } ) ; ( void ) Read2 ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; CI . I -> eraseFromParent ( ) ; Paired . I -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Read2 << '\n'


<s> const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , Paired . I , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI . InstsToMove ) ; CI . I -> eraseFromParent ( ) ; Paired . I ->


<s> const unsigned Opcode = getNewOpcode ( CI , Paired ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset , Paired . Offset ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC ) . addImm ( CI . DLC ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent (


<s> MachineBasicBlock :: iterator <ISA_LIT>LoadStoreOptimizer :: mergeTBufferLoadPair ( CombineInfo & CI , CombineInfo & Paired ) { MachineBasicBlock * MBB = CI . I -> getParent ( ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI , Paired ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset , Paired . Offset ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , * STI ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( JoinedFormat ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand (


<s> const unsigned Opcode = getNewOpcode ( CI , Paired ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI , Paired ) ; Register DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset , Paired . Offset ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , * STI ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * Paired . I -> memoperands_begin ( ) ; MachineInstr * New = MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( MergedOffset ) . addImm ( JoinedFormat ) . addImm ( CI . GLC ) . addImm ( CI . SLC ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC ) . addImm ( <NUM_LIT> ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI , Paired ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> >


<s> Register SrcReg = MRI -> createVirtualRegister ( SuperRC ) ; const auto * Src0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdata ) ; const auto * Src1 = TII -> getNamedOperand ( * Paired . I , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , Paired . I , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode , * TII ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; unsigned JoinedFormat = getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , * STI ) ; assert ( CI . I -> hasOneMemOperand ( ) && Paired . I -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin


<s> const MCInstrDesc & Write2Desc = TII -> get ( Opc ) ; DebugLoc DL = CI . I -> getDebugLoc ( ) ; Register BaseReg = AddrReg -> getReg ( ) ; unsigned BaseSubReg = AddrReg -> getSubReg ( ) ; unsigned BaseRegFlags = <NUM_LIT> ; if ( CI . BaseOff ) { Register ImmReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , Paired . I , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , Paired . I , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , Paired . I , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * Paired . I } ) ; moveInstsAfter ( Write2 , CI . InstsToMove ) ; CI . I -> eraseFromParent ( ) ; Paired . I -> eraseFromParent


<s> if ( Info0 -> BitsPerComp != <NUM_LIT> ) return false ; if ( getBufferFormatWithCompCount ( CI . Format , CI . Width + Paired . Width , STI ) == <NUM_LIT> ) return false ; } unsigned EltOffset0 = CI . Offset / CI . EltSize ; unsigned EltOffset1 = Paired . Offset / CI . EltSize ; CI . UseST64 = false ; CI . BaseOff = <NUM_LIT> ; if ( ( CI . InstClass != DS_READ ) && ( CI . InstClass != DS_WRITE ) ) { return ( EltOffset0 + CI . Width == EltOffset1 || EltOffset1 + Paired . Width == EltOffset0 ) && CI . GLC == Paired . GLC && CI . DLC == Paired . DLC && ( CI . InstClass == S_BUFFER_LOAD_IMM || CI . SLC == Paired . SLC ) ; } if ( ( EltOffset0 % <NUM_LIT> == <NUM_LIT> ) && ( EltOffset1 % <NUM_LIT> ) == <NUM_LIT> && isUInt < <NUM_LIT> > ( EltOffset0 / <NUM_LIT> ) && isUInt < <NUM_LIT> > ( EltOffset1 / <NUM_LIT> ) ) { CI . Offset = EltOffset0 / <NUM_LIT> ; Paired . Offset = EltOffset1 / <NUM_LIT> ; CI . UseST64 = true ; return true ; } if ( isUInt < <NUM_LIT> > ( EltOffset0 ) && isUInt < <NUM_LIT> > ( EltOffset1 ) ) { CI . Offset = EltOffset0 ; Paired . Offset = EltOffset1 ; return true ; } unsigned OffsetDiff = std :: abs ( ( int ) EltOffset1 - ( int ) EltOffset0 ) ; CI . BaseOff = std :: min ( CI . Offset , Paired . Offset ) ; if ( ( OffsetDiff % <NUM_LIT> == <NUM_LIT> ) && isUInt < <NUM_LIT> > ( OffsetDiff / <NUM_LIT> ) ) { CI . Offset = ( EltOffset0 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; Paired . Offset = ( EltOffset1 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; CI . UseST64 = true ; return


<s> CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeBufferLoadPair ( CI , Paired ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case BUFFER_STORE : { MachineBasicBlock :: iterator NewMI = mergeBufferStorePair ( CI , Paired ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case MIMG : { MachineBasicBlock :: iterator NewMI = mergeImagePair ( CI , Paired ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case TBUFFER_LOAD : { MachineBasicBlock :: iterator NewMI = mergeTBufferLoadPair ( CI , Paired ) ; CI . setMI ( NewMI , * TII , * STM ) ; OptimizeListAgain |= ( CI . Width + Paired . Width ) < <NUM_LIT> ; break ; } case TBUFFER_STORE : { MachineBasicBlock :: iterator NewMI = mergeTBufferStorePair (


<s> for ( MachineOperand & Use : MI . operands ( ) ) { if ( Use . isReg ( ) && ( ( Use . readsReg ( ) && RegDefs . count ( Use . getReg ( ) ) ) || ( Use . isDef ( ) && RegDefs . count ( Use . getReg ( ) ) ) || ( Use . isDef ( ) && TargetRegisterInfo :: isPhysicalRegister ( Use .

getReg ( ) ) && PhysRegUses . count ( Use . getReg ( ) ) ) ) ) {


<s> static bool addToListsIfDependent ( MachineInstr & MI , DenseSet < unsigned > & RegDefs , DenseSet < unsigned > & PhysRegUses , SmallVectorImpl < MachineInstr * > & Insts ) { for ( MachineOperand & Use : MI . operands ( ) ) { if ( Use . isReg ( ) && ( ( Use . readsReg ( ) && RegDefs . count ( Use . getReg ( ) ) ) || ( Use . isDef ( ) && RegDefs . count ( Use . getReg ( ) ) ) || ( Use . isDef ( ) && TargetRegisterInfo :: isPhysicalRegister ( Use . getReg ( ) ) && PhysRegUses . count ( Use . getReg ( ) ) ) ) ) { Insts . push_back ( & MI ) ; addDefsUsesToList ( MI , RegDefs , PhysRegUses ) ; return true


<s> static bool canMoveInstsAcrossMemOp ( MachineInstr & MemOp , ArrayRef < MachineInstr * > InstsToMove , AliasAnalysis * AA ) { assert ( MemOp . mayLoadOrStore ( ) ) ; for ( MachineInstr * InstToMove : InstsToMove ) { if ( ! InstToMove -> mayLoadOrStore ( ) ) continue ; if ( ! memAccessesCanBeReordered ( MemOp , * InstToMove , AA ) ) return false ; } return true


<s> MachineInstr * HiHalf = BuildMI ( * MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , DestSub1 ) . addReg ( DeadCarryReg , RegState :: Define | RegState :: Dead ) . addReg ( Addr . Base . HiReg , <NUM_LIT> , Addr . Base . HiSubReg ) . add ( OffsetHi ) . addReg ( CarryReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; ( void ) HiHalf ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; HiHalf -> dump ( ) ; ) ; unsigned FullDestReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * FullBase = BuildMI ( * MBB , MBBI , DL , TII -> get ( TargetOpcode :: REG_SEQUENCE ) , FullDestReg ) . addReg

( DestSub0 ) . addImm ( <STR_LIT> ) . addReg ( DestSub1 ) . addImm ( <STR_LIT> ) ;


<s> unsigned Reg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; MachineInstr * Mov = BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Reg ) . addImm ( Val ) ; ( void ) Mov ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ; Mov -> dump ( ) ) ; return MachineOperand :: CreateReg ( Reg


<s> MachineOperand <ISA_LIT>LoadStoreOptimizer :: createRegOrImm ( int32_t Val , MachineInstr & MI ) { APInt V ( <NUM_LIT> , Val , true ) ; if ( TII -> isInlineConstant ( V ) ) return MachineOperand :: CreateImm


<s> return new <ISA_LIT>LoadStoreOptimizer


create<ISA_LIT>LoadStoreOptimizerPass ( ) {


<s> MachineInstr * Def = MRI -> getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Def || Def -> getOpcode ( ) != <STR_LIT> || ! Def -> getOperand ( <NUM_LIT> ) . isImm ( ) ) return None ; return Def -> getOperand ( <NUM_LIT> ) .


<s> MachineInstr * Def = MRI -> getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Def || Def -> getOpcode ( ) != <STR_LIT> || ! Def -> getOperand

( <NUM_LIT> ) . isImm ( ) ) return None ;


<s> void getAnalysisUsage ( AnalysisUsage &


<s> switch ( baseOpcode ) { default : return UNKNOWN ; case <STR_LIT> : return BUFFER_LOAD_OFFEN ; case <STR_LIT> : return BUFFER_LOAD_OFFSET ; case <STR_LIT> : return BUFFER_STORE_OFFEN ; case <STR_LIT> : return BUFFER_STORE_OFFSET ; case <STR_LIT> : return BUFFER_LOAD_OFFEN_exact ; case <STR_LIT> : return BUFFER_LOAD_OFFSET_exact ; case <STR_LIT> : return BUFFER_STORE_OFFEN_exact ; case <STR_LIT> : return BUFFER_STORE_OFFSET_exact ; } } switch ( Opc ) { default : return UNKNOWN ; case


<s> default : return <STR_LIT> ( CI . InstClass , Width ) ; case UNKNOWN : llvm_unreachable ( <STR_LIT> ) ; case S_BUFFER_LOAD_IMM : switch ( Width


<s> unsigned <ISA_LIT>LoadStoreOptimizer :: getNewOpcode ( const CombineInfo & CI ) { const unsigned Width = CI . Width0 + CI . Width1 ; switch ( CI . InstClass ) { default : return <STR_LIT> ( CI . InstClass , Width ) ; case UNKNOWN : llvm_unreachable (


<s> default : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT> ; case <STR_LIT> : return


<s> const unsigned Opc = MI . getOpcode ( ) ; if ( TII -> isMUBUF ( MI ) ) { return <STR_LIT> ( Opc ) ; } switch ( Opc ) { default : return <NUM_LIT> ; case <STR_LIT> : return <NUM_LIT>


<s> return ! ( A -> mayStore ( ) || B -> mayStore ( ) ) || ! A -> mayAlias ( AA , * B


<s> static bool memAccessesCanBeReordered ( MachineBasicBlock :: iterator A , MachineBasicBlock :: iterator B , AliasAnalysis * AA ) { return ! ( A -> mayStore ( ) || B -> mayStore ( ) ) || ! A -> mayAlias


<s> DebugLoc DL = CI . I -> getDebugLoc ( ) ; const unsigned Opcode = getNewOpcode ( CI ) ; const TargetRegisterClass * SuperRC = getTargetRegisterClass ( CI ) ; unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) ; const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI


<s> const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: soffset ) ) . addImm ( std :: min ( CI . Offset0 , CI . Offset1 ) ) . addImm ( CI . GLC0 ) . addImm ( CI . SLC0 ) . addImm ( <NUM_LIT> ) . addImm ( CI . DLC0 ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; moveInstsAfter ( MIB , CI .


<s> BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , SrcReg ) . add ( * Src0 ) . addImm ( SubRegIdx0 ) . add ( * Src1 ) . addImm ( SubRegIdx1 ) ; auto MIB = BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) ) . addReg ( SrcReg , RegState :: Kill ) ; const unsigned Regs = getRegs ( Opcode ) ; if ( Regs & VADDR ) MIB . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vaddr ) ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI .


<s> const auto * AddrReg = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: addr ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: vdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: vdst ) ; unsigned NewOffset0 = CI . Offset0 ; unsigned NewOffset1 = CI . Offset1 ; unsigned Opc = CI . UseST64 ? read2ST64Opcode ( CI . EltSize ) : read2Opcode ( CI . EltSize ) ; unsigned SubRegIdx0 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; unsigned SubRegIdx1 = ( CI . EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; if ( NewOffset0 > NewOffset1 ) { std :: swap ( NewOffset0 , NewOffset1 ) ; std :: swap ( SubRegIdx0 , SubRegIdx1 ) ; } assert ( ( isUInt < <NUM_LIT> > ( NewOffset0 ) && isUInt < <NUM_LIT> > ( NewOffset1 ) ) && ( NewOffset0 != NewOffset1 ) && <STR_LIT> ) ; const MCInstrDesc & Read2Desc = TII -> get ( Opc ) ; const TargetRegisterClass * SuperRC = ( CI . EltSize

== <NUM_LIT> ) ? & <STR_LIT> : & <STR_LIT> ;


<s> unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . DLC0 ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg , RegState :: Kill , SubRegIdx1 ) ; moveInstsAfter ( Copy1 , CI


<s> unsigned DestReg = MRI -> createVirtualRegister ( SuperRC ) ; unsigned MergedOffset = std :: min ( CI . Offset0 , CI . Offset1 ) ; assert ( CI . I -> hasOneMemOperand ( ) && CI . Paired -> hasOneMemOperand ( ) ) ; const MachineMemOperand * MMOa = * CI . I -> memoperands_begin ( ) ; const MachineMemOperand * MMOb = * CI . Paired -> memoperands_begin ( ) ; BuildMI ( * MBB , CI . Paired , DL , TII -> get ( Opcode ) , DestReg ) . add ( * TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sbase ) ) . addImm ( MergedOffset ) . addImm ( CI . GLC0 ) . addImm ( CI . DLC0 ) . addMemOperand ( combineKnownAdjacentMMOs ( * MBB -> getParent ( ) , MMOa , MMOb ) ) ; std :: pair < unsigned , unsigned > SubRegIdx = getSubRegIdxs ( CI ) ; const unsigned SubRegIdx0 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const unsigned SubRegIdx1 = std :: get < <NUM_LIT> > ( SubRegIdx ) ; const MCInstrDesc & CopyDesc = TII -> get ( TargetOpcode :: COPY ) ; const auto * Dest0 = TII -> getNamedOperand ( * CI . I , <STR_LIT> :: sdst ) ; const auto * Dest1 = TII -> getNamedOperand ( * CI . Paired , <STR_LIT> :: sdst ) ; BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest0 ) . addReg ( DestReg , <NUM_LIT> , SubRegIdx0 ) ; MachineInstr * Copy1 = BuildMI ( * MBB , CI . Paired , DL , CopyDesc ) . add ( * Dest1 ) . addReg ( DestReg ,


<s> BuildMI ( * MBB , CI . Paired , DL , TII -> get ( <STR_LIT> ) , ImmReg ) . addImm ( CI . BaseOff ) ; BaseReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; BaseRegFlags = RegState :: Kill ; TII -> getAddNoCarry ( * MBB , CI . Paired , DL , BaseReg ) . addReg ( ImmReg ) . addReg ( AddrReg -> getReg ( ) , <NUM_LIT> , BaseSubReg ) . addImm ( <NUM_LIT> ) ; BaseSubReg = <NUM_LIT> ; } MachineInstrBuilder Write2 = BuildMI ( * MBB , CI . Paired , DL , Write2Desc ) . addReg ( BaseReg , BaseRegFlags , BaseSubReg ) . add ( * Data0 ) . add ( * Data1 ) . addImm ( NewOffset0 ) . addImm ( NewOffset1 ) . addImm ( <NUM_LIT> ) . cloneMergedMemRefs ( { & * CI . I , & * CI . Paired } ) ; moveInstsAfter ( Write2 , CI . InstsToMove ) ; MachineBasicBlock :: iterator Next = std :: next ( CI . I ) ; CI . I -> eraseFromParent ( ) ; CI . Paired -> eraseFromParent ( ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << * Write2 <<


<s> MachineBasicBlock * MBB = I -> getParent ( ) ; ++ I ; for ( MachineInstr * MI : InstsToMove ) { MI -> removeFromParent ( ) ; MBB -> insert ( I


<s> CI . Offset0 = EltOffset0 / <NUM_LIT> ; CI . Offset1 = EltOffset1 / <NUM_LIT> ; CI . UseST64 = true ; return true ; } if ( isUInt < <NUM_LIT> > ( EltOffset0 ) && isUInt < <NUM_LIT> > ( EltOffset1 ) ) { CI . Offset0 = EltOffset0 ; CI . Offset1 = EltOffset1 ; return true ; } unsigned OffsetDiff = std :: abs ( ( int ) EltOffset1 - ( int ) EltOffset0 ) ; CI . BaseOff = std :: min ( CI . Offset0 , CI . Offset1 ) ; if ( ( OffsetDiff % <NUM_LIT> == <NUM_LIT> ) && isUInt < <NUM_LIT> > ( OffsetDiff / <NUM_LIT> ) ) { CI . Offset0 = ( EltOffset0 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; CI . Offset1 = ( EltOffset1 - CI . BaseOff / CI . EltSize ) / <NUM_LIT> ; CI . UseST64 = true ; return


<s> unsigned EltOffset1 = CI . Offset1 / CI . EltSize ; CI . UseST64 = false ; CI . BaseOff = <NUM_LIT> ; if ( ( CI . InstClass != DS_READ ) && ( CI . InstClass != DS_WRITE ) ) { return ( EltOffset0 + CI . Width0 == EltOffset1 || EltOffset1 + CI . Width1 == EltOffset0 ) && CI . GLC0 == CI . GLC1 && CI . DLC0 == CI . DLC1 && ( CI . InstClass == S_BUFFER_LOAD_IMM || CI . SLC0 == CI . SLC1 ) ; } if ( ( EltOffset0 % <NUM_LIT> == <NUM_LIT> ) && ( EltOffset1 % <NUM_LIT> ) == <NUM_LIT> && isUInt < <NUM_LIT> > ( EltOffset0 / <NUM_LIT> ) && isUInt < <NUM_LIT> > ( EltOffset1 / <NUM_LIT> ) ) { CI . Offset0 = EltOffset0 / <NUM_LIT> ; CI . Offset1 = EltOffset1 / <NUM_LIT> ; CI . UseST64 = true ; return true ; } if ( isUInt < <NUM_LIT> > ( EltOffset0 ) && isUInt < <NUM_LIT> > ( EltOffset1 ) ) { CI . Offset0 = EltOffset0 ; CI . Offset1 =


<s> continue ; } const unsigned Opc = MI . getOpcode ( ) ; CombineInfo CI ; CI . I = I ; CI . InstClass = getInstClass ( Opc ) ; switch ( CI . InstClass ) { default : break ; case DS_READ : CI . EltSize = ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; if ( findMatchingInst ( CI ) ) { Modified = true ; I = mergeRead2Pair ( CI ) ; } else { ++ I ; } continue ; case DS_WRITE : CI . EltSize = ( Opc == <STR_LIT> || Opc == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; if ( findMatchingInst ( CI ) ) { Modified = true ; I = mergeWrite2Pair ( CI ) ; } else { ++ I ; } continue ; case S_BUFFER_LOAD_IMM : CI . EltSize = <STR_LIT> ( * STM , <NUM_LIT> ) ; if ( findMatchingInst ( CI ) ) { Modified = true ; I = mergeSBufferLoadImmPair ( CI ) ; OptimizeAgain |= ( CI . Width0 + CI . Width1 ) < <NUM_LIT> ; } else { ++ I ; } continue ; case BUFFER_LOAD_OFFEN : case BUFFER_LOAD_OFFSET : case BUFFER_LOAD_OFFEN_exact : case BUFFER_LOAD_OFFSET_exact


<s> MachineInstr * Def = MRI -> getUniqueVRegDef ( Base . getReg ( ) ) ; if ( ! Def || Def -> getOpcode ( ) != <STR_LIT> || Def -> getNumOperands ( ) != <NUM_LIT> ) return ; MachineOperand BaseLo = Def -> getOperand ( <NUM_LIT> ) ; MachineOperand BaseHi = Def -> getOperand ( <NUM_LIT> ) ; if ( ! BaseLo . isReg ( ) || ! BaseHi . isReg ( ) ) return ; MachineInstr * BaseLoDef = MRI -> getUniqueVRegDef ( BaseLo . getReg ( ) ) ; MachineInstr * BaseHiDef = MRI -> getUniqueVRegDef ( BaseHi . getReg ( ) ) ; if ( ! BaseLoDef || BaseLoDef -> getOpcode ( ) != <STR_LIT> || ! BaseHiDef

|| BaseHiDef -> getOpcode ( ) != <STR_LIT> ) return ;


<s> Src1 = TII -> getNamedOperand ( * BaseHiDef , <STR_LIT> :: src1 ) ; if ( Src0 -> isImm ( ) ) std :: swap ( Src0 , Src1 ) ; if ( ! Src1 -> isImm ( ) ) return ; uint64_t Offset1 = Src1 -> getImm ( ) ; BaseHi = * Src0 ; Addr . Base . LoReg = BaseLo . getReg ( ) ; Addr . Base . HiReg = BaseHi . getReg ( ) ; Addr . Base . LoSubReg = BaseLo . getSubReg ( ) ; Addr . Base . HiSubReg = BaseHi . getSubReg ( ) ; Addr . Offset = ( * Offset0P & <NUM_LIT> ) | (


<s> if ( STM -> ldsRequiresM0Init ( ) ) return ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; return ( EltSize == <NUM_LIT> )


<s> if ( STM -> ldsRequiresM0Init ( ) ) return ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; return ( EltSize == <NUM_LIT> ) ?


<s> if ( STM -> ldsRequiresM0Init ( ) ) return ( EltSize == <NUM_LIT> ) ?


<s> unsigned <ISA_LIT>LoadStoreOptimizer :: write2ST64Opcode ( unsigned EltSize ) const { if ( STM -> ldsRequiresM0Init ( ) ) return ( EltSize == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; return ( EltSize == <NUM_LIT>


<s> void <ISA_LIT>LowerControlFlowPass :: Branch ( MachineInstr & MI ) { if ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) == MI . getParent

( ) -> getNextNode ( ) ) MI . eraseFromParent ( ) ;


<s> unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( <STR_LIT> )


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> )


<s> void <ISA_LIT>LowerControlFlowPass :: computeIndirectRegAndOffset ( unsigned VecReg , unsigned & Reg , int & Offset ) { unsigned SubReg = TRI -> getSubReg ( VecReg , <STR_LIT> ) ; if ( ! SubReg ) SubReg = VecReg ; const TargetRegisterClass * RC = TRI -> getPhysRegClass ( SubReg ) ; int RegIdx = TRI -> getHWRegIndex ( SubReg


<s> const TargetRegisterClass * RC = TRI -> getPhysRegClass ( SubReg ) ; int RegIdx = TRI -> getHWRegIndex ( SubReg ) + Offset ; if ( RegIdx < <NUM_LIT> ) { Offset = RegIdx ; RegIdx = <NUM_LIT> ; } else { Offset =


<s> return new <ISA_LIT>LowerControlFlowPass


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Saved = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Saved ) . addReg ( Src ) ; MI . eraseFromParent


<s> unsigned Saved = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Saved ) . addReg ( Src ) ; MI . eraseFromParent (


<s> DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Vcc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( Vcc ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( <STR_LIT> ) . addReg ( Reg ) ; Skip ( MI , MI .


<s> unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Vcc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Vcc


<s> unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Vcc ) . addReg ( Src ) ; MI . eraseFromParent


<s> const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MBB . getParent ( ) -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; assert ( MFI -> getShaderType ( ) == ShaderType :: PIXEL || MFI -> getShaderType ( ) == ShaderType :: GEOMETRY ) ; if ( ( Op . isImm ( ) ) ) { if ( Op . getImm ( ) & <NUM_LIT> ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addImm ( <NUM_LIT> ) ; } } else


<s> assert ( MFI -> getShaderType ( ) == ShaderType :: PIXEL || MFI -> getShaderType ( ) == ShaderType :: GEOMETRY ) ; if ( ( Op . isImm ( ) ) ) { if ( Op . getImm ( ) & <NUM_LIT> ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT>


<s> BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Idx ) . addImm ( Offset ) ; } else { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Idx ) ; } MBB . insert ( I , MovRel ) ; } else { assert ( <STR_LIT> . contains ( Save ) ) ; assert ( <STR_LIT> . contains ( Idx ) ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Save ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( Idx ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( <STR_LIT> ) . addReg ( Idx ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; if ( Offset ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addImm ( Offset ) ; } MBB . insert ( I , MovRel ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( - <NUM_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) .


<s> unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) .


<s> void <ISA_LIT>LowerControlFlowPass :: Loop ( MachineInstr & MI ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Src ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) .

addOperand ( MI . getOperand ( <NUM_LIT> ) ) ;


<s> break ; case <STR_LIT> : Else ( MI ) ; break ; case <STR_LIT> : Break ( MI ) ; break ; case <STR_LIT> : IfBreak ( MI ) ; break ; case <STR_LIT> : ElseBreak ( MI ) ; break ; case <STR_LIT> : ++ Depth ; Loop ( MI ) ; break ; case <STR_LIT> : if ( -- Depth == <NUM_LIT> && HaveKill ) { SkipIfDead ( MI ) ; HaveKill = false ; } EndCf ( MI ) ; break ; case <STR_LIT> : if ( Depth == <NUM_LIT> ) SkipIfDead ( MI ) ; else HaveKill = true ; Kill ( MI ) ; break ; case <STR_LIT> : Branch ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : IndirectSrc ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : IndirectDst ( MI ) ; break ; } } } if ( NeedWQM && MFI -> getShaderType ( ) == ShaderType :: PIXEL ) { MachineBasicBlock & MBB = MF . front ( ) ; BuildMI ( MBB , MBB . getFirstNonPHI ( ) , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) ; } if ( NeedFlat && MFI -> IsKernel ) { const MachineFrameInfo * FrameInfo = MF . getFrameInfo ( ) ; uint64_t StackSizeBytes = FrameInfo -> getStackSize ( ) ; int IndirectBegin = static_cast < const <ISA_LIT>InstrInfo * > ( TII ) -> getIndirectIndexBegin ( MF ) ; uint64_t StackOffset = IndirectBegin < <NUM_LIT> ? <NUM_LIT> : ( <NUM_LIT> * IndirectBegin / <NUM_LIT> ) ; assert ( ( StackSizeBytes < <NUM_LIT> ) && StackOffset < <NUM_LIT> && <STR_LIT> ) ; MachineBasicBlock & MBB = MF . front ( ) ; DebugLoc NoDL ; MachineBasicBlock :: iterator Start = MBB .


<s> SmallVector < MachineBasicBlock * , <NUM_LIT> > Worklist ( Begin -> succ_begin ( ) , Begin -> succ_end ( ) ) ; while ( ! Worklist . empty ( ) ) { MachineBasicBlock * MBB = Worklist . pop_back_val


<s> break ; default : assert ( false && <STR_LIT> ) ; break ; } MachineBasicBlock :: iterator Next ; for ( I = Prev ? Prev -> getIterator ( ) : MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MaskMI = * I ; switch ( MaskMI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : combineMasks ( MaskMI ) ; break ; default : I = MBB . end ( ) ; break ; } } return SplitBB


<s> bool Redundant = true ; for ( auto & I : MBB . instrs ( ) ) { if ( ! I . isDebugInstr ( ) && ! I . isUnconditionalBranch ( ) ) Redundant = false ; } if ( Redundant ) { MachineBasicBlock * Succ = * MBB . succ_begin ( ) ; SmallVector < MachineBasicBlock * , <NUM_LIT> > Preds ( MBB . predecessors ( ) ) ; for ( auto P : Preds ) { P -> replaceSuccessor ( & MBB , Succ ) ; MachineBasicBlock :: iterator I ( P -> getFirstInstrTerminator ( ) ) ; while ( I != P -> end ( ) ) { if ( I -> isBranch ( ) ) { if ( TII -> getBranchDestBlock ( * I ) == & MBB ) { I -> getOperand ( <NUM_LIT> ) . setMBB ( Succ ) ; break ; } } I ++ ; } if ( I == P -> end ( ) ) { MachineFunction * MF = P -> getParent ( ) ; MachineFunction :: iterator InsertPt = P -> getNextNode ( ) ? MachineFunction :: iterator ( P -> getNextNode ( ) ) : MF -> end ( ) ; MF -> splice ( InsertPt , Succ ) ; } } MBB . removeSuccessor ( Succ ) ; if ( LIS ) { for ( auto & I : MBB . instrs ( ) ) LIS -> RemoveMachineInstrFromMaps ( I ) ; } MBB . clear ( ) ; MBB . eraseFromParent ( ) ; return true


<s> MachineBasicBlock * B = & MBB ; do { if ( ! Visited . insert ( B ) . second ) return MBB . end ( ) ; auto E = B -> end ( ) ; for ( ; It != E ; ++ It ) { if ( It -> getOpcode ( ) == <STR_LIT> ) continue ; if ( TII -> mayReadEXEC ( * MRI , * It ) ) break ; } if ( It != E ) return It ; if ( B -> succ_size ( ) != <NUM_LIT> ) return MBB . end ( ) ; MachineBasicBlock * Succ = * B -> succ_begin ( ) ; It = Succ -> begin ( ) ; B = Succ ; } while (


<s> MachineBasicBlock :: iterator <ISA_LIT>LowerControlFlow :: skipIgnoreExecInstsTrivialSucc ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator It ) const { SmallSet < const MachineBasicBlock * , <NUM_LIT> > Visited ; MachineBasicBlock * B = & MBB ; do { if ( ! Visited . insert ( B ) . second ) return MBB . end ( ) ; auto E = B -> end ( ) ; for ( ; It != E ; ++ It ) { if ( It -> getOpcode ( ) == <STR_LIT> ) continue ; if ( TII -> mayReadEXEC ( * MRI , * It ) ) break ; } if ( It != E )


<s> void <ISA_LIT>LowerControlFlow :: Branch ( MachineInstr & MI ) { if ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) == MI . getParent ( ) -> getNextNode


<s> unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ,

Dst ) . addReg ( <STR_LIT> ) . addReg ( Src ) ;


<s> unsigned SubReg = TRI -> getSubReg ( VecReg , <STR_LIT> ) ; if ( ! SubReg ) SubReg = VecReg ; const TargetRegisterClass * RC = TRI -> getPhysRegClass ( SubReg ) ; int RegIdx = TRI -> getHWRegIndex ( SubReg )


<s> FunctionPass * llvm :: create<ISA_LIT>LowerControlFlowPass ( ) { return new <ISA_LIT>LowerControlFlow (


<s> unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , MBB . getFirstNonPHI ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Src ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg (


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , MBB . getFirstNonPHI ( ) , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Reg ) ; MI . eraseFromParent


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg (


<s> void <ISA_LIT>LowerControlFlow :: If ( MachineInstr & MI ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Vcc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( Vcc ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> )

, Reg ) . addReg ( <STR_LIT> ) . addReg ( Reg ) ;


<s> unsigned Reg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Vcc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Reg ) . addReg ( Vcc ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> )


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Vcc = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Src = MI . getOperand ( <NUM_LIT>


<s> computeIndirectRegAndOffset ( Dst , Reg , Off ) ; MachineInstr * MovRel = BuildMI ( * MBB . getParent ( ) , DL , TII -> get ( <STR_LIT> ) ) . addReg ( Reg , RegState :: Define ) . addReg ( Val )

. addReg ( Dst , RegState :: Implicit ) ;


<s> unsigned Vec = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; int Off = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; unsigned Reg ; computeIndirectRegAndOffset ( Vec , Reg , Off ) ; MachineInstr * MovRel = BuildMI ( * MBB . getParent ( ) , DL , TII -> get ( <STR_LIT> )

, Dst ) . addReg ( Reg ) . addReg ( Vec , RegState :: Implicit ) ;


<s> MachineInstr * MovRel = BuildMI ( * MBB . getParent ( ) , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( Reg ) . addReg ( Vec , RegState :: Implicit ) ; LoadM0 ( MI , MovRel


<s> BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( Src ) ; BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addOperand ( MI .


<s> TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MF . getSubtarget ( ) . getRegisterInfo ( ) ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; bool HaveKill = false ; bool NeedWQM = false ; bool NeedFlat = false ; unsigned Depth = <NUM_LIT> ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; ++ BI ) { MachineBasicBlock * EmptyMBBAtEnd = NULL ; MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; if ( TII -> isWQM ( MI ) || TII -> isDS ( MI ) ) NeedWQM = true ; if ( TII -> isFLAT ( MI ) ) NeedFlat = true ; switch ( MI . getOpcode ( ) ) { default : break ; case <STR_LIT> : ++ Depth ; If ( MI ) ; break ; case <STR_LIT> : Else ( MI ) ; break ; case <STR_LIT> : Break ( MI ) ; break ; case <STR_LIT> : IfBreak ( MI ) ; break ; case <STR_LIT> : ElseBreak ( MI ) ; break ; case <STR_LIT> : ++ Depth ; Loop ( MI ) ; break ; case <STR_LIT> : if ( -- Depth == <NUM_LIT> && HaveKill ) { SkipIfDead ( MI ) ; HaveKill = false ; } EndCf ( MI ) ; break ; case <STR_LIT> : if ( Depth == <NUM_LIT> ) SkipIfDead ( MI ) ; else HaveKill = true ; Kill ( MI ) ; break ; case <STR_LIT> : Branch ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : IndirectSrc ( MI ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : IndirectDst ( MI ) ; break ; case <STR_LIT> : { if ( MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) break ; if ( BI != -- MF . end ( ) || I != MBB . getFirstTerminator ( ) ) { if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB . addSuccessor ( EmptyMBBAtEnd ) ; BuildMI ( * BI , I , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; } I -> eraseFromParent ( ) ; break ; } } } } if ( NeedWQM && MFI -> getShaderType ( ) == ShaderType


<s> for ( MachineBasicBlock * MBB = From ; MBB != To && ! MBB -> succ_empty ( ) ; MBB = * MBB -> succ_begin ( ) ) { for ( MachineBasicBlock :: iterator I = MBB -> begin (


<s> bool <ISA_LIT>LowerControlFlow :: shouldSkip ( MachineBasicBlock * From , MachineBasicBlock * To ) { unsigned NumInstr = <NUM_LIT> ; for ( MachineBasicBlock * MBB = From ; MBB != To && ! MBB -> succ_empty ( ) ; MBB = * MBB -> succ_begin ( ) ) { for ( MachineBasicBlock :: iterator I = MBB -> begin (


<s> if ( ! shouldSkip ( * From . getParent ( ) -> succ_begin


<s> if ( MBB . getParent ( ) -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> getShaderType ( ) != ShaderType :: PIXEL || ! shouldSkip ( & MBB , & MBB . getParent ( ) -> back ( ) ) ) return ; MachineBasicBlock :: iterator Insert = & MI ; ++ Insert ; BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg (

<STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( <STR_LIT> ) ;


<s> Register SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( XorTermrOpc ) , Exec ) . addReg ( Exec ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS ) { MI . eraseFromParent


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; Register DstReg = getSaveExec ( & MI ) ; bool ExecModified = MI . getOperand ( <NUM_LIT> ) . getImm ( ) != <NUM_LIT> ; MachineBasicBlock :: iterator Start = MBB . begin ( ) ; Register CopyReg = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , Start , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Register SaveReg = ExecModified ? MRI -> createVirtualRegister ( BoolRC ) : DstReg ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . addReg ( CopyReg ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; if ( ExecModified ) { MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( XorTermrOpc ) , Exec ) . addReg ( Exec ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS


<s> const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator InsPt = Def && Def -> getParent ( ) == & MBB ? std :: next ( MachineBasicBlock :: iterator ( Def ) ) : MBB . begin ( ) ; MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( OrOpc ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI ,


<s> unsigned CFMask = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * Def = MRI . getUniqueVRegDef ( CFMask ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator InsPt = Def && Def -> getParent ( ) == & MBB ? std :: next ( MachineBasicBlock :: iterator ( Def ) ) : MBB . begin ( ) ; MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( OrOpc ) , Exec ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * NewMI ) ; MI . eraseFromParent ( ) ; if ( LIS ) LIS -> handleMove


<s> Register CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( Exec ) . addReg ( Exec , RegState :: ImplicitDefine ) ; Register Tmp = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( AndOpc ) , Tmp ) . addReg ( CopyReg ) . add ( Cond ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = nullptr ; if ( ! SimpleIf ) { Xor = BuildMI ( MBB , I , DL , TII -> get ( XorOpc ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; } MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( MovTermOpc ) , Exec ) . addReg ( Tmp , RegState :: Kill ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * And ) ; if ( ! SimpleIf ) LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * SetExec ) ; LIS -> InsertMachineInstrInMaps ( * NewBr ) ; LIS -> removeAllRegUnitsForPhysReg ( <STR_LIT> ) ; MI . eraseFromParent ( ) ; LIS -> removeInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( Tmp ) ; if ( ! SimpleIf ) LIS ->

createAndComputeVirtRegInterval ( CopyReg ) ;


<s> MachineOperand & Cond = MI . getOperand ( <NUM_LIT> ) ; assert ( Cond . getSubReg ( ) == <STR_LIT> ) ; MachineOperand & ImpDefSCC = MI . getOperand ( <NUM_LIT> ) ; assert ( ImpDefSCC . getReg ( ) == <STR_LIT> && ImpDefSCC . isDef ( ) ) ; bool SimpleIf = isSimpleIf ( MI , MRI , TII ) ; Register CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( Exec ) . addReg ( Exec , RegState :: ImplicitDefine ) ; Register Tmp = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( AndOpc ) , Tmp ) . addReg ( CopyReg ) . add ( Cond ) ; setImpSCCDefDead ( * And


<s> } } MachineInstr * And = nullptr , * Or = nullptr ; if ( ! SkipAnding ) { Register AndReg = MRI -> createVirtualRegister ( BoolRC ) ; And = BuildMI ( MBB , & MI , DL , TII -> get ( AndOpc ) , AndReg ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . addReg ( AndReg ) .


<s> MachineInstr * And = nullptr , * Or = nullptr ; if ( ! SkipAnding ) { Register AndReg = MRI -> createVirtualRegister ( BoolRC ) ; And = BuildMI ( MBB , & MI , DL , TII -> get ( AndOpc ) , AndReg ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . addReg ( AndReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> createAndComputeVirtRegInterval ( AndReg ) ; } else Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) { if ( And ) LIS -> InsertMachineInstrInMaps ( * And ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * Or ) ; } MI . eraseFromParent (


<s> MachineInstr * Branch = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) { LIS -> ReplaceMachineInstrInMaps ( MI


<s> MachineBasicBlock :: iterator J = std :: next ( I ) ; if ( J != MBB -> end ( ) && J -> getOpcode ( ) == FalseTermOpc && J -> getOperand ( <NUM_LIT> ) . isReg ( ) && J -> getOperand ( <NUM_LIT> ) . getReg ( ) == SaveExecReg ) { SaveExecReg = J -> getOperand ( <NUM_LIT> ) . getReg ( ) ; J -> eraseFromParent ( )


<s> MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; if ( MBB == EMBB || ! Visited . insert ( MBB ) . second ) continue ; for ( auto & Term : MBB -> terminators ( ) ) if ( TII -> isKillTerminator ( Term . getOpcode ( ) ) ) return false ; Worklist . append ( MBB -> succ_begin ( ) , MBB -> succ_end ( )


<s> MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * OrSaveExec ) ; LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * Branch ) ; LIS -> removeInterval ( DstReg ) ; LIS -> createAndComputeVirtRegInterval ( DstReg ) ; LIS -> createAndComputeVirtRegInterval ( SaveReg ) ; LIS -> removeAllRegUnitsForPhysReg ( <STR_LIT>


<s> Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock :: iterator Start = MBB . begin ( ) ; Register SaveReg = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LV ) LV -> replaceKillInstruction ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MI , * OrSaveExec ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS ->


<s> if ( SimpleIf ) { auto UseMI = MRI -> use_instr_nodbg_begin ( SaveExecReg ) ; SimpleIf = ! hasKill ( MI . getParent ( ) , UseMI -> getParent ( ) ) ; } Register CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( Exec ) . addReg ( Exec , RegState :: ImplicitDefine ) ; LoweredIf . insert ( CopyReg ) ; Register Tmp = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( AndOpc ) , Tmp ) . addReg ( CopyReg ) . add ( Cond ) ; if ( LV ) LV -> replaceKillInstruction ( Cond . getReg ( ) , MI , * And ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = nullptr ; if ( ! SimpleIf ) { Xor = BuildMI ( MBB , I , DL , TII -> get ( XorOpc ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; } MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get

( MovTermOpc ) , Exec ) . addReg ( Tmp , RegState :: Kill ) ;


<s> SimpleIf = ! hasKill ( MI . getParent ( ) , UseMI -> getParent ( ) ) ; } Register CopyReg = SimpleIf ? SaveExecReg : MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * CopyExec = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , CopyReg ) . addReg ( Exec ) . addReg ( Exec , RegState :: ImplicitDefine ) ; LoweredIf . insert ( CopyReg ) ; Register Tmp = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * And = BuildMI ( MBB , I , DL , TII -> get ( AndOpc ) , Tmp ) . addReg ( CopyReg ) . add ( Cond ) ; if ( LV ) LV -> replaceKillInstruction ( Cond . getReg ( ) , MI , * And ) ; setImpSCCDefDead ( * And , true ) ; MachineInstr * Xor = nullptr ; if ( ! SimpleIf ) { Xor = BuildMI ( MBB , I , DL , TII -> get ( XorOpc ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; } MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( MovTermOpc ) , Exec ) . addReg ( Tmp , RegState :: Kill ) ; if ( LV ) LV -> getVarInfo ( Tmp ) . Kills . push_back ( SetExec ) ; I = skipToUncondBrOrEnd ( MBB , I ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * And ) ; if ( ! SimpleIf ) LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * SetExec ) ; LIS -> InsertMachineInstrInMaps ( * NewBr ) ; LIS -> removeAllRegUnitsForPhysReg ( <STR_LIT> ) ; MI . eraseFromParent ( ) ; LIS -> removeInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( SaveExecReg ) ; LIS -> createAndComputeVirtRegInterval ( Tmp ) ; if ( ! SimpleIf ) LIS -> createAndComputeVirtRegInterval (


<s> Register AndReg = MRI -> createVirtualRegister ( BoolRC ) ; And = BuildMI ( MBB , & MI , DL , TII -> get ( AndOpc ) , AndReg ) . addReg ( Exec ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LV ) LV -> replaceKillInstruction ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , MI , * And ) ; Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . addReg ( AndReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> createAndComputeVirtRegInterval ( AndReg ) ; } else { Or = BuildMI ( MBB , & MI , DL , TII -> get ( OrOpc ) , Dst ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LV ) LV -> replaceKillInstruction ( MI . getOperand ( <NUM_LIT> ) . getReg (


<s> BuildMI ( * MBB , MBB -> begin ( ) , MI . getDebugLoc ( ) , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addImm ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) ) ; if ( LIS ) LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; return ; } Register InputReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * FirstMI = & * MBB -> begin ( ) ; if ( InputReg . isVirtual ( ) ) { MachineInstr * DefInstr = MRI -> getVRegDef ( InputReg ) ; assert ( DefInstr && DefInstr -> isCopy ( ) ) ; if ( DefInstr -> getParent ( ) == MBB ) { if ( DefInstr != FirstMI ) { DefInstr -> removeFromParent ( ) ; MBB -> insert ( FirstMI , DefInstr ) ; if ( LIS ) LIS -> handleMove ( * DefInstr ) ; } else { FirstMI = & * std :: next ( FirstMI -> getIterator ( ) ) ; } } } const DebugLoc DL = MI . getDebugLoc ( ) ; const unsigned WavefrontSize = ST . getWavefrontSize ( ) ; const unsigned Mask = ( WavefrontSize << <NUM_LIT> ) - <NUM_LIT> ; Register CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; auto BfeMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( <STR_LIT> ) , CountReg ) . addReg ( InputReg ) . addImm ( ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) & Mask ) | <NUM_LIT> ) ; if ( LV ) LV -> recomputeForSingleDefVirtReg ( InputReg ) ; auto BfmMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( CountReg ) . addImm ( <NUM_LIT> ) ; auto CmpMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( CountReg , RegState :: Kill ) . addImm ( WavefrontSize ) ; if ( LV ) LV -> getVarInfo ( CountReg ) . Kills . push_back ( CmpMI ) ; auto CmovMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addImm ( - <NUM_LIT> ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * BfeMI ) ; LIS -> InsertMachineInstrInMaps ( *


<s> MBB -> insert ( FirstMI , DefInstr ) ; if ( LIS ) LIS -> handleMove ( * DefInstr ) ; } else { FirstMI = & * std :: next ( FirstMI -> getIterator ( ) ) ; } } } const DebugLoc DL = MI . getDebugLoc ( ) ; const unsigned WavefrontSize = ST . getWavefrontSize ( ) ; const unsigned Mask = ( WavefrontSize << <NUM_LIT> ) - <NUM_LIT> ; Register CountReg = MRI -> createVirtualRegister ( & <STR_LIT> ) ; auto BfeMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( <STR_LIT> ) , CountReg ) . addReg ( InputReg ) . addImm ( ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) & Mask ) | <NUM_LIT> ) ; if ( LV ) LV -> recomputeForSingleDefVirtReg ( InputReg ) ; auto BfmMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addReg ( CountReg ) . addImm ( <NUM_LIT> ) ; auto CmpMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( CountReg , RegState :: Kill ) . addImm ( WavefrontSize ) ; if ( LV ) LV -> getVarInfo ( CountReg ) . Kills . push_back ( CmpMI ) ; auto CmovMI = BuildMI ( * MBB , FirstMI , DL , TII -> get ( IsWave32 ? <STR_LIT> : <STR_LIT> ) , Exec ) . addImm ( - <NUM_LIT> ) ; if ( ! LIS ) { MI . eraseFromParent


<s> if ( Next == MBB . end ( ) || ! LoweredEndCf . count ( & * Next ) ) continue ; Register SavedExec = TII -> getNamedOperand ( * Next , <STR_LIT> :: src1 ) -> getReg ( ) ; assert ( SavedExec . isVirtual ( ) && <STR_LIT> ) ; const MachineInstr * Def = MRI -> getUniqueVRegDef ( SavedExec


<s> } if ( MDT ) { if ( MDT -> dominates ( & MBB , Succ ) ) MDT -> changeImmediateDominator ( MDT -> getNode ( Succ ) , MDT -> getNode ( & MBB ) -> getIDom ( ) ) ; MDT -> eraseNode ( & MBB ) ; } MBB . clear ( ) ; MBB . eraseFromParent ( ) ; if ( FallThrough && ! FallThrough -> isLayoutSuccessor ( Succ ) ) { if ( ! Succ -> canFallThrough ( ) ) { MachineFunction * MF = FallThrough -> getParent ( ) ; MachineFunction :: iterator FallThroughPos


<s> MachineBasicBlock * FallThrough = nullptr ; while ( ! MBB . predecessors ( ) . empty ( ) ) { MachineBasicBlock * P = * MBB . pred_begin ( ) ; if ( P -> getFallThrough ( ) == & MBB ) FallThrough = P ; P -> ReplaceUsesOfBlockWith ( & MBB , Succ ) ; } MBB . removeSuccessor ( Succ ) ; if ( LIS ) { for ( auto & I : MBB . instrs ( ) ) LIS -> RemoveMachineInstrFromMaps ( I ) ; } if ( MDT ) { if ( MDT -> dominates ( & MBB , Succ ) ) MDT -> changeImmediateDominator ( MDT -> getNode ( Succ ) , MDT -> getNode ( & MBB ) -> getIDom ( ) ) ; MDT -> eraseNode ( & MBB


<s> MDT = getAnalysisIfAvailable < MachineDominatorTree > ( ) ; MRI = & MF . getRegInfo ( ) ; BoolRC = TRI -> getBoolRC ( ) ; if ( ST . isWave32 ( ) ) { AndOpc = <STR_LIT> ; OrOpc = <STR_LIT> ; XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } else { AndOpc = <STR_LIT> ; OrOpc = <STR_LIT> ; XorOpc = <STR_LIT> ; MovTermOpc = <STR_LIT> ; Andn2TermOpc = <STR_LIT> ; XorTermrOpc = <STR_LIT> ; OrTermrOpc = <STR_LIT> ; OrSaveExecOpc = <STR_LIT> ; Exec = <STR_LIT> ; } const bool CanDemote = MF . getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ; for ( auto & MBB : MF ) { bool IsKillBlock = false ; for ( auto & Term : MBB . terminators ( ) ) { if ( TII -> isKillTerminator ( Term . getOpcode ( ) ) ) { KillBlocks . insert ( & MBB ) ; IsKillBlock = true ; break ; } } if ( CanDemote && ! IsKillBlock ) { for ( auto & MI : MBB ) { if ( MI . getOpcode ( ) == <STR_LIT> ) { KillBlocks . insert ( & MBB ) ; break ; } } } } MachineFunction :: iterator NextBB ; for ( MachineFunction :: iterator BI = MF . begin ( ) ; BI != MF . end ( ) ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock * MBB = & * BI ; MachineBasicBlock :: iterator I , E , Next ; E = MBB -> end ( ) ; for ( I = MBB -> begin ( ) ; I != E ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; MachineBasicBlock * SplitMBB = MBB ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : SplitMBB = process ( MI ) ; break ; case <STR_LIT> : case <STR_LIT>


<s> void <ISA_LIT>LowerControlFlow :: combineMasks ( MachineInstr & MI ) { assert ( MI . getNumExplicitOperands ( ) == <NUM_LIT> ) ; SmallVector < MachineOperand , <NUM_LIT> > Ops ; unsigned OpToReplace = <NUM_LIT> ; findMaskOperands ( MI , <NUM_LIT> , Ops ) ; if ( Ops . size ( ) == <NUM_LIT> ) OpToReplace = <NUM_LIT> ; findMaskOperands ( MI , <NUM_LIT> , Ops ) ; if ( Ops . size ( ) != <NUM_LIT> ) return ; unsigned UniqueOpndIdx ; if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else


<s> if ( Ops . size ( ) != <NUM_LIT> ) return ; unsigned UniqueOpndIdx ; if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else if ( Ops [ <NUM_LIT> ] . isIdenticalTo ( Ops [ <NUM_LIT> ] ) ) UniqueOpndIdx = <NUM_LIT> ; else return ; unsigned Reg = MI . getOperand ( OpToReplace ) . getReg ( ) ; MI . RemoveOperand (


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; unsigned Dst = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineInstr * Or = BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , Dst ) . addReg ( <STR_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps ( MI , * Or ) ; MI . eraseFromParent (


<s> MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) , DstReg ) . addReg ( <STR_LIT> ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; } MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . addReg ( DstReg ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> InsertMachineInstrInMaps ( * OrSaveExec ) ; LIS -> InsertMachineInstrInMaps ( * Xor


<s> void <ISA_LIT>LowerControlFlow :: emitEndCf ( MachineInstr & MI ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; MachineBasicBlock :: iterator InsPt = MBB . begin ( ) ; MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( LIS ) LIS -> ReplaceMachineInstrInMaps


<s> MachineInstr * NewMI = BuildMI ( MBB , InsPt , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> )

. addReg ( <STR_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) ;


<s> MI . setDesc ( TII -> get ( <STR_LIT> )


<s> void <ISA_LIT>LowerControlFlow :: findMaskOperands ( MachineInstr & MI , unsigned OpNo , SmallVectorImpl < MachineOperand > & Src ) const { MachineOperand & Op = MI . getOperand ( OpNo ) ; if ( ! Op . isReg ( ) || ! TargetRegisterInfo :: isVirtualRegister ( Op . getReg ( ) ) ) { Src . push_back ( Op ) ; return ; } MachineInstr * Def = MRI -> getUniqueVRegDef ( Op . getReg ( ) ) ; if ( ! Def || Def -> getParent ( ) != MI . getParent ( ) || ! ( Def -> isFullCopy ( ) || ( Def -> getOpcode ( ) == MI . getOpcode ( ) ) ) ) return ; for ( auto I = Def -> getIterator ( ) ; I != MI . getIterator ( ) ; ++ I ) if ( I -> modifiesRegister ( <STR_LIT> , TRI ) && ! ( I -> isCopy ( ) && I -> getOperand ( <NUM_LIT> ) . getReg ( ) != <STR_LIT> ) ) return ; for ( const auto & SrcOp : Def -> explicit_operands ( ) ) if ( SrcOp . isUse ( ) && ( ! SrcOp . isReg ( ) || TargetRegisterInfo :: isVirtualRegister ( SrcOp . getReg ( ) ) || SrcOp . getReg

( ) == <STR_LIT> ) ) Src . push_back ( SrcOp ) ;


<s> DenseSet < const MachineBasicBlock * > Visited ; SmallVector < MachineBasicBlock * , <NUM_LIT> > Worklist ( SMBB -> succ_begin ( ) , SMBB -> succ_end ( ) ) ; while ( ! Worklist . empty ( ) ) { MachineBasicBlock * MBB = Worklist . pop_back_val ( ) ; if ( MBB == EMBB || ! Visited . insert ( MBB ) .


<s> if ( U == MRI -> use_instr_nodbg_end ( ) || std :: next ( U ) != MRI -> use_instr_nodbg_end ( ) || U -> getOpcode ( ) != <STR_LIT> ) return false ; auto SMBB = MI . getParent ( ) ; auto EMBB = U -> getParent ( ) ; DenseSet < const MachineBasicBlock * > Visited ; SmallVector < MachineBasicBlock * , <NUM_LIT> > Worklist ( SMBB -> succ_begin ( ) , SMBB -> succ_end


<s> const DebugLoc & DL = MI . getDebugLoc ( ) ; Register DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; MachineBasicBlock :: iterator Start = MBB . begin ( ) ; Register SaveReg = MRI -> createVirtualRegister ( BoolRC ) ; MachineInstr * OrSaveExec = BuildMI ( MBB , Start , DL , TII -> get ( OrSaveExecOpc ) , SaveReg ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; MachineBasicBlock * DestBB = MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ; MachineBasicBlock :: iterator ElsePt ( MI ) ; MachineInstr * And = BuildMI ( MBB , ElsePt , DL , TII -> get ( AndOpc ) , DstReg ) . addReg ( Exec ) . addReg ( SaveReg ) ; if ( LIS ) LIS -> InsertMachineInstrInMaps ( * And ) ; MachineInstr * Xor = BuildMI ( MBB , ElsePt , DL , TII -> get ( XorTermrOpc ) , Exec ) . addReg ( Exec ) . addReg ( DstReg ) ; ElsePt = skipToUncondBrOrEnd ( MBB , ElsePt ) ; MachineInstr * Branch = BuildMI ( MBB , ElsePt , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( DestBB ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> RemoveMachineInstrFromMaps ( MI ) ; MI . eraseFromParent ( ) ; LIS -> InsertMachineInstrInMaps ( * OrSaveExec ) ; LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * Branch ) ; LIS -> removeInterval ( DstReg


<s> if ( ! SimpleIf ) { Xor = BuildMI ( MBB , I , DL , TII -> get ( XorOpc ) , SaveExecReg ) . addReg ( Tmp ) . addReg ( CopyReg ) ; setImpSCCDefDead ( * Xor , ImpDefSCC . isDead ( ) ) ; } MachineInstr * SetExec = BuildMI ( MBB , I , DL , TII -> get ( MovTermOpc ) , Exec ) . addReg ( Tmp , RegState :: Kill ) ; I = skipToUncondBrOrEnd ( MBB , I ) ; MachineInstr * NewBr = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; if ( ! LIS ) { MI . eraseFromParent ( ) ; return ; } LIS -> InsertMachineInstrInMaps ( * CopyExec ) ; LIS -> ReplaceMachineInstrInMaps ( MI , * And ) ; if ( ! SimpleIf ) LIS -> InsertMachineInstrInMaps ( * Xor ) ; LIS -> InsertMachineInstrInMaps ( * SetExec ) ; LIS -> InsertMachineInstrInMaps ( * NewBr ) ; LIS -> removeAllRegUnitsForPhysReg ( <STR_LIT> )