<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; unsigned MaxNumSGPRs = getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = getMaxNumVGPRs ( MF ) ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumVGPRs ; i < TotalNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg (


<s> unsigned <ISA_LIT>RegisterInfo :: getTotalNumSGPRs ( const <ISA_LIT>Subtarget & ST ) const { if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return


<s> unsigned RegCount = getMaxNumSGPRs ( MF ) ; unsigned Reg ; if ( RegCount & <NUM_LIT> ) { Reg = RegCount - <NUM_LIT> ; } else { Reg = RegCount -


<s> if ( ! RC ) return false ; return ! hasVGPRs (


<s> bool isSGPRClass ( const TargetRegisterClass * RC )


<s> if ( static_cast < int > ( RCID ) == -


<s> if ( static_cast < int > ( RCID ) == - <NUM_LIT> ) return false ; return isSGPRClass ( getRegClass ( RCID )


<s> return SubReg ? alignTo ( getSubRegIdxOffset ( SubReg ) , <NUM_LIT> )


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; const DebugLoc & DL = MI -> getDebugLoc ( ) ; bool IsStore = MI -> mayStore ( ) ; unsigned Opc = MI -> getOpcode ( ) ; int LoadStoreOp = IsStore ? getOffsetMUBUFStore ( Opc ) : getOffsetMUBUFLoad ( Opc ) ; if ( LoadStoreOp == - <NUM_LIT> ) return false ; unsigned Reg = TII -> getNamedOperand ( * MI , <STR_LIT> :: vdata ) ->


<s> unsigned Reg = TII -> getNamedOperand ( * MI , <STR_LIT> :: vdata ) -> getReg ( ) ; BuildMI ( * MBB , MI , DL , TII -> get ( LoadStoreOp ) ) . addReg ( Reg , getDefRegState ( ! IsStore ) ) . addOperand ( * TII -> getNamedOperand ( * MI , <STR_LIT> :: srsrc ) ) . addOperand ( * TII -> getNamedOperand ( * MI , <STR_LIT> :: soffset ) ) . addImm ( Offset ) . addImm (

<NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . setMemRefs ( MI -> memoperands_begin ( ) , MI -> memoperands_end ( ) ) ;


<s> unsigned getSGPR32PressureSet ( ) const { return SGPR32SetID


<s> unsigned getVGPR32PressureSet ( ) const {


<s> } for ( auto Reg : <STR_LIT> ) { Reserved . set ( getSubReg ( Reg , <STR_LIT> ) ) ; Register Low = getSubReg ( Reg , <STR_LIT> ) ; if ( ! <STR_LIT> . contains ( Low ) ) Reserved . set ( Low ) ; } for ( auto Reg : <STR_LIT> ) { Reserved . set ( getSubReg ( Reg , <STR_LIT> ) ) ; } if ( ! ST . hasMAIInsts ( ) ) { for ( unsigned i = <NUM_LIT> ; i < MaxNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register ScratchRSrcReg = MFI -> getScratchRSrcReg ( ) ; if ( ScratchRSrcReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchRSrcReg ) ; } MCRegister StackPtrReg = MFI -> getStackPtrOffsetReg ( ) ; if ( StackPtrReg ) { reserveRegisterTuples ( Reserved , StackPtrReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , StackPtrReg ) ) ; } MCRegister FrameReg = MFI -> getFrameOffsetReg ( ) ; if ( FrameReg ) { reserveRegisterTuples ( Reserved , FrameReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , FrameReg ) ) ; } if ( hasBasePointer ( MF ) ) { MCRegister BasePtrReg = getBaseRegister ( ) ; reserveRegisterTuples ( Reserved , BasePtrReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , BasePtrReg ) ) ; } for ( auto Reg : MFI -> WWMReservedRegs ) { reserveRegisterTuples ( Reserved , Reg .


<s> bool <ISA_LIT>RegisterInfo :: hasAGPRs ( const TargetRegisterClass *


<s> MIB . addReg ( SB . SuperReg , RegState :: ImplicitDefine ) ; } if ( SB . NumSubRegs > <NUM_LIT> ) MIB . addReg ( SB . SuperReg , getKillRegState ( UseKill ) | RegState :: Implicit ) ; } } else { SB . prepare ( ) ; unsigned SubKillState = getKillRegState ( ( SB . NumSubRegs == <NUM_LIT> ) && SB . IsKill ) ; auto PVD = SB . getPerVGPRData ( ) ; for ( unsigned Offset = <NUM_LIT> ; Offset < PVD . NumVGPRs ; ++ Offset ) { unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; MachineInstrBuilder WriteLane = BuildMI ( SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SB . TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PVD . PerVGPR ) . addReg ( SB . TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ; if ( LIS ) { if ( i == <NUM_LIT> ) LIS -> ReplaceMachineInstrInMaps ( * MI , * WriteLane ) ; else LIS -> InsertMachineInstrInMaps ( * WriteLane ) ; } if ( SB . NumSubRegs > <NUM_LIT> ) { unsigned SuperKillState = <NUM_LIT> ; if ( i + <NUM_LIT> == SB . NumSubRegs ) SuperKillState |= getKillRegState ( SB . IsKill ) ; WriteLane . addReg ( SB . SuperReg , RegState :: Implicit | SuperKillState ) ; } } SB . readWriteTmpVGPR ( Offset , false ) ; } SB . restore ( )


<s> bool <ISA_LIT>RegisterInfo :: isFrameOffsetLegal ( const MachineInstr * MI , Register BaseReg , int64_t Offset ) const { if ( ! <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) && ! <ISA_LIT>InstrInfo :: isFLATScratch ( * MI ) ) return false ; int64_t NewOffset = Offset + getScratchInstrOffset ( MI ) ; if ( <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) ) return <ISA_LIT>InstrInfo :: isLegalMUBUFImmOffset


<s> int64_t FullOffset = Offset + getScratchInstrOffset ( MI ) ; if ( <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) ) return ! <ISA_LIT>InstrInfo :: isLegalMUBUFImmOffset ( FullOffset ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo


<s> assert ( TII -> isLegalFLATOffset ( NewOffset , <ISA_LIT>AS :: PRIVATE_ADDRESS , true ) && <STR_LIT> ) ; FIOp -> ChangeToRegister ( BaseReg , false ) ; OffsetOp -> setImm ( NewOffset ) ; return ; } MachineOperand * SOffset = TII -> getNamedOperand ( MI , <STR_LIT> :: soffset ) ; assert ( SOffset -> isImm ( ) && SOffset -> getImm ( ) == <NUM_LIT> ) ; assert ( <ISA_LIT>InstrInfo :: isLegalMUBUFImmOffset ( NewOffset ) && <STR_LIT> ) ; FIOp -> ChangeToRegister ( BaseReg , false ) ; OffsetOp -> setImm (


<s> if ( Reg == <STR_LIT> ) return MachineInstrBuilder ( ) ; bool IsStore = MI -> mayStore ( ) ; MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; auto * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; unsigned Dst = IsStore ? Reg : ValueReg ; unsigned Src = IsStore ? ValueReg : Reg ; unsigned Opc = ( IsStore ^ TRI -> isVGPR ( MRI , Reg ) ) ? <STR_LIT> : <STR_LIT> ; auto MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( Opc ) , Dst ) . addReg ( Src , getKillRegState ( IsKill ) ) ; MIB -> setAsmPrinterFlag ( MachineInstr :: ReloadReuse ) ; return MIB


<s> if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT>


<s> if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return &


<s> if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return &


<s> if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; return


<s> if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT>


<s> static const TargetRegisterClass * const BaseClasses [ ] = { & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , } ; for ( const TargetRegisterClass * BaseClass : BaseClasses ) { if ( BaseClass -> contains ( Reg )


<s> if ( SpillToVGPR ) { for ( unsigned i = <NUM_LIT> , e = SB . NumSubRegs ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; auto MIB = BuildMI ( SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SubReg ) . addReg ( Spill . VGPR ) . addImm ( Spill . Lane ) ; if ( SB . NumSubRegs > <NUM_LIT> && i == <NUM_LIT> ) MIB . addReg ( SB . SuperReg , RegState :: ImplicitDefine ) ; } } else { SB . prepare ( ) ; auto PVD = SB . getPerVGPRData ( ) ; for ( unsigned Offset = <NUM_LIT> ; Offset < PVD . NumVGPRs ; ++ Offset ) { SB . readWriteTmpVGPR ( Offset , true ) ; for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; bool LastSubReg = ( i +


<s> SGPRSpillBuilder SB ( * this , * ST . getInstrInfo ( ) , isWave32 , MI , Index , RS ) ; ArrayRef < <ISA_LIT>MachineFunctionInfo :: SpilledReg > VGPRSpills = SB . MFI . getSGPRToVGPRSpills ( Index ) ; bool SpillToVGPR = ! VGPRSpills . empty ( ) ; if ( OnlyToVGPR && ! SpillToVGPR ) return false ; if ( SpillToVGPR ) { for ( unsigned i = <NUM_LIT> , e = SB . NumSubRegs ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; auto MIB = BuildMI ( SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SubReg ) . addReg ( Spill . VGPR ) . addImm ( Spill . Lane ) ; if ( SB . NumSubRegs > <NUM_LIT> && i == <NUM_LIT> ) MIB . addReg ( SB . SuperReg , RegState :: ImplicitDefine ) ; } } else { SB . prepare ( ) ; auto PVD = SB . getPerVGPRData ( ) ; for ( unsigned Offset = <NUM_LIT> ; Offset < PVD . NumVGPRs ; ++ Offset ) { SB . readWriteTmpVGPR ( Offset , true ) ; for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; bool LastSubReg = ( i + <NUM_LIT> == e ) ; auto MIB = BuildMI ( SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SubReg ) . addReg ( SB . TmpVGPR , getKillRegState ( LastSubReg ) ) . addImm ( i ) ; if ( SB . NumSubRegs > <NUM_LIT> && i == <NUM_LIT> ) MIB . addReg ( SB . SuperReg , RegState :: ImplicitDefine ) ; } } SB . restore (


<s> if ( i == <NUM_LIT> && SB . NumSubRegs > <NUM_LIT> ) { MIB . addReg ( SB . SuperReg , RegState :: ImplicitDefine ) ; } if ( SB . NumSubRegs > <NUM_LIT> ) MIB . addReg ( SB . SuperReg , getKillRegState ( UseKill ) | RegState :: Implicit ) ; } } else { SB . prepare ( ) ; unsigned SubKillState = getKillRegState ( ( SB . NumSubRegs == <NUM_LIT> ) && SB . IsKill ) ; auto PVD = SB . getPerVGPRData ( ) ; for ( unsigned Offset = <NUM_LIT> ; Offset < PVD . NumVGPRs ; ++ Offset ) { unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; MachineInstrBuilder WriteLane = BuildMI ( SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SB . TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PVD . PerVGPR ) . addReg ( SB . TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ; if ( SB .


<s> SB . prepare ( ) ; unsigned SubKillState = getKillRegState ( ( SB . NumSubRegs == <NUM_LIT> ) && SB . IsKill ) ; auto PVD = SB . getPerVGPRData ( ) ; for ( unsigned Offset = <NUM_LIT> ; Offset < PVD . NumVGPRs ; ++ Offset ) { unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; MachineInstrBuilder WriteLane = BuildMI ( SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SB . TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PVD . PerVGPR ) . addReg ( SB . TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ; if ( SB . NumSubRegs > <NUM_LIT> ) { unsigned SuperKillState = <NUM_LIT> ; if ( i + <NUM_LIT> == SB . NumSubRegs ) SuperKillState |= getKillRegState ( SB . IsKill ) ; WriteLane . addReg ( SB . SuperReg , RegState :: Implicit | SuperKillState ) ; } } SB . readWriteTmpVGPR ( Offset , false ) ; } SB . restore (


<s> switch ( RB . getID ( ) ) { case <STR_LIT> : return & <STR_LIT> ; case <STR_LIT> : return & <STR_LIT> ; case <STR_LIT> : return & <STR_LIT> ; case <STR_LIT> : return & <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; } } case <NUM_LIT> : return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; default : if ( Size < <NUM_LIT> ) return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; assert ( Size < <NUM_LIT> && <STR_LIT> ) ; return getRegClassForSizeOnBank ( PowerOf2Ceil ( Size )


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; if ( isWave32 ) { Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; } unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } for ( auto Reg : <STR_LIT> ) { Reserved . set ( getSubReg ( Reg , <STR_LIT> ) ) ; Register Low = getSubReg ( Reg , <STR_LIT> ) ; if ( ! <STR_LIT> . contains ( Low ) ) Reserved . set ( Low ) ; } Register ScratchRSrcReg = MFI -> getScratchRSrcReg ( ) ; if ( ScratchRSrcReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchRSrcReg ) ; } MCRegister StackPtrReg = MFI -> getStackPtrOffsetReg ( ) ; if ( StackPtrReg ) { reserveRegisterTuples ( Reserved , StackPtrReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , StackPtrReg ) ) ; } MCRegister FrameReg = MFI -> getFrameOffsetReg ( ) ; if ( FrameReg ) { reserveRegisterTuples ( Reserved , FrameReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; unsigned SuperReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool IsKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; const DebugLoc & DL = MI -> getDebugLoc ( ) ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; bool SpillToSMEM = spillSGPRToSMEM ( ) ; if ( SpillToSMEM && OnlyToVGPR ) return false ; assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ; unsigned OffsetReg = <STR_LIT> ; unsigned M0CopyReg = <STR_LIT> ; if ( SpillToSMEM ) { if ( RS -> isRegUsed ( <STR_LIT> ) ) { M0CopyReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , M0CopyReg ) . addReg ( <STR_LIT> ) ; } } unsigned ScalarStoreOp ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; if ( SpillToSMEM && isSGPRClass ( RC ) ) { std :: tie ( EltSize , ScalarStoreOp ) = getSpillEltSize ( RC -> getSize ( ) , true ) ; } ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { unsigned SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; if ( SpillToSMEM ) { int64_t FrOffset = FrameInfo . getObjectOffset ( Index ) ; unsigned Align = FrameInfo . getObjectAlignment ( Index


<s> } } else { Register TmpVGPR = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpVGPR ) ; unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ; unsigned PerVGPR = <NUM_LIT> ; unsigned NumVGPRs = ( NumSubRegs + ( PerVGPR - <NUM_LIT> ) ) / PerVGPR ; int64_t VGPRLanes = ( <NUM_LIT> << std :: min ( PerVGPR , NumSubRegs ) ) - <NUM_LIT> ; for ( unsigned Offset = <NUM_LIT> ; Offset < NumVGPRs ; ++ Offset ) { unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PerVGPR , NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : Register ( getSubReg ( SuperReg , SplitParts [ i ] ) ) ; MachineInstrBuilder WriteLane = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PerVGPR ) . addReg ( TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ; if ( NumSubRegs > <NUM_LIT> ) { unsigned SuperKillState = <NUM_LIT> ; if ( i + <NUM_LIT> == NumSubRegs ) SuperKillState |= getKillRegState ( IsKill ) ; WriteLane . addReg ( SuperReg , RegState :: Implicit | SuperKillState ) ; } } buildSGPRSpillLoadStore ( MI , Index , Offset , EltSize , TmpVGPR , VGPRLanes , RS , false ) ; } } MI -> eraseFromParent ( ) ; MFI -> addToSpilledSGPRs ( NumSubRegs


<s> switch ( BitWidth ) { case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return &


<s> case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; default : return


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getSGPRClassForBitWidth ( unsigned BitWidth ) { switch ( BitWidth ) { case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; default : return nullptr


<s> return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT>


<s> } Register TmpReg ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i != e ; ++ i , Offset += EltSize ) { Register SubReg = NumSubRegs == <NUM_LIT> ? Register ( ValueReg ) : getSubReg ( ValueReg , getSubRegFromChannel ( i ) ) ; unsigned SOffsetRegState = <NUM_LIT> ; unsigned SrcDstRegState = getDefRegState ( ! IsStore ) ; if ( i + <NUM_LIT> == e ) { SOffsetRegState |= getKillRegState ( Scavenged ) ; SrcDstRegState |= getKillRegState ( IsKill ) ; } auto MIB = spillVGPRtoAGPR ( ST , MI , Index , i , SubReg , IsKill ) ; if ( ! MIB . getInstr ( ) ) { unsigned FinalReg = SubReg ; if ( hasAGPRs ( RC ) ) { if ( ! TmpReg ) { assert ( RS && <STR_LIT> ) ; TmpReg = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpReg ) ; } if ( IsStore ) BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SubReg , getKillRegState ( IsKill ) ) ; SubReg = TmpReg ; } MachinePointerInfo PInfo = BasePtrInfo . getWithOffset ( EltSize * i ) ; MachineMemOperand * NewMMO = MF -> getMachineMemOperand ( PInfo , MMO -> getFlags ( ) , EltSize , commonAlignment ( Alignment , EltSize * i ) ) ; MIB = BuildMI ( * MBB , MI , DL , Desc ) . addReg ( SubReg , getDefRegState ( ! IsStore ) | getKillRegState ( IsKill ) ) . addReg ( ScratchRsrcReg ) ; if ( SOffset == <STR_LIT> ) { MIB . addImm ( <NUM_LIT> ) ; } else { MIB . addReg (


<s> <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; DenseSet < Register > SGPRSpillVGPRDefinedSet ; ArrayRef < <ISA_LIT>MachineFunctionInfo :: SpilledReg > VGPRSpills = MFI -> getSGPRToVGPRSpills ( Index ) ; bool SpillToVGPR = ! VGPRSpills . empty ( ) ; if ( OnlyToVGPR && ! SpillToVGPR ) return false ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; Register SuperReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool IsKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; const DebugLoc & DL = MI -> getDebugLoc ( ) ; assert ( SpillToVGPR || ( SuperReg != MFI -> getStackPtrOffsetReg ( ) && SuperReg != MFI -> getFrameOffsetReg ( ) ) ) ; assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ; assert ( SuperReg != <STR_LIT> && SuperReg != <STR_LIT> && SuperReg != <STR_LIT> && <STR_LIT> ) ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; if ( SpillToVGPR ) { for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; bool VGPRDefined = true ; if ( MBB == & MF -> front ( ) ) VGPRDefined = ! SGPRSpillVGPRDefinedSet . insert ( Spill . VGPR )


<s> bool IsKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; const DebugLoc & DL = MI -> getDebugLoc ( ) ; assert ( SpillToVGPR || ( SuperReg != MFI -> getStackPtrOffsetReg ( ) && SuperReg != MFI -> getFrameOffsetReg ( ) ) ) ; assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ; assert ( SuperReg != <STR_LIT> && SuperReg != <STR_LIT> && SuperReg != <STR_LIT> && <STR_LIT> ) ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; if ( SpillToVGPR ) { for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; bool VGPRDefined = true ; if ( MBB == & MF -> front ( ) ) VGPRDefined = ! SGPRSpillVGPRDefinedSet . insert ( Spill . VGPR ) . second ; BuildMI ( * MBB , MI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , Spill . VGPR ) . addReg ( SubReg , getKillRegState ( IsKill ) ) . addImm ( Spill . Lane ) . addReg ( Spill . VGPR , VGPRDefined ? <NUM_LIT> : RegState :: Undef ) ; } } else { Register TmpVGPR = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpVGPR ) ; unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT>


<s> } else { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , SOffset ) . addReg ( ScratchOffsetReg ) . addImm ( Offset ) ; } Offset = <NUM_LIT> ; } for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i != e ; ++ i , Offset += EltSize ) { Register SubReg = NumSubRegs == <NUM_LIT> ? Register ( ValueReg ) : getSubReg ( ValueReg , getSubRegFromChannel ( i ) ) ; unsigned SOffsetRegState = <NUM_LIT> ; unsigned SrcDstRegState = getDefRegState ( ! IsStore ) ; if ( i + <NUM_LIT> == e ) { SOffsetRegState |= getKillRegState ( Scavenged ) ; SrcDstRegState |= getKillRegState ( IsKill ) ; } auto MIB = spillVGPRtoAGPR ( ST , MI , Index , i , SubReg , IsKill ) ; if ( ! MIB . getInstr ( ) ) { unsigned FinalReg = SubReg ; if ( TmpReg != <STR_LIT> ) { if ( IsStore ) BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SubReg , getKillRegState ( IsKill ) ) ; SubReg = TmpReg ; } MachinePointerInfo PInfo = BasePtrInfo . getWithOffset ( EltSize * i ) ; MachineMemOperand * NewMMO = MF -> getMachineMemOperand ( PInfo , MMO -> getFlags ( ) , EltSize , MinAlign ( Align , EltSize * i ) ) ; MIB = BuildMI ( * MBB , MI , DL , Desc ) . addReg ( SubReg , getDefRegState ( ! IsStore ) | getKillRegState ( IsKill ) ) . addReg ( ScratchRsrcReg ) ; if ( SOffset == <STR_LIT> ) { MIB . addImm ( <NUM_LIT> ) ; } else


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned BaseIdx = alignDown ( ST . getMaxNumSGPRs ( MF ) , <NUM_LIT> ) - <NUM_LIT> ; unsigned BaseReg ( <STR_LIT> . getRegister ( BaseIdx


<s> unsigned BaseReg ( <STR_LIT> . getRegister ( BaseIdx ) ) ; return getMatchingSuperReg ( BaseReg , <STR_LIT> , &


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; if ( isWave32 ) { Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; } unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumVGPRs ; i < TotalNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } for ( auto Reg : <STR_LIT> ) { Reserved . set ( getSubReg ( Reg , <STR_LIT> ) ) ; Register Low = getSubReg ( Reg , <STR_LIT> ) ; if ( ! <STR_LIT> . contains ( Low ) ) Reserved . set ( Low ) ; } for ( auto Reg : <STR_LIT> ) { Reserved . set ( getSubReg ( Reg , <STR_LIT> ) ) ; } if ( ! ST . hasMAIInsts ( ) ) { for ( unsigned i = <NUM_LIT> ; i < MaxNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo


<s> bool isVGPRClass ( const TargetRegisterClass


<s> bool isVGPRClass ( const TargetRegisterClass *


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ; const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF -> getSubtarget ( ) . getInstrInfo ( ) ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; bool IsStore = TII -> get ( LoadStoreOp ) . mayStore ( ) ; bool RanOutOfSGPRs = false ; bool Scavenged = false ; unsigned SOffset = ScratchOffset ; unsigned OriginalImmOffset = Offset ; unsigned NumSubRegs = getNumSubRegsForSpillOp ( MI -> getOpcode ( ) ) ; unsigned Size = NumSubRegs * <NUM_LIT> ; if ( ! isUInt < <NUM_LIT> > ( Offset + Size ) ) { SOffset = <STR_LIT> ; if ( RS ) SOffset = RS -> FindUnusedReg ( & <STR_LIT> ) ; if ( SOffset == <STR_LIT> ) { RanOutOfSGPRs = true ; SOffset = ScratchOffset ; } else { Scavenged = true ; } BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , SOffset ) . addReg ( ScratchOffset ) . addImm ( Offset ) ; Offset = <NUM_LIT> ; } for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i != e ; ++ i , Offset += <NUM_LIT> ) { unsigned SubReg = NumSubRegs > <NUM_LIT> ? getPhysRegSubReg


<s> } else { unsigned Align = FrameInfo -> getObjectAlignment ( Index ) ; unsigned Size = FrameInfo -> getObjectSize ( Index ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , Index ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addFrameIndex ( Index ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( MFI -> getScratchWaveOffsetReg ( ) ) . addImm ( i * <NUM_LIT> ) . addMemOperand ( MMO ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , SubReg ) . addReg ( TmpReg , RegState :: Kill ) . addReg ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) , RegState :: ImplicitDefine ) ; } } MI -> eraseFromParent ( ) ; break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : buildScratchLoadStore ( MI , <STR_LIT> , TII -> getNamedOperand ( * MI , <STR_LIT> :: src ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: scratch_rsrc ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: scratch_offset ) -> getReg ( ) , FrameInfo -> getObjectOffset ( Index ) + TII -> getNamedOperand ( * MI , <STR_LIT> :: offset ) -> getImm ( ) , RS ) ; MI -> eraseFromParent ( ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { buildScratchLoadStore ( MI , <STR_LIT> , TII -> getNamedOperand ( * MI , <STR_LIT> :: dst ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: scratch_rsrc ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: scratch_offset ) -> getReg ( ) , FrameInfo -> getObjectOffset ( Index ) + TII -> getNamedOperand ( * MI , <STR_LIT> :: offset ) -> getImm ( ) , RS ) ; MI -> eraseFromParent ( ) ; break ; } default : { int64_t Offset = FrameInfo -> getObjectOffset ( Index ) ; FIOp . ChangeToImmediate ( Offset ) ; if ( ! TII -> isImmOperandLegal ( MI , FIOperandNum


<s> const <ISA_LIT>MachineFunctionInfo & MFI = * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; unsigned <ISA_LIT>MDPerCU = <NUM_LIT> ; unsigned MaxInvocationsPerWave = <ISA_LIT>MDPerCU * ST . getWavefrontSize ( ) ; return alignTo ( MFI . getMaximumWorkGroupSize ( MF ) , MaxInvocationsPerWave


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; unsigned MaxWaveCountPer<ISA_LIT>MD = getMaxWaveCountPer<ISA_LIT>MD ( MF ) ; unsigned TotalSGPRCountPer<ISA_LIT>MD , AddressableSGPRCount , SGPRUsageAlignment ; unsigned ReservedSGPRCount ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { TotalSGPRCountPer<ISA_LIT>MD = <NUM_LIT> ; AddressableSGPRCount = <NUM_LIT> ; SGPRUsageAlignment = <NUM_LIT> ; ReservedSGPRCount = <NUM_LIT> ; } else { TotalSGPRCountPer<ISA_LIT>MD = <NUM_LIT> ; AddressableSGPRCount = <NUM_LIT> ; SGPRUsageAlignment = <NUM_LIT> ; ReservedSGPRCount = <NUM_LIT> ; } unsigned MaxSGPRCount = (


<s> if ( SGPRPressureSets . test ( Idx ) && VGPRPressureSets . test ( Idx ) ) { return VSLimit ; } if ( SGPRPressureSets . test ( Idx ) ) return SGPRLimit ; return VGPRLimit


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; unsigned MaxWorkGroupSGPRCount = getMaxWorkGroupSGPRCount ( MF ) ; unsigned MaxWorkGroupVGPRCount = getMaxWorkGroupVGPRCount ( MF ) ; unsigned NumSGPRs = <STR_LIT> . getNumRegs ( ) ; unsigned NumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxWorkGroupSGPRCount ; i < NumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } for ( unsigned i = MaxWorkGroupVGPRCount ; i < NumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; if ( ScratchWaveOffsetReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchWaveOffsetReg ) ; } unsigned ScratchRSrcReg = MFI -> getScratchRSrcReg ( ) ; if ( ScratchRSrcReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchRSrcReg


<s> MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; unsigned UnusedCarry = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , Ins , DL , TII -> get ( <STR_LIT> ) , BaseReg ) . addReg ( UnusedCarry ,

RegState :: Define | RegState :: Dead ) . addImm ( Offset ) . addFrameIndex ( FrameIdx ) ;


<s> if ( SeenFI ) llvm_unreachable ( <STR_LIT> ) ; SeenFI = true ; } } MachineOperand * FIOp = TII -> getNamedOperand ( MI , <STR_LIT> :: vaddr ) ; assert ( FIOp && FIOp -> isFI ( ) && <STR_LIT> ) ; assert ( TII -> isMUBUF ( MI ) ) ; MachineOperand * OffsetOp = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) ; int64_t NewOffset = OffsetOp -> getImm ( ) + Offset ; if ( isUInt < <NUM_LIT> > ( NewOffset ) ) { FIOp -> ChangeToRegister ( BaseReg


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand * VData = TII -> getNamedOperand ( * MI , <STR_LIT> :: vdata ) ; buildSpillLoadStore ( MI , <STR_LIT> , Index , VData -> getReg ( ) , VData -> isKill ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: srsrc ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: soffset ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: offset ) -> getImm ( ) , * MI -> memoperands_begin ( ) , RS ) ; MI -> eraseFromParent ( ) ; break ; } default : { const DebugLoc & DL = MI -> getDebugLoc ( ) ; bool IsMUBUF = TII -> isMUBUF ( * MI ) ; if ( ! IsMUBUF && MFI -> getFrameOffsetReg ( ) != MFI -> getScratchWaveOffsetReg ( ) ) { unsigned DiffReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; bool IsCopy = MI -> getOpcode ( ) == <STR_LIT> ; unsigned ResultReg = IsCopy ? MI -> getOperand ( <NUM_LIT> ) . getReg ( ) : MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , DiffReg ) . addReg ( MFI -> getFrameOffsetReg ( ) ) . addReg ( MFI -> getScratchWaveOffsetReg ( ) ) ; int64_t Offset = FrameInfo . getObjectOffset ( Index ) ; if ( Offset == <NUM_LIT> ) { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , ResultReg ) . addImm ( Log2_32 ( ST . getWavefrontSize ( ) ) ) . addReg ( DiffReg ) ; } else { unsigned ScaledReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , ScaledReg ) . addImm ( Log2_32 ( ST

. getWavefrontSize ( ) ) ) . addReg ( DiffReg , RegState :: Kill ) ;


<s> if ( SpillToSMEM && OnlyToVGPR ) return false ; assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ; unsigned OffsetReg = <STR_LIT> ; unsigned M0CopyReg = <STR_LIT> ; if ( SpillToSMEM ) { if ( RS -> isRegUsed ( <STR_LIT> ) ) { M0CopyReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , M0CopyReg ) . addReg ( <STR_LIT> ) ; } } unsigned EltSize = <NUM_LIT> ; unsigned ScalarLoadOp ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; if ( SpillToSMEM && isSGPRClass ( RC ) ) { std :: tie ( EltSize , ScalarLoadOp ) = getSpillEltSize ( getRegSizeInBits ( * RC ) / <NUM_LIT> , false ) ; } ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; int64_t FrOffset = FrameInfo . getObjectOffset ( Index ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { unsigned SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; if ( SpillToSMEM ) { unsigned Align = FrameInfo . getObjectAlignment ( Index ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , Index , EltSize * i ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , EltSize , MinAlign ( Align , EltSize * i ) ) ; int64_t Offset = ( ST . getWavefrontSize ( ) * FrOffset ) + ( EltSize * i ) ; if ( Offset != <NUM_LIT> ) { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , OffsetReg ) . addReg ( MFI -> getFrameOffsetReg ( ) ) . addImm ( Offset ) ; } else { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , OffsetReg ) . addReg ( MFI -> getFrameOffsetReg ( ) ) ; } auto MIB = BuildMI ( * MBB , MI , DL , TII -> get ( ScalarLoadOp ) , SubReg ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( OffsetReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT>


<s> bool <ISA_LIT>RegisterInfo :: spillSGPR ( MachineBasicBlock :: iterator MI , int Index , RegScavenger * RS , bool OnlyToVGPR ) const { MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineFunction * MF = MBB -> getParent ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; DenseSet < unsigned > SGPRSpillVGPRDefinedSet ; ArrayRef < <ISA_LIT>MachineFunctionInfo :: SpilledReg > VGPRSpills = MFI -> getSGPRToVGPRSpills ( Index ) ; bool SpillToVGPR = ! VGPRSpills . empty ( ) ; if ( OnlyToVGPR && ! SpillToVGPR ) return false ; MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; unsigned SuperReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool IsKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; const DebugLoc & DL = MI -> getDebugLoc ( ) ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; bool SpillToSMEM = spillSGPRToSMEM ( ) ; if ( SpillToSMEM && OnlyToVGPR ) return false ; assert ( SpillToVGPR || ( SuperReg != MFI -> getStackPtrOffsetReg ( ) && SuperReg != MFI -> getFrameOffsetReg ( ) && SuperReg != MFI -> getScratchWaveOffsetReg ( ) ) ) ; assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ; unsigned OffsetReg = <STR_LIT> ; unsigned M0CopyReg = <STR_LIT> ; if ( SpillToSMEM ) { if ( RS -> isRegUsed ( <STR_LIT> ) ) { M0CopyReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , M0CopyReg ) . addReg ( <STR_LIT> ) ; } } unsigned ScalarStoreOp ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; if ( SpillToSMEM && isSGPRClass


<s> } unsigned ScalarStoreOp ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; if ( SpillToSMEM && isSGPRClass ( RC ) ) { std :: tie ( EltSize , ScalarStoreOp ) = getSpillEltSize ( getRegSizeInBits ( * RC ) / <NUM_LIT> , true ) ; } ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { unsigned SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; if ( SpillToSMEM ) { int64_t FrOffset = FrameInfo . getObjectOffset ( Index ) ; unsigned Align = FrameInfo . getObjectAlignment ( Index ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , Index , EltSize * i ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , EltSize , MinAlign ( Align , EltSize * i ) ) ; int64_t Offset = ( ST . getWavefrontSize ( ) * FrOffset ) + ( EltSize * i ) ; if ( Offset != <NUM_LIT> ) { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , OffsetReg ) . addReg ( MFI -> getFrameOffsetReg ( ) ) . addImm ( Offset ) ; } else { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , OffsetReg ) . addReg ( MFI -> getFrameOffsetReg ( ) ) ; } BuildMI ( * MBB , MI , DL , TII -> get ( ScalarStoreOp ) ) . addReg ( SubReg , getKillRegState ( IsKill ) ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( OffsetReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; continue ; } if ( SpillToVGPR ) { <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; bool VGPRDefined = true ; if ( MBB == & MF -> front ( ) ) VGPRDefined = ! SGPRSpillVGPRDefinedSet . insert ( Spill . VGPR ) . second ; BuildMI ( * MBB , MI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , Spill . VGPR ) . addReg ( SubReg , getKillRegState ( IsKill ) ) . addImm ( Spill . Lane ) . addReg ( Spill . VGPR , VGPRDefined ? <NUM_LIT> : RegState :: Undef ) ; } else { if ( OnlyToVGPR ) return false ; unsigned TmpReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; MachineInstrBuilder Mov = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; const MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ; const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF -> getSubtarget ( ) . getInstrInfo ( ) ) ; LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; bool IsLoad = TII -> get ( LoadStoreOp ) . mayLoad ( ) ; bool RanOutOfSGPRs = false ; unsigned SOffset = ScratchOffset ; unsigned NumSubRegs = getNumSubRegsForSpillOp ( MI -> getOpcode ( ) ) ; unsigned Size = NumSubRegs * <NUM_LIT> ; if ( ! isUInt < <NUM_LIT> > ( Offset + Size ) ) { SOffset = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; if ( SOffset == <STR_LIT> ) { RanOutOfSGPRs = true ; SOffset = <STR_LIT> ; } BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , SOffset ) . addReg ( ScratchOffset ) . addImm ( Offset ) ; Offset = <NUM_LIT> ; } if ( RanOutOfSGPRs ) Ctx . emitError ( <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i != e ; ++ i , Offset += <NUM_LIT> ) { unsigned SubReg = NumSubRegs > <NUM_LIT> ? getPhysRegSubReg


<s> I != E ; ++ I ) { if ( ! MRI . isPhysRegUsed ( *


<s> unsigned <ISA_LIT>RegisterInfo :: findUnusedRegister ( const MachineRegisterInfo & MRI , const TargetRegisterClass * RC ) const { for ( TargetRegisterClass :: iterator I = RC -> begin ( ) , E = RC -> end ( ) ; I != E ; ++ I ) { if ( ! MRI . isPhysRegUsed ( * I ) ) return * I ; } return


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getCFGStructurizerRegClass ( MVT VT ) const { switch ( VT .


<s> return SRC ; } else if ( SRC == & <STR_LIT> ) { return & <STR_LIT> ; } else if ( getCommonSubClass ( SRC , & <STR_LIT> ) ) { return & <STR_LIT> ; } else if ( getCommonSubClass ( SRC , & <STR_LIT> ) ) { return & <STR_LIT> ; } else if ( getCommonSubClass ( SRC , &


<s> BitVector Reserved ( getNumRegs ( ) ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; if ( MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . hasSGPRInitBug ( ) ) { unsigned NumSGPRs = <STR_LIT> . getNumRegs ( ) ; unsigned Limit = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> ; for ( unsigned i = Limit ; i < NumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i


<s> Reserved . set ( <STR_LIT> ) ; if ( MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . hasSGPRInitBug ( ) ) { unsigned NumSGPRs = <STR_LIT> . getNumRegs ( ) ; unsigned Limit = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> ; for ( unsigned i = Limit ; i < NumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister (


<s> return getCommonSubClass ( & <STR_LIT> , RC ) || getCommonSubClass ( & <STR_LIT> , RC ) || getCommonSubClass ( & <STR_LIT> , RC ) || getCommonSubClass ( & <STR_LIT> , RC ) || getCommonSubClass ( & <STR_LIT> ,

RC ) || getCommonSubClass ( & <STR_LIT> , RC ) ;


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumVGPRs ; i < TotalNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved ,


<s> NeedSuperRegImpOperand = true ; unsigned State = SrcDstRegState ; if ( Lane != LaneE ) State &= ~ RegState :: Kill ; MIB . addReg ( ValueReg , RegState :: Implicit | State ) ; } RemEltSize -= <NUM_LIT> ; } if ( ! RemEltSize ) continue ; if ( RemEltSize != EltSize ) { assert ( IsFlat && EltSize > <NUM_LIT> ) ; unsigned NumRegs = RemEltSize / <NUM_LIT> ; SubReg = Register ( getSubReg ( ValueReg , getSubRegFromChannel ( RegOffset / <NUM_LIT> , NumRegs ) ) ) ; unsigned Opc = getFlatScratchSpillOpcode ( TII , LoadStoreOp , RemEltSize ) ; Desc = & TII -> get ( Opc ) ; } unsigned FinalReg = SubReg ; if ( IsAGPR ) { assert ( EltSize == <NUM_LIT> ) ; if ( ! TmpReg ) { assert ( RS && <STR_LIT> ) ; TmpReg = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpReg ) ; } if ( IsStore ) { auto AccRead = BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SubReg , getKillRegState ( IsKill ) ) ; if ( NeedSuperRegDef ) AccRead . addReg ( ValueReg , RegState :: ImplicitDefine ) ; AccRead -> setAsmPrinterFlag ( MachineInstr :: ReloadReuse ) ; } SubReg = TmpReg ; } MachinePointerInfo PInfo = BasePtrInfo . getWithOffset ( RegOffset ) ; MachineMemOperand * NewMMO = MF -> getMachineMemOperand ( PInfo , MMO -> getFlags ( ) , RemEltSize , commonAlignment ( Alignment , RegOffset ) ) ; auto MIB = BuildMI ( MBB , MI , DL , * Desc ) . addReg ( SubReg , getDefRegState ( ! IsStore ) | getKillRegState ( IsKill ) ) ; if ( ! IsFlat ) MIB . addReg ( FuncInfo -> getScratchRSrcReg ( ) ) ; if ( SOffset == <STR_LIT> ) { if ( ! IsFlat ) MIB . addImm ( <NUM_LIT> ) ; } else { MIB . addReg ( SOffset , SOffsetRegState ) ; } MIB . addImm ( Offset + RegOffset ) . addImm ( <NUM_LIT> ) ; if ( ! IsFlat ) MIB . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; MIB . addMemOperand ( NewMMO ) ; if ( ! IsAGPR && NeedSuperRegDef ) MIB . addReg ( ValueReg , RegState :: ImplicitDefine ) ; if ( ! IsStore && TmpReg != <STR_LIT> ) { MIB = BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , FinalReg ) . addReg ( TmpReg , RegState :: Kill ) ; MIB -> setAsmPrinterFlag ( MachineInstr :: ReloadReuse ) ; } if ( NeedSuperRegImpOperand ) MIB . addReg ( ValueReg , RegState :: Implicit |


<s> BuildMI ( * MBB , MI , DL , TII . get ( MovOpc ) , SavedExecReg ) . addReg ( ExecReg ) ; auto I = BuildMI ( * MBB , MI , DL , TII . get ( MovOpc ) , ExecReg ) . addImm ( VGPRLanes ) ; if ( ! TmpVGPRLive ) I . addReg ( TmpVGPR , RegState :: ImplicitDefine ) ; TRI . buildVGPRSpillLoadStore ( * this , TmpVGPRIndex , <NUM_LIT> , false ) ; } else { if ( TmpVGPRLive ) TRI . buildVGPRSpillLoadStore ( * this , TmpVGPRIndex , <NUM_LIT> , false , false ) ; auto I = BuildMI ( * MBB , MI , DL , TII . get ( NotOpc ) , ExecReg ) . addReg ( ExecReg ) ; if ( ! TmpVGPRLive ) I . addReg ( TmpVGPR ,


<s> } else { TRI . buildVGPRSpillLoadStore ( * this , Index , Offset , IsLoad , false ) ; auto Not0 = BuildMI ( * MBB , MI , DL , TII . get ( NotOpc ) , ExecReg ) . addReg ( ExecReg ) ; Not0 -> getOperand ( <NUM_LIT> ) .


<s> TRI . buildVGPRSpillLoadStore ( * this , TmpVGPRIndex , <NUM_LIT> , true , false ) ; auto I = BuildMI ( * MBB , MI , DL , TII . get ( MovOpc ) , ExecReg ) . addReg ( SavedExecReg , RegState :: Kill ) ; if ( ! TmpVGPRLive ) { I . addReg ( TmpVGPR , RegState :: ImplicitKill ) ; } } else { TRI . buildVGPRSpillLoadStore ( * this , TmpVGPRIndex , <NUM_LIT> , true


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumVGPRs ; i < TotalNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; if ( ScratchWaveOffsetReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchWaveOffsetReg ) ; } unsigned ScratchRSrcReg = MFI -> getScratchRSrcReg ( ) ; if ( ScratchRSrcReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchRSrcReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , ScratchWaveOffsetReg ) ) ; } unsigned StackPtrReg = MFI -> getStackPtrOffsetReg ( ) ; if ( StackPtrReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , StackPtrReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , StackPtrReg ) ) ; } unsigned FrameReg = MFI -> getFrameOffsetReg


<s> bool OnlyExecLo = isWave32 || NumSubRegs == <NUM_LIT> || SuperReg == <STR_LIT> ; unsigned ExecMovOpc = OnlyExecLo ? <STR_LIT> : <STR_LIT> ; Register ExecReg = OnlyExecLo ? <STR_LIT> : <STR_LIT> ; Register SavedExecReg ; if ( OnlyExecLo ) { SavedExecReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ FirstPart + ExecLane ] ) ; } else { for ( ; ExecLane < ( NumSubRegs - <NUM_LIT> ) ; ++ ExecLane ) { SavedExecReg = getMatchingSuperReg ( getSubReg ( SuperReg , SplitParts [ FirstPart + ExecLane ] ) , <STR_LIT> , & <STR_LIT> ) ; if ( SavedExecReg ) break ; } } assert ( SavedExecReg ) ; BuildMI ( * MBB , MI , DL , TII -> get ( ExecMovOpc ) , SavedExecReg ) . addReg ( ExecReg ) ; BuildMI ( * MBB , MI , DL , TII -> get ( ExecMovOpc ) , ExecReg ) . addImm ( VGPRLanes ) ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; assert ( FrameInfo . getStackID ( Index ) != TargetStackID :: SGPRSpill ) ; Register FrameReg = FrameInfo . isFixedObjectIndex ( Index ) && hasBasePointer ( * MF ) ? getBaseRegister ( ) : getFrameRegister ( * MF ) ; Align Alignment = FrameInfo . getObjectAlign ( Index ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , Index ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , IsLoad ? MachineMemOperand :: MOLoad : MachineMemOperand :: MOStore , EltSize , Alignment ) ; if ( IsLoad ) { unsigned Opc = ST . enableFlatScratch ( ) ? <STR_LIT> : <STR_LIT> ; buildSpillLoadStore ( MI , Opc , Index , VGPR , false , MFI -> getScratchRSrcReg ( ) , FrameReg , Offset * EltSize , MMO , RS ) ; } else { unsigned Opc = ST . enableFlatScratch ( ) ? <STR_LIT> : <STR_LIT> ; buildSpillLoadStore ( MI , Opc , Index , VGPR , IsKill , MFI -> getScratchRSrcReg ( ) , FrameReg , Offset * EltSize , MMO , RS ) ; MFI -> addToSpilledVGPRs (


<s> if ( i + <NUM_LIT> == e ) { SOffsetRegState |= getKillRegState ( Scavenged ) ; SrcDstRegState |= getKillRegState ( IsKill ) ; } const bool NeedSuperRegDef = NumSubRegs > <NUM_LIT> && IsStore && i == <NUM_LIT> ; auto MIB = spillVGPRtoAGPR ( ST , MI , Index , i , SubReg , IsKill ) ; if ( ! MIB . getInstr ( ) ) { unsigned FinalReg = SubReg ; const bool IsAGPR = hasAGPRs ( RC ) ; if ( IsAGPR ) { if ( ! TmpReg ) { assert ( RS && <STR_LIT> ) ; TmpReg = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpReg ) ; } if ( IsStore ) { auto AccRead = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SubReg , getKillRegState ( IsKill ) ) ; if ( NeedSuperRegDef ) AccRead . addReg ( ValueReg , RegState :: ImplicitDefine ) ; } SubReg = TmpReg ; } MachinePointerInfo PInfo = BasePtrInfo . getWithOffset ( EltSize * i ) ; MachineMemOperand * NewMMO = MF -> getMachineMemOperand ( PInfo , MMO -> getFlags ( ) , EltSize , commonAlignment ( Alignment , EltSize * i ) ) ; MIB = BuildMI ( * MBB , MI , DL , * Desc ) . addReg ( SubReg , getDefRegState ( ! IsStore ) | getKillRegState ( IsKill ) ) ; if ( ! IsFlat ) MIB . addReg ( ScratchRsrcReg ) ; if ( SOffset == <STR_LIT> ) { if ( ! IsFlat ) MIB . addImm ( <NUM_LIT> ) ; } else { MIB . addReg ( SOffset , SOffsetRegState ) ; } MIB . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; if ( ! IsFlat ) MIB . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; MIB . addMemOperand ( NewMMO ) ; if ( ! IsAGPR && NeedSuperRegDef ) MIB . addReg ( ValueReg , RegState :: ImplicitDefine ) ; if ( ! IsStore && TmpReg != <STR_LIT> ) MIB = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , FinalReg ) . addReg ( TmpReg , RegState :: Kill ) ; }


<s> return SubReg ? divideCeil ( getSubRegIdxOffset ( SubReg ) , <NUM_LIT>


<s> } else { Register TmpVGPR = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpVGPR ) ; unsigned PerVGPR = <NUM_LIT> ; unsigned NumVGPRs = ( NumSubRegs + ( PerVGPR - <NUM_LIT> ) ) / PerVGPR ; int64_t VGPRLanes = ( <NUM_LIT> << std :: min ( PerVGPR , NumSubRegs ) ) - <NUM_LIT> ; for ( unsigned Offset = <NUM_LIT> ; Offset < NumVGPRs ; ++ Offset ) { buildSGPRSpillLoadStore ( MI , Index , Offset , EltSize , TmpVGPR , VGPRLanes , RS , true ) ; for ( unsigned i = Offset * PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PerVGPR , NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; bool LastSubReg = ( i + <NUM_LIT> == e ) ; auto MIB = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , SubReg ) . addReg ( TmpVGPR , getKillRegState ( LastSubReg ) ) . addImm ( i ) ; if ( NumSubRegs > <NUM_LIT> && i == <NUM_LIT> ) MIB . addReg ( SuperReg , RegState :: ImplicitDefine ) ; } } } MI -> eraseFromParent (


<s> RegPressureIgnoredUnits . resize ( getNumRegUnits ( ) ) ; RegPressureIgnoredUnits . set ( * MCRegUnitIterator ( <STR_LIT> , this ) ) ; for ( auto Reg : <STR_LIT> ) RegPressureIgnoredUnits . set ( * MCRegUnitIterator ( Reg , this ) ) ; static llvm :: once_flag InitializeRegSplitPartsFlag ; static auto InitializeRegSplitPartsOnce = [ this ] ( ) { for ( unsigned Idx = <NUM_LIT> , E = getNumSubRegIndices ( ) - <NUM_LIT> ; Idx < E ; ++ Idx ) { unsigned Size = getSubRegIdxSize ( Idx ) ; if ( Size & <NUM_LIT> ) continue ; std :: vector < int16_t > & Vec = RegSplitParts [ Size / <NUM_LIT> - <NUM_LIT> ] ; unsigned Pos = getSubRegIdxOffset ( Idx ) ; if ( Pos % Size ) continue ; Pos /= Size ; if ( Vec . empty ( ) ) { unsigned MaxNumParts = <NUM_LIT> / Size ; Vec . resize ( MaxNumParts ) ; } Vec [ Pos ] = Idx ; } } ; static llvm :: once_flag InitializeSubRegFromChannelTableFlag


<s> <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; bool UseKill = IsKill && i == NumSubRegs - <NUM_LIT> ; bool VGPRDefined = true ; if ( MBB == & MF -> front ( ) ) VGPRDefined = ! SGPRSpillVGPRDefinedSet . insert ( Spill . VGPR ) . second ; auto MIB = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , Spill . VGPR ) . addReg ( SubReg , getKillRegState ( UseKill ) ) . addImm ( Spill . Lane ) . addReg ( Spill . VGPR , VGPRDefined ? <NUM_LIT> : RegState :: Undef ) ; if ( i == <NUM_LIT> && NumSubRegs > <NUM_LIT> ) { MIB . addReg ( SuperReg , RegState :: ImplicitDefine ) ; } if ( NumSubRegs > <NUM_LIT> ) MIB . addReg ( SuperReg , getKillRegState ( UseKill ) | RegState :: Implicit ) ; } } else { Register TmpVGPR = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpVGPR ) ; unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ; unsigned PerVGPR = <NUM_LIT> ; unsigned NumVGPRs = ( NumSubRegs + ( PerVGPR - <NUM_LIT> ) ) / PerVGPR ; int64_t VGPRLanes = ( <NUM_LIT> << std :: min ( PerVGPR , NumSubRegs ) ) - <NUM_LIT> ; for ( unsigned Offset = <NUM_LIT> ; Offset < NumVGPRs ; ++ Offset ) { unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PerVGPR , NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; MachineInstrBuilder WriteLane = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PerVGPR ) . addReg ( TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ; if ( NumSubRegs > <NUM_LIT> ) { unsigned SuperKillState = <NUM_LIT> ; if ( i + <NUM_LIT> == NumSubRegs ) SuperKillState |=


<s> bool SpillToVGPR = ! VGPRSpills . empty ( ) ; if ( OnlyToVGPR && ! SpillToVGPR ) return false ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const DebugLoc & DL = MI -> getDebugLoc ( ) ; Register SuperReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ; unsigned M0CopyReg = <STR_LIT> ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; Register TmpVGPR ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; if ( SpillToVGPR ) { <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; auto MIB = BuildMI ( * MBB , MI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , SubReg ) . addReg ( Spill . VGPR ) . addImm ( Spill . Lane ) ; if ( NumSubRegs > <NUM_LIT> && i == <NUM_LIT> ) MIB .

addReg ( SuperReg , RegState :: ImplicitDefine ) ;


<s> DenseSet < unsigned > SGPRSpillVGPRDefinedSet ; ArrayRef < <ISA_LIT>MachineFunctionInfo :: SpilledReg > VGPRSpills = MFI -> getSGPRToVGPRSpills ( Index ) ; bool SpillToVGPR = ! VGPRSpills . empty ( ) ; if ( OnlyToVGPR && ! SpillToVGPR ) return false ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; Register SuperReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool IsKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; const DebugLoc & DL = MI -> getDebugLoc ( ) ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; assert ( SpillToVGPR || ( SuperReg != MFI -> getStackPtrOffsetReg ( ) && SuperReg != MFI -> getFrameOffsetReg ( ) && SuperReg != MFI -> getScratchWaveOffsetReg ( ) ) ) ; assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ; unsigned M0CopyReg = <STR_LIT> ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; Register TmpVGPR ; unsigned SubKillState = getKillRegState ( ( NumSubRegs


<s> unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; MachineInstrBuilder WriteLane = BuildMI ( SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SB . TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PVD . PerVGPR ) . addReg ( SB . TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ; if ( LIS ) { if ( i == <NUM_LIT> ) LIS -> ReplaceMachineInstrInMaps ( * MI , * WriteLane ) ; else LIS -> InsertMachineInstrInMaps ( * WriteLane ) ; } if ( SB . NumSubRegs > <NUM_LIT> ) { unsigned SuperKillState = <NUM_LIT> ; if ( i + <NUM_LIT> == SB . NumSubRegs ) SuperKillState |= getKillRegState ( SB . IsKill ) ; WriteLane . addReg ( SB . SuperReg , RegState :: Implicit | SuperKillState ) ; } } SB . readWriteTmpVGPR ( Offset , false ) ; } SB . restore ( ) ; } MI -> eraseFromParent ( ) ; SB . MFI . addToSpilledSGPRs


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumVGPRs ; i < TotalNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; if ( ScratchWaveOffsetReg != <STR_LIT>


<s> switch ( Size ) { case <NUM_LIT> : AltName = <STR_LIT> ; break ; case <NUM_LIT> : AltName = <STR_LIT> ; break ; case <NUM_LIT> : AltName = <STR_LIT> ; break ; case <NUM_LIT> : AltName = <STR_LIT> ; break ; case <NUM_LIT> : AltName


<s> bool Scavenged = false ; unsigned SOffset = ScratchOffset ; unsigned OriginalImmOffset = Offset ; unsigned NumSubRegs = getNumSubRegsForSpillOp ( MI -> getOpcode ( ) ) ; unsigned Size = NumSubRegs * <NUM_LIT> ; if ( ! isUInt < <NUM_LIT> > ( Offset + Size ) ) { SOffset = <STR_LIT> ; if ( RS ) SOffset = RS -> FindUnusedReg ( & <STR_LIT> ) ; if ( SOffset == <STR_LIT> ) { RanOutOfSGPRs = true ; SOffset = ScratchOffset ; } else { Scavenged = true ; } BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , SOffset ) . addReg ( ScratchOffset ) . addImm ( Offset ) ; Offset = <NUM_LIT> ; } for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i != e ; ++ i , Offset += <NUM_LIT> ) { unsigned SubReg = NumSubRegs > <NUM_LIT> ? getPhysRegSubReg ( Value , & <STR_LIT> , i ) : Value ; unsigned SOffsetRegState = <NUM_LIT> ; unsigned SrcDstRegState = getDefRegState ( ! IsStore ) ; if ( i + <NUM_LIT> == e ) { SOffsetRegState |= getKillRegState ( Scavenged ) ; SrcDstRegState |= getKillRegState ( IsKill ) ; } BuildMI ( * MBB , MI , DL , TII -> get ( LoadStoreOp ) ) . addReg ( SubReg , getDefRegState ( ! IsStore ) ) . addReg ( ScratchRsrcReg ) . addReg ( SOffset , SOffsetRegState ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm (

<NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( Value , RegState :: Implicit | SrcDstRegState ) . setMemRefs ( MI -> memoperands_begin ( ) , MI -> memoperands_end ( ) ) ;


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; const MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ; const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF -> getSubtarget ( ) . getInstrInfo ( ) ) ; LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; bool IsLoad = TII -> get ( LoadStoreOp ) . mayLoad ( ) ; bool RanOutOfSGPRs = false ; unsigned SOffset = ScratchOffset ; unsigned NumSubRegs = getNumSubRegsForSpillOp ( MI -> getOpcode ( ) ) ; unsigned Size = NumSubRegs * <NUM_LIT> ; if ( ! isUInt < <NUM_LIT> > ( Offset + Size ) ) { SOffset = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; if ( SOffset


<s> if ( isSGPRClass ( * I ) ) { Limit = SGPRLimit / NumSubRegs ; } else { Limit = VGPRLimit / NumSubRegs ; } const int * Sets = getRegClassPressureSets ( * I ) ; assert ( Sets ) ; for ( unsigned i = <NUM_LIT> ; Sets [ i ] != - <NUM_LIT> ; ++ i ) { if ( Sets [ i ] == ( int ) Idx ) return Limit ; } } return <NUM_LIT>


<s> for ( regclass_iterator I = regclass_begin ( ) , E = regclass_end ( ) ; I != E ; ++ I ) { unsigned NumSubRegs = std :: max ( ( int ) ( * I ) -> getSize ( ) / <NUM_LIT> , <NUM_LIT> ) ; unsigned Limit ; if ( isSGPRClass ( * I ) ) { Limit = SGPRLimit / NumSubRegs ; } else { Limit = VGPRLimit / NumSubRegs ; } const int * Sets = getRegClassPressureSets ( * I ) ; assert ( Sets ) ; for ( unsigned i = <NUM_LIT> ; Sets [ i ] != - <NUM_LIT> ; ++ i ) { if ( Sets [ i ] == (


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; if ( MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . hasSGPRInitBug ( ) ) { unsigned NumSGPRs = <STR_LIT> .


<s> if ( TII -> hasUnwantedEffectsWhenEXECEmpty ( * I ) ) return true ; if ( TII -> isSMRD ( * I ) || TII -> isVMEM ( * I ) || TII -> isFLAT ( * I ) || I -> getOpcode ( ) == <STR_LIT> ) return true ; ++ NumInstr ; if ( NumInstr >= SkipThreshold ) return true ; } } return


<s> if ( TII -> analyzeBranch ( SrcMBB , TrueMBB , FalseMBB , Cond ) ) return false ; if ( ! FalseMBB ) FalseMBB = SrcMBB . getNextNode ( ) ; return true


<s> if ( ( SrcMBB . getNumber ( ) >= TrueMBB -> getNumber ( ) ) || mustRetainExeczBranch ( * FalseMBB , * TrueMBB ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MI . eraseFromParent (


<s> MachineBasicBlock * TrueMBB = nullptr ; MachineBasicBlock * FalseMBB = nullptr ; SmallVector < MachineOperand , <NUM_LIT> > Cond ; if ( ! getBlockDestinations ( SrcMBB , TrueMBB , FalseMBB , Cond ) ) return false ; if ( ( SrcMBB . getNumber ( ) >= TrueMBB -> getNumber ( ) ) || mustRetainExeczBranch ( * FalseMBB , * TrueMBB ) ) return false ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MI ) ; MI . eraseFromParent ( ) ; SrcMBB . removeSuccessor ( TrueMBB


<s> static MachineOperand copyRegOperandAsImplicit ( const MachineOperand & Orig ) { assert ( ! Orig . isImplicit ( ) ) ; return MachineOperand :: CreateReg ( Orig . getReg ( ) , Orig . isDef ( ) , true , Orig . isKill ( ) , Orig . isDead ( ) , Orig . isUndef ( ) , Orig . isEarlyClobber ( ) , Orig . getSubReg ( ) , Orig . isDebug


<s> assert ( ! Orig . isImplicit ( ) ) ; return MachineOperand :: CreateReg ( Orig . getReg ( ) , Orig . isDef ( ) , true , Orig . isKill ( ) , Orig . isDead ( ) , Orig

. isUndef ( ) , Orig . isEarlyClobber ( ) , Orig . getSubReg ( ) , Orig . isDebug ( ) , Orig . isInternalRead ( ) ) ;


<s> return new <ISA_LIT>ShrinkInstructions


<s> if ( Src0 . isImm ( ) && TII -> isLiteralConstant ( Src0 , TII -> getOpSize ( MI , Src0Idx ) ) ) return ; if ( Src0 . isReg ( ) && ! isVGPR ( & Src0 , TRI , MRI ) ) return ; if ( Src0 . isReg ( ) && MRI . hasOneUse ( Src0 . getReg ( ) ) ) { unsigned Reg = Src0 . getReg ( ) ; MachineInstr * Def = MRI . getUniqueVRegDef ( Reg ) ; if ( Def && Def -> isMoveImmediate ( )


<s> const char * getPassName ( ) const override


<s> static bool isVGPR ( const MachineOperand * MO , const <ISA_LIT>RegisterInfo & TRI , const MachineRegisterInfo & MRI ) { if ( ! MO -> isReg ( ) ) return false ; if ( TargetRegisterInfo :: isVirtualRegister ( MO -> getReg ( ) ) ) return TRI . hasVGPRs ( MRI . getRegClass ( MO -> getReg ( ) ) ) ; return TRI . hasVGPRs ( TRI . getPhysRegClass ( MO


<s> Next = std :: next ( I ) ; MachineInstr & MI = * I ; if ( MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isImm ( ) ) { if ( isInt < <NUM_LIT> > ( Src . getImm ( ) ) && ! TII -> isInlineConstant ( Src , <NUM_LIT> ) ) MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; } continue ; } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; if ( ! canShrink ( MI , TII , TRI , MRI ) ) { if ( ! MI . isCommutable ( ) || ! TII -> commuteInstruction ( & MI ) || ! canShrink ( MI , TII , TRI , MRI ) ) continue ; } if ( ! TII -> hasVALU32BitEncoding ( MI . getOpcode ( ) ) ) continue ; int Op32 = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( TII -> isVOPC ( Op32 ) ) { unsigned DstReg = MI . getOperand ( <NUM_LIT> ) . getReg ( ) ; if ( TargetRegisterInfo :: isVirtualRegister ( DstReg ) ) { MRI . setRegAllocationHint ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) , <NUM_LIT> , <STR_LIT> ) ; continue ; } if ( DstReg != <STR_LIT> ) continue ; } if ( Op32 == <STR_LIT> ) { const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( ! Src2 -> isReg


<s> int Src0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: src0 ) ; MachineOperand & Src0 = MI . getOperand ( Src0Idx ) ; if ( Src0 . isReg ( ) ) { Register Reg = Src0 . getReg ( ) ; if ( Reg . isVirtual ( ) && MRI . hasOneUse ( Reg ) ) { MachineInstr * Def = MRI . getUniqueVRegDef ( Reg ) ; if ( Def && Def -> isMoveImmediate ( ) ) { MachineOperand & MovSrc = Def -> getOperand ( <NUM_LIT> ) ; bool ConstantFolded = false ; if ( MovSrc . isImm ( ) && ( isInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) || isUInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) ) ) { Src0 . ChangeToImmediate ( MovSrc . getImm ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isFI ( ) ) { Src0 . ChangeToFrameIndex ( MovSrc . getIndex ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isGlobal ( ) ) { Src0 . ChangeToGA ( MovSrc . getGlobal ( ) , MovSrc . getOffset ( ) , MovSrc . getTargetFlags ( ) ) ; ConstantFolded = true ; } if ( ConstantFolded ) { assert ( MRI . use_empty ( Reg ) ) ; Def -> eraseFromParent ( ) ; ++ NumLiteralConstantsFolded ; return true ; } } } } if ( TryToCommute && MI . isCommutable (


<s> if ( Reg . isVirtual ( ) && MRI . hasOneUse ( Reg ) ) { MachineInstr * Def = MRI . getUniqueVRegDef ( Reg ) ; if ( Def && Def -> isMoveImmediate ( ) ) { MachineOperand & MovSrc = Def -> getOperand ( <NUM_LIT> ) ; bool ConstantFolded = false ; if ( MovSrc . isImm ( ) && ( isInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) || isUInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) ) ) { Src0 . ChangeToImmediate ( MovSrc . getImm ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isFI ( ) ) { Src0 . ChangeToFrameIndex ( MovSrc . getIndex ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isGlobal ( ) ) { Src0 . ChangeToGA ( MovSrc . getGlobal ( ) , MovSrc . getOffset ( ) , MovSrc . getTargetFlags ( ) ) ; ConstantFolded = true ; } if ( ConstantFolded ) { assert ( MRI . use_empty ( Reg ) ) ; Def -> eraseFromParent ( ) ; ++ NumLiteralConstantsFolded ; return true ; } } } } if ( TryToCommute && MI . isCommutable ( ) ) { if ( TII -> commuteInstruction ( MI ) ) { if ( foldImmediates ( MI , TII , MRI , false ) ) return true ; TII -> commuteInstruction ( MI ) ; } } return


<s> } } } if ( ST . hasSwap ( ) && ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) ) { if ( auto * NextMI = matchSwap ( MI , MRI , TII ) ) { Next = NextMI -> getIterator ( ) ; continue ; } } if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand * Dest = & MI . getOperand ( <NUM_LIT> ) ; MachineOperand * Src0 = & MI . getOperand ( <NUM_LIT> ) ; MachineOperand * Src1 = & MI . getOperand ( <NUM_LIT> ) ; if ( ! Src0 -> isReg ( ) && Src1 -> isReg ( ) ) { if ( TII -> commuteInstruction ( MI , false , <NUM_LIT> , <NUM_LIT> ) ) std :: swap ( Src0 , Src1 ) ; } if ( Dest -> getReg ( ) . isVirtual ( ) && Src0 -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , Src0 -> getReg ( ) ) ; MRI . setRegAllocationHint ( Src0 -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; continue ; } if ( Src0 -> isReg ( ) && Src0 -> getReg ( ) == Dest -> getReg ( ) ) { if ( Src1 -> isImm ( ) && isKImmOperand ( TII , * Src1 ) ) { unsigned Opc = ( MI . getOpcode ( ) == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; MI . setDesc ( TII -> get ( Opc ) ) ; MI . tieOperands ( <NUM_LIT> , <NUM_LIT> ) ; } } } if ( MI . isCompare ( ) && TII -> isSOPC ( MI ) ) { shrinkScalarCompare ( TII , MI ) ; continue ; } if ( MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isImm ( ) && Dst . getReg ( ) . isPhysical ( ) ) { int32_t ReverseImm ; if ( isKImmOperand ( TII , Src ) ) MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; else if ( isReverseInlineImm ( TII , Src , ReverseImm ) ) { MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; Src . setImm


<s> if ( i == <NUM_LIT> ) { VgprBase = Vgpr ; } else if ( VgprBase + i != Vgpr ) return ; if ( ! Op . isUndef ( ) ) IsUndef = false ; if ( ! Op . isKill ( ) ) IsKill = false ; } if ( VgprBase + NewAddrDwords > <NUM_LIT> ) return ; int TFEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: tfe ) ; int LWEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: lwe ) ; unsigned TFEVal = ( TFEIdx == - <NUM_LIT> ) ? <NUM_LIT> : MI . getOperand ( TFEIdx ) . getImm ( ) ; unsigned LWEVal = ( LWEIdx == - <NUM_LIT> ) ? <NUM_LIT> : MI . getOperand ( LWEIdx ) . getImm ( ) ; int ToUntie = - <NUM_LIT> ; if ( TFEVal || LWEVal ) { for ( unsigned i = LWEIdx + <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( MI . getOperand ( i ) . isReg ( ) && MI . getOperand ( i

) . isTied ( ) && MI . getOperand ( i ) . isImplicit ( ) ) {


<s> default : return false ; case <STR_LIT> : case <STR_LIT> : if ( TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) -> isImm ( ) ) return false ; return true ; case <STR_LIT> : case <STR_LIT> : if ( ! isVGPR ( Src2 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ; break ; case <STR_LIT> : break ; } } const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 && ( ! isVGPR ( Src1 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) ) ) return false ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) ) return false ; return ! TII -> hasModifiersSet ( MI , <STR_LIT> :: omod ) && ! TII -> hasModifiersSet ( MI


<s> if ( ! isVGPR ( Src2 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ; break ; case <STR_LIT> : break ; } } const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; if ( Src1 && ( ! isVGPR ( Src1 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src1_modifiers ) )


<s> const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( Src2 ) { switch ( MI . getOpcode ( ) ) { default : return false ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( ! isVGPR ( TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) , TRI , MRI ) ) return false ; return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( ! isVGPR ( Src2 , TRI , MRI ) || TII ->

hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ;


<s> static bool canShrink ( MachineInstr & MI , const <ISA_LIT>InstrInfo * TII , const <ISA_LIT>RegisterInfo & TRI , const MachineRegisterInfo & MRI ) { const MachineOperand * Src2 = TII -> getNamedOperand ( MI , <STR_LIT> :: src2 ) ; if ( Src2 ) { switch ( MI . getOpcode ( ) ) { default : return false ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( ! isVGPR ( TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) , TRI , MRI ) ) return false ; return true ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( ! isVGPR ( Src2 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ; break ; case <STR_LIT> : break ; } } const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> ::


<s> MachineOperand & MovSrc = Def -> getOperand ( <NUM_LIT> ) ; bool ConstantFolded = false ; if ( MovSrc . isImm ( ) && ( isInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) || isUInt < <NUM_LIT> > ( MovSrc . getImm ( ) ) ) ) { Src0 . setSubReg ( <NUM_LIT> ) ; Src0 . ChangeToImmediate ( MovSrc . getImm ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isFI ( ) ) { Src0 . setSubReg ( <NUM_LIT> ) ; Src0 . ChangeToFrameIndex ( MovSrc . getIndex ( ) ) ; ConstantFolded = true ; } if ( ConstantFolded ) { assert ( MRI . use_empty ( Reg ) ) ; Def -> eraseFromParent ( ) ; ++ NumLiteralConstantsFolded ; return true ; } } } } if ( TryToCommute && MI . isCommutable ( ) ) { if ( TII -> commuteInstruction (


<s> if ( TRI . getRegSizeInBits ( Reg , MRI ) != <NUM_LIT> ) { if ( TargetRegisterInfo :: isPhysicalRegister ( Reg ) ) { Reg = TRI . getSubReg ( Reg , TRI . getSubRegFromChannel ( I ) ) ; } else { LaneBitmask LM = TRI . getSubRegIndexLaneMask ( Sub ) ; Sub = TRI . getSubRegFromChannel ( I + countTrailingZeros ( LM .


<s> case <STR_LIT> : case <STR_LIT> : return true ; case <STR_LIT> : case <STR_LIT> : if ( ! isVGPR ( Src2 , TRI , MRI ) || TII -> hasModifiersSet ( MI , <STR_LIT> :: src2_modifiers ) ) return false ; break ; case <STR_LIT> : break ; } } const MachineOperand * Src1 = TII -> getNamedOperand ( MI , <STR_LIT> :: src1 ) ; const MachineOperand * Src1Mod = TII -> getNamedOperand ( MI , <STR_LIT> :: src1_modifiers ) ; if ( Src1 && ( ! isVGPR ( Src1 , TRI , MRI ) || ( Src1Mod && Src1Mod -> getImm ( ) != <NUM_LIT> ) ) ) return false ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> :: src0_modifiers ) ) return false ; if ( TII -> hasModifiersSet ( MI , <STR_LIT> ::


<s> MachineBasicBlock & MBB = * BI ; MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; if ( MI . getOpcode ( ) == <STR_LIT> ) { MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isImm ( ) && TargetRegisterInfo :: isPhysicalRegister ( MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { int32_t ReverseImm ; if ( isReverseInlineImm ( TII , Src , ReverseImm ) ) { MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; Src . setImm ( ReverseImm ) ; continue ; } } } if ( MI . getOpcode ( ) == <STR_LIT> && Next != MBB . end ( ) && ( * Next ) . getOpcode ( ) == <STR_LIT> ) { MachineInstr & NextMI = * Next ; uint8_t Nop0 = MI . getOperand ( <NUM_LIT> ) . getImm ( ) + <NUM_LIT> ; uint8_t Nop1 = NextMI . getOperand ( <NUM_LIT> ) . getImm ( ) + <NUM_LIT> ; if ( Nop0 + Nop1 <= <NUM_LIT> ) { NextMI . getOperand ( <NUM_LIT> ) . setImm ( Nop0 + Nop1 - <NUM_LIT> ) ; MI . eraseFromParent ( ) ; } continue ; } if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand * Dest = & MI . getOperand ( <NUM_LIT> ) ; MachineOperand * Src0 = & MI . getOperand ( <NUM_LIT> ) ; MachineOperand * Src1 = & MI . getOperand ( <NUM_LIT> ) ; if ( ! Src0 -> isReg ( ) && Src1 -> isReg ( ) ) { if ( TII -> commuteInstruction ( MI , false , <NUM_LIT> , <NUM_LIT> ) ) std :: swap ( Src0 , Src1 ) ; } if ( TargetRegisterInfo :: isVirtualRegister ( Dest -> getReg ( ) ) && Src0 -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , Src0 -> getReg ( ) ) ; MRI . setRegAllocationHint ( Src0 -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; continue ; } if ( Src0 -> isReg ( ) && Src0 -> getReg ( ) == Dest -> getReg ( ) ) { if ( Src1 -> isImm ( ) && isKImmOperand ( TII , * Src1 ) ) { unsigned Opc = ( MI . getOpcode ( ) == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; MI . setDesc ( TII -> get ( Opc ) ) ; MI . tieOperands ( <NUM_LIT> , <NUM_LIT> ) ; } } } if ( MI . isCompare ( ) && TII -> isSOPC ( MI


<s> static void dropInstructionKeepingImpDefs ( MachineInstr & MI , const <ISA_LIT>InstrInfo * TII ) { for ( unsigned i = MI . getDesc ( ) . getNumOperands ( ) + MI . getDesc ( ) . getNumImplicitUses ( ) + MI . getDesc ( ) . getNumImplicitDefs ( )


<s> StringRef getPassName ( ) const override { return <STR_LIT>


<s> if ( Reg . isPhysical ( ) && MO . getReg ( ) . isPhysical ( ) ) { if ( TRI . regsOverlap ( Reg , MO . getReg ( ) ) ) return true ; } else if ( MO . getReg ( ) == Reg && Reg . isVirtual ( ) ) { LaneBitmask Overlap = TRI . getSubRegIndexLaneMask ( SubReg ) & TRI . getSubRegIndexLaneMask ( MO . getSubReg ( ) ) ; if ( Overlap . any ( ) ) return true ; } } return


<s> static bool instModifiesReg ( const MachineInstr * MI , unsigned Reg , unsigned SubReg , const <ISA_LIT>RegisterInfo & TRI ) { return instAccessReg ( MI -> defs ( ) , Reg ,


<s> return isInt < <NUM_LIT> > ( Src . getImm ( ) ) && ! TII -> isInlineConstant ( * Src . getParent ( ) , Src

. getParent ( ) -> getOperandNo ( & Src ) ) ;


<s> if ( ! isInt < <NUM_LIT> > ( Src . getImm ( ) ) || TII -> isInlineConstant ( Src ) ) return false ; ReverseImm = reverseBits < int32_t > ( static_cast < int32_t > ( Src . getImm ( ) ) ) ; return ReverseImm >= - <NUM_LIT> &&


<s> if ( ! MovX && instModifiesReg ( & * I , X , Xsub , TRI ) ) { MovX = nullptr ; break ; } continue ; } if ( MovX || ( I -> getOpcode ( ) != <STR_LIT> && I -> getOpcode ( ) != <STR_LIT> ) || I -> getOperand ( <NUM_LIT> ) . getReg ( ) != X || I -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != Xsub ) { MovX = nullptr ; break ; } if ( I -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) continue ; if ( Size > <NUM_LIT> && ( I -> getNumImplicitOperands ( ) > ( I -> isCopy ( ) ? <NUM_LIT> : <NUM_LIT> ) ) ) continue ; MovX = & * I ; } if ( ! MovX ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MovT << * MovX << * MovY ) ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) { TargetInstrInfo :: RegSubRegPair X1 , Y1 ; X1 = getSubRegForIndex ( X , Xsub , I , TRI , MRI ) ; Y1 = getSubRegForIndex ( Y , Ysub , I , TRI , MRI ) ; MachineBasicBlock & MBB = * MovT . getParent ( ) ; auto MIB = BuildMI ( MBB , MovX -> getIterator ( ) , MovT . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addDef ( X1 . Reg , <NUM_LIT> , X1 . SubReg ) . addDef ( Y1 . Reg , <NUM_LIT> , Y1 . SubReg ) . addReg ( Y1 . Reg , <NUM_LIT> , Y1 . SubReg ) . addReg ( X1 . Reg , <NUM_LIT> , X1 . SubReg ) . getInstr ( ) ; if ( MovX -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) { MIB -> RemoveOperand ( MIB -> getNumExplicitOperands ( ) ) ; MIB -> copyImplicitOps ( * MBB . getParent ( ) , * MovX ) ; } } MovX -> eraseFromParent ( ) ; dropInstructionKeepingImpDefs ( * MovY , TII ) ; MachineInstr * Next = & * std :: next ( MovT . getIterator ( ) ) ; if ( T . isVirtual ( ) && MRI . use_nodbg_empty ( T ) ) { dropInstructionKeepingImpDefs ( MovT , TII ) ; } else { Xop . setIsKill ( false ) ; for ( int I = MovT . getNumImplicitOperands ( ) - <NUM_LIT> ; I >= <NUM_LIT> ; -- I ) { unsigned OpNo = MovT . getNumExplicitOperands ( ) + I ; const MachineOperand & Op = MovT . getOperand (


<s> if ( ! Src1 . isImm ( ) ) return ; int SOPKOpc = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( SOPKOpc == - <NUM_LIT> ) return ; if ( SOPKOpc == <STR_LIT> || SOPKOpc == <STR_LIT> ) { bool HasUImm ; if ( isKImmOrKUImmOperand ( TII , Src1 , HasUImm ) ) { if ( ! HasUImm ) { SOPKOpc = ( SOPKOpc == <STR_LIT> ) ?


<s> assert ( MovT . getOpcode ( ) == <STR_LIT> || MovT . getOpcode ( ) == <STR_LIT> ) ; Register T = MovT . getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Tsub = MovT . getOperand ( <NUM_LIT> ) . getSubReg ( ) ; MachineOperand & Xop = MovT . getOperand ( <NUM_LIT> ) ; if ( ! Xop . isReg ( ) ) return nullptr ; Register X = Xop . getReg ( ) ; unsigned Xsub = Xop . getSubReg ( ) ; unsigned Size = TII -> getOpSize ( MovT , <NUM_LIT> ) / <NUM_LIT> ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; if ( ! TRI . isVGPR ( MRI , X ) ) return nullptr ; if ( MovT . hasRegisterImplicitUseOperand ( <STR_LIT> ) ) return nullptr ; const unsigned SearchLimit = <NUM_LIT> ; unsigned Count = <NUM_LIT> ; bool KilledT = false ; for ( auto Iter = std :: next ( MovT . getIterator ( ) ) , E = MovT . getParent ( ) -> instr_end ( ) ; Iter != E && Count < SearchLimit && ! KilledT ; ++ Iter , ++ Count ) { MachineInstr * MovY = & * Iter ; KilledT = MovY -> killsRegister ( T , & TRI ) ; if ( ( MovY -> getOpcode ( ) != <STR_LIT> && MovY -> getOpcode ( ) != <STR_LIT> ) || ! MovY -> getOperand ( <NUM_LIT> ) . isReg ( ) || MovY -> getOperand ( <NUM_LIT> ) . getReg ( ) != T || MovY -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != Tsub || MovY -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) continue ; Register Y = MovY -> getOperand ( <NUM_LIT> ) . getReg ( ) ; unsigned Ysub = MovY -> getOperand ( <NUM_LIT> ) . getSubReg ( ) ; if ( ! TRI . isVGPR ( MRI , Y ) ) continue ; MachineInstr * MovX = nullptr ; for ( auto IY = MovY -> getIterator ( ) , I = std :: next ( MovT


<s> if ( MI . getOpcode ( ) == <STR_LIT> ) { MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isImm ( ) && MI . getOperand ( <NUM_LIT> ) . getReg ( ) . isPhysical ( ) ) { int32_t ReverseImm ; if ( isReverseInlineImm ( TII , Src , ReverseImm ) ) { MI . setDesc ( TII -> get ( <STR_LIT> ) ) ; Src . setImm ( ReverseImm ) ; continue ; } } } if ( ST . hasSwap ( ) && ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) ) { if ( auto * NextMI = matchSwap ( MI , MRI , TII ) ) { Next = NextMI -> getIterator ( ) ; continue ; } } if ( MI . getOpcode ( ) == <STR_LIT> || MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand * Dest = & MI . getOperand ( <NUM_LIT> ) ; MachineOperand * Src0 = & MI . getOperand ( <NUM_LIT> ) ; MachineOperand * Src1 = & MI . getOperand ( <NUM_LIT> ) ; if ( ! Src0 -> isReg ( ) && Src1 -> isReg ( ) ) { if ( TII -> commuteInstruction ( MI , false , <NUM_LIT> , <NUM_LIT> ) ) std :: swap ( Src0 , Src1 ) ; } if ( Dest -> getReg ( ) . isVirtual ( ) && Src0 -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , Src0 -> getReg ( ) ) ; MRI . setRegAllocationHint ( Src0 -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; continue ; } if ( Src0 -> isReg ( ) && Src0 -> getReg ( ) == Dest -> getReg ( ) ) { if ( Src1 -> isImm ( ) && isKImmOperand ( TII , * Src1 ) ) { unsigned Opc = ( MI . getOpcode ( ) == <STR_LIT> ) ? <STR_LIT> : <STR_LIT> ; MI . setDesc ( TII -> get ( Opc ) ) ; MI . tieOperands ( <NUM_LIT> , <NUM_LIT> ) ; } } } if ( MI . isCompare ( ) && TII -> isSOPC ( MI ) ) { shrinkScalarCompare ( TII , MI ) ; continue ; } if ( MI . getOpcode ( ) == <STR_LIT> ) { const MachineOperand & Dst = MI . getOperand ( <NUM_LIT> ) ; MachineOperand & Src = MI . getOperand ( <NUM_LIT> ) ; if ( Src . isImm ( ) && Dst . getReg ( ) . isPhysical ( ) ) { int32_t ReverseImm ; if ( isKImmOperand ( TII , Src ) ) MI


<s> void <ISA_LIT>ShrinkInstructions :: shrinkMIMG ( MachineInstr & MI ) { const <STR_LIT> * Info = <STR_LIT> ( MI . getOpcode ( ) ) ; if ( ! Info || Info -> MIMGEncoding != <STR_LIT> ) return ; MachineFunction * MF = MI . getParent ( ) -> getParent ( ) ; const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; int VAddr0Idx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr0 ) ; unsigned NewAddrDwords = Info -> VAddrDwords ; const TargetRegisterClass * RC ; if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else { RC = & <STR_LIT> ; NewAddrDwords = <NUM_LIT> ; } unsigned VgprBase = <NUM_LIT> ; bool IsUndef = true ; bool IsKill = NewAddrDwords == Info -> VAddrDwords ; for ( unsigned i = <NUM_LIT> ; i < Info -> VAddrDwords ; ++ i ) { const MachineOperand & Op = MI . getOperand ( VAddr0Idx + i ) ; unsigned Vgpr = TRI . getHWRegIndex ( Op . getReg ( ) ) ; if ( i == <NUM_LIT> ) { VgprBase = Vgpr ; } else if ( VgprBase + i != Vgpr ) return ; if ( ! Op . isUndef ( ) ) IsUndef = false ; if ( ! Op . isKill ( ) ) IsKill = false ; } if ( VgprBase + NewAddrDwords > <NUM_LIT> ) return ; int TFEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: tfe ) ; int LWEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: lwe ) ; unsigned TFEVal = ( TFEIdx == - <NUM_LIT> ) ? <NUM_LIT> : MI . getOperand ( TFEIdx ) . getImm ( ) ; unsigned LWEVal = ( LWEIdx == - <NUM_LIT> ) ? <NUM_LIT> : MI . getOperand ( LWEIdx ) .


<s> Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( isPowerOf2_32 ( Imm ) ) { NewImm = countTrailingZeros ( Imm ) ; Opc = <STR_LIT> ; } else if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else if ( Opc == <STR_LIT> ) { if ( <STR_LIT> ( ~ Imm , ST . hasInv2PiInlineImm ( ) ) ) { NewImm = ~ Imm ; Opc = <STR_LIT> ; } } else { llvm_unreachable ( <STR_LIT> ) ; } if ( NewImm != <NUM_LIT> ) { if ( Dest -> getReg ( ) . isVirtual ( ) && SrcReg -> isReg ( ) ) { MRI . setRegAllocationHint ( Dest -> getReg ( ) , <NUM_LIT> , SrcReg -> getReg ( ) ) ; MRI . setRegAllocationHint ( SrcReg -> getReg ( ) , <NUM_LIT> , Dest -> getReg ( ) ) ; return true ; } if ( SrcReg -> isReg ( ) && SrcReg -> getReg ( ) == Dest -> getReg ( ) ) { const bool IsUndef = SrcReg -> isUndef ( ) ; const bool IsKill = SrcReg -> isKill ( ) ; MI . setDesc ( TII -> get ( Opc


<s> MachineFunction & MF = * MI . getMF ( ) ; for ( unsigned i = MI . getDesc ( ) . getNumOperands ( ) + MI . getDesc ( ) . getNumImplicitUses ( ) + MI . getDesc ( ) . getNumImplicitDefs ( ) , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { const MachineOperand & MO = MI . getOperand (


<s> const MachineOperand & MO = MI . getOperand ( i ) ; if ( ( MO . isReg ( ) && MO . isImplicit ( ) ) || MO .

isRegMask ( ) ) NewMI . addOperand ( MF , MO ) ;


<s> for ( unsigned i = MI . getDesc ( ) . getNumOperands ( ) + MI . getDesc ( ) . getNumImplicitUses ( ) + MI . getDesc ( ) . getNumImplicitDefs ( ) , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { const MachineOperand & Op = MI . getOperand ( i ) ; if ( ! Op . isDef ( ) ) continue ; BuildMI ( * MI . getParent ( ) , MI . getIterator ( ) , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) , Op . getReg ( ) ) ; } MI . eraseFromParent ( )


<s> Src0 . ChangeToImmediate ( MovSrc . getImm ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isFI ( ) ) { Src0 . ChangeToFrameIndex ( MovSrc . getIndex ( ) ) ; ConstantFolded = true ; } else if ( MovSrc . isGlobal ( ) ) { Src0 . ChangeToGA ( MovSrc . getGlobal ( ) , MovSrc . getOffset ( ) , MovSrc . getTargetFlags ( ) ) ; ConstantFolded = true ; } } if ( ConstantFolded ) { if ( MRI -> use_nodbg_empty ( Reg ) ) Def -> eraseFromParent ( ) ; ++ NumLiteralConstantsFolded


<s> for ( const MachineOperand & MO : R ) { if ( ! MO . isReg ( ) ) continue ; if ( Reg . isPhysical ( ) && MO . getReg ( ) . isPhysical ( ) ) { if ( TRI -> regsOverlap ( Reg , MO


<s> return instAccessReg ( MI -> defs ( )


<s> return instAccessReg ( MI -> defs ( ) , Reg , SubReg


<s> return instAccessReg ( MI -> uses ( ) ,


<s> bool <ISA_LIT>ShrinkInstructions :: instReadsReg ( const MachineInstr * MI , unsigned Reg ,


<s> bool <ISA_LIT>ShrinkInstructions :: isKImmOperand ( const MachineOperand & Src ) const { return isInt < <NUM_LIT> > ( Src . getImm ( ) ) && ! TII -> isInlineConstant ( * Src

. getParent ( ) , Src . getParent ( ) -> getOperandNo ( & Src ) ) ;


<s> bool <ISA_LIT>ShrinkInstructions :: isKImmOrKUImmOperand ( const MachineOperand & Src , bool & IsUnsigned ) const { if ( isInt < <NUM_LIT> > ( Src . getImm ( ) ) ) { IsUnsigned = false ; return ! TII -> isInlineConstant ( Src ) ; } if ( isUInt < <NUM_LIT> > ( Src . getImm ( )


<s> bool <ISA_LIT>ShrinkInstructions :: isKUImmOperand ( const MachineOperand & Src ) const { return isUInt < <NUM_LIT> > ( Src . getImm ( ) ) && ! TII -> isInlineConstant ( * Src . getParent ( ) , Src .

getParent ( ) -> getOperandNo ( & Src ) ) ;


<s> bool <ISA_LIT>ShrinkInstructions :: isReverseInlineImm ( const MachineOperand & Src , int32_t & ReverseImm ) const { if ( ! isInt < <NUM_LIT> > ( Src . getImm ( ) ) || TII -> isInlineConstant ( Src ) ) return false ; ReverseImm = reverseBits < int32_t > ( static_cast < int32_t > ( Src . getImm


<s> MovX = nullptr ; break ; } if ( ! instReadsReg ( & * I , Y , Ysub ) ) { if ( ! MovX && instModifiesReg ( & * I , X , Xsub ) ) { MovX = nullptr ; break ; } continue ; } if ( MovX || ( I -> getOpcode ( ) != <STR_LIT> && I -> getOpcode ( ) != <STR_LIT> ) || I -> getOperand ( <NUM_LIT> ) . getReg ( ) != X || I -> getOperand ( <NUM_LIT> ) . getSubReg ( ) != Xsub ) { MovX = nullptr ; break ; } if ( I -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) continue ; if ( Size > <NUM_LIT> && ( I -> getNumImplicitOperands ( ) > ( I -> isCopy ( ) ? <NUM_LIT> : <NUM_LIT> ) ) ) continue ; MovX = & * I ; } if ( ! MovX ) continue ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << MovT << * MovX << * MovY ) ; for ( unsigned I = <NUM_LIT> ; I < Size ; ++ I ) { TargetInstrInfo :: RegSubRegPair X1 , Y1 ; X1 = getSubRegForIndex ( X , Xsub , I ) ; Y1 = getSubRegForIndex ( Y , Ysub , I ) ; MachineBasicBlock & MBB = * MovT . getParent ( ) ; auto MIB = BuildMI ( MBB , MovX -> getIterator ( ) , MovT . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addDef ( X1 . Reg , <NUM_LIT> , X1 . SubReg ) . addDef ( Y1 . Reg , <NUM_LIT> , Y1 . SubReg ) . addReg ( Y1 . Reg , <NUM_LIT> , Y1 . SubReg ) . addReg ( X1 . Reg , <NUM_LIT> , X1 . SubReg ) . getInstr ( ) ; if ( MovX -> hasRegisterImplicitUseOperand ( <STR_LIT> ) ) { MIB -> removeOperand ( MIB -> getNumExplicitOperands ( ) ) ; MIB -> copyImplicitOps ( * MBB . getParent ( ) , * MovX ) ; } } MovX -> eraseFromParent ( ) ; dropInstructionKeepingImpDefs ( * MovY ) ; MachineInstr * Next = & * std :: next ( MovT . getIterator ( ) ) ; if ( T . isVirtual ( ) &&


<s> NewOpcode = <STR_LIT> ; break ; } } if ( Src2 . isReg ( ) && TRI -> isVGPR ( * MRI , Src2 . getReg ( ) ) ) { if ( Src1 . isImm ( ) && ! TII -> isInlineConstant ( Src1 ) ) Swap = false ; else if ( Src0 . isImm ( ) && ! TII -> isInlineConstant ( Src0 ) ) Swap = true ; else return ; switch ( Opcode ) { default : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : NewOpcode = <STR_LIT> ; break ; case <STR_LIT> : NewOpcode = <STR_LIT> ; break ; case <STR_LIT> : NewOpcode = <STR_LIT> ; break ; case <STR_LIT> : NewOpcode = <STR_LIT> ; break ; } } if ( NewOpcode == <STR_LIT> ) return ; if ( Swap ) { BuildMI ( * MI . getParent ( ) , MI , MI . getDebugLoc ( ) , TII -> get ( NewOpcode ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) . add ( Src1 ) . add ( Src0 ) . add ( Src2 ) . setMIFlags ( MI . getFlags ( ) ) ; MI . eraseFromParent ( ) ; } else


<s> RC = & <STR_LIT> ; } else if ( Info -> VAddrDwords == <NUM_LIT> ) { RC = & <STR_LIT> ; } else { RC = & <STR_LIT> ; NewAddrDwords = <NUM_LIT> ; } unsigned VgprBase = <NUM_LIT> ; bool IsUndef = true ; bool IsKill = NewAddrDwords == Info -> VAddrDwords ; for ( unsigned i = <NUM_LIT> ; i < Info -> VAddrDwords ; ++ i ) { const MachineOperand & Op = MI . getOperand ( VAddr0Idx + i ) ; unsigned Vgpr = TRI -> getHWRegIndex ( Op . getReg ( ) ) ; if ( i == <NUM_LIT> ) { VgprBase = Vgpr ; } else if ( VgprBase + i != Vgpr ) return ; if ( ! Op . isUndef ( ) ) IsUndef = false ; if ( ! Op . isKill ( ) ) IsKill = false ; } if ( VgprBase + NewAddrDwords > <NUM_LIT> ) return ; int TFEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: tfe ) ; int LWEIdx = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: lwe ) ; unsigned TFEVal = ( TFEIdx == - <NUM_LIT> ) ? <NUM_LIT> : MI . getOperand ( TFEIdx ) . getImm ( ) ; unsigned LWEVal = ( LWEIdx == - <NUM_LIT> ) ? <NUM_LIT> : MI . getOperand ( LWEIdx ) . getImm ( ) ; int ToUntie = - <NUM_LIT> ; if ( TFEVal || LWEVal ) { for ( unsigned i = LWEIdx + <NUM_LIT> , e = MI . getNumOperands ( ) ; i != e ; ++ i ) { if ( MI . getOperand ( i ) . isReg ( ) && MI . getOperand ( i ) . isTied ( ) && MI . getOperand ( i ) . isImplicit ( ) ) { assert ( ToUntie == - <NUM_LIT> && <STR_LIT> ) ; ToUntie = i ; MI . untieRegOperand ( ToUntie ) ; } } } unsigned NewOpcode = <STR_LIT> ( Info -> BaseOpcode , <STR_LIT> , Info -> VDataDwords , NewAddrDwords ) ; MI . setDesc ( TII -> get ( NewOpcode ) ) ; MI . getOperand ( VAddr0Idx ) . setReg ( RC -> getRegister ( VgprBase ) ) ; MI . getOperand ( VAddr0Idx ) . setIsUndef ( IsUndef ) ; MI . getOperand ( VAddr0Idx ) . setIsKill ( IsKill ) ; for ( unsigned i = <NUM_LIT> ; i < Info -> VAddrDwords ; ++ i ) MI . removeOperand ( VAddr0Idx + <NUM_LIT> ) ; if ( ToUntie >=