<s> case <ISA_LIT>RegisterInfo :: DISPATCH_ID : assert ( MFI -> hasDispatchID ( ) ) ; return MFI -> DispatchIDUserSGPR ; case <ISA_LIT>RegisterInfo :: FLAT_SCRATCH_INIT : assert ( MFI -> hasFlatScratchInit ( ) ) ; return MFI -> FlatScratchInitUserSGPR ; case <ISA_LIT>RegisterInfo :: DISPATCH_PTR : assert ( MFI -> hasDispatchPtr ( ) ) ; return MFI -> DispatchPtrUserSGPR ; case <ISA_LIT>RegisterInfo :: QUEUE_PTR : assert ( MFI -> hasQueuePtr ( ) ) ; return MFI -> QueuePtrUserSGPR ; case <ISA_LIT>RegisterInfo :: WORKITEM_ID_X : assert ( MFI -> hasWorkItemIDX ( ) ) ; return <STR_LIT> ; case <ISA_LIT>RegisterInfo :: WORKITEM_ID_Y : assert ( MFI -> hasWorkItemIDY ( ) ) ; return <STR_LIT> ; case <ISA_LIT>RegisterInfo :: WORKITEM_ID_Z : assert ( MFI -> hasWorkItemIDZ ( ) ) ; return <STR_LIT> ; }


<s> const MachineOperand * Reg = TII -> getNamedOperand ( * MI , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , MI , DL , TII -> get ( LoadStoreOp ) ) . add ( * Reg ) . add ( * TII -> getNamedOperand ( * MI , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * MI , <STR_LIT> :: soffset ) ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . setMemRefs ( MI -> memoperands_begin ( ) , MI -> memoperands_end ( ) ) ;


<s> if ( LoadStoreOp == - <NUM_LIT> ) return false ; const MachineOperand * Reg = TII -> getNamedOperand ( * MI , <STR_LIT> :: vdata ) ; BuildMI ( * MBB , MI , DL , TII -> get ( LoadStoreOp ) ) . add ( * Reg ) . add ( * TII -> getNamedOperand ( * MI , <STR_LIT> :: srsrc ) ) . add ( * TII -> getNamedOperand ( * MI , <STR_LIT> :: soffset ) ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . setMemRefs ( MI -> memoperands_begin ( ) , MI -> memoperands_end ( ) ) ;


<s> for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumVGPRs ; i < TotalNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; if ( ScratchWaveOffsetReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchWaveOffsetReg ) ; } unsigned ScratchRSrcReg = MFI -> getScratchRSrcReg ( ) ; if ( ScratchRSrcReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchRSrcReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , ScratchWaveOffsetReg ) ) ; } unsigned StackPtrReg = MFI -> getStackPtrOffsetReg ( ) ; if ( StackPtrReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , StackPtrReg ) ;

assert ( ! isSubRegister ( ScratchRSrcReg , StackPtrReg ) ) ;


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = ST . getMaxNumVGPRs ( MF ) ;

unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ;


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumVGPRs ; i < TotalNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; }

const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;


<s> unsigned M0CopyReg = <STR_LIT> ; if ( SpillToSMEM ) { if ( RS -> isRegUsed ( <STR_LIT> ) ) { M0CopyReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , M0CopyReg ) . addReg ( <STR_LIT> ) ; } } unsigned ScalarStoreOp ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; if ( SpillToSMEM && isSGPRClass ( RC ) ) { std :: tie ( EltSize , ScalarStoreOp ) = getSpillEltSize ( getRegSizeInBits ( * RC ) / <NUM_LIT> , true ) ; } ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { unsigned SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; if ( SpillToSMEM ) { int64_t FrOffset = FrameInfo . getObjectOffset ( Index ) ; unsigned Align = FrameInfo . getObjectAlignment ( Index ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , Index , EltSize * i ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , EltSize , MinAlign ( Align , EltSize * i ) ) ; int64_t Offset = ( ST . getWavefrontSize ( ) * FrOffset ) + ( EltSize * i ) ; if ( Offset != <NUM_LIT> ) { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , OffsetReg ) . addReg ( MFI -> getScratchWaveOffsetReg ( ) ) . addImm ( Offset ) ; } else { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , OffsetReg ) . addReg ( MFI -> getScratchWaveOffsetReg ( ) ) ; } BuildMI ( * MBB , MI , DL , TII -> get ( ScalarStoreOp ) ) . addReg ( SubReg , getKillRegState ( IsKill ) ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( OffsetReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; continue ; } if ( SpillToVGPR ) { <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; BuildMI ( * MBB , MI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , Spill . VGPR ) . addReg ( SubReg , getKillRegState ( IsKill ) ) . addImm ( Spill . Lane ) ; } else {


<s> SOffset = ScratchOffset ; } else { Scavenged = true ; } BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , SOffset ) . addReg ( ScratchOffset ) . addImm ( Offset ) ; Offset = <NUM_LIT> ; } for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i != e ; ++ i , Offset += <NUM_LIT> ) { unsigned SubReg = NumSubRegs > <NUM_LIT> ? getPhysRegSubReg ( Value , & <STR_LIT> , i ) : Value ; unsigned SOffsetRegState = <NUM_LIT> ; if ( i + <NUM_LIT> == e && Scavenged ) SOffsetRegState |= RegState :: Kill ; BuildMI ( * MBB , MI , DL , TII -> get ( LoadStoreOp ) ) . addReg ( SubReg , getDefRegState ( IsLoad ) ) . addReg ( ScratchRsrcReg ) . addReg ( SOffset , SOffsetRegState ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( Value , RegState :: Implicit | getDefRegState ( IsLoad ) ) . setMemRefs ( MI -> memoperands_begin ( ) , MI -> memoperands_end ( ) ) ; } if ( RanOutOfSGPRs ) { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , ScratchOffset ) . addReg ( ScratchOffset ) . addImm ( OriginalImmOffset ) ; }


<s> return getRegClassForSizeOnBank ( Ty . getSizeInBits ( ) , Bank ) ;


<s> const TargetRegisterClass * getRegClassForTypeOnBank ( LLT Ty , const RegisterBank & Bank ) const {

return getRegClassForSizeOnBank ( Ty . getSizeInBits ( ) , Bank ) ;


<s> assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ; unsigned OffsetReg = <STR_LIT> ; unsigned M0CopyReg = <STR_LIT> ; if ( SpillToSMEM ) { if ( RS -> isRegUsed ( <STR_LIT> ) ) { M0CopyReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , M0CopyReg ) . addReg ( <STR_LIT> ) ; } } unsigned ScalarStoreOp ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; if ( SpillToSMEM && isSGPRClass ( RC ) ) { std :: tie ( EltSize , ScalarStoreOp ) = getSpillEltSize ( getRegSizeInBits ( * RC ) / <NUM_LIT> , true ) ; } ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { unsigned SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; if ( SpillToSMEM ) { int64_t FrOffset = FrameInfo . getObjectOffset ( Index ) ; unsigned Align = FrameInfo . getObjectAlignment ( Index ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , Index , EltSize * i ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , EltSize , MinAlign ( Align , EltSize * i ) ) ; int64_t Offset = ( ST . getWavefrontSize ( ) * FrOffset ) + ( EltSize * i ) ; if ( Offset != <NUM_LIT> ) { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , OffsetReg ) . addReg ( MFI -> getFrameOffsetReg ( ) ) . addImm ( Offset ) ; } else { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , OffsetReg ) . addReg ( MFI -> getFrameOffsetReg ( ) ) ; } BuildMI ( * MBB , MI , DL , TII -> get ( ScalarStoreOp ) ) . addReg ( SubReg , getKillRegState ( IsKill ) ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( OffsetReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> unsigned getCSRFirstUseCost ( const MachineFunction & MF ) const override {


<s> const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF -> getSubtarget ( ) . getInstrInfo ( ) ) ; LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; bool IsLoad = TII -> get ( LoadStoreOp ) . mayLoad ( ) ; bool RanOutOfSGPRs = false ; bool Scavenged = false ; unsigned SOffset = ScratchOffset ; unsigned NumSubRegs = getNumSubRegsForSpillOp ( MI -> getOpcode ( ) ) ; unsigned Size = NumSubRegs * <NUM_LIT> ; if ( ! isUInt < <NUM_LIT> > ( Offset + Size ) ) { SOffset = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; if ( SOffset == <STR_LIT> ) { RanOutOfSGPRs = true ; SOffset = <STR_LIT> ; } else { Scavenged = true ; } BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , SOffset ) . addReg ( ScratchOffset ) . addImm ( Offset ) ; Offset = <NUM_LIT> ; } if ( RanOutOfSGPRs ) Ctx . emitError ( <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i != e ; ++ i , Offset += <NUM_LIT> ) { unsigned SubReg = NumSubRegs > <NUM_LIT> ? getPhysRegSubReg ( Value , & <STR_LIT> , i ) : Value ; unsigned SOffsetRegState = <NUM_LIT> ; if ( i + <NUM_LIT> == e && Scavenged ) SOffsetRegState |= RegState :: Kill ; BuildMI ( * MBB , MI , DL , TII -> get ( LoadStoreOp ) ) . addReg ( SubReg , getDefRegState ( IsLoad ) ) . addReg ( ScratchRsrcReg ) . addReg ( SOffset , SOffsetRegState ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( Value , RegState :: Implicit | getDefRegState ( IsLoad ) ) . setMemRefs ( MI -> memoperands_begin ( ) , MI -> memoperands_end ( ) ) ; }


<s> if ( Register :: isVirtualRegister ( Reg ) ) RC = MRI . getRegClass ( Reg ) ; else RC = getPhysRegClass ( Reg ) ;


<s> if ( Requested && ( Requested <= getNumReservedSGPRs ( ST , MFI ) ) ) Requested = <NUM_LIT> ; unsigned NumInputSGPRs = MFI . getNumPreloadedSGPRs ( ) ; if ( Requested && Requested < NumInputSGPRs ) Requested = NumInputSGPRs ; if ( Requested && Requested > getMaxNumSGPRs ( ST , WavesPerEU . first , false ) ) Requested = <NUM_LIT> ; if ( WavesPerEU . second && Requested && Requested < getMinNumSGPRs ( ST , WavesPerEU . second ) ) Requested = <NUM_LIT> ; if ( Requested ) MaxNumSGPRs = Requested ; }

if ( ST . hasSGPRInitBug ( ) ) MaxNumSGPRs = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG ;


<s> unsigned Requested = <STR_LIT> ( F , <STR_LIT> , MaxNumVGPRs ) ; if ( Requested && Requested <= getNumDebuggerReservedVGPRs ( ST ) ) Requested = <NUM_LIT> ; if ( Requested && Requested > getMaxNumVGPRs ( WavesPerEU . first ) ) Requested = <NUM_LIT> ; if ( WavesPerEU . second && Requested && Requested < getMinNumVGPRs ( WavesPerEU . second ) ) Requested = <NUM_LIT> ;

if ( Requested ) MaxNumVGPRs = Requested ;


<s> if ( Requested && Requested > getMaxNumVGPRs ( WavesPerEU . first ) ) Requested = <NUM_LIT> ; if ( WavesPerEU . second && Requested && Requested < getMinNumVGPRs ( WavesPerEU . second ) ) Requested = <NUM_LIT> ; if ( Requested ) MaxNumVGPRs = Requested ; }

return MaxNumVGPRs - getNumDebuggerReservedVGPRs ( ST ) ;


<s> } else { switch ( WavesPerEU ) { case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ;


<s> case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; default : return <NUM_LIT> ; }


<s> case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ; case <NUM_LIT> : return <NUM_LIT> ;


<s> unsigned <ISA_LIT>RegisterInfo :: getNumAddressableSGPRs ( const <ISA_LIT>Subtarget & ST ) const { if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> unsigned <ISA_LIT>RegisterInfo :: getNumAddressableSGPRs ( const <ISA_LIT>Subtarget & ST ) const {

if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return <NUM_LIT> ;


<s> if ( ST . debuggerReserveRegs ( ) ) return <NUM_LIT> ;


<s> unsigned <ISA_LIT>RegisterInfo :: getNumDebuggerReservedVGPRs ( const <ISA_LIT>Subtarget & ST ) const {

if ( ST . debuggerReserveRegs ( ) ) return <NUM_LIT> ;


<s> if ( MFI . hasFlatScratchInit ( ) ) { if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return <NUM_LIT> ; if ( ST . getGeneration ( ) == <ISA_LIT>Subtarget :: SEA_ISLANDS ) return <NUM_LIT> ; } if ( ST . isXNACKEnabled ( ) ) return <NUM_LIT> ;


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; unsigned MaxNumSGPRs = getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = getMaxNumVGPRs ( MF ) ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumVGPRs ; i < TotalNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ;


<s> unsigned <ISA_LIT>RegisterInfo :: getTotalNumSGPRs ( const <ISA_LIT>Subtarget & ST ) const {


<s> if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) return <NUM_LIT> ; return <NUM_LIT> ;


<s> unsigned BaseReg ( <STR_LIT> . getRegister ( BaseIdx ) ) ; return getMatchingSuperReg ( BaseReg , <STR_LIT> , & <STR_LIT> ) ;


<s> } else { Reg = RegCount - <NUM_LIT> ; } return <STR_LIT> . getRegister ( Reg ) ;


<s> unsigned RegCount = getMaxNumSGPRs ( MF ) ; unsigned Reg ; if ( RegCount & <NUM_LIT> ) { Reg = RegCount - <NUM_LIT> ; } else {


<s> int64_t Offset = FrameInfo . getObjectOffset ( Index ) ; if ( Offset == <NUM_LIT> ) { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , ResultReg ) . addImm ( ST . getWavefrontSizeLog2 ( ) ) . addReg ( FrameReg ) ; } else { if ( auto MIB = TII -> getAddNoCarry ( * MBB , MI , DL , ResultReg , * RS ) ) { Register ScaledReg = ResultReg ; BuildMI ( * MBB , * MIB , DL , TII -> get ( <STR_LIT> ) , ScaledReg ) . addImm ( ST . getWavefrontSizeLog2 ( ) ) . addReg ( FrameReg ) ; const bool IsVOP2 = MIB -> getOpcode ( ) == <STR_LIT> ; if ( IsVOP2 || <STR_LIT> ( Offset , ST . hasInv2PiInlineImm ( ) ) ) { MIB . addImm ( Offset ) ; MIB . addReg ( ScaledReg , RegState :: Kill ) ; if ( ! IsVOP2 ) MIB . addImm ( <NUM_LIT> ) ; } else { assert ( MIB -> getOpcode ( ) == <STR_LIT> && <STR_LIT> ) ; Register ConstOffsetReg ; if ( ! isWave32 ) ConstOffsetReg = getSubReg ( MIB . getReg ( <NUM_LIT> ) , <STR_LIT> ) ; else ConstOffsetReg = MIB . getReg ( <NUM_LIT> ) ; BuildMI ( * MBB , * MIB , DL , TII -> get ( <STR_LIT> ) , ConstOffsetReg ) . addImm ( Offset ) ; MIB . addReg ( ConstOffsetReg , RegState :: Kill ) ; MIB . addReg ( ScaledReg , RegState :: Kill ) ; MIB . addImm ( <NUM_LIT> ) ; } } else { Register TmpScaledReg = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> , false ) ; Register ScaledReg = TmpScaledReg . isValid ( ) ? TmpScaledReg : FrameReg ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , ScaledReg ) . addReg ( FrameReg ) . addImm ( ST . getWavefrontSizeLog2 ( ) ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , ScaledReg ) . addReg ( ScaledReg , RegState :: Kill ) . addImm ( Offset ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , ResultReg ) . addReg ( ScaledReg , RegState :: Kill ) ; if ( ! TmpScaledReg . isValid ( ) ) { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , ScaledReg ) . addReg ( ScaledReg , RegState :: Kill ) . addImm ( Offset ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , ScaledReg ) . addReg ( FrameReg ) . addImm ( ST . getWavefrontSizeLog2 ( ) ) ; } } } if ( IsCopy ) MI -> eraseFromParent ( ) ; else FIOp . ChangeToRegister ( ResultReg , false , false , true ) ; return ; }


<s> if ( static_cast < int > ( RCID ) == - <NUM_LIT> ) return false ;

return isSGPRClass ( getRegClass ( RCID ) ) ;


<s> unsigned getChannelFromSubReg ( unsigned SubReg ) const { return SubReg ? alignTo ( getSubRegIdxOffset ( SubReg ) , <NUM_LIT> ) / <NUM_LIT> : <NUM_LIT> ;


<s> static unsigned getNumCoveredRegs ( LaneBitmask LM ) { return LM . getNumLanes ( ) ;


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; const DebugLoc & DL = MI -> getDebugLoc ( ) ; bool IsStore = MI -> mayStore ( ) ; unsigned Opc = MI -> getOpcode ( ) ; int LoadStoreOp = IsStore ? getOffsetMUBUFStore ( Opc ) : getOffsetMUBUFLoad ( Opc ) ; if ( LoadStoreOp == - <NUM_LIT> ) return false ;

unsigned Reg = TII -> getNamedOperand ( * MI , <STR_LIT> :: vdata ) -> getReg ( ) ;


<s> unsigned Reg = TII -> getNamedOperand ( * MI , <STR_LIT> :: vdata ) -> getReg ( ) ;

BuildMI ( * MBB , MI , DL , TII -> get ( LoadStoreOp ) ) . addReg ( Reg , getDefRegState ( ! IsStore ) ) . addOperand ( * TII -> getNamedOperand ( * MI , <STR_LIT> :: srsrc ) ) . addOperand ( * TII -> getNamedOperand ( * MI , <STR_LIT> :: soffset ) ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . setMemRefs ( MI -> memoperands_begin ( ) , MI -> memoperands_end ( ) ) ;


<s> } for ( auto Reg : <STR_LIT> ) { Reserved . set ( getSubReg ( Reg , <STR_LIT> ) ) ; Register Low = getSubReg ( Reg , <STR_LIT> ) ; if ( ! <STR_LIT> . contains ( Low ) ) Reserved . set ( Low ) ; } for ( auto Reg : <STR_LIT> ) { Reserved . set ( getSubReg ( Reg , <STR_LIT> ) ) ; } if ( ! ST . hasMAIInsts ( ) ) { for ( unsigned i = <NUM_LIT> ; i < MaxNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register ScratchRSrcReg = MFI -> getScratchRSrcReg ( ) ; if ( ScratchRSrcReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchRSrcReg ) ; } MCRegister StackPtrReg = MFI -> getStackPtrOffsetReg ( ) ; if ( StackPtrReg ) { reserveRegisterTuples ( Reserved , StackPtrReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , StackPtrReg ) ) ; } MCRegister FrameReg = MFI -> getFrameOffsetReg ( ) ; if ( FrameReg ) { reserveRegisterTuples ( Reserved , FrameReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , FrameReg ) ) ; } if ( hasBasePointer ( MF ) ) { MCRegister BasePtrReg = getBaseRegister ( ) ; reserveRegisterTuples ( Reserved , BasePtrReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , BasePtrReg ) ) ; } for ( auto Reg : MFI -> WWMReservedRegs ) {

reserveRegisterTuples ( Reserved , Reg . first ) ;


<s> MIB . addReg ( SB . SuperReg , RegState :: ImplicitDefine ) ; } if ( SB . NumSubRegs > <NUM_LIT> ) MIB . addReg ( SB . SuperReg , getKillRegState ( UseKill ) | RegState :: Implicit ) ; } } else { SB . prepare ( ) ; unsigned SubKillState = getKillRegState ( ( SB . NumSubRegs == <NUM_LIT> ) && SB . IsKill ) ; auto PVD = SB . getPerVGPRData ( ) ; for ( unsigned Offset = <NUM_LIT> ; Offset < PVD . NumVGPRs ; ++ Offset ) { unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; MachineInstrBuilder WriteLane = BuildMI ( SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SB . TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PVD . PerVGPR ) . addReg ( SB . TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ; if ( LIS ) { if ( i == <NUM_LIT> ) LIS -> ReplaceMachineInstrInMaps ( * MI , * WriteLane ) ; else LIS -> InsertMachineInstrInMaps ( * WriteLane ) ; } if ( SB . NumSubRegs > <NUM_LIT> ) { unsigned SuperKillState = <NUM_LIT> ; if ( i + <NUM_LIT> == SB . NumSubRegs ) SuperKillState |= getKillRegState ( SB . IsKill ) ; WriteLane . addReg ( SB . SuperReg , RegState :: Implicit | SuperKillState ) ; } } SB . readWriteTmpVGPR ( Offset , false ) ; }


<s> if ( ! <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) && ! <ISA_LIT>InstrInfo :: isFLATScratch ( * MI ) ) return false ; int64_t NewOffset = Offset + getScratchInstrOffset ( MI ) ; if ( <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) ) return <ISA_LIT>InstrInfo :: isLegalMUBUFImmOffset ( NewOffset ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; return TII -> isLegalFLATOffset ( NewOffset , <ISA_LIT>AS :: PRIVATE_ADDRESS , true ) ;


<s> bool <ISA_LIT>RegisterInfo :: isFrameOffsetLegal ( const MachineInstr * MI , Register BaseReg , int64_t Offset ) const { if ( ! <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) && ! <ISA_LIT>InstrInfo :: isFLATScratch ( * MI ) ) return false ; int64_t NewOffset = Offset + getScratchInstrOffset ( MI ) ;

if ( <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) ) return <ISA_LIT>InstrInfo :: isLegalMUBUFImmOffset ( NewOffset ) ;


<s> MachineFunction * MF = MBB -> getParent ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; unsigned MovOpc = ST . enableFlatScratch ( ) ? <STR_LIT> : <STR_LIT> ; Register BaseReg = MRI . createVirtualRegister ( ST . enableFlatScratch ( ) ? & <STR_LIT> : & <STR_LIT> ) ; if ( Offset == <NUM_LIT> ) { BuildMI ( * MBB , Ins , DL , TII -> get ( MovOpc ) , BaseReg ) . addFrameIndex ( FrameIdx ) ; return BaseReg ;


<s> int64_t FullOffset = Offset + getScratchInstrOffset ( MI ) ; if ( <ISA_LIT>InstrInfo :: isMUBUF ( * MI ) ) return ! <ISA_LIT>InstrInfo :: isLegalMUBUFImmOffset ( FullOffset ) ;

const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ;


<s> assert ( TII -> isLegalFLATOffset ( NewOffset , <ISA_LIT>AS :: PRIVATE_ADDRESS , true ) && <STR_LIT> ) ; FIOp -> ChangeToRegister ( BaseReg , false ) ; OffsetOp -> setImm ( NewOffset ) ; return ; } MachineOperand * SOffset = TII -> getNamedOperand ( MI , <STR_LIT> :: soffset ) ; assert ( SOffset -> isImm ( ) && SOffset -> getImm ( ) == <NUM_LIT> ) ; assert ( <ISA_LIT>InstrInfo :: isLegalMUBUFImmOffset ( NewOffset ) && <STR_LIT> ) ; FIOp -> ChangeToRegister ( BaseReg , false ) ;


<s> void <ISA_LIT>RegisterInfo :: resolveFrameIndex ( MachineInstr & MI , Register BaseReg , int64_t Offset ) const { const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; bool IsFlat = TII -> isFLATScratch ( MI ) ; bool SeenFI = false ; for ( const MachineOperand & MO : MI . operands ( ) ) { if ( MO . isFI ( ) ) { if ( SeenFI ) llvm_unreachable ( <STR_LIT> ) ; SeenFI = true ; } } MachineOperand * FIOp = TII -> getNamedOperand ( MI , IsFlat ? <STR_LIT> :: saddr : <STR_LIT> :: vaddr ) ; MachineOperand * OffsetOp = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) ; int64_t NewOffset = OffsetOp -> getImm ( ) + Offset ; assert ( FIOp && FIOp -> isFI ( ) && <STR_LIT> ) ; assert ( TII -> isMUBUF ( MI ) || TII -> isFLATScratch ( MI ) ) ; if ( IsFlat ) { assert ( TII -> isLegalFLATOffset ( NewOffset , <ISA_LIT>AS :: PRIVATE_ADDRESS , true ) && <STR_LIT> ) ; FIOp -> ChangeToRegister ( BaseReg , false ) ; OffsetOp -> setImm ( NewOffset ) ; return ;


<s> if ( Reg == <STR_LIT> ) return MachineInstrBuilder ( ) ; bool IsStore = MI -> mayStore ( ) ; MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; auto * TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( MRI . getTargetRegisterInfo ( ) ) ; unsigned Dst = IsStore ? Reg : ValueReg ; unsigned Src = IsStore ? ValueReg : Reg ; unsigned Opc = ( IsStore ^ TRI -> isVGPR ( MRI , Reg ) ) ? <STR_LIT> : <STR_LIT> ; auto MIB = BuildMI ( * MBB , MI , MI -> getDebugLoc ( ) , TII -> get ( Opc ) , Dst ) . addReg ( Src , getKillRegState ( IsKill ) ) ; MIB -> setAsmPrinterFlag ( MachineInstr :: ReloadReuse ) ;


<s> if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ;


<s> if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ;


<s> if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ; if ( BitWidth <= <NUM_LIT> ) return & <STR_LIT> ;


<s> static const TargetRegisterClass * const BaseClasses [ ] = { & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , & <STR_LIT> , } ; for ( const TargetRegisterClass * BaseClass : BaseClasses ) {


<s> if ( SpillToVGPR ) { for ( unsigned i = <NUM_LIT> , e = SB . NumSubRegs ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; auto MIB = BuildMI ( SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SubReg ) . addReg ( Spill . VGPR ) . addImm ( Spill . Lane ) ; if ( SB . NumSubRegs > <NUM_LIT> && i == <NUM_LIT> ) MIB . addReg ( SB . SuperReg , RegState :: ImplicitDefine ) ; } } else { SB . prepare ( ) ; auto PVD = SB . getPerVGPRData ( ) ; for ( unsigned Offset = <NUM_LIT> ; Offset < PVD . NumVGPRs ; ++ Offset ) { SB . readWriteTmpVGPR ( Offset , true ) ; for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ;

bool LastSubReg = ( i + <NUM_LIT> == e ) ;


<s> SGPRSpillBuilder SB ( * this , * ST . getInstrInfo ( ) , isWave32 , MI , Index , RS ) ; ArrayRef < <ISA_LIT>MachineFunctionInfo :: SpilledReg > VGPRSpills = SB . MFI . getSGPRToVGPRSpills ( Index ) ; bool SpillToVGPR = ! VGPRSpills . empty ( ) ; if ( OnlyToVGPR && ! SpillToVGPR ) return false ; if ( SpillToVGPR ) { for ( unsigned i = <NUM_LIT> , e = SB . NumSubRegs ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; auto MIB = BuildMI ( SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SubReg ) . addReg ( Spill . VGPR ) . addImm ( Spill . Lane ) ; if ( SB . NumSubRegs > <NUM_LIT> && i == <NUM_LIT> ) MIB . addReg ( SB . SuperReg , RegState :: ImplicitDefine ) ; } } else { SB . prepare ( ) ; auto PVD = SB . getPerVGPRData ( ) ; for ( unsigned Offset = <NUM_LIT> ; Offset < PVD . NumVGPRs ; ++ Offset ) { SB . readWriteTmpVGPR ( Offset , true ) ; for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; bool LastSubReg = ( i + <NUM_LIT> == e ) ; auto MIB = BuildMI ( SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SubReg ) . addReg ( SB . TmpVGPR , getKillRegState ( LastSubReg ) ) . addImm ( i ) ; if ( SB . NumSubRegs > <NUM_LIT> && i == <NUM_LIT> ) MIB . addReg ( SB . SuperReg , RegState :: ImplicitDefine ) ; } }


<s> if ( i == <NUM_LIT> && SB . NumSubRegs > <NUM_LIT> ) { MIB . addReg ( SB . SuperReg , RegState :: ImplicitDefine ) ; } if ( SB . NumSubRegs > <NUM_LIT> ) MIB . addReg ( SB . SuperReg , getKillRegState ( UseKill ) | RegState :: Implicit ) ; } } else { SB . prepare ( ) ; unsigned SubKillState = getKillRegState ( ( SB . NumSubRegs == <NUM_LIT> ) && SB . IsKill ) ; auto PVD = SB . getPerVGPRData ( ) ; for ( unsigned Offset = <NUM_LIT> ; Offset < PVD . NumVGPRs ; ++ Offset ) { unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; MachineInstrBuilder WriteLane = BuildMI ( SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SB . TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PVD . PerVGPR ) . addReg ( SB . TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ;


<s> SB . prepare ( ) ; unsigned SubKillState = getKillRegState ( ( SB . NumSubRegs == <NUM_LIT> ) && SB . IsKill ) ; auto PVD = SB . getPerVGPRData ( ) ; for ( unsigned Offset = <NUM_LIT> ; Offset < PVD . NumVGPRs ; ++ Offset ) { unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PVD . PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PVD . PerVGPR , SB . NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = SB . NumSubRegs == <NUM_LIT> ? SB . SuperReg : Register ( getSubReg ( SB . SuperReg , SB . SplitParts [ i ] ) ) ; MachineInstrBuilder WriteLane = BuildMI ( SB . MBB , MI , SB . DL , SB . TII . get ( <STR_LIT> ) , SB . TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PVD . PerVGPR ) . addReg ( SB . TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ; if ( SB . NumSubRegs > <NUM_LIT> ) { unsigned SuperKillState = <NUM_LIT> ; if ( i + <NUM_LIT> == SB . NumSubRegs ) SuperKillState |= getKillRegState ( SB . IsKill ) ; WriteLane . addReg ( SB . SuperReg , RegState :: Implicit | SuperKillState ) ; } } SB . readWriteTmpVGPR ( Offset , false ) ; }


<s> switch ( RB . getID ( ) ) { case <STR_LIT> : return & <STR_LIT> ; case <STR_LIT> : return & <STR_LIT> ; case <STR_LIT> : return & <STR_LIT> ; case <STR_LIT> : return & <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ; } } case <NUM_LIT> : return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; default : if ( Size < <NUM_LIT> ) return RB . getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; assert ( Size < <NUM_LIT> && <STR_LIT> ) ;

return getRegClassForSizeOnBank ( PowerOf2Ceil ( Size ) , RB , MRI ) ;


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; if ( isWave32 ) { Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; } unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } for ( auto Reg : <STR_LIT> ) { Reserved . set ( getSubReg ( Reg , <STR_LIT> ) ) ; Register Low = getSubReg ( Reg , <STR_LIT> ) ; if ( ! <STR_LIT> . contains ( Low ) ) Reserved . set ( Low ) ; } Register ScratchRSrcReg = MFI -> getScratchRSrcReg ( ) ; if ( ScratchRSrcReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchRSrcReg ) ; } MCRegister StackPtrReg = MFI -> getStackPtrOffsetReg ( ) ; if ( StackPtrReg ) { reserveRegisterTuples ( Reserved , StackPtrReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , StackPtrReg ) ) ; } MCRegister FrameReg = MFI -> getFrameOffsetReg ( ) ; if ( FrameReg ) { reserveRegisterTuples ( Reserved , FrameReg ) ;

assert ( ! isSubRegister ( ScratchRSrcReg , FrameReg ) ) ;


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumVGPRs ; i < TotalNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; if ( ScratchWaveOffsetReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchWaveOffsetReg ) ;


<s> unsigned OffsetReg = <STR_LIT> ; unsigned M0CopyReg = <STR_LIT> ; if ( SpillToSMEM ) { if ( RS -> isRegUsed ( <STR_LIT> ) ) { M0CopyReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , M0CopyReg ) . addReg ( <STR_LIT> ) ; } } unsigned EltSize = <NUM_LIT> ; unsigned ScalarLoadOp ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; if ( SpillToSMEM && isSGPRClass ( RC ) ) { std :: tie ( EltSize , ScalarLoadOp ) = getSpillEltSize ( RC -> getSize ( ) , false ) ; } ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; int64_t FrOffset = FrameInfo . getObjectOffset ( Index ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { unsigned SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; if ( SpillToSMEM ) { unsigned Align = FrameInfo . getObjectAlignment ( Index ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , Index , EltSize * i ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , EltSize , MinAlign ( Align , EltSize * i ) ) ; int64_t Offset = ( ST . getWavefrontSize ( ) * FrOffset ) + ( EltSize * i ) ; if ( Offset != <NUM_LIT> ) { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , OffsetReg ) . addReg ( MFI -> getScratchWaveOffsetReg ( ) ) . addImm ( Offset ) ; } else { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , OffsetReg ) . addReg ( MFI -> getScratchWaveOffsetReg ( ) ) ; } auto MIB = BuildMI ( * MBB , MI , DL , TII -> get ( ScalarLoadOp ) , SubReg ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( OffsetReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; if ( NumSubRegs > <NUM_LIT> ) MIB . addReg ( SuperReg , RegState :: ImplicitDefine ) ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; unsigned SuperReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool IsKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; const DebugLoc & DL = MI -> getDebugLoc ( ) ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; bool SpillToSMEM = spillSGPRToSMEM ( ) ; if ( SpillToSMEM && OnlyToVGPR ) return false ; assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ; unsigned OffsetReg = <STR_LIT> ; unsigned M0CopyReg = <STR_LIT> ; if ( SpillToSMEM ) { if ( RS -> isRegUsed ( <STR_LIT> ) ) { M0CopyReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , M0CopyReg ) . addReg ( <STR_LIT> ) ; } } unsigned ScalarStoreOp ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; if ( SpillToSMEM && isSGPRClass ( RC ) ) { std :: tie ( EltSize , ScalarStoreOp ) = getSpillEltSize ( RC -> getSize ( ) , true ) ; } ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { unsigned SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; if ( SpillToSMEM ) { int64_t FrOffset = FrameInfo . getObjectOffset ( Index ) ;

unsigned Align = FrameInfo . getObjectAlignment ( Index ) ;


<s> bool VGPRDefined = true ; if ( MBB == & MF -> front ( ) ) VGPRDefined = ! SGPRSpillVGPRDefinedSet . insert ( Spill . VGPR ) . second ; auto MIB = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , Spill . VGPR ) . addReg ( SubReg , getKillRegState ( UseKill ) ) . addImm ( Spill . Lane ) . addReg ( Spill . VGPR , VGPRDefined ? <NUM_LIT> : RegState :: Undef ) ; if ( i == <NUM_LIT> && NumSubRegs > <NUM_LIT> ) { MIB . addReg ( SuperReg , RegState :: ImplicitDefine ) ; } if ( NumSubRegs > <NUM_LIT> ) MIB . addReg ( SuperReg , getKillRegState ( UseKill ) | RegState :: Implicit ) ; } } else { Register TmpVGPR = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpVGPR ) ; unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ; unsigned PerVGPR = <NUM_LIT> ; unsigned NumVGPRs = ( NumSubRegs + ( PerVGPR - <NUM_LIT> ) ) / PerVGPR ; int64_t VGPRLanes = ( <NUM_LIT> << std :: min ( PerVGPR , NumSubRegs ) ) - <NUM_LIT> ; for ( unsigned Offset = <NUM_LIT> ; Offset < NumVGPRs ; ++ Offset ) { unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PerVGPR , NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : Register ( getSubReg ( SuperReg , SplitParts [ i ] ) ) ; MachineInstrBuilder WriteLane = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PerVGPR ) . addReg ( TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ; if ( NumSubRegs > <NUM_LIT> ) { unsigned SuperKillState = <NUM_LIT> ; if ( i + <NUM_LIT> == NumSubRegs ) SuperKillState |= getKillRegState ( IsKill ) ; WriteLane . addReg ( SuperReg , RegState :: Implicit | SuperKillState ) ; } } buildSGPRSpillLoadStore ( MI , Index , Offset , EltSize , TmpVGPR , VGPRLanes , RS , false ) ; }


<s> } } else { Register TmpVGPR = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpVGPR ) ; unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ; unsigned PerVGPR = <NUM_LIT> ; unsigned NumVGPRs = ( NumSubRegs + ( PerVGPR - <NUM_LIT> ) ) / PerVGPR ; int64_t VGPRLanes = ( <NUM_LIT> << std :: min ( PerVGPR , NumSubRegs ) ) - <NUM_LIT> ; for ( unsigned Offset = <NUM_LIT> ; Offset < NumVGPRs ; ++ Offset ) { unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PerVGPR , NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : Register ( getSubReg ( SuperReg , SplitParts [ i ] ) ) ; MachineInstrBuilder WriteLane = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PerVGPR ) . addReg ( TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ; if ( NumSubRegs > <NUM_LIT> ) { unsigned SuperKillState = <NUM_LIT> ; if ( i + <NUM_LIT> == NumSubRegs ) SuperKillState |= getKillRegState ( IsKill ) ; WriteLane . addReg ( SuperReg , RegState :: Implicit | SuperKillState ) ; } } buildSGPRSpillLoadStore ( MI , Index , Offset , EltSize , TmpVGPR , VGPRLanes , RS , false ) ; } } MI -> eraseFromParent ( ) ;

MFI -> addToSpilledSGPRs ( NumSubRegs ) ;


<s> switch ( BitWidth ) { case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> :


<s> case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; default :


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getSGPRClassForBitWidth ( unsigned BitWidth ) { switch ( BitWidth ) { case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; default :


<s> return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> : return & <STR_LIT> ; case <NUM_LIT> :


<s> } Register TmpReg ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i != e ; ++ i , Offset += EltSize ) { Register SubReg = NumSubRegs == <NUM_LIT> ? Register ( ValueReg ) : getSubReg ( ValueReg , getSubRegFromChannel ( i ) ) ; unsigned SOffsetRegState = <NUM_LIT> ; unsigned SrcDstRegState = getDefRegState ( ! IsStore ) ; if ( i + <NUM_LIT> == e ) { SOffsetRegState |= getKillRegState ( Scavenged ) ; SrcDstRegState |= getKillRegState ( IsKill ) ; } auto MIB = spillVGPRtoAGPR ( ST , MI , Index , i , SubReg , IsKill ) ; if ( ! MIB . getInstr ( ) ) { unsigned FinalReg = SubReg ; if ( hasAGPRs ( RC ) ) { if ( ! TmpReg ) { assert ( RS && <STR_LIT> ) ; TmpReg = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpReg ) ; } if ( IsStore ) BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SubReg , getKillRegState ( IsKill ) ) ; SubReg = TmpReg ; } MachinePointerInfo PInfo = BasePtrInfo . getWithOffset ( EltSize * i ) ; MachineMemOperand * NewMMO = MF -> getMachineMemOperand ( PInfo , MMO -> getFlags ( ) , EltSize , commonAlignment ( Alignment , EltSize * i ) ) ; MIB = BuildMI ( * MBB , MI , DL , Desc ) . addReg ( SubReg , getDefRegState ( ! IsStore ) | getKillRegState ( IsKill ) ) . addReg ( ScratchRsrcReg ) ; if ( SOffset == <STR_LIT> ) { MIB . addImm ( <NUM_LIT> ) ; } else {

MIB . addReg ( SOffset , SOffsetRegState ) ;


<s> <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; DenseSet < Register > SGPRSpillVGPRDefinedSet ; ArrayRef < <ISA_LIT>MachineFunctionInfo :: SpilledReg > VGPRSpills = MFI -> getSGPRToVGPRSpills ( Index ) ; bool SpillToVGPR = ! VGPRSpills . empty ( ) ; if ( OnlyToVGPR && ! SpillToVGPR ) return false ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; Register SuperReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool IsKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; const DebugLoc & DL = MI -> getDebugLoc ( ) ; assert ( SpillToVGPR || ( SuperReg != MFI -> getStackPtrOffsetReg ( ) && SuperReg != MFI -> getFrameOffsetReg ( ) ) ) ; assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ; assert ( SuperReg != <STR_LIT> && SuperReg != <STR_LIT> && SuperReg != <STR_LIT> && <STR_LIT> ) ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; if ( SpillToVGPR ) { for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; bool VGPRDefined = true ;

if ( MBB == & MF -> front ( ) ) VGPRDefined = ! SGPRSpillVGPRDefinedSet . insert ( Spill . VGPR ) . second ;


<s> bool VGPRDefined = true ; if ( MBB == & MF -> front ( ) ) VGPRDefined = ! SGPRSpillVGPRDefinedSet . insert ( Spill . VGPR ) . second ; auto MIB = BuildMI ( * MBB , MI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , Spill . VGPR ) . addReg ( SubReg , getKillRegState ( IsKill ) ) . addImm ( Spill . Lane ) . addReg ( Spill . VGPR , VGPRDefined ? <NUM_LIT> : RegState :: Undef ) ; if ( i == <NUM_LIT> && NumSubRegs > <NUM_LIT> ) { MIB . addReg ( SuperReg , RegState :: ImplicitDefine ) ; } } } else { Register TmpVGPR = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpVGPR ) ; unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ; unsigned PerVGPR = <NUM_LIT> ; unsigned NumVGPRs = ( NumSubRegs + ( PerVGPR - <NUM_LIT> ) ) / PerVGPR ; int64_t VGPRLanes = ( <NUM_LIT> << std :: min ( PerVGPR , NumSubRegs ) ) - <NUM_LIT> ; for ( unsigned Offset = <NUM_LIT> ; Offset < NumVGPRs ; ++ Offset ) { unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PerVGPR , NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; MachineInstrBuilder WriteLane = BuildMI ( * MBB , MI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PerVGPR ) . addReg ( TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ; if ( NumSubRegs > <NUM_LIT> ) { unsigned SuperKillState = <NUM_LIT> ; if ( i + <NUM_LIT> == NumSubRegs ) SuperKillState |= getKillRegState ( IsKill ) ; WriteLane . addReg ( SuperReg , RegState :: Implicit | SuperKillState ) ; } } buildSGPRSpillLoadStore ( MI , Index , Offset , EltSize , TmpVGPR , VGPRLanes , RS , false ) ; }


<s> unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; if ( SpillToVGPR ) { for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; bool VGPRDefined = true ; if ( MBB == & MF -> front ( ) ) VGPRDefined = ! SGPRSpillVGPRDefinedSet . insert ( Spill . VGPR ) . second ; BuildMI ( * MBB , MI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , Spill . VGPR ) . addReg ( SubReg , getKillRegState ( IsKill ) ) . addImm ( Spill . Lane ) . addReg ( Spill . VGPR , VGPRDefined ? <NUM_LIT> : RegState :: Undef ) ; } } else { Register TmpVGPR = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpVGPR ) ; unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ; unsigned PerVGPR = <NUM_LIT> ; unsigned NumVGPRs = ( NumSubRegs + ( PerVGPR - <NUM_LIT> ) ) / PerVGPR ; int64_t VGPRLanes = ( <NUM_LIT> << std :: min ( PerVGPR , NumSubRegs ) ) - <NUM_LIT> ; for ( unsigned Offset = <NUM_LIT> ; Offset < NumVGPRs ; ++ Offset ) { unsigned TmpVGPRFlags = RegState :: Undef ; for ( unsigned i = Offset * PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PerVGPR , NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; MachineInstrBuilder WriteLane = BuildMI ( * MBB , MI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , TmpVGPR ) . addReg ( SubReg , SubKillState ) . addImm ( i % PerVGPR ) . addReg ( TmpVGPR , TmpVGPRFlags ) ; TmpVGPRFlags = <NUM_LIT> ; if ( NumSubRegs > <NUM_LIT> ) { unsigned SuperKillState = <NUM_LIT> ; if ( i + <NUM_LIT> == NumSubRegs ) SuperKillState |= getKillRegState ( IsKill ) ; WriteLane . addReg ( SuperReg , RegState :: Implicit | SuperKillState ) ;


<s> bool IsKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; const DebugLoc & DL = MI -> getDebugLoc ( ) ; assert ( SpillToVGPR || ( SuperReg != MFI -> getStackPtrOffsetReg ( ) && SuperReg != MFI -> getFrameOffsetReg ( ) ) ) ; assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ; assert ( SuperReg != <STR_LIT> && SuperReg != <STR_LIT> && SuperReg != <STR_LIT> && <STR_LIT> ) ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; if ( SpillToVGPR ) { for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; bool VGPRDefined = true ; if ( MBB == & MF -> front ( ) ) VGPRDefined = ! SGPRSpillVGPRDefinedSet . insert ( Spill . VGPR ) . second ; BuildMI ( * MBB , MI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , Spill . VGPR ) . addReg ( SubReg , getKillRegState ( IsKill ) ) . addImm ( Spill . Lane ) . addReg ( Spill . VGPR , VGPRDefined ? <NUM_LIT> : RegState :: Undef ) ; } } else { Register TmpVGPR = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpVGPR ) ;

unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ;


<s> } else { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , SOffset ) . addReg ( ScratchOffsetReg ) . addImm ( Offset ) ; } Offset = <NUM_LIT> ; } for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i != e ; ++ i , Offset += EltSize ) { Register SubReg = NumSubRegs == <NUM_LIT> ? Register ( ValueReg ) : getSubReg ( ValueReg , getSubRegFromChannel ( i ) ) ; unsigned SOffsetRegState = <NUM_LIT> ; unsigned SrcDstRegState = getDefRegState ( ! IsStore ) ; if ( i + <NUM_LIT> == e ) { SOffsetRegState |= getKillRegState ( Scavenged ) ; SrcDstRegState |= getKillRegState ( IsKill ) ; } auto MIB = spillVGPRtoAGPR ( ST , MI , Index , i , SubReg , IsKill ) ; if ( ! MIB . getInstr ( ) ) { unsigned FinalReg = SubReg ; if ( TmpReg != <STR_LIT> ) { if ( IsStore ) BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SubReg , getKillRegState ( IsKill ) ) ; SubReg = TmpReg ; } MachinePointerInfo PInfo = BasePtrInfo . getWithOffset ( EltSize * i ) ; MachineMemOperand * NewMMO = MF -> getMachineMemOperand ( PInfo , MMO -> getFlags ( ) , EltSize , MinAlign ( Align , EltSize * i ) ) ; MIB = BuildMI ( * MBB , MI , DL , Desc ) . addReg ( SubReg , getDefRegState ( ! IsStore ) | getKillRegState ( IsKill ) ) . addReg ( ScratchRsrcReg ) ; if ( SOffset == <STR_LIT> ) { MIB . addImm ( <NUM_LIT> ) ;


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; if ( isWave32 ) { Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; } unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumVGPRs ; i < TotalNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ;


<s> <ISA_LIT>RegisterInfo :: <ISA_LIT>RegisterInfo ( const GCNSubtarget & ST ) : <ISA_LIT>GenRegisterInfo ( <NUM_LIT> ) , ST ( ST ) , SpillSGPRToVGPR ( EnableSpillSGPRToVGPR ) , isWave32 ( ST . isWave32 ( ) ) {


<s> const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; Register TmpVGPR ; unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; if ( SpillToVGPR ) { <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; bool VGPRDefined = true ; if ( MBB == & MF -> front ( ) ) VGPRDefined = ! SGPRSpillVGPRDefinedSet . insert ( Spill . VGPR ) . second ; BuildMI ( * MBB , MI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , Spill . VGPR ) . addReg ( SubReg , getKillRegState ( IsKill ) ) . addImm ( Spill . Lane ) . addReg ( Spill . VGPR , VGPRDefined ? <NUM_LIT> : RegState :: Undef ) ; } else { if ( OnlyToVGPR ) return false ; if ( ! TmpVGPR . isValid ( ) ) TmpVGPR = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; MachineInstrBuilder Mov = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpVGPR ) . addReg ( SubReg , SubKillState ) ; if ( NumSubRegs > <NUM_LIT> ) { unsigned SuperKillState = <NUM_LIT> ; if ( i + <NUM_LIT> == e ) SuperKillState |= getKillRegState ( IsKill ) ; Mov . addReg ( SuperReg , RegState :: Implicit | SuperKillState ) ; } unsigned Align = FrameInfo . getObjectAlignment ( Index ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , Index , EltSize * i ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , EltSize , MinAlign ( Align , EltSize * i ) ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addReg ( TmpVGPR , RegState :: Kill ) . addFrameIndex ( Index ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( MFI -> getStackPtrOffsetReg ( ) ) . addImm ( i * <NUM_LIT> ) . addMemOperand ( MMO ) ; } } MI -> eraseFromParent ( ) ; MFI -> addToSpilledSGPRs ( NumSubRegs ) ; return true ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned BaseIdx = alignDown ( ST . getMaxNumSGPRs ( MF ) , <NUM_LIT> ) - <NUM_LIT> ;

unsigned BaseReg ( <STR_LIT> . getRegister ( BaseIdx ) ) ;


<s> unsigned BaseReg ( <STR_LIT> . getRegister ( BaseIdx ) ) ;

return getMatchingSuperReg ( BaseReg , <STR_LIT> , & <STR_LIT> ) ;


<s> if ( MO . isFI ( ) ) { if ( SeenFI ) llvm_unreachable ( <STR_LIT> ) ; SeenFI = true ; } } MachineOperand * FIOp = TII -> getNamedOperand ( MI , <STR_LIT> :: vaddr ) ; MachineBasicBlock * MBB = MI . getParent ( ) ; MachineFunction * MF = MBB -> getParent ( ) ; assert ( FIOp && FIOp -> isFI ( ) && <STR_LIT> ) ; assert ( TII -> isMUBUF ( MI ) ) ; MachineOperand * SOffset = TII -> getNamedOperand ( MI , <STR_LIT> :: soffset ) ; assert ( SOffset -> getReg ( ) == MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> getStackPtrOffsetReg ( ) && <STR_LIT> ) ; MachineOperand * OffsetOp = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) ; int64_t NewOffset = OffsetOp -> getImm ( ) + Offset ; assert ( isUInt < <NUM_LIT> > ( NewOffset ) && <STR_LIT> ) ; FIOp -> ChangeToRegister ( BaseReg , false ) ; OffsetOp -> setImm ( NewOffset ) ; SOffset -> ChangeToImmediate ( <NUM_LIT> ) ;


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getRegClassForSizeOnBank ( unsigned Size , const RegisterBank & RB , const MachineRegisterInfo & MRI ) const { switch ( RB . getID ( ) ) { case <STR_LIT> : return getVGPRClassForBitWidth ( std :: max ( <NUM_LIT> , Size ) ) ; case <STR_LIT> : assert ( Size == <NUM_LIT> ) ; return isWave32 ? & <STR_LIT> : & <STR_LIT> ; case <STR_LIT> : return getSGPRClassForBitWidth ( std :: max ( <NUM_LIT> , Size ) ) ; default : llvm_unreachable ( <STR_LIT> ) ;


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; if ( isWave32 ) { Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; } unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumVGPRs ; i < TotalNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } for ( auto Reg : <STR_LIT> ) { Reserved . set ( getSubReg ( Reg , <STR_LIT> ) ) ; Register Low = getSubReg ( Reg , <STR_LIT> ) ; if ( ! <STR_LIT> . contains ( Low ) ) Reserved . set ( Low ) ; } for ( auto Reg : <STR_LIT> ) { Reserved . set ( getSubReg ( Reg , <STR_LIT> ) ) ; } if ( ! ST . hasMAIInsts ( ) ) { for ( unsigned i = <NUM_LIT> ; i < MaxNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } }

const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ; const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF -> getSubtarget ( ) . getInstrInfo ( ) ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; bool IsStore = TII -> get ( LoadStoreOp ) . mayStore ( ) ; bool RanOutOfSGPRs = false ; bool Scavenged = false ; unsigned SOffset = ScratchOffset ; unsigned OriginalImmOffset = Offset ; unsigned NumSubRegs = getNumSubRegsForSpillOp ( MI -> getOpcode ( ) ) ; unsigned Size = NumSubRegs * <NUM_LIT> ; if ( ! isUInt < <NUM_LIT> > ( Offset + Size ) ) { SOffset = <STR_LIT> ; if ( RS ) SOffset = RS -> FindUnusedReg ( & <STR_LIT> ) ; if ( SOffset == <STR_LIT> ) { RanOutOfSGPRs = true ; SOffset = ScratchOffset ; } else { Scavenged = true ; } BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , SOffset ) . addReg ( ScratchOffset ) . addImm ( Offset ) ; Offset = <NUM_LIT> ; } for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i != e ; ++ i , Offset += <NUM_LIT> ) {

unsigned SubReg = NumSubRegs > <NUM_LIT> ? getPhysRegSubReg ( Value , & <STR_LIT> , i ) :


<s> } else { unsigned Align = FrameInfo -> getObjectAlignment ( Index ) ; unsigned Size = FrameInfo -> getObjectSize ( Index ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , Index ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , Size , Align ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addFrameIndex ( Index ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( MFI -> getScratchWaveOffsetReg ( ) ) . addImm ( i * <NUM_LIT> ) . addMemOperand ( MMO ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , SubReg ) . addReg ( TmpReg , RegState :: Kill ) . addReg ( MI -> getOperand ( <NUM_LIT> ) . getReg ( ) , RegState :: ImplicitDefine ) ; } } MI -> eraseFromParent ( ) ; break ; } case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : buildScratchLoadStore ( MI , <STR_LIT> , TII -> getNamedOperand ( * MI , <STR_LIT> :: src ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: scratch_rsrc ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: scratch_offset ) -> getReg ( ) , FrameInfo -> getObjectOffset ( Index ) + TII -> getNamedOperand ( * MI , <STR_LIT> :: offset ) -> getImm ( ) , RS ) ; MI -> eraseFromParent ( ) ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { buildScratchLoadStore ( MI , <STR_LIT> , TII -> getNamedOperand ( * MI , <STR_LIT> :: dst ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: scratch_rsrc ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: scratch_offset ) -> getReg ( ) , FrameInfo -> getObjectOffset ( Index ) + TII -> getNamedOperand ( * MI , <STR_LIT> :: offset ) -> getImm ( ) , RS ) ; MI -> eraseFromParent ( ) ; break ; } default : { int64_t Offset = FrameInfo -> getObjectOffset ( Index ) ; FIOp . ChangeToImmediate ( Offset ) ;

if ( ! TII -> isImmOperandLegal ( MI , FIOperandNum , FIOp ) ) {


<s> const <ISA_LIT>MachineFunctionInfo & MFI = * MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; unsigned <ISA_LIT>MDPerCU = <NUM_LIT> ; unsigned MaxInvocationsPerWave = <ISA_LIT>MDPerCU * ST . getWavefrontSize ( ) ;

return alignTo ( MFI . getMaximumWorkGroupSize ( MF ) , MaxInvocationsPerWave ) / MaxInvocationsPerWave ;


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; unsigned MaxWaveCountPer<ISA_LIT>MD = getMaxWaveCountPer<ISA_LIT>MD ( MF ) ; unsigned TotalSGPRCountPer<ISA_LIT>MD , AddressableSGPRCount , SGPRUsageAlignment ; unsigned ReservedSGPRCount ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { TotalSGPRCountPer<ISA_LIT>MD = <NUM_LIT> ; AddressableSGPRCount = <NUM_LIT> ; SGPRUsageAlignment = <NUM_LIT> ; ReservedSGPRCount = <NUM_LIT> ; } else { TotalSGPRCountPer<ISA_LIT>MD = <NUM_LIT> ; AddressableSGPRCount = <NUM_LIT> ; SGPRUsageAlignment = <NUM_LIT> ; ReservedSGPRCount = <NUM_LIT> ; }

unsigned MaxSGPRCount = ( TotalSGPRCountPer<ISA_LIT>MD / MaxWaveCountPer<ISA_LIT>MD ) ;


<s> unsigned SGPRLimit = getNumSGPRsAllowed ( STI . getGeneration ( ) , STI . getMaxWavesPerCU ( ) ) ; unsigned VGPRLimit = getNumVGPRsAllowed ( STI . getMaxWavesPerCU ( ) ) ; unsigned VSLimit = SGPRLimit + VGPRLimit ; if ( SGPRPressureSets . test ( Idx ) && VGPRPressureSets . test ( Idx ) ) { return VSLimit ; } if ( SGPRPressureSets . test ( Idx ) ) return SGPRLimit ; return VGPRLimit ;


<s> if ( SGPRPressureSets . test ( Idx ) && VGPRPressureSets . test ( Idx ) ) { return VSLimit ; } if ( SGPRPressureSets . test ( Idx ) ) return SGPRLimit ;


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; unsigned MaxWorkGroupSGPRCount = getMaxWorkGroupSGPRCount ( MF ) ; unsigned MaxWorkGroupVGPRCount = getMaxWorkGroupVGPRCount ( MF ) ; unsigned NumSGPRs = <STR_LIT> . getNumRegs ( ) ; unsigned NumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxWorkGroupSGPRCount ; i < NumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } for ( unsigned i = MaxWorkGroupVGPRCount ; i < NumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; if ( ScratchWaveOffsetReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchWaveOffsetReg ) ; } unsigned ScratchRSrcReg = MFI -> getScratchRSrcReg ( ) ; if ( ScratchRSrcReg != <STR_LIT> ) {

reserveRegisterTuples ( Reserved , ScratchRSrcReg ) ;


<s> } for ( unsigned i = MaxWorkGroupVGPRCount ; i < NumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; if ( ScratchWaveOffsetReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchWaveOffsetReg ) ; } unsigned ScratchRSrcReg = MFI -> getScratchRSrcReg ( ) ; if ( ScratchRSrcReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchRSrcReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , ScratchWaveOffsetReg ) ) ; } const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ST . debuggerReserveRegs ( ) ) { unsigned ReservedVGPRFirst = MaxWorkGroupVGPRCount - MFI -> getDebuggerReservedVGPRCount ( ) ; for ( unsigned i = ReservedVGPRFirst ; i < MaxWorkGroupVGPRCount ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } } return Reserved ;


<s> MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; unsigned UnusedCarry = MRI . createVirtualRegister ( & <STR_LIT> ) ;

BuildMI ( * MBB , Ins , DL , TII -> get ( <STR_LIT> ) , BaseReg ) . addReg ( UnusedCarry , RegState :: Define | RegState :: Dead ) . addImm ( Offset ) . addFrameIndex ( FrameIdx ) ;


<s> assert ( isUInt < <NUM_LIT> > ( Offset ) && <STR_LIT> ) ; if ( Offset == <NUM_LIT> ) { BuildMI ( * MBB , Ins , DL , TII -> get ( <STR_LIT> ) , BaseReg ) . addFrameIndex ( FrameIdx ) ; return ; } MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; unsigned UnusedCarry = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , Ins , DL , TII -> get ( <STR_LIT> ) , BaseReg ) . addReg ( UnusedCarry , RegState :: Define | RegState :: Dead ) . addImm ( Offset ) . addFrameIndex ( FrameIdx ) ;


<s> if ( SeenFI ) llvm_unreachable ( <STR_LIT> ) ; SeenFI = true ; } } MachineOperand * FIOp = TII -> getNamedOperand ( MI , <STR_LIT> :: vaddr ) ; assert ( FIOp && FIOp -> isFI ( ) && <STR_LIT> ) ; assert ( TII -> isMUBUF ( MI ) ) ; MachineOperand * OffsetOp = TII -> getNamedOperand ( MI , <STR_LIT> :: offset ) ; int64_t NewOffset = OffsetOp -> getImm ( ) + Offset ; if ( isUInt < <NUM_LIT> > ( NewOffset ) ) {

FIOp -> ChangeToRegister ( BaseReg , false ) ;


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; if ( isWave32 ) { Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; } unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } for ( auto Reg : <STR_LIT> ) { Reserved . set ( getSubReg ( Reg , <STR_LIT> ) ) ; Register Low = getSubReg ( Reg , <STR_LIT> ) ; if ( ! <STR_LIT> . contains ( Low ) ) Reserved . set ( Low ) ; } Register ScratchRSrcReg = MFI -> getScratchRSrcReg ( ) ; if ( ScratchRSrcReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchRSrcReg ) ; } MCRegister StackPtrReg = MFI -> getStackPtrOffsetReg ( ) ; if ( StackPtrReg ) { reserveRegisterTuples ( Reserved , StackPtrReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , StackPtrReg ) ) ; } MCRegister FrameReg = MFI -> getFrameOffsetReg ( ) ; if ( FrameReg ) { reserveRegisterTuples ( Reserved , FrameReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , FrameReg ) ) ; } if ( hasBasePointer ( MF ) ) { MCRegister BasePtrReg = getBaseRegister ( ) ; reserveRegisterTuples ( Reserved , BasePtrReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , BasePtrReg ) ) ;


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: VOLCANIC_ISLANDS ) { reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; } if ( ST . hasSGPRInitBug ( ) ) { unsigned NumSGPRs = <STR_LIT> . getNumRegs ( ) ; unsigned Limit = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> ; for ( unsigned i = Limit ; i < NumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ;


<s> } if ( ST . hasSGPRInitBug ( ) ) { unsigned NumSGPRs = <STR_LIT> . getNumRegs ( ) ; unsigned Limit = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> ; for ( unsigned i = Limit ; i < NumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ;


<s> case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : { const MachineOperand * VData = TII -> getNamedOperand ( * MI , <STR_LIT> :: vdata ) ; buildSpillLoadStore ( MI , <STR_LIT> , Index , VData -> getReg ( ) , VData -> isKill ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: srsrc ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: soffset ) -> getReg ( ) , TII -> getNamedOperand ( * MI , <STR_LIT> :: offset ) -> getImm ( ) , * MI -> memoperands_begin ( ) , RS ) ; MI -> eraseFromParent ( ) ; break ; } default : { const DebugLoc & DL = MI -> getDebugLoc ( ) ; bool IsMUBUF = TII -> isMUBUF ( * MI ) ; if ( ! IsMUBUF && MFI -> getFrameOffsetReg ( ) != MFI -> getScratchWaveOffsetReg ( ) ) { unsigned DiffReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; bool IsCopy = MI -> getOpcode ( ) == <STR_LIT> ; unsigned ResultReg = IsCopy ? MI -> getOperand ( <NUM_LIT> ) . getReg ( ) : MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , DiffReg ) . addReg ( MFI -> getFrameOffsetReg ( ) ) . addReg ( MFI -> getScratchWaveOffsetReg ( ) ) ; int64_t Offset = FrameInfo . getObjectOffset ( Index ) ; if ( Offset == <NUM_LIT> ) { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , ResultReg ) . addImm ( Log2_32 ( ST . getWavefrontSize ( ) ) ) . addReg ( DiffReg ) ; } else { unsigned ScaledReg = MRI . createVirtualRegister ( & <STR_LIT> ) ;

BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , ScaledReg ) . addImm ( Log2_32 ( ST . getWavefrontSize ( ) ) ) . addReg ( DiffReg , RegState :: Kill ) ;


<s> case <NUM_LIT> : return RB -> getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB -> getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB -> getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB -> getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB -> getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB -> getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; case <NUM_LIT> : return RB -> getID ( ) == <STR_LIT> ? & <STR_LIT> : & <STR_LIT> ; default : llvm_unreachable ( <STR_LIT> ) ;


<s> unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumVGPRs ; i < TotalNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; if ( ScratchWaveOffsetReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchWaveOffsetReg ) ; } unsigned ScratchRSrcReg = MFI -> getScratchRSrcReg ( ) ; if ( ScratchRSrcReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchRSrcReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , ScratchWaveOffsetReg ) ) ; } unsigned StackPtrReg = MFI -> getStackPtrOffsetReg ( ) ; if ( StackPtrReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , StackPtrReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , StackPtrReg ) ) ; } unsigned FrameReg = MFI -> getFrameOffsetReg ( ) ; if ( FrameReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , FrameReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , FrameReg ) ) ;


<s> if ( SpillToSMEM && OnlyToVGPR ) return false ; assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ; unsigned OffsetReg = <STR_LIT> ; unsigned M0CopyReg = <STR_LIT> ; if ( SpillToSMEM ) { if ( RS -> isRegUsed ( <STR_LIT> ) ) { M0CopyReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , M0CopyReg ) . addReg ( <STR_LIT> ) ; } } unsigned EltSize = <NUM_LIT> ; unsigned ScalarLoadOp ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; if ( SpillToSMEM && isSGPRClass ( RC ) ) { std :: tie ( EltSize , ScalarLoadOp ) = getSpillEltSize ( getRegSizeInBits ( * RC ) / <NUM_LIT> , false ) ; } ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; int64_t FrOffset = FrameInfo . getObjectOffset ( Index ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { unsigned SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; if ( SpillToSMEM ) { unsigned Align = FrameInfo . getObjectAlignment ( Index ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , Index , EltSize * i ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , EltSize , MinAlign ( Align , EltSize * i ) ) ; int64_t Offset = ( ST . getWavefrontSize ( ) * FrOffset ) + ( EltSize * i ) ; if ( Offset != <NUM_LIT> ) { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , OffsetReg ) . addReg ( MFI -> getFrameOffsetReg ( ) ) . addImm ( Offset ) ; } else { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , OffsetReg ) . addReg ( MFI -> getFrameOffsetReg ( ) ) ; }

auto MIB = BuildMI ( * MBB , MI , DL , TII -> get ( ScalarLoadOp ) , SubReg ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( OffsetReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> bool <ISA_LIT>RegisterInfo :: spillSGPR ( MachineBasicBlock :: iterator MI , int Index , RegScavenger * RS , bool OnlyToVGPR ) const { MachineBasicBlock * MBB = MI -> getParent ( ) ; MachineFunction * MF = MBB -> getParent ( ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; DenseSet < unsigned > SGPRSpillVGPRDefinedSet ; ArrayRef < <ISA_LIT>MachineFunctionInfo :: SpilledReg > VGPRSpills = MFI -> getSGPRToVGPRSpills ( Index ) ; bool SpillToVGPR = ! VGPRSpills . empty ( ) ; if ( OnlyToVGPR && ! SpillToVGPR ) return false ; MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; const GCNSubtarget & ST = MF -> getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; unsigned SuperReg = MI -> getOperand ( <NUM_LIT> ) . getReg ( ) ; bool IsKill = MI -> getOperand ( <NUM_LIT> ) . isKill ( ) ; const DebugLoc & DL = MI -> getDebugLoc ( ) ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; bool SpillToSMEM = spillSGPRToSMEM ( ) ; if ( SpillToSMEM && OnlyToVGPR ) return false ; assert ( SpillToVGPR || ( SuperReg != MFI -> getStackPtrOffsetReg ( ) && SuperReg != MFI -> getFrameOffsetReg ( ) && SuperReg != MFI -> getScratchWaveOffsetReg ( ) ) ) ; assert ( SuperReg != <STR_LIT> && <STR_LIT> ) ; unsigned OffsetReg = <STR_LIT> ; unsigned M0CopyReg = <STR_LIT> ; if ( SpillToSMEM ) { if ( RS -> isRegUsed ( <STR_LIT> ) ) { M0CopyReg = MRI . createVirtualRegister ( & <STR_LIT> ) ; BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , M0CopyReg ) . addReg ( <STR_LIT> ) ; } } unsigned ScalarStoreOp ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ;

if ( SpillToSMEM && isSGPRClass ( RC ) ) {


<s> } unsigned ScalarStoreOp ; unsigned EltSize = <NUM_LIT> ; const TargetRegisterClass * RC = getPhysRegClass ( SuperReg ) ; if ( SpillToSMEM && isSGPRClass ( RC ) ) { std :: tie ( EltSize , ScalarStoreOp ) = getSpillEltSize ( getRegSizeInBits ( * RC ) / <NUM_LIT> , true ) ; } ArrayRef < int16_t > SplitParts = getRegSplitParts ( RC , EltSize ) ; unsigned NumSubRegs = SplitParts . empty ( ) ? <NUM_LIT> : SplitParts . size ( ) ; unsigned SubKillState = getKillRegState ( ( NumSubRegs == <NUM_LIT> ) && IsKill ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i < e ; ++ i ) { unsigned SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; if ( SpillToSMEM ) { int64_t FrOffset = FrameInfo . getObjectOffset ( Index ) ; unsigned Align = FrameInfo . getObjectAlignment ( Index ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , Index , EltSize * i ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , EltSize , MinAlign ( Align , EltSize * i ) ) ; int64_t Offset = ( ST . getWavefrontSize ( ) * FrOffset ) + ( EltSize * i ) ; if ( Offset != <NUM_LIT> ) { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , OffsetReg ) . addReg ( MFI -> getFrameOffsetReg ( ) ) . addImm ( Offset ) ; } else { BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , OffsetReg ) . addReg ( MFI -> getFrameOffsetReg ( ) ) ; } BuildMI ( * MBB , MI , DL , TII -> get ( ScalarStoreOp ) ) . addReg ( SubReg , getKillRegState ( IsKill ) ) . addReg ( MFI -> getScratchRSrcReg ( ) ) . addReg ( OffsetReg , RegState :: Kill ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; continue ; } if ( SpillToVGPR ) { <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill = VGPRSpills [ i ] ; bool VGPRDefined = true ; if ( MBB == & MF -> front ( ) ) VGPRDefined = ! SGPRSpillVGPRDefinedSet . insert ( Spill . VGPR ) . second ; BuildMI ( * MBB , MI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , Spill . VGPR ) . addReg ( SubReg , getKillRegState ( IsKill ) ) . addImm ( Spill . Lane ) . addReg ( Spill . VGPR , VGPRDefined ? <NUM_LIT> : RegState :: Undef ) ; } else { if ( OnlyToVGPR ) return false ; unsigned TmpReg = MRI . createVirtualRegister ( & <STR_LIT> ) ;

MachineInstrBuilder Mov = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SubReg , SubKillState ) ;


<s> MachineBasicBlock * MBB = MI -> getParent ( ) ; const MachineFunction * MF = MI -> getParent ( ) -> getParent ( ) ; const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF -> getSubtarget ( ) . getInstrInfo ( ) ) ; LLVMContext & Ctx = MF -> getFunction ( ) -> getContext ( ) ; DebugLoc DL = MI -> getDebugLoc ( ) ; bool IsLoad = TII -> get ( LoadStoreOp ) . mayLoad ( ) ; bool RanOutOfSGPRs = false ; unsigned SOffset = ScratchOffset ; unsigned NumSubRegs = getNumSubRegsForSpillOp ( MI -> getOpcode ( ) ) ; unsigned Size = NumSubRegs * <NUM_LIT> ; if ( ! isUInt < <NUM_LIT> > ( Offset + Size ) ) { SOffset = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; if ( SOffset == <STR_LIT> ) { RanOutOfSGPRs = true ; SOffset = <STR_LIT> ; } BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , SOffset ) . addReg ( ScratchOffset ) . addImm ( Offset ) ; Offset = <NUM_LIT> ; } if ( RanOutOfSGPRs ) Ctx . emitError ( <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = NumSubRegs ; i != e ; ++ i , Offset += <NUM_LIT> ) {


if ( ! MRI . isPhysRegUsed ( * I ) ) return * I ;


<s> unsigned <ISA_LIT>RegisterInfo :: findUnusedRegister ( const MachineRegisterInfo & MRI , const TargetRegisterClass * RC ) const { for ( TargetRegisterClass :: iterator I = RC -> begin ( ) , E = RC -> end ( ) ; I != E ; ++ I ) { if ( ! MRI . isPhysRegUsed ( * I ) ) return * I ; }


<s> const TargetRegisterClass * <ISA_LIT>RegisterInfo :: getCFGStructurizerRegClass ( MVT VT ) const {


<s> return SRC ; } else if ( SRC == & <STR_LIT> ) { return & <STR_LIT> ; } else if ( getCommonSubClass ( SRC , & <STR_LIT> ) ) { return & <STR_LIT> ; } else if ( getCommonSubClass ( SRC , & <STR_LIT> ) ) { return & <STR_LIT> ;


<s> BitVector Reserved ( getNumRegs ( ) ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; Reserved . set ( <STR_LIT> ) ; if ( MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . hasSGPRInitBug ( ) ) { unsigned NumSGPRs = <STR_LIT> . getNumRegs ( ) ; unsigned Limit = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> ; for ( unsigned i = Limit ; i < NumSGPRs ; ++ i ) {

unsigned Reg = <STR_LIT> . getRegister ( i ) ;


<s> Reserved . set ( <STR_LIT> ) ; if ( MF . getSubtarget < <ISA_LIT>Subtarget > ( ) . hasSGPRInitBug ( ) ) { unsigned NumSGPRs = <STR_LIT> . getNumRegs ( ) ; unsigned Limit = <ISA_LIT>Subtarget :: FIXED_SGPR_COUNT_FOR_INIT_BUG - <NUM_LIT> ; for ( unsigned i = Limit ; i < NumSGPRs ; ++ i ) {


<s> return getCommonSubClass ( & <STR_LIT> , RC ) || getCommonSubClass ( & <STR_LIT> , RC ) || getCommonSubClass ( & <STR_LIT> , RC ) || getCommonSubClass ( & <STR_LIT> , RC ) || getCommonSubClass ( & <STR_LIT> , RC ) || getCommonSubClass ( & <STR_LIT> , RC ) ;


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumVGPRs ; i < TotalNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ;


<s> NeedSuperRegImpOperand = true ; unsigned State = SrcDstRegState ; if ( Lane != LaneE ) State &= ~ RegState :: Kill ; MIB . addReg ( ValueReg , RegState :: Implicit | State ) ; } RemEltSize -= <NUM_LIT> ; } if ( ! RemEltSize ) continue ; if ( RemEltSize != EltSize ) { assert ( IsFlat && EltSize > <NUM_LIT> ) ; unsigned NumRegs = RemEltSize / <NUM_LIT> ; SubReg = Register ( getSubReg ( ValueReg , getSubRegFromChannel ( RegOffset / <NUM_LIT> , NumRegs ) ) ) ; unsigned Opc = getFlatScratchSpillOpcode ( TII , LoadStoreOp , RemEltSize ) ; Desc = & TII -> get ( Opc ) ; } unsigned FinalReg = SubReg ; if ( IsAGPR ) { assert ( EltSize == <NUM_LIT> ) ; if ( ! TmpReg ) { assert ( RS && <STR_LIT> ) ; TmpReg = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpReg ) ; } if ( IsStore ) { auto AccRead = BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SubReg , getKillRegState ( IsKill ) ) ; if ( NeedSuperRegDef ) AccRead . addReg ( ValueReg , RegState :: ImplicitDefine ) ; AccRead -> setAsmPrinterFlag ( MachineInstr :: ReloadReuse ) ; } SubReg = TmpReg ; } MachinePointerInfo PInfo = BasePtrInfo . getWithOffset ( RegOffset ) ; MachineMemOperand * NewMMO = MF -> getMachineMemOperand ( PInfo , MMO -> getFlags ( ) , RemEltSize , commonAlignment ( Alignment , RegOffset ) ) ; auto MIB = BuildMI ( MBB , MI , DL , * Desc ) . addReg ( SubReg , getDefRegState ( ! IsStore ) | getKillRegState ( IsKill ) ) ; if ( ! IsFlat ) MIB . addReg ( FuncInfo -> getScratchRSrcReg ( ) ) ; if ( SOffset == <STR_LIT> ) { if ( ! IsFlat ) MIB . addImm ( <NUM_LIT> ) ; } else { MIB . addReg ( SOffset , SOffsetRegState ) ; } MIB . addImm ( Offset + RegOffset ) . addImm ( <NUM_LIT> ) ; if ( ! IsFlat ) MIB . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; MIB . addMemOperand ( NewMMO ) ; if ( ! IsAGPR && NeedSuperRegDef ) MIB . addReg ( ValueReg , RegState :: ImplicitDefine ) ; if ( ! IsStore && TmpReg != <STR_LIT> ) { MIB = BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) , FinalReg ) . addReg ( TmpReg , RegState :: Kill ) ; MIB -> setAsmPrinterFlag ( MachineInstr :: ReloadReuse ) ; }

if ( NeedSuperRegImpOperand ) MIB . addReg ( ValueReg , RegState :: Implicit | SrcDstRegState ) ;


<s> if ( RC == & <STR_LIT> || RC == & <STR_LIT> ) return & <STR_LIT> ; if ( RC == & <STR_LIT> || RC == & <STR_LIT> ) return & <STR_LIT> ; if ( RC == & <STR_LIT> || RC == & <STR_LIT> ) return & <STR_LIT> ; if ( RC == & <STR_LIT> || RC == & <STR_LIT> ) return & <STR_LIT> ; if ( RC == & <STR_LIT> || RC == & <STR_LIT> ) return & <STR_LIT> ; if ( RC == & <STR_LIT> || RC == & <STR_LIT> ) return & <STR_LIT> ; if ( RC == & <STR_LIT> || RC == & <STR_LIT> ) return & <STR_LIT> ; if ( RC == & <STR_LIT> || RC == & <STR_LIT> ) return & <STR_LIT> ; } return TargetRegisterInfo :: getLargestLegalSuperClass ( RC , MF ) ;


<s> BuildMI ( * MBB , MI , DL , TII . get ( MovOpc ) , SavedExecReg ) . addReg ( ExecReg ) ; auto I = BuildMI ( * MBB , MI , DL , TII . get ( MovOpc ) , ExecReg ) . addImm ( VGPRLanes ) ; if ( ! TmpVGPRLive ) I . addReg ( TmpVGPR , RegState :: ImplicitDefine ) ; TRI . buildVGPRSpillLoadStore ( * this , TmpVGPRIndex , <NUM_LIT> , false ) ; } else { if ( TmpVGPRLive ) TRI . buildVGPRSpillLoadStore ( * this , TmpVGPRIndex , <NUM_LIT> , false , false ) ; auto I = BuildMI ( * MBB , MI , DL , TII . get ( NotOpc ) , ExecReg ) . addReg ( ExecReg ) ;


<s> } else { TRI . buildVGPRSpillLoadStore ( * this , Index , Offset , IsLoad , false ) ; auto Not0 = BuildMI ( * MBB , MI , DL , TII . get ( NotOpc ) , ExecReg ) . addReg ( ExecReg ) ;

Not0 -> getOperand ( <NUM_LIT> ) . setIsDead ( ) ;


<s> TRI . buildVGPRSpillLoadStore ( * this , TmpVGPRIndex , <NUM_LIT> , true , false ) ; auto I = BuildMI ( * MBB , MI , DL , TII . get ( MovOpc ) , ExecReg ) . addReg ( SavedExecReg , RegState :: Kill ) ; if ( ! TmpVGPRLive ) { I . addReg ( TmpVGPR , RegState :: ImplicitKill ) ; } } else {


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; reserveRegisterTuples ( Reserved , <STR_LIT> ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ;

unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ;


<s> reserveRegisterTuples ( Reserved , <STR_LIT> ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned MaxNumSGPRs = ST . getMaxNumSGPRs ( MF ) ; unsigned TotalNumSGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumSGPRs ; i < TotalNumSGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } unsigned MaxNumVGPRs = ST . getMaxNumVGPRs ( MF ) ; unsigned TotalNumVGPRs = <STR_LIT> . getNumRegs ( ) ; for ( unsigned i = MaxNumVGPRs ; i < TotalNumVGPRs ; ++ i ) { unsigned Reg = <STR_LIT> . getRegister ( i ) ; reserveRegisterTuples ( Reserved , Reg ) ; } const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; if ( ScratchWaveOffsetReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchWaveOffsetReg ) ; } unsigned ScratchRSrcReg = MFI -> getScratchRSrcReg ( ) ; if ( ScratchRSrcReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , ScratchRSrcReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , ScratchWaveOffsetReg ) ) ; } unsigned StackPtrReg = MFI -> getStackPtrOffsetReg ( ) ; if ( StackPtrReg != <STR_LIT> ) { reserveRegisterTuples ( Reserved , StackPtrReg ) ; assert ( ! isSubRegister ( ScratchRSrcReg , StackPtrReg ) ) ; }

unsigned FrameReg = MFI -> getFrameOffsetReg ( ) ;


<s> case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; default : break ; } const TargetRegisterClass * RC = getMinimalPhysRegClass ( Reg ) ; unsigned Size = getRegSizeInBits ( * RC ) ; unsigned AltName = <STR_LIT> ; switch ( Size ) { case <NUM_LIT> : AltName = <STR_LIT> ; break ; case <NUM_LIT> : AltName = <STR_LIT> ; break ; case <NUM_LIT> : AltName = <STR_LIT> ; break ; case <NUM_LIT> : AltName = <STR_LIT> ; break ; case <NUM_LIT> : AltName = <STR_LIT> ; break ; case <NUM_LIT> : AltName = <STR_LIT> ; break ; case <NUM_LIT> : AltName = <STR_LIT> ; break ; case <NUM_LIT> : AltName = <STR_LIT> ; break ;


<s> bool OnlyExecLo = isWave32 || NumSubRegs == <NUM_LIT> || SuperReg == <STR_LIT> ; unsigned ExecMovOpc = OnlyExecLo ? <STR_LIT> : <STR_LIT> ; Register ExecReg = OnlyExecLo ? <STR_LIT> : <STR_LIT> ; Register SavedExecReg ; if ( OnlyExecLo ) { SavedExecReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ FirstPart + ExecLane ] ) ; } else { for ( ; ExecLane < ( NumSubRegs - <NUM_LIT> ) ; ++ ExecLane ) { SavedExecReg = getMatchingSuperReg ( getSubReg ( SuperReg , SplitParts [ FirstPart + ExecLane ] ) , <STR_LIT> , & <STR_LIT> ) ; if ( SavedExecReg ) break ; } } assert ( SavedExecReg ) ; BuildMI ( * MBB , MI , DL , TII -> get ( ExecMovOpc ) , SavedExecReg ) . addReg ( ExecReg ) ; BuildMI ( * MBB , MI , DL , TII -> get ( ExecMovOpc ) , ExecReg ) . addImm ( VGPRLanes ) ; MachineFrameInfo & FrameInfo = MF -> getFrameInfo ( ) ; assert ( FrameInfo . getStackID ( Index ) != TargetStackID :: SGPRSpill ) ; Register FrameReg = FrameInfo . isFixedObjectIndex ( Index ) && hasBasePointer ( * MF ) ? getBaseRegister ( ) : getFrameRegister ( * MF ) ; Align Alignment = FrameInfo . getObjectAlign ( Index ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( * MF , Index ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( PtrInfo , IsLoad ? MachineMemOperand :: MOLoad : MachineMemOperand :: MOStore , EltSize , Alignment ) ; if ( IsLoad ) { unsigned Opc = ST . enableFlatScratch ( ) ? <STR_LIT> : <STR_LIT> ; buildSpillLoadStore ( MI , Opc , Index , VGPR , false , MFI -> getScratchRSrcReg ( ) , FrameReg , Offset * EltSize , MMO , RS ) ; } else { unsigned Opc = ST . enableFlatScratch ( ) ? <STR_LIT> : <STR_LIT> ; buildSpillLoadStore ( MI , Opc , Index , VGPR , IsKill , MFI -> getScratchRSrcReg ( ) , FrameReg , Offset * EltSize , MMO , RS ) ;

MFI -> addToSpilledVGPRs ( <NUM_LIT> ) ;


<s> if ( i + <NUM_LIT> == e ) { SOffsetRegState |= getKillRegState ( Scavenged ) ; SrcDstRegState |= getKillRegState ( IsKill ) ; } const bool NeedSuperRegDef = NumSubRegs > <NUM_LIT> && IsStore && i == <NUM_LIT> ; auto MIB = spillVGPRtoAGPR ( ST , MI , Index , i , SubReg , IsKill ) ; if ( ! MIB . getInstr ( ) ) { unsigned FinalReg = SubReg ; const bool IsAGPR = hasAGPRs ( RC ) ; if ( IsAGPR ) { if ( ! TmpReg ) { assert ( RS && <STR_LIT> ) ; TmpReg = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpReg ) ; } if ( IsStore ) { auto AccRead = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , TmpReg ) . addReg ( SubReg , getKillRegState ( IsKill ) ) ; if ( NeedSuperRegDef ) AccRead . addReg ( ValueReg , RegState :: ImplicitDefine ) ; } SubReg = TmpReg ; } MachinePointerInfo PInfo = BasePtrInfo . getWithOffset ( EltSize * i ) ; MachineMemOperand * NewMMO = MF -> getMachineMemOperand ( PInfo , MMO -> getFlags ( ) , EltSize , commonAlignment ( Alignment , EltSize * i ) ) ; MIB = BuildMI ( * MBB , MI , DL , * Desc ) . addReg ( SubReg , getDefRegState ( ! IsStore ) | getKillRegState ( IsKill ) ) ; if ( ! IsFlat ) MIB . addReg ( ScratchRsrcReg ) ; if ( SOffset == <STR_LIT> ) { if ( ! IsFlat ) MIB . addImm ( <NUM_LIT> ) ; } else { MIB . addReg ( SOffset , SOffsetRegState ) ; } MIB . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; if ( ! IsFlat ) MIB . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; MIB . addMemOperand ( NewMMO ) ; if ( ! IsAGPR && NeedSuperRegDef ) MIB . addReg ( ValueReg , RegState :: ImplicitDefine ) ; if ( ! IsStore && TmpReg != <STR_LIT> ) MIB = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , FinalReg ) . addReg ( TmpReg , RegState :: Kill ) ;


<s> unsigned getChannelFromSubReg ( unsigned SubReg ) const { return SubReg ? divideCeil ( getSubRegIdxOffset ( SubReg ) , <NUM_LIT> ) : <NUM_LIT> ;


<s> } else { Register TmpVGPR = RS -> scavengeRegister ( & <STR_LIT> , MI , <NUM_LIT> ) ; RS -> setRegUsed ( TmpVGPR ) ; unsigned PerVGPR = <NUM_LIT> ; unsigned NumVGPRs = ( NumSubRegs + ( PerVGPR - <NUM_LIT> ) ) / PerVGPR ; int64_t VGPRLanes = ( <NUM_LIT> << std :: min ( PerVGPR , NumSubRegs ) ) - <NUM_LIT> ; for ( unsigned Offset = <NUM_LIT> ; Offset < NumVGPRs ; ++ Offset ) { buildSGPRSpillLoadStore ( MI , Index , Offset , EltSize , TmpVGPR , VGPRLanes , RS , true ) ; for ( unsigned i = Offset * PerVGPR , e = std :: min ( ( Offset + <NUM_LIT> ) * PerVGPR , NumSubRegs ) ; i < e ; ++ i ) { Register SubReg = NumSubRegs == <NUM_LIT> ? SuperReg : getSubReg ( SuperReg , SplitParts [ i ] ) ; bool LastSubReg = ( i + <NUM_LIT> == e ) ; auto MIB = BuildMI ( * MBB , MI , DL , TII -> get ( <STR_LIT> ) , SubReg ) . addReg ( TmpVGPR , getKillRegState ( LastSubReg ) ) . addImm ( i ) ; if ( NumSubRegs > <NUM_LIT> && i == <NUM_LIT> ) MIB . addReg ( SuperReg , RegState :: ImplicitDefine ) ; } } }


<s> RegPressureIgnoredUnits . resize ( getNumRegUnits ( ) ) ; RegPressureIgnoredUnits . set ( * MCRegUnitIterator ( <STR_LIT> , this ) ) ; for ( auto Reg : <STR_LIT> ) RegPressureIgnoredUnits . set ( * MCRegUnitIterator ( Reg , this ) ) ; static llvm :: once_flag InitializeRegSplitPartsFlag ; static auto InitializeRegSplitPartsOnce = [ this ] ( ) { for ( unsigned Idx = <NUM_LIT> , E = getNumSubRegIndices ( ) - <NUM_LIT> ; Idx < E ; ++ Idx ) { unsigned Size = getSubRegIdxSize ( Idx ) ; if ( Size & <NUM_LIT> ) continue ; std :: vector < int16_t > & Vec = RegSplitParts [ Size / <NUM_LIT> - <NUM_LIT> ] ; unsigned Pos = getSubRegIdxOffset ( Idx ) ; if ( Pos % Size ) continue ; Pos /= Size ; if ( Vec . empty ( ) ) { unsigned MaxNumParts = <NUM_LIT> / Size ; Vec . resize ( MaxNumParts ) ; } Vec [ Pos ] = Idx ; } } ;

static llvm :: once_flag InitializeSubRegFromChannelTableFlag ;