<s> } for ( MachineBasicBlock & OtherBB : MF ) { if ( & OtherBB == & MBB ) continue ; if ( OffsetRegUsed || FPAdjusted ) OtherBB . addLiveIn ( ScratchWaveOffsetReg ) ; if ( ResourceRegUsed ) OtherBB . addLiveIn ( ScratchRsrcReg ) ; } DebugLoc DL ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; bool CopyBuffer = ResourceRegUsed && PreloadedPrivateBufferReg != <STR_LIT> && ST . isAmdHsaOrMesa ( F ) && ScratchRsrcReg != PreloadedPrivateBufferReg ; bool CopyBufferFirst = TRI -> isSubRegisterEq ( PreloadedPrivateBufferReg , ScratchWaveOffsetReg ) ; if ( CopyBuffer && CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } unsigned SPReg = MFI -> getStackPtrOffsetReg ( ) ; assert ( SPReg != <STR_LIT> ) ; const bool HasFP = hasFP ( MF ) ; if ( HasFP || OffsetRegUsed ) { assert ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , HasFP ? RegState :: Kill : <NUM_LIT> ) ; } if ( CopyBuffer && ! CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } if ( ResourceRegUsed ) { emitEntryFunctionScratchSetup ( ST , MF , MBB , MFI , I , PreloadedPrivateBufferReg , ScratchRsrcReg ) ; } if ( HasFP ) { DebugLoc DL ; const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; int64_t StackSize = FrameInfo . getStackSize ( ) ; if ( StackSize == <NUM_LIT> ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SPReg ) . addReg ( MFI


<s> assert ( ! ST . isAmdHsaOrMesa ( Fn ) ) ; const MCInstrDesc & SMovB32 = TII -> get ( <STR_LIT> ) ; Register Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; Register Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; if ( MFI -> hasImplicitBufferPtr ( ) ) { Register Rsrc01 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; if ( <STR_LIT> ( MF . getFunction ( ) . getCallingConv ( ) ) ) { const MCInstrDesc & Mov64 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , Mov64 , Rsrc01 ) . addReg ( MFI -> getImplicitBufferPtrUserSGPR ( ) ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } else { const MCInstrDesc & LoadDwordX2 = TII -> get ( <STR_LIT> ) ; PointerType * PtrTy = PointerType :: get ( Type :: getInt64Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; auto MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant | MachineMemOperand :: MODereferenceable , <NUM_LIT> , <NUM_LIT> ) ; BuildMI ( MBB , I , DL , LoadDwordX2 , Rsrc01 ) . addReg ( MFI -> getImplicitBufferPtrUserSGPR ( ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; MF . getRegInfo ( ) . addLiveIn ( MFI -> getImplicitBufferPtrUserSGPR ( ) ) ; MBB . addLiveIn ( MFI -> getImplicitBufferPtrUserSGPR ( ) ) ; } } else { Register Rsrc0 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; Register Rsrc1 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc0 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc1 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } BuildMI ( MBB , I , DL , SMovB32 , Rsrc2 ) . addImm ( Rsrc23 & <NUM_LIT> ) . addReg ( ScratchRsrcReg ,


<s> BuildMI ( MBB , I , DL , SMovB32 , RsrcLo ) . addReg ( GitPtrLo ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; PointerType * PtrTy = PointerType :: get ( Type :: getInt64Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; const MCInstrDesc & LoadDwordX4 = TII -> get ( <STR_LIT> ) ; auto MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant | MachineMemOperand :: MODereferenceable , <NUM_LIT> , <NUM_LIT> ) ; unsigned Offset = Fn . getCallingConv ( ) == CallingConv :: <ISA_LIT>_CS ? <NUM_LIT> : <NUM_LIT> ; const GCNSubtarget & Subtarget = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned EncodedOffset = <STR_LIT> ( Subtarget , Offset ) ; BuildMI ( MBB , I , DL , LoadDwordX4 , ScratchRsrcReg ) . addReg ( Rsrc01 ) . addImm ( EncodedOffset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) . addMemOperand ( MMO ) ; return ; } if ( ST . isMesaGfxShader ( Fn ) || ( PreloadedPrivateBufferReg == <STR_LIT> ) ) { assert ( ! ST . isAmdHsaOrMesa ( Fn ) ) ; const MCInstrDesc & SMovB32 = TII -> get ( <STR_LIT> ) ; Register Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; Register Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; if ( MFI -> hasImplicitBufferPtr ( ) ) { Register Rsrc01 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; if ( <STR_LIT> ( MF . getFunction ( ) . getCallingConv ( ) ) ) { const MCInstrDesc & Mov64 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , Mov64 , Rsrc01 ) . addReg ( MFI -> getImplicitBufferPtrUserSGPR ( ) ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } else { const MCInstrDesc & LoadDwordX2 = TII -> get ( <STR_LIT> ) ; PointerType * PtrTy = PointerType :: get ( Type :: getInt64Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; auto MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant | MachineMemOperand :: MODereferenceable , <NUM_LIT> , <NUM_LIT> ) ; BuildMI ( MBB , I , DL , LoadDwordX2 , Rsrc01 ) . addReg ( MFI -> getImplicitBufferPtrUserSGPR ( ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; MF . getRegInfo ( ) . addLiveIn

( MFI -> getImplicitBufferPtrUserSGPR ( ) ) ;


<s> MRI . addLiveIn ( FlatScratchInitReg ) ; MBB . addLiveIn ( FlatScratchInitReg ) ; Register FlatScrInitLo = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; Register FlatScrInitHi = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; if ( ST . flatScratchIsPointer ( ) ) { if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX10 ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitLo ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitHi ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( FlatScrInitLo ) . addImm ( int16_t ( <STR_LIT> :: ID_FLAT_SCR_LO | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ) ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( FlatScrInitHi ) . addImm ( int16_t ( <STR_LIT> :: ID_FLAT_SCR_HI | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ) ) ; return ; } BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; return ; } assert ( ST . getGeneration ( ) < <ISA_LIT>Subtarget :: GFX10 ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT>

) . addReg ( FlatScrInitHi , RegState :: Kill ) ;


<s> LivePhysRegs LiveRegs ; LiveRegs . init ( * MRI .


<s> assert ( MFI -> isEntryFunction ( ) ) ; if ( ScratchWaveOffsetReg == <STR_LIT> || ( ! hasFP ( MF ) && ! MRI . isPhysRegUsed ( ScratchWaveOffsetReg ) ) ) { return std :: make_pair ( <STR_LIT> , false ) ; } if ( ST . hasSGPRInitBug ( ) ) return std :: make_pair ( ScratchWaveOffsetReg , false ) ; unsigned NumPreloaded = MFI -> getNumPreloadedSGPRs ( ) ; ArrayRef < MCPhysReg > AllSGPRs = getAllSGPRs ( ST , MF ) ; if ( NumPreloaded > AllSGPRs . size ( ) ) return std :: make_pair ( ScratchWaveOffsetReg , false ) ; AllSGPRs = AllSGPRs . slice ( NumPreloaded ) ; unsigned ReservedRegCount = <NUM_LIT> ; if ( AllSGPRs . size ( ) < ReservedRegCount ) return std :: make_pair ( ScratchWaveOffsetReg , false ) ; bool HandledScratchWaveOffsetReg = ScratchWaveOffsetReg != TRI -> reservedPrivateSegmentWaveByteOffsetReg ( MF ) ; bool FPAdjusted = false ; for ( MCPhysReg Reg : AllSGPRs . drop_back ( ReservedRegCount ) ) { if ( ! MRI . isPhysRegUsed ( Reg ) && MRI .


<s> if ( MFI . hasCalls ( ) ) { if ( MFI . getStackSize ( ) != <NUM_LIT> ) return true ; if ( MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> isEntryFunction ( ) ) return true ; } return MFI . hasVarSizedObjects ( ) || MFI . isFrameAddressTaken ( ) || MFI . hasStackMap ( ) || MFI . hasPatchPoint ( ) || MF . getSubtarget < GCNSubtarget >

( ) . getRegisterInfo ( ) -> needsStackRealignment ( MF ) || MF . getTarget ( ) . Options . DisableFramePointerElim ( MF ) ;


<s> } return MFI . hasVarSizedObjects ( ) || MFI . isFrameAddressTaken ( ) || MFI . hasStackMap ( ) || MFI . hasPatchPoint ( ) || MF . getSubtarget < GCNSubtarget > ( ) . getRegisterInfo ( ) -> needsStackRealignment ( MF ) ||

MF . getTarget ( ) . Options . DisableFramePointerElim ( MF ) ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; FuncInfo -> removeDeadFrameIndices ( MFI ) ; assert ( allSGPRSpillsAreDead ( MFI , None ) && <STR_LIT> ) ; if ( ! allStackObjectsAreDead ( MFI ) ) { assert ( RS && <STR_LIT> ) ; if ( FuncInfo -> isEntryFunction ( ) ) { int ScavengeFI = MFI . CreateFixedObject ( TRI -> getSpillSize ( <STR_LIT> ) , <NUM_LIT> , false ) ; RS -> addScavengingFrameIndex ( ScavengeFI ) ; } else


<s> void <ISA_LIT>FrameLowering :: determineCalleeSavesSGPR ( MachineFunction & MF , BitVector & SavedRegs , RegScavenger * RS ) const { TargetFrameLowering :: determineCalleeSaves ( MF , SavedRegs , RS ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( MFI -> isEntryFunction ( ) ) return ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; SavedRegs . reset ( MFI -> getStackPtrOffsetReg ( ) ) ; const BitVector AllSavedRegs = SavedRegs ; SavedRegs . clearBitsInMask ( TRI -> getAllVectorRegMask ( ) ) ; MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; const bool WillHaveFP = FrameInfo . hasCalls ( ) && ( AllSavedRegs . any ( ) || MFI -> hasSpilledSGPRs ( ) ) ; if ( WillHaveFP || hasFP ( MF ) ) SavedRegs . reset ( MFI -> getFrameOffsetReg ( ) ) ; const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; Register RetAddrReg = TRI -> getReturnAddressReg ( MF ) ; if ( ! MFI -> isEntryFunction ( ) && ( FrameInfo . hasCalls

( ) || MRI . isPhysRegModified ( RetAddrReg ) ) ) {


<s> static void buildPrologSpill ( const GCNSubtarget & ST , const <ISA_LIT>RegisterInfo & TRI , const <ISA_LIT>MachineFunctionInfo & FuncInfo , LivePhysRegs & LiveRegs , MachineFunction & MF , MachineBasicBlock & MBB , MachineBasicBlock :: iterator I , Register SpillReg , int FI ) { unsigned Opc = ST . enableFlatScratch ( ) ? <STR_LIT> : <STR_LIT> ; MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( MF , FI ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , FrameInfo . getObjectSize (

FI ) , FrameInfo . getObjectAlign ( FI ) ) ;


<s> Register ScratchWaveOffsetReg ; if ( TRI -> isSubRegisterEq ( ScratchRsrcReg , PreloadedScratchWaveOffsetReg ) ) { ArrayRef < MCPhysReg > AllSGPRs = TRI -> getAllSGPR32 ( MF ) ; unsigned NumPreloaded = MFI -> getNumPreloadedSGPRs ( ) ; AllSGPRs = AllSGPRs . slice ( std :: min ( static_cast < unsigned > ( AllSGPRs . size ( ) ) , NumPreloaded ) ) ; Register GITPtrLoReg = MFI -> getGITPtrLoReg ( MF ) ; for ( MCPhysReg Reg : AllSGPRs ) { if ( ! MRI . isPhysRegUsed ( Reg ) && MRI . isAllocatable ( Reg ) && ! TRI -> isSubRegisterEq ( ScratchRsrcReg , Reg ) && GITPtrLoReg != Reg ) { ScratchWaveOffsetReg = Reg ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , RegState :: Kill ) ; break ; } } } else { ScratchWaveOffsetReg = PreloadedScratchWaveOffsetReg ; } assert ( ScratchWaveOffsetReg ) ; if ( requiresStackPointerReference ( MF ) ) { Register SPReg = MFI -> getStackPtrOffsetReg ( ) ; assert ( SPReg != <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SPReg ) . addImm ( FrameInfo . getStackSize ( ) * getScratchScaleFactor ( ST ) ) ; } if ( hasFP ( MF ) ) { Register FPReg = MFI -> getFrameOffsetReg ( ) ; assert ( FPReg != <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FPReg ) . addImm ( <NUM_LIT> ) ; } bool NeedsFlatScratchInit = MFI -> hasFlatScratchInit ( ) && ( MRI . isPhysRegUsed ( <STR_LIT> ) || FrameInfo . hasCalls ( ) || ( ! allStackObjectsAreDead ( FrameInfo ) && ST . enableFlatScratch ( ) ) ) ; if ( ( NeedsFlatScratchInit || ScratchRsrcReg ) && ! ST . flatScratchIsArchitected ( ) ) { MRI . addLiveIn ( PreloadedScratchWaveOffsetReg ) ; MBB . addLiveIn ( PreloadedScratchWaveOffsetReg ) ; } if ( NeedsFlatScratchInit ) { emitEntryFunctionFlatScratchInit ( MF , MBB , I ,


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const bool SpillVGPRToAGPR = ST . hasMAIInsts ( ) && FuncInfo -> hasSpilledVGPRs ( ) && EnableSpillVGPRToAGPR ; if ( SpillVGPRToAGPR ) { BitVector SpillFIs ( MFI . getObjectIndexEnd ( ) , false ) ; bool SeenDbgInstr = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( MI . isDebugInstr ( ) ) SeenDbgInstr = true ; if ( TII -> isVGPRSpill ( MI ) ) { unsigned FIOp = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; int FI = MI . getOperand ( FIOp ) . getIndex ( ) ; Register VReg = TII -> getNamedOperand ( MI , <STR_LIT> :: vdata ) -> getReg ( ) ; if ( FuncInfo -> allocateVGPRSpillToAGPR ( MF , FI , TRI -> isAGPR ( MRI , VReg ) ) ) { RS -> enterBasicBlock ( MBB ) ; TRI -> eliminateFrameIndex ( MI , <NUM_LIT> , FIOp


<s> TargetFrameLowering :: determineCalleeSaves ( MF , SavedRegs , RS ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( MFI -> isEntryFunction ( ) ) return ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; SavedRegs . reset ( MFI -> getStackPtrOffsetReg ( ) ) ; const BitVector AllSavedRegs = SavedRegs ; SavedRegs . clearBitsInMask ( TRI -> getAllVectorRegMask ( )


<s> unsigned EncodedOffset = <STR_LIT> ( Subtarget , Offset ) ; BuildMI ( MBB , I , DL , LoadDwordX2 , FlatScrInit ) . addReg ( FlatScrInit ) . addImm ( EncodedOffset ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; const MCInstrDesc & SAndB32 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , SAndB32 , FlatScrInitHi ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; } else { Register FlatScratchInitReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; assert ( FlatScratchInitReg ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MRI . addLiveIn ( FlatScratchInitReg ) ; MBB . addLiveIn ( FlatScratchInitReg ) ; FlatScrInitLo = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; FlatScrInitHi = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; } if ( ST . flatScratchIsPointer ( ) ) { if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX10 ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitLo ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitHi ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( FlatScrInitLo ) . addImm ( int16_t ( <STR_LIT> :: ID_FLAT_SCR_LO | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ) ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( FlatScrInitHi ) . addImm ( int16_t ( <STR_LIT> :: ID_FLAT_SCR_HI | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ) ) ; return ; } BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; return ; } assert ( ST . getGeneration ( ) < <ISA_LIT>Subtarget :: GFX9 ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitHi , RegState :: Kill ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitLo ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitLo , RegState ::


<s> MBB . addLiveIn ( PreloadedScratchRsrcReg ) ; } } DebugLoc DL ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; Register ScratchWaveOffsetReg ; if ( TRI -> isSubRegisterEq ( ScratchRsrcReg , PreloadedScratchWaveOffsetReg ) ) { ArrayRef < MCPhysReg > AllSGPRs = TRI -> getAllSGPR32 ( MF ) ; unsigned NumPreloaded = MFI -> getNumPreloadedSGPRs ( ) ; AllSGPRs = AllSGPRs . slice ( std :: min ( static_cast < unsigned > ( AllSGPRs . size ( ) ) , NumPreloaded ) ) ; Register GITPtrLoReg = MFI -> getGITPtrLoReg ( MF ) ; for ( MCPhysReg Reg : AllSGPRs ) { if ( ! MRI . isPhysRegUsed ( Reg ) && MRI . isAllocatable ( Reg ) && ! TRI -> isSubRegisterEq ( ScratchRsrcReg , Reg ) && GITPtrLoReg != Reg ) { ScratchWaveOffsetReg = Reg ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , RegState :: Kill ) ; break ; } } } else { ScratchWaveOffsetReg = PreloadedScratchWaveOffsetReg ; } assert ( ScratchWaveOffsetReg ) ; if ( requiresStackPointerReference ( MF ) ) { Register SPReg = MFI -> getStackPtrOffsetReg ( ) ; assert ( SPReg != <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SPReg ) . addImm ( MF . getFrameInfo ( ) . getStackSize ( ) * getScratchScaleFactor ( ST ) ) ; } if ( hasFP ( MF ) ) { Register FPReg = MFI -> getFrameOffsetReg ( ) ; assert ( FPReg != <STR_LIT>


<s> BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , FramePtrReg ) . addReg ( FuncInfo -> SGPRForFPSaveRestoreCopy ) . setMIFlag ( MachineInstr :: FrameDestroy ) ; } if ( FuncInfo -> SGPRForBPSaveRestoreCopy ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , BasePtrReg ) . addReg ( FuncInfo -> SGPRForBPSaveRestoreCopy ) . setMIFlag ( MachineInstr :: FrameDestroy ) ; } if ( FPSaveIndex ) { const int FramePtrFI = * FPSaveIndex ; assert ( ! MFI . isDeadObjectIndex ( FramePtrFI ) ) ; if ( spilledToMemory ( MF , FramePtrFI ) ) { initLiveRegs ( LiveRegs , TRI , FuncInfo , MF , MBB , MBBI , false ) ; MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister ( MRI , LiveRegs , <STR_LIT> ) ; if ( ! TmpVGPR ) report_fatal_error ( <STR_LIT> ) ; buildEpilogRestore ( ST , TRI , * FuncInfo , LiveRegs , MF , MBB , MBBI , TmpVGPR , FramePtrFI ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , FramePtrReg ) . addReg ( TmpVGPR , RegState :: Kill ) ; } else { assert ( MFI . getStackID ( FramePtrFI ) == TargetStackID :: SGPRSpill ) ; ArrayRef < <ISA_LIT>MachineFunctionInfo :: SpilledReg > Spill = FuncInfo -> getSGPRToVGPRSpills ( FramePtrFI ) ; assert ( Spill . size ( ) == <NUM_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , FramePtrReg ) . addReg ( Spill [ <NUM_LIT> ] . VGPR ) . addImm ( Spill [ <NUM_LIT> ] . Lane ) ; } } if ( BPSaveIndex ) { const int BasePtrFI = * BPSaveIndex ; assert ( ! MFI . isDeadObjectIndex ( BasePtrFI ) ) ; if ( spilledToMemory ( MF , BasePtrFI ) ) { initLiveRegs ( LiveRegs , TRI , FuncInfo , MF , MBB , MBBI , false ) ; MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister ( MRI , LiveRegs , <STR_LIT> ) ; if ( ! TmpVGPR ) report_fatal_error ( <STR_LIT> ) ; buildEpilogRestore ( ST , TRI , * FuncInfo , LiveRegs , MF , MBB , MBBI , TmpVGPR , BasePtrFI ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , BasePtrReg ) . addReg ( TmpVGPR , RegState :: Kill ) ; } else { assert ( MFI . getStackID ( BasePtrFI ) == TargetStackID :: SGPRSpill ) ; ArrayRef < <ISA_LIT>MachineFunctionInfo :: SpilledReg > Spill = FuncInfo -> getSGPRToVGPRSpills ( BasePtrFI ) ; assert ( Spill . size ( ) == <NUM_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , BasePtrReg ) . addReg ( Spill [ <NUM_LIT> ] . VGPR ) . addImm ( Spill [ <NUM_LIT> ] . Lane ) ; } } Register ScratchExecCopy


<s> case TargetStackID :: NoAlloc : case TargetStackID :: SGPRSpill : return true ; case TargetStackID :: ScalableVector


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; FuncInfo -> removeDeadFrameIndices ( MFI ) ; assert ( allSGPRSpillsAreDead ( MF ) &&


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; FuncInfo -> removeDeadFrameIndices ( MFI ) ; assert ( allSGPRSpillsAreDead ( MF ) && <STR_LIT> ) ; if ( ! allStackObjectsAreDead


<s> for ( int I = MFI . getObjectIndexBegin ( ) , E = MFI . getObjectIndexEnd ( ) ; I != E ;


<s> if ( ! Reg . FI . hasValue ( ) ) continue ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; if ( ScratchExecCopy == <STR_LIT> ) { LivePhysRegs LiveRegs ( * ST . getRegisterInfo ( ) ) ; LiveRegs . addLiveOuts ( MBB ) ; LiveRegs . stepBackward ( * MBBI ) ; ScratchExecCopy = findScratchNonCalleeSaveRegister ( MF , LiveRegs , * TRI . getWaveMaskRegClass ( ) ) ; const unsigned OrSaveExec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , MBBI , DL , TII -> get ( OrSaveExec ) , ScratchExecCopy ) . addImm ( - <NUM_LIT> ) ; } TII -> loadRegFromStackSlot ( MBB , MBBI , Reg . VGPR , Reg . FI . getValue ( ) , & <STR_LIT> ,


<s> BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitLo ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitHi ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( FlatScrInitLo ) . addImm ( int16_t ( <STR_LIT> :: ID_FLAT_SCR_LO | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ) ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( FlatScrInitHi ) . addImm ( int16_t ( <STR_LIT> :: ID_FLAT_SCR_HI | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ) ) ; return ; } BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitLo ) .


<s> unsigned ScratchSPReg = findScratchNonCalleeSaveRegister ( MF , LiveRegs , <STR_LIT> ) ; assert ( ScratchSPReg != <STR_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , ScratchSPReg ) . addReg ( StackPtrReg ) . addImm ( ( Alignment - <NUM_LIT> ) * ST . getWavefrontSize ( ) ) . setMIFlag ( MachineInstr :: FrameSetup ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , FramePtrReg ) . addReg ( ScratchSPReg , RegState :: Kill ) . addImm ( - Alignment * ST . getWavefrontSize ( ) ) . setMIFlag ( MachineInstr :: FrameSetup ) ; FuncInfo -> setIsStackRealigned ( true ) ; } else if ( ( HasFP = hasFP ( MF ) ) ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , FramePtrReg ) . addReg ( StackPtrReg ) . setMIFlag ( MachineInstr :: FrameSetup ) ; } if ( HasFP && RoundedSize != <NUM_LIT> ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , StackPtrReg ) . addReg ( StackPtrReg ) . addImm ( RoundedSize * ST . getWavefrontSize ( ) ) . setMIFlag ( MachineInstr :: FrameSetup ) ; } unsigned ScratchExecCopy = <STR_LIT> ; for ( const <ISA_LIT>MachineFunctionInfo :: SGPRSpillVGPRCSR & Reg : FuncInfo -> getSGPRSpillVGPRs ( ) ) { if ( ! Reg . FI . hasValue ( ) ) continue ; if ( ScratchExecCopy == <STR_LIT> ) { if ( LiveRegs . empty ( ) ) { LiveRegs . init ( TRI ) ; LiveRegs . addLiveIns ( MBB ) ; } ScratchExecCopy = findScratchNonCalleeSaveRegister ( MF , LiveRegs , * TRI . getWaveMaskRegClass ( ) ) ; const unsigned OrSaveExec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , MBBI , DL , TII -> get ( OrSaveExec ) , ScratchExecCopy ) . addImm ( - <NUM_LIT> ) ; } TII -> storeRegToStackSlot ( MBB , MBBI , Reg . VGPR , true , Reg . FI . getValue ( ) , & <STR_LIT> , & TII -> getRegisterInfo ( ) ) ; } if ( ScratchExecCopy != <STR_LIT> ) { unsigned ExecMov = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , MBBI , DL , TII -> get ( ExecMov

) , Exec ) . addReg ( ScratchExecCopy ) ;


<s> const MCPhysReg * CSRegs = MRI . getCalleeSavedRegs ( ) ; for ( unsigned i = <NUM_LIT> ; CSRegs [ i ] ;

++ i ) LiveRegs . addReg ( CSRegs [ i ] ) ;


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const MCPhysReg * CSRegs = MRI . getCalleeSavedRegs ( ) ; for ( unsigned i = <NUM_LIT> ; CSRegs [ i ] ; ++ i ) LiveRegs . addReg ( CSRegs [ i ] ) ; for ( unsigned Reg : RC ) { if ( LiveRegs . available ( MRI


<s> DebugLoc DL ; if ( PreloadedScratchWaveOffsetReg != ScratchWaveOffsetReg ) { BuildMI ( MBB , I , DL , SMovB32 , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , RegState :: Kill ) ; } if ( ST . isAmdHsaOS ( ) ) { assert ( ! TRI -> isSubRegisterEq ( PreloadedPrivateBufferReg , ScratchRsrcReg ) && ! TRI -> isSubRegisterEq ( PreloadedPrivateBufferReg , ScratchWaveOffsetReg ) ) ; unsigned Rsrc01 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc23 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Lo = TRI -> getSubReg ( PreloadedPrivateBufferReg , <STR_LIT> ) ; unsigned Hi = TRI -> getSubReg ( PreloadedPrivateBufferReg , <STR_LIT> ) ; const MCInstrDesc & SMovB64 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , SMovB64 , Rsrc01 ) . addReg ( Lo , RegState :: Kill ) ; BuildMI ( MBB , I , DL , SMovB64 , Rsrc23 ) . addReg ( Hi , RegState :: Kill ) ; } else { unsigned Rsrc0 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc1 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc0 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc1 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc2 ) . addImm ( Rsrc23 & <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc3 ) . addImm ( Rsrc23 >> <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } for ( MachineBasicBlock &


<s> if ( ST . debuggerEmitPrologue ( ) ) emitDebuggerPrologue ( MF , MBB ) ; if ( ! MF . getFrameInfo ( ) . hasStackObjects ( ) ) return ; assert ( & MF . front ( ) == & MBB && <STR_LIT> ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( hasOnlySGPRSpills ( MFI , MF . getFrameInfo ( ) ) ) return ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo * TRI = & TII -> getRegisterInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; unsigned ScratchRsrcReg = MFI -> getScratchRSrcReg ( ) ; assert ( ScratchRsrcReg != <STR_LIT> ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; assert ( ScratchWaveOffsetReg != <STR_LIT> ) ; unsigned PreloadedScratchWaveOffsetReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: PRIVATE_SEGMENT_WAVE_BYTE_OFFSET ) ; unsigned PreloadedPrivateBufferReg = <STR_LIT> ; if ( ST . isAmdHsaOS ( ) ) { PreloadedPrivateBufferReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: PRIVATE_SEGMENT_BUFFER ) ; } if ( MFI -> hasFlatScratchInit ( ) ) { DebugLoc DL ; unsigned FlatScratchInitReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: FLAT_SCRATCH_INIT ) ; MRI . addLiveIn ( FlatScratchInitReg ) ; MBB . addLiveIn ( FlatScratchInitReg ) ; unsigned FlatScrInitHi = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitHi , RegState :: Kill ) ; unsigned FlatScrInitLo = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitLo ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitLo , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; } if ( ScratchRsrcReg == PreloadedPrivateBufferReg ) { assert ( ScratchWaveOffsetReg == PreloadedScratchWaveOffsetReg && <STR_LIT> ) ; return ; } MRI . addLiveIn ( PreloadedScratchWaveOffsetReg ) ; MBB . addLiveIn ( PreloadedScratchWaveOffsetReg ) ; if ( ST . isAmdHsaOS ( ) ) { MRI . addLiveIn ( PreloadedPrivateBufferReg ) ; MBB . addLiveIn ( PreloadedPrivateBufferReg ) ; } if ( ! ST . hasSGPRInitBug ( ) ) { if ( ScratchRsrcReg == TRI -> reservedPrivateSegmentBufferReg ( MF ) ) { MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned NumPreloaded = MFI -> getNumPreloadedSGPRs ( ) / <NUM_LIT> ; for ( MCPhysReg Reg : getAllSGPR128 ( ) . drop_back ( <NUM_LIT> ) . slice ( NumPreloaded ) ) { if ( ! MRI . isPhysRegUsed ( Reg ) ) { assert ( MRI . isAllocatable ( Reg ) ) ; MRI . replaceRegWith ( ScratchRsrcReg , Reg


<s> MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; if ( ! MFI . hasStackObjects ( ) ) return ; bool MayNeedScavengingEmergencySlot = MFI . hasStackObjects ( ) ; assert ( ( RS || ! MayNeedScavengingEmergencySlot ) && <STR_LIT> ) ; if ( MayNeedScavengingEmergencySlot ) { int ScavengeFI = MFI . CreateSpillStackObject ( <STR_LIT> . getSize ( ) , <STR_LIT> . getAlignment ( ) ) ; RS -> addScavengingFrameIndex (


<s> MRI . addLiveIn ( PreloadedPrivateBufferReg ) ; MBB . addLiveIn ( PreloadedPrivateBufferReg ) ; } for ( MachineBasicBlock & OtherBB : MF ) { if ( & OtherBB == & MBB ) continue ; if ( OffsetRegUsed ) OtherBB . addLiveIn ( ScratchWaveOffsetReg ) ; if ( ResourceRegUsed ) OtherBB . addLiveIn ( ScratchRsrcReg ) ; } DebugLoc DL ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; bool CopyBuffer = ResourceRegUsed && PreloadedPrivateBufferReg != <STR_LIT> && ST . isAmdCodeObjectV2 ( F ) && ScratchRsrcReg != PreloadedPrivateBufferReg ; bool CopyBufferFirst = TRI -> isSubRegisterEq ( PreloadedPrivateBufferReg , ScratchWaveOffsetReg ) ; if ( CopyBuffer && CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } if ( OffsetRegUsed && PreloadedScratchWaveOffsetReg != ScratchWaveOffsetReg ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , MRI . isPhysRegUsed ( ScratchWaveOffsetReg ) ? <NUM_LIT> : RegState :: Kill ) ; } if ( CopyBuffer && ! CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } if ( ResourceRegUsed ) emitEntryFunctionScratchSetup ( ST , MF , MBB ,

MFI , I , PreloadedPrivateBufferReg , ScratchRsrcReg ) ;


<s> assert ( ST . isAmdCodeObjectV2 ( MF ) || ST . isMesaGfxShader ( MF ) ) ; MRI . addLiveIn ( PreloadedPrivateBufferReg ) ; MBB . addLiveIn ( PreloadedPrivateBufferReg ) ; } for ( MachineBasicBlock & OtherBB : MF ) { if ( & OtherBB == & MBB ) continue ; if ( OffsetRegUsed ) OtherBB . addLiveIn ( ScratchWaveOffsetReg ) ; if ( ResourceRegUsed ) OtherBB . addLiveIn ( ScratchRsrcReg ) ; } DebugLoc DL ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; bool CopyBuffer = ResourceRegUsed && PreloadedPrivateBufferReg != <STR_LIT> && ST . isAmdCodeObjectV2 ( MF ) && ScratchRsrcReg != PreloadedPrivateBufferReg ; bool CopyBufferFirst = TRI -> isSubRegisterEq ( PreloadedPrivateBufferReg , ScratchWaveOffsetReg ) ; if ( CopyBuffer && CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } if ( OffsetRegUsed && PreloadedScratchWaveOffsetReg != ScratchWaveOffsetReg ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , RegState :: Kill ) ; } if ( CopyBuffer && ! CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } if ( ResourceRegUsed && ( ST . isMesaGfxShader ( MF ) || ( PreloadedPrivateBufferReg == <STR_LIT> ) ) ) { assert ( ! ST . isAmdCodeObjectV2 ( MF ) ) ; const MCInstrDesc & SMovB32 = TII -> get ( <STR_LIT> ) ; unsigned Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; if ( MFI -> hasPrivateMemoryInputPtr ( ) ) { unsigned Rsrc01 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; if ( <STR_LIT> ( MF . getFunction ( ) -> getCallingConv ( ) ) ) { const MCInstrDesc & Mov64 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , Mov64 , Rsrc01 ) . addReg ( PreloadedPrivateBufferReg ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } else { const MCInstrDesc & LoadDwordX2 = TII -> get ( <STR_LIT> ) ; PointerType * PtrTy = PointerType :: get ( Type :: getInt64Ty ( MF . getFunction ( ) -> getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; auto MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant | MachineMemOperand

:: MODereferenceable , <NUM_LIT> , <NUM_LIT> ) ;


<s> unsigned ScratchRsrcReg = MFI -> getScratchRSrcReg ( ) ; if ( ScratchRsrcReg == <STR_LIT> ) return <STR_LIT> ; if ( ST . hasSGPRInitBug ( ) || ScratchRsrcReg != TRI -> reservedPrivateSegmentBufferReg ( MF ) ) return ScratchRsrcReg ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned NumPreloaded = ( MFI -> getNumPreloadedSGPRs ( ) + <NUM_LIT> ) / <NUM_LIT> ; ArrayRef < MCPhysReg > AllSGPR128s = getAllSGPR128 ( ST , MF ) ; AllSGPR128s = AllSGPR128s . slice ( std :: min ( static_cast < unsigned > ( AllSGPR128s . size (


<s> if ( ST . hasSGPRInitBug ( ) || ScratchWaveOffsetReg != TRI -> reservedPrivateSegmentWaveByteOffsetReg ( MF ) ) return ScratchWaveOffsetReg ; unsigned ScratchRsrcReg = MFI -> getScratchRSrcReg ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned NumPreloaded = MFI -> getNumPreloadedSGPRs ( ) ; ArrayRef < MCPhysReg > AllSGPRs = getAllSGPRs ( ST , MF ) ; if ( NumPreloaded > AllSGPRs . size ( ) ) return ScratchWaveOffsetReg ; AllSGPRs = AllSGPRs . slice ( NumPreloaded ) ; if ( AllSGPRs . size ( ) < <NUM_LIT> ) return ScratchWaveOffsetReg ; for ( MCPhysReg Reg : AllSGPRs . drop_back ( <NUM_LIT> ) ) { if ( ! MRI . isPhysRegUsed ( Reg ) ) { if ( ! MRI . isAllocatable ( Reg ) || TRI -> isSubRegisterEq ( ScratchRsrcReg , Reg ) ) continue ; MRI . replaceRegWith ( ScratchWaveOffsetReg ,


<s> MachineFunction * MF = MBB . getParent ( ) ; MachineFrameInfo & MFI = MF -> getFrameInfo ( ) ; int64_t Offset = MFI . getObjectOffset ( FI ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( MachinePointerInfo :: getFixedStack ( * MF , FI ) , MachineMemOperand :: MOLoad , <NUM_LIT> , MFI . getObjectAlign ( FI ) ) ; if ( ST . enableFlatScratch ( ) ) { if ( TII -> isLegalFLATOffset ( Offset , <ISA_LIT>AS :: PRIVATE_ADDRESS , true ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SpillReg ) . addReg ( SPReg ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; return ; } MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister ( MF -> getRegInfo ( ) , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT>

) , OffsetReg ) . addReg ( SPReg ) . addImm ( Offset ) ;


<s> return ; } if ( <ISA_LIT>InstrInfo :: isLegalMUBUFImmOffset ( Offset ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SpillReg ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; return ; } MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister ( MF -> getRegInfo ( ) , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , OffsetReg ) . addImm ( Offset ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SpillReg ) . addReg ( OffsetReg , RegState :: Kill ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm (


<s> int64_t Amount = I -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( Amount == <NUM_LIT> ) return MBB . erase ( I ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const DebugLoc & DL = I -> getDebugLoc ( ) ; unsigned Opc = I -> getOpcode ( ) ; bool IsDestroy = Opc == TII -> getCallFrameDestroyOpcode ( ) ; uint64_t CalleePopAmount = IsDestroy ? I -> getOperand ( <NUM_LIT> ) . getImm ( ) : <NUM_LIT> ; if ( ! hasReservedCallFrame ( MF ) ) { Amount = alignTo ( Amount , getStackAlign ( ) ) ; assert ( isUInt < <NUM_LIT> > ( Amount ) && <STR_LIT> ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register SPReg = MFI -> getStackPtrOffsetReg ( ) ; unsigned Op = IsDestroy ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , I , DL , TII -> get ( Op ) , SPReg ) . addReg ( SPReg ) . addImm ( Amount * getScratchScaleFactor ( ST ) ) ; } else if ( CalleePopAmount != <NUM_LIT> ) { llvm_unreachable ( <STR_LIT> ) ; } return MBB . erase


<s> MBB . addLiveIn ( PreloadedScratchRsrcReg ) ; } } DebugLoc DL ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; Register ScratchWaveOffsetReg ; if ( TRI -> isSubRegisterEq ( ScratchRsrcReg , PreloadedScratchWaveOffsetReg ) ) { ArrayRef < MCPhysReg > AllSGPRs = TRI -> getAllSGPR32 ( MF ) ; unsigned NumPreloaded = MFI -> getNumPreloadedSGPRs ( ) ; AllSGPRs = AllSGPRs . slice ( std :: min ( static_cast < unsigned > ( AllSGPRs . size ( ) ) , NumPreloaded ) ) ; Register GITPtrLoReg = MFI -> getGITPtrLoReg ( MF ) ; for ( MCPhysReg Reg : AllSGPRs ) { if ( ! MRI . isPhysRegUsed ( Reg ) && MRI . isAllocatable ( Reg ) && ! TRI -> isSubRegisterEq ( ScratchRsrcReg , Reg ) && GITPtrLoReg != Reg ) { ScratchWaveOffsetReg = Reg ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , RegState :: Kill ) ; break ; } } } else { ScratchWaveOffsetReg =


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; MachineBasicBlock :: iterator MBBI = MBB . getFirstTerminator ( ) ; LivePhysRegs LiveRegs ; DebugLoc DL ; const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; uint32_t NumBytes = MFI . getStackSize ( ) ; uint32_t RoundedSize = FuncInfo -> isStackRealigned ( ) ? NumBytes + MFI . getMaxAlign ( ) . value ( ) : NumBytes ; const Register StackPtrReg = FuncInfo -> getStackPtrOffsetReg ( ) ; const Register FramePtrReg = FuncInfo -> getFrameOffsetReg ( ) ; const Register BasePtrReg = TRI . hasBasePointer ( MF ) ? TRI . getBaseRegister ( ) : Register ( ) ; bool HasFPSaveIndex = FuncInfo -> FramePointerSaveIndex . hasValue ( ) ; bool SpillFPToMemory = false ; if ( HasFPSaveIndex ) { SpillFPToMemory = MFI . getStackID ( * FuncInfo -> FramePointerSaveIndex ) != TargetStackID :: SGPRSpill ; } bool HasBPSaveIndex = FuncInfo -> BasePointerSaveIndex . hasValue ( ) ; bool SpillBPToMemory = false ; if ( HasBPSaveIndex ) { SpillBPToMemory = MFI . getStackID ( * FuncInfo -> BasePointerSaveIndex ) != TargetStackID :: SGPRSpill ; } if ( RoundedSize != <NUM_LIT> && hasFP ( MF ) ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , StackPtrReg ) . addReg ( StackPtrReg ) . addImm ( RoundedSize * getScratchScaleFactor ( ST ) ) . setMIFlag ( MachineInstr :: FrameDestroy ) ; } if ( FuncInfo -> SGPRForFPSaveRestoreCopy ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , FramePtrReg ) . addReg ( FuncInfo -> SGPRForFPSaveRestoreCopy ) . setMIFlag ( MachineInstr :: FrameSetup ) ; } if ( FuncInfo -> SGPRForBPSaveRestoreCopy ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , BasePtrReg ) . addReg ( FuncInfo -> SGPRForBPSaveRestoreCopy ) . setMIFlag ( MachineInstr :: FrameSetup ) ; } Register ScratchExecCopy ; if ( HasFPSaveIndex )


<s> if ( ! TempSGPR ) { int NewFI = FrameInfo . CreateStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) , true , nullptr , TargetStackID :: SGPRSpill ) ; if ( MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) { FrameIndex = NewFI ; LLVM_DEBUG ( auto Spill = MFI -> getSGPRToVGPRSpills ( NewFI ) . front ( ) ; dbgs ( ) << ( IsFP ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << printReg ( Spill . VGPR , TRI ) << ':' << Spill . Lane << '\n' ; ) ; } else { MF . getFrameInfo ( ) . RemoveStackObject ( NewFI ) ; FrameIndex = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << FrameIndex << <STR_LIT> << ( IsFP ? <STR_LIT> : <STR_LIT> ) << '\n' ) ; } } else


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; if ( MFI -> haveFreeLanesForSGPRSpill ( MF , <NUM_LIT> ) ) { int NewFI = FrameInfo . CreateStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) , true , nullptr , TargetStackID :: SGPRSpill ) ; if ( ! MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) llvm_unreachable ( <STR_LIT> ) ; FrameIndex = NewFI ; LLVM_DEBUG ( auto Spill = MFI -> getSGPRToVGPRSpills ( NewFI ) . front ( ) ; dbgs ( ) << <STR_LIT> << ( IsFP ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << printReg ( Spill . VGPR , TRI ) << ':' << Spill . Lane << '\n' ) ; return ; } TempSGPR = findScratchNonCalleeSaveRegister ( MF . getRegInfo ( ) , LiveRegs , <STR_LIT> , true ) ; if ( ! TempSGPR ) { int NewFI = FrameInfo . CreateStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) , true , nullptr , TargetStackID :: SGPRSpill ) ; if ( MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) { FrameIndex =


<s> const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; if ( MFI . hasCalls ( ) && ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ->


<s> MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; FuncInfo -> removeDeadFrameIndices ( MFI ) ; assert ( allSGPRSpillsAreDead ( MF ) && <STR_LIT> ) ; if ( ! allStackObjectsAreDead ( MFI ) ) { assert ( RS && <STR_LIT> ) ; if ( FuncInfo -> isEntryFunction ( ) ) { int ScavengeFI = MFI . CreateFixedObject ( TRI -> getSpillSize ( <STR_LIT> ) , <NUM_LIT> , false ) ; RS -> addScavengingFrameIndex


<s> void <ISA_LIT>FrameLowering :: processFunctionBeforeFrameFinalized ( MachineFunction & MF , RegScavenger * RS ) const { MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; FuncInfo -> removeDeadFrameIndices ( MFI ) ; assert ( allSGPRSpillsAreDead ( MF ) && <STR_LIT> ) ; if ( ! allStackObjectsAreDead ( MFI ) ) { assert ( RS && <STR_LIT> ) ; if ( FuncInfo -> isEntryFunction ( ) ) { int ScavengeFI = MFI . CreateFixedObject ( TRI -> getSpillSize ( <STR_LIT> ) ,


<s> } } DebugLoc DL ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; Register ScratchWaveOffsetReg ; if ( TRI -> isSubRegisterEq ( ScratchRsrcReg , PreloadedScratchWaveOffsetReg ) ) { ArrayRef < MCPhysReg > AllSGPRs = TRI -> getAllSGPR32 ( MF ) ; unsigned NumPreloaded = MFI -> getNumPreloadedSGPRs ( ) ; AllSGPRs = AllSGPRs . slice ( std :: min ( static_cast < unsigned > ( AllSGPRs . size ( ) ) , NumPreloaded ) ) ; Register GITPtrLoReg = MFI -> getGITPtrLoReg ( MF ) ; for ( MCPhysReg Reg : AllSGPRs ) { if ( ! MRI . isPhysRegUsed ( Reg ) && MRI . isAllocatable ( Reg ) && ! TRI -> isSubRegisterEq ( ScratchRsrcReg , Reg ) && GITPtrLoReg != Reg ) { ScratchWaveOffsetReg = Reg ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , RegState :: Kill ) ; break ; } } } else { ScratchWaveOffsetReg = PreloadedScratchWaveOffsetReg ; } assert ( ScratchWaveOffsetReg ) ; if ( requiresStackPointerReference ( MF ) ) { Register SPReg = MFI -> getStackPtrOffsetReg ( ) ; assert ( SPReg != <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SPReg ) . addImm ( MF . getFrameInfo ( ) . getStackSize ( ) * getScratchScaleFactor ( ST ) ) ; } if ( hasFP ( MF ) ) { Register FPReg = MFI -> getFrameOffsetReg ( ) ; assert ( FPReg != <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> )


<s> } else { const MCInstrDesc & LoadDwordX2 = TII -> get ( <STR_LIT> ) ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; auto MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant | MachineMemOperand :: MODereferenceable , <NUM_LIT> , Align ( <NUM_LIT> ) ) ; BuildMI ( MBB , I , DL , LoadDwordX2 , Rsrc01 ) . addReg ( MFI -> getImplicitBufferPtrUserSGPR ( ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; MF . getRegInfo ( ) . addLiveIn ( MFI -> getImplicitBufferPtrUserSGPR ( ) ) ; MBB . addLiveIn ( MFI -> getImplicitBufferPtrUserSGPR ( ) ) ; } } else { Register Rsrc0 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; Register Rsrc1 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc0 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc1 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } BuildMI ( MBB , I , DL , SMovB32 , Rsrc2 ) . addImm ( Rsrc23 & <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc3 ) . addImm ( Rsrc23 >> <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } else if ( ST . isAmdHsaOrMesa ( Fn ) ) { assert ( PreloadedScratchRsrcReg ) ; if ( ScratchRsrcReg != PreloadedScratchRsrcReg ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg

) . addReg ( PreloadedScratchRsrcReg , RegState :: Kill ) ;


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( ! FuncInfo -> isEntryFunction ( ) ) { FuncInfo -> allocateWWMReservedSpillSlots ( MFI , * TRI ) ; } const bool SpillVGPRToAGPR = ST . hasMAIInsts ( ) && FuncInfo -> hasSpilledVGPRs ( ) && EnableSpillVGPRToAGPR ; if ( SpillVGPRToAGPR ) { BitVector SpillFIs ( MFI . getObjectIndexEnd ( ) , false ) ; BitVector NonVGPRSpillFIs ( MFI . getObjectIndexEnd ( ) , false ) ; bool SeenDbgInstr = false ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : llvm :: make_early_inc_range ( MBB ) ) { int FrameIndex ; if ( MI . isDebugInstr ( ) ) SeenDbgInstr = true ; if ( TII -> isVGPRSpill ( MI ) ) { unsigned FIOp = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; int FI = MI . getOperand ( FIOp ) . getIndex ( ) ; Register VReg = TII -> getNamedOperand ( MI , <STR_LIT> :: vdata ) -> getReg ( ) ; if ( FuncInfo -> allocateVGPRSpillToAGPR ( MF , FI , TRI -> isAGPR ( MRI , VReg ) ) ) { RS -> enterBasicBlock ( MBB ) ; TRI -> eliminateFrameIndex ( MI , <NUM_LIT> , FIOp , RS ) ; SpillFIs . set ( FI ) ; continue ; } } else if ( TII -> isStoreToStackSlot ( MI , FrameIndex ) || TII -> isLoadFromStackSlot ( MI , FrameIndex ) ) if ( ! MFI . isFixedObjectIndex ( FrameIndex ) ) NonVGPRSpillFIs . set ( FrameIndex ) ; } } for ( unsigned FI : SpillFIs . set_bits ( ) ) if ( ! NonVGPRSpillFIs . test ( FI ) ) FuncInfo -> setVGPRToAGPRSpillDead ( FI ) ; for ( MachineBasicBlock & MBB : MF ) { for ( MCPhysReg Reg : FuncInfo -> getVGPRSpillAGPRs ( ) ) MBB . addLiveIn ( Reg ) ; for ( MCPhysReg Reg : FuncInfo -> getAGPRSpillVGPRs ( ) ) MBB . addLiveIn ( Reg ) ; MBB . sortUniqueLiveIns ( ) ; if ( ! SpillFIs . empty ( ) && SeenDbgInstr ) { for ( MachineInstr & MI : MBB ) { if ( MI . isDebugValue ( ) && MI . getOperand ( <NUM_LIT> ) . isFI ( ) && SpillFIs [ MI . getOperand ( <NUM_LIT> ) . getIndex ( ) ] ) { MI . getOperand ( <NUM_LIT> ) . ChangeToRegister ( Register ( ) , false ) ; } } } } } bool HaveSGPRToVMemSpill = FuncInfo -> removeDeadFrameIndices ( MFI , true ) ; assert ( allSGPRSpillsAreDead ( MF ) && <STR_LIT> ) ; if ( ! allStackObjectsAreDead ( MFI


<s> MRI . addLiveIn ( FlatScratchInitReg ) ; MBB . addLiveIn ( FlatScratchInitReg ) ; Register FlatScrInitLo = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; Register FlatScrInitHi = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; if ( ST . flatScratchIsPointer ( ) ) { if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX10 ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitLo ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitHi ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( FlatScrInitLo ) . addImm ( int16_t ( <STR_LIT> :: ID_FLAT_SCR_LO | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ) ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( FlatScrInitHi ) . addImm ( int16_t ( <STR_LIT> :: ID_FLAT_SCR_HI | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ) ) ; return ; } BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; return ; } assert ( ST . getGeneration ( ) < <ISA_LIT>Subtarget :: GFX9 ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitHi , RegState :: Kill ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitLo ) . addReg ( FlatScrInitLo )


<s> if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX10 ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitLo ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitHi ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( FlatScrInitLo ) . addImm ( int16_t ( <STR_LIT> :: ID_FLAT_SCR_LO | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ) ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( FlatScrInitHi ) . addImm ( int16_t ( <STR_LIT> :: ID_FLAT_SCR_HI | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ) ) ; return ; } BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; return ; } assert ( ST . getGeneration ( ) < <ISA_LIT>Subtarget


<s> static void getVGPRSpillLaneOrTempRegister ( MachineFunction & MF , LivePhysRegs & LiveRegs , Register & TempSGPR , Optional < int > & FrameIndex , bool IsFP ) { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; if ( MFI -> haveFreeLanesForSGPRSpill ( MF , <NUM_LIT> ) ) { int NewFI = FrameInfo . CreateStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) , true , nullptr , TargetStackID :: SGPRSpill ) ; if ( ! MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) llvm_unreachable ( <STR_LIT> ) ; FrameIndex = NewFI ; LLVM_DEBUG ( auto Spill = MFI -> getSGPRToVGPRSpills ( NewFI ) . front ( ) ; dbgs ( ) << <STR_LIT> << ( IsFP ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << printReg ( Spill . VGPR , TRI ) << ':' << Spill . Lane << '\n' ) ; return ; } TempSGPR = findScratchNonCalleeSaveRegister ( MF . getRegInfo ( ) , LiveRegs , <STR_LIT> , true ) ; if ( ! TempSGPR ) { int NewFI = FrameInfo . CreateStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) , true , nullptr , TargetStackID :: SGPRSpill ) ; if ( MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) { FrameIndex = NewFI ; } else


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; FuncInfo -> removeDeadFrameIndices ( MFI ) ; assert ( allSGPRSpillsAreDead ( MFI , None


<s> BuildMI ( MBB , I , DL , SMovB32 , RsrcHi ) . addImm ( MFI -> getGITPtrHigh ( ) ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } else { const MCInstrDesc & GetPC64 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , GetPC64 , Rsrc01 ) ; } auto GitPtrLo = <STR_LIT> ; if ( ST . hasMergedShaders ( ) ) { switch ( MF . getFunction ( ) . getCallingConv ( ) ) { case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_GS : GitPtrLo = <STR_LIT> ; break ; default : break ; } } MF . getRegInfo ( ) . addLiveIn ( GitPtrLo ) ; MF . front ( ) . addLiveIn ( GitPtrLo ) ; BuildMI ( MBB , I , DL , SMovB32 , RsrcLo ) . addReg ( GitPtrLo ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; PointerType * PtrTy = PointerType :: get ( Type :: getInt64Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; const MCInstrDesc & LoadDwordX4 = TII -> get ( <STR_LIT> ) ; auto MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant | MachineMemOperand :: MODereferenceable , <NUM_LIT> , <NUM_LIT> ) ; unsigned Offset = Fn . getCallingConv ( ) == CallingConv :: <ISA_LIT>_CS ? <NUM_LIT> : <NUM_LIT> ; const GCNSubtarget & Subtarget = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned EncodedOffset = <STR_LIT> ( Subtarget , Offset ) ; BuildMI ( MBB , I , DL , LoadDwordX4 , ScratchRsrcReg ) . addReg ( Rsrc01 ) . addImm ( EncodedOffset ) . addImm ( <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) . addMemOperand ( MMO ) ; return ; } if ( ST . isMesaGfxShader ( Fn ) || ( PreloadedPrivateBufferReg == <STR_LIT> ) ) { assert ( ! ST . isAmdHsaOrMesa ( Fn ) ) ; const MCInstrDesc & SMovB32 = TII -> get ( <STR_LIT> ) ; unsigned Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; if ( MFI -> hasImplicitBufferPtr ( ) ) { unsigned Rsrc01 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; if ( <STR_LIT> ( MF . getFunction ( ) . getCallingConv ( ) ) ) { const MCInstrDesc & Mov64 = TII -> get


<s> if ( isUInt < <NUM_LIT> > ( Offset ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( SpillReg , RegState :: Kill ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; return ; } MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister ( MF -> getRegInfo ( ) , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ,


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const Function & F = MF . getFunction ( ) ; assert ( MFI -> isEntryFunction ( ) ) ; Register ScratchWaveOffsetReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; if ( ScratchWaveOffsetReg == <STR_LIT> ) return ; Register ScratchRsrcReg = getEntryFunctionReservedScratchRsrcReg ( MF , ScratchWaveOffsetReg ) ; if ( ScratchRsrcReg != <STR_LIT> ) { for ( MachineBasicBlock & OtherBB : MF ) { if ( & OtherBB != & MBB ) { OtherBB . addLiveIn ( ScratchRsrcReg ) ; } } } Register PreloadedScratchRsrcReg = <STR_LIT> ; if ( ST . isAmdHsaOrMesa ( F ) ) { PreloadedScratchRsrcReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; if ( ScratchRsrcReg != <STR_LIT> && PreloadedScratchRsrcReg != <STR_LIT> ) { MRI . addLiveIn ( PreloadedScratchRsrcReg ) ; MBB . addLiveIn ( PreloadedScratchRsrcReg ) ; } } DebugLoc DL ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; if ( MF . getFrameInfo ( ) . hasCalls (


<s> if ( ST . hasSGPRInitBug ( ) || ScratchRsrcReg != TRI -> reservedPrivateSegmentBufferReg ( MF ) ) return ScratchRsrcReg ; unsigned NumPreloaded = ( MFI -> getNumPreloadedSGPRs ( ) + <NUM_LIT> ) / <NUM_LIT> ; ArrayRef < MCPhysReg > AllSGPR128s = getAllSGPR128 ( ST , MF ) ; AllSGPR128s = AllSGPR128s . slice ( std :: min ( static_cast < unsigned > ( AllSGPR128s . size ( ) ) , NumPreloaded ) ) ; for ( MCPhysReg Reg :


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MachineBasicBlock :: iterator MBBI = MBB . getFirstTerminator ( ) ; LivePhysRegs LiveRegs ; DebugLoc DL ; const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; uint32_t NumBytes = MFI . getStackSize ( ) ; uint32_t RoundedSize = FuncInfo -> isStackRealigned ( ) ? NumBytes + MFI . getMaxAlign ( ) . value ( ) : NumBytes ; if ( RoundedSize != <NUM_LIT> && hasFP ( MF ) ) { const Register StackPtrReg = FuncInfo -> getStackPtrOffsetReg ( ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , StackPtrReg ) . addReg ( StackPtrReg ) . addImm ( RoundedSize * ST . getWavefrontSize ( ) ) . setMIFlag ( MachineInstr :: FrameDestroy ) ; } if ( FuncInfo -> SGPRForFPSaveRestoreCopy ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , FuncInfo -> getFrameOffsetReg ( ) ) . addReg ( FuncInfo -> SGPRForFPSaveRestoreCopy ) . setMIFlag ( MachineInstr :: FrameSetup ) ; } if ( FuncInfo -> FramePointerSaveIndex ) { const int FI = FuncInfo -> FramePointerSaveIndex . getValue ( ) ; assert ( ! MF . getFrameInfo ( ) . isDeadObjectIndex ( FI ) && MF . getFrameInfo ( ) . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; ArrayRef < <ISA_LIT>MachineFunctionInfo :: SpilledReg > Spill = FuncInfo -> getSGPRToVGPRSpills ( FI ) ; assert ( Spill . size ( ) == <NUM_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , FuncInfo -> getFrameOffsetReg ( ) ) . addReg ( Spill [ <NUM_LIT> ] . VGPR ) . addImm ( Spill [ <NUM_LIT> ] . Lane ) ; } Register ScratchExecCopy ; for ( const <ISA_LIT>MachineFunctionInfo :: SGPRSpillVGPRCSR & Reg : FuncInfo -> getSGPRSpillVGPRs ( ) ) { if ( ! Reg . FI . hasValue ( ) ) continue ; const <ISA_LIT>RegisterInfo & TRI = TII


<s> Register ScratchExecCopy ; if ( FuncInfo -> SGPRForFPSaveRestoreCopy ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , FuncInfo -> SGPRForFPSaveRestoreCopy ) . addReg ( FramePtrReg ) . setMIFlag ( MachineInstr :: FrameSetup ) ; for ( MachineBasicBlock & MBB : MF ) MBB . addLiveIn ( FuncInfo -> SGPRForFPSaveRestoreCopy ) ; } for ( const <ISA_LIT>MachineFunctionInfo :: SGPRSpillVGPRCSR & Reg : FuncInfo -> getSGPRSpillVGPRs ( ) ) { if ( ! Reg . FI . hasValue ( ) ) continue ; if ( ! ScratchExecCopy ) { if ( LiveRegs . empty ( ) ) { LiveRegs . init ( TRI ) ; LiveRegs . addLiveIns ( MBB ) ; if ( FuncInfo -> SGPRForFPSaveRestoreCopy ) LiveRegs . removeReg ( FuncInfo -> SGPRForFPSaveRestoreCopy ) ; } ScratchExecCopy = findScratchNonCalleeSaveRegister ( MRI , LiveRegs , * TRI . getWaveMaskRegClass ( ) ) ; assert ( FuncInfo -> SGPRForFPSaveRestoreCopy != ScratchExecCopy ) ; const unsigned OrSaveExec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , MBBI , DL , TII -> get ( OrSaveExec ) , ScratchExecCopy ) . addImm ( - <NUM_LIT> ) ; } buildPrologSpill ( LiveRegs , MBB , MBBI , TII , Reg . VGPR , FuncInfo -> getScratchRSrcReg ( ) , StackPtrReg , Reg . FI . getValue ( ) ) ; } if ( ScratchExecCopy != <STR_LIT> ) { unsigned ExecMov = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; MCRegister Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , MBBI , DL , TII -> get ( ExecMov ) , Exec ) . addReg ( ScratchExecCopy , RegState :: Kill ) ; LiveRegs . addReg ( ScratchExecCopy ) ; } if ( FuncInfo -> FramePointerSaveIndex ) { const int FI = FuncInfo -> FramePointerSaveIndex .


<s> if ( MFI -> isEntryFunction ( ) ) emitEntryFunctionPrologue


<s> if ( MFI -> isEntryFunction ( ) ) emitEntryFunctionPrologue (


<s> if ( ST . isMesaGfxShader ( MF ) || ( PreloadedPrivateBufferReg == <STR_LIT> ) ) { assert ( ! ST . isAmdCodeObjectV2 ( MF ) ) ; const MCInstrDesc & SMovB32 = TII -> get ( <STR_LIT> ) ; unsigned Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; if ( MFI -> hasImplicitBufferPtr ( ) ) { unsigned Rsrc01 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; if ( <STR_LIT> ( MF . getFunction ( ) . getCallingConv ( ) ) ) { const MCInstrDesc & Mov64 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , Mov64 , Rsrc01 ) . addReg ( MFI -> getImplicitBufferPtrUserSGPR ( ) ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } else { const MCInstrDesc & LoadDwordX2 = TII -> get ( <STR_LIT> ) ; PointerType * PtrTy = PointerType :: get ( Type :: getInt64Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; auto MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant | MachineMemOperand :: MODereferenceable , <NUM_LIT> , <NUM_LIT> ) ; BuildMI ( MBB , I , DL , LoadDwordX2 , Rsrc01 ) . addReg ( MFI -> getImplicitBufferPtrUserSGPR ( ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } } else { unsigned Rsrc0 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc1 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc0 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc1 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } BuildMI ( MBB , I , DL , SMovB32 , Rsrc2 ) . addImm (

Rsrc23 & <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ;


<s> PointerType * PtrTy = PointerType :: get ( Type :: getInt64Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; const MCInstrDesc & LoadDwordX4 = TII -> get ( <STR_LIT> ) ; auto MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant | MachineMemOperand :: MODereferenceable , <NUM_LIT> , <NUM_LIT> ) ; BuildMI ( MBB , I , DL , LoadDwordX4 , ScratchRsrcReg ) . addReg ( Rsrc01 ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) . addMemOperand ( MMO ) ; return ; } if ( ST . isMesaGfxShader ( MF ) || ( PreloadedPrivateBufferReg == <STR_LIT> ) ) { assert ( ! ST . isAmdCodeObjectV2 ( MF ) ) ; const MCInstrDesc & SMovB32 = TII -> get ( <STR_LIT> ) ; unsigned Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; if ( MFI -> hasImplicitBufferPtr ( ) ) { unsigned Rsrc01 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; if ( <STR_LIT> ( MF . getFunction ( ) . getCallingConv ( ) ) ) { const MCInstrDesc & Mov64 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , Mov64 , Rsrc01 ) . addReg ( MFI -> getImplicitBufferPtrUserSGPR ( ) ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } else


<s> bool MayNeedScavengingEmergencySlot = MFI -> hasStackObjects ( ) ; assert ( ( RS || ! MayNeedScavengingEmergencySlot ) && <STR_LIT> ) ; if (


<s> unsigned ScratchExecCopy = <STR_LIT> ; for ( const <ISA_LIT>MachineFunctionInfo :: SGPRSpillVGPRCSR & Reg : FuncInfo -> getSGPRSpillVGPRs ( ) ) { if ( ! Reg . FI . hasValue ( ) ) continue ; if ( ScratchExecCopy == <STR_LIT> ) { LivePhysRegs LiveRegs ( * ST . getRegisterInfo ( ) ) ; LiveRegs . addLiveIns ( MBB ) ; ScratchExecCopy = findScratchNonCalleeSaveRegister ( MF , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , ScratchExecCopy ) . addImm ( - <NUM_LIT> ) ; } TII -> loadRegFromStackSlot ( MBB , MBBI , Reg . VGPR , Reg . FI . getValue ( ) , & <STR_LIT> , & TII -> getRegisterInfo


<s> } else if ( NeedFP ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , FramePtrReg ) . addReg ( StackPtrReg ) . setMIFlag ( MachineInstr :: FrameSetup ) ; } if ( RoundedSize != <NUM_LIT> && hasSP ( MF ) ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , StackPtrReg ) . addReg ( StackPtrReg ) . addImm ( RoundedSize * ST . getWavefrontSize ( ) ) . setMIFlag ( MachineInstr :: FrameSetup ) ; } unsigned ScratchExecCopy = <STR_LIT> ; for ( const <ISA_LIT>MachineFunctionInfo :: SGPRSpillVGPRCSR & Reg : FuncInfo -> getSGPRSpillVGPRs ( ) ) { if ( ! Reg . FI . hasValue ( ) ) continue ; if ( ScratchExecCopy == <STR_LIT> ) { if ( LiveRegs . empty ( ) ) { LiveRegs . init ( TRI ) ; LiveRegs . addLiveIns ( MBB ) ; } ScratchExecCopy = findScratchNonCalleeSaveRegister ( MF , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , ScratchExecCopy ) . addImm ( - <NUM_LIT> ) ; } TII -> storeRegToStackSlot ( MBB , MBBI , Reg . VGPR , true , Reg . FI . getValue ( ) , & <STR_LIT> , & TII -> getRegisterInfo


<s> static unsigned findScratchNonCalleeSaveRegister ( MachineFunction & MF , LivePhysRegs & LiveRegs , const TargetRegisterClass & RC ) { const GCNSubtarget & Subtarget = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo & TRI = * Subtarget . getRegisterInfo ( ) ; const MCPhysReg * CSRegs = TRI . getCalleeSavedRegs ( & MF ) ; for ( unsigned i = <NUM_LIT> ; CSRegs [ i ] ; ++ i ) LiveRegs . addReg ( CSRegs [ i ] ) ; MachineRegisterInfo & MRI = MF


<s> case CallingConv :: <ISA_LIT>_HS : case CallingConv :: <ISA_LIT>_GS : GitPtrLo = <STR_LIT> ; break ; default : break ; } } MF . getRegInfo ( ) . addLiveIn ( GitPtrLo ) ; MF . front ( ) . addLiveIn ( GitPtrLo ) ; BuildMI ( MBB , I , DL , SMovB32 , RsrcLo ) . addReg ( GitPtrLo ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; PointerType * PtrTy = PointerType :: get ( Type :: getInt64Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; const MCInstrDesc & LoadDwordX4 = TII -> get ( <STR_LIT> ) ; auto MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant | MachineMemOperand :: MODereferenceable , <NUM_LIT> , <NUM_LIT> ) ; unsigned Offset = Fn . getCallingConv ( ) == CallingConv :: <ISA_LIT>_CS ? <NUM_LIT> : <NUM_LIT> ; BuildMI ( MBB , I , DL , LoadDwordX4 , ScratchRsrcReg ) . addReg ( Rsrc01 ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) . addMemOperand ( MMO ) ; return ; } if ( ST . isMesaGfxShader ( Fn ) || ( PreloadedPrivateBufferReg == <STR_LIT> ) ) { assert ( ! ST . isAmdHsaOrMesa ( Fn ) ) ; const MCInstrDesc & SMovB32 = TII -> get ( <STR_LIT> ) ; unsigned Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; if ( MFI -> hasImplicitBufferPtr ( ) ) { unsigned Rsrc01 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; if ( <STR_LIT> ( MF . getFunction ( ) . getCallingConv ( ) ) ) { const MCInstrDesc & Mov64 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , Mov64 , Rsrc01 ) . addReg ( MFI -> getImplicitBufferPtrUserSGPR ( ) ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } else { const MCInstrDesc & LoadDwordX2 = TII -> get ( <STR_LIT> ) ; PointerType * PtrTy = PointerType :: get ( Type :: getInt64Ty ( MF . getFunction ( ) . getContext ( ) ) , <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; auto MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant | MachineMemOperand :: MODereferenceable , <NUM_LIT>


<s> BuildMI ( MBB , I , DL , SMovB32 , RsrcLo ) . addReg ( GitPtrLo ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; MachinePointerInfo PtrInfo ( <ISA_LIT>AS :: CONSTANT_ADDRESS ) ; const MCInstrDesc & LoadDwordX4 = TII -> get ( <STR_LIT> ) ; auto MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant | MachineMemOperand :: MODereferenceable , <NUM_LIT> , <NUM_LIT> ) ; unsigned Offset = Fn . getCallingConv ( ) == CallingConv :: <ISA_LIT>_CS ? <NUM_LIT> : <NUM_LIT> ; const GCNSubtarget & Subtarget = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned EncodedOffset = <STR_LIT> ( Subtarget , Offset ) ; BuildMI ( MBB , I , DL , LoadDwordX4 , ScratchRsrcReg ) . addReg ( Rsrc01 ) . addImm ( EncodedOffset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) . addMemOperand ( MMO ) ; return ; } if ( ST . isMesaGfxShader ( Fn ) || ( PreloadedPrivateBufferReg == <STR_LIT> ) ) { assert ( ! ST . isAmdHsaOrMesa ( Fn ) ) ; const MCInstrDesc & SMovB32 = TII -> get ( <STR_LIT> ) ; Register Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; Register Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; if ( MFI -> hasImplicitBufferPtr ( ) ) { Register Rsrc01 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; if ( <STR_LIT> ( MF . getFunction ( ) . getCallingConv ( ) ) ) { const MCInstrDesc & Mov64 = TII -> get ( <STR_LIT>


<s> unsigned FlatScratchInitReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: FLAT_SCRATCH_INIT ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MRI . addLiveIn ( FlatScratchInitReg ) ; MBB . addLiveIn ( FlatScratchInitReg ) ; unsigned FlatScrInitHi = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitHi , RegState :: Kill ) ; unsigned FlatScrInitLo = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitLo ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitLo , RegState :: Kill ) .


<s> unsigned PreloadedPrivateBufferReg = <STR_LIT> ; if ( ST . isAmdCodeObjectV2 ( MF ) || ST . isMesaGfxShader ( MF ) ) { PreloadedPrivateBufferReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: PRIVATE_SEGMENT_BUFFER ) ; } bool OffsetRegUsed = ! MRI . use_empty ( ScratchWaveOffsetReg ) ; bool ResourceRegUsed = ! MRI . use_empty ( ScratchRsrcReg ) ; if ( OffsetRegUsed ) { assert ( PreloadedScratchWaveOffsetReg != <STR_LIT> && <STR_LIT> ) ; MRI . addLiveIn ( PreloadedScratchWaveOffsetReg ) ; MBB . addLiveIn ( PreloadedScratchWaveOffsetReg ) ; } if ( ResourceRegUsed && PreloadedPrivateBufferReg != <STR_LIT> ) { assert ( ST . isAmdCodeObjectV2 ( MF ) || ST . isMesaGfxShader ( MF ) ) ; MRI . addLiveIn ( PreloadedPrivateBufferReg ) ; MBB . addLiveIn ( PreloadedPrivateBufferReg ) ; } for ( MachineBasicBlock & OtherBB : MF ) { if ( & OtherBB == & MBB ) continue ; if ( OffsetRegUsed ) OtherBB . addLiveIn ( ScratchWaveOffsetReg ) ; if ( ResourceRegUsed ) OtherBB . addLiveIn ( ScratchRsrcReg ) ; } DebugLoc DL ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; bool CopyBuffer = ResourceRegUsed && PreloadedPrivateBufferReg != <STR_LIT> && ST . isAmdCodeObjectV2 ( MF ) && ScratchRsrcReg != PreloadedPrivateBufferReg ; bool CopyBufferFirst = TRI -> isSubRegisterEq ( PreloadedPrivateBufferReg , ScratchWaveOffsetReg ) ; if ( CopyBuffer && CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } if ( OffsetRegUsed && PreloadedScratchWaveOffsetReg != ScratchWaveOffsetReg ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , RegState :: Kill ) ; } if ( CopyBuffer && ! CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } if ( ResourceRegUsed && ( ST . isMesaGfxShader ( MF ) || ( PreloadedPrivateBufferReg == <STR_LIT> ) ) ) { assert ( ! ST . isAmdCodeObjectV2 ( MF ) ) ; const MCInstrDesc & SMovB32 = TII -> get ( <STR_LIT> ) ; unsigned Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; if ( MFI -> hasPrivateMemoryInputPtr ( ) ) { unsigned Rsrc01 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; if ( <STR_LIT> ( MF . getFunction ( )


<s> return makeArrayRef ( <STR_LIT> . begin ( ) , TRI

-> getMaxNumSGPRs ( MF ) / <NUM_LIT> ) ;


<s> static ArrayRef < MCPhysReg > getAllSGPRs ( const MachineFunction & MF , const <ISA_LIT>RegisterInfo * TRI ) { return makeArrayRef ( <STR_LIT> . begin ( ) , TRI


<s> return makeArrayRef ( <STR_LIT> . begin ( ) , TRI -> getMaxNumSGPRs ( MF )


<s> if ( ST . hasSGPRInitBug ( ) || ScratchRsrcReg != TRI -> reservedPrivateSegmentBufferReg ( MF ) ) return ScratchRsrcReg ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned NumPreloaded = ( MFI -> getNumPreloadedSGPRs ( ) + <NUM_LIT> ) / <NUM_LIT> ; ArrayRef < MCPhysReg > AllSGPR128s = getAllSGPR128 ( MF , TRI ) ; AllSGPR128s = AllSGPR128s . slice ( std :: min ( static_cast < unsigned > ( AllSGPR128s . size ( ) ) , NumPreloaded ) ) ; for ( MCPhysReg Reg : AllSGPR128s ) { if ( ! MRI . isPhysRegUsed ( Reg ) &&


<s> unsigned ScratchRsrcReg = MFI -> getScratchRSrcReg ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned NumPreloaded = MFI -> getNumPreloadedSGPRs ( ) ; ArrayRef < MCPhysReg > AllSGPRs = getAllSGPRs ( MF , TRI ) ; if ( NumPreloaded > AllSGPRs . size ( ) ) return ScratchWaveOffsetReg ; AllSGPRs = AllSGPRs . slice ( NumPreloaded ) ; if ( AllSGPRs . size ( ) < <NUM_LIT> ) return ScratchWaveOffsetReg ; for ( MCPhysReg Reg : AllSGPRs . drop_back


<s> void <ISA_LIT>FrameLowering :: processFunctionBeforeFrameFinalized ( MachineFunction & MF , RegScavenger * RS ) const { MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; if ( ! MFI . hasStackObjects ( ) ) return ; bool MayNeedScavengingEmergencySlot = MFI . hasStackObjects ( ) ; assert ( ( RS || ! MayNeedScavengingEmergencySlot ) && <STR_LIT>


<s> if ( CopyBuffer && CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } if ( OffsetRegUsed && PreloadedScratchWaveOffsetReg != ScratchWaveOffsetReg ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , RegState :: Kill ) ; } if ( CopyBuffer && ! CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } if ( ResourceRegUsed && ( ST . isMesaGfxShader ( MF ) || ( PreloadedPrivateBufferReg == <STR_LIT> ) ) ) { assert ( ! ST . isAmdCodeObjectV2 ( MF ) ) ; const MCInstrDesc & SMovB32 = TII -> get ( <STR_LIT> ) ; unsigned Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; if ( MFI -> hasPrivateMemoryInputPtr ( ) ) { unsigned Rsrc01 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; if ( <STR_LIT> ( MF . getFunction ( ) -> getCallingConv ( ) ) ) { const MCInstrDesc & Mov64 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , Mov64 , Rsrc01 ) . addReg ( PreloadedPrivateBufferReg ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } else { const MCInstrDesc & LoadDwordX2 = TII -> get ( <STR_LIT> ) ; PointerType * PtrTy = PointerType :: get ( Type :: getInt64Ty ( MF . getFunction ( ) -> getContext ( ) ) , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; auto MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant | MachineMemOperand :: MODereferenceable , <NUM_LIT> , <NUM_LIT> ) ; BuildMI ( MBB , I , DL , LoadDwordX2 , Rsrc01 ) . addReg ( PreloadedPrivateBufferReg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } } else { unsigned Rsrc0 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc1 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT>


<s> const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( MFI -> isEntryFunction ( ) ) return ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; SavedRegs . reset ( MFI -> getStackPtrOffsetReg ( ) ) ; const BitVector AllSavedRegs = SavedRegs


<s> MachineBasicBlock :: iterator MBBI = MBB . getFirstTerminator ( ) ; DebugLoc DL ; unsigned ScratchExecCopy = <STR_LIT> ; for ( const <ISA_LIT>MachineFunctionInfo :: SGPRSpillVGPRCSR & Reg : FuncInfo -> getSGPRSpillVGPRs ( ) ) { if ( ! Reg . FI . hasValue ( ) ) continue ; if ( ScratchExecCopy == <STR_LIT> ) { LivePhysRegs LiveRegs ( * ST . getRegisterInfo ( ) ) ; LiveRegs . addLiveIns ( MBB ) ; ScratchExecCopy = findScratchNonCalleeSaveRegister ( MF , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , ScratchExecCopy ) . addImm ( - <NUM_LIT> ) ; } TII -> loadRegFromStackSlot ( MBB , MBBI , Reg . VGPR , Reg . FI . getValue ( ) , & <STR_LIT> , & TII -> getRegisterInfo ( ) ) ; } if ( ScratchExecCopy != <STR_LIT> ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( ScratchExecCopy ) ; } if ( hasFP ( MF ) ) { const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; uint32_t NumBytes = MFI . getStackSize ( ) ; uint32_t RoundedSize = FuncInfo -> isStackRealigned ( ) ? NumBytes + MFI . getMaxAlignment ( ) : NumBytes ; const unsigned StackPtrReg = FuncInfo -> getStackPtrOffsetReg ( ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , StackPtrReg ) . addReg ( StackPtrReg ) . addImm ( RoundedSize *

ST . getWavefrontSize ( ) ) . setMIFlag ( MachineInstr :: FrameDestroy ) ;


<s> if ( MFI . getStackSize ( ) != <NUM_LIT> ) return true ; if ( MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> isEntryFunction ( ) ) return true ; if ( MF . getTarget ( ) . Options . DisableFramePointerElim ( MF ) ) return true ; } return MFI . hasVarSizedObjects ( ) || MFI . isFrameAddressTaken ( ) || MFI . hasStackMap ( ) || MFI . hasPatchPoint ( ) || MF .

getSubtarget < GCNSubtarget > ( ) . getRegisterInfo ( ) -> needsStackRealignment ( MF ) ;


<s> AllSGPRSpilledToVGPRs = true ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( TII -> isSGPRSpill ( MI ) ) { int FI = TII -> getNamedOperand ( MI , <STR_LIT> :: addr ) -> getIndex ( ) ; assert ( MFI . getStackID ( FI ) == <STR_LIT> ) ; if ( FuncInfo -> allocateSGPRSpillToVGPR ( MF , FI ) ) { bool Spilled = TRI . eliminateSGPRToVGPRSpillFrameIndex ( MI , FI , RS ) ; ( void ) Spilled ; assert ( Spilled && <STR_LIT> ) ; } else AllSGPRSpilledToVGPRs = false ; } } } } FuncInfo -> removeSGPRToVGPRFrameIndices ( MFI ) ; if ( FuncInfo -> hasNonSpillStackObjects ( ) || FuncInfo -> hasSpilledVGPRs ( ) || ! AllSGPRSpilledToVGPRs || ! allStackObjectsAreDead ( MFI ) ) { assert ( RS && <STR_LIT> ) ; if ( FuncInfo -> isEntryFunction


<s> int64_t Offset = MFI . getObjectOffset ( FI ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( MachinePointerInfo :: getFixedStack ( * MF , FI ) , MachineMemOperand :: MOStore , <NUM_LIT> , MFI . getObjectAlign ( FI ) ) ; if ( isUInt < <NUM_LIT> > ( Offset ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( SpillReg , RegState :: Kill ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; return ; } LiveRegs . addReg ( SpillReg ) ; MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister ( MF -> getRegInfo ( ) , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , OffsetReg ) . addImm ( Offset ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( SpillReg , RegState :: Kill ) . addReg ( OffsetReg , RegState :: Kill ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( <NUM_LIT> ) . addImm


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const bool SpillVGPRToAGPR = ST . hasMAIInsts ( ) && FuncInfo -> hasSpilledVGPRs ( ) && EnableSpillVGPRToAGPR ; if ( SpillVGPRToAGPR ) { BitVector SpillFIs ( MFI . getObjectIndexEnd ( ) , false ) ; bool SeenDbgInstr = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( MI . isDebugInstr ( ) ) SeenDbgInstr = true ; if ( TII -> isVGPRSpill ( MI ) ) { unsigned FIOp = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; int FI = MI . getOperand ( FIOp ) . getIndex ( ) ; Register VReg = TII -> getNamedOperand ( MI , <STR_LIT> :: vdata ) -> getReg ( ) ; if ( FuncInfo -> allocateVGPRSpillToAGPR ( MF , FI , TRI -> isAGPR ( MRI


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const bool SpillVGPRToAGPR = ST . hasMAIInsts ( ) && FuncInfo -> hasSpilledVGPRs ( ) && EnableSpillVGPRToAGPR ; if ( SpillVGPRToAGPR ) { BitVector SpillFIs ( MFI . getObjectIndexEnd ( ) , false ) ; bool SeenDbgInstr = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( MI . isDebugInstr ( ) ) SeenDbgInstr = true ; if ( TII -> isVGPRSpill ( MI ) ) { unsigned FIOp = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; int FI = MI . getOperand ( FIOp ) . getIndex ( ) ; Register VReg = TII -> getNamedOperand ( MI , <STR_LIT> :: vdata ) -> getReg ( ) ; if ( FuncInfo -> allocateVGPRSpillToAGPR ( MF , FI , TRI -> isAGPR ( MRI , VReg ) ) ) { RS -> enterBasicBlock ( MBB ) ; TRI -> eliminateFrameIndex ( MI , <NUM_LIT> , FIOp , RS ) ; SpillFIs . set ( FI ) ; continue ; } } } } for ( MachineBasicBlock & MBB : MF


<s> const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( MFI -> isEntryFunction ( ) ) return ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; SavedRegs . reset ( MFI -> getStackPtrOffsetReg ( ) ) ; const BitVector AllSavedRegs


<s> if ( <ISA_LIT>InstrInfo :: isLegalMUBUFImmOffset ( Offset ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( SpillReg , RegState :: Kill ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; return ; } LiveRegs . addReg ( SpillReg ) ; MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister ( MF -> getRegInfo ( ) , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , OffsetReg ) . addImm ( Offset ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( SpillReg , RegState :: Kill ) . addReg ( OffsetReg , RegState :: Kill ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm

( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> LiveRegs . addReg ( SpillReg ) ; MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister ( MF -> getRegInfo ( ) , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , OffsetReg ) . addImm ( Offset ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( SpillReg , RegState :: Kill ) . addReg ( OffsetReg , RegState :: Kill ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT>

) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> DebugLoc DL ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; bool CopyBuffer = ResourceRegUsed && PreloadedPrivateBufferReg != <STR_LIT> && ST . isAmdHsaOrMesa ( F ) && ScratchRsrcReg != PreloadedPrivateBufferReg ; bool CopyBufferFirst = TRI -> isSubRegisterEq ( PreloadedPrivateBufferReg , ScratchWaveOffsetReg ) ; if ( CopyBuffer && CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } unsigned SPReg = MFI -> getStackPtrOffsetReg ( ) ; assert ( SPReg != <STR_LIT> ) ; const bool HasFP = hasFP ( MF ) ; if ( HasFP || OffsetRegUsed ) { assert ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , HasFP ? RegState :: Kill : <NUM_LIT> ) ; } if ( CopyBuffer && ! CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } if ( ResourceRegUsed ) { emitEntryFunctionScratchSetup ( ST , MF , MBB , MFI , I , PreloadedPrivateBufferReg , ScratchRsrcReg ) ; } if ( HasFP ) { DebugLoc DL ; const MachineFrameInfo & FrameInfo = MF . getFrameInfo (


<s> if ( ScratchWaveOffsetReg == TRI -> reservedPrivateSegmentWaveByteOffsetReg ( MF ) ) { MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned NumPreloaded = MFI -> getNumPreloadedSGPRs ( ) ; for ( MCPhysReg Reg : getAllSGPRs ( ) . drop_back ( <NUM_LIT> ) . slice ( NumPreloaded ) ) { if ( ! MRI . isPhysRegUsed ( Reg ) ) { if ( ! MRI . isAllocatable ( Reg ) || TRI -> isSubRegisterEq ( ScratchRsrcReg , Reg ) ) continue ; MRI . replaceRegWith ( ScratchWaveOffsetReg , Reg ) ; ScratchWaveOffsetReg = Reg ; MFI -> setScratchWaveOffsetReg ( ScratchWaveOffsetReg ) ; break ; } } } } assert ( ! TRI -> isSubRegister ( ScratchRsrcReg , ScratchWaveOffsetReg ) ) ; const MCInstrDesc & SMovB32 = TII -> get ( <STR_LIT> ) ; DebugLoc DL ; if ( PreloadedScratchWaveOffsetReg != ScratchWaveOffsetReg ) { BuildMI ( MBB , I , DL , SMovB32 , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , RegState :: Kill ) ; } if ( ST . isAmdHsaOS ( ) ) { assert ( ! TRI -> isSubRegisterEq ( PreloadedPrivateBufferReg , ScratchRsrcReg ) && ! TRI -> isSubRegisterEq ( PreloadedPrivateBufferReg , ScratchWaveOffsetReg ) ) ; unsigned Rsrc01 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc23 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Lo = TRI -> getSubReg ( PreloadedPrivateBufferReg , <STR_LIT> ) ; unsigned Hi = TRI -> getSubReg ( PreloadedPrivateBufferReg , <STR_LIT> ) ; const MCInstrDesc & SMovB64 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , SMovB64 , Rsrc01 ) . addReg ( Lo , RegState :: Kill ) ; BuildMI ( MBB , I , DL , SMovB64 , Rsrc23 ) . addReg ( Hi , RegState :: Kill ) ; } else { unsigned Rsrc0 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc1 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc0 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc1 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc2 ) . addImm ( Rsrc23 & <NUM_LIT> ) . addReg


<s> if ( TII -> isSGPRSpill ( MI ) ) { int FI = TII -> getNamedOperand ( MI , <STR_LIT> :: addr ) -> getIndex ( ) ; assert ( MFI . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; if ( FuncInfo -> allocateSGPRSpillToVGPR ( MF , FI ) ) { bool Spilled = TRI . eliminateSGPRToVGPRSpillFrameIndex ( MI , FI , RS ) ; ( void ) Spilled ; assert ( Spilled && <STR_LIT> ) ; } else AllSGPRSpilledToVGPRs = false ; } } } } FuncInfo -> removeSGPRToVGPRFrameIndices ( MFI ) ; if ( FuncInfo -> hasNonSpillStackObjects ( ) || FuncInfo -> hasSpilledVGPRs ( ) || ! AllSGPRSpilledToVGPRs || ! allStackObjectsAreDead ( MFI ) ) { assert ( RS && <STR_LIT> ) ; if ( FuncInfo -> isEntryFunction ( ) ) { int ScavengeFI = MFI . CreateFixedObject ( TRI . getSpillSize ( <STR_LIT> ) , <NUM_LIT> , false ) ; RS -> addScavengingFrameIndex (


<s> if ( ! MF . getFrameInfo ( ) -> hasStackObjects ( ) ) return ; assert ( & MF . front ( ) == & MBB && <STR_LIT> ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( hasOnlySGPRSpills ( MFI , MF . getFrameInfo ( ) ) ) return ; const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ; const <ISA_LIT>RegisterInfo * TRI = & TII -> getRegisterInfo ( ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; unsigned ScratchRsrcReg = MFI -> getScratchRSrcReg ( ) ; assert ( ScratchRsrcReg != <STR_LIT> ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; assert ( ScratchWaveOffsetReg != <STR_LIT> ) ; unsigned PreloadedScratchWaveOffsetReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: PRIVATE_SEGMENT_WAVE_BYTE_OFFSET ) ; unsigned PreloadedPrivateBufferReg = <STR_LIT> ; if ( ST . isAmdHsaOS ( ) ) { PreloadedPrivateBufferReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: PRIVATE_SEGMENT_BUFFER ) ; } if ( MFI -> hasFlatScratchInit ( ) ) { DebugLoc DL ; unsigned FlatScratchInitReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: FLAT_SCRATCH_INIT ) ; MRI . addLiveIn ( FlatScratchInitReg ) ; MBB . addLiveIn ( FlatScratchInitReg ) ; unsigned FlatScrInitHi = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitHi , RegState :: Kill ) ; unsigned FlatScrInitLo = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitLo ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitLo , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; } if ( ScratchRsrcReg == PreloadedPrivateBufferReg ) { assert ( ScratchWaveOffsetReg == PreloadedScratchWaveOffsetReg &&


<s> return FuncInfo -> hasSpilledSGPRs ( ) && ( ! FuncInfo -> hasSpilledVGPRs ( ) && ! FuncInfo


<s> MachineMemOperand * MMO = MF -> getMachineMemOperand ( MachinePointerInfo :: getFixedStack ( * MF , FI ) , MachineMemOperand :: MOLoad , <NUM_LIT> , MFI . getObjectAlign ( FI ) ) ; if ( isUInt < <NUM_LIT> > ( Offset ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SpillReg ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; return ; } MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister ( MF -> getRegInfo ( ) , LiveRegs


<s> assert ( isUInt < <NUM_LIT> > ( Amount ) && <STR_LIT> ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned SPReg = MFI -> getStackPtrOffsetReg ( ) ; unsigned Op = IsDestroy ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , I , DL , TII -> get ( Op ) , SPReg ) . addReg ( SPReg ) . addImm ( Amount * ST . getWavefrontSize ( ) ) ; } else if ( CalleePopAmount != <NUM_LIT> ) { llvm_unreachable ( <STR_LIT> ) ; } return MBB . erase


<s> } if ( FuncInfo -> FramePointerSaveIndex ) { const int FI = FuncInfo -> FramePointerSaveIndex . getValue ( ) ; assert ( ! MF . getFrameInfo ( ) . isDeadObjectIndex ( FI ) && MF . getFrameInfo ( ) . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; ArrayRef < <ISA_LIT>MachineFunctionInfo :: SpilledReg > Spill = FuncInfo -> getSGPRToVGPRSpills ( FI ) ; assert ( Spill . size ( ) == <NUM_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , FuncInfo -> getFrameOffsetReg ( ) ) . addReg ( Spill [ <NUM_LIT> ] . VGPR ) . addImm ( Spill [ <NUM_LIT> ] . Lane ) ; } unsigned ScratchExecCopy = <STR_LIT> ; for ( const <ISA_LIT>MachineFunctionInfo :: SGPRSpillVGPRCSR & Reg : FuncInfo -> getSGPRSpillVGPRs ( ) ) { if ( ! Reg . FI . hasValue ( ) ) continue ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; if ( ScratchExecCopy == <STR_LIT> ) { if ( LiveRegs . empty ( ) ) { LiveRegs . init ( * ST . getRegisterInfo ( ) ) ; LiveRegs . addLiveOuts


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MachineBasicBlock :: iterator MBBI = MBB . getFirstTerminator ( ) ; LivePhysRegs LiveRegs ; DebugLoc DL ; const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; uint32_t NumBytes = MFI . getStackSize ( ) ; uint32_t RoundedSize = FuncInfo -> isStackRealigned ( ) ? NumBytes + MFI . getMaxAlign ( ) . value ( ) : NumBytes ; if ( RoundedSize != <NUM_LIT> && hasFP ( MF ) ) { const unsigned StackPtrReg = FuncInfo -> getStackPtrOffsetReg ( ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , StackPtrReg ) . addReg ( StackPtrReg ) . addImm ( RoundedSize * ST . getWavefrontSize ( ) ) . setMIFlag ( MachineInstr :: FrameDestroy ) ; } if ( FuncInfo -> SGPRForFPSaveRestoreCopy != <STR_LIT> ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , FuncInfo -> getFrameOffsetReg ( ) ) . addReg ( FuncInfo -> SGPRForFPSaveRestoreCopy ) . setMIFlag ( MachineInstr :: FrameSetup ) ; } if ( FuncInfo -> FramePointerSaveIndex ) { const int FI = FuncInfo -> FramePointerSaveIndex . getValue ( ) ; assert ( ! MF . getFrameInfo ( ) . isDeadObjectIndex ( FI ) && MF . getFrameInfo ( ) . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; ArrayRef < <ISA_LIT>MachineFunctionInfo :: SpilledReg > Spill = FuncInfo -> getSGPRToVGPRSpills ( FI ) ; assert ( Spill . size ( ) == <NUM_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , FuncInfo -> getFrameOffsetReg ( ) ) . addReg ( Spill [ <NUM_LIT> ] . VGPR ) . addImm ( Spill [ <NUM_LIT> ] . Lane ) ; } unsigned ScratchExecCopy = <STR_LIT> ; for ( const <ISA_LIT>MachineFunctionInfo :: SGPRSpillVGPRCSR & Reg : FuncInfo ->


<s> SavedVGPRs . clearBitsNotInMask ( TRI -> getAllVGPRRegMask ( ) ) ; const bool WillHaveFP = FrameInfo . hasCalls ( ) && ( SavedVGPRs . any ( ) || ! allStackObjectsAreDead ( FrameInfo ) ) ; for ( auto SSpill : MFI -> getSGPRSpillVGPRs ( ) ) SavedVGPRs . reset ( SSpill . VGPR ) ; const bool HasFP = WillHaveFP || hasFP ( MF ) ; if ( ! HasFP ) return ; if ( MFI -> haveFreeLanesForSGPRSpill ( MF , <NUM_LIT> ) ) { int NewFI = MF . getFrameInfo ( ) . CreateStackObject ( <NUM_LIT> , <NUM_LIT> , true , nullptr , TargetStackID :: SGPRSpill ) ; if ( ! MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) llvm_unreachable ( <STR_LIT> ) ; MFI -> FramePointerSaveIndex


<s> const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo


<s> void <ISA_LIT>FrameLowering :: determineCalleeSavesSGPR ( MachineFunction & MF , BitVector & SavedRegs , RegScavenger * RS ) const { TargetFrameLowering :: determineCalleeSaves ( MF , SavedRegs , RS ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; SavedRegs . reset ( MFI -> getStackPtrOffsetReg ( ) ) ; SavedRegs . clearBitsInMask ( TRI ->


<s> int NewFI = MF . getFrameInfo ( ) . CreateStackObject ( TRI -> getSpillSize ( * RC ) , TRI -> getSpillAlign ( * RC ) , true , nullptr , TargetStackID :: SGPRSpill ) ; if ( TRI -> spillSGPRToVGPR ( ) && MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) { FI = NewFI ; } else { MF . getFrameInfo ( ) . RemoveStackObject ( NewFI ) ; FI = MF . getFrameInfo ( ) . CreateSpillStackObject ( TRI -> getSpillSize ( * RC )

, Align ( TRI -> getSpillAlign ( * RC ) ) ) ;


<s> if ( MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) { FI = NewFI ; } } else { int NewFI = MF . getFrameInfo ( ) . CreateStackObject ( TRI -> getSpillSize ( * RC ) , TRI -> getSpillAlign ( * RC ) , true , nullptr , TargetStackID :: SGPRSpill ) ; if ( TRI -> spillSGPRToVGPR ( ) && MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) { FI =


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; for ( int I = MFI . getObjectIndexBegin ( ) , E = MFI . getObjectIndexEnd ( ) ; I != E ;


<s> static bool allStackObjectsAreDead ( const MachineFunction & MF ) { const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo <