<s> if ( CopyBuffer && CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } if ( OffsetRegUsed && PreloadedScratchWaveOffsetReg != ScratchWaveOffsetReg ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , RegState :: Kill ) ; } if ( CopyBuffer && ! CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } if ( ResourceRegUsed && ( ST . isMesaGfxShader ( MF ) || ( PreloadedPrivateBufferReg == <STR_LIT> ) ) ) { assert ( ! ST . isAmdCodeObjectV2 ( MF ) ) ; const MCInstrDesc & SMovB32 = TII -> get ( <STR_LIT> ) ; unsigned Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; if ( MFI -> hasPrivateMemoryInputPtr ( ) ) { unsigned Rsrc01 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; if ( <STR_LIT> ( MF . getFunction ( ) -> getCallingConv ( ) ) ) { const MCInstrDesc & Mov64 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , Mov64 , Rsrc01 ) . addReg ( PreloadedPrivateBufferReg ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } else { const MCInstrDesc & LoadDwordX2 = TII -> get ( <STR_LIT> ) ; PointerType * PtrTy = PointerType :: get ( Type :: getInt64Ty ( MF . getFunction ( ) -> getContext ( ) ) , AMDGPUA<ISA_LIT> . CONSTANT_ADDRESS ) ; MachinePointerInfo PtrInfo ( UndefValue :: get ( PtrTy ) ) ; auto MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad | MachineMemOperand :: MOInvariant | MachineMemOperand :: MODereferenceable , <NUM_LIT> , <NUM_LIT> ) ; BuildMI ( MBB , I , DL , LoadDwordX2 , Rsrc01 ) . addReg ( PreloadedPrivateBufferReg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; } } else { unsigned Rsrc0 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ;

unsigned Rsrc1 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ;


<s> const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( MFI -> isEntryFunction ( ) ) return ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; SavedRegs . reset ( MFI -> getStackPtrOffsetReg ( ) ) ;

const BitVector AllSavedRegs = SavedRegs ;


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const bool SpillVGPRToAGPR = ST . hasMAIInsts ( ) && FuncInfo -> hasSpilledVGPRs ( ) && EnableSpillVGPRToAGPR ; if ( SpillVGPRToAGPR ) { BitVector SpillFIs ( MFI . getObjectIndexEnd ( ) , false ) ; BitVector NonVGPRSpillFIs ( MFI . getObjectIndexEnd ( ) , false ) ; bool SeenDbgInstr = false ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : llvm :: make_early_inc_range ( MBB ) ) { int FrameIndex ; if ( MI . isDebugInstr ( ) ) SeenDbgInstr = true ; if ( TII -> isVGPRSpill ( MI ) ) { unsigned FIOp = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; int FI = MI . getOperand ( FIOp ) . getIndex ( ) ; Register VReg = TII -> getNamedOperand ( MI , <STR_LIT> :: vdata ) -> getReg ( ) ; if ( FuncInfo -> allocateVGPRSpillToAGPR ( MF , FI , TRI -> isAGPR ( MRI , VReg ) ) ) { RS -> enterBasicBlock ( MBB ) ; TRI -> eliminateFrameIndex ( MI , <NUM_LIT> , FIOp , RS ) ; SpillFIs . set ( FI ) ; continue ; } } else if ( TII -> isStoreToStackSlot ( MI , FrameIndex ) || TII -> isLoadFromStackSlot ( MI , FrameIndex ) ) if ( ! MFI . isFixedObjectIndex ( FrameIndex ) ) NonVGPRSpillFIs . set ( FrameIndex ) ; } } for ( unsigned FI : SpillFIs . set_bits ( ) ) if ( ! NonVGPRSpillFIs . test ( FI ) ) FuncInfo -> setVGPRToAGPRSpillDead ( FI ) ; for ( MachineBasicBlock & MBB : MF ) { for ( MCPhysReg Reg : FuncInfo -> getVGPRSpillAGPRs ( ) ) MBB . addLiveIn ( Reg ) ; for ( MCPhysReg Reg : FuncInfo -> getAGPRSpillVGPRs ( ) ) MBB . addLiveIn ( Reg ) ; MBB . sortUniqueLiveIns ( ) ; if ( ! SpillFIs . empty ( ) && SeenDbgInstr ) { for ( MachineInstr & MI : MBB ) { if ( MI . isDebugValue ( ) && MI . getOperand ( <NUM_LIT> ) . isFI ( ) && SpillFIs [ MI . getOperand ( <NUM_LIT> ) . getIndex ( ) ] ) { MI . getOperand ( <NUM_LIT> ) . ChangeToRegister ( Register ( ) , false ) ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; MachineBasicBlock :: iterator MBBI = MBB . getFirstTerminator ( ) ; DebugLoc DL ; unsigned ScratchExecCopy = <STR_LIT> ; for ( const <ISA_LIT>MachineFunctionInfo :: SGPRSpillVGPRCSR & Reg : FuncInfo -> getSGPRSpillVGPRs ( ) ) { if ( ! Reg . FI . hasValue ( ) ) continue ; if ( ScratchExecCopy == <STR_LIT> ) { LivePhysRegs LiveRegs ( * ST . getRegisterInfo ( ) ) ; LiveRegs . addLiveIns ( MBB ) ; ScratchExecCopy = findScratchNonCalleeSaveRegister ( MF , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , ScratchExecCopy ) . addImm ( - <NUM_LIT> ) ; } TII -> loadRegFromStackSlot ( MBB , MBBI , Reg . VGPR , Reg . FI . getValue ( ) , & <STR_LIT> , & TII -> getRegisterInfo ( ) ) ;


<s> MachineBasicBlock :: iterator MBBI = MBB . getFirstTerminator ( ) ; DebugLoc DL ; unsigned ScratchExecCopy = <STR_LIT> ; for ( const <ISA_LIT>MachineFunctionInfo :: SGPRSpillVGPRCSR & Reg : FuncInfo -> getSGPRSpillVGPRs ( ) ) { if ( ! Reg . FI . hasValue ( ) ) continue ; if ( ScratchExecCopy == <STR_LIT> ) { LivePhysRegs LiveRegs ( * ST . getRegisterInfo ( ) ) ; LiveRegs . addLiveIns ( MBB ) ; ScratchExecCopy = findScratchNonCalleeSaveRegister ( MF , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , ScratchExecCopy ) . addImm ( - <NUM_LIT> ) ; } TII -> loadRegFromStackSlot ( MBB , MBBI , Reg . VGPR , Reg . FI . getValue ( ) , & <STR_LIT> , & TII -> getRegisterInfo ( ) ) ; } if ( ScratchExecCopy != <STR_LIT> ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( ScratchExecCopy ) ; } if ( hasFP ( MF ) ) { const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; uint32_t NumBytes = MFI . getStackSize ( ) ; uint32_t RoundedSize = FuncInfo -> isStackRealigned ( ) ? NumBytes + MFI . getMaxAlignment ( ) : NumBytes ; const unsigned StackPtrReg = FuncInfo -> getStackPtrOffsetReg ( ) ;

BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , StackPtrReg ) . addReg ( StackPtrReg ) . addImm ( RoundedSize * ST . getWavefrontSize ( ) ) . setMIFlag ( MachineInstr :: FrameDestroy ) ;


<s> if ( MFI . getStackSize ( ) != <NUM_LIT> ) return true ; if ( MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> isEntryFunction ( ) ) return true ; if ( MF . getTarget ( ) . Options . DisableFramePointerElim ( MF ) ) return true ; }

return MFI . hasVarSizedObjects ( ) || MFI . isFrameAddressTaken ( ) || MFI . hasStackMap ( ) || MFI . hasPatchPoint ( ) || MF . getSubtarget < GCNSubtarget > ( ) . getRegisterInfo ( ) -> needsStackRealignment ( MF ) ;


<s> if ( MFI . getStackSize ( ) != <NUM_LIT> ) return true ; if ( MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> isEntryFunction ( ) ) return true ; if ( MF . getTarget ( ) . Options . DisableFramePointerElim ( MF ) ) return true ; } return MFI . hasVarSizedObjects ( ) || MFI . isFrameAddressTaken ( ) || MFI . hasStackMap ( ) || MFI . hasPatchPoint ( ) || MF . getSubtarget < GCNSubtarget > ( ) . getRegisterInfo ( ) -> needsStackRealignment ( MF ) ;


<s> AllSGPRSpilledToVGPRs = true ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( TII -> isSGPRSpill ( MI ) ) { int FI = TII -> getNamedOperand ( MI , <STR_LIT> :: addr ) -> getIndex ( ) ; assert ( MFI . getStackID ( FI ) == <STR_LIT> ) ; if ( FuncInfo -> allocateSGPRSpillToVGPR ( MF , FI ) ) { bool Spilled = TRI . eliminateSGPRToVGPRSpillFrameIndex ( MI , FI , RS ) ; ( void ) Spilled ; assert ( Spilled && <STR_LIT> ) ; } else AllSGPRSpilledToVGPRs = false ; } } } } FuncInfo -> removeSGPRToVGPRFrameIndices ( MFI ) ; if ( FuncInfo -> hasNonSpillStackObjects ( ) || FuncInfo -> hasSpilledVGPRs ( ) || ! AllSGPRSpilledToVGPRs || ! allStackObjectsAreDead ( MFI ) ) { assert ( RS && <STR_LIT> ) ;

if ( FuncInfo -> isEntryFunction ( ) ) {


<s> int64_t Offset = MFI . getObjectOffset ( FI ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( MachinePointerInfo :: getFixedStack ( * MF , FI ) , MachineMemOperand :: MOStore , <NUM_LIT> , MFI . getObjectAlign ( FI ) ) ; if ( isUInt < <NUM_LIT> > ( Offset ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( SpillReg , RegState :: Kill ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; return ; } LiveRegs . addReg ( SpillReg ) ; MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister ( MF -> getRegInfo ( ) , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , OffsetReg ) . addImm ( Offset ) ;

BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( SpillReg , RegState :: Kill ) . addReg ( OffsetReg , RegState :: Kill ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const bool SpillVGPRToAGPR = ST . hasMAIInsts ( ) && FuncInfo -> hasSpilledVGPRs ( ) && EnableSpillVGPRToAGPR ; if ( SpillVGPRToAGPR ) { BitVector SpillFIs ( MFI . getObjectIndexEnd ( ) , false ) ; bool SeenDbgInstr = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( MI . isDebugInstr ( ) ) SeenDbgInstr = true ; if ( TII -> isVGPRSpill ( MI ) ) { unsigned FIOp = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; int FI = MI . getOperand ( FIOp ) . getIndex ( ) ; Register VReg = TII -> getNamedOperand ( MI , <STR_LIT> :: vdata ) -> getReg ( ) ;

if ( FuncInfo -> allocateVGPRSpillToAGPR ( MF , FI , TRI -> isAGPR ( MRI , VReg ) ) ) {


<s> MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const bool SpillVGPRToAGPR = ST . hasMAIInsts ( ) && FuncInfo -> hasSpilledVGPRs ( ) && EnableSpillVGPRToAGPR ; if ( SpillVGPRToAGPR ) { BitVector SpillFIs ( MFI . getObjectIndexEnd ( ) , false ) ; bool SeenDbgInstr = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( MI . isDebugInstr ( ) ) SeenDbgInstr = true ; if ( TII -> isVGPRSpill ( MI ) ) { unsigned FIOp = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; int FI = MI . getOperand ( FIOp ) . getIndex ( ) ; Register VReg = TII -> getNamedOperand ( MI , <STR_LIT> :: vdata ) -> getReg ( ) ; if ( FuncInfo -> allocateVGPRSpillToAGPR ( MF , FI , TRI -> isAGPR ( MRI , VReg ) ) ) { RS -> enterBasicBlock ( MBB ) ; TRI -> eliminateFrameIndex ( MI , <NUM_LIT> , FIOp , RS ) ; SpillFIs . set ( FI ) ; continue ; } } } }


<s> if ( WillHaveFP || hasFP ( MF ) ) { assert ( ! MFI -> SGPRForFPSaveRestoreCopy && ! MFI -> FramePointerSaveIndex && <STR_LIT> ) ; getVGPRSpillLaneOrTempRegister ( MF , LiveRegs , MFI -> SGPRForFPSaveRestoreCopy , MFI -> FramePointerSaveIndex , true ) ; } if ( TRI -> hasBasePointer ( MF ) ) { if ( MFI -> SGPRForFPSaveRestoreCopy ) LiveRegs . addReg ( MFI -> SGPRForFPSaveRestoreCopy ) ; assert ( ! MFI -> SGPRForBPSaveRestoreCopy && ! MFI -> BasePointerSaveIndex && <STR_LIT> ) ; getVGPRSpillLaneOrTempRegister ( MF , LiveRegs , MFI -> SGPRForBPSaveRestoreCopy , MFI -> BasePointerSaveIndex , false ) ; }


<s> MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; SavedVGPRs . clearBitsNotInMask ( TRI -> getAllVGPRRegMask ( ) ) ; const bool WillHaveFP = FrameInfo . hasCalls ( ) && ( SavedVGPRs . any ( ) || ! allStackObjectsAreDead ( FrameInfo ) ) ; for ( auto SSpill : MFI -> getSGPRSpillVGPRs ( ) ) SavedVGPRs . reset ( SSpill . VGPR ) ; LivePhysRegs LiveRegs ; LiveRegs . init ( * TRI ) ; if ( WillHaveFP || hasFP ( MF ) ) { assert ( ! MFI -> SGPRForFPSaveRestoreCopy && ! MFI -> FramePointerSaveIndex && <STR_LIT> ) ; getVGPRSpillLaneOrTempRegister ( MF , LiveRegs , MFI -> SGPRForFPSaveRestoreCopy , MFI -> FramePointerSaveIndex , true ) ;


<s> MachineMemOperand * MMO = MF -> getMachineMemOperand ( MachinePointerInfo :: getFixedStack ( * MF , FI ) , MachineMemOperand :: MOLoad , <NUM_LIT> , MFI . getObjectAlign ( FI ) ) ; if ( <ISA_LIT>InstrInfo :: isLegalMUBUFImmOffset ( Offset ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SpillReg ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; return ;


<s> if ( <ISA_LIT>InstrInfo :: isLegalMUBUFImmOffset ( Offset ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( SpillReg , RegState :: Kill ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; return ; } LiveRegs . addReg ( SpillReg ) ; MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister ( MF -> getRegInfo ( ) , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , OffsetReg ) . addImm ( Offset ) ;


<s> LiveRegs . addReg ( SpillReg ) ; MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister ( MF -> getRegInfo ( ) , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , OffsetReg ) . addImm ( Offset ) ;


<s> DebugLoc DL ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; bool CopyBuffer = ResourceRegUsed && PreloadedPrivateBufferReg != <STR_LIT> && ST . isAmdHsaOrMesa ( F ) && ScratchRsrcReg != PreloadedPrivateBufferReg ; bool CopyBufferFirst = TRI -> isSubRegisterEq ( PreloadedPrivateBufferReg , ScratchWaveOffsetReg ) ; if ( CopyBuffer && CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } unsigned SPReg = MFI -> getStackPtrOffsetReg ( ) ; assert ( SPReg != <STR_LIT> ) ; const bool HasFP = hasFP ( MF ) ; if ( HasFP || OffsetRegUsed ) { assert ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , HasFP ? RegState :: Kill : <NUM_LIT> ) ; } if ( CopyBuffer && ! CopyBufferFirst ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchRsrcReg ) . addReg ( PreloadedPrivateBufferReg , RegState :: Kill ) ; } if ( ResourceRegUsed ) { emitEntryFunctionScratchSetup ( ST , MF , MBB , MFI , I , PreloadedPrivateBufferReg , ScratchRsrcReg ) ; } if ( HasFP ) { DebugLoc DL ;

const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ;


<s> if ( ScratchWaveOffsetReg == TRI -> reservedPrivateSegmentWaveByteOffsetReg ( MF ) ) { MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned NumPreloaded = MFI -> getNumPreloadedSGPRs ( ) ; for ( MCPhysReg Reg : getAllSGPRs ( ) . drop_back ( <NUM_LIT> ) . slice ( NumPreloaded ) ) { if ( ! MRI . isPhysRegUsed ( Reg ) ) { if ( ! MRI . isAllocatable ( Reg ) || TRI -> isSubRegisterEq ( ScratchRsrcReg , Reg ) ) continue ; MRI . replaceRegWith ( ScratchWaveOffsetReg , Reg ) ; ScratchWaveOffsetReg = Reg ; MFI -> setScratchWaveOffsetReg ( ScratchWaveOffsetReg ) ; break ; } } } } assert ( ! TRI -> isSubRegister ( ScratchRsrcReg , ScratchWaveOffsetReg ) ) ; const MCInstrDesc & SMovB32 = TII -> get ( <STR_LIT> ) ; DebugLoc DL ; if ( PreloadedScratchWaveOffsetReg != ScratchWaveOffsetReg ) { BuildMI ( MBB , I , DL , SMovB32 , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , RegState :: Kill ) ; } if ( ST . isAmdHsaOS ( ) ) { assert ( ! TRI -> isSubRegisterEq ( PreloadedPrivateBufferReg , ScratchRsrcReg ) && ! TRI -> isSubRegisterEq ( PreloadedPrivateBufferReg , ScratchWaveOffsetReg ) ) ; unsigned Rsrc01 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc23 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Lo = TRI -> getSubReg ( PreloadedPrivateBufferReg , <STR_LIT> ) ; unsigned Hi = TRI -> getSubReg ( PreloadedPrivateBufferReg , <STR_LIT> ) ; const MCInstrDesc & SMovB64 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , SMovB64 , Rsrc01 ) . addReg ( Lo , RegState :: Kill ) ; BuildMI ( MBB , I , DL , SMovB64 , Rsrc23 ) . addReg ( Hi , RegState :: Kill ) ; } else { unsigned Rsrc0 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc1 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc2 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; unsigned Rsrc3 = TRI -> getSubReg ( ScratchRsrcReg , <STR_LIT> ) ; uint64_t Rsrc23 = TII -> getScratchRsrcWords23 ( ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc0 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ; BuildMI ( MBB , I , DL , SMovB32 , Rsrc1 ) . addExternalSymbol ( <STR_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ;

BuildMI ( MBB , I , DL , SMovB32 , Rsrc2 ) . addImm ( Rsrc23 & <NUM_LIT> ) . addReg ( ScratchRsrcReg , RegState :: ImplicitDefine ) ;


<s> ScratchExecCopy = findScratchNonCalleeSaveRegister ( MF , LiveRegs , * TRI . getWaveMaskRegClass ( ) ) ; const unsigned OrSaveExec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , MBBI , DL , TII -> get ( OrSaveExec ) , ScratchExecCopy ) . addImm ( - <NUM_LIT> ) ; } TII -> loadRegFromStackSlot ( MBB , MBBI , Reg . VGPR , Reg . FI . getValue ( ) , & <STR_LIT> , & TII -> getRegisterInfo ( ) ) ; } if ( ScratchExecCopy != <STR_LIT> ) { unsigned ExecMov = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; unsigned Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , MBBI , DL , TII -> get ( ExecMov ) , Exec ) . addReg ( ScratchExecCopy ) ; } if ( hasFP ( MF ) ) { const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; uint32_t NumBytes = MFI . getStackSize ( ) ; uint32_t RoundedSize = FuncInfo -> isStackRealigned ( ) ? NumBytes + MFI . getMaxAlignment ( ) : NumBytes ; const unsigned StackPtrReg = FuncInfo -> getStackPtrOffsetReg ( ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , StackPtrReg ) . addReg ( StackPtrReg ) . addImm ( RoundedSize * ST . getWavefrontSize ( ) ) . setMIFlag ( MachineInstr :: FrameDestroy ) ;


<s> if ( TII -> isSGPRSpill ( MI ) ) { int FI = TII -> getNamedOperand ( MI , <STR_LIT> :: addr ) -> getIndex ( ) ; assert ( MFI . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; if ( FuncInfo -> allocateSGPRSpillToVGPR ( MF , FI ) ) { bool Spilled = TRI . eliminateSGPRToVGPRSpillFrameIndex ( MI , FI , RS ) ; ( void ) Spilled ; assert ( Spilled && <STR_LIT> ) ; } else AllSGPRSpilledToVGPRs = false ; } } } } FuncInfo -> removeSGPRToVGPRFrameIndices ( MFI ) ; if ( FuncInfo -> hasNonSpillStackObjects ( ) || FuncInfo -> hasSpilledVGPRs ( ) || ! AllSGPRSpilledToVGPRs || ! allStackObjectsAreDead ( MFI ) ) { assert ( RS && <STR_LIT> ) ; if ( FuncInfo -> isEntryFunction ( ) ) { int ScavengeFI = MFI . CreateFixedObject ( TRI . getSpillSize ( <STR_LIT> ) , <NUM_LIT> , false ) ;

RS -> addScavengingFrameIndex ( ScavengeFI ) ;


<s> <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const bool SpillVGPRToAGPR = ST . hasMAIInsts ( ) && FuncInfo -> hasSpilledVGPRs ( ) && EnableSpillVGPRToAGPR ; if ( SpillVGPRToAGPR ) { BitVector SpillFIs ( MFI . getObjectIndexEnd ( ) , false ) ; BitVector NonVGPRSpillFIs ( MFI . getObjectIndexEnd ( ) , false ) ; bool SeenDbgInstr = false ; for ( MachineBasicBlock & MBB : MF ) { for ( MachineInstr & MI : llvm :: make_early_inc_range ( MBB ) ) { int FrameIndex ; if ( MI . isDebugInstr ( ) ) SeenDbgInstr = true ; if ( TII -> isVGPRSpill ( MI ) ) { unsigned FIOp = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; int FI = MI . getOperand ( FIOp ) . getIndex ( ) ; Register VReg = TII -> getNamedOperand ( MI , <STR_LIT> :: vdata ) -> getReg ( ) ; if ( FuncInfo -> allocateVGPRSpillToAGPR ( MF , FI , TRI -> isAGPR ( MRI , VReg ) ) ) { RS -> enterBasicBlock ( MBB ) ; TRI -> eliminateFrameIndex ( MI , <NUM_LIT> , FIOp , RS ) ; SpillFIs . set ( FI ) ; continue ; } } else if ( TII -> isStoreToStackSlot ( MI , FrameIndex ) || TII -> isLoadFromStackSlot ( MI , FrameIndex ) ) NonVGPRSpillFIs . set ( FrameIndex ) ; }


<s> if ( ! MF . getFrameInfo ( ) -> hasStackObjects ( ) ) return ; assert ( & MF . front ( ) == & MBB && <STR_LIT> ) ; <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; if ( hasOnlySGPRSpills ( MFI , MF . getFrameInfo ( ) ) ) return ; const <ISA_LIT>InstrInfo * TII = static_cast < const <ISA_LIT>InstrInfo * > ( MF . getSubtarget ( ) . getInstrInfo ( ) ) ; const <ISA_LIT>RegisterInfo * TRI = & TII -> getRegisterInfo ( ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MachineBasicBlock :: iterator I = MBB . begin ( ) ; unsigned ScratchRsrcReg = MFI -> getScratchRSrcReg ( ) ; assert ( ScratchRsrcReg != <STR_LIT> ) ; unsigned ScratchWaveOffsetReg = MFI -> getScratchWaveOffsetReg ( ) ; assert ( ScratchWaveOffsetReg != <STR_LIT> ) ; unsigned PreloadedScratchWaveOffsetReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: PRIVATE_SEGMENT_WAVE_BYTE_OFFSET ) ; unsigned PreloadedPrivateBufferReg = <STR_LIT> ; if ( ST . isAmdHsaOS ( ) ) { PreloadedPrivateBufferReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: PRIVATE_SEGMENT_BUFFER ) ; } if ( MFI -> hasFlatScratchInit ( ) ) { DebugLoc DL ; unsigned FlatScratchInitReg = TRI -> getPreloadedValue ( MF , <ISA_LIT>RegisterInfo :: FLAT_SCRATCH_INIT ) ; MRI . addLiveIn ( FlatScratchInitReg ) ; MBB . addLiveIn ( FlatScratchInitReg ) ; unsigned FlatScrInitHi = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitHi , RegState :: Kill ) ; unsigned FlatScrInitLo = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitLo ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitLo , RegState :: Kill ) . addImm ( <NUM_LIT> ) ; } if ( ScratchRsrcReg == PreloadedPrivateBufferReg ) {

assert ( ScratchWaveOffsetReg == PreloadedScratchWaveOffsetReg && <STR_LIT> ) ;


<s> static bool hasOnlySGPRSpills ( const <ISA_LIT>MachineFunctionInfo * FuncInfo , const MachineFrameInfo * FrameInfo ) { return FuncInfo -> hasSpilledSGPRs ( ) && ( ! FuncInfo -> hasSpilledVGPRs ( ) && ! FuncInfo -> hasNonSpillStackObjects ( ) ) ;


<s> MachineFrameInfo & MFI = MF -> getFrameInfo ( ) ; int64_t Offset = MFI . getObjectOffset ( FI ) ; MachineMemOperand * MMO = MF -> getMachineMemOperand ( MachinePointerInfo :: getFixedStack ( * MF , FI ) , MachineMemOperand :: MOLoad , <NUM_LIT> , MFI . getObjectAlign ( FI ) ) ; if ( isUInt < <NUM_LIT> > ( Offset ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SpillReg ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; return ; } MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister ( MF -> getRegInfo ( ) , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , OffsetReg ) . addImm ( Offset ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SpillReg ) . addReg ( OffsetReg , RegState :: Kill ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> MachineMemOperand * MMO = MF -> getMachineMemOperand ( MachinePointerInfo :: getFixedStack ( * MF , FI ) , MachineMemOperand :: MOLoad , <NUM_LIT> , MFI . getObjectAlign ( FI ) ) ; if ( isUInt < <NUM_LIT> > ( Offset ) ) { BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , SpillReg ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( Offset ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ; return ; }

MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister ( MF -> getRegInfo ( ) , LiveRegs , <STR_LIT> ) ;


<s> MCPhysReg OffsetReg = findScratchNonCalleeSaveRegister ( MF -> getRegInfo ( ) , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) , OffsetReg ) . addImm ( Offset ) ; BuildMI ( MBB , I , DebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addReg ( SpillReg , RegState :: Kill ) . addReg ( OffsetReg , RegState :: Kill ) . addReg ( ScratchRsrcReg ) . addReg ( SPReg ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addMemOperand ( MMO ) ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const DebugLoc & DL = I -> getDebugLoc ( ) ; unsigned Opc = I -> getOpcode ( ) ; bool IsDestroy = Opc == TII -> getCallFrameDestroyOpcode ( ) ; uint64_t CalleePopAmount = IsDestroy ? I -> getOperand ( <NUM_LIT> ) . getImm ( ) : <NUM_LIT> ; if ( ! hasReservedCallFrame ( MF ) ) { Amount = alignTo ( Amount , getStackAlign ( ) ) ; assert ( isUInt < <NUM_LIT> > ( Amount ) && <STR_LIT> ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned SPReg = MFI -> getStackPtrOffsetReg ( ) ; unsigned Op = IsDestroy ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , I , DL , TII -> get ( Op ) , SPReg ) . addReg ( SPReg ) . addImm ( Amount * ST . getWavefrontSize ( ) ) ; } else if ( CalleePopAmount != <NUM_LIT> ) { llvm_unreachable ( <STR_LIT> ) ; } return MBB . erase ( I ) ;


<s> assert ( isUInt < <NUM_LIT> > ( Amount ) && <STR_LIT> ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned SPReg = MFI -> getStackPtrOffsetReg ( ) ; unsigned Op = IsDestroy ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , I , DL , TII -> get ( Op ) , SPReg ) . addReg ( SPReg ) . addImm ( Amount * ST . getWavefrontSize ( ) ) ; } else if ( CalleePopAmount != <NUM_LIT> ) { llvm_unreachable ( <STR_LIT> ) ; }


<s> } if ( FuncInfo -> FramePointerSaveIndex ) { const int FI = FuncInfo -> FramePointerSaveIndex . getValue ( ) ; assert ( ! MF . getFrameInfo ( ) . isDeadObjectIndex ( FI ) && MF . getFrameInfo ( ) . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; ArrayRef < <ISA_LIT>MachineFunctionInfo :: SpilledReg > Spill = FuncInfo -> getSGPRToVGPRSpills ( FI ) ; assert ( Spill . size ( ) == <NUM_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , FuncInfo -> getFrameOffsetReg ( ) ) . addReg ( Spill [ <NUM_LIT> ] . VGPR ) . addImm ( Spill [ <NUM_LIT> ] . Lane ) ; } unsigned ScratchExecCopy = <STR_LIT> ; for ( const <ISA_LIT>MachineFunctionInfo :: SGPRSpillVGPRCSR & Reg : FuncInfo -> getSGPRSpillVGPRs ( ) ) { if ( ! Reg . FI . hasValue ( ) ) continue ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; if ( ScratchExecCopy == <STR_LIT> ) { if ( LiveRegs . empty ( ) ) { LiveRegs . init ( * ST . getRegisterInfo ( ) ) ;


<s> const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MachineBasicBlock :: iterator MBBI = MBB . getFirstTerminator ( ) ; LivePhysRegs LiveRegs ; DebugLoc DL ; const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; uint32_t NumBytes = MFI . getStackSize ( ) ; uint32_t RoundedSize = FuncInfo -> isStackRealigned ( ) ? NumBytes + MFI . getMaxAlign ( ) . value ( ) : NumBytes ; if ( RoundedSize != <NUM_LIT> && hasFP ( MF ) ) { const unsigned StackPtrReg = FuncInfo -> getStackPtrOffsetReg ( ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , StackPtrReg ) . addReg ( StackPtrReg ) . addImm ( RoundedSize * ST . getWavefrontSize ( ) ) . setMIFlag ( MachineInstr :: FrameDestroy ) ; } if ( FuncInfo -> SGPRForFPSaveRestoreCopy != <STR_LIT> ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , FuncInfo -> getFrameOffsetReg ( ) ) . addReg ( FuncInfo -> SGPRForFPSaveRestoreCopy ) . setMIFlag ( MachineInstr :: FrameSetup ) ; } if ( FuncInfo -> FramePointerSaveIndex ) { const int FI = FuncInfo -> FramePointerSaveIndex . getValue ( ) ; assert ( ! MF . getFrameInfo ( ) . isDeadObjectIndex ( FI ) && MF . getFrameInfo ( ) . getStackID ( FI ) == TargetStackID :: SGPRSpill ) ; ArrayRef < <ISA_LIT>MachineFunctionInfo :: SpilledReg > Spill = FuncInfo -> getSGPRToVGPRSpills ( FI ) ; assert ( Spill . size ( ) == <NUM_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> getMCOpcodeFromPseudo ( <STR_LIT> ) , FuncInfo -> getFrameOffsetReg ( ) ) . addReg ( Spill [ <NUM_LIT> ] . VGPR ) . addImm ( Spill [ <NUM_LIT> ] . Lane ) ; } unsigned ScratchExecCopy = <STR_LIT> ;

for ( const <ISA_LIT>MachineFunctionInfo :: SGPRSpillVGPRCSR & Reg : FuncInfo -> getSGPRSpillVGPRs ( ) ) {


<s> SavedVGPRs . clearBitsNotInMask ( TRI -> getAllVGPRRegMask ( ) ) ; const bool WillHaveFP = FrameInfo . hasCalls ( ) && ( SavedVGPRs . any ( ) || ! allStackObjectsAreDead ( FrameInfo ) ) ; for ( auto SSpill : MFI -> getSGPRSpillVGPRs ( ) ) SavedVGPRs . reset ( SSpill . VGPR ) ; const bool HasFP = WillHaveFP || hasFP ( MF ) ; if ( ! HasFP ) return ; if ( MFI -> haveFreeLanesForSGPRSpill ( MF , <NUM_LIT> ) ) { int NewFI = MF . getFrameInfo ( ) . CreateStackObject ( <NUM_LIT> , <NUM_LIT> , true , nullptr , TargetStackID :: SGPRSpill ) ; if ( ! MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) llvm_unreachable ( <STR_LIT> ) ;


<s> const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ;

const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ;


<s> void <ISA_LIT>FrameLowering :: determineCalleeSavesSGPR ( MachineFunction & MF , BitVector & SavedRegs , RegScavenger * RS ) const { TargetFrameLowering :: determineCalleeSaves ( MF , SavedRegs , RS ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; SavedRegs . reset ( MFI -> getStackPtrOffsetReg ( ) ) ;

SavedRegs . clearBitsInMask ( TRI -> getAllVGPRRegMask ( ) ) ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; assert ( allSGPRSpillsAreDead ( MFI , None ) && <STR_LIT> ) ; if ( ! allStackObjectsAreDead ( MFI ) ) { assert ( RS && <STR_LIT> ) ; if ( FuncInfo -> isEntryFunction ( ) ) { int ScavengeFI = MFI . CreateFixedObject ( TRI -> getSpillSize ( <STR_LIT> ) , <NUM_LIT> , false ) ; RS -> addScavengingFrameIndex ( ScavengeFI ) ; } else { int ScavengeFI = MFI . CreateStackObject ( TRI -> getSpillSize ( <STR_LIT> ) , TRI -> getSpillAlignment ( <STR_LIT> ) , false ) ; RS -> addScavengingFrameIndex ( ScavengeFI ) ; } }


<s> int NewFI = MF . getFrameInfo ( ) . CreateStackObject ( TRI -> getSpillSize ( * RC ) , TRI -> getSpillAlign ( * RC ) , true , nullptr , TargetStackID :: SGPRSpill ) ; if ( TRI -> spillSGPRToVGPR ( ) && MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) { FI = NewFI ; } else { MF . getFrameInfo ( ) . RemoveStackObject ( NewFI ) ;

FI = MF . getFrameInfo ( ) . CreateSpillStackObject ( TRI -> getSpillSize ( * RC ) , Align ( TRI -> getSpillAlign ( * RC ) ) ) ;


<s> if ( MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) { FI = NewFI ; } } else { int NewFI = MF . getFrameInfo ( ) . CreateStackObject ( TRI -> getSpillSize ( * RC ) , TRI -> getSpillAlign ( * RC ) , true , nullptr , TargetStackID :: SGPRSpill ) ; if ( TRI -> spillSGPRToVGPR ( ) && MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) {


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; const <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; for ( int I = MFI . getObjectIndexBegin ( ) , E = MFI . getObjectIndexEnd ( ) ;


<s> static bool allStackObjectsAreDead ( const MachineFunction & MF ) { const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ;

const <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * RI = ST . getRegisterInfo ( ) ; Register FramePtrReg = FuncInfo -> getFrameOffsetReg ( ) ; Register BasePtrReg = RI -> getBaseRegister ( ) ; unsigned NumModifiedRegs = <NUM_LIT> ; if ( FuncInfo -> SGPRForFPSaveRestoreCopy ) NumModifiedRegs ++ ; if ( FuncInfo -> SGPRForBPSaveRestoreCopy ) NumModifiedRegs ++ ; for ( auto & CS : C<ISA_LIT> ) { if ( CS . getReg ( ) == FramePtrReg && FuncInfo -> SGPRForFPSaveRestoreCopy ) { CS . setDstReg ( FuncInfo -> SGPRForFPSaveRestoreCopy ) ; if ( -- NumModifiedRegs ) break ; } else if ( CS . getReg ( ) == BasePtrReg && FuncInfo -> SGPRForBPSaveRestoreCopy ) { CS . setDstReg ( FuncInfo -> SGPRForBPSaveRestoreCopy ) ; if ( -- NumModifiedRegs ) break ; }


<s> MachineInstr * <ISA_LIT>FrameLowering :: buildCFI ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator MBBI , const DebugLoc & DL , const MCCFIInstruction & CFIInst ) const { MachineFunction & MF = * MBB . getParent ( ) ; const <ISA_LIT>InstrInfo * TII = MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) ;

return BuildMI ( MBB , MBBI , DL , TII -> get ( TargetOpcode :: CFI_INSTRUCTION ) ) . addCFIIndex ( MF . addFrameInst ( CFIInst ) ) . setMIFlag ( MachineInstr :: FrameSetup ) ;


<s> raw_svector_ostream OSCFIInst ( CFIInst ) ; SmallString < <NUM_LIT> > Block ; raw_svector_ostream OSBlock ( Block ) ; OSCFIInst << uint8_t ( dwarf :: DW_CFA_expression ) ; encodeULEB128 ( DwarfSGPR , OSCFIInst ) ; for ( <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill : VGPRSpills ) { encodeDwarfRegisterLocation ( MCRI . getDwarfRegNum ( Spill . VGPR , false ) , OSBlock ) ; OSBlock << uint8_t ( dwarf :: DW_OP_bit_piece ) ; const unsigned SGPRBitSize = <NUM_LIT> ; encodeULEB128 ( SGPRBitSize , OSBlock ) ; encodeULEB128 ( SGPRBitSize * Spill . Lane , OSBlock ) ; }

encodeULEB128 ( Block . size ( ) , OSCFIInst ) ;


<s> MachineFunction & MF = * MBB . getParent ( ) ; const MCRegisterInfo & MCRI = * MF . getMMI ( ) . getContext ( ) . getRegisterInfo ( ) ; int DwarfSGPR = MCRI . getDwarfRegNum ( SGPR , false ) ; SmallString < <NUM_LIT> > CFIInst ; raw_svector_ostream OSCFIInst ( CFIInst ) ; SmallString < <NUM_LIT> > Block ; raw_svector_ostream OSBlock ( Block ) ; OSCFIInst << uint8_t ( dwarf :: DW_CFA_expression ) ; encodeULEB128 ( DwarfSGPR , OSCFIInst ) ; for ( <ISA_LIT>MachineFunctionInfo :: SpilledReg Spill : VGPRSpills ) { encodeDwarfRegisterLocation ( MCRI . getDwarfRegNum ( Spill . VGPR , false ) , OSBlock ) ; OSBlock << uint8_t ( dwarf :: DW_OP_bit_piece ) ; const unsigned SGPRBitSize = <NUM_LIT> ; encodeULEB128 ( SGPRBitSize , OSBlock ) ; encodeULEB128 ( SGPRBitSize * Spill . Lane , OSBlock ) ; } encodeULEB128 ( Block . size ( ) , OSCFIInst ) ; OSCFIInst << Block ; buildCFI ( MBB , MBBI , DL , MCCFIInstruction :: createEscape ( nullptr , OSCFIInst . str ( ) ) ) ;


<s> int DwarfVGPR = MCRI . getDwarfRegNum ( VGPR , false ) ; SmallString < <NUM_LIT> > CFIInst ; raw_svector_ostream OSCFIInst ( CFIInst ) ; SmallString < <NUM_LIT> > Block ; raw_svector_ostream OSBlock ( Block ) ; OSCFIInst << uint8_t ( dwarf :: DW_CFA_expression ) ; encodeULEB128 ( DwarfVGPR , OSCFIInst ) ; encodeDwarfRegisterLocation ( DwarfVGPR , OSBlock ) ; OSBlock << uint8_t ( dwarf :: DW_OP_swap ) ; OSBlock << uint8_t ( dwarf :: DW_OP_LLVM_offset_uconst ) ; encodeULEB128 ( Offset , OSBlock ) ; OSBlock << uint8_t ( dwarf :: DW_OP_LLVM_call_frame_entry_reg ) ; encodeULEB128 ( MCRI . getDwarfRegNum ( ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> , false ) , OSBlock ) ; OSBlock << uint8_t ( dwarf :: DW_OP_deref_size ) ; OSBlock << uint8_t ( ST . getWavefrontSize ( ) / CHAR_BIT ) ;

OSBlock << uint8_t ( dwarf :: DW_OP_LLVM_select_bit_piece ) ;


<s> MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( MF , FI ) ; MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOLoad , FrameInfo . getObjectSize ( FI ) , FrameInfo . getObjectAlign ( FI ) ) ; TRI . buildSpillLoadStore ( MBB , I , Opc , FI , SpillReg , false , FuncInfo . getStackPtrOffsetReg ( ) , <NUM_LIT> , MMO , nullptr , & LiveRegs ) ;


<s> Register TargetLo = TRI -> getSubReg ( TargetReg , <STR_LIT> ) ; Register TargetHi = TRI -> getSubReg ( TargetReg , <STR_LIT> ) ; if ( MFI -> getGITPtrHigh ( ) != <NUM_LIT> ) { BuildMI ( MBB , I , DL , SMovB32 , TargetHi ) . addImm ( MFI -> getGITPtrHigh ( ) ) . addReg ( TargetReg , RegState :: ImplicitDefine ) ; } else { const MCInstrDesc & GetPC64 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , GetPC64 , TargetReg ) ; } Register GitPtrLo = MFI -> getGITPtrLoReg ( * MF ) ; MF -> getRegInfo ( ) . addLiveIn ( GitPtrLo ) ; MBB . addLiveIn ( GitPtrLo ) ; BuildMI ( MBB , I , DL , SMovB32 , TargetLo ) . addReg ( GitPtrLo ) ;


<s> MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; MachinePointerInfo PtrInfo = MachinePointerInfo :: getFixedStack ( MF , FI ) ;

MachineMemOperand * MMO = MF . getMachineMemOperand ( PtrInfo , MachineMemOperand :: MOStore , FrameInfo . getObjectSize ( FI ) , FrameInfo . getObjectAlign ( FI ) ) ;


<s> static Register buildScratchExecCopy ( LivePhysRegs & LiveRegs , MachineFunction & MF , MachineBasicBlock & MBB , MachineBasicBlock :: iterator MBBI , bool IsProlog ) { Register ScratchExecCopy ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const <ISA_LIT>RegisterInfo & TRI = TII -> getRegisterInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; DebugLoc DL ; initLiveRegs ( LiveRegs , TRI , FuncInfo , MF , MBB , MBBI , IsProlog ) ; ScratchExecCopy = findScratchNonCalleeSaveRegister ( MRI , LiveRegs , * TRI . getWaveMaskRegClass ( ) ) ; if ( ! ScratchExecCopy ) report_fatal_error ( <STR_LIT> ) ;


<s> MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; SavedVGPRs . clearBitsNotInMask ( TRI -> getAllVectorRegMask ( ) ) ; if ( ! ST . hasGFX90AInsts ( ) ) SavedVGPRs . clearBitsInMask ( TRI -> getAllAGPRRegMask ( ) ) ; if ( TRI -> isCFISavedRegsSpillEnabled ( ) ) { allocateCFISave ( MF , MFI -> ReturnAddressSaveIndex , TRI -> getReturnAddressReg ( MF ) ) ; allocateCFISave ( MF , MFI -> EXECSaveIndex , ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ) ;


<s> const DebugLoc & DL = I -> getDebugLoc ( ) ; unsigned Opc = I -> getOpcode ( ) ; bool IsDestroy = Opc == TII -> getCallFrameDestroyOpcode ( ) ; uint64_t CalleePopAmount = IsDestroy ? I -> getOperand ( <NUM_LIT> ) . getImm ( ) : <NUM_LIT> ; if ( ! hasReservedCallFrame ( MF ) ) { Amount = alignTo ( Amount , getStackAlign ( ) ) ; assert ( isUInt < <NUM_LIT> > ( Amount ) && <STR_LIT> ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ;

Register SPReg = MFI -> getStackPtrOffsetReg ( ) ;


<s> MachineBasicBlock :: iterator <ISA_LIT>FrameLowering :: eliminateCallFramePseudoInstr ( MachineFunction & MF , MachineBasicBlock & MBB , MachineBasicBlock :: iterator I ) const { int64_t Amount = I -> getOperand ( <NUM_LIT> ) . getImm ( ) ; if ( Amount == <NUM_LIT> ) return MBB . erase ( I ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>InstrInfo * TII = ST . getInstrInfo ( ) ; const DebugLoc & DL = I -> getDebugLoc ( ) ; unsigned Opc = I -> getOpcode ( ) ; bool IsDestroy = Opc == TII -> getCallFrameDestroyOpcode ( ) ; uint64_t CalleePopAmount = IsDestroy ? I -> getOperand ( <NUM_LIT> ) . getImm ( ) : <NUM_LIT> ; if ( ! hasReservedCallFrame ( MF ) ) { Amount = alignTo ( Amount , getStackAlign ( ) ) ; assert ( isUInt < <NUM_LIT> > ( Amount ) && <STR_LIT> ) ; const <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register SPReg = MFI -> getStackPtrOffsetReg ( ) ; Amount *= getScratchScaleFactor ( ST ) ; if ( IsDestroy ) Amount = - Amount ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SPReg ) . addReg ( SPReg ) . addImm ( Amount ) ; } else if ( CalleePopAmount != <NUM_LIT> ) { llvm_unreachable ( <STR_LIT> ) ; }


<s> initLiveRegs ( LiveRegs , TRI , FuncInfo , MF , MBB , MBBI , true ) ; MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister ( MRI , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , TmpVGPR ) . addReg ( TRI . getSubReg ( RetAddrReg , <STR_LIT> ) ) ; int DwordOff = <NUM_LIT> ; buildPrologSpill ( ST , TRI , * FuncInfo , LiveRegs , MF , MBB , MBBI , TmpVGPR , FI , DwordOff ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , TmpVGPR ) . addReg ( TRI . getSubReg ( RetAddrReg , <STR_LIT> ) ) ; DwordOff = <NUM_LIT> ; buildPrologSpill ( ST , TRI , * FuncInfo , LiveRegs , MF , MBB , MBBI , TmpVGPR , FI , DwordOff ) ; buildCFI ( MBB , MBBI , DL , MCCFIInstruction :: createOffset ( nullptr , MCRI -> getDwarfRegNum ( <STR_LIT> , false ) , MFI . getObjectOffset ( FI ) * ST . getWavefrontSize ( ) ) ) ; } if ( EXECSaveIndex && spilledToMemory ( MF , * EXECSaveIndex ) ) { const int FI = * EXECSaveIndex ; assert ( ! MFI . isDeadObjectIndex ( FI ) ) ; initLiveRegs ( LiveRegs , TRI , FuncInfo , MF , MBB , MBBI , true ) ; MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister ( MRI , LiveRegs , <STR_LIT> ) ; BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , TmpVGPR ) . addReg ( TRI . getSubReg ( <STR_LIT> , <STR_LIT> ) ) ; int DwordOff = <NUM_LIT> ; buildPrologSpill ( ST , TRI , * FuncInfo , LiveRegs , MF , MBB , MBBI , TmpVGPR , FI , DwordOff ) ; if ( ! ST . isWave32 ( ) ) { BuildMI ( MBB , MBBI , DL , TII -> get ( <STR_LIT> ) , TmpVGPR ) . addReg ( TRI . getSubReg ( <STR_LIT> , <STR_LIT> ) ) ; DwordOff = <NUM_LIT> ; buildPrologSpill ( ST , TRI , * FuncInfo , LiveRegs , MF , MBB , MBBI , TmpVGPR , FI , DwordOff ) ; } buildCFI ( MBB , MBBI , DL , MCCFIInstruction :: createOffset ( nullptr , MCRI -> getDwarfRegNum ( <STR_LIT> , false ) , MFI . getObjectOffset ( FI ) * ST . getWavefrontSize ( ) ) ) ; } } if ( ! emitSpillsToMem ) { if ( RASaveIndex && ! spilledToMemory ( MF , * RASaveIndex ) ) { MCRegister RetAddrReg = TRI . getReturnAddressReg ( MF ) ;

if ( ! MBB . isLiveIn ( RetAddrReg ) ) MBB . addLiveIn ( RetAddrReg ) ;


<s> const MCInstrDesc & SAndB32 = TII -> get ( <STR_LIT> ) ; BuildMI ( MBB , I , DL , SAndB32 , FlatScrInitHi ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; } else { Register FlatScratchInitReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; assert ( FlatScratchInitReg ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; MRI . addLiveIn ( FlatScratchInitReg ) ; MBB . addLiveIn ( FlatScratchInitReg ) ; FlatScrInitLo = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; FlatScrInitHi = TRI -> getSubReg ( FlatScratchInitReg , <STR_LIT> ) ; } if ( ST . flatScratchIsPointer ( ) ) { if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX10 ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitLo ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FlatScrInitHi ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( FlatScrInitLo ) . addImm ( int16_t ( <STR_LIT> :: ID_FLAT_SCR_LO | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ) ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addReg ( FlatScrInitHi ) . addImm ( int16_t ( <STR_LIT> :: ID_FLAT_SCR_HI | ( <NUM_LIT> << <STR_LIT> :: WIDTH_M1_SHIFT_ ) ) ) ; return ; } BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitLo ) . addReg ( ScratchWaveOffsetReg ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitHi ) . addImm ( <NUM_LIT> ) ; return ; } assert ( ST . getGeneration ( ) < <ISA_LIT>Subtarget :: GFX9 ) ;

BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addReg ( FlatScrInitHi , RegState :: Kill ) ;


<s> if ( ST . isAmdHsaOrMesa ( F ) ) { PreloadedScratchRsrcReg = MFI -> getPreloadedReg ( <STR_LIT> ) ; if ( ScratchRsrcReg && PreloadedScratchRsrcReg ) { MRI . addLiveIn ( PreloadedScratchRsrcReg ) ; MBB . addLiveIn ( PreloadedScratchRsrcReg ) ; } } Register ScratchWaveOffsetReg ; if ( TRI -> isSubRegisterEq ( ScratchRsrcReg , PreloadedScratchWaveOffsetReg ) ) { ArrayRef < MCPhysReg > AllSGPRs = TRI -> getAllSGPR32 ( MF ) ; unsigned NumPreloaded = MFI -> getNumPreloadedSGPRs ( ) ; AllSGPRs = AllSGPRs . slice ( std :: min ( static_cast < unsigned > ( AllSGPRs . size ( ) ) , NumPreloaded ) ) ; Register GITPtrLoReg = MFI -> getGITPtrLoReg ( MF ) ; for ( MCPhysReg Reg : AllSGPRs ) { if ( ! MRI . isPhysRegUsed ( Reg ) && MRI . isAllocatable ( Reg ) && ! TRI -> isSubRegisterEq ( ScratchRsrcReg , Reg ) && GITPtrLoReg != Reg ) { ScratchWaveOffsetReg = Reg ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , ScratchWaveOffsetReg ) . addReg ( PreloadedScratchWaveOffsetReg , RegState :: Kill ) ; break ; } } } else { ScratchWaveOffsetReg = PreloadedScratchWaveOffsetReg ; } assert ( ScratchWaveOffsetReg ) ; if ( requiresStackPointerReference ( MF ) ) { Register SPReg = MFI -> getStackPtrOffsetReg ( ) ; assert ( SPReg != <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , SPReg ) . addImm ( FrameInfo . getStackSize ( ) * getScratchScaleFactor ( ST ) ) ; } if ( hasFP ( MF ) ) { Register FPReg = MFI -> getFrameOffsetReg ( ) ; assert ( FPReg != <STR_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) , FPReg ) . addImm ( <NUM_LIT> ) ; } bool NeedsFlatScratchInit = MFI -> hasFlatScratchInit ( ) && ( MRI . isPhysRegUsed ( <STR_LIT> ) || FrameInfo . hasCalls ( ) || ( ! allStackObjectsAreDead ( MF ) && ST . enableFlatScratch ( ) ) ) ; if ( ( NeedsFlatScratchInit || ScratchRsrcReg ) && ! ST . flatScratchIsArchitected ( ) ) { MRI . addLiveIn ( PreloadedScratchWaveOffsetReg ) ; MBB . addLiveIn ( PreloadedScratchWaveOffsetReg ) ; } if ( NeedsFlatScratchInit ) { emitEntryFunctionFlatScratchInit ( MF , MBB , I , DL , ScratchWaveOffsetReg ) ; } if ( ScratchRsrcReg ) {

emitEntryFunctionScratchRsrcRegSetup ( MF , MBB , I , DL , PreloadedScratchRsrcReg , ScratchRsrcReg , ScratchWaveOffsetReg ) ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo & TRI = ST . getInstrInfo ( ) -> getRegisterInfo ( ) ; Register StackPtrReg = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> getStackPtrOffsetReg ( ) ; buildCFI ( MBB , MBBI , DL , MCCFIInstruction :: createLLVMDefAspaceCfa ( nullptr , MCRI -> getDwarfRegNum ( StackPtrReg , false ) , <NUM_LIT> , <NUM_LIT> ) ) ; static const char PCEncodedInst [ ] = { dwarf :: DW_CFA_expression , <NUM_LIT> , <NUM_LIT> , static_cast < char > ( dwarf :: DW_OP_regx ) , <NUM_LIT> , static_cast < char > ( dwarf :: DW_OP_piece ) , <NUM_LIT> , static_cast < char > ( dwarf :: DW_OP_regx ) , <NUM_LIT> , static_cast < char > ( dwarf :: DW_OP_piece ) , <NUM_LIT> } ; buildCFI ( MBB , MBBI , DL , MCCFIInstruction :: createEscape ( nullptr , StringRef ( PCEncodedInst , sizeof ( PCEncodedInst ) ) ) ) ; BitVector IsCalleeSaved ( TRI . getNumRegs ( ) ) ; const MCPhysReg * CSRegs = MRI . getCalleeSavedRegs ( ) ; for ( unsigned I = <NUM_LIT> ; CSRegs [ I ] ; ++ I ) { IsCalleeSaved . set ( CSRegs [ I ] ) ; } auto ProcessReg = [ & ] ( MCPhysReg Reg ) { if ( IsCalleeSaved . test ( Reg ) || ! MRI . isPhysRegModified ( Reg ) ) return ; MCRegister DwarfReg = MCRI -> getDwarfRegNum ( Reg , false ) ; buildCFI ( MBB , MBBI , DL , MCCFIInstruction :: createUndefined ( nullptr , DwarfReg ) ) ; } ; for_each ( <STR_LIT> . getRegisters ( ) , ProcessReg ) ; if ( ST . hasMAIInsts ( ) ) { for_each ( <STR_LIT> . getRegisters ( ) , ProcessReg ) ; } for_each ( <STR_LIT> . getRegisters ( ) , ProcessReg ) ;


<s> const <ISA_LIT>RegisterInfo & TRI = ST . getInstrInfo ( ) -> getRegisterInfo ( ) ; Register StackPtrReg = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> getStackPtrOffsetReg ( ) ; buildCFI ( MBB , MBBI , DL , MCCFIInstruction :: createLLVMDefAspaceCfa ( nullptr , MCRI -> getDwarfRegNum ( StackPtrReg , false ) , <NUM_LIT> , <NUM_LIT> ) ) ; static const char PCEncodedInst [ ] = { dwarf :: DW_CFA_expression , <NUM_LIT> , <NUM_LIT> , static_cast < char > ( dwarf :: DW_OP_regx ) , <NUM_LIT> , static_cast < char > ( dwarf :: DW_OP_piece ) , <NUM_LIT> , static_cast < char > ( dwarf :: DW_OP_regx ) , <NUM_LIT> , static_cast < char > ( dwarf :: DW_OP_piece ) , <NUM_LIT> } ; buildCFI ( MBB , MBBI , DL , MCCFIInstruction :: createEscape ( nullptr , StringRef ( PCEncodedInst , sizeof ( PCEncodedInst ) ) ) ) ;

BitVector IsCalleeSaved ( TRI . getNumRegs ( ) ) ;


<s> OS << uint8_t ( dwarf :: DW_OP_regx ) ; encodeULEB128 ( DwarfReg , OS ) ; }


<s> static void encodeDwarfRegisterLocation ( int DwarfReg , raw_ostream & OS ) { if ( DwarfReg < <NUM_LIT> ) { OS << uint8_t ( dwarf :: DW_OP_reg0 + DwarfReg ) ; } else {


<s> static MCRegister findScratchNonCalleeSaveRegister ( MachineRegisterInfo & MRI , LivePhysRegs & LiveRegs , const TargetRegisterClass & RC , bool Unused = false ) { const MCPhysReg * CSRegs = MRI . getCalleeSavedRegs ( ) ; for ( unsigned i = <NUM_LIT> ; CSRegs [ i ] ; ++ i ) LiveRegs . addReg ( CSRegs [ i ] ) ; if ( Unused ) { for ( MCRegister Reg : RC ) { if ( ! MRI . isPhysRegUsed ( Reg ) && LiveRegs . available ( MRI , Reg ) ) return Reg ;


<s> static MCRegister findScratchNonCalleeSaveRegister ( MachineRegisterInfo & MRI , LivePhysRegs & LiveRegs , const TargetRegisterClass & RC , bool Unused = false ) { const MCPhysReg * CSRegs = MRI . getCalleeSavedRegs ( ) ; for ( unsigned i = <NUM_LIT> ; CSRegs [ i ] ; ++ i ) LiveRegs . addReg ( CSRegs [ i ] ) ; if ( Unused ) { for ( MCRegister Reg : RC ) { if ( ! MRI . isPhysRegUsed ( Reg ) && LiveRegs . available ( MRI , Reg ) ) return Reg ; } } else { for ( MCRegister Reg : RC ) { if ( LiveRegs . available ( MRI , Reg ) ) return Reg ;


<s> static bool frameTriviallyRequiresSP ( const MachineFrameInfo & MFI ) { return MFI . hasVarSizedObjects ( ) || MFI . hasStackMap ( ) || MFI . hasPatchPoint ( ) ;


<s> <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; assert ( MFI -> isEntryFunction ( ) ) ; Register ScratchRsrcReg = MFI -> getScratchRSrcReg ( ) ; if ( ! ScratchRsrcReg || ( ! MRI . isPhysRegUsed ( ScratchRsrcReg ) && allStackObjectsAreDead ( MF ) ) ) return Register ( ) ; if ( ST . hasSGPRInitBug ( ) || ScratchRsrcReg != TRI -> reservedPrivateSegmentBufferReg ( MF ) ) return ScratchRsrcReg ; unsigned NumPreloaded = ( MFI -> getNumPreloadedSGPRs ( ) + <NUM_LIT> ) / <NUM_LIT> ; ArrayRef < MCPhysReg > AllSGPR128s = TRI -> getAllSGPR128 ( MF ) ; AllSGPR128s = AllSGPR128s . slice ( std :: min ( static_cast < unsigned > ( AllSGPR128s . size ( ) ) , NumPreloaded ) ) ; Register GITPtrLoReg = MFI -> getGITPtrLoReg ( MF ) ; for ( MCPhysReg Reg : AllSGPR128s ) { if ( ! MRI . isPhysRegUsed ( Reg ) && MRI . isAllocatable ( Reg ) && ! TRI -> isSubRegisterEq ( Reg , GITPtrLoReg ) ) { MRI . replaceRegWith ( ScratchRsrcReg , Reg ) ; MFI -> setScratchRSrcReg ( Reg ) ; return Reg ; }


<s> ArrayRef < MCPhysReg > AllSGPR128s = TRI -> getAllSGPR128 ( MF ) ; AllSGPR128s = AllSGPR128s . slice ( std :: min ( static_cast < unsigned > ( AllSGPR128s . size ( ) ) , NumPreloaded ) ) ; Register GITPtrLoReg = MFI -> getGITPtrLoReg ( MF ) ; for ( MCPhysReg Reg : AllSGPR128s ) {

if ( ! MRI . isPhysRegUsed ( Reg ) && MRI . isAllocatable ( Reg ) && ! TRI -> isSubRegisterEq ( Reg , GITPtrLoReg ) ) {


<s> FrameReg = RI -> getFrameRegister ( MF ) ;

return StackOffset :: getFixed ( MF . getFrameInfo ( ) . getObjectOffset ( FI ) ) ;


<s> const <ISA_LIT>RegisterInfo * RI = MF . getSubtarget < GCNSubtarget > ( ) . getRegisterInfo ( ) ;

FrameReg = RI -> getFrameRegister ( MF ) ;


<s> if ( ! MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) llvm_unreachable ( <STR_LIT> ) ; FrameIndex = NewFI ; LLVM_DEBUG ( auto Spill = MFI -> getSGPRToVGPRSpills ( NewFI ) . front ( ) ; dbgs ( ) << <STR_LIT> << ( IsFP ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << printReg ( Spill . VGPR , TRI ) << ':' << Spill . Lane << '\n' ) ; return ; } TempSGPR = findScratchNonCalleeSaveRegister ( MF . getRegInfo ( ) , LiveRegs , <STR_LIT> , true ) ; if ( ! TempSGPR ) { int NewFI = FrameInfo . CreateStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) , true , nullptr , TargetStackID :: SGPRSpill ) ; if ( TRI -> spillSGPRToVGPR ( ) && MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) { FrameIndex = NewFI ; LLVM_DEBUG ( auto Spill = MFI -> getSGPRToVGPRSpills ( NewFI ) . front ( ) ; dbgs ( ) << ( IsFP ? <STR_LIT> : <STR_LIT> ) << <STR_LIT> << printReg ( Spill . VGPR , TRI ) << ':' << Spill . Lane << '\n' ; ) ;


<s> static void getVGPRSpillLaneOrTempRegister ( MachineFunction & MF , LivePhysRegs & LiveRegs , Register & TempSGPR , Optional < int > & FrameIndex , bool IsFP ) { <ISA_LIT>MachineFunctionInfo * MFI = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; if ( MFI -> haveFreeLanesForSGPRSpill ( MF , <NUM_LIT> ) ) { int NewFI = FrameInfo . CreateStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) , true , nullptr , TargetStackID :: SGPRSpill ) ;

if ( ! MFI -> allocateSGPRSpillToVGPR ( MF , NewFI ) ) llvm_unreachable ( <STR_LIT> ) ;


<s> return MFI . getStackSize ( ) != <NUM_LIT> ; }

return frameTriviallyRequiresSP ( MFI ) || MFI . isFrameAddressTaken ( ) || MF . getSubtarget < GCNSubtarget > ( ) . getRegisterInfo ( ) -> hasStackRealignment ( MF ) || MF . getTarget ( ) . Options . DisableFramePointerElim ( MF ) ;


<s> LiveRegs . addLiveIns ( MBB ) ; } else { LiveRegs . addLiveOuts ( MBB ) ; LiveRegs . stepBackward ( * MBBI ) ; } }


<s> bool SeenDbgInstr = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator Next ; for ( auto I = MBB . begin ( ) , E = MBB . end ( ) ; I != E ; I = Next ) { MachineInstr & MI = * I ; Next = std :: next ( I ) ; if ( MI . isDebugInstr ( ) ) SeenDbgInstr = true ; if ( TII -> isVGPRSpill ( MI ) ) { unsigned FIOp = <STR_LIT> ( MI . getOpcode ( ) , <STR_LIT> :: vaddr ) ; int FI = MI . getOperand ( FIOp ) . getIndex ( ) ; Register VReg = TII -> getNamedOperand ( MI , <STR_LIT> :: vdata ) -> getReg ( ) ; if ( FuncInfo -> allocateVGPRSpillToAGPR ( MF , FI , TRI -> isAGPR ( MRI , VReg ) ) ) { RS -> enterBasicBlock ( MBB ) ; TRI -> eliminateFrameIndex ( MI , <NUM_LIT> , FIOp , RS ) ; SpillFIs . set ( FI ) ; continue ; } } } } for ( MachineBasicBlock & MBB : MF ) { for ( MCPhysReg Reg : FuncInfo -> getVGPRSpillAGPRs ( ) ) MBB . addLiveIn ( Reg ) ;

for ( MCPhysReg Reg : FuncInfo -> getAGPRSpillVGPRs ( ) ) MBB . addLiveIn ( Reg ) ;


<s> if ( MFI . hasCalls ( ) ) return true ; return frameTriviallyRequiresSP ( MFI ) ;


<s> <ISA_LIT>FrameLowering ( StackDirection D , Align StackAl , int LAO , Align TransAl = Align ( <NUM_LIT> ) ) : <ISA_LIT>FrameLowering ( D , StackAl , LAO , TransAl ) {


<s> const <ISA_LIT>InstrInfo * TII = MF . getSubtarget < GCNSubtarget > ( ) . getInstrInfo ( ) ; for ( const CalleeSavedInfo & CS : C<ISA_LIT> ) { unsigned Reg = CS . getReg ( ) ; if ( CS . isSpilledToReg ( ) ) {

BuildMI ( MBB , MBBI , DebugLoc ( ) , TII -> get ( TargetOpcode :: COPY ) , CS . getDstReg ( ) ) . addReg ( Reg , getKillRegState ( true ) ) ;


<s> static bool spilledToMemory ( const MachineFunction & MF , int SaveIndex ) { const MachineFrameInfo & MFI = MF . getFrameInfo ( ) ;

return MFI . getStackID ( SaveIndex ) != TargetStackID :: SGPRSpill ;


<s> auto & MBB = * CI . First -> getParent ( ) ;

auto ClauseMI = BuildMI ( MBB , * CI . First , DebugLoc ( ) , <ISA_LIT>I -> get ( <STR_LIT> ) ) . addImm ( CI . Length - <NUM_LIT> ) ;


<s> if ( CI . First == CI . Last ) return false ; assert ( CI . Length <= <NUM_LIT> && <STR_LIT> ) ;

auto & MBB = * CI . First -> getParent ( ) ;


<s> } if ( <ISA_LIT>InstrInfo :: isFLAT ( MI ) ) return HARDCLAUSE_FLAT ; if ( <ISA_LIT>InstrInfo :: isSMRD ( MI ) ) return HARDCLAUSE_SMEM ; } if ( MI . getOpcode ( ) == <STR_LIT> ) return HARDCLAUSE_INTERNAL ;

if ( MI . isMetaInstruction ( ) ) return HARDCLAUSE_IGNORE ;


<s> bool Dummy2 ; unsigned Dummy3 ; SmallVector < const MachineOperand * , <NUM_LIT> > BaseOps ; if ( Type <= LAST_REAL_HARDCLAUSE_TYPE ) { if ( ! <ISA_LIT>I -> getMemOperandsWithOffsetWidth ( MI , BaseOps , Dummy1 , Dummy2 , Dummy3 , TRI ) ) { Type = HARDCLAUSE_ILLEGAL ; } } if ( CI . Length == <NUM_LIT> || ( CI . Length && Type != HARDCLAUSE_INTERNAL && Type != HARDCLAUSE_IGNORE && ( Type != CI . Type || ! <ISA_LIT>I -> shouldClusterMemOps ( CI . BaseOps , BaseOps , <NUM_LIT> , <NUM_LIT> ) ) ) ) {

Changed |= emitClause ( CI , <ISA_LIT>I ) ;


<s> HardClauseType Type = getHardClauseType ( MI ) ; int64_t Dummy1 ; bool Dummy2 ; unsigned Dummy3 ; SmallVector < const MachineOperand * , <NUM_LIT> > BaseOps ; if ( Type <= LAST_REAL_HARDCLAUSE_TYPE ) { if ( ! <ISA_LIT>I -> getMemOperandsWithOffsetWidth ( MI , BaseOps , Dummy1 , Dummy2 , Dummy3 , TRI ) ) { Type = HARDCLAUSE_ILLEGAL ; } } if ( CI . Length == <NUM_LIT> || ( CI . Length && Type != HARDCLAUSE_INTERNAL && Type != HARDCLAUSE_IGNORE && ( Type != CI . Type || ! <ISA_LIT>I -> shouldClusterMemOps ( CI . BaseOps , BaseOps , <NUM_LIT> , <NUM_LIT> ) ) ) ) { Changed |= emitClause ( CI , <ISA_LIT>I ) ; CI = ClauseInfo ( ) ; } if ( CI . Length ) { if ( Type != HARDCLAUSE_IGNORE ) { if ( Type == HARDCLAUSE_INTERNAL ) { ++ CI . TrailingInternalLength ; } else { ++ CI . Length ; CI . Length += CI . TrailingInternalLength ; CI . TrailingInternalLength = <NUM_LIT> ; CI . Last = & MI ; CI . BaseOps = std :: move ( BaseOps ) ; } } } else if ( Type <= LAST_REAL_HARDCLAUSE_TYPE ) { CI = ClauseInfo { Type , & MI , & MI , <NUM_LIT> , <NUM_LIT> , std :: move ( BaseOps ) } ; } } if ( CI . Length ) Changed |= emitClause ( CI , <ISA_LIT>I ) ;


<s> Changed |= emitClause ( CI , <ISA_LIT>I ) ; CI = ClauseInfo ( ) ; } if ( CI . Length ) { ++ CI . Length ; if ( Type != HARDCLAUSE_INTERNAL ) { CI . Last = & MI ; CI . BaseOps = std :: move ( BaseOps ) ; } } else if ( Type <= LAST_REAL_HARDCLAUSE_TYPE ) { CI = ClauseInfo { Type , & MI , & MI , <NUM_LIT> , std :: move ( BaseOps ) } ; } } if ( CI . Length ) Changed |= emitClause ( CI , <ISA_LIT>I ) ; } return Changed ;


<s> const TargetRegisterInfo * TRI = ST . getRegisterInfo ( ) ; bool Changed = false ; for ( auto & MBB : MF ) { ClauseInfo CI ; for ( auto & MI : MBB ) { HardClauseType Type = getHardClauseType ( MI ) ; int64_t Dummy1 ; bool Dummy2 ; SmallVector < const MachineOperand * , <NUM_LIT> > BaseOps ; if ( Type <= LAST_REAL_HARDCLAUSE_TYPE ) { if ( ! <ISA_LIT>I -> getMemOperandsWithOffset ( MI , BaseOps , Dummy1 , Dummy2 , TRI ) ) { Type = HARDCLAUSE_ILLEGAL ; } } if ( CI . Length == <NUM_LIT> || ( CI . Length && Type != HARDCLAUSE_INTERNAL && ( Type != CI . Type || ! <ISA_LIT>I -> shouldClusterMemOps ( CI . BaseOps , BaseOps , <NUM_LIT> ) ) ) ) { Changed |= emitClause ( CI , <ISA_LIT>I ) ; CI = ClauseInfo ( ) ; } if ( CI . Length ) { ++ CI . Length ; if ( Type != HARDCLAUSE_INTERNAL ) { CI . Last = & MI ;

CI . BaseOps = std :: move ( BaseOps ) ;


<s> for ( auto & MBB : MF ) { ClauseInfo CI ; for ( auto & MI : MBB ) { HardClauseType Type = getHardClauseType ( MI ) ; int64_t Dummy1 ; bool Dummy2 ; unsigned Dummy3 ; SmallVector < const MachineOperand * , <NUM_LIT> > BaseOps ; if ( Type <= LAST_REAL_HARDCLAUSE_TYPE ) { if ( ! <ISA_LIT>I -> getMemOperandsWithOffsetWidth ( MI , BaseOps , Dummy1 , Dummy2 , Dummy3 , TRI ) ) { Type = HARDCLAUSE_ILLEGAL ; } }

if ( CI . Length == <NUM_LIT> || ( CI . Length && Type != HARDCLAUSE_INTERNAL && ( Type != CI . Type || ! <ISA_LIT>I -> shouldClusterMemOps ( CI . BaseOps , BaseOps , <NUM_LIT> , <NUM_LIT> ) ) ) ) {


<s> assert ( Size <= <NUM_LIT> && <STR_LIT> ) ; auto & MBB = * CI . First -> getParent ( ) ; auto ClauseMI = BuildMI ( MBB , * CI . First , DebugLoc ( ) , <ISA_LIT>I -> get ( <STR_LIT> ) ) . addImm ( Size - <NUM_LIT> ) ; finalizeBundle ( MBB , ClauseMI -> getIterator ( ) , std :: next ( CI . Last -> getIterator ( ) ) ) ; return true ;


<s> auto ClauseMI = BuildMI ( MBB , * CI . First , DebugLoc ( ) , <ISA_LIT>I -> get ( <STR_LIT> ) ) . addImm ( Size - <NUM_LIT> ) ;

finalizeBundle ( MBB , ClauseMI -> getIterator ( ) , std :: next ( CI . Last -> getIterator ( ) ) ) ;


<s> HardClauseType getHardClauseType ( const MachineInstr & MI ) { if ( MI . mayLoad ( ) ) { if ( <ISA_LIT>InstrInfo :: isVMEM ( MI ) || <ISA_LIT>InstrInfo :: isSegmentSpecificFLAT ( MI ) ) return HARDCLAUSE_VMEM ;

if ( <ISA_LIT>InstrInfo :: isFLAT ( MI ) ) return HARDCLAUSE_FLAT ;


<s> HardClauseType getHardClauseType ( const MachineInstr & MI ) { if ( MI . mayLoad ( ) ) {

if ( <ISA_LIT>InstrInfo :: isVMEM ( MI ) || <ISA_LIT>InstrInfo :: isSegmentSpecificFLAT ( MI ) ) return HARDCLAUSE_VMEM ;


<s> for ( auto & MI : MBB ) { HardClauseType Type = getHardClauseType ( MI ) ; int64_t Dummy1 ; bool Dummy2 ; unsigned Dummy3 ; SmallVector < const MachineOperand * , <NUM_LIT> > BaseOps ; if ( Type <= LAST_REAL_HARDCLAUSE_TYPE ) { if ( ! <ISA_LIT>I -> getMemOperandsWithOffsetWidth ( MI , BaseOps , Dummy1 , Dummy2 , Dummy3 , TRI ) ) { Type = HARDCLAUSE_ILLEGAL ; } } if ( CI . Length == <NUM_LIT> || ( CI . Length && Type != HARDCLAUSE_INTERNAL && ( Type != CI . Type || ! <ISA_LIT>I -> shouldClusterMemOps ( CI . BaseOps , BaseOps , <NUM_LIT> , <NUM_LIT> ) ) ) ) { Changed |= emitClause ( CI , <ISA_LIT>I ) ; CI = ClauseInfo ( ) ; } if ( CI . Length ) { ++ CI . Length ; if ( Type != HARDCLAUSE_INTERNAL ) { CI . Last = & MI ;


<s> <ISA_LIT>InsertHardClauses ( ) : MachineFunctionPass ( ID ) {


<s> FunctionPass * llvm :: create<ISA_LIT>InsertNopsPass ( ) { return new <ISA_LIT>InsertNops ( ) ;


<s> if ( MI -> isDebugValue ( ) || ! MI -> getDebugLoc ( ) ) { continue ; } auto DL = MI -> getDebugLoc ( ) ; auto CL = DL . getLine ( ) ; auto LineToInstEntry = LineToInst . find ( CL ) ; if ( LineToInstEntry == LineToInst . end ( ) ) { BuildMI ( * MBB , * MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; LineToInst . insert ( std :: make_pair ( CL , MI ) ) ; } else { LineToInstEntry -> second = MI ; } } } for ( auto LineToInstEntry = LineToInst . begin ( ) ; LineToInstEntry != LineToInst . end ( ) ; ++ LineToInstEntry ) { auto MBB = LineToInstEntry -> second -> getParent ( ) ; auto DL = LineToInstEntry -> second -> getDebugLoc ( ) ; MachineBasicBlock :: iterator MI = LineToInstEntry -> second ; ++ MI ; if ( MI != MBB -> end ( ) ) { BuildMI ( * MBB , * MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; } } MachineBasicBlock & MBB = MF . front ( ) ;


<s> <ISA_LIT>InsertNops ( ) : MachineFunctionPass ( ID ) {


<s> MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> void getAnalysisUsage ( AnalysisUsage & AU ) const override {

MachineFunctionPass :: getAnalysisUsage ( AU ) ;


<s> MachineBasicBlock * <ISA_LIT>InsertSkips :: insertSkipBlock ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator I ) const {

MachineFunction * MF = MBB . getParent ( ) ;


<s> break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) ; if ( TRI -> isVGPR ( MBB . getParent ( ) -> getRegInfo ( ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Opcode = <STR_LIT> ( Opcode ) ; BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else { BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . addReg ( <STR_LIT> , RegState :: Define ) . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <NUM_LIT> ) ; } break ; } case <STR_LIT> : { const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; int64_t KillVal = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; assert ( KillVal == <NUM_LIT> || KillVal == - <NUM_LIT> ) ; if ( Op . isImm ( ) ) { int64_t Imm = Op . getImm ( ) ; assert ( Imm == <NUM_LIT> || Imm == - <NUM_LIT> ) ; if ( Imm == KillVal ) BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addImm ( <NUM_LIT> ) ; break ; } unsigned Opcode = KillVal ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . add ( Op ) ; break ; } default :


<s> switch ( Opc ) { case TargetOpcode :: IMPLICIT_DEF : case TargetOpcode :: KILL : case TargetOpcode :: BUNDLE : case TargetOpcode :: CFI_INSTRUCTION : case TargetOpcode :: EH_LABEL :


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : ExecBranchStack . push_back ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { MI . eraseFromParent ( ) ; } else if ( HaveSkipBlock ) { MI . eraseFromParent ( ) ; } break ; case <STR_LIT> : case <STR_LIT> : MadeChange = true ; kill ( MI ) ; if ( ExecBranchStack . empty ( ) ) { if ( skipIfDead ( MI , * NextBB ) ) { HaveSkipBlock = true ; NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; } } else { HaveKill = true ; } MI . eraseFromParent ( ) ; break ; case <STR_LIT> : assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( BI != -- MF . end ( ) || I != MBB . getFirstTerminator ( ) ) { if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ;

MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ;


<s> bool HaveKill = false ; bool MadeChange = false ; SmallVector < MachineBasicBlock * , <NUM_LIT> > ExecBranchStack ; MachineFunction :: iterator NextBB ; MachineBasicBlock * EmptyMBBAtEnd = nullptr ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock & MBB = * BI ; bool HaveSkipBlock = false ; if ( ! ExecBranchStack . empty ( ) && ExecBranchStack . back ( ) == & MBB ) { ExecBranchStack . pop_back ( ) ; } if ( HaveKill && ExecBranchStack . empty ( ) ) { HaveKill = false ; } MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : ExecBranchStack . push_back ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { MI . eraseFromParent ( ) ; } else if ( HaveSkipBlock ) { MI . eraseFromParent ( ) ; } break ; case <STR_LIT> : case <STR_LIT> : MadeChange = true ; kill ( MI ) ; if ( ExecBranchStack . empty ( ) ) { if ( skipIfDead ( MI , * NextBB ) ) { HaveSkipBlock = true ; NextBB = std :: next ( BI ) ; BE = MF . end ( ) ;


<s> unsigned NumInstr = <NUM_LIT> ; const MachineFunction * MF = From . getParent ( ) ; for ( MachineFunction :: const_iterator MBBI ( & From ) , ToI ( & To ) , End = MF -> end ( ) ; MBBI != End && MBBI != ToI ; ++ MBBI ) { const MachineBasicBlock & MBB = * MBBI ; for ( MachineBasicBlock :: const_iterator I = MBB . begin ( ) , E = MBB . end ( ) ; NumInstr < SkipThreshold && I != E ; ++ I ) { if ( opcodeEmitsNoInsts ( I -> getOpcode ( ) ) ) continue ; if ( I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) return true ; if ( ( I -> getOpcode ( ) == <STR_LIT> ) || ( I -> getOpcode ( ) == <STR_LIT> ) ) { return true ; } if ( I -> isInlineAsm ( ) ) {

const MCAsmInfo * MAI = MF -> getTarget ( ) . getMCAsmInfo ( ) ;


<s> BuildMI ( & MBB , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( & NextBB ) ; MachineBasicBlock :: iterator Insert = SkipBB -> begin ( ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ;

BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) ;


<s> for ( ; M != E ; ++ M ) { if ( M -> definesRegister ( SReg , TRI ) ) break ; if ( M -> modifiesRegister ( SReg , TRI ) ) return Changed ; ReadsSreg |= M -> readsRegister ( SReg , TRI ) ; } if ( M == E || ! M -> isMoveImmediate ( ) || ! M -> getOperand ( <NUM_LIT> ) . isImm ( ) || M -> getOperand ( <NUM_LIT> ) . getImm ( ) != - <NUM_LIT> ) return Changed ; if ( ! ReadsSreg && Op2 . isKill ( ) ) { A -> getOperand ( <NUM_LIT> ) . ChangeToImmediate ( - <NUM_LIT> ) ; M -> eraseFromParent ( ) ; } } if ( ! ReadsCond && A -> registerDefIsDead ( <STR_LIT> ) && MI . killsRegister ( CondReg , TRI ) ) A -> eraseFromParent ( ) ; bool IsVCCZ = MI . getOpcode ( ) == <STR_LIT> ; if ( SReg == ExecReg ) { if ( IsVCCZ ) { MI . eraseFromParent ( ) ; return true ; } MI . setDesc ( TII -> get ( <STR_LIT> ) ) ;


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : ExecBranchStack . push_back ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { MI . eraseFromParent ( ) ; } else if ( HaveSkipBlock ) { MI . eraseFromParent ( ) ; } break ; case <STR_LIT> : case <STR_LIT> : MadeChange = true ; kill ( MI ) ; if ( ExecBranchStack . empty ( ) ) { if ( NextBB != BE && skipIfDead ( MI , * NextBB ) ) { HaveSkipBlock = true ; NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; } } else { HaveKill = true ; } MI . eraseFromParent ( ) ; break ; case <STR_LIT> : assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( BI != -- MF . end ( ) || I != MBB . getFirstTerminator ( ) ) { if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB . addSuccessor ( EmptyMBBAtEnd ) ; BuildMI ( * BI , I , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; I -> eraseFromParent ( ) ; } break ; case <STR_LIT> : case <STR_LIT> : MadeChange |= optimizeVccBranch ( MI ) ; break ;


<s> for ( MachineBasicBlock :: const_iterator I = MBB . begin ( ) , E = MBB . end ( ) ; NumInstr < SkipThreshold && I != E ; ++ I ) { if ( opcodeEmitsNoInsts ( I -> getOpcode ( ) ) ) continue ; if ( I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) return true ; if ( TII -> hasUnwantedEffectsWhenEXECEmpty ( * I ) ) return true ; if ( TII -> isSMRD ( * I ) || TII -> isVMEM ( * I ) || I -> getOpcode ( ) == <STR_LIT> ) return true ;


<s> MachineBasicBlock * SkipBB = insertSkipBlock ( MBB , MI . getIterator ( ) ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; BuildMI ( & MBB , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( & NextBB ) ; MachineBasicBlock :: iterator Insert = SkipBB -> begin ( ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; return true ;


<s> const DebugLoc & DL = MI . getDebugLoc ( ) ; BuildMI ( & MBB , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( & NextBB ) ; MachineBasicBlock :: iterator Insert = SkipBB -> begin ( ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ; return true ;


<s> const MachineBasicBlock & MBB = * MBBI ; for ( MachineBasicBlock :: const_iterator I = MBB . begin ( ) , E = MBB . end ( ) ; NumInstr < SkipThreshold && I != E ; ++ I ) { if ( opcodeEmitsNoInsts ( I -> getOpcode ( ) ) ) continue ; if ( I -> getOpcode ( ) == <STR_LIT> || I -> getOpcode ( ) == <STR_LIT> ) return true ; if ( TII -> hasUnwantedEffectsWhenEXECEmpty ( * I ) ) return true ; ++ NumInstr ; if ( NumInstr >= SkipThreshold ) return true ; }


<s> bool ReadsCond = false ; unsigned Threshold = <NUM_LIT> ; for ( ++ A ; A != E ; ++ A ) { if ( ! -- Threshold ) return false ; if ( A -> modifiesRegister ( ExecReg , TRI ) ) return false ; if ( A -> modifiesRegister ( CondReg , TRI ) ) { if ( ! A -> definesRegister ( CondReg , TRI ) || A -> getOpcode ( ) != And ) return false ; break ; } ReadsCond |= A -> readsRegister ( CondReg , TRI ) ; } if ( A == E ) return false ; MachineOperand & Op1 = A -> getOperand ( <NUM_LIT> ) ; MachineOperand & Op2 = A -> getOperand ( <NUM_LIT> ) ; if ( Op1 . getReg ( ) != ExecReg && Op2 . isReg ( ) && Op2 . getReg ( ) == ExecReg ) { TII -> commuteInstruction ( * A ) ; Changed = true ; } if ( Op1 . getReg ( ) != ExecReg ) return Changed ; if ( Op2 . isImm ( ) && Op2 . getImm ( ) != - <NUM_LIT> ) return Changed ; unsigned SReg = <STR_LIT> ; if ( Op2 . isReg ( ) ) {


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; const GCNSubtarget & ST = MBB . getParent ( ) -> getSubtarget < GCNSubtarget > ( ) ; const bool IsWave32 = ST . isWave32 ( ) ; const unsigned CondReg = TRI -> getVCC ( ) ; const unsigned ExecReg = IsWave32 ? <STR_LIT> : <STR_LIT> ; const unsigned And = IsWave32 ? <STR_LIT> : <STR_LIT> ; MachineBasicBlock :: reverse_iterator A = MI . getReverseIterator ( ) , E = MBB . rend ( ) ; bool ReadsCond = false ; unsigned Threshold = <NUM_LIT> ; for ( ++ A ; A != E ; ++ A ) { if ( ! -- Threshold ) return false ; if ( A -> modifiesRegister ( ExecReg , TRI ) ) return false ; if ( A -> modifiesRegister ( CondReg , TRI ) ) { if ( ! A -> definesRegister ( CondReg , TRI ) || A -> getOpcode ( ) != And ) return false ; break ; } ReadsCond |= A -> readsRegister ( CondReg , TRI ) ; } if ( A == E ) return false ; MachineOperand & Op1 = A -> getOperand ( <NUM_LIT> ) ; MachineOperand & Op2 = A -> getOperand ( <NUM_LIT> ) ; if ( Op1 . getReg ( ) != ExecReg && Op2 . isReg ( ) && Op2 . getReg ( ) == ExecReg ) { TII -> commuteInstruction ( * A ) ; Changed = true ; } if ( Op1 . getReg ( ) != ExecReg ) return Changed ; if ( Op2 . isImm ( ) && Op2 . getImm ( ) != - <NUM_LIT> ) return Changed ; unsigned SReg = <STR_LIT> ; if ( Op2 . isReg ( ) ) { SReg = Op2 . getReg ( ) ; auto M = std :: next ( A ) ; bool ReadsSreg = false ; for ( ; M != E ; ++ M ) { if ( M -> definesRegister ( SReg , TRI ) ) break ; if ( M -> modifiesRegister ( SReg , TRI ) ) return Changed ; ReadsSreg |= M -> readsRegister ( SReg , TRI ) ; }

if ( M == E || ! M -> isMoveImmediate ( ) || ! M -> getOperand ( <NUM_LIT> ) . isImm ( ) || M -> getOperand ( <NUM_LIT> ) . getImm ( ) != - <NUM_LIT> ) return Changed ;


<s> MachineBasicBlock & MBB = * BI ; bool HaveSkipBlock = false ; if ( ! ExecBranchStack . empty ( ) && ExecBranchStack . back ( ) == & MBB ) { ExecBranchStack . pop_back ( ) ; } if ( HaveKill && ExecBranchStack . empty ( ) ) { HaveKill = false ; } MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : ExecBranchStack . push_back ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; break ; case <STR_LIT> : ExecBranchStack . push_back ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { MI . eraseFromParent ( ) ; } else if ( HaveSkipBlock ) { MI . eraseFromParent ( ) ; }


<s> CallingConv :: ID CallConv = MBB . getParent ( ) -> getFunction ( ) -> getCallingConv ( ) ; assert ( CallConv == CallingConv :: <ISA_LIT>_PS || CallConv == CallingConv :: <ISA_LIT>_GS ) ; if ( Op . isImm ( ) ) {


<s> const MachineFunction * MF = From . getParent ( ) ; for ( MachineFunction :: const_iterator MBBI ( & From ) , ToI ( & To ) , End = MF -> end ( ) ; MBBI != End && MBBI != ToI ; ++ MBBI ) { const MachineBasicBlock & MBB = * MBBI ;

for ( MachineBasicBlock :: const_iterator I = MBB . begin ( ) , E = MBB . end ( ) ;


MachineFunctionPass ( ID ) , TRI ( nullptr ) , TII ( nullptr ) , SkipThreshold ( <NUM_LIT> ) {


<s> MachineFunctionPass ( ID ) , TRI ( nullptr ) , TII ( nullptr ) , SkipThreshold ( <NUM_LIT> ) {


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; MachineFunction * MF = MBB . getParent ( ) ; if ( MF -> getFunction ( ) -> getCallingConv ( ) != CallingConv :: <ISA_LIT>_PS || ! shouldSkip ( MBB , MBB . getParent ( ) -> back ( ) ) ) return false ; MachineBasicBlock * SkipBB = insertSkipBlock ( MBB , MI . getIterator ( ) ) ; const DebugLoc & DL = MI . getDebugLoc ( ) ; BuildMI ( & MBB , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( & NextBB ) ; MachineBasicBlock :: iterator Insert = SkipBB -> begin ( ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ;


<s> MachineBasicBlock :: iterator Insert = SkipBB -> begin ( ) ;

BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ;


<s> void <ISA_LIT>InsertSkips :: kill ( MachineInstr & MI ) { MachineBasicBlock & MBB = * MI . getParent ( ) ; DebugLoc DL = MI . getDebugLoc ( ) ; const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; CallingConv :: ID CallConv = MBB . getParent ( ) -> getFunction ( ) -> getCallingConv ( ) ; assert ( CallConv == CallingConv :: <ISA_LIT>_PS || CallConv == CallingConv :: <ISA_LIT>_GS ) ; if ( Op . isImm ( ) ) {


<s> assert ( CallConv == CallingConv :: <ISA_LIT>_PS || CallConv == CallingConv :: <ISA_LIT>_GS ) ; if ( Op . isImm ( ) ) { if ( Op . getImm ( ) & <NUM_LIT> ) { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addImm ( <NUM_LIT> ) ; } } else { BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . add ( Op ) ;


<s> SmallVector < MachineBasicBlock * , <NUM_LIT> > ExecBranchStack ; MachineFunction :: iterator NextBB ; MachineBasicBlock * EmptyMBBAtEnd = nullptr ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock & MBB = * BI ; bool HaveSkipBlock = false ; if ( ! ExecBranchStack . empty ( ) && ExecBranchStack . back ( ) == & MBB ) { ExecBranchStack . pop_back ( ) ; } if ( HaveKill && ExecBranchStack . empty ( ) ) { HaveKill = false ; } MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : ExecBranchStack . push_back ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { MI . eraseFromParent ( ) ; } else if ( HaveSkipBlock ) { MI . eraseFromParent ( ) ; } break ; case <STR_LIT> : MadeChange = true ;


<s> SmallVector < MachineBasicBlock * , <NUM_LIT> > ExecBranchStack ; MachineFunction :: iterator NextBB ; MachineBasicBlock * EmptyMBBAtEnd = nullptr ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock & MBB = * BI ; bool HaveSkipBlock = false ; if ( ! ExecBranchStack . empty ( ) && ExecBranchStack . back ( ) == & MBB ) { ExecBranchStack . pop_back ( ) ; } if ( HaveKill && ExecBranchStack . empty ( ) ) { HaveKill = false ; } MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : ExecBranchStack . push_back ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) {


<s> case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } const GCNSubtarget & ST = MBB . getParent ( ) -> getSubtarget < GCNSubtarget > ( ) ; if ( ST . hasNoSdstCMPX ( ) ) Opcode = <STR_LIT> ( Opcode ) ; assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) ; if ( TRI -> isVGPR ( MBB . getParent ( ) -> getRegInfo ( ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Opcode = <STR_LIT> ( Opcode ) ; BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else { auto I = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) ; if ( ! ST . hasNoSdstCMPX ( ) ) I . addReg ( <STR_LIT> , RegState :: Define ) ; I . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) ;


<s> EarlyExitBlock = MF -> CreateMachineBasicBlock ( ) ; MF -> insert ( MF -> end ( ) , EarlyExitBlock ) ;

generatePsEndPgm ( * EarlyExitBlock , EarlyExitBlock -> end ( ) , DL , TII ) ;


<s> BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ;


<s> static void generatePsEndPgm ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator I , DebugLoc DL , const <ISA_LIT>InstrInfo * TII ) {

BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ;


<s> auto NextBBI = std :: next ( MBB . getIterator ( ) ) ; bool NoSuccessor = I == MBB . end ( ) && llvm :: find ( MBB . successors ( ) , & * NextBBI ) == MBB . succ_end ( ) ; if ( NoSuccessor ) { generatePsEndPgm ( MBB , I , DL , TII ) ; } else { if ( ! EarlyExitBlock ) { createEarlyExitBlock ( MBB ) ; NextBBI = std :: next ( MBB . getIterator ( ) ) ;


<s> ( void ) MF ; assert ( MF -> getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ) ; auto NextBBI = std :: next ( MBB . getIterator ( ) ) ; bool NoSuccessor = I == MBB . end ( ) && llvm :: find ( MBB . successors ( ) , & * NextBBI ) == MBB . succ_end ( ) ; if ( NoSuccessor ) { generatePsEndPgm ( MBB , I , DL , TII ) ; } else { if ( ! EarlyExitBlock ) { createEarlyExitBlock ( MBB ) ; NextBBI = std :: next ( MBB . getIterator ( ) ) ; } MachineInstr * BranchMI = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( EarlyExitBlock ) ; auto Next = std :: next ( BranchMI -> getIterator ( ) ) ; if ( Next != MBB . end ( ) && ! Next -> isTerminator ( ) ) splitBlock ( MBB , * BranchMI , MDT ) ; MBB . addSuccessor ( EarlyExitBlock ) ; MDT -> getBase ( ) . insertEdge ( & MBB , EarlyExitBlock ) ;


<s> case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { assert ( & MI == & MBB . back ( ) ) ; MI . eraseFromParent ( ) ; MadeChange = true ; } break ; case <STR_LIT> : case <STR_LIT> : { MadeChange = true ; bool CanKill = kill ( MI ) ; if ( CanKill && MF . getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && dominatesAllReachable ( MBB ) ) { KillInstrs . push_back ( & MI ) ; } else { MI . eraseFromParent ( ) ; } break ; } case <STR_LIT> : assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( & MBB != & MF . back ( ) || & MI != & MBB . back ( ) ) { if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB . addSuccessor ( EmptyMBBAtEnd ) ; BuildMI ( MBB , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; MI . eraseFromParent ( ) ; MDT -> getBase ( ) . insertEdge ( & MBB , EmptyMBBAtEnd ) ; } break ; default : break ; } } } for ( MachineInstr * Kill : KillInstrs ) { skipIfDead ( * Kill -> getParent ( ) , std :: next ( Kill -> getIterator ( ) ) , Kill -> getDebugLoc ( ) ) ; Kill -> eraseFromParent ( ) ; }


<s> auto NextBBI = std :: next ( MBB . getIterator ( ) ) ; bool NoSuccessor = llvm :: find ( MBB . successors ( ) , & * NextBBI ) == MBB . succ_end ( ) ; MachineBasicBlock * SkipBB ; if ( NoSuccessor ) { SkipBB = & MBB ; } else { SkipBB = MF -> CreateMachineBasicBlock ( ) ; MF -> insert ( NextBBI , SkipBB ) ; BuildMI ( & MBB , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( & * NextBBI ) ; MBB . addSuccessor ( SkipBB ) ;


<s> MachineBasicBlock :: iterator Insert = SkipBB -> begin ( ) ; BuildMI ( * SkipBB , Insert , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) ;


<s> createEarlyExitBlock ( MBB ) ; NextBBI = std :: next ( MBB . getIterator ( ) ) ; } auto BranchMI = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( EarlyExitBlock ) ; auto Next = std :: next ( BranchMI -> getIterator ( ) ) ; if ( Next != MBB . end ( ) && ! Next -> isTerminator ( ) ) { MachineBasicBlock * SplitBB = MF -> CreateMachineBasicBlock ( MBB . getBasicBlock ( ) ) ; MF -> insert ( NextBBI , SplitBB ) ;

SplitBB -> splice ( SplitBB -> begin ( ) , & MBB , I , MBB . end ( ) ) ;


<s> assert ( MF -> getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ) ; auto NextBBI = std :: next ( MBB . getIterator ( ) ) ; bool NoSuccessor = I == MBB . end ( ) && llvm :: find ( MBB . successors ( ) , & * NextBBI ) == MBB . succ_end ( ) ; if ( NoSuccessor ) { generatePsEndPgm ( MBB , I , DL , TII ) ; } else { ensureEarlyExitBlock ( MBB , false ) ; MachineInstr * BranchMI = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( EarlyExitBlock ) ; auto Next = std :: next ( BranchMI -> getIterator ( ) ) ; if ( Next != MBB . end ( ) && ! Next -> isTerminator ( ) ) splitBlock ( MBB , * BranchMI , MDT ) ; MBB . addSuccessor ( EarlyExitBlock ) ; MDT -> getBase ( ) . insertEdge ( & MBB , EarlyExitBlock ) ; }


<s> MachineFunction :: iterator NextBB ; MachineBasicBlock * EmptyMBBAtEnd = nullptr ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock & MBB = * BI ; bool HaveSkipBlock = false ; if ( ! ExecBranchStack . empty ( ) && ExecBranchStack . back ( ) == & MBB ) { ExecBranchStack . pop_back ( ) ; } if ( HaveKill && ExecBranchStack . empty ( ) ) { HaveKill = false ; } MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : ExecBranchStack . push_back ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { MI . eraseFromParent ( ) ; } else if ( HaveSkipBlock ) { MI . eraseFromParent ( ) ; } break ; case <STR_LIT> : MadeChange = true ; kill ( MI ) ; if ( ExecBranchStack . empty ( ) ) { if ( skipIfDead ( MI , * NextBB ) ) { HaveSkipBlock = true ; NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; } } else { HaveKill = true ; } MI . eraseFromParent ( ) ;


<s> MachineFunction :: iterator NextBB ; MachineBasicBlock * EmptyMBBAtEnd = nullptr ; for ( MachineFunction :: iterator BI = MF . begin ( ) , BE = MF . end ( ) ; BI != BE ; BI = NextBB ) { NextBB = std :: next ( BI ) ; MachineBasicBlock & MBB = * BI ; bool HaveSkipBlock = false ; if ( ! ExecBranchStack . empty ( ) && ExecBranchStack . back ( ) == & MBB ) { ExecBranchStack . pop_back ( ) ; } if ( HaveKill && ExecBranchStack . empty ( ) ) { HaveKill = false ; } MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : ExecBranchStack . push_back ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ; MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { MI . eraseFromParent ( ) ; } else if ( HaveSkipBlock ) { MI . eraseFromParent ( ) ; } break ; case <STR_LIT> : MadeChange = true ; kill ( MI ) ; if ( ExecBranchStack . empty ( ) ) { if ( skipIfDead ( MI , * NextBB ) ) { HaveSkipBlock = true ; NextBB = std :: next ( BI ) ; BE = MF . end ( ) ; } } else { HaveKill = true ; }


<s> switch ( MI . getOpcode ( ) ) { case <STR_LIT> : { unsigned Opcode = <NUM_LIT> ; switch ( MI . getOperand ( <NUM_LIT> ) . getImm ( ) ) { case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } assert ( MI . getOperand ( <NUM_LIT> ) . isReg ( ) ) ; MachineInstr * NewMI ; if ( TRI -> isVGPR ( MBB . getParent ( ) -> getRegInfo ( ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) { Opcode = <STR_LIT> ( Opcode ) ; NewMI = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; } else { NewMI = BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . addReg ( <STR_LIT> , RegState :: Define ) . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <NUM_LIT> ) . add ( MI . getOperand ( <NUM_LIT> ) ) . addImm ( <NUM_LIT> ) ; } if ( NewMI -> hasExtraSrcRegAllocReq ( ) ) for ( MachineOperand & NewMO : NewMI -> uses ( ) ) if ( NewMO . isReg ( ) && NewMO . isUse ( ) ) NewMO . setIsRenamable ( false ) ; break ; } case <STR_LIT> : {

const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ;


<s> Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; case <STR_LIT> : Opcode = <STR_LIT> ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } if ( ! MI . getOperand ( <NUM_LIT> ) . isReg ( ) || ! TRI -> isVGPR ( MBB . getParent ( ) -> getRegInfo ( ) , MI . getOperand ( <NUM_LIT> ) . getReg ( ) ) ) llvm_unreachable ( <STR_LIT> ) ; BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) . add ( MI . getOperand ( <NUM_LIT> ) ) ; break ; } case <STR_LIT> : { const MachineOperand & Op = MI . getOperand ( <NUM_LIT> ) ; int64_t KillVal = MI . getOperand ( <NUM_LIT> ) . getImm ( ) ; assert ( KillVal == <NUM_LIT> || KillVal == - <NUM_LIT> ) ; if ( Op . isImm ( ) ) { int64_t Imm = Op . getImm ( ) ; assert ( Imm == <NUM_LIT> || Imm == - <NUM_LIT> ) ; if ( Imm == KillVal ) BuildMI ( MBB , & MI , DL , TII -> get ( <STR_LIT> ) , <STR_LIT> ) . addImm ( <NUM_LIT> ) ; break ; } unsigned Opcode = KillVal ? <STR_LIT> : <STR_LIT> ; BuildMI ( MBB , & MI , DL , TII -> get ( Opcode ) , <STR_LIT> ) . addReg ( <STR_LIT> ) . add ( Op ) ; break ;


<s> MachineBasicBlock & MBB = * MI . getParent ( ) ; const DebugLoc DL = MI . getDebugLoc ( ) ; ensureEarlyExitBlock ( MBB , true ) ;

auto BranchMI = BuildMI ( MBB , MI , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( EarlyExitBlock ) ;


<s> unsigned Mov = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; Register Exec = ST . isWave32 ( ) ? <STR_LIT> : <STR_LIT> ; auto ExitI = EarlyExitBlock -> getFirstNonPHI ( ) ; BuildMI ( * EarlyExitBlock , ExitI , DL , TII -> get ( Mov ) , Exec ) . addImm ( <NUM_LIT> ) ; EarlyExitClearsExec = true ; }


<s> if ( IsPS ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <STR_LIT> :: ET_NULL ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ; }

BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <NUM_LIT> ) ;


<s> static void generateEndPgm ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator I , DebugLoc DL , const <ISA_LIT>InstrInfo * TII , bool IsPS ) { if ( IsPS ) { BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addImm ( <STR_LIT> :: ET_NULL ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addReg ( <STR_LIT> , RegState :: Undef ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) . addImm ( <NUM_LIT> ) ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; TII = ST . getInstrInfo ( ) ; TRI = & TII -> getRegisterInfo ( ) ; MDT = & getAnalysis < MachineDominatorTree > ( ) ; SkipThreshold = SkipThresholdFlag ; SmallVector < MachineInstr * , <NUM_LIT> > KillInstrs ; SmallVector < MachineInstr * , <NUM_LIT> > EarlyTermInstrs ; bool MadeChange = false ; for ( MachineBasicBlock & MBB : MF ) { MachineBasicBlock :: iterator I , Next ; for ( I = MBB . begin ( ) ; I != MBB . end ( ) ; I = Next ) { Next = std :: next ( I ) ; MachineInstr & MI = * I ; switch ( MI . getOpcode ( ) ) { case <STR_LIT> : MadeChange |= skipMaskBranch ( MI , MBB ) ; break ; case <STR_LIT> : if ( MBB . isLayoutSuccessor ( MI . getOperand ( <NUM_LIT> ) . getMBB ( ) ) ) { assert ( & MI == & MBB . back ( ) ) ; MI . eraseFromParent ( ) ; MadeChange = true ; } break ;


<s> void <ISA_LIT>InsertSkips :: skipIfDead ( MachineBasicBlock & MBB , MachineBasicBlock :: iterator I , DebugLoc DL ) { MachineFunction * MF = MBB . getParent ( ) ; ( void ) MF ; assert ( MF -> getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS ) ; auto NextBBI = std :: next ( MBB . getIterator ( ) ) ; bool NoSuccessor = I == MBB . end ( ) && ! llvm :: is_contained ( MBB . successors ( ) , & * NextBBI ) ; if ( NoSuccessor ) { generateEndPgm ( MBB , I , DL , TII , true ) ; } else { ensureEarlyExitBlock ( MBB , false ) ; MachineInstr * BranchMI = BuildMI ( MBB , I , DL , TII -> get ( <STR_LIT> ) ) . addMBB ( EarlyExitBlock ) ; auto Next = std :: next ( BranchMI -> getIterator ( ) ) ; if ( Next != MBB . end ( ) && ! Next -> isTerminator ( ) ) splitBlock ( MBB , * BranchMI , MDT ) ; MBB . addSuccessor ( EarlyExitBlock ) ; MDT -> getBase ( ) . insertEdge ( & MBB , EarlyExitBlock ) ; }


<s> } break ; case <STR_LIT> : case <STR_LIT> : { MadeChange = true ; bool CanKill = kill ( MI ) ; if ( CanKill && MF . getFunction ( ) . getCallingConv ( ) == CallingConv :: <ISA_LIT>_PS && dominatesAllReachable ( MBB ) ) { KillInstrs . push_back ( & MI ) ; } else { MI . eraseFromParent ( ) ; } break ; } case <STR_LIT> : assert ( ! MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) -> returnsVoid ( ) ) ; if ( & MBB != & MF . back ( ) || & MI != & MBB . back ( ) ) { if ( ! EmptyMBBAtEnd ) { EmptyMBBAtEnd = MF . CreateMachineBasicBlock ( ) ; MF . insert ( MF . end ( ) , EmptyMBBAtEnd ) ; } MBB . addSuccessor ( EmptyMBBAtEnd ) ; BuildMI ( MBB , & MI , MI . getDebugLoc ( ) , TII -> get ( <STR_LIT> ) ) . addMBB ( EmptyMBBAtEnd ) ; MI . eraseFromParent ( ) ; MDT -> getBase ( ) . insertEdge ( & MBB , EmptyMBBAtEnd ) ; } break ; case <STR_LIT> : case <STR_LIT> : MadeChange |= optimizeVccBranch ( MI ) ; break ; default : break ; } } } for ( MachineInstr * Kill : KillInstrs ) {

skipIfDead ( * Kill -> getParent ( ) , std :: next ( Kill -> getIterator ( ) ) , Kill -> getDebugLoc ( ) ) ;