<s> if ( CC == CallingConv :: <ISA_LIT>_KERNEL || CC == CallingConv :: SPIR_KERNEL ) { if ( ! F . arg_empty ( ) ) KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } else if ( CC == CallingConv :: <ISA_LIT>_PS ) { P<ISA_LIT>nputAddr = <STR_LIT> ( F ) ; } if ( ! isEntryFunction ( ) ) { ScratchRSrcReg = <STR_LIT> ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } } if ( UseFixedABI ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } bool HasStackObjects = F . hasFnAttribute ( <STR_LIT> ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa ) { PrivateSegmentBuffer = true ; if ( UseFixedABI ) { DispatchPtr = true ; QueuePtr = true ; DispatchID = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } } else if ( ST . isMesaGfxShader ( F ) ) { ImplicitBufferPtr = true ; } if ( UseFixedABI || F . hasFnAttribute ( <STR_LIT> ) ) KernargSegmentPtr = true ; if ( ST . hasFlatAddressSpace ( ) && isEntryFunction ( ) && isAmdHsaOrMesa ) { if ( HasCalls || HasStackObjects ) FlatScratchInit = true ; } Attribute A = F . getFnAttribute ( <STR_LIT> ) ; StringRef S = A . getValueAsString ( ) ; if ( ! S . empty ( ) ) S . consumeInteger ( <NUM_LIT> , GITPtrHigh ) ; A = F . getFnAttribute ( <STR_LIT> ) ;


<s> assert ( BufferRsrc ) ; auto PSV = BufferPSVs . try_emplace ( BufferRsrc , std :: make_unique < <ISA_LIT>BufferPseudoSourceValue > ( TII ) ) ;

return PSV . first -> second . get ( ) ;


<s> const <ISA_LIT>BufferPseudoSourceValue * getBufferPSV ( const <ISA_LIT>InstrInfo & TII , const Value * BufferRsrc ) { assert ( BufferRsrc ) ; auto PSV = BufferPSVs . try_emplace ( BufferRsrc , std :: make_unique < <ISA_LIT>BufferPseudoSourceValue > ( TII ) ) ; return PSV . first -> second . get ( ) ;


<s> auto PSV = ImagePSVs . try_emplace ( ImgRsrc , std :: make_unique < <ISA_LIT>ImagePseudoSourceValue > ( TII ) ) ; return PSV . first -> second . get ( ) ;


<s> auto Arg = ArgInfo . getPreloadedValue ( Value ) . first ;

return Arg ? Arg -> getRegister ( ) : Register ( ) ;


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . NoSignedZerosFPMath , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ;

YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ;


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ArgInfo ) ;

YamlIO . mapOptional ( <STR_LIT> , MFI . Mode , <ISA_LIT>Mode ( ) ) ;


<s> if ( Code . startswith ( <STR_LIT> ) ) return true ; } } continue ; } const Function * Callee = dyn_cast < Function > ( CB -> getCalledOperand ( ) -> stripPointerCasts ( ) ) ; if ( ! Callee ) return true ; if ( Callee -> getIntrinsicID ( ) == <STR_LIT> ) return true ; } }


<s> for ( StringRef Code : CI . Codes ) { Code . consume_front ( <STR_LIT> ) ; if ( Code . startswith ( <STR_LIT> ) ) return true ; } } continue ; } const Function * Callee = dyn_cast < Function > ( CB -> getCalledOperand ( ) -> stripPointerCasts ( ) ) ; if ( ! Callee ) return true ;

if ( Callee -> getIntrinsicID ( ) == <STR_LIT> ) return true ;


<s> MFI . RemoveStackObject ( R . first ) ; SGPRToVGPRSpills . erase ( R . first ) ; } } bool HaveSGPRToMemory = false ; if ( ResetSGPRSpillStackIDs ) { for ( int i = MFI . getObjectIndexBegin ( ) , e = MFI . getObjectIndexEnd ( ) ; i != e ; ++ i ) { if ( i != FramePointerSaveIndex && i != BasePointerSaveIndex ) { if ( MFI . getStackID ( i ) == TargetStackID :: SGPRSpill ) { MFI . setStackID ( i , TargetStackID :: Default ) ; HaveSGPRToMemory = true ; } } } } for ( auto & R : VGPRToAGPRSpills ) { if ( R . second . IsDead ) MFI . RemoveStackObject ( R . first ) ; } return HaveSGPRToMemory ;


<s> void ReserveWWMRegister ( Register Reg ) {


<s> SGPRSpillVGPRCSR ( Register V , Optional < int > F ) : VGPR ( V ) , FI ( F ) {


<s> if ( CC != CallingConv :: <ISA_LIT>_Gfx ) ArgInfo = <ISA_LIT>ArgumentUsageInfo :: FixedABIFunctionInfo ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; if ( ! ST . enableFlatScratch ( ) ) { ScratchRSrcReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; } if ( ! F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { ImplicitArgPtr = false ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; if ( ST . hasGFX90AInsts ( ) && ST . getMaxNumVGPRs ( F ) <= <STR_LIT> . getNumRegs ( ) && ! mayUseAGPRs ( MF ) ) MayNeedAGPRs = false ; } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa && ! ST . enableFlatScratch ( ) ) PrivateSegmentBuffer = true ; else if ( ST . isMesaGfxShader ( F ) ) ImplicitBufferPtr = true ; if ( ! <STR_LIT> ( CC ) ) { if ( IsKernel || ! F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( IsKernel || ! F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) && ST . getMaxWorkitemID ( F , <NUM_LIT> ) != <NUM_LIT> ) WorkItemIDY = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) && ST . getMaxWorkitemID ( F , <NUM_LIT> ) != <NUM_LIT> ) WorkItemIDZ = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } bool HasStackObjects = F . hasFnAttribute ( <STR_LIT> ) ; if ( ST . hasFlatAddressSpace ( ) && isEntryFunction ( ) && ( isAmdHsaOrMesa || ST . enableFlatScratch ( ) ) && ( HasCalls || HasStackObjects || ST . enableFlatScratch ( ) ) && ! ST . flatScratchIsArchitected ( ) ) { FlatScratchInit = true ; } if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ;

if ( ! ST . flatScratchIsArchitected ( ) ) {


<s> } const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; for ( unsigned I = <NUM_LIT> , E = MRI . getNumVirtRegs ( ) ; I != E ; ++ I ) { const Register Reg = Register :: index2VirtReg ( I ) ; const TargetRegisterClass * RC = MRI . getRegClassOrNull ( Reg ) ; if ( RC && <ISA_LIT>RegisterInfo :: isAGPRClass ( RC ) ) { UsesAGPRs = true ; return true ; } else if ( ! RC && ! MRI . use_empty ( Reg ) && MRI . getType ( Reg ) . isValid ( ) ) { return true ; }


<s> <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned Size = FrameInfo . getObjectSize ( FI ) ; unsigned NumLanes = Size / <NUM_LIT> ; if ( NumLanes > WaveSize ) return false ; assert ( Size >= <NUM_LIT> && <STR_LIT> ) ; assert ( TRI -> spillSGPRToVGPR ( ) && <STR_LIT> ) ; const MCPhysReg * CSRegs = MRI . getCalleeSavedRegs ( ) ; for ( unsigned I = <NUM_LIT> ; I < NumLanes ; ++ I , ++ NumVGPRSpillLanes ) { Register LaneVGPR ; unsigned VGPRIndex = ( NumVGPRSpillLanes % WaveSize ) ; if ( FuncInfo -> VGPRReservedForSGPRSpill && NumVGPRSpillLanes < WaveSize ) { assert ( FuncInfo -> VGPRReservedForSGPRSpill == SpillVGPRs . back ( ) . VGPR ) ; LaneVGPR = FuncInfo -> VGPRReservedForSGPRSpill ; } else if ( VGPRIndex == <NUM_LIT> ) { LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , MF ) ; if ( LaneVGPR == <STR_LIT> ) { SGPRToVGPRSpills . erase ( FI ) ; NumVGPRSpillLanes -= I ; return false ; } Optional < int > CSRSpillFI ; if ( ( FrameInfo . hasCalls ( ) || ! isEntryFunction ( ) ) && CSRegs && isCalleeSavedReg ( CSRegs , LaneVGPR ) ) { CSRSpillFI = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) ) ; } SpillVGPRs . push_back ( SGPRSpillVGPRCSR ( LaneVGPR , CSRSpillFI ) ) ; for ( MachineBasicBlock & BB : MF ) BB . addLiveIn ( LaneVGPR ) ; } else { LaneVGPR = SpillVGPRs . back ( ) . VGPR ; } SpillLanes . push_back ( SpilledReg ( LaneVGPR , VGPRIndex ) ) ;


<s> MaxKernArgAlign = assumeAligned ( YamlMFI . MaxKernArgAlign ) ; LDSSize = YamlMFI . LDSSize ; DynLDSAlign = YamlMFI . DynLDSAlign ; HighBitsOf32BitAddress = YamlMFI . HighBitsOf32BitAddress ; Occupancy = YamlMFI . Occupancy ; IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ;


<s> MaxKernArgAlign = assumeAligned ( YamlMFI . MaxKernArgAlign ) ; LDSSize = YamlMFI . LDSSize ; DynLDSAlign = YamlMFI . DynLDSAlign ; HighBitsOf32BitAddress = YamlMFI . HighBitsOf32BitAddress ; Occupancy = YamlMFI . Occupancy ; IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ;


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchWaveOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ;

YamlIO . mapOptional ( <STR_LIT> , MFI . ArgInfo ) ;


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register LaneVGPR = TRI -> findUnusedRegister ( MF . getRegInfo ( ) , & <STR_LIT> , MF , true ) ; if ( LaneVGPR == Register ( ) ) return false ; SpillVGPRs . push_back ( SGPRSpillVGPRCSR ( LaneVGPR , None ) ) ;

FuncInfo -> VGPRReservedForSGPRSpill = LaneVGPR ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register LaneVGPR = TRI -> findUnusedRegister ( MF . getRegInfo ( ) , & <STR_LIT> , MF , true ) ; if ( LaneVGPR == Register ( ) ) return false ;

SpillVGPRs . push_back ( SGPRSpillVGPRCSR ( LaneVGPR , None ) ) ;


<s> assert ( Reg != <NUM_LIT> && <STR_LIT> ) ; ScratchWaveOffsetReg = Reg ;


<s> if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } } if ( UseFixedABI ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } bool HasStackObjects = F . hasFnAttribute ( <STR_LIT> ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa ) { if ( ! ST . enableFlatScratch ( ) ) PrivateSegmentBuffer = true ; if ( UseFixedABI ) { DispatchPtr = true ; QueuePtr = true ; DispatchID = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } } else if ( ST . isMesaGfxShader ( F ) ) { ImplicitBufferPtr = true ; } if ( UseFixedABI || F . hasFnAttribute ( <STR_LIT> ) ) KernargSegmentPtr = true ; if ( ST . hasFlatAddressSpace ( ) && isEntryFunction ( ) && ( isAmdHsaOrMesa || ST . enableFlatScratch ( ) ) ) {

if ( HasCalls || HasStackObjects || ST . enableFlatScratch ( ) ) FlatScratchInit = true ;


<s> <ISA_LIT>Mode ( const <STR_LIT> & Mode ) { IEEE = Mode . IEEE ; DX10Clamp = Mode . DX10Clamp ;


<s> IEEE = Mode . IEEE ; DX10Clamp = Mode . DX10Clamp ;


<s> if ( FuncInfo -> VGPRReservedForSGPRSpill && NumVGPRSpillLanes < WaveSize ) { assert ( FuncInfo -> VGPRReservedForSGPRSpill == SpillVGPRs . back ( ) . VGPR ) ; LaneVGPR = FuncInfo -> VGPRReservedForSGPRSpill ; } else if ( VGPRIndex == <NUM_LIT> ) { LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , MF ) ; if ( LaneVGPR == <STR_LIT> ) { SGPRToVGPRSpills . erase ( FI ) ; NumVGPRSpillLanes -= I ; return false ; } Optional < int > SpillFI ; if ( ! isEntryFunction ( ) ) { SpillFI = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) ) ; } SpillVGPRs . push_back ( SGPRSpillVGPR ( LaneVGPR , SpillFI ) ) ; for ( MachineBasicBlock & BB : MF ) BB . addLiveIn ( LaneVGPR ) ; } else {

LaneVGPR = SpillVGPRs . back ( ) . VGPR ;


<s> ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } bool HasStackObjects = F . hasFnAttribute ( <STR_LIT> ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; if ( ! ST . flatScratchIsArchitected ( ) ) { PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa ) { if ( ! ST . enableFlatScratch ( ) ) PrivateSegmentBuffer = true ; if ( UseFixedABI ) { DispatchPtr = true ; QueuePtr = true ; DispatchID = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } } else if ( ST . isMesaGfxShader ( F ) ) { ImplicitBufferPtr = true ; } if ( UseFixedABI || F . hasFnAttribute ( <STR_LIT> ) ) KernargSegmentPtr = true ; if ( ST . hasFlatAddressSpace ( ) && isEntryFunction ( ) && ( isAmdHsaOrMesa || ST . enableFlatScratch ( ) ) && ! ST . flatScratchIsArchitected ( ) ) { if ( HasCalls || HasStackObjects || ST . enableFlatScratch ( ) ) FlatScratchInit = true ; } Attribute A = F . getFnAttribute ( <STR_LIT> ) ; StringRef S = A . getValueAsString ( ) ;

if ( ! S . empty ( ) ) S . consumeInteger ( <NUM_LIT> , GITPtrHigh ) ;


<s> const bool HasCalls = F . hasFnAttribute ( <STR_LIT> ) ; const bool UseFixedABI = <ISA_LIT>TargetMachine :: EnableFixedFunctionABI && CC != CallingConv :: <ISA_LIT>_Gfx && ( ! isEntryFunction ( ) || HasCalls ) ; if ( CC == CallingConv :: <ISA_LIT>_KERNEL || CC == CallingConv :: SPIR_KERNEL ) { if ( ! F . arg_empty ( ) ) KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } else if ( CC == CallingConv :: <ISA_LIT>_PS ) { P<ISA_LIT>nputAddr = <STR_LIT> ( F ) ; } if ( ! isEntryFunction ( ) ) { if ( UseFixedABI ) ArgInfo = <ISA_LIT>ArgumentUsageInfo :: FixedABIFunctionInfo ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; if ( ! ST . enableFlatScratch ( ) ) { ScratchRSrcReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; } if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } } if ( UseFixedABI ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ;

if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ;


<s> for ( auto & R : SGPRToVGPRSpills ) MFI . RemoveStackObject ( R . first ) ;

for ( unsigned i = MFI . getObjectIndexBegin ( ) , e = MFI . getObjectIndexEnd ( ) ;


<s> for ( unsigned i = MFI . getObjectIndexBegin ( ) , e = MFI . getObjectIndexEnd ( ) ;

i != e ; ++ i ) MFI . setStackID ( i , <NUM_LIT> ) ;


<s> ScratchWaveOffsetReg = <STR_LIT> ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( ScratchWaveOffsetReg ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } } if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa ) { PrivateSegmentBuffer = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } else if ( ST . isMesaGfxShader ( F ) ) { ImplicitBufferPtr = true ;


<s> if ( isAmdHsaOrMesa && ! ST . enableFlatScratch ( ) ) PrivateSegmentBuffer = true ; else if ( ST . isMesaGfxShader ( F ) ) ImplicitBufferPtr = true ; if ( ! <STR_LIT> ( CC ) ) { if ( IsKernel || ! F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( IsKernel || ! F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) && ST . getMaxWorkitemID ( F , <NUM_LIT> ) != <NUM_LIT> ) WorkItemIDY = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) && ST . getMaxWorkitemID ( F , <NUM_LIT> ) != <NUM_LIT> ) WorkItemIDZ = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( ! F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } bool HasStackObjects = F . hasFnAttribute ( <STR_LIT> ) ; if ( ST . hasFlatAddressSpace ( ) && isEntryFunction ( ) && ( isAmdHsaOrMesa || ST . enableFlatScratch ( ) ) && ( HasCalls || HasStackObjects || ST . enableFlatScratch ( ) ) && ! ST . flatScratchIsArchitected ( ) ) { FlatScratchInit = true ; } if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; if ( ! ST . flatScratchIsArchitected ( ) ) { PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ;


<s> const MCPhysReg * CSRegs = MRI . getCalleeSavedRegs ( ) ; for ( unsigned I = <NUM_LIT> ; I < NumLanes ; ++ I , ++ NumVGPRSpillLanes ) { Register LaneVGPR ; unsigned VGPRIndex = ( NumVGPRSpillLanes % WaveSize ) ; if ( VGPRIndex == <NUM_LIT> && ! FuncInfo -> VGPRReservedForSGPRSpill ) { LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , MF ) ; if ( LaneVGPR == <STR_LIT> ) { SGPRToVGPRSpills . erase ( FI ) ; NumVGPRSpillLanes -= I ; return false ; } Optional < int > CSRSpillFI ; if ( ( FrameInfo . hasCalls ( ) || ! isEntryFunction ( ) ) && CSRegs && isCalleeSavedReg ( CSRegs , LaneVGPR ) ) { CSRSpillFI = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) ) ; }

SpillVGPRs . push_back ( SGPRSpillVGPRCSR ( LaneVGPR , CSRSpillFI ) ) ;


<s> void <ISA_LIT>MachineFunctionInfo :: removeDeadFrameIndices ( MachineFrameInfo & MFI ) { for ( auto & R : SGPRToVGPRSpills ) { if ( R . first != FramePointerSaveIndex && R . first != BasePointerSaveIndex ) MFI . RemoveStackObject ( R . first ) ;


<s> Register LaneVGPR = TRI -> findUnusedRegister ( MF . getRegInfo ( ) , & <STR_LIT> , MF , true ) ; SpillVGPRs . push_back ( SGPRSpillVGPRCSR ( LaneVGPR , None ) ) ; FuncInfo -> VGPRReservedForSGPRSpill = LaneVGPR ;


<s> Register LaneVGPR = TRI -> findUnusedRegister ( MF . getRegInfo ( ) , & <STR_LIT> , MF , true ) ; SpillVGPRs . push_back ( SGPRSpillVGPRCSR ( LaneVGPR , None ) ) ; FuncInfo -> VGPRReservedForSGPRSpill = LaneVGPR ; return true ;


<s> FlatWorkGroupSizes = ST . getFlatWorkGroupSizes ( F ) ; WavesPerEU = ST . getWavesPerEU ( F ) ; Occupancy = ST . computeOccupancy ( F , getLDSSize ( ) ) ; CallingConv :: ID CC = F . getCallingConv ( ) ; const bool HasCalls = F . hasFnAttribute ( <STR_LIT> ) ; const bool UseFixedABI = <ISA_LIT>TargetMachine :: EnableFixedFunctionABI && ( ! isEntryFunction ( ) || HasCalls ) ; if ( CC == CallingConv :: <ISA_LIT>_KERNEL || CC == CallingConv :: SPIR_KERNEL ) { if ( ! F . arg_empty ( ) ) KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } else if ( CC == CallingConv :: <ISA_LIT>_PS ) { P<ISA_LIT>nputAddr = <STR_LIT> ( F ) ; } if ( ! isEntryFunction ( ) ) { ScratchRSrcReg = <STR_LIT> ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } } if ( UseFixedABI ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; }

bool HasStackObjects = F . hasFnAttribute ( <STR_LIT> ) ;


<s> } Optional < int > CSRSpillFI ; if ( ( FrameInfo . hasCalls ( ) || ! isEntryFunction ( ) ) && CSRegs && isCalleeSavedReg ( CSRegs , LaneVGPR ) ) { CSRSpillFI = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , <NUM_LIT> ) ; } SpillVGPRs . push_back ( SGPRSpillVGPRCSR ( LaneVGPR , CSRSpillFI ) ) ; for ( MachineBasicBlock & BB : MF ) BB . addLiveIn ( LaneVGPR ) ; } else { LaneVGPR = SpillVGPRs . back ( ) . VGPR ; } SpillLanes . push_back ( SpilledReg ( LaneVGPR , VGPRIndex ) ) ; } return true ;


<s> const Function & F = MF . getFunction ( ) ; FlatWorkGroupSizes = ST . getFlatWorkGroupSizes ( F ) ; WavesPerEU = ST . getWavesPerEU ( F ) ; Occupancy = getMaxWavesPerEU ( ) ; limitOccupancy ( MF ) ; CallingConv :: ID CC = F . getCallingConv ( ) ; if ( CC == CallingConv :: <ISA_LIT>_KERNEL || CC == CallingConv :: SPIR_KERNEL ) { if ( ! F . arg_empty ( ) ) KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } else if ( CC == CallingConv :: <ISA_LIT>_PS ) { P<ISA_LIT>nputAddr = <STR_LIT> ( F ) ; } if ( ! isEntryFunction ( ) ) { ScratchRSrcReg = <STR_LIT> ; ScratchWaveOffsetReg = <STR_LIT> ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( ScratchWaveOffsetReg ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } } if ( ST . debuggerEmitPrologue ( ) ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; bool MaySpill = ST . isVGPRSpillingEnabled ( F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; if ( HasStackObjects || MaySpill ) { PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } } bool IsCOV2 = ST . isAmdCodeObjectV2 ( F ) ; if ( IsCOV2 ) { if ( HasStackObjects || MaySpill ) PrivateSegmentBuffer = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } else if ( ST . isMesaGfxShader ( F ) ) {

if ( HasStackObjects || MaySpill ) ImplicitBufferPtr = true ;


<s> } } if ( ST . debuggerEmitPrologue ( ) ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; bool MaySpill = ST . isVGPRSpillingEnabled ( F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; if ( HasStackObjects || MaySpill ) { PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } } bool IsCOV2 = ST . isAmdCodeObjectV2 ( F ) ;


<s> if ( ! GWSResourcePSV ) { GWSResourcePSV = llvm :: make_unique < <ISA_LIT>GWSResourcePseudoSourceValue > ( TII ) ;


<s> auto Arg = ArgInfo . getPreloadedValue ( Value ) . first ; return Arg ? Arg -> getRegister ( ) : <NUM_LIT> ;


<s> auto Arg = ArgInfo . getPreloadedValue ( Value ) . first ;

return Arg ? Arg -> getRegister ( ) : <NUM_LIT> ;


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchWaveOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ;

YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ;


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const Function * F = MF . getFunction ( ) ; P<ISA_LIT>nputAddr = <STR_LIT> ( * F ) ; const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; if ( ! <STR_LIT> ( F -> getCallingConv ( ) ) ) { KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDZ = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDZ = true ; if ( WorkItemIDZ ) WorkItemIDY = true ; bool MaySpill = ST . isVGPRSpillingEnabled ( * F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( HasStackObjects || MaySpill ) PrivateSegmentWaveByteOffset = true ; if ( ST . isAmdCodeObjectV2 ( MF ) ) { if ( HasStackObjects || MaySpill ) PrivateSegmentBuffer = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } else if ( ST . isMesaGfxShader ( MF ) ) { if ( HasStackObjects || MaySpill ) PrivateMemoryInputPtr = true ;


<s> const Function * F = MF . getFunction ( ) ; P<ISA_LIT>nputAddr = <STR_LIT> ( * F ) ; const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; if ( ! <STR_LIT> ( F -> getCallingConv ( ) ) ) { KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDZ = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDZ = true ; if ( WorkItemIDZ ) WorkItemIDY = true ; bool MaySpill = ST . isVGPRSpillingEnabled ( * F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( HasStackObjects || MaySpill ) PrivateSegmentWaveByteOffset = true ; if ( ST . isAmdCodeObjectV2 ( MF ) ) { if ( HasStackObjects || MaySpill ) PrivateSegmentBuffer = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } else if ( ST . isMesaGfxShader ( MF ) ) {

if ( HasStackObjects || MaySpill ) PrivateMemoryInputPtr = true ;


<s> Register getVGPRForAGPRCopy ( ) const { assert ( VGPRForAGPRCopy && <STR_LIT> ) ; return VGPRForAGPRCopy ;


<s> assert ( VGPRForAGPRCopy && <STR_LIT> ) ; return VGPRForAGPRCopy ;


<s> HighBitsOf32BitAddress = YamlMFI . HighBitsOf32BitAddress ; IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ; WaveLimiter = YamlMFI . WaveLimiter ; HasSpilledSGPRs = YamlMFI . HasSpilledSGPRs ;

HasSpilledVGPRs = YamlMFI . HasSpilledVGPRs ;


<s> void <ISA_LIT>MachineFunctionInfo :: allocateWWMReservedSpillSlots ( MachineFrameInfo & MFI , const <ISA_LIT>RegisterInfo & TRI ) { assert ( WWMReservedFrameIndexes . empty ( ) ) ; WWMReservedFrameIndexes . resize ( WWMReservedRegs . size ( ) ) ; int I = <NUM_LIT> ; for ( Register VGPR : WWMReservedRegs ) { const TargetRegisterClass * RC = TRI . getPhysRegClass ( VGPR ) ;

WWMReservedFrameIndexes [ I ++ ] = MFI . CreateSpillStackObject ( TRI . getSpillSize ( * RC ) , TRI . getSpillAlign ( * RC ) ) ;


<s> assert ( WWMReservedFrameIndexes . empty ( ) ) ; WWMReservedFrameIndexes . resize ( WWMReservedRegs . size ( ) ) ; int I = <NUM_LIT> ; for ( Register VGPR : WWMReservedRegs ) { const TargetRegisterClass * RC = TRI . getPhysRegClass ( VGPR ) ; WWMReservedFrameIndexes [ I ++ ] = MFI . CreateSpillStackObject ( TRI . getSpillSize ( * RC ) , TRI . getSpillAlign ( * RC ) ) ;


<s> LDSSize = YamlMFI . LDSSize ; GDSSize = YamlMFI . GDSSize ; DynLDSAlign = YamlMFI . DynLDSAlign ; HighBitsOf32BitAddress = YamlMFI . HighBitsOf32BitAddress ; Occupancy = YamlMFI . Occupancy ; IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ; WaveLimiter = YamlMFI . WaveLimiter ; HasSpilledSGPRs = YamlMFI . HasSpilledSGPRs ; HasSpilledVGPRs = YamlMFI . HasSpilledVGPRs ; BytesInStackArgArea = YamlMFI . BytesInStackArgArea ; ReturnsVoid = YamlMFI . ReturnsVoid ; if ( YamlMFI . ScavengeFI ) { auto FIOrErr = YamlMFI . ScavengeFI -> getFI ( MF . getFrameInfo ( ) ) ; if ( ! FIOrErr ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , <NUM_LIT> , SourceMgr :: DK_Error , toString ( FIOrErr . takeError ( ) ) , <STR_LIT> , None , None ) ; SourceRange = YamlMFI . ScavengeFI -> SourceRange ; return true ; } ScavengeFI = * FIOrErr ;


<s> auto FIOrErr = YamlMFI . ScavengeFI -> getFI ( MF . getFrameInfo ( ) ) ; if ( ! FIOrErr ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , <NUM_LIT> , SourceMgr :: DK_Error , toString ( FIOrErr . takeError ( ) ) , <STR_LIT> , None , None ) ; SourceRange = YamlMFI . ScavengeFI -> SourceRange ; return true ; }


<s> static void mapping ( IO & YamlIO , <ISA_LIT>Mode & Mode ) { YamlIO . mapOptional ( <STR_LIT> , Mode . IEEE , true ) ; YamlIO . mapOptional ( <STR_LIT> , Mode . DX10Clamp , true ) ; YamlIO . mapOptional ( <STR_LIT> , Mode . FP32Denormals , true ) ; YamlIO . mapOptional ( <STR_LIT> , Mode . FP64FP16Denormals , true ) ;


<s> YamlIO . mapOptional ( <STR_LIT> , Mode . IEEE , true ) ; YamlIO . mapOptional ( <STR_LIT> , Mode . DX10Clamp , true ) ; YamlIO . mapOptional ( <STR_LIT> , Mode . FP32Denormals , true ) ;

YamlIO . mapOptional ( <STR_LIT> , Mode . FP64FP16Denormals , true ) ;


<s> DX10Clamp = Mode . DX10Clamp ; FP32Denormals = Mode . FP32Denormals ; FP64FP16Denormals = Mode . FP64FP16Denormals ;


<s> Optional < int > CSRSpillFI ; if ( ( FrameInfo . hasCalls ( ) || ! isEntryFunction ( ) ) && CSRegs && isCalleeSavedReg ( CSRegs , LaneVGPR ) ) { CSRSpillFI = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) ) ; } SpillVGPRs . push_back ( SGPRSpillVGPRCSR ( LaneVGPR , CSRSpillFI ) ) ; for ( MachineBasicBlock & BB : MF ) BB . addLiveIn ( LaneVGPR ) ; } else { LaneVGPR = SpillVGPRs . back ( ) . VGPR ; } SpillLanes . push_back ( SpilledReg ( LaneVGPR , VGPRIndex ) ) ; } return true ;


<s> unsigned WaveSize = ST . getWavefrontSize ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; unsigned Size = FrameInfo . getObjectSize ( FI ) ; assert ( Size >= <NUM_LIT> && Size <= <NUM_LIT> && <STR_LIT> ) ; assert ( TRI -> spillSGPRToVGPR ( ) && <STR_LIT> ) ; int NumLanes = Size / <NUM_LIT> ; const MCPhysReg * CSRegs = MRI . getCalleeSavedRegs ( ) ; for ( int I = <NUM_LIT> ; I < NumLanes ; ++ I , ++ NumVGPRSpillLanes ) { Register LaneVGPR ; unsigned VGPRIndex = ( NumVGPRSpillLanes % WaveSize ) ; if ( VGPRIndex == <NUM_LIT> && ! FuncInfo -> VGPRReservedForSGPRSpill ) { LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , MF ) ; if ( LaneVGPR == <STR_LIT> ) { SGPRToVGPRSpills . erase ( FI ) ; NumVGPRSpillLanes -= I ; return false ;


<s> HighBitsOf32BitAddress = YamlMFI . HighBitsOf32BitAddress ; IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ;


<s> std :: vector < SpilledReg > & SpillLanes = SGPRToVGPRSpills [ FI ] ; if ( ! SpillLanes . empty ( ) ) return true ; const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; unsigned WaveSize = ST . getWavefrontSize ( ) ; unsigned Size = FrameInfo . getObjectSize ( FI ) ; assert ( Size >= <NUM_LIT> && Size <= <NUM_LIT> && <STR_LIT> ) ; assert ( TRI -> spillSGPRToVGPR ( ) && <STR_LIT> ) ; int NumLanes = Size / <NUM_LIT> ;

const MCPhysReg * CSRegs = MRI . getCalleeSavedRegs ( ) ;


<s> int NumLanes = Size / <NUM_LIT> ; const MCPhysReg * CSRegs = MRI . getCalleeSavedRegs ( ) ; for ( int I = <NUM_LIT> ; I < NumLanes ; ++ I , ++ NumVGPRSpillLanes ) { Register LaneVGPR ; unsigned VGPRIndex = ( NumVGPRSpillLanes % WaveSize ) ; if ( VGPRIndex == <NUM_LIT> ) { LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , MF ) ; if ( LaneVGPR == <STR_LIT> ) { SGPRToVGPRSpills . erase ( FI ) ; NumVGPRSpillLanes -= I ; return false ; } Optional < int > CSRSpillFI ; if ( ( FrameInfo . hasCalls ( ) || ! isEntryFunction ( ) ) && CSRegs && isCalleeSavedReg ( CSRegs , LaneVGPR ) ) { CSRSpillFI = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , <NUM_LIT> ) ;


<s> WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa ) { PrivateSegmentBuffer = true ; if ( UseFixedABI ) { DispatchPtr = true ; QueuePtr = true ; DispatchID = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } } else if ( ST . isMesaGfxShader ( F ) ) { ImplicitBufferPtr = true ; } if ( UseFixedABI || F . hasFnAttribute ( <STR_LIT> ) ) KernargSegmentPtr = true ; if ( ST . hasFlatAddressSpace ( ) && isEntryFunction ( ) && isAmdHsaOrMesa ) { auto hasNonSpillStackObjects = [ & ] ( ) { if ( ! HasStackObjects ) return false ; for ( auto OI = FrameInfo . getObjectIndexBegin ( ) , OE = FrameInfo . getObjectIndexEnd ( ) ; OI != OE ; ++ OI ) if ( ! FrameInfo . isSpillSlotObjectIndex ( OI ) ) return true ; return false ; } ; if ( HasCalls || hasNonSpillStackObjects ( ) ) FlatScratchInit = true ; } Attribute A = F . getFnAttribute ( <STR_LIT> ) ; StringRef S = A . getValueAsString ( ) ; if ( ! S . empty ( ) ) S . consumeInteger ( <NUM_LIT> , GITPtrHigh ) ; A = F . getFnAttribute ( <STR_LIT> ) ; S = A . getValueAsString ( ) ; if ( ! S . empty ( ) ) S . consumeInteger ( <NUM_LIT> , HighBitsOf32BitAddress ) ;

S = F . getFnAttribute ( <STR_LIT> ) . getValueAsString ( ) ;


<s> if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa ) { PrivateSegmentBuffer = true ; if ( UseFixedABI ) { DispatchPtr = true ; QueuePtr = true ; DispatchID = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } } else if ( ST . isMesaGfxShader ( F ) ) { ImplicitBufferPtr = true ; } if ( UseFixedABI || F . hasFnAttribute ( <STR_LIT> ) ) KernargSegmentPtr = true ; if ( ST . hasFlatAddressSpace ( ) && isEntryFunction ( ) && isAmdHsaOrMesa ) { auto hasNonSpillStackObjects = [ & ] ( ) { if ( ! HasStackObjects ) return false ; for ( auto OI = FrameInfo . getObjectIndexBegin ( ) , OE = FrameInfo . getObjectIndexEnd ( ) ; OI != OE ; ++ OI ) if ( ! FrameInfo . isSpillSlotObjectIndex ( OI ) ) return true ; return false ; } ; if ( HasCalls || hasNonSpillStackObjects ( ) ) FlatScratchInit = true ; } Attribute A = F . getFnAttribute ( <STR_LIT> ) ; StringRef S = A . getValueAsString ( ) ; if ( ! S . empty ( ) ) S . consumeInteger ( <NUM_LIT> , GITPtrHigh ) ; A = F . getFnAttribute ( <STR_LIT> ) ; S = A . getValueAsString ( ) ; if ( ! S . empty ( ) ) S . consumeInteger ( <NUM_LIT> , HighBitsOf32BitAddress ) ;


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ArgInfo ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . Mode , <ISA_LIT>Mode ( ) ) ;

YamlIO . mapOptional ( <STR_LIT> , MFI . HighBitsOf32BitAddress , <NUM_LIT> ) ;


<s> static void mapping ( IO & YamlIO , <ISA_LIT>MachineFunctionInfo & MFI ) { YamlIO . mapOptional ( <STR_LIT> , MFI . ExplicitKernArgSize , UINT64_C ( <NUM_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MaxKernArgAlign , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . LDSSize , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . IsEntryFunction , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . NoSignedZerosFPMath , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledSGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledVGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ;


<s> <ISA_LIT>MachineFunctionInfo :: <ISA_LIT>MachineFunctionInfo ( const MachineFunction & MF ) : <ISA_LIT>MachineFunction ( MF ) , TIDReg ( <STR_LIT> ) , ScratchRSrcReg ( <STR_LIT> ) , ScratchWaveOffsetReg ( <STR_LIT> ) , FrameOffsetReg ( <STR_LIT> ) , StackPtrOffsetReg ( <STR_LIT> ) , PrivateSegmentBufferUserSGPR ( <STR_LIT> ) , DispatchPtrUserSGPR ( <STR_LIT> ) , QueuePtrUserSGPR ( <STR_LIT> ) , KernargSegmentPtrUserSGPR ( <STR_LIT> ) , DispatchIDUserSGPR ( <STR_LIT> ) , FlatScratchInitUserSGPR ( <STR_LIT> ) , PrivateSegmentSizeUserSGPR ( <STR_LIT> ) , GridWorkGroupCountXUserSGPR ( <STR_LIT> ) , GridWorkGroupCountYUserSGPR ( <STR_LIT> ) , GridWorkGroupCountZUserSGPR ( <STR_LIT> ) , WorkGroupIDXSystemSGPR ( <STR_LIT> ) , WorkGroupIDYSystemSGPR ( <STR_LIT> ) , WorkGroupIDZSystemSGPR ( <STR_LIT> ) , WorkGroupInfoSystemSGPR ( <STR_LIT> ) , PrivateSegmentWaveByteOffsetSystemSGPR ( <STR_LIT> ) , WorkItemIDXVGPR ( <STR_LIT> ) , WorkItemIDYVGPR ( <STR_LIT> ) , WorkItemIDZVGPR ( <STR_LIT> ) , P<ISA_LIT>nputAddr ( <NUM_LIT> ) , P<ISA_LIT>nputEnable ( <NUM_LIT> ) , ReturnsVoid ( true ) , FlatWorkGroupSizes ( <NUM_LIT> , <NUM_LIT> ) , WavesPerEU ( <NUM_LIT> , <NUM_LIT> ) , DebuggerWorkGroupIDStackObjectIndices ( { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ) , DebuggerWorkItemIDStackObjectIndices ( { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ) , LDSWaveSpillSize ( <NUM_LIT> ) , NumUserSGPRs ( <NUM_LIT> ) , NumSystemSGPRs ( <NUM_LIT> ) , HasSpilledSGPRs ( false ) , HasSpilledVGPRs ( false ) , HasNonSpillStackObjects ( false ) , NumSpilledSGPRs ( <NUM_LIT> ) , NumSpilledVGPRs ( <NUM_LIT> ) , PrivateSegmentBuffer ( false ) , DispatchPtr ( false ) , QueuePtr ( false ) , KernargSegmentPtr ( false ) , DispatchID ( false ) , FlatScratchInit ( false ) , GridWorkgroupCountX ( false ) , GridWorkgroupCountY ( false ) , GridWorkgroupCountZ ( false ) , WorkGroupIDX ( false ) , WorkGroupIDY ( false ) , WorkGroupIDZ ( false ) , WorkGroupInfo ( false ) , PrivateSegmentWaveByteOffset ( false ) , WorkItemIDX ( false ) , WorkItemIDY ( false ) , WorkItemIDZ ( false ) , ImplicitBufferPtr ( false ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const Function * F = MF . getFunction ( ) ; FlatWorkGroupSizes = ST . getFlatWorkGroupSizes ( * F ) ; WavesPerEU = ST . getWavesPerEU ( * F ) ; if ( ! isEntryFunction ( ) ) { ScratchRSrcReg = <STR_LIT> ; ScratchWaveOffsetReg = <STR_LIT> ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; PrivateSegmentWaveByteOffsetSystemSGPR = ScratchWaveOffsetReg ; }

CallingConv :: ID CC = F -> getCallingConv ( ) ;


<s> <ISA_LIT>MachineFunctionInfo :: <ISA_LIT>MachineFunctionInfo ( const MachineFunction & MF ) : <ISA_LIT>MachineFunction ( MF ) , TIDReg ( <STR_LIT> ) , ScratchRSrcReg ( <STR_LIT> ) , ScratchWaveOffsetReg ( <STR_LIT> ) , PrivateSegmentBufferUserSGPR ( <STR_LIT> ) , DispatchPtrUserSGPR ( <STR_LIT> ) , QueuePtrUserSGPR ( <STR_LIT> ) , KernargSegmentPtrUserSGPR ( <STR_LIT> ) , DispatchIDUserSGPR ( <STR_LIT> ) , FlatScratchInitUserSGPR ( <STR_LIT> ) , PrivateSegmentSizeUserSGPR ( <STR_LIT> ) , GridWorkGroupCountXUserSGPR ( <STR_LIT> ) , GridWorkGroupCountYUserSGPR ( <STR_LIT> ) , GridWorkGroupCountZUserSGPR ( <STR_LIT> ) , WorkGroupIDXSystemSGPR ( <STR_LIT> ) , WorkGroupIDYSystemSGPR ( <STR_LIT> ) , WorkGroupIDZSystemSGPR ( <STR_LIT> ) , WorkGroupInfoSystemSGPR ( <STR_LIT> ) , PrivateSegmentWaveByteOffsetSystemSGPR ( <STR_LIT> ) , P<ISA_LIT>nputAddr ( <NUM_LIT> ) , ReturnsVoid ( true ) , LDSWaveSpillSize ( <NUM_LIT> ) , P<ISA_LIT>nputEna ( <NUM_LIT> ) , NumUserSGPRs ( <NUM_LIT> ) , NumSystemSGPRs ( <NUM_LIT> ) , HasSpilledSGPRs ( false ) , HasSpilledVGPRs ( false ) , PrivateSegmentBuffer ( false ) , DispatchPtr ( false ) , QueuePtr ( false ) , DispatchID ( false ) , KernargSegmentPtr ( false ) , FlatScratchInit ( false ) , GridWorkgroupCountX ( false ) , GridWorkgroupCountY ( false ) , GridWorkgroupCountZ ( false ) , WorkGroupIDX ( true ) , WorkGroupIDY ( false ) , WorkGroupIDZ ( false ) , WorkGroupInfo ( false ) , PrivateSegmentWaveByteOffset ( false ) , WorkItemIDX ( true ) , WorkItemIDY ( false ) , WorkItemIDZ ( false ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const Function * F = MF . getFunction ( ) ;

P<ISA_LIT>nputAddr = <STR_LIT> ( * F ) ;


<s> <ISA_LIT>MachineFunctionInfo :: <ISA_LIT>MachineFunctionInfo ( const MachineFunction & MF ) : <ISA_LIT>MachineFunction ( MF ) , TIDReg ( <STR_LIT> ) , HasSpilledSGPRs ( false ) , HasSpilledVGPRs ( false ) , P<ISA_LIT>nputAddr ( <NUM_LIT> ) , NumUserSGPRs ( <NUM_LIT> ) , LDSWaveSpillSize ( <NUM_LIT> ) {


<s> const <ISA_LIT>Subtarget & ST = MF -> getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; MachineFrameInfo * FrameInfo = MF -> getFrameInfo ( ) ; MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; int64_t Offset = FrameInfo -> getObjectOffset ( FrameIndex ) ; Offset += SubIdx * <NUM_LIT> ; unsigned LaneVGPRIdx = Offset / ( <NUM_LIT> * <NUM_LIT> ) ;

unsigned Lane = ( Offset / <NUM_LIT> ) % <NUM_LIT> ;


<s> ScratchRSrcReg = <STR_LIT> ; ScratchWaveOffsetReg = <STR_LIT> ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( ScratchWaveOffsetReg ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } } if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa ) { PrivateSegmentBuffer = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ;

} else if ( ST . isMesaGfxShader ( F ) ) {


<s> unsigned getDebuggerReservedVGPRCount ( ) const {


<s> Offset += SubIdx * <NUM_LIT> ; unsigned LaneVGPRIdx = Offset / ( <NUM_LIT> * <NUM_LIT> ) ; unsigned Lane = ( Offset / <NUM_LIT> ) % <NUM_LIT> ; struct SpilledReg Spill ; Spill . Lane = Lane ; if ( ! LaneVGPRs . count ( LaneVGPRIdx ) ) { unsigned LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , * MF ) ; if ( LaneVGPR == <STR_LIT> ) return Spill ; LaneVGPRs [ LaneVGPRIdx ] = LaneVGPR ;

for ( MachineFunction :: iterator BI = MF -> begin ( ) , BE = MF -> end ( ) ;


<s> if ( ! LaneVGPRs . count ( LaneVGPRIdx ) ) { unsigned LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , * MF ) ; if ( LaneVGPR == <STR_LIT> ) return Spill ; LaneVGPRs [ LaneVGPRIdx ] = LaneVGPR ; for ( MachineFunction :: iterator BI = MF -> begin ( ) , BE = MF -> end ( ) ; BI != BE ; ++ BI ) { BI -> addLiveIn ( LaneVGPR ) ; } }

Spill . VGPR = LaneVGPRs [ LaneVGPRIdx ] ;


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const Function * F = MF . getFunction ( ) ; P<ISA_LIT>nputAddr = <STR_LIT> ( * F ) ; const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; if ( ! <STR_LIT> ( F -> getCallingConv ( ) ) ) { KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDZ = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDZ = true ; if ( WorkItemIDZ ) WorkItemIDY = true ; bool MaySpill = ST . isVGPRSpillingEnabled ( * F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( HasStackObjects || MaySpill ) PrivateSegmentWaveByteOffset = true ; if ( ST . isAmdHsaOS ( ) ) { if ( HasStackObjects || MaySpill ) PrivateSegmentBuffer = true ;

if ( F -> hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ;


<s> if ( CC == CallingConv :: <ISA_LIT>_KERNEL || CC == CallingConv :: SPIR_KERNEL ) { if ( ! F . arg_empty ( ) ) KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } else if ( CC == CallingConv :: <ISA_LIT>_PS ) { P<ISA_LIT>nputAddr = <STR_LIT> ( F ) ; } if ( ! isEntryFunction ( ) ) { ScratchRSrcReg = <STR_LIT> ; ScratchWaveOffsetReg = <STR_LIT> ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( ScratchWaveOffsetReg ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } } if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) {


<s> limitOccupancy ( getMaxWavesPerEU ( ) ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ;

limitOccupancy ( ST . getOccupancyWithLocalMemSize ( getLDSSize ( ) , MF . getFunction ( ) ) ) ;


<s> Occupancy = getMaxWavesPerEU ( ) ; limitOccupancy ( MF ) ; if ( ! isEntryFunction ( ) ) { ScratchRSrcReg = <STR_LIT> ; ScratchWaveOffsetReg = <STR_LIT> ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( ScratchWaveOffsetReg ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; assert ( MaxKernArgAlign == <NUM_LIT> ) ; MaxKernArgAlign = ST . getAlignmentForImplicitArgPtr ( ) ; } } CallingConv :: ID CC = F . getCallingConv ( ) ; if ( CC == CallingConv :: <ISA_LIT>_KERNEL || CC == CallingConv :: SPIR_KERNEL ) { if ( ! F . arg_empty ( ) ) KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } else if ( CC == CallingConv :: <ISA_LIT>_PS ) { P<ISA_LIT>nputAddr = <STR_LIT> ( F ) ; } if ( ST . debuggerEmitPrologue ( ) ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; bool MaySpill = ST . isVGPRSpillingEnabled ( F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; if ( HasStackObjects || MaySpill ) { PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } } bool IsCOV2 = ST . isAmdCodeObjectV2 ( F ) ; if ( IsCOV2 ) { if ( HasStackObjects || MaySpill ) PrivateSegmentBuffer = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } else if ( ST . isMesaGfxShader ( F ) ) {

if ( HasStackObjects || MaySpill ) ImplicitBufferPtr = true ;


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . LDSSize , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . DynLDSAlign , Align ( ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . IsEntryFunction , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . NoSignedZerosFPMath , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledSGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledVGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ;

YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ;


<s> <ISA_LIT>MachineFunctionInfo :: <ISA_LIT>MachineFunctionInfo ( const MachineFunction & MF ) : <ISA_LIT>MachineFunction ( MF ) , TIDReg ( <STR_LIT> ) , ScratchRSrcReg ( <STR_LIT> ) , ScratchWaveOffsetReg ( <STR_LIT> ) , PrivateSegmentBufferUserSGPR ( <STR_LIT> ) , DispatchPtrUserSGPR ( <STR_LIT> ) , QueuePtrUserSGPR ( <STR_LIT> ) , KernargSegmentPtrUserSGPR ( <STR_LIT> ) , DispatchIDUserSGPR ( <STR_LIT> ) , FlatScratchInitUserSGPR ( <STR_LIT> ) , PrivateSegmentSizeUserSGPR ( <STR_LIT> ) , GridWorkGroupCountXUserSGPR ( <STR_LIT> ) , GridWorkGroupCountYUserSGPR ( <STR_LIT> ) , GridWorkGroupCountZUserSGPR ( <STR_LIT> ) , WorkGroupIDXSystemSGPR ( <STR_LIT> ) , WorkGroupIDYSystemSGPR ( <STR_LIT> ) , WorkGroupIDZSystemSGPR ( <STR_LIT> ) , WorkGroupInfoSystemSGPR ( <STR_LIT> ) , PrivateSegmentWaveByteOffsetSystemSGPR ( <STR_LIT> ) , P<ISA_LIT>nputAddr ( <NUM_LIT> ) , ReturnsVoid ( true ) , MaximumWorkGroupSize ( <NUM_LIT> ) , DebuggerReservedVGPRCount ( <NUM_LIT> ) , DebuggerWorkGroupIDStackObjectIndices ( { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ) , DebuggerWorkItemIDStackObjectIndices ( { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ) , LDSWaveSpillSize ( <NUM_LIT> ) , P<ISA_LIT>nputEna ( <NUM_LIT> ) , NumUserSGPRs ( <NUM_LIT> ) , NumSystemSGPRs ( <NUM_LIT> ) , HasSpilledSGPRs ( false ) , HasSpilledVGPRs ( false ) , HasNonSpillStackObjects ( false ) , HasFlatInstructions ( false ) , PrivateSegmentBuffer ( false ) , DispatchPtr ( false ) , QueuePtr ( false ) , DispatchID ( false ) , KernargSegmentPtr ( false ) , FlatScratchInit ( false ) , GridWorkgroupCountX ( false ) , GridWorkgroupCountY ( false ) , GridWorkgroupCountZ ( false ) , WorkGroupIDX ( false ) , WorkGroupIDY ( false ) , WorkGroupIDZ ( false ) , WorkGroupInfo ( false ) , PrivateSegmentWaveByteOffset ( false ) , WorkItemIDX ( false ) , WorkItemIDY ( false ) , WorkItemIDZ ( false ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const Function * F = MF . getFunction ( ) ; P<ISA_LIT>nputAddr = <STR_LIT> ( * F ) ; const MachineFrameInfo * FrameInfo = MF . getFrameInfo ( ) ; if ( ! <STR_LIT> ( F -> getCallingConv ( ) ) ) { KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDZ = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDZ = true ; if ( WorkItemIDZ ) WorkItemIDY = true ;

bool MaySpill = ST . isVGPRSpillingEnabled ( * F ) ;


<s> LaneVGPRs [ LaneVGPRIdx ] = LaneVGPR ; MRI . setPhysRegUsed ( LaneVGPR ) ; for ( MachineFunction :: iterator BI = MF -> begin ( ) , BE = MF -> end ( ) ; BI != BE ; ++ BI ) { BI -> addLiveIn ( LaneVGPR ) ; } } Spill . VGPR = LaneVGPRs [ LaneVGPRIdx ] ; Spill . Lane = Lane ;


<s> struct SpilledReg Spill ; if ( ! LaneVGPRs . count ( LaneVGPRIdx ) ) { unsigned LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> ) ; LaneVGPRs [ LaneVGPRIdx ] = LaneVGPR ; MRI . setPhysRegUsed ( LaneVGPR ) ; for ( MachineFunction :: iterator BI = MF -> begin ( ) , BE = MF -> end ( ) ;


<s> <ISA_LIT>MachineFunctionInfo :: <ISA_LIT>MachineFunctionInfo ( const MachineFunction & MF ) : <ISA_LIT>MachineFunction ( MF ) , TIDReg ( <STR_LIT> ) , HasSpilledVGPRs ( false ) , P<ISA_LIT>nputAddr ( <NUM_LIT> ) , NumUserSGPRs ( <NUM_LIT> ) , LDSWaveSpillSize ( <NUM_LIT> ) {


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . MaxKernArgAlign , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . LDSSize , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . IsEntryFunction , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . NoSignedZerosFPMath , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchWaveOffsetReg , StringValue ( <STR_LIT> ) ) ;

YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ;


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . LDSSize , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . IsEntryFunction , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . NoSignedZerosFPMath , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchWaveOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ;


<s> WorkItemIDX = true ; } if ( ST . debuggerEmitPrologue ( ) ) { WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDY = true ; WorkItemIDZ = true ; } else { if ( F -> hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } if ( WorkItemIDZ ) WorkItemIDY = true ; const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; bool MaySpill = ST . isVGPRSpillingEnabled ( * F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( HasStackObjects || MaySpill ) PrivateSegmentWaveByteOffset = true ; if ( ST . isAmdCodeObjectV2 ( MF ) ) { if ( HasStackObjects || MaySpill ) PrivateSegmentBuffer = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } else if ( ST . isMesaGfxShader ( MF ) ) {

if ( HasStackObjects || MaySpill ) PrivateMemoryInputPtr = true ;


<s> const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; if ( ! <STR_LIT> ( F -> getCallingConv ( ) ) ) { KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDZ = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDZ = true ; if ( WorkItemIDZ ) WorkItemIDY = true ; bool MaySpill = ST . isVGPRSpillingEnabled ( * F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( HasStackObjects || MaySpill ) PrivateSegmentWaveByteOffset = true ; if ( ST . isAmdHsaOS ( ) ) { if ( HasStackObjects || MaySpill ) PrivateSegmentBuffer = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ;

if ( F -> hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ;


<s> Register getPreloadedReg ( <STR_LIT> Value ) const { auto Arg = std :: get < <NUM_LIT> > ( ArgInfo . getPreloadedValue ( Value ) ) ;

return Arg ? Arg -> getRegister ( ) : Register ( ) ;


<s> return Arg ? Arg -> getRegister ( ) : Register ( ) ;


<s> Spill . Lane = Lane ; if ( ! LaneVGPRs . count ( LaneVGPRIdx ) ) { unsigned LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> ) ; if ( LaneVGPR == <STR_LIT> ) return Spill ; LaneVGPRs [ LaneVGPRIdx ] = LaneVGPR ; for ( MachineFunction :: iterator BI = MF -> begin ( ) , BE = MF -> end ( ) ; BI != BE ; ++ BI ) { BI -> addLiveIn ( LaneVGPR ) ; } } Spill . VGPR = LaneVGPRs [ LaneVGPRIdx ] ;


<s> MaxKernArgAlign = YamlMFI . MaxKernArgAlign ; LDSSize = YamlMFI . LDSSize ; HighBitsOf32BitAddress = YamlMFI . HighBitsOf32BitAddress ; IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ;


<s> if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; bool MaySpill = ST . isVGPRSpillingEnabled ( F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; if ( HasStackObjects || MaySpill ) { PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa ) { if ( HasStackObjects || MaySpill ) PrivateSegmentBuffer = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } else if ( ST . isMesaGfxShader ( F ) ) { if ( HasStackObjects || MaySpill ) ImplicitBufferPtr = true ; } if ( F . hasFnAttribute ( <STR_LIT> ) ) KernargSegmentPtr = true ; if ( ST . hasFlatAddressSpace ( ) && isEntryFunction ( ) && isAmdHsaOrMesa ) { if ( HasStackObjects || F . hasFnAttribute ( <STR_LIT> ) ) FlatScratchInit = true ;


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchWaveOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ArgInfo ) ;


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchWaveOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ;

YamlIO . mapOptional ( <STR_LIT> , MFI . ArgInfo ) ;


<s> Register <ISA_LIT>MachineFunctionInfo :: addDispatchID ( const <ISA_LIT>RegisterInfo & TRI ) { ArgInfo . DispatchID = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs += <NUM_LIT> ;

return ArgInfo . DispatchID . getRegister ( ) ;


<s> ArgInfo . FlatScratchInit = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs += <NUM_LIT> ; return ArgInfo . FlatScratchInit . getRegister ( ) ;


<s> Register <ISA_LIT>MachineFunctionInfo :: addFlatScratchInit ( const <ISA_LIT>RegisterInfo & TRI ) { ArgInfo . FlatScratchInit = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs += <NUM_LIT> ;

return ArgInfo . FlatScratchInit . getRegister ( ) ;


<s> ArgInfo . ImplicitBufferPtr = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs += <NUM_LIT> ; return ArgInfo . ImplicitBufferPtr . getRegister ( ) ;


<s> ArgInfo . KernargSegmentPtr = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs += <NUM_LIT> ;

return ArgInfo . KernargSegmentPtr . getRegister ( ) ;


<s> Register <ISA_LIT>MachineFunctionInfo :: addPrivateSegmentBuffer ( const <ISA_LIT>RegisterInfo & TRI ) { ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs += <NUM_LIT> ; return ArgInfo . PrivateSegmentBuffer . getRegister ( ) ;


<s> ArgInfo . QueuePtr = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs += <NUM_LIT> ;

return ArgInfo . QueuePtr . getRegister ( ) ;


<s> if ( FuncInfo -> VGPRReservedForSGPRSpill && NumVGPRSpillLanes < WaveSize ) { assert ( FuncInfo -> VGPRReservedForSGPRSpill == SpillVGPRs . back ( ) . VGPR ) ; LaneVGPR = FuncInfo -> VGPRReservedForSGPRSpill ; } else if ( VGPRIndex == <NUM_LIT> ) { LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , MF ) ; if ( LaneVGPR == <STR_LIT> ) { SGPRToVGPRSpills . erase ( FI ) ; NumVGPRSpillLanes -= I ; DiagnosticInfoResourceLimit DiagOutOfRegs ( MF . getFunction ( ) , <STR_LIT> , <NUM_LIT> , DS_Error ) ; MF . getFunction ( ) . getContext ( ) . diagnose ( DiagOutOfRegs ) ; return false ; } Optional < int > SpillFI ; if ( ! isEntryFunction ( ) ) {

SpillFI = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) ) ;


<s> unsigned NumLanes = Size / <NUM_LIT> ; if ( NumLanes > WaveSize ) return false ; assert ( Size >= <NUM_LIT> && <STR_LIT> ) ; assert ( TRI -> spillSGPRToVGPR ( ) && <STR_LIT> ) ; for ( unsigned I = <NUM_LIT> ; I < NumLanes ; ++ I , ++ NumVGPRSpillLanes ) { Register LaneVGPR ; unsigned VGPRIndex = ( NumVGPRSpillLanes % WaveSize ) ; if ( FuncInfo -> VGPRReservedForSGPRSpill && NumVGPRSpillLanes < WaveSize ) { assert ( FuncInfo -> VGPRReservedForSGPRSpill == SpillVGPRs . back ( ) . VGPR ) ; LaneVGPR = FuncInfo -> VGPRReservedForSGPRSpill ; } else if ( VGPRIndex == <NUM_LIT> ) { LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , MF ) ; if ( LaneVGPR == <STR_LIT> ) { SGPRToVGPRSpills . erase ( FI ) ; NumVGPRSpillLanes -= I ; DiagnosticInfoResourceLimit DiagOutOfRegs ( MF . getFunction ( ) , <STR_LIT> , <NUM_LIT> , DS_Error ) ; MF . getFunction ( ) . getContext ( ) . diagnose ( DiagOutOfRegs ) ; return false ; } Optional < int > SpillFI ; if ( ! isEntryFunction ( ) ) { SpillFI = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) ) ; }

SpillVGPRs . push_back ( SGPRSpillVGPR ( LaneVGPR , SpillFI ) ) ;


<s> const uint32_t * CSRMask = TRI -> getCallPreservedMask ( MF , MF . getFunction ( ) . getCallingConv ( ) ) ; if ( CSRMask ) OtherUsedRegs . setBitsInMask ( CSRMask ) ; for ( MCPhysReg Reg : SpillAGPR ) OtherUsedRegs . set ( Reg ) ; for ( MCPhysReg Reg : SpillVGPR ) OtherUsedRegs . set ( Reg ) ; SmallVectorImpl < MCPhysReg > :: const_iterator NextSpillReg = Regs . begin ( ) ; for ( int I = NumLanes - <NUM_LIT> ; I >= <NUM_LIT> ; -- I ) { NextSpillReg = std :: find_if ( NextSpillReg , Regs . end ( ) , [ & MRI , & OtherUsedRegs ] ( MCPhysReg Reg ) { return MRI . isAllocatable ( Reg ) && ! MRI . isPhysRegUsed ( Reg ) && ! OtherUsedRegs [ Reg ] ; } ) ; if ( NextSpillReg == Regs . end ( ) ) { Spill . FullyAllocated = false ; break ; }

OtherUsedRegs . set ( * NextSpillReg ) ;


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; Spill . FullyAllocated = true ; BitVector OtherUsedRegs ; OtherUsedRegs . resize ( TRI -> getNumRegs ( ) ) ; const uint32_t * CSRMask = TRI -> getCallPreservedMask ( MF , MF . getFunction ( ) . getCallingConv ( ) ) ; if ( CSRMask ) OtherUsedRegs . setBitsInMask ( CSRMask ) ; for ( MCPhysReg Reg : SpillAGPR ) OtherUsedRegs . set ( Reg ) ; for ( MCPhysReg Reg : SpillVGPR ) OtherUsedRegs . set ( Reg ) ; SmallVectorImpl < MCPhysReg > :: const_iterator NextSpillReg = Regs . begin ( ) ; for ( int I = NumLanes - <NUM_LIT> ; I >= <NUM_LIT> ; -- I ) { NextSpillReg = std :: find_if ( NextSpillReg , Regs . end ( ) , [ & MRI , & OtherUsedRegs ] ( MCPhysReg Reg ) { return MRI . isAllocatable ( Reg ) && ! MRI . isPhysRegUsed ( Reg ) && ! OtherUsedRegs [ Reg ] ;


<s> explicit <ISA_LIT>BufferPseudoSourceValue ( const TargetInstrInfo & TII ) : <ISA_LIT>PseudoSourceValue ( PSVBuffer , TII ) {


<s> explicit <ISA_LIT>GWSResourcePseudoSourceValue ( const TargetInstrInfo & TII ) : <ISA_LIT>PseudoSourceValue ( GWSResource , TII ) {


<s> explicit <ISA_LIT>ImagePseudoSourceValue ( const TargetInstrInfo & TII ) : <ISA_LIT>PseudoSourceValue ( PSVImage , TII ) {


<s> <ISA_LIT>PseudoSourceValue ( unsigned Kind , const TargetInstrInfo & TII ) : PseudoSourceValue ( Kind , TII ) {


<s> Any |= convertArg ( AI . QueuePtr , ArgInfo . QueuePtr ) ; Any |= convertArg ( AI . KernargSegmentPtr , ArgInfo . KernargSegmentPtr ) ; Any |= convertArg ( AI . DispatchID , ArgInfo . DispatchID ) ; Any |= convertArg ( AI . FlatScratchInit , ArgInfo . FlatScratchInit ) ; Any |= convertArg ( AI . PrivateSegmentSize , ArgInfo . PrivateSegmentSize ) ; Any |= convertArg ( AI . WorkGroupIDX , ArgInfo . WorkGroupIDX ) ; Any |= convertArg ( AI . WorkGroupIDY , ArgInfo . WorkGroupIDY ) ; Any |= convertArg ( AI . WorkGroupIDZ , ArgInfo . WorkGroupIDZ ) ; Any |= convertArg ( AI . WorkGroupInfo , ArgInfo . WorkGroupInfo ) ; Any |= convertArg ( AI . PrivateSegmentWaveByteOffset , ArgInfo . PrivateSegmentWaveByteOffset ) ; Any |= convertArg ( AI . ImplicitArgPtr , ArgInfo . ImplicitArgPtr ) ;

Any |= convertArg ( AI . ImplicitBufferPtr , ArgInfo . ImplicitBufferPtr ) ;


<s> unsigned getGDSSize ( ) const { return GDSSize ;


<s> if ( ! GWSResourcePSV ) { GWSResourcePSV = std :: make_unique < <ISA_LIT>GWSResourcePseudoSourceValue > ( TII ) ;


<s> const <ISA_LIT>ImagePseudoSourceValue * getImagePSV ( const <ISA_LIT>InstrInfo & TII ) { if ( ! ImagePSV ) ImagePSV = std :: make_unique < <ISA_LIT>ImagePseudoSourceValue > ( TII ) ; return ImagePSV . get ( ) ;


<s> if ( ! ImagePSV ) ImagePSV = std :: make_unique < <ISA_LIT>ImagePseudoSourceValue > ( TII ) ;


<s> MCPhysReg <ISA_LIT>MachineFunctionInfo :: getNextSystemSGPR ( ) const { return <STR_LIT> + NumUserSGPRs + NumSystemSGPRs ;


<s> MCPhysReg <ISA_LIT>MachineFunctionInfo :: getNextSystemSGPR ( ) const {

return <STR_LIT> + NumUserSGPRs + NumSystemSGPRs ;


<s> MCPhysReg <ISA_LIT>MachineFunctionInfo :: getNextUserSGPR ( ) const {

assert ( NumSystemSGPRs == <NUM_LIT> && <STR_LIT> ) ;


<s> ScavengeFI = MFI . CreateFixedObject ( TRI . getSpillSize ( <STR_LIT> ) , <NUM_LIT> , false ) ; } else { ScavengeFI = MFI . CreateStackObject ( TRI . getSpillSize ( <STR_LIT> ) , TRI . getSpillAlign ( <STR_LIT> ) , false ) ;


<s> bool <ISA_LIT>MachineFunctionInfo :: haveFreeLanesForSGPRSpill ( const MachineFunction & MF , unsigned NumNeed ) const { const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned WaveSize = ST . getWavefrontSize ( ) ;

return NumVGPRSpillLanes + NumNeed <= WaveSize * SpillVGPRs . size ( ) ;


<s> IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ; WaveLimiter = YamlMFI . WaveLimiter ; HasSpilledSGPRs = YamlMFI . HasSpilledSGPRs ; HasSpilledVGPRs = YamlMFI . HasSpilledVGPRs ; if ( YamlMFI . ScavengeFI ) { auto FIOrErr = YamlMFI . ScavengeFI -> getFI ( MF . getFrameInfo ( ) ) ; if ( ! FIOrErr ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , <NUM_LIT> , SourceMgr :: DK_Error , toString ( FIOrErr . takeError ( ) ) , <STR_LIT> , None , None ) ; SourceRange = YamlMFI . ScavengeFI -> SourceRange ; return true ;


<s> NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ; WaveLimiter = YamlMFI . WaveLimiter ; HasSpilledSGPRs = YamlMFI . HasSpilledSGPRs ; HasSpilledVGPRs = YamlMFI . HasSpilledVGPRs ; if ( YamlMFI . ScavengeFI ) { auto FIOrErr = YamlMFI . ScavengeFI -> getFI ( MF . getFrameInfo ( ) ) ; if ( ! FIOrErr ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , <NUM_LIT> , SourceMgr :: DK_Error , toString ( FIOrErr . takeError ( ) ) , <STR_LIT> , None , None ) ; SourceRange = YamlMFI . ScavengeFI -> SourceRange ;


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; limitOccupancy ( ST . getOccupancyWithLocalMemSize ( getLDSSize ( ) , MF . getFunction ( ) ) ) ;


<s> limitOccupancy ( ST . getOccupancyWithLocalMemSize ( getLDSSize ( ) , MF . getFunction ( ) ) ) ;


<s> static void mapping ( IO & YamlIO , <ISA_LIT>MachineFunctionInfo & MFI ) { YamlIO . mapOptional ( <STR_LIT> , MFI . ExplicitKernArgSize , UINT64_C ( <NUM_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MaxKernArgAlign , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . LDSSize , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . DynLDSAlign , Align ( ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . IsEntryFunction , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . NoSignedZerosFPMath , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledSGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledVGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ArgInfo ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . Mode , <ISA_LIT>Mode ( ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HighBitsOf32BitAddress , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . Occupancy , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScavengeFI ) ;


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledSGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledVGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ArgInfo ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . Mode , <ISA_LIT>Mode ( ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HighBitsOf32BitAddress , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . Occupancy , <NUM_LIT> ) ;

YamlIO . mapOptional ( <STR_LIT> , MFI . ScavengeFI ) ;


raw_string_ostream OS ( Dest . Value ) ;


<s> } } for ( int i = MFI . getObjectIndexBegin ( ) , e = MFI . getObjectIndexEnd ( ) ; i != e ; ++ i ) if ( i != FramePointerSaveIndex && i != BasePointerSaveIndex && ( ! TRI -> isCFISavedRegsSpillEnabled ( ) || ( i != ReturnAddressSaveIndex && i != EXECSaveIndex ) ) ) MFI . setStackID ( i , TargetStackID :: Default ) ;


<s> for ( MachineBasicBlock & MBB : MF ) { MBB . removeLiveIn ( ReservedVGPR ) ; MBB . sortUniqueLiveIns ( ) ; } this -> VGPRReservedForSGPRSpill = <STR_LIT> ; return true ; } }


<s> SpillVGPRs . erase ( i ) ; for ( MachineBasicBlock & MBB : MF ) { MBB . removeLiveIn ( ReservedVGPR ) ; MBB . sortUniqueLiveIns ( ) ; } this -> VGPRReservedForSGPRSpill = <STR_LIT> ; return true ; } }


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register LaneVGPR = TRI -> findUnusedRegister ( MF . getRegInfo ( ) , & <STR_LIT> , MF , true ) ; if ( LaneVGPR == Register ( ) ) return false ; SpillVGPRs . push_back ( SGPRSpillVGPR ( LaneVGPR , None ) ) ; FuncInfo -> VGPRReservedForSGPRSpill = LaneVGPR ;


<s> WWMReservedRegs . insert ( std :: make_pair ( Reg , FI ) ) ;


<s> void setSGPRSpillVGPRs ( Register NewVGPR , Optional < int > newFI , int Index ) {


<s> SpillVGPRs [ Index ] . VGPR = NewVGPR ;


<s> void setTIDReg ( Register Reg ) { TIDReg = Reg ;


<s> if ( UseFixedABI ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } bool HasStackObjects = F . hasFnAttribute ( <STR_LIT> ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; if ( ! ST . flatScratchIsArchitected ( ) ) { PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa && ! ST . enableFlatScratch ( ) ) PrivateSegmentBuffer = true ; else if ( ST . isMesaGfxShader ( F ) ) ImplicitBufferPtr = true ; if ( ! <STR_LIT> ( CC ) ) { if ( UseFixedABI ) { DispatchPtr = true ; QueuePtr = true ; DispatchID = true ;


<s> return true ; } const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; for ( unsigned I = <NUM_LIT> , E = MRI . getNumVirtRegs ( ) ; I != E ; ++ I ) { const Register Reg = Register :: index2VirtReg ( I ) ; const TargetRegisterClass * RC = MRI . getRegClassOrNull ( Reg ) ; if ( RC && <ISA_LIT>RegisterInfo :: isAGPRClass ( RC ) ) { UsesAGPRs = true ; return true ; } } for ( MCRegister Reg : <STR_LIT> ) { if ( MRI . isPhysRegUsed ( Reg ) ) { UsesAGPRs = true ; return true ; } }


<s> const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; for ( unsigned I = <NUM_LIT> , E = MRI . getNumVirtRegs ( ) ; I != E ; ++ I ) { const Register Reg = Register :: index2VirtReg ( I ) ; const TargetRegisterClass * RC = MRI . getRegClassOrNull ( Reg ) ; if ( RC && <ISA_LIT>RegisterInfo :: isAGPRClass ( RC ) ) { UsesAGPRs = true ; return true ; } } for ( MCRegister Reg : <STR_LIT> ) { if ( MRI . isPhysRegUsed ( Reg ) ) { UsesAGPRs = true ; return true ;


<s> Preds . push_back ( Pred ) ; assert ( none_of ( Succs , [ = ] ( std :: pair < <ISA_LIT>ScheduleBlock * , <ISA_LIT>ScheduleBlockLinkKind > S ) { return PredID == S . first -> getID ( ) ;


<s> if ( PredID == P -> getID ( ) ) return ; } Preds . push_back ( Pred ) ; assert ( none_of ( Succs , [ = ] ( std :: pair < <ISA_LIT>ScheduleBlock * , <ISA_LIT>ScheduleBlockLinkKind > S ) { return PredID == S . first -> getID ( ) ;


<s> Succs . push_back ( std :: make_pair ( Succ , Kind ) ) ; assert ( none_of ( Preds , [ = ] ( <ISA_LIT>ScheduleBlock * P ) { return SuccID == P -> getID ( ) ; } ) && <STR_LIT> ) ;


<s> return ; } } if ( Succ -> isHighLatencyBlock ( ) ) ++ NumHighLatencySuccessors ; Succs . push_back ( std :: make_pair ( Succ , Kind ) ) ; assert ( none_of ( Preds , [ = ] ( <ISA_LIT>ScheduleBlock * P ) { return SuccID == P -> getID ( ) ; } ) && <STR_LIT> ) ;


<s> decreaseLiveRegs ( Block , Block -> getInRegs ( ) ) ; addLiveRegs ( Block -> getOutRegs ( ) ) ; releaseBlockSuccs ( Block ) ; for ( const auto & RegP : LiveOutRegsNumUsages [ Block -> getID ( ) ] ) {

assert ( LiveRegsConsumers . find ( RegP . first ) == LiveRegsConsumers . end ( ) || LiveRegsConsumers [ RegP . first ] == <NUM_LIT> ) ;


<s> } for ( Register Reg : OutRegs ) { if ( ! Reg . isVirtual ( ) ) continue ; PSetIterator PSetI = DAG -> getMRI ( ) -> getPressureSets ( Reg ) ; for ( ; PSetI . isValid ( ) ; ++ PSetI ) { DiffSetPressure [ * PSetI ] += PSetI . getWeight ( ) ;


if ( SUColors . size ( ) == <NUM_LIT> && SUColorsPending . size ( ) == <NUM_LIT> ) PendingColoring [ SU -> NodeNum ] = * SUColors . begin ( ) ;


<s> if ( SuccDep . isWeak ( ) || SuccSU -> NodeNum >= DAG -> SUnits . size ( ) ) { continue ; } assert ( SuccSU -> isInstr ( ) && <STR_LIT> ) ; if ( ! <ISA_LIT>InstrInfo :: isEXP ( * SuccSU -> getInstr ( ) ) ) { return ; } }


<s> const SUnit * SuccSU = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || SuccSU -> NodeNum >= DAG -> SUnits . size ( ) ) { continue ; } assert ( SuccSU -> isInstr ( ) && <STR_LIT> ) ;

if ( ! <ISA_LIT>InstrInfo :: isEXP ( * SuccSU -> getInstr ( ) ) ) {


<s> std :: set < unsigned > SeenColors ; if ( DAGSize <= <NUM_LIT> ) return ; PreviousColor = CurrentColoring [ <NUM_LIT> ] ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { SUnit * SU = & DAG -> SUnits [ i ] ; unsigned CurrentColor = CurrentColoring [ i ] ; unsigned PreviousColorSave = PreviousColor ; assert ( i == SU -> NodeNum ) ; if ( CurrentColor != PreviousColor ) SeenColors . insert ( PreviousColor ) ;


<s> void <ISA_LIT>ScheduleBlockCreator :: colorHighLatenciesAlone ( ) { unsigned DAGSize = DAG -> SUnits . size ( ) ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) {


<s> unsigned DAGSize = DAG -> SUnits . size ( ) ;

for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) {


<s> const SUnit & SU = DAG -> SUnits [ SUNum ] ; if ( DAG -> IsHighLatencySU [ SU . NodeNum ] ) { unsigned CompatibleGroup = true ; int ProposedColor = Color ; std :: vector < int > AdditionalElements ; for ( unsigned j : FormingGroup ) { bool HasSubGraph ; std :: vector < int > SubGraph ; SubGraph = DAG -> GetTopo ( ) -> GetSubGraph ( SU , DAG -> SUnits [ j ] , HasSubGraph ) ; assert ( ! HasSubGraph ) ; SubGraph = DAG -> GetTopo ( ) -> GetSubGraph ( DAG -> SUnits [ j ] , SU , HasSubGraph ) ; if ( ! HasSubGraph ) continue ; else if ( SubGraph . size ( ) > <NUM_LIT> ) { CompatibleGroup = false ; break ; } else { for ( unsigned k : SubGraph ) { if ( DAG -> IsHighLatencySU [ k ] || ( CurrentColoring [ k ] != ProposedColor && CurrentColoring [ k ] != <NUM_LIT> ) ) { CompatibleGroup = false ; break ; } if ( hasDataDependencyPred ( DAG -> SUnits [ k ] , DAG -> SUnits [ j ] ) ) { CompatibleGroup = false ; break ; } } if ( ! CompatibleGroup ) break ; if ( hasDataDependencyPred ( SU , DAG -> SUnits [ j ] ) ) { CompatibleGroup = false ; break ; } llvm :: append_range ( AdditionalElements , SubGraph ) ; } } if ( CompatibleGroup ) {

FormingGroup . insert ( SU . NodeNum ) ;


<s> std :: set < unsigned > SUColors ; if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ; if ( SU -> Preds . size ( ) > <NUM_LIT> && ! DAG -> IsLowLatencySU [ SU -> NodeNum ] ) continue ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ;


<s> for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ; }

if ( SUColors . size ( ) == <NUM_LIT> ) CurrentColoring [ SU -> NodeNum ] = * SUColors . begin ( ) ;


<s> SUnit * SU = & DAG -> SUnits [ SUNum ] ; std :: set < unsigned > SUColors ; if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ; } if ( SUColors . size ( ) == <NUM_LIT> && * SUColors . begin ( ) <= DAGSize ) CurrentColoring [ SU -> NodeNum ] = * SUColors . begin ( ) ; }


<s> for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ;

SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ;


<s> SUnit * SU = & DAG -> SUnits [ SUNum ] ; unsigned color = CurrentColoring [ SU -> NodeNum ] ; std :: set < unsigned > SUColors ; if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ; if ( ColorCount [ color ] > <NUM_LIT> ) continue ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ;


<s> } for ( unsigned SUNum : DAG -> BottomUpIndex2SU ) { SUnit * SU = & DAG -> SUnits [ SUNum ] ; unsigned color = CurrentColoring [ SU -> NodeNum ] ; std :: set < unsigned > SUColors ; if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ; if ( ColorCount [ color ] > <NUM_LIT> ) continue ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ; } if ( SUColors . size ( ) == <NUM_LIT> && * SUColors . begin ( ) != color ) { -- ColorCount [ color ] ;

CurrentColoring [ SU -> NodeNum ] = * SUColors . begin ( ) ;


<s> BlockPtrs . push_back ( std :: make_unique < <ISA_LIT>ScheduleBlock > ( DAG , this , ID ) ) ; CurrentBlocks . push_back ( BlockPtrs . rbegin ( ) -> get ( ) ) ; RealID [ Color ] = ID ; } CurrentBlocks [ RealID [ Color ] ] -> addUnit ( SU ) ; Node2CurrentBlock [ SU -> NodeNum ] = RealID [ Color ] ; } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { SUnit * SU = & DAG -> SUnits [ i ] ; int SUID = Node2CurrentBlock [ i ] ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; if ( Node2CurrentBlock [ Succ -> NodeNum ] != SUID ) CurrentBlocks [ SUID ] -> addSucc ( CurrentBlocks [ Node2CurrentBlock [ Succ -> NodeNum ] ] , SuccDep . isCtrl ( ) ? NoData : Data ) ; } for ( SDep & PredDep : SU -> Preds ) { SUnit * Pred = PredDep . getSUnit ( ) ;

if ( PredDep . isWeak ( ) || Pred -> NodeNum >= DAGSize ) continue ;


<s> void <ISA_LIT>ScheduleBlockCreator :: cutHugeBlocks ( ) {


<s> void <ISA_LIT>ScheduleBlockScheduler :: decreaseLiveRegs ( <ISA_LIT>ScheduleBlock * Block , std :: set < unsigned > & Regs ) { for ( unsigned Reg : Regs ) { std :: set < unsigned > :: iterator Pos = LiveRegs . find ( Reg ) ; assert ( Pos != LiveRegs . end ( ) && LiveRegsConsumers . find ( Reg ) != LiveRegsConsumers . end ( ) && LiveRegsConsumers [ Reg ] >= <NUM_LIT> ) ; -- LiveRegsConsumers [ Reg ] ; if ( LiveRegsConsumers [ Reg ] == <NUM_LIT> ) LiveRegs . erase ( Pos ) ; }


<s> if ( ! SU -> NumPredsLeft ) TopReadySUs . push_back ( SU ) ; } while ( ! TopReadySUs . empty ( ) ) { SUnit * SU = TopReadySUs [ <NUM_LIT> ] ; ScheduledSUnits . push_back ( SU ) ; nodeScheduled ( SU ) ; } Scheduled = true ;


<s> } while ( ! TopReadySUs . empty ( ) ) { SUnit * SU = TopReadySUs [ <NUM_LIT> ] ;


<s> <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; if ( Block -> getPreds ( ) . empty ( ) ) Block -> Depth = <NUM_LIT> ; else { unsigned Depth = <NUM_LIT> ; for ( <ISA_LIT>ScheduleBlock * Pred : Block -> getPreds ( ) ) { if ( Depth < Pred -> Depth + Pred -> getCost ( ) ) Depth = Pred -> Depth + Pred -> getCost ( ) ; } Block -> Depth = Depth ;


<s> } } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { int BlockIndice = BottomUpIndex2Block [ i ] ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; if ( Block -> getSuccs ( ) . empty ( ) ) Block -> Height = <NUM_LIT> ; else { unsigned Height = <NUM_LIT> ; for ( const auto & Succ : Block -> getSuccs ( ) ) Height = std :: max ( Height , Succ . first -> Height + Succ . first -> getCost ( ) ) ; Block -> Height = Height ;


<s> for ( _Iterator RegI = First ; RegI != End ; ++ RegI ) { Register Reg = * RegI ; if ( ! Reg . isVirtual ( ) ) continue ; PSetIterator PSetI = MRI . getPressureSets ( Reg ) ; for ( ; PSetI . isValid ( ) ; ++ PSetI ) { if ( * PSetI == <STR_LIT> :: VGPR_32 ) VgprUsage += PSetI . getWeight ( ) ;

else if ( * PSetI == <STR_LIT> :: SReg_32 ) SgprUsage += PSetI . getWeight ( ) ;


<s> for ( SUnit * SU : SUnits ) { releaseSuccessors ( SU , false ) ; if ( DAG -> IsHighLatencySU [ SU -> NodeNum ] ) HighLatencyBlock = true ; } HasLowLatencyNonWaitedParent . resize ( SUnits . size ( ) , <NUM_LIT> ) ;


<s> Res . TopDownIndex2Block = TopDownIndex2Block ; Res . TopDownBlock2Index = TopDownBlock2Index ; Blocks [ BlockVariant ] = Res ; return Res ; } else { return B -> second ; }


<s> createBlocksForVariant ( BlockVariant ) ; topologicalSort ( ) ; scheduleInsideBlocks ( ) ; fillStats ( ) ; Res . Blocks = CurrentBlocks ; Res . TopDownIndex2Block = TopDownIndex2Block ; Res . TopDownBlock2Index = TopDownBlock2Index ; Blocks [ BlockVariant ] = Res ; return Res ; } else { return B -> second ;


<s> static const char * getReasonStr ( <ISA_LIT>ScheduleCandReason Reason ) { switch ( Reason ) { case NoCand : return <STR_LIT> ; case RegUsage : return <STR_LIT> ; case Latency : return <STR_LIT> ; case Successor : return <STR_LIT> ;


<s> static const char * getReasonStr ( <ISA_LIT>ScheduleCandReason Reason ) { switch ( Reason ) { case NoCand : return <STR_LIT> ; case RegUsage : return <STR_LIT> ; case Latency : return <STR_LIT> ; case Successor : return <STR_LIT> ; case Depth : return <STR_LIT> ; case NodeOrder : return <STR_LIT> ; } llvm_unreachable ( <STR_LIT> ) ;


<s> static bool hasDataDependencyPred ( const SUnit & SU , const SUnit & FromSU ) {

for ( const auto & PredDep : SU . Preds ) {


<s> static bool hasDataDependencyPred ( const SUnit & SU , const SUnit & FromSU ) { for ( const auto & PredDep : SU . Preds ) { if ( PredDep . getSUnit ( ) == & FromSU && PredDep . getKind ( ) == llvm :: SDep :: Data ) return true ; }


<s> } LiveOutRegs . clear ( ) ; for ( const auto & RegMaskPair : RPTracker . getPressure ( ) . LiveOutRegs ) { Register Reg = RegMaskPair . RegUnit ; if ( Reg . isVirtual ( ) && isDefBetween ( Reg , LIS -> getInstructionIndex ( * BeginBlock ) . getRegSlot ( ) , LIS -> getInstructionIndex ( * EndBlock ) . getRegSlot ( ) , MRI , LIS ) ) { LiveOutRegs . insert ( Reg ) ; } } LiveInPressure = TopPressure . MaxSetPressure ; LiveOutPressure = BotPressure . MaxSetPressure ;


<s> if ( MI -> isDebugValue ( ) ) continue ; SlotIndex InstSlot = LIS -> getInstructionIndex ( * MI ) . getRegSlot ( ) ;

if ( InstSlot >= First && InstSlot <= Last ) return true ;


<s> bool <ISA_LIT>ScheduleBlockCreator :: isSUInBlock ( SUnit * SU , unsigned ID ) { if ( SU -> NodeNum >= DAG -> SUnits . size ( ) ) return false ; return CurrentBlocks [ Node2CurrentBlock [ SU -> NodeNum ] ] -> getID ( ) == ID ;


<s> ScheduledSUnits [ BestPos ] = SU -> NodeNum ; ScheduledSUnitsInv [ SU -> NodeNum ] = BestPos ; } LastLowLatencyPos = BestPos ; if ( IsLowLatencyUser ) LastLowLatencyUser = BestPos ; } else if ( IsLowLatencyUser ) { LastLowLatencyUser = i ; } else if ( SU -> getInstr ( ) -> getOpcode ( ) == <STR_LIT> ) { bool CopyForLowLat = false ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * Succ -> getInstr ( ) ) ) { CopyForLowLat = true ; } } if ( ! CopyForLowLat ) continue ; if ( MinPos < i ) { for ( unsigned u = i ; u > MinPos ; -- u ) { ++ ScheduledSUnitsInv [ ScheduledSUnits [ u - <NUM_LIT> ] ] ; ScheduledSUnits [ u ] = ScheduledSUnits [ u - <NUM_LIT> ] ; } ScheduledSUnits [ MinPos ] = SU -> NodeNum ;

ScheduledSUnitsInv [ SU -> NodeNum ] = MinPos ;


<s> } if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * SU -> getInstr ( ) ) ) { unsigned BestPos = LastLowLatencyUser + <NUM_LIT> ; if ( ( int ) BestPos <= LastLowLatencyPos ) BestPos = LastLowLatencyPos + <NUM_LIT> ; if ( BestPos < MinPos ) BestPos = MinPos ; if ( BestPos < i ) { for ( unsigned u = i ; u > BestPos ; -- u ) { ++ ScheduledSUnitsInv [ ScheduledSUnits [ u - <NUM_LIT> ] ] ; ScheduledSUnits [ u ] = ScheduledSUnits [ u - <NUM_LIT> ] ; } ScheduledSUnits [ BestPos ] = SU -> NodeNum ; ScheduledSUnitsInv [ SU -> NodeNum ] = BestPos ; } LastLowLatencyPos = BestPos ; if ( IsLowLatencyUser ) LastLowLatencyUser = BestPos ; } else if ( IsLowLatencyUser ) { LastLowLatencyUser = i ; } else if ( SU -> getInstr ( ) -> getOpcode ( ) == <STR_LIT> ) { bool CopyForLowLat = false ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ;

if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ;


<s> if ( ! I -> isDebugInstr ( ) ) break ; } return I ;


<s> assert ( ! SU -> NumPredsLeft ) ; std :: vector < SUnit * > :: iterator I = llvm :: find ( TopReadySUs , SU ) ; if ( I == TopReadySUs . end ( ) ) { dbgs ( ) << <STR_LIT> ; llvm_unreachable ( nullptr ) ; } TopReadySUs . erase ( I ) ; releaseSuccessors ( SU , true ) ; if ( HasLowLatencyNonWaitedParent [ NodeNum2Index [ SU -> NodeNum ] ] ) HasLowLatencyNonWaitedParent . assign ( SUnits . size ( ) , <NUM_LIT> ) ; if ( DAG -> IsLowLatencySU [ SU -> NodeNum ] ) { for ( SDep & Succ : SU -> Succs ) { std :: map < unsigned , unsigned > :: iterator I = NodeNum2Index . find ( Succ . getSUnit ( ) -> NodeNum ) ; if ( I != NodeNum2Index . end ( ) ) HasLowLatencyNonWaitedParent [ I -> second ] = <NUM_LIT> ; }


<s> Cand . Block = nullptr ; for ( std :: vector < <ISA_LIT>ScheduleBlock * > :: iterator I = ReadyBlocks . begin ( ) , E = ReadyBlocks . end ( ) ; I != E ; ++ I ) { <ISA_LIT>BlockSchedCandidate TryCand ; TryCand . Block = * I ; TryCand . IsHighLatency = TryCand . Block -> isHighLatencyBlock ( ) ; TryCand . VGPRUsageDiff = checkRegUsageImpact ( TryCand . Block -> getInRegs ( ) , TryCand . Block -> getOutRegs ( ) ) [ <STR_LIT> :: VGPR_32 ] ; TryCand . NumSuccessors = TryCand . Block -> getSuccs ( ) . size ( ) ; TryCand . NumHighLatencySuccessors = TryCand . Block -> getNumHighLatencySuccessors ( ) ; TryCand . LastPosHighLatParentScheduled = ( unsigned int ) std :: max < int > ( <NUM_LIT> , LastPosHighLatencyParentScheduled [ TryCand . Block -> getID ( ) ] - LastPosWaitedHighLatency ) ; TryCand . Height = TryCand . Block -> Height ; if ( VregCurrentUsage > <NUM_LIT> || Variant != <STR_LIT> ) { if ( ! tryCandidateRegUsage ( Cand , TryCand ) && Variant != <STR_LIT> ) tryCandidateLatency ( Cand , TryCand ) ; } else { if ( ! tryCandidateLatency ( Cand , TryCand ) ) tryCandidateRegUsage ( Cand , TryCand ) ;


<s> TryCand . LowLatencyOffset = DAG -> LowLatencyOffset [ SU -> NodeNum ] ; TryCand . HasLowLatencyNonWaitedParent = HasLowLatencyNonWaitedParent [ NodeNum2Index [ SU -> NodeNum ] ] ; tryCandidateTopDown ( TopCand , TryCand ) ;

if ( TryCand . Reason != NoCand ) TopCand . setBest ( TryCand ) ;