<s> WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa ) { PrivateSegmentBuffer = true ; if ( UseFixedABI ) { DispatchPtr = true ; QueuePtr = true ; DispatchID = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } } else if ( ST . isMesaGfxShader ( F ) ) { ImplicitBufferPtr = true ; } if ( UseFixedABI || F . hasFnAttribute ( <STR_LIT> ) ) KernargSegmentPtr = true ; if ( ST . hasFlatAddressSpace ( ) && isEntryFunction ( ) && isAmdHsaOrMesa ) { auto hasNonSpillStackObjects = [ & ] ( ) { if ( ! HasStackObjects ) return false ; for ( auto OI = FrameInfo . getObjectIndexBegin ( ) , OE = FrameInfo . getObjectIndexEnd ( ) ; OI != OE ; ++ OI ) if ( ! FrameInfo . isSpillSlotObjectIndex ( OI ) ) return true ; return false ; } ; if ( HasCalls || hasNonSpillStackObjects ( ) ) FlatScratchInit = true ; } Attribute A = F . getFnAttribute ( <STR_LIT> ) ; StringRef S = A . getValueAsString ( ) ; if ( ! S . empty ( ) ) S . consumeInteger ( <NUM_LIT> , GITPtrHigh ) ; A = F . getFnAttribute ( <STR_LIT> ) ; S = A . getValueAsString ( ) ; if ( ! S . empty ( ) ) S . consumeInteger ( <NUM_LIT> , HighBitsOf32BitAddress ) ; S = F . getFnAttribute ( <STR_LIT> ) .


<s> if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa ) { PrivateSegmentBuffer = true ; if ( UseFixedABI ) { DispatchPtr = true ; QueuePtr = true ; DispatchID = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } } else if ( ST . isMesaGfxShader ( F ) ) { ImplicitBufferPtr = true ; } if ( UseFixedABI || F . hasFnAttribute ( <STR_LIT> ) ) KernargSegmentPtr = true ; if ( ST . hasFlatAddressSpace ( ) && isEntryFunction ( ) && isAmdHsaOrMesa ) { auto hasNonSpillStackObjects = [ & ] ( ) { if ( ! HasStackObjects ) return false ; for ( auto OI = FrameInfo . getObjectIndexBegin ( ) , OE = FrameInfo . getObjectIndexEnd ( ) ; OI != OE ; ++ OI ) if ( ! FrameInfo . isSpillSlotObjectIndex ( OI ) ) return true ; return false ; } ; if ( HasCalls || hasNonSpillStackObjects ( ) ) FlatScratchInit = true ; } Attribute A = F . getFnAttribute ( <STR_LIT> ) ; StringRef S = A . getValueAsString ( ) ; if ( ! S . empty ( ) ) S . consumeInteger ( <NUM_LIT> , GITPtrHigh ) ; A = F . getFnAttribute ( <STR_LIT> ) ; S = A . getValueAsString ( ) ; if ( ! S . empty ( ) ) S . consumeInteger ( <NUM_LIT> , HighBitsOf32BitAddress ) ; S = F . getFnAttribute ( <STR_LIT> ) .


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ArgInfo ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . Mode , <ISA_LIT>Mode ( ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HighBitsOf32BitAddress


<s> static void mapping ( IO & YamlIO , <ISA_LIT>MachineFunctionInfo & MFI ) { YamlIO . mapOptional ( <STR_LIT> , MFI . ExplicitKernArgSize , UINT64_C ( <NUM_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MaxKernArgAlign , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . LDSSize , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . IsEntryFunction , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . NoSignedZerosFPMath , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledSGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledVGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg


<s> <ISA_LIT>MachineFunctionInfo :: <ISA_LIT>MachineFunctionInfo ( const MachineFunction & MF ) : <ISA_LIT>MachineFunction ( MF ) , TIDReg ( <STR_LIT> ) , ScratchRSrcReg ( <STR_LIT> ) , ScratchWaveOffsetReg ( <STR_LIT> ) , FrameOffsetReg ( <STR_LIT> ) , StackPtrOffsetReg ( <STR_LIT> ) , PrivateSegmentBufferUserSGPR ( <STR_LIT> ) , DispatchPtrUserSGPR ( <STR_LIT> ) , QueuePtrUserSGPR ( <STR_LIT> ) , KernargSegmentPtrUserSGPR ( <STR_LIT> ) , DispatchIDUserSGPR ( <STR_LIT> ) , FlatScratchInitUserSGPR ( <STR_LIT> ) , PrivateSegmentSizeUserSGPR ( <STR_LIT> ) , GridWorkGroupCountXUserSGPR ( <STR_LIT> ) , GridWorkGroupCountYUserSGPR ( <STR_LIT> ) , GridWorkGroupCountZUserSGPR ( <STR_LIT> ) , WorkGroupIDXSystemSGPR ( <STR_LIT> ) , WorkGroupIDYSystemSGPR ( <STR_LIT> ) , WorkGroupIDZSystemSGPR ( <STR_LIT> ) , WorkGroupInfoSystemSGPR ( <STR_LIT> ) , PrivateSegmentWaveByteOffsetSystemSGPR ( <STR_LIT> ) , WorkItemIDXVGPR ( <STR_LIT> ) , WorkItemIDYVGPR ( <STR_LIT> ) , WorkItemIDZVGPR ( <STR_LIT> ) , P<ISA_LIT>nputAddr ( <NUM_LIT> ) , P<ISA_LIT>nputEnable ( <NUM_LIT> ) , ReturnsVoid ( true ) , FlatWorkGroupSizes ( <NUM_LIT> , <NUM_LIT> ) , WavesPerEU ( <NUM_LIT> , <NUM_LIT> ) , DebuggerWorkGroupIDStackObjectIndices ( { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ) , DebuggerWorkItemIDStackObjectIndices ( { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ) , LDSWaveSpillSize ( <NUM_LIT> ) , NumUserSGPRs ( <NUM_LIT> ) , NumSystemSGPRs ( <NUM_LIT> ) , HasSpilledSGPRs ( false ) , HasSpilledVGPRs ( false ) , HasNonSpillStackObjects ( false ) , NumSpilledSGPRs ( <NUM_LIT> ) , NumSpilledVGPRs ( <NUM_LIT> ) , PrivateSegmentBuffer ( false ) , DispatchPtr ( false ) , QueuePtr ( false ) , KernargSegmentPtr ( false ) , DispatchID ( false ) , FlatScratchInit ( false ) , GridWorkgroupCountX ( false ) , GridWorkgroupCountY ( false ) , GridWorkgroupCountZ ( false ) , WorkGroupIDX ( false ) , WorkGroupIDY ( false ) , WorkGroupIDZ ( false ) , WorkGroupInfo ( false ) , PrivateSegmentWaveByteOffset ( false ) , WorkItemIDX ( false ) , WorkItemIDY ( false ) , WorkItemIDZ ( false ) , ImplicitBufferPtr ( false ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const Function * F = MF . getFunction ( ) ; FlatWorkGroupSizes = ST . getFlatWorkGroupSizes ( * F ) ; WavesPerEU = ST . getWavesPerEU ( * F ) ; if ( ! isEntryFunction ( ) ) { ScratchRSrcReg = <STR_LIT> ; ScratchWaveOffsetReg = <STR_LIT> ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; PrivateSegmentWaveByteOffsetSystemSGPR = ScratchWaveOffsetReg ; } CallingConv :: ID CC = F ->


<s> <ISA_LIT>MachineFunctionInfo :: <ISA_LIT>MachineFunctionInfo ( const MachineFunction & MF ) : <ISA_LIT>MachineFunction ( MF ) , TIDReg ( <STR_LIT> ) , ScratchRSrcReg ( <STR_LIT> ) , ScratchWaveOffsetReg ( <STR_LIT> ) , PrivateSegmentBufferUserSGPR ( <STR_LIT> ) , DispatchPtrUserSGPR ( <STR_LIT> ) , QueuePtrUserSGPR ( <STR_LIT> ) , KernargSegmentPtrUserSGPR ( <STR_LIT> ) , DispatchIDUserSGPR ( <STR_LIT> ) , FlatScratchInitUserSGPR ( <STR_LIT> ) , PrivateSegmentSizeUserSGPR ( <STR_LIT> ) , GridWorkGroupCountXUserSGPR ( <STR_LIT> ) , GridWorkGroupCountYUserSGPR ( <STR_LIT> ) , GridWorkGroupCountZUserSGPR ( <STR_LIT> ) , WorkGroupIDXSystemSGPR ( <STR_LIT> ) , WorkGroupIDYSystemSGPR ( <STR_LIT> ) , WorkGroupIDZSystemSGPR ( <STR_LIT> ) , WorkGroupInfoSystemSGPR ( <STR_LIT> ) , PrivateSegmentWaveByteOffsetSystemSGPR ( <STR_LIT> ) , P<ISA_LIT>nputAddr ( <NUM_LIT> ) , ReturnsVoid ( true ) , LDSWaveSpillSize ( <NUM_LIT> ) , P<ISA_LIT>nputEna ( <NUM_LIT> ) , NumUserSGPRs ( <NUM_LIT> ) , NumSystemSGPRs ( <NUM_LIT> ) , HasSpilledSGPRs ( false ) , HasSpilledVGPRs ( false ) , PrivateSegmentBuffer ( false ) , DispatchPtr ( false ) , QueuePtr ( false ) , DispatchID ( false ) , KernargSegmentPtr ( false ) , FlatScratchInit ( false ) , GridWorkgroupCountX ( false ) , GridWorkgroupCountY ( false ) , GridWorkgroupCountZ ( false ) , WorkGroupIDX ( true ) , WorkGroupIDY ( false ) , WorkGroupIDZ ( false ) , WorkGroupInfo ( false ) , PrivateSegmentWaveByteOffset ( false ) , WorkItemIDX ( true ) , WorkItemIDY ( false ) , WorkItemIDZ ( false ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const Function * F = MF . getFunction ( ) ; P<ISA_LIT>nputAddr = <STR_LIT> ( * F )


<s> const <ISA_LIT>Subtarget & ST = MF -> getSubtarget < <ISA_LIT>Subtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; MachineFrameInfo * FrameInfo = MF -> getFrameInfo ( ) ; MachineRegisterInfo & MRI = MF -> getRegInfo ( ) ; int64_t Offset = FrameInfo -> getObjectOffset ( FrameIndex ) ; Offset += SubIdx * <NUM_LIT> ; unsigned LaneVGPRIdx = Offset / ( <NUM_LIT> * <NUM_LIT> ) ; unsigned Lane = ( Offset /


<s> ScratchRSrcReg = <STR_LIT> ; ScratchWaveOffsetReg = <STR_LIT> ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( ScratchWaveOffsetReg ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } } if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa ) { PrivateSegmentBuffer = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } else if ( ST . isMesaGfxShader ( F )


<s> unsigned getDebuggerReservedVGPRCount ( ) const { return


<s> Offset += SubIdx * <NUM_LIT> ; unsigned LaneVGPRIdx = Offset / ( <NUM_LIT> * <NUM_LIT> ) ; unsigned Lane = ( Offset / <NUM_LIT> ) % <NUM_LIT> ; struct SpilledReg Spill ; Spill . Lane = Lane ; if ( ! LaneVGPRs . count ( LaneVGPRIdx ) ) { unsigned LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , * MF ) ; if ( LaneVGPR == <STR_LIT> ) return Spill ; LaneVGPRs [ LaneVGPRIdx ] = LaneVGPR ; for ( MachineFunction :: iterator BI = MF -> begin (


<s> if ( ! LaneVGPRs . count ( LaneVGPRIdx ) ) { unsigned LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , * MF ) ; if ( LaneVGPR == <STR_LIT> ) return Spill ; LaneVGPRs [ LaneVGPRIdx ] = LaneVGPR ; for ( MachineFunction :: iterator BI = MF -> begin ( ) , BE = MF -> end ( ) ; BI != BE ; ++ BI ) { BI -> addLiveIn ( LaneVGPR ) ; } } Spill . VGPR = LaneVGPRs [


<s> const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const Function * F = MF . getFunction ( ) ; P<ISA_LIT>nputAddr = <STR_LIT> ( * F ) ; const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; if ( ! <STR_LIT> ( F -> getCallingConv ( ) ) ) { KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDZ = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDZ = true ; if ( WorkItemIDZ ) WorkItemIDY = true ; bool MaySpill = ST . isVGPRSpillingEnabled ( * F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( HasStackObjects || MaySpill ) PrivateSegmentWaveByteOffset = true ; if ( ST . isAmdHsaOS ( ) ) { if ( HasStackObjects || MaySpill ) PrivateSegmentBuffer = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) DispatchPtr


<s> if ( CC == CallingConv :: <ISA_LIT>_KERNEL || CC == CallingConv :: SPIR_KERNEL ) { if ( ! F . arg_empty ( ) ) KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } else if ( CC == CallingConv :: <ISA_LIT>_PS ) { P<ISA_LIT>nputAddr = <STR_LIT> ( F ) ; } if ( ! isEntryFunction ( ) ) { ScratchRSrcReg = <STR_LIT> ; ScratchWaveOffsetReg = <STR_LIT> ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( ScratchWaveOffsetReg ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; MaxKernArgAlign = std :: max ( ST . getAlignmentForImplicitArgPtr ( ) , MaxKernArgAlign ) ; } } if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY =


<s> limitOccupancy ( ST . getOccupancyWithLocalMemSize ( getLDSSize ( ) , MF . getFunction ( ) )


<s> limitOccupancy ( getMaxWavesPerEU ( ) ) ; const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; limitOccupancy ( ST . getOccupancyWithLocalMemSize ( getLDSSize ( ) ,


<s> Occupancy = getMaxWavesPerEU ( ) ; limitOccupancy ( MF ) ; if ( ! isEntryFunction ( ) ) { ScratchRSrcReg = <STR_LIT> ; ScratchWaveOffsetReg = <STR_LIT> ; FrameOffsetReg = <STR_LIT> ; StackPtrOffsetReg = <STR_LIT> ; ArgInfo . PrivateSegmentBuffer = ArgDescriptor :: createRegister ( ScratchRSrcReg ) ; ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( ScratchWaveOffsetReg ) ; if ( F . hasFnAttribute ( <STR_LIT> ) ) ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) { KernargSegmentPtr = true ; assert ( MaxKernArgAlign == <NUM_LIT> ) ; MaxKernArgAlign = ST . getAlignmentForImplicitArgPtr ( ) ; } } CallingConv :: ID CC = F . getCallingConv ( ) ; if ( CC == CallingConv :: <ISA_LIT>_KERNEL || CC == CallingConv :: SPIR_KERNEL ) { if ( ! F . arg_empty ( ) ) KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } else if ( CC == CallingConv :: <ISA_LIT>_PS ) { P<ISA_LIT>nputAddr = <STR_LIT> ( F ) ; } if ( ST . debuggerEmitPrologue ( ) ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; bool MaySpill = ST . isVGPRSpillingEnabled ( F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; if ( HasStackObjects || MaySpill ) { PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } } bool IsCOV2 = ST . isAmdCodeObjectV2 ( F ) ; if ( IsCOV2 ) { if ( HasStackObjects || MaySpill ) PrivateSegmentBuffer = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } else if ( ST . isMesaGfxShader ( F ) ) { if ( HasStackObjects || MaySpill )


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . LDSSize , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . DynLDSAlign , Align ( ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . IsEntryFunction , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . NoSignedZerosFPMath , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledSGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledVGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI .

StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ;


<s> <ISA_LIT>MachineFunctionInfo :: <ISA_LIT>MachineFunctionInfo ( const MachineFunction & MF ) : <ISA_LIT>MachineFunction ( MF ) , TIDReg ( <STR_LIT> ) , ScratchRSrcReg ( <STR_LIT> ) , ScratchWaveOffsetReg ( <STR_LIT> ) , PrivateSegmentBufferUserSGPR ( <STR_LIT> ) , DispatchPtrUserSGPR ( <STR_LIT> ) , QueuePtrUserSGPR ( <STR_LIT> ) , KernargSegmentPtrUserSGPR ( <STR_LIT> ) , DispatchIDUserSGPR ( <STR_LIT> ) , FlatScratchInitUserSGPR ( <STR_LIT> ) , PrivateSegmentSizeUserSGPR ( <STR_LIT> ) , GridWorkGroupCountXUserSGPR ( <STR_LIT> ) , GridWorkGroupCountYUserSGPR ( <STR_LIT> ) , GridWorkGroupCountZUserSGPR ( <STR_LIT> ) , WorkGroupIDXSystemSGPR ( <STR_LIT> ) , WorkGroupIDYSystemSGPR ( <STR_LIT> ) , WorkGroupIDZSystemSGPR ( <STR_LIT> ) , WorkGroupInfoSystemSGPR ( <STR_LIT> ) , PrivateSegmentWaveByteOffsetSystemSGPR ( <STR_LIT> ) , P<ISA_LIT>nputAddr ( <NUM_LIT> ) , ReturnsVoid ( true ) , MaximumWorkGroupSize ( <NUM_LIT> ) , DebuggerReservedVGPRCount ( <NUM_LIT> ) , DebuggerWorkGroupIDStackObjectIndices ( { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ) , DebuggerWorkItemIDStackObjectIndices ( { { <NUM_LIT> , <NUM_LIT> , <NUM_LIT> } } ) , LDSWaveSpillSize ( <NUM_LIT> ) , P<ISA_LIT>nputEna ( <NUM_LIT> ) , NumUserSGPRs ( <NUM_LIT> ) , NumSystemSGPRs ( <NUM_LIT> ) , HasSpilledSGPRs ( false ) , HasSpilledVGPRs ( false ) , HasNonSpillStackObjects ( false ) , HasFlatInstructions ( false ) , PrivateSegmentBuffer ( false ) , DispatchPtr ( false ) , QueuePtr ( false ) , DispatchID ( false ) , KernargSegmentPtr ( false ) , FlatScratchInit ( false ) , GridWorkgroupCountX ( false ) , GridWorkgroupCountY ( false ) , GridWorkgroupCountZ ( false ) , WorkGroupIDX ( false ) , WorkGroupIDY ( false ) , WorkGroupIDZ ( false ) , WorkGroupInfo ( false ) , PrivateSegmentWaveByteOffset ( false ) , WorkItemIDX ( false ) , WorkItemIDY ( false ) , WorkItemIDZ ( false ) { const <ISA_LIT>Subtarget & ST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; const Function * F = MF . getFunction ( ) ; P<ISA_LIT>nputAddr = <STR_LIT> ( * F ) ; const MachineFrameInfo * FrameInfo = MF . getFrameInfo ( ) ; if ( ! <STR_LIT> ( F -> getCallingConv ( ) ) ) { KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDZ = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDZ = true ; if ( WorkItemIDZ ) WorkItemIDY = true ; bool MaySpill = ST . isVGPRSpillingEnabled


<s> LaneVGPRs [ LaneVGPRIdx ] = LaneVGPR ; MRI . setPhysRegUsed ( LaneVGPR ) ; for ( MachineFunction :: iterator BI = MF -> begin ( ) , BE = MF -> end ( ) ; BI != BE ; ++ BI ) { BI -> addLiveIn ( LaneVGPR ) ; } } Spill . VGPR = LaneVGPRs [ LaneVGPRIdx ] ; Spill . Lane = Lane ; return


<s> struct SpilledReg Spill ; if ( ! LaneVGPRs . count ( LaneVGPRIdx ) ) { unsigned LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> ) ; LaneVGPRs [ LaneVGPRIdx ] = LaneVGPR ; MRI . setPhysRegUsed ( LaneVGPR ) ; for ( MachineFunction :: iterator BI = MF -> begin ( ) , BE = MF -> end ( ) ; BI != BE ; ++


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . MaxKernArgAlign , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . LDSSize , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . IsEntryFunction , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . NoSignedZerosFPMath , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchWaveOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI .

FrameOffsetReg , StringValue ( <STR_LIT> ) ) ;


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . LDSSize , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . IsEntryFunction , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . NoSignedZerosFPMath , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchWaveOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT>


<s> WorkItemIDX = true ; } if ( ST . debuggerEmitPrologue ( ) ) { WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDY = true ; WorkItemIDZ = true ; } else { if ( F -> hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } if ( WorkItemIDZ ) WorkItemIDY = true ; const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; bool MaySpill = ST . isVGPRSpillingEnabled ( * F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( HasStackObjects || MaySpill ) PrivateSegmentWaveByteOffset = true ; if ( ST . isAmdCodeObjectV2 ( MF ) ) { if ( HasStackObjects || MaySpill ) PrivateSegmentBuffer = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) QueuePtr = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) DispatchID = true ; } else if ( ST . isMesaGfxShader ( MF ) ) { if ( HasStackObjects || MaySpill )


<s> const MachineFrameInfo & FrameInfo = MF . getFrameInfo ( ) ; if ( ! <STR_LIT> ( F -> getCallingConv ( ) ) ) { KernargSegmentPtr = true ; WorkGroupIDX = true ; WorkItemIDX = true ; } if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkGroupIDZ = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDY = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) || ST . debuggerEmitPrologue ( ) ) WorkItemIDZ = true ; if ( WorkItemIDZ ) WorkItemIDY = true ; bool MaySpill = ST . isVGPRSpillingEnabled ( * F ) ; bool HasStackObjects = FrameInfo . hasStackObjects ( ) ; if ( HasStackObjects || MaySpill ) PrivateSegmentWaveByteOffset = true ; if ( ST . isAmdHsaOS ( ) ) { if ( HasStackObjects || MaySpill ) PrivateSegmentBuffer = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) ) DispatchPtr = true ; if ( F -> hasFnAttribute ( <STR_LIT> ) )


<s> Register getPreloadedReg ( <STR_LIT> Value ) const { auto Arg = std :: get < <NUM_LIT> > ( ArgInfo . getPreloadedValue ( Value ) ) ; return Arg ? Arg -> getRegister ( ) : Register


<s> Spill . Lane = Lane ; if ( ! LaneVGPRs . count ( LaneVGPRIdx ) ) { unsigned LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> ) ; if ( LaneVGPR == <STR_LIT> ) return Spill ; LaneVGPRs [ LaneVGPRIdx ] = LaneVGPR ; for ( MachineFunction :: iterator BI = MF -> begin ( ) , BE = MF -> end ( ) ; BI != BE ; ++ BI ) { BI -> addLiveIn ( LaneVGPR ) ; } } Spill . VGPR = LaneVGPRs [ LaneVGPRIdx ] ; return


<s> MaxKernArgAlign = YamlMFI . MaxKernArgAlign ; LDSSize = YamlMFI . LDSSize ; HighBitsOf32BitAddress = YamlMFI . HighBitsOf32BitAddress ; IsEntryFunction = YamlMFI . IsEntryFunction ; NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ; WaveLimiter = YamlMFI


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchWaveOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI .


<s> Register <ISA_LIT>MachineFunctionInfo :: addDispatchID ( const <ISA_LIT>RegisterInfo & TRI ) { ArgInfo . DispatchID = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs += <NUM_LIT> ; return ArgInfo . DispatchID . getRegister


<s> ArgInfo . DispatchPtr = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> ,


<s> Register <ISA_LIT>MachineFunctionInfo :: addFlatScratchInit ( const <ISA_LIT>RegisterInfo & TRI ) { ArgInfo . FlatScratchInit = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs += <NUM_LIT> ; return ArgInfo . FlatScratchInit . getRegister


<s> ArgInfo . KernargSegmentPtr = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs += <NUM_LIT> ; return ArgInfo . KernargSegmentPtr .


<s> ArgInfo . QueuePtr = ArgDescriptor :: createRegister ( TRI . getMatchingSuperReg ( getNextUserSGPR ( ) , <STR_LIT> , & <STR_LIT> ) ) ; NumUserSGPRs += <NUM_LIT> ; return ArgInfo . QueuePtr . getRegister


<s> if ( FuncInfo -> VGPRReservedForSGPRSpill && NumVGPRSpillLanes < WaveSize ) { assert ( FuncInfo -> VGPRReservedForSGPRSpill == SpillVGPRs . back ( ) . VGPR ) ; LaneVGPR = FuncInfo -> VGPRReservedForSGPRSpill ; } else if ( VGPRIndex == <NUM_LIT> ) { LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , MF ) ; if ( LaneVGPR == <STR_LIT> ) { SGPRToVGPRSpills . erase ( FI ) ; NumVGPRSpillLanes -= I ; DiagnosticInfoResourceLimit DiagOutOfRegs ( MF . getFunction ( ) , <STR_LIT> , <NUM_LIT> , DS_Error ) ; MF . getFunction ( ) . getContext ( ) . diagnose ( DiagOutOfRegs ) ; return false ; } Optional < int > SpillFI ; if ( ! isEntryFunction ( ) ) { SpillFI = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , Align (


<s> unsigned NumLanes = Size / <NUM_LIT> ; if ( NumLanes > WaveSize ) return false ; assert ( Size >= <NUM_LIT> && <STR_LIT> ) ; assert ( TRI -> spillSGPRToVGPR ( ) && <STR_LIT> ) ; for ( unsigned I = <NUM_LIT> ; I < NumLanes ; ++ I , ++ NumVGPRSpillLanes ) { Register LaneVGPR ; unsigned VGPRIndex = ( NumVGPRSpillLanes % WaveSize ) ; if ( FuncInfo -> VGPRReservedForSGPRSpill && NumVGPRSpillLanes < WaveSize ) { assert ( FuncInfo -> VGPRReservedForSGPRSpill == SpillVGPRs . back ( ) . VGPR ) ; LaneVGPR = FuncInfo -> VGPRReservedForSGPRSpill ; } else if ( VGPRIndex == <NUM_LIT> ) { LaneVGPR = TRI -> findUnusedRegister ( MRI , & <STR_LIT> , MF ) ; if ( LaneVGPR == <STR_LIT> ) { SGPRToVGPRSpills . erase ( FI ) ; NumVGPRSpillLanes -= I ; DiagnosticInfoResourceLimit DiagOutOfRegs ( MF . getFunction ( ) , <STR_LIT> , <NUM_LIT> , DS_Error ) ; MF . getFunction ( ) . getContext ( ) . diagnose ( DiagOutOfRegs ) ; return false ; } Optional < int > SpillFI ; if ( ! isEntryFunction ( ) ) { SpillFI = FrameInfo . CreateSpillStackObject ( <NUM_LIT> , Align ( <NUM_LIT> ) ) ; } SpillVGPRs . push_back ( SGPRSpillVGPR ( LaneVGPR , SpillFI


<s> const uint32_t * CSRMask = TRI -> getCallPreservedMask ( MF , MF . getFunction ( ) . getCallingConv ( ) ) ; if ( CSRMask ) OtherUsedRegs . setBitsInMask ( CSRMask ) ; for ( MCPhysReg Reg : SpillAGPR ) OtherUsedRegs . set ( Reg ) ; for ( MCPhysReg Reg : SpillVGPR ) OtherUsedRegs . set ( Reg ) ; SmallVectorImpl < MCPhysReg > :: const_iterator NextSpillReg = Regs . begin ( ) ; for ( int I = NumLanes - <NUM_LIT> ; I >= <NUM_LIT> ; -- I ) { NextSpillReg = std :: find_if ( NextSpillReg , Regs . end ( ) , [ & MRI , & OtherUsedRegs ] ( MCPhysReg Reg ) { return MRI . isAllocatable ( Reg ) && ! MRI . isPhysRegUsed ( Reg ) && ! OtherUsedRegs [ Reg ] ; } ) ; if ( NextSpillReg == Regs . end ( ) ) { Spill . FullyAllocated = false ; break ; } OtherUsedRegs . set (


<s> const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; Spill . FullyAllocated = true ; BitVector OtherUsedRegs ; OtherUsedRegs . resize ( TRI -> getNumRegs ( ) ) ; const uint32_t * CSRMask = TRI -> getCallPreservedMask ( MF , MF . getFunction ( ) . getCallingConv ( ) ) ; if ( CSRMask ) OtherUsedRegs . setBitsInMask ( CSRMask ) ; for ( MCPhysReg Reg : SpillAGPR ) OtherUsedRegs . set ( Reg ) ; for ( MCPhysReg Reg : SpillVGPR ) OtherUsedRegs . set ( Reg ) ; SmallVectorImpl < MCPhysReg > :: const_iterator NextSpillReg = Regs . begin ( ) ; for ( int I = NumLanes - <NUM_LIT> ; I >= <NUM_LIT> ; -- I ) { NextSpillReg = std :: find_if ( NextSpillReg , Regs . end ( ) , [ & MRI , & OtherUsedRegs ] ( MCPhysReg Reg ) { return MRI . isAllocatable ( Reg ) && ! MRI . isPhysRegUsed ( Reg ) && ! OtherUsedRegs [ Reg ] ; } )


<s> Any |= convertArg ( AI . QueuePtr , ArgInfo . QueuePtr ) ; Any |= convertArg ( AI . KernargSegmentPtr , ArgInfo . KernargSegmentPtr ) ; Any |= convertArg ( AI . DispatchID , ArgInfo . DispatchID ) ; Any |= convertArg ( AI . FlatScratchInit , ArgInfo . FlatScratchInit ) ; Any |= convertArg ( AI . PrivateSegmentSize , ArgInfo . PrivateSegmentSize ) ; Any |= convertArg ( AI . WorkGroupIDX , ArgInfo . WorkGroupIDX ) ; Any |= convertArg ( AI . WorkGroupIDY , ArgInfo . WorkGroupIDY ) ; Any |= convertArg ( AI . WorkGroupIDZ , ArgInfo . WorkGroupIDZ ) ; Any |= convertArg ( AI . WorkGroupInfo , ArgInfo . WorkGroupInfo ) ; Any |= convertArg ( AI . PrivateSegmentWaveByteOffset , ArgInfo . PrivateSegmentWaveByteOffset ) ; Any |= convertArg ( AI . ImplicitArgPtr , ArgInfo . ImplicitArgPtr ) ; Any |= convertArg ( AI . ImplicitBufferPtr


<s> const <ISA_LIT>BufferPseudoSourceValue * getBufferPSV ( const


<s> if ( ! BufferPSV ) BufferPSV = std :: make_unique < <ISA_LIT>BufferPseudoSourceValue > (


<s> if ( ! ImagePSV ) ImagePSV = std :: make_unique < <ISA_LIT>ImagePseudoSourceValue > ( TII ) ; return ImagePSV . get


<s> MCPhysReg <ISA_LIT>MachineFunctionInfo :: getNextSystemSGPR ( ) const { return <STR_LIT> + NumUserSGPRs +


<s> assert ( NumSystemSGPRs == <NUM_LIT> && <STR_LIT>


<s> MCPhysReg <ISA_LIT>MachineFunctionInfo :: getNextUserSGPR ( ) const { assert ( NumSystemSGPRs == <NUM_LIT> &&


<s> bool hasCalculatedTID ( ) const { return


<s> bool <ISA_LIT>MachineFunctionInfo :: haveFreeLanesForSGPRSpill ( const MachineFunction & MF , unsigned NumNeed ) const { const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; unsigned WaveSize = ST . getWavefrontSize ( ) ; return NumVGPRSpillLanes + NumNeed <= WaveSize * SpillVGPRs .


<s> NoSignedZerosFPMath = YamlMFI . NoSignedZerosFPMath ; MemoryBound = YamlMFI . MemoryBound ; WaveLimiter = YamlMFI . WaveLimiter ; HasSpilledSGPRs = YamlMFI . HasSpilledSGPRs ; HasSpilledVGPRs = YamlMFI . HasSpilledVGPRs ; if ( YamlMFI . ScavengeFI ) { auto FIOrErr = YamlMFI . ScavengeFI -> getFI ( MF . getFrameInfo ( ) ) ; if ( ! FIOrErr ) { const MemoryBuffer & Buffer = * PFS . SM -> getMemoryBuffer ( PFS . SM -> getMainFileID ( ) ) ; Error = SMDiagnostic ( * PFS . SM , SMLoc ( ) , Buffer . getBufferIdentifier ( ) , <NUM_LIT> , <NUM_LIT> , SourceMgr :: DK_Error , toString ( FIOrErr . takeError ( ) ) , <STR_LIT> , None , None ) ; SourceRange = YamlMFI . ScavengeFI -> SourceRange ; return true


<s> YamlIO . mapOptional ( <STR_LIT> , MFI . MemoryBound , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . WaveLimiter , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledSGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HasSpilledVGPRs , false ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ScratchRSrcReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . FrameOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . StackPtrOffsetReg , StringValue ( <STR_LIT> ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . ArgInfo ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . Mode , <ISA_LIT>Mode ( ) ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . HighBitsOf32BitAddress , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI . Occupancy , <NUM_LIT> ) ; YamlIO . mapOptional ( <STR_LIT> , MFI


<s> OS << printReg ( Reg , & TRI ) ; } return


<s> yaml :: StringValue Dest ; { raw_string_ostream OS ( Dest . Value


<s> } } for ( int i = MFI . getObjectIndexBegin ( ) , e = MFI . getObjectIndexEnd ( ) ; i != e ; ++ i ) if ( i != FramePointerSaveIndex && i != BasePointerSaveIndex && ( ! TRI -> isCFISavedRegsSpillEnabled ( ) || ( i != ReturnAddressSaveIndex && i != EXECSaveIndex ) ) ) MFI . setStackID ( i , TargetStackID :: Default ) ; for ( auto & R :


<s> for ( MachineBasicBlock & MBB : MF ) { MBB . removeLiveIn ( ReservedVGPR ) ; MBB . sortUniqueLiveIns ( ) ; } this -> VGPRReservedForSGPRSpill = <STR_LIT> ; return true ; } } return


<s> SpillVGPRs . erase ( i ) ; for ( MachineBasicBlock & MBB : MF ) { MBB . removeLiveIn ( ReservedVGPR ) ; MBB . sortUniqueLiveIns ( ) ; } this -> VGPRReservedForSGPRSpill = <STR_LIT> ; return true ; } } return false


<s> const GCNSubtarget & ST = MF . getSubtarget < GCNSubtarget > ( ) ; const <ISA_LIT>RegisterInfo * TRI = ST . getRegisterInfo ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF . getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; Register LaneVGPR = TRI -> findUnusedRegister ( MF . getRegInfo ( ) , & <STR_LIT> , MF , true ) ; if ( LaneVGPR == Register ( ) ) return false ; SpillVGPRs . push_back ( SGPRSpillVGPR ( LaneVGPR , None ) ) ; FuncInfo -> VGPRReservedForSGPRSpill = LaneVGPR ; return true


<s> void reserveWWMRegister ( Register Reg , Optional <


<s> void setSGPRSpillVGPRs ( Register NewVGPR , Optional < int > newFI , int Index ) { SpillVGPRs [ Index ] . VGPR


<s> SpillVGPRs [ Index ] . VGPR = NewVGPR ; SpillVGPRs [ Index ]


<s> if ( UseFixedABI ) { WorkGroupIDX = true ; WorkGroupIDY = true ; WorkGroupIDZ = true ; WorkItemIDX = true ; WorkItemIDY = true ; WorkItemIDZ = true ; ImplicitArgPtr = true ; } else { if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkGroupIDZ = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDX = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDY = true ; if ( F . hasFnAttribute ( <STR_LIT> ) ) WorkItemIDZ = true ; } bool HasStackObjects = F . hasFnAttribute ( <STR_LIT> ) ; if ( isEntryFunction ( ) ) { if ( WorkItemIDZ ) WorkItemIDY = true ; if ( ! ST . flatScratchIsArchitected ( ) ) { PrivateSegmentWaveByteOffset = true ; if ( ST . getGeneration ( ) >= <ISA_LIT>Subtarget :: GFX9 && ( CC == CallingConv :: <ISA_LIT>_HS || CC == CallingConv :: <ISA_LIT>_GS ) ) ArgInfo . PrivateSegmentWaveByteOffset = ArgDescriptor :: createRegister ( <STR_LIT> ) ; } } bool isAmdHsaOrMesa = ST . isAmdHsaOrMesa ( F ) ; if ( isAmdHsaOrMesa && ! ST . enableFlatScratch ( ) ) PrivateSegmentBuffer = true ; else if ( ST . isMesaGfxShader ( F ) ) ImplicitBufferPtr = true ; if ( ! <STR_LIT> ( CC ) ) { if ( UseFixedABI ) { DispatchPtr = true ; QueuePtr = true ; DispatchID = true ; }


<s> return true ; } const MachineRegisterInfo & MRI = MF . getRegInfo ( ) ; for ( unsigned I = <NUM_LIT> , E = MRI . getNumVirtRegs ( ) ; I != E ; ++ I ) { const Register Reg = Register :: index2VirtReg ( I ) ; const TargetRegisterClass * RC = MRI . getRegClassOrNull ( Reg ) ; if ( RC && <ISA_LIT>RegisterInfo :: isAGPRClass ( RC ) ) { UsesAGPRs = true ; return true ; } } for ( MCRegister Reg : <STR_LIT> ) { if ( MRI . isPhysRegUsed ( Reg ) ) { UsesAGPRs = true ; return true ; } } UsesAGPRs = false


<s> void <ISA_LIT>ScheduleBlockScheduler :: addLiveRegs ( std :: set < unsigned


<s> Preds . push_back ( Pred ) ; assert ( none_of ( Succs , [ = ] ( std :: pair < <ISA_LIT>ScheduleBlock * , <ISA_LIT>ScheduleBlockLinkKind > S ) { return PredID == S . first -> getID ( ) ; } ) &&


<s> if ( PredID == P -> getID ( ) ) return ; } Preds . push_back ( Pred ) ; assert ( none_of ( Succs , [ = ] ( std :: pair < <ISA_LIT>ScheduleBlock * , <ISA_LIT>ScheduleBlockLinkKind > S ) { return PredID == S . first -> getID ( ) ; } ) && <STR_LIT>


<s> NodeNum2Index [ SU -> NodeNum ] = SUnits . size


<s> void <ISA_LIT>ScheduleBlock :: addUnit (


<s> decreaseLiveRegs ( Block , Block -> getInRegs ( ) ) ; addLiveRegs ( Block -> getOutRegs ( ) ) ; releaseBlockSuccs ( Block ) ; for ( const auto & RegP : LiveOutRegsNumUsages [ Block -> getID ( ) ] ) { assert ( LiveRegsConsumers . find ( RegP . first ) == LiveRegsConsumers . end ( ) || LiveRegsConsumers [ RegP . first


<s> if ( SuccDep . isWeak ( ) || SuccSU -> NodeNum >= DAG -> SUnits . size ( ) ) { continue ; } assert ( SuccSU -> isInstr ( ) && <STR_LIT> ) ; if ( ! <ISA_LIT>InstrInfo :: isEXP ( * SuccSU -> getInstr ( ) ) ) { return ; } } ExpGroup . push_back ( SUNum


<s> const SUnit * SuccSU = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || SuccSU -> NodeNum >= DAG -> SUnits . size ( ) ) { continue ; } assert ( SuccSU -> isInstr ( ) && <STR_LIT> ) ; if ( ! <ISA_LIT>InstrInfo :: isEXP ( * SuccSU


<s> std :: set < unsigned > SeenColors ; if ( DAGSize <= <NUM_LIT> ) return ; PreviousColor = CurrentColoring [ <NUM_LIT> ] ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { SUnit * SU = & DAG -> SUnits [ i ] ; unsigned CurrentColor = CurrentColoring [ i ] ; unsigned PreviousColorSave = PreviousColor ; assert ( i == SU -> NodeNum ) ; if ( CurrentColor != PreviousColor ) SeenColors . insert ( PreviousColor ) ; PreviousColor = CurrentColor


<s> void <ISA_LIT>ScheduleBlockCreator :: colorHighLatenciesAlone ( ) { unsigned DAGSize = DAG -> SUnits . size ( ) ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { SUnit * SU = & DAG -> SUnits


<s> unsigned DAGSize = DAG -> SUnits . size ( ) ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i


<s> const SUnit & SU = DAG -> SUnits [ SUNum ] ; if ( DAG -> IsHighLatencySU [ SU . NodeNum ] ) { unsigned CompatibleGroup = true ; int ProposedColor = Color ; std :: vector < int > AdditionalElements ; for ( unsigned j : FormingGroup ) { bool HasSubGraph ; std :: vector < int > SubGraph ; SubGraph = DAG -> GetTopo ( ) -> GetSubGraph ( SU , DAG -> SUnits [ j ] , HasSubGraph ) ; assert ( ! HasSubGraph ) ; SubGraph = DAG -> GetTopo ( ) -> GetSubGraph ( DAG -> SUnits [ j ] , SU , HasSubGraph ) ; if ( ! HasSubGraph ) continue ; else if ( SubGraph . size ( ) > <NUM_LIT> ) { CompatibleGroup = false ; break ; } else { for ( unsigned k : SubGraph ) { if ( DAG -> IsHighLatencySU [ k ] || ( CurrentColoring [ k ] != ProposedColor && CurrentColoring [ k ] != <NUM_LIT> ) ) { CompatibleGroup = false ; break ; } if ( hasDataDependencyPred ( DAG -> SUnits [ k ] , DAG -> SUnits [ j ] ) ) { CompatibleGroup = false ; break ; } } if ( ! CompatibleGroup ) break ; if ( hasDataDependencyPred ( SU , DAG -> SUnits [ j ] ) ) { CompatibleGroup = false ; break ; } llvm :: append_range ( AdditionalElements , SubGraph ) ; } } if ( CompatibleGroup ) { FormingGroup . insert ( SU .


<s> for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ; } if ( SUColors . size ( ) == <NUM_LIT> ) CurrentColoring [ SU -> NodeNum ]


<s> for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum


<s> } for ( unsigned SUNum : DAG -> BottomUpIndex2SU ) { SUnit * SU = & DAG -> SUnits [ SUNum ] ; unsigned color = CurrentColoring [ SU -> NodeNum ] ; std :: set < unsigned > SUColors ; if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ; if ( ColorCount [ color ] > <NUM_LIT> ) continue ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ; } if ( SUColors . size ( ) == <NUM_LIT> && * SUColors . begin ( ) != color ) { -- ColorCount [ color ] ; CurrentColoring [ SU -> NodeNum ] = *


<s> BlockPtrs . push_back ( std :: make_unique < <ISA_LIT>ScheduleBlock > ( DAG , this , ID ) ) ; CurrentBlocks . push_back ( BlockPtrs . rbegin ( ) -> get ( ) ) ; RealID [ Color ] = ID ; } CurrentBlocks [ RealID [ Color ] ] -> addUnit ( SU ) ; Node2CurrentBlock [ SU -> NodeNum ] = RealID [ Color ] ; } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { SUnit * SU = & DAG -> SUnits [ i ] ; int SUID = Node2CurrentBlock [ i ] ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; if ( Node2CurrentBlock [ Succ -> NodeNum ] != SUID ) CurrentBlocks [ SUID ] -> addSucc ( CurrentBlocks [ Node2CurrentBlock [ Succ -> NodeNum ] ] , SuccDep . isCtrl ( ) ? NoData : Data ) ; } for ( SDep & PredDep : SU -> Preds ) { SUnit * Pred = PredDep . getSUnit ( ) ; if ( PredDep . isWeak ( ) ||


<s> } while ( ! TopReadySUs . empty ( ) ) { SUnit * SU = TopReadySUs [ <NUM_LIT> ] ; ScheduledSUnits . push_back (


<s> for ( _Iterator RegI = First ; RegI != End ; ++ RegI ) { Register Reg = * RegI ; if ( ! Reg . isVirtual ( ) ) continue ; PSetIterator PSetI = MRI . getPressureSets ( Reg ) ; for ( ; PSetI . isValid ( ) ; ++ PSetI ) { if ( * PSetI == <STR_LIT> :: VGPR_32 ) VgprUsage += PSetI . getWeight ( ) ; else if ( * PSetI == <STR_LIT> :: SReg_32 ) SgprUsage += PSetI .


<s> static const char * getReasonStr ( <ISA_LIT>ScheduleCandReason Reason ) { switch ( Reason ) { case NoCand : return <STR_LIT> ; case RegUsage : return <STR_LIT> ; case Latency : return <STR_LIT> ; case Successor : return <STR_LIT> ; case Depth :


<s> static bool hasDataDependencyPred ( const SUnit & SU , const SUnit & FromSU ) { for ( const auto & PredDep : SU .


<s> static bool hasDataDependencyPred ( const SUnit & SU , const SUnit & FromSU ) { for ( const auto & PredDep : SU . Preds ) { if ( PredDep . getSUnit ( ) == & FromSU && PredDep . getKind ( ) == llvm :: SDep :: Data ) return true ; } return


<s> } LiveOutRegs . clear ( ) ; for ( const auto & RegMaskPair : RPTracker . getPressure ( ) . LiveOutRegs ) { Register Reg = RegMaskPair . RegUnit ; if ( Reg . isVirtual ( ) && isDefBetween ( Reg , LIS -> getInstructionIndex ( * BeginBlock ) . getRegSlot ( ) , LIS -> getInstructionIndex ( * EndBlock ) . getRegSlot ( ) , MRI , LIS ) ) { LiveOutRegs . insert ( Reg ) ; } } LiveInPressure = TopPressure . MaxSetPressure ; LiveOutPressure = BotPressure . MaxSetPressure ; TopRPTracker . closeTop


<s> if ( MI -> isDebugValue ( ) ) continue ; SlotIndex InstSlot = LIS -> getInstructionIndex ( * MI ) . getRegSlot ( ) ; if ( InstSlot >= First && InstSlot <=


<s> ScheduledSUnits [ BestPos ] = SU -> NodeNum ; ScheduledSUnitsInv [ SU -> NodeNum ] = BestPos ; } LastLowLatencyPos = BestPos ; if ( IsLowLatencyUser ) LastLowLatencyUser = BestPos ; } else if ( IsLowLatencyUser ) { LastLowLatencyUser = i ; } else if ( SU -> getInstr ( ) -> getOpcode ( ) == <STR_LIT> ) { bool CopyForLowLat = false ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * Succ -> getInstr ( ) ) ) { CopyForLowLat = true ; } } if ( ! CopyForLowLat ) continue ; if ( MinPos < i ) { for ( unsigned u = i ; u > MinPos ; -- u ) { ++ ScheduledSUnitsInv [ ScheduledSUnits [ u - <NUM_LIT> ] ] ; ScheduledSUnits [ u ] = ScheduledSUnits [ u - <NUM_LIT> ] ; } ScheduledSUnits [ MinPos ] = SU -> NodeNum ; ScheduledSUnitsInv [ SU -> NodeNum ]


<s> } if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * SU -> getInstr ( ) ) ) { unsigned BestPos = LastLowLatencyUser + <NUM_LIT> ; if ( ( int ) BestPos <= LastLowLatencyPos ) BestPos = LastLowLatencyPos + <NUM_LIT> ; if ( BestPos < MinPos ) BestPos = MinPos ; if ( BestPos < i ) { for ( unsigned u = i ; u > BestPos ; -- u ) { ++ ScheduledSUnitsInv [ ScheduledSUnits [ u - <NUM_LIT> ] ] ; ScheduledSUnits [ u ] = ScheduledSUnits [ u - <NUM_LIT> ] ; } ScheduledSUnits [ BestPos ] = SU -> NodeNum ; ScheduledSUnitsInv [ SU -> NodeNum ] = BestPos ; } LastLowLatencyPos = BestPos ; if ( IsLowLatencyUser ) LastLowLatencyUser = BestPos ; } else if ( IsLowLatencyUser ) { LastLowLatencyUser = i ; } else if ( SU -> getInstr ( ) -> getOpcode ( ) == <STR_LIT> ) { bool CopyForLowLat = false ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ


<s> TryCand . LowLatencyOffset = DAG -> LowLatencyOffset [ SU -> NodeNum ] ; TryCand . HasLowLatencyNonWaitedParent = HasLowLatencyNonWaitedParent [ NodeNum2Index [ SU -> NodeNum ] ] ; tryCandidateTopDown ( TopCand , TryCand ) ; if ( TryCand . Reason != NoCand ) TopCand .


<s> void <ISA_LIT>ScheduleBlockScheduler :: releaseBlockSuccs ( <ISA_LIT>ScheduleBlock * Parent ) { for ( const auto & Block : Parent -> getSuccs ( ) ) { if ( -- BlockNumPredsLeft [ Block . first -> getID ( ) ] == <NUM_LIT>

) ReadyBlocks . push_back ( Block . first ) ;


<s> } if ( SuccSU -> NumPredsLeft == <NUM_LIT> ) { dbgs ( ) << <STR_LIT> ; DAG -> dumpNode ( * SuccSU ) ; dbgs ( )


<s> SUnit * SuccSU = Succ . getSUnit ( ) ; if ( SuccSU -> NodeNum >= DAG -> SUnits . size ( ) ) continue ; if ( BC -> isSUInBlock ( SuccSU , ID ) != InOrOutBlock ) continue ; releaseSucc ( SU , & Succ ) ; if ( SuccSU -> NumPredsLeft == <NUM_LIT> && InOrOutBlock )


<s> void <ISA_LIT>ScheduleDAGMI :: restoreSULinksLeft ( ) { for ( unsigned i = <NUM_LIT> , e = SUnits . size ( ) ; i != e ; ++ i ) { SUnits [ i ] . isScheduled = false ; SUnits [ i ] . WeakPredsLeft = SUnitsLinksBackup [ i ] . WeakPredsLeft ; SUnits [ i ] . NumPredsLeft = SUnitsLinksBackup [ i ] . NumPredsLeft ; SUnits [ i ] . WeakSuccsLeft = SUnitsLinksBackup [ i ] . WeakSuccsLeft ; SUnits [ i ] . NumSuccsLeft = SUnitsLinksBackup [ i ] .


<s> if ( <ISA_LIT>TII -> getMemOperandWithOffset ( * SU -> getInstr ( ) , BaseLatOp , OffLatReg , OffsetIsScalable , TRI ) ) LowLatencyOffset [ i ] = OffLatReg ; } else if ( <ISA_LIT>TII -> isHighLatencyDef ( SU -> getInstr ( ) -> getOpcode ( ) ) ) IsHighLatencySU [ i ] = <NUM_LIT> ; } <ISA_LIT>Scheduler Scheduler ( this ) ; Best = Scheduler . scheduleVariant ( <STR_LIT> , <STR_LIT> ) ; if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsageLatency } , { LatenciesGrouped , BlockLatencyRegUsage } , { LatenciesAlonePlusConsecutive , BlockLatencyRegUsage } , } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , { LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , { LatenciesAlonePlusConsecutive , BlockRegUsage } } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } ScheduledSUnits = Best . SUs ; ScheduledSUnitsInv . resize ( SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { ScheduledSUnitsInv [ ScheduledSUnits [ i ] ] = i ; } moveLowLatencies ( ) ; assert ( TopRPTracker . getPos ( ) == RegionBegin && <STR_LIT> ) ; TopRPTracker . setPos ( CurrentTop ) ; for ( unsigned I : ScheduledSUnits ) { SUnit * SU = & SUnits [ I ] ; scheduleMI ( SU , true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << * SU -> getInstr ( ) ) ; } assert ( CurrentTop == CurrentBottom && <STR_LIT> ) ; placeDebugValues ( ) ; LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> << printMBBReference ( * begin ( )


<s> MachineBasicBlock :: iterator CurrentTopFastSched = DAG -> getCurrentTop ( ) ; std :: vector < MachineBasicBlock :: iterator > PosOld ; std :: vector < MachineBasicBlock :: iterator > PosNew ; PosOld . reserve ( DAG -> SUnits . size ( ) ) ; PosNew . reserve ( DAG -> SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { int BlockIndice = TopDownIndex2Block [ i ] ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; std :: vector < SUnit * > SUs = Block -> getScheduledUnits ( ) ; for ( SUnit * SU : SUs ) { MachineInstr * MI = SU -> getInstr ( ) ; MachineBasicBlock :: iterator Pos = MI ; PosOld . push_back ( Pos ) ; if ( & * CurrentTopFastSched == MI ) { PosNew . push_back ( Pos ) ; CurrentTopFastSched = nextIfDebug ( ++ CurrentTopFastSched , DAG -> getCurrentBottom ( ) ) ; } else { DAG -> getBB ( ) -> splice ( CurrentTopFastSched , DAG -> getBB ( ) , MI ) ; DAG -> getLIS ( ) -> handleMove ( * MI , true ) ; PosNew . push_back (


<s> struct <ISA_LIT>ScheduleBlockResult <STR_LIT> ( <ISA_LIT>SchedulerBlockCreatorVariant BlockVariant , <ISA_LIT>SchedulerBlockSchedulerVariant ScheduleVariant ) { <ISA_LIT>ScheduleBlocks Blocks = BlockCreator . getBlocks ( BlockVariant ) ; <ISA_LIT>ScheduleBlockScheduler Scheduler ( DAG , ScheduleVariant , Blocks ) ; std :: vector < <ISA_LIT>ScheduleBlock * > ScheduledBlocks ; struct <ISA_LIT>ScheduleBlockResult


<s> int topoInd = - <NUM_LIT> ; for ( <ISA_LIT>ScheduleBlock * Pred : Block -> getPreds ( ) ) { std :: set < unsigned > PredOutRegs = Pred -> getOutRegs ( ) ; std :: set < unsigned > :: iterator RegPos = PredOutRegs . find ( Reg ) ; if ( RegPos != PredOutRegs . end ( ) ) { Found = true ; if ( topoInd < BlocksStruct . TopDownBlock2Index [ Pred -> getID ( ) ] ) { topoInd = BlocksStruct . TopDownBlock2Index [ Pred -> getID ( ) ] ; } } } if ( ! Found ) continue ; int PredID = BlocksStruct . TopDownIndex2Block [ topoInd ] ; ++ LiveOutRegsNumUsages [ PredID ] [ Reg ] ; } } LastPosHighLatencyParentScheduled . resize ( Blocks . size ( ) , <NUM_LIT> ) ; BlockNumPredsLeft . resize ( Blocks . size ( ) ) ; BlockNumSuccsLeft . resize ( Blocks . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = Blocks [ i ] ; BlockNumPredsLeft [ i ] = Block -> getPreds ( ) . size ( ) ; BlockNumSuccsLeft [ i ] = Block -> getSuccs ( ) . size ( ) ; } for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = Blocks [ i ] ; assert ( Block -> getID ( ) == i ) ; } std :: set < unsigned > InRegs = DAG -> getInRegs ( ) ; addLiveRegs ( InRegs ) ; for ( unsigned Reg : DAG -> getOutRegs ( ) ) { for ( unsigned i = <NUM_LIT> , e = Blocks . size ( ) ; i != e ; ++ i ) { int ID = BlocksStruct . TopDownIndex2Block [ Blocks . size ( ) - <NUM_LIT> - i ] ; <ISA_LIT>ScheduleBlock * Block = Blocks [ ID ] ; const std :: set < unsigned > & OutRegs = Block -> getOutRegs ( ) ; if ( OutRegs . find ( Reg ) == OutRegs . end ( ) ) continue ; ++ LiveOutRegsNumUsages [ ID ] [ Reg ] ; break ; } } for ( <ISA_LIT>ScheduleBlock * Block : Blocks ) { for ( unsigned Reg : Block -> getInRegs ( ) ) { bool Found = false ; for ( <ISA_LIT>ScheduleBlock * Pred : Block -> getPreds ( ) ) { std :: set < unsigned > PredOutRegs = Pred -> getOutRegs ( ) ; std :: set < unsigned > :: iterator RegPos = PredOutRegs . find ( Reg ) ; if ( RegPos != PredOutRegs . end ( )


<s> <ISA_LIT>TII = static_cast < const <ISA_LIT>InstrInfo * > ( TII ) ; <ISA_LIT>TRI = static_cast < const <ISA_LIT>RegisterInfo * >


<s> BottomUpIndex2SU = std :: vector < int > ( Topo . rbegin ( ) , Topo . rend (


<s> dbgs ( ) << <STR_LIT> << Cand . SU -> NodeNum << <STR_LIT>


<s> if ( TryCand . IsHighLatency && <STR_LIT> ( TryCand . Height , Cand . Height , TryCand , Cand , Depth ) ) return true ; if ( <STR_LIT> ( TryCand . NumHighLatencySuccessors , Cand . NumHighLatencySuccessors , TryCand , Cand , Successor ) ) return true ; return false


<s> if ( <STR_LIT> ( TryCand . VGPRUsageDiff > <NUM_LIT> , Cand . VGPRUsageDiff > <NUM_LIT> , TryCand , Cand , RegUsage ) ) return true ; if ( <STR_LIT> ( TryCand . NumSuccessors > <NUM_LIT> , Cand . NumSuccessors > <NUM_LIT> , TryCand , Cand , Successor ) ) return true ; if ( <STR_LIT> ( TryCand . Height , Cand . Height , TryCand , Cand , Depth ) ) return true ; if ( <STR_LIT> ( TryCand . VGPRUsageDiff , Cand . VGPRUsageDiff , TryCand , Cand , RegUsage ) ) return true ; return false


<s> if ( <STR_LIT> ( TryCand . NumSuccessors > <NUM_LIT> , Cand . NumSuccessors > <NUM_LIT> , TryCand , Cand , Successor ) ) return true ; if ( <STR_LIT> ( TryCand . Height , Cand . Height , TryCand , Cand , Depth ) ) return true ; if ( <STR_LIT> ( TryCand . VGPRUsageDiff , Cand . VGPRUsageDiff , TryCand , Cand , RegUsage ) ) return true ; return


<s> TryCand . Reason = NodeOrder ; return ; } if ( Cand . SGPRUsage > <NUM_LIT> && <STR_LIT> ( TryCand . SGPRUsage , Cand . SGPRUsage , TryCand , Cand , RegUsage ) ) return ; if ( <STR_LIT> ( TryCand . HasLowLatencyNonWaitedParent , Cand . HasLowLatencyNonWaitedParent , TryCand , Cand , <STR_LIT> ) ) return ; if ( <STR_LIT> ( TryCand . IsLowLatency , Cand . IsLowLatency , TryCand


<s> TryCand . Reason = NodeOrder ; return ; } if ( Cand . SGPRUsage > <NUM_LIT> && <STR_LIT> ( TryCand . SGPRUsage , Cand . SGPRUsage , TryCand , Cand , RegUsage ) ) return ; if ( <STR_LIT> ( TryCand . HasLowLatencyNonWaitedParent , Cand . HasLowLatencyNonWaitedParent , TryCand , Cand , <STR_LIT> ) ) return ; if ( <STR_LIT> ( TryCand . IsLowLatency , Cand . IsLowLatency , TryCand , Cand , <STR_LIT> ) ) return ; if ( TryCand . IsLowLatency && <STR_LIT> ( TryCand . LowLatencyOffset , Cand . LowLatencyOffset , TryCand , Cand , <STR_LIT> ) ) return ; if ( <STR_LIT> ( TryCand . VGPRUsage , Cand . VGPRUsage , TryCand , Cand , RegUsage ) ) return ; if ( TryCand . SU -> NodeNum < Cand . SU -> NodeNum ) { TryCand . Reason


<s> if ( TryVal < CandVal ) { TryCand . Reason = Reason ; return true ; } if ( TryVal > CandVal )


<s> for ( SUnit * SU : SUnits ) { SU -> isScheduled = false ; for ( SDep & Succ : SU ->


<s> SUnit & getEntrySU ( ) { return EntrySU


<s> void <ISA_LIT>ScheduleBlock :: addPred ( <ISA_LIT>ScheduleBlock * Pred ) { unsigned PredID = Pred -> getID ( ) ; for ( <ISA_LIT>ScheduleBlock * P : Preds ) { if ( PredID == P -> getID ( ) ) return ; } Preds . push_back (


<s> void <ISA_LIT>ScheduleBlockScheduler :: blockScheduled ( <ISA_LIT>ScheduleBlock * Block ) { decreaseLiveRegs ( Block , Block -> getInRegs ( ) ) ; addLiveRegs ( Block -> getOutRegs ( ) ) ; releaseBlockSuccs ( Block ) ; for ( std :: map < unsigned , unsigned > :: iterator RegI = LiveOutRegsNumUsages [ Block -> getID ( ) ] . begin ( ) , E = LiveOutRegsNumUsages [ Block -> getID ( ) ] . end ( ) ; RegI != E ; ++ RegI ) { std :: pair < unsigned , unsigned > RegP = * RegI ; if ( LiveRegsConsumers . find ( RegP . first ) == LiveRegsConsumers . end ( ) ) LiveRegsConsumers [ RegP . first ] = RegP . second ; else { assert ( LiveRegsConsumers [ RegP . first ] == <NUM_LIT> ) ; LiveRegsConsumers [ RegP . first ] += RegP . second ; } } if ( LastPosHighLatencyParentScheduled [ Block -> getID ( ) ] > ( unsigned ) LastPosWaitedHighLatency

) LastPosWaitedHighLatency = LastPosHighLatencyParentScheduled [ Block -> getID ( ) ] ;


<s> if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ; if ( ColorCount [ color ] > <NUM_LIT> ) continue ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ; } if ( SUColors . size ( ) == <NUM_LIT> && * SUColors . begin ( ) != color ) { -- ColorCount [ color


<s>
if
(
ReadyBlocks
.
empty
(
)
)
return
nullptr
;
DAG
->
fillVgprSgprCost
(
LiveRegs
.
begin
(
)
,
LiveRegs
.
end
(
)
,
VregCurrentUsage
,
SregCurrentUsage
)
;
if
(
VregCurrentUsage
>
maxVregUsage
)
maxVregUsage
=
VregCurrentUsage
;
if
(
VregCurrentUsage
>
maxSregUsage
)
maxSregUsage
=
VregCurrentUsage
;
DEBUG
(
dbgs
(
)
<<
<STR_LIT>
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
<ISA_LIT>ScheduleBlock
*
Block
:
ReadyBlocks
)
dbgs
(
)
<<
Block
->
getID
(
)
<<
' '
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveRegs
)
dbgs
(
)
<<
PrintVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
dbgs
(
)
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
VregCurrentUsage
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
SregCurrentUsage
<<
'\n'
;
)
;
Cand
.
Block
=
nullptr
;
for
(
std
::
vector
<
<ISA_LIT>ScheduleBlock
*
>
::
iterator
I
=
ReadyBlocks
.
begin
(
)
,
E
=
ReadyBlocks
.
end
(
)
;
I
!=
E
;
++
I
)
{
<ISA_LIT>BlockSchedCandidate
TryCand
;
TryCand
.
Block
=
*
I
;
TryCand
.
IsHighLatency
=
TryCand
.
Block
->
isHighLatencyBlock
(
)
;
TryCand
.
VGPRUsageDiff
=
checkRegUsageImpact
(
TryCand
.
Block
->
getInRegs
(
)
,
TryCand
.
Block
->
getOutRegs
(
)
)
[
DAG
->
getVGPRSetID
(
)
]
;
TryCand
.
NumSuccessors
=
TryCand
.
Block
->
getSuccs
(
)
.
size
(
)
;
TryCand
.
NumHighLatencySuccessors
=
TryCand
.
Block
->
getNumHighLatencySuccessors
(
)
;
TryCand
.
LastPosHighLatParentScheduled
=
(
unsigned
int
)
std
::
max
<
int
>
(
<NUM_LIT>
,
LastPosHighLatencyParentScheduled
[
TryCand
.
Block
->
getID
(
)
]
LastPosWaitedHighLatency
)
;
TryCand
.
Height
=
TryCand
.


<s>
for
(
unsigned
Reg
:
LiveInRegs
)
dbgs
(
)
<<
PrintVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveOutRegs
)
dbgs
(
)
<<
PrintVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
}
dbgs
(
)
<<
<STR_LIT>
;
if
(
!
Scheduled
)
{
for
(
SUnit
*
SU
:
SUnits
)
{
SU
->
dump
(
DAG
)
;
}
}
else
{
for
(
SUnit
*
SU
:


<s> std :: vector < unsigned > & getInternalAdditionalRegUsage ( ) { return


<s>
LLVM_DEBUG
(
dbgs
(
)
<<
<STR_LIT>
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
<ISA_LIT>ScheduleBlock
*
Block
:
ReadyBlocks
)
dbgs
(
)
<<
Block
->
getID
(
)
<<
' '
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
dbgs
(
)
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
VregCurrentUsage
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
SregCurrentUsage
<<
'\n'
;
)
;
Cand
.
Block
=
nullptr
;
for
(
std
::
vector
<
<ISA_LIT>ScheduleBlock
*
>
::
iterator
I
=
ReadyBlocks
.
begin
(
)
,
E
=
ReadyBlocks
.
end
(
)
;
I
!=
E
;
++
I
)
{
<ISA_LIT>BlockSchedCandidate
TryCand
;
TryCand
.
Block
=
*
I
;
TryCand
.
IsHighLatency
=
TryCand
.
Block
->
isHighLatencyBlock
(
)
;
TryCand
.
VGPRUsageDiff
=
checkRegUsageImpact
(
TryCand
.
Block
->
getInRegs
(
)
,
TryCand
.
Block
->
getOutRegs
(
)
)
[
DAG
->
getVGPRSetID
(
)
]
;
TryCand
.
NumSuccessors
=
TryCand
.
Block
->
getSuccs
(
)
.
size
(
)
;
TryCand
.
NumHighLatencySuccessors
=
TryCand
.
Block
->
getNumHighLatencySuccessors
(
)
;
TryCand
.
LastPosHighLatParentScheduled
=
(
unsigned
int
)
std
::
max
<
int
>
(
<NUM_LIT>
,
LastPosHighLatencyParentScheduled
[
TryCand
.
Block
->
getID
(
)
]
LastPosWaitedHighLatency
)
;
TryCand
.
Height
=
TryCand
.
Block
->
Height
;
if
(
VregCurrentUsage
>
<NUM_LIT>
||
Variant
!=
<STR_LIT>
)
{
if
(
!
tryCandidateRegUsage
(
Cand
,
TryCand
)
&&
Variant
!=
<STR_LIT>
)
tryCandidateLatency
(
Cand
,
TryCand
)
;
}
else
{
if
(
!
tryCandidateLatency
(
Cand
,
TryCand
)
)
tryCandidateRegUsage
(
Cand
,
TryCand
)
;
}
if
(
TryCand
.
Reason
!=
NoCand


<s>
dbgs
(
)
<<
<STR_LIT>
<<
LiveOutPressure
[
DAG
->
getSGPRSetID
(
)
]
<<
' '
<<
LiveOutPressure
[
DAG
->
getVGPRSetID
(
)
]
<<
<STR_LIT>
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveInRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveOutRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
}
dbgs
(
)
<<
<STR_LIT>


<s> SUnit * SU = & DAG -> SUnits [ DAG -> BottomUpIndex2SU [ i ] ] ; std :: set < unsigned > SUColors ; if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; SUColors . insert ( CurrentColoring [ Succ -> NodeNum ] ) ; } if ( SUColors . size ( ) == <NUM_LIT> && * SUColors . begin ( ) <= DAGSize

) CurrentColoring [ SU -> NodeNum ] = * SUColors . begin ( ) ;


<s> return new <ISA_LIT>ScheduleDAGMI ( C


<s> ScheduleDAGInstrs * llvm :: create<ISA_LIT>MachineScheduler ( MachineSchedContext * C ) { return new <ISA_LIT>ScheduleDAGMI


<s> } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { int BlockIndice = BottomUpIndex2Block [ i ] ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; if ( Block -> getSuccs ( ) . size ( ) == <NUM_LIT> ) Block -> Height = <NUM_LIT> ; else { unsigned Height = <NUM_LIT> ; for ( <ISA_LIT>ScheduleBlock * Succ : Block -> getSuccs ( ) ) { if ( Height < Succ -> Height + <NUM_LIT> ) Height = Succ ->


<s> if ( MI -> isDebugValue ( ) ) continue ; SlotIndex InstSlot = LIS -> getInstructionIndex ( MI ) . getRegSlot ( ) ; if ( InstSlot >= First && InstSlot <= Last ) return true ; } return


<s> return MachineBasicBlock :: instr_iterator ( const_cast < MachineInstr * > ( & * nextIfDebug ( MachineBasicBlock :: const_iterator ( I )


<s> static MachineBasicBlock :: iterator nextIfDebug ( MachineBasicBlock :: iterator I , MachineBasicBlock :: const_iterator End ) { return MachineBasicBlock :: instr_iterator ( const_cast < MachineInstr * > ( & * nextIfDebug (

MachineBasicBlock :: const_iterator ( I ) , End ) ) ) ;


<s> if ( HasLowLatencyNonWaitedParent [ NodeNum2Index [ SU -> NodeNum ] ] ) HasLowLatencyNonWaitedParent . assign ( SUnits . size ( ) , <NUM_LIT> ) ; if ( DAG -> IsLowLatencySU [ SU -> NodeNum ] ) { for ( SDep & Succ : SU -> Succs ) { std :: map < unsigned , unsigned > :: iterator I = NodeNum2Index . find ( Succ . getSUnit (


<s> llvm_unreachable ( nullptr ) ; } TopReadySUs . erase ( I ) ; releaseSuccessors ( SU , true ) ; if ( HasLowLatencyNonWaitedParent [ NodeNum2Index [ SU -> NodeNum ] ] ) HasLowLatencyNonWaitedParent . assign ( SUnits . size ( ) , <NUM_LIT> ) ; if ( DAG -> IsLowLatencySU [ SU -> NodeNum ] ) { for ( SDep & Succ : SU -> Succs ) { std :: map < unsigned , unsigned > :: iterator I = NodeNum2Index .

find ( Succ . getSUnit ( ) -> NodeNum ) ;


<s> void <ISA_LIT>ScheduleBlockCreator :: regroupNoUserInstructions ( ) { unsigned DAGSize = DAG -> SUnits . size ( ) ; int GroupID = NextNonReservedID ++ ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { SUnit * SU = & DAG -> SUnits [ DAG -> BottomUpIndex2SU [ i ] ] ; bool hasSuccessor = false ; if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ; for ( SDep & SuccDep :


<s> if ( CurrentColoring [ SU -> NodeNum ] <= ( int ) DAGSize ) continue ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( SuccDep . isWeak ( ) || Succ -> NodeNum >= DAGSize ) continue ; hasSuccessor = true


<s> SUnit * SuccSU = Succ . getSUnit ( ) ; if ( BC -> isSUInBlock ( SuccSU ,


<s> { LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , { LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , { LatenciesAlonePlusConsecutive , BlockRegUsage } } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } ScheduledSUnits = Best . SUs ; ScheduledSUnitsInv . resize ( SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { ScheduledSUnitsInv [ ScheduledSUnits [ i ] ] = i ; } moveLowLatencies ( ) ; assert ( TopRPTracker . getPos ( ) == RegionBegin && <STR_LIT> ) ; TopRPTracker . setPos ( CurrentTop ) ; for ( std :: vector < unsigned > :: iterator I = ScheduledSUnits . begin ( ) , E = ScheduledSUnits . end ( ) ; I != E ; ++ I ) { SUnit * SU = & SUnits [ * I ] ; scheduleMI ( SU , true ) ; DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << * SU -> getInstr ( ) ) ; } assert ( CurrentTop == CurrentBottom && <STR_LIT> ) ; placeDebugValues ( ) ; DEBUG ( { unsigned BBNum = begin ( ) -> getParent ( ) -> getNumber ( ) ; dbgs ( ) << <STR_LIT> << BBNum << <STR_LIT> ; dumpSchedule ( ) ; dbgs ( ) <<


<s> if ( & * CurrentTopFastSched == MI ) { PosNew . push_back ( Pos ) ; CurrentTopFastSched = nextIfDebug ( ++ CurrentTopFastSched , DAG -> getCurrentBottom ( ) ) ; } else { DAG -> getBB ( ) -> splice ( CurrentTopFastSched , DAG -> getBB ( ) , MI ) ; DAG -> getLIS ( ) -> handleMove ( MI , true ) ; PosNew . push_back ( CurrentTopFastSched ) ; } } } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ i ] ; std :: vector < SUnit * > SUs = Block -> getScheduledUnits ( ) ; Block -> schedule ( ( * SUs . begin ( ) ) -> getInstr ( ) , ( * SUs . rbegin ( ) ) -> getInstr ( ) ) ; } DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( unsigned i = PosOld . size ( ) , e = <NUM_LIT> ; i != e ; -- i ) { MachineBasicBlock :: iterator POld = PosOld [ i - <NUM_LIT> ] ; MachineBasicBlock :: iterator PNew = PosNew [ i - <NUM_LIT> ] ; if ( PNew != POld ) { DAG -> getBB ( ) -> splice ( POld , DAG -> getBB ( ) , PNew ) ; DAG -> getLIS ( ) -> handleMove ( POld , true ) ; } } DEBUG ( for ( unsigned i = <NUM_LIT> , e = CurrentBlocks . size ( ) ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ i ] ; Block -> printDebug (


<s> <ISA_LIT>TRI = static_cast < const <ISA_LIT>RegisterInfo * > ( TRI ) ; VGPRSetID = <ISA_LIT>TRI -> getVGPR32PressureSet


<s> if ( SU -> NodeNum < DAGSize && ! -- TopDownSU2Index [ SU -> NodeNum ] ) WorkList . push_back ( SU ) ; } } BottomUpIndex2SU = std :: vector < int > ( TopDownIndex2SU . rbegin ( ) , TopDownIndex2SU . rend ( ) ) ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { SUnit * SU = & SUnits [ i ] ; for ( SDep & Pred : SU -> Preds ) { if ( Pred . getSUnit ( ) -> NodeNum >= DAGSize ) continue ; assert ( TopDownSU2Index [ SU -> NodeNum ] > TopDownSU2Index [ Pred . getSUnit ( ) -> NodeNum ] && <STR_LIT> ) ; } } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { SUnit * SU = & SUnits [ i ] ; for ( SDep & Succ : SU -> Succs ) { if ( Succ . getSUnit ( ) -> NodeNum >= DAGSize ) continue ; assert ( TopDownSU2Index [ SU -> NodeNum ] < TopDownSU2Index [ Succ . getSUnit ( ) -> NodeNum


<s> LLVM_DEBUG ( for ( unsigned i = <NUM_LIT> , e = CurrentBlocks . size


<s> std :: vector < <ISA_LIT>ScheduleBlock * > getBlocks ( ) {


<s> std :: vector < <ISA_LIT>ScheduleBlock * > getBlocks ( )


<s> MachineBasicBlock :: iterator getCurrentBottom ( ) { return


<s> MachineBasicBlock :: iterator getCurrentTop ( ) { return CurrentTop


<s> MachineBasicBlock :: iterator getCurrentTop ( ) { return


<s> unsigned getSGPRSetID ( ) const { return SGPRSetID


<s> unsigned getSGPRUsage ( ) { return maxSregUsage


<s> unsigned getVGPRSetID ( ) const { return VGPRSetID


<s> bool OffsetIsScalable ; if ( <ISA_LIT>TII -> getMemOperandWithOffset ( * SU -> getInstr ( ) , BaseLatOp , OffLatReg , OffsetIsScalable , TRI ) ) LowLatencyOffset [ i ] = OffLatReg ; } else if ( <ISA_LIT>TII -> isHighLatencyDef ( SU -> getInstr ( ) -> getOpcode ( ) ) ) IsHighLatencySU [ i ] = <NUM_LIT> ; } <ISA_LIT>Scheduler Scheduler ( this ) ; Best = Scheduler . scheduleVariant ( <STR_LIT> , <STR_LIT> ) ; if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsageLatency } , { LatenciesGrouped , BlockLatencyRegUsage } , { LatenciesAlonePlusConsecutive , BlockLatencyRegUsage } , } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , { LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , { LatenciesAlonePlusConsecutive , BlockRegUsage } } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } ScheduledSUnits = Best . SUs ; ScheduledSUnitsInv . resize ( SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { ScheduledSUnitsInv [ ScheduledSUnits [ i ] ] = i ; } moveLowLatencies ( ) ; assert ( TopRPTracker . getPos ( ) == RegionBegin && <STR_LIT> ) ; TopRPTracker . setPos ( CurrentTop ) ; for ( std :: vector < unsigned > :: iterator I = ScheduledSUnits . begin ( ) , E = ScheduledSUnits . end ( ) ; I != E ; ++ I ) { SUnit * SU = & SUnits [ * I ] ; scheduleMI ( SU , true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << * SU -> getInstr ( ) ) ; } assert ( CurrentTop == CurrentBottom && <STR_LIT>


<s> { LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , { LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , { LatenciesAlonePlusConsecutive , BlockRegUsage } } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } ScheduledSUnits = Best . SUs ; ScheduledSUnitsInv . resize ( SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { ScheduledSUnitsInv [ ScheduledSUnits [ i ] ] = i ; } moveLowLatencies ( ) ; assert ( TopRPTracker . getPos ( ) == RegionBegin && <STR_LIT> ) ; TopRPTracker . setPos ( CurrentTop ) ; for ( std :: vector < unsigned > :: iterator I = ScheduledSUnits . begin ( ) , E = ScheduledSUnits . end ( ) ; I != E ; ++ I ) { SUnit * SU = & SUnits [ * I ] ; scheduleMI ( SU , true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum <<


<s> for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ i ] ; Block -> fastSchedule ( ) ; } MachineBasicBlock :: iterator CurrentTopFastSched = DAG -> getCurrentTop ( ) ; std :: vector < MachineBasicBlock :: iterator > PosOld ; std :: vector < MachineBasicBlock :: iterator > PosNew ; PosOld . reserve ( DAG -> SUnits . size ( ) ) ; PosNew . reserve ( DAG -> SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { int BlockIndice = TopDownIndex2Block [ i ] ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; std :: vector < SUnit * > SUs = Block -> getScheduledUnits ( ) ; for ( SUnit * SU : SUs ) { MachineInstr * MI = SU -> getInstr ( ) ; MachineBasicBlock :: iterator Pos = MI ; PosOld . push_back ( Pos ) ; if ( & * CurrentTopFastSched == MI ) { PosNew . push_back ( Pos ) ; CurrentTopFastSched = nextIfDebug ( ++ CurrentTopFastSched , DAG -> getCurrentBottom ( ) ) ; } else { DAG -> getBB ( ) -> splice ( CurrentTopFastSched , DAG -> getBB ( ) , MI ) ; DAG -> getLIS ( ) -> handleMove ( * MI , true ) ; PosNew . push_back ( CurrentTopFastSched ) ; } } } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ i ] ; std :: vector < SUnit * > SUs = Block -> getScheduledUnits ( ) ; Block -> schedule ( ( * SUs . begin ( ) ) -> getInstr ( ) , ( * SUs . rbegin ( ) ) -> getInstr ( ) ) ; } LLVM_DEBUG ( dbgs ( ) << <STR_LIT> ) ; for ( unsigned i = PosOld . size ( ) , e = <NUM_LIT> ; i != e ; -- i ) { MachineBasicBlock :: iterator POld = PosOld [


<s> <ISA_LIT>ScheduleBlock ( <ISA_LIT>ScheduleDAGMI * DAG , <ISA_LIT>ScheduleBlockCreator * BC , unsigned ID ) : DAG ( DAG ) , BC ( BC ) , SUnits ( ) , TopReadySUs ( ) , ScheduledSUnits ( ) , TopRPTracker ( TopPressure ) , Scheduled ( false ) , HighLatencyBlock ( false ) , ID ( ID ) , Preds ( )

, Succs ( ) , NumHighLatencySuccessors ( <NUM_LIT> ) { }


<s> <ISA_LIT>ScheduleBlock ( <ISA_LIT>ScheduleDAGMI * DAG , <ISA_LIT>ScheduleBlockCreator * BC , unsigned ID ) : DAG ( DAG ) , BC ( BC ) , SUnits ( ) , TopReadySUs ( ) , ScheduledSUnits ( ) , TopRPTracker ( TopPressure ) , Scheduled ( false ) , HighLatencyBlock (

false ) , ID ( ID ) , Preds ( ) , Succs ( ) , NumHighLatencySuccessors ( <NUM_LIT> ) { }


<s> <ISA_LIT>Scheduler ( <ISA_LIT>ScheduleDAGMI * DAG ) : DAG ( DAG ) ,


<s> <ISA_LIT>Scheduler ( <ISA_LIT>ScheduleDAGMI * DAG ) : DAG ( DAG ) , BlockCreator ( DAG


<s> ~ <ISA_LIT>ScheduleBlockScheduler ( )


<s> void <ISA_LIT>ScheduleDAGMI :: fillVgprSgprCost ( _Iterator First , _Iterator End , unsigned & VgprUsage , unsigned & SgprUsage ) { VgprUsage = <NUM_LIT> ; SgprUsage = <NUM_LIT> ; for ( _Iterator RegI = First ; RegI != End ; ++ RegI ) { unsigned Reg = * RegI ; if ( ! Register :: isVirtualRegister ( Reg ) ) continue ; PSetIterator PSetI = MRI . getPressureSets ( Reg ) ; for ( ; PSetI . isValid ( ) ; ++ PSetI ) { if ( * PSetI == <STR_LIT> :: VGPR_32 ) VgprUsage += PSetI . getWeight ( ) ; else if ( * PSetI == <STR_LIT> :: SReg_32 ) SgprUsage += PSetI .


<s> RPTracker . advance ( ) ; } RPTracker . closeRegion ( ) ; TopRPTracker . addLiveRegs ( RPTracker . getPressure ( ) . LiveInRegs ) ; BotRPTracker . addLiveRegs ( RPTracker . getPressure ( ) . LiveOutRegs ) ; for ( const auto & RegMaskPair : RPTracker . getPressure ( ) . LiveInRegs ) { if ( TargetRegisterInfo :: isVirtualRegister ( RegMaskPair . RegUnit ) ) LiveInRegs . insert ( RegMaskPair . RegUnit ) ; } LiveOutRegs . clear ( ) ; for ( const auto & RegMaskPair : RPTracker . getPressure ( ) . LiveOutRegs ) { unsigned Reg = RegMaskPair . RegUnit ; if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) && isDefBetween ( Reg , LIS -> getInstructionIndex ( BeginBlock ) . getRegSlot ( ) , LIS -> getInstructionIndex ( EndBlock ) . getRegSlot ( ) , MRI


<s> void <ISA_LIT>ScheduleBlock :: initRegPressure ( MachineBasicBlock :: iterator BeginBlock , MachineBasicBlock :: iterator EndBlock ) { IntervalPressure Pressure , BotPressure ; RegPressureTracker RPTracker ( Pressure ) , BotRPTracker ( BotPressure ) ; LiveIntervals * LIS = DAG -> getLIS ( ) ; MachineRegisterInfo * MRI = DAG -> getMRI ( ) ; DAG -> initRPTracker ( TopRPTracker ) ; DAG -> initRPTracker ( BotRPTracker ) ; DAG -> initRPTracker ( RPTracker ) ; for ( SUnit * SU : ScheduledSUnits ) { RPTracker . setPos ( SU -> getInstr ( ) ) ; RPTracker . advance ( ) ; } RPTracker . closeRegion ( ) ; TopRPTracker . addLiveRegs ( RPTracker . getPressure (


<s> for ( std :: vector < <ISA_LIT>ScheduleBlock * > :: iterator I = ReadyBlocks . begin ( ) , E = ReadyBlocks . end ( ) ; I != E ; ++ I ) { <ISA_LIT>BlockSchedCandidate TryCand ; TryCand . Block = * I ; TryCand . IsHighLatency = TryCand . Block -> isHighLatencyBlock ( ) ; TryCand . VGPRUsageDiff = checkRegUsageImpact ( TryCand . Block -> getInRegs ( ) , TryCand . Block -> getOutRegs ( ) ) [ DAG -> getVGPRSetID ( ) ] ; TryCand . NumSuccessors = TryCand . Block -> getSuccs ( ) . size ( ) ; TryCand . NumHighLatencySuccessors = TryCand . Block -> getNumHighLatencySuccessors ( ) ; TryCand . LastPosHighLatParentScheduled = ( unsigned int ) std :: max < int > ( <NUM_LIT> , LastPosHighLatencyParentScheduled [ TryCand . Block -> getID ( ) ] - LastPosWaitedHighLatency ) ; TryCand . Height = TryCand . Block -> Height ; if ( VregCurrentUsage > <NUM_LIT> || Variant != <STR_LIT> ) { if ( ! tryCandidateRegUsage ( Cand , TryCand ) && Variant != <STR_LIT> ) tryCandidateLatency ( Cand , TryCand ) ; } else { if ( ! tryCandidateLatency ( Cand , TryCand ) ) tryCandidateRegUsage ( Cand , TryCand ) ; } if ( TryCand . Reason != NoCand ) { Cand . setBest ( TryCand )


<s>
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
dbgs
(
)
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
VregCurrentUsage
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
SregCurrentUsage
<<
'\n'
;
)
;
Cand
.
Block
=
nullptr
;
for
(
std
::
vector
<
<ISA_LIT>ScheduleBlock
*
>
::
iterator
I
=
ReadyBlocks
.
begin
(
)
,
E
=
ReadyBlocks
.
end
(
)
;
I
!=
E
;
++
I
)
{
<ISA_LIT>BlockSchedCandidate
TryCand
;
TryCand
.
Block
=
*
I
;
TryCand
.
IsHighLatency
=
TryCand
.
Block
->
isHighLatencyBlock
(
)
;
TryCand
.
VGPRUsageDiff
=
checkRegUsageImpact
(
TryCand
.
Block
->
getInRegs
(
)
,
TryCand
.
Block
->
getOutRegs
(
)
)
[
DAG
->
getVGPRSetID
(
)
]
;
TryCand
.
NumSuccessors
=
TryCand
.
Block
->
getSuccs
(
)
.
size
(
)
;
TryCand
.
NumHighLatencySuccessors
=
TryCand
.
Block
->
getNumHighLatencySuccessors
(
)
;
TryCand
.
LastPosHighLatParentScheduled
=
(
unsigned
int
)
std
::
max
<
int
>
(
<NUM_LIT>
,
LastPosHighLatencyParentScheduled
[
TryCand
.
Block
->
getID
(
)
]
LastPosWaitedHighLatency
)
;
TryCand
.
Height
=
TryCand
.
Block
->
Height
;
if
(
VregCurrentUsage
>
<NUM_LIT>
||
Variant
!=
<STR_LIT>
)
{
if
(
!
tryCandidateRegUsage
(
Cand
,
TryCand
)
&&
Variant
!=
<STR_LIT>
)
tryCandidateLatency
(
Cand
,
TryCand
)
;
}
else
{
if
(
!
tryCandidateLatency
(
Cand
,
TryCand
)
)
tryCandidateRegUsage
(
Cand
,
TryCand
)
;
}
if
(
TryCand
.
Reason
!=
NoCand
)
{
Cand
.
setBest
(
TryCand
)
;
Best
=
I
;
DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
Cand
.
Block
->
getID
(
)
<<
' '
<<
getReasonStr
(
Cand
.
Reason
)
<<
'\n'
)
;
}
}
DEBUG
(
dbgs
(
)
<<
<STR_LIT>
<<
Cand
.
Block
->
getID


<s> static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsageLatency } , { LatenciesGrouped , BlockLatencyRegUsage } , { LatenciesAlonePlusConsecutive , BlockLatencyRegUsage } , } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , { LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , { LatenciesAlonePlusConsecutive , BlockRegUsage } } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } ScheduledSUnits = Best . SUs ; ScheduledSUnitsInv . resize ( SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { ScheduledSUnitsInv [ ScheduledSUnits [ i ] ] = i ; } moveLowLatencies ( ) ; assert ( TopRPTracker . getPos ( ) == RegionBegin && <STR_LIT> ) ; TopRPTracker . setPos ( CurrentTop ) ; for ( std :: vector < unsigned > :: iterator I = ScheduledSUnits . begin ( ) , E = ScheduledSUnits . end ( ) ; I != E ; ++ I ) { SUnit * SU = & SUnits [ * I ] ; scheduleMI ( SU , true ) ; DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << * SU -> getInstr ( ) ) ; } assert ( CurrentTop == CurrentBottom && <STR_LIT> ) ; placeDebugValues (


<s> buildDAGWithRegPressure ( ) ; DEBUG ( for ( SUnit & SU : SUnits ) SU . dumpAll ( this ) ) ; topologicalSort ( ) ; findRootsAndBiasEdges ( TopRoots , BotRoots ) ; SchedImpl -> initialize ( this ) ; initQueues ( TopRoots , BotRoots ) ; SUnitsLinksBackup = SUnits ; IsLowLatencySU . clear ( ) ; LowLatencyOffset . clear ( ) ; IsHighLatencySU . clear ( ) ; IsLowLatencySU . resize ( SUnits . size ( ) , <NUM_LIT> ) ; LowLatencyOffset . resize ( SUnits . size ( ) , <NUM_LIT> ) ; IsHighLatencySU . resize ( SUnits . size ( ) , <NUM_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { SUnit * SU = & SUnits [ i ] ; unsigned BaseLatReg ; int64_t OffLatReg ; if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * SU -> getInstr ( ) ) ) { IsLowLatencySU [ i ] = <NUM_LIT> ; if ( <ISA_LIT>TII -> getMemOpBaseRegImmOfs ( * SU -> getInstr ( ) , BaseLatReg , OffLatReg , TRI ) ) LowLatencyOffset [ i ] = OffLatReg ; } else if ( <ISA_LIT>TII -> isHighLatencyInstruction ( * SU -> getInstr ( ) ) ) IsHighLatencySU [ i ] = <NUM_LIT> ; } <ISA_LIT>Scheduler Scheduler ( this ) ; Best = Scheduler . scheduleVariant ( <STR_LIT> , <STR_LIT> ) ; if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsageLatency } , { LatenciesGrouped , BlockLatencyRegUsage } , { LatenciesAlonePlusConsecutive , BlockLatencyRegUsage } , } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } if ( Best . MaxVGPRUsage > <NUM_LIT> ) { static const std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > Variants [ ] = { { LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , {

LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , {


<s> } ScheduledSUnits = Best . SUs ; ScheduledSUnitsInv . resize ( SUnits . size ( ) ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { ScheduledSUnitsInv [ ScheduledSUnits [ i ] ] = i ; } moveLowLatencies ( ) ; assert ( TopRPTracker . getPos ( ) == RegionBegin && <STR_LIT> ) ; TopRPTracker . setPos ( CurrentTop ) ; for ( std :: vector < unsigned > :: iterator I = ScheduledSUnits . begin ( ) , E = ScheduledSUnits . end ( ) ; I != E ; ++ I ) { SUnit * SU = & SUnits [ * I ] ; scheduleMI ( SU , true ) ; LLVM_DEBUG ( dbgs ( ) << <STR_LIT> << SU -> NodeNum << <STR_LIT> << * SU -> getInstr ( ) ) ; } assert ( CurrentTop == CurrentBottom && <STR_LIT> ) ; placeDebugValues ( ) ; LLVM_DEBUG ( { dbgs ( ) << <STR_LIT> << printMBBReference ( * begin ( ) -> getParent ( ) ) << <STR_LIT> ; dumpSchedule ( ) ; dbgs ( )


<s>
dbgs
(
)
<<
<STR_LIT>
;
for
(
std
::
pair
<
<ISA_LIT>ScheduleBlock
*
,
<ISA_LIT>ScheduleBlockLinkKind
>
S
:
Succs
)
{
if
(
S
.
second
==
<STR_LIT>
)
dbgs
(
)
<<
<STR_LIT>
;
S
.
first
->
printDebug
(
false
)
;
}
if
(
Scheduled
)
{
dbgs
(
)
<<
<STR_LIT>
<<
LiveInPressure
[
DAG
->
getSGPRSetID
(
)
]
<<
' '
<<
LiveInPressure
[
DAG
->
getVGPRSetID
(
)
]
<<
'\n'
;
dbgs
(
)
<<
<STR_LIT>
<<
LiveOutPressure
[
DAG
->
getSGPRSetID
(
)
]
<<
' '
<<
LiveOutPressure
[
DAG
->
getVGPRSetID
(
)
]
<<
<STR_LIT>
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveInRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
dbgs
(
)
<<
<STR_LIT>
;
for
(
unsigned
Reg
:
LiveOutRegs
)
dbgs
(
)
<<
printVRegOrUnit
(
Reg
,
DAG
->
getTRI
(
)
)
<<
' '
;
}
dbgs
(
)
<<
<STR_LIT>
;
if
(
!
Scheduled
)
{
for
(
SUnit
*
SU
:
SUnits
)
{
SU
->
dump
(
DAG
)
;
}
}
else
{
for
(
SUnit
*
SU
:
SUnits
)
{
SU
->
dump
(
DAG
)
;
}
}
dbgs
(
)
<<
<STR_LIT>


<s> if ( I == TopReadySUs . end ( ) ) { dbgs ( ) << <STR_LIT> ; llvm_unreachable ( nullptr ) ; } TopReadySUs . erase ( I ) ; releaseSuccessors ( SU , true ) ; if ( HasLowLatencyNonWaitedParent [ NodeNum2Index [ SU -> NodeNum ] ] ) HasLowLatencyNonWaitedParent . assign ( SUnits . size ( ) , <NUM_LIT> ) ; if ( DAG -> IsLowLatencySU [ SU -> NodeNum ] ) { for ( SDep & Succ : SU ->


<s> buildDAGWithRegPressure ( ) ; DEBUG ( for ( SUnit & SU : SUnits ) SU . dumpAll ( this ) ) ; topologicalSort ( ) ; findRootsAndBiasEdges ( TopRoots , BotRoots ) ; SchedImpl -> initialize ( this ) ; initQueues ( TopRoots , BotRoots ) ; SUnitsLinksBackup = SUnits ; IsLowLatencySU . clear ( ) ; LowLatencyOffset . clear ( ) ; IsHighLatencySU . clear ( ) ; IsLowLatencySU . resize ( SUnits . size ( ) , <NUM_LIT> ) ; LowLatencyOffset . resize ( SUnits . size ( ) , <NUM_LIT> ) ; IsHighLatencySU . resize ( SUnits . size ( ) , <NUM_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = ( unsigned ) SUnits . size ( ) ; i != e ; ++ i ) { SUnit * SU = & SUnits [ i ] ; unsigned BaseLatReg ; int64_t OffLatReg ; if ( <ISA_LIT>TII -> isLowLatencyInstruction ( SU -> getInstr ( ) ) ) { IsLowLatencySU [ i ] = <NUM_LIT> ; if ( <ISA_LIT>TII -> getMemOpBaseRegImmOfs ( SU -> getInstr ( ) , BaseLatReg , OffLatReg , TRI ) ) LowLatencyOffset [ i ] = OffLatReg ; } else if ( <ISA_LIT>TII -> isHighLatencyInstruction ( SU -> getInstr ( ) ) ) IsHighLatencySU [ i ] = <NUM_LIT> ; } <ISA_LIT>Scheduler Scheduler ( this ) ; Best = Scheduler . scheduleVariant ( <STR_LIT> , <STR_LIT> ) ; if ( Best . MaxVGPRUsage > <NUM_LIT> ) { std :: vector < std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant >> Variants = { { LatenciesAlone , BlockRegUsageLatency } , { LatenciesGrouped , BlockLatencyRegUsage } , { LatenciesAlonePlusConsecutive , BlockLatencyRegUsage } , } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } if ( Best . MaxVGPRUsage > <NUM_LIT> ) { std :: vector < std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant >> Variants = { { LatenciesAlone , BlockRegUsage } , { LatenciesGrouped , BlockRegUsageLatency } , { LatenciesGrouped , BlockRegUsage } , { LatenciesAlonePlusConsecutive , BlockRegUsageLatency } , { LatenciesAlonePlusConsecutive , BlockRegUsage } } ; for ( std :: pair < <ISA_LIT>SchedulerBlockCreatorVariant , <ISA_LIT>SchedulerBlockSchedulerVariant > v : Variants ) { Temp = Scheduler . scheduleVariant ( v . first , v . second ) ; if ( Temp . MaxVGPRUsage < Best . MaxVGPRUsage ) Best = Temp ; } } ScheduledSUnits = Best . SUs


<s> SgprUsage = <NUM_LIT> ; for ( _Iterator RegI = First ; RegI != End ; ++ RegI ) { unsigned Reg = * RegI ; if ( ! Register :: isVirtualRegister ( Reg ) ) continue ; PSetIterator PSetI = MRI . getPressureSets ( Reg ) ; for ( ; PSetI . isValid ( ) ; ++


<s> <ISA_LIT>TII = static_cast < const <ISA_LIT>InstrInfo * > ( TII ) ; <ISA_LIT>TRI = static_cast < const <ISA_LIT>RegisterInfo * >


<s> <ISA_LIT>ScheduleBlock ( <ISA_LIT>ScheduleDAGMI * DAG , <ISA_LIT>ScheduleBlockCreator * BC , unsigned ID ) : DAG ( DAG ) , BC ( BC ) , TopRPTracker ( TopPressure ) , Scheduled ( false ) , HighLatencyBlock ( false ) , ID ( ID ) ,

NumHighLatencySuccessors ( <NUM_LIT> ) {


<s> std :: set < unsigned > InRegs ( RPTracker . getPressure ( ) . LiveInRegs . begin ( ) , RPTracker . getPressure ( ) . LiveInRegs . end ( ) ) ; return


<s> RPTracker . init ( & MF , RegClassInfo , LIS


<s> void initRPTracker ( RegPressureTracker & RPTracker )


<s> if ( ! TargetRegisterInfo :: isVirtualRegister ( Reg ) ) continue ; ( void ) LiveRegs .


<s> if ( ! TargetRegisterInfo :: isVirtualRegister ( Reg ) ) continue ; ( void ) LiveRegs . insert (


<s> DiffSetPressure . assign ( DAG -> getTRI ( ) -> getNumRegPressureSets ( ) , <NUM_LIT> ) ; for ( unsigned Reg : InRegs ) { if ( ! TargetRegisterInfo :: isVirtualRegister ( Reg ) ) continue ; if ( LiveRegsConsumers [ Reg ] > <NUM_LIT> ) continue ; PSetIterator PSetI = DAG -> getMRI ( ) ->


<s> } else { for ( unsigned k : SubGraph ) { if ( DAG -> IsHighLatencySU [ k ] || ( CurrentColoring [ k ] != ProposedColor && CurrentColoring [ k ] != <NUM_LIT> ) ) { CompatibleGroup = false ; break ; } if ( hasDataDependencyPred ( DAG -> SUnits [ k ] , DAG -> SUnits [ j ] ) ) { CompatibleGroup = false ; break ; } } if ( ! CompatibleGroup ) break ; if ( hasDataDependencyPred ( SU , DAG -> SUnits [ j ] ) ) { CompatibleGroup = false ; break ; } AdditionalElements . insert ( AdditionalElements . end ( ) , SubGraph . begin ( ) , SubGraph . end ( ) ) ; } } if ( CompatibleGroup ) { FormingGroup . insert ( SU . NodeNum ) ; for ( unsigned j : AdditionalElements ) CurrentColoring [ j ] = ProposedColor ; CurrentColoring [ SU . NodeNum ] = ProposedColor ; ++ Count ; } if ( ! CompatibleGroup ) { FormingGroup . clear ( ) ; Color = ++ NextReservedID ; ProposedColor = Color ; FormingGroup . insert ( SU . NodeNum ) ; CurrentColoring [ SU . NodeNum ] = ProposedColor ; Count = <NUM_LIT> ; } else if ( Count == GroupSize ) { FormingGroup . clear ( ) ; Color = ++ NextReservedID ; ProposedColor =


<s> for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { SUnit * SU = & DAG -> SUnits [ i ] ; if ( DAG -> IsHighLatencySU [ SU -> NodeNum ] ) ++ NumHighLatencies ; } if ( NumHighLatencies == <NUM_LIT> ) return ; if ( NumHighLatencies <= <NUM_LIT> ) GroupSize = <NUM_LIT> ; else if ( NumHighLatencies <= <NUM_LIT> ) GroupSize = <NUM_LIT> ; else GroupSize = <NUM_LIT> ; for ( unsigned SUNum : DAG -> TopDownIndex2SU ) { const SUnit & SU = DAG -> SUnits [ SUNum ] ; if ( DAG -> IsHighLatencySU [ SU . NodeNum ] ) { unsigned CompatibleGroup = true ; int ProposedColor = Color ; std :: vector < int > AdditionalElements ; for ( unsigned j : FormingGroup ) { bool HasSubGraph ; std :: vector < int > SubGraph ; SubGraph = DAG -> GetTopo ( ) -> GetSubGraph ( SU , DAG -> SUnits [ j ] , HasSubGraph ) ; assert ( ! HasSubGraph ) ; SubGraph = DAG -> GetTopo ( ) -> GetSubGraph ( DAG -> SUnits [ j ] , SU , HasSubGraph ) ; if ( ! HasSubGraph ) continue ; else if ( SubGraph . size ( ) > <NUM_LIT> ) { CompatibleGroup = false ; break ; } else { for ( unsigned k : SubGraph ) { if ( DAG -> IsHighLatencySU [ k ] || ( CurrentColoring [ k ] != ProposedColor && CurrentColoring [ k ] != <NUM_LIT> ) ) { CompatibleGroup = false ; break ; } if ( hasDataDependencyPred ( DAG -> SUnits [ k ] , DAG -> SUnits [ j ] ) ) { CompatibleGroup = false ; break ; } } if ( ! CompatibleGroup ) break ; if ( hasDataDependencyPred ( SU , DAG -> SUnits [ j ] ) ) { CompatibleGroup = false ; break ; } AdditionalElements . insert ( AdditionalElements . end ( ) , SubGraph . begin ( ) , SubGraph . end ( ) ) ; } } if ( CompatibleGroup ) { FormingGroup . insert ( SU . NodeNum ) ; for ( unsigned j : AdditionalElements ) CurrentColoring [ j ] = ProposedColor ; CurrentColoring [ SU . NodeNum ] =


<s> for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { int BlockIndice = TopDownIndex2Block [ i ] ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; if ( Block -> getPreds ( ) . empty ( ) ) Block -> Depth = <NUM_LIT> ; else { unsigned Depth = <NUM_LIT> ; for ( <ISA_LIT>ScheduleBlock * Pred : Block -> getPreds ( ) ) { if ( Depth < Pred -> Depth + <NUM_LIT> ) Depth = Pred -> Depth + <NUM_LIT> ; } Block -> Depth = Depth ; } } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { int BlockIndice = BottomUpIndex2Block [ i ] ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; if ( Block -> getSuccs ( ) . empty ( ) ) Block -> Height = <NUM_LIT> ; else { unsigned Height = <NUM_LIT> ; for ( const auto & Succ : Block -> getSuccs ( ) ) Height = std :: min ( Height , Succ . first -> Height + <NUM_LIT> ) ; Block -> Height


<s> void <ISA_LIT>ScheduleBlockCreator :: fillStats ( ) { unsigned DAGSize = CurrentBlocks . size ( ) ; for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { int BlockIndice = TopDownIndex2Block [ i ] ; <ISA_LIT>ScheduleBlock * Block = CurrentBlocks [ BlockIndice ] ; if ( Block -> getPreds ( ) . empty ( ) ) Block -> Depth = <NUM_LIT> ; else { unsigned Depth = <NUM_LIT> ; for ( <ISA_LIT>ScheduleBlock * Pred : Block -> getPreds ( ) ) { if ( Depth < Pred -> Depth + <NUM_LIT> ) Depth = Pred -> Depth + <NUM_LIT> ; } Block -> Depth = Depth ; } } for ( unsigned i = <NUM_LIT> , e = DAGSize ; i != e ; ++ i ) { int BlockIndice = BottomUpIndex2Block


<s> if ( ! TargetRegisterInfo :: isVirtualRegister ( Reg ) ) continue ; PSetIterator PSetI = MRI . getPressureSets ( Reg ) ; for ( ; PSetI . isValid ( ) ; ++ PSetI ) { if ( * PSetI == VGPRSetID ) VgprUsage += PSetI


<s> DEBUG ( for ( unsigned i = <NUM_LIT> , e = CurrentBlocks . size ( ) ; i !=


<s> std :: vector < <ISA_LIT>ScheduleBlock *


<s> MachineBasicBlock :: iterator getCurrentTop ( ) { return CurrentTop


<s> std :: set < unsigned > InRegs ; for ( const auto & RegMaskPair : RPTracker . getPressure (


<s> std :: vector < unsigned > & getInternalAdditionnalRegUsage ( ) { return InternalAdditionnalPressure


<s> unsigned getNumHighLatencySuccessors ( ) const


<s> const std :: vector < <ISA_LIT>ScheduleBlock * > &


<s> std :: vector < SUnit * > getScheduledUnits ( ) { return ScheduledSUnits


<s> ArrayRef < std :: pair < <ISA_LIT>ScheduleBlock * , <ISA_LIT>ScheduleBlockLinkKind >> getSuccs ( ) const { return


<s> unsigned getVGPRUsage ( ) { return maxVregUsage


<s> for ( SUnit * SU : ScheduledSUnits ) { RPTracker . setPos ( SU -> getInstr ( ) ) ; RPTracker . advance ( ) ; } RPTracker . closeRegion ( ) ; TopRPTracker . addLiveRegs ( RPTracker . getPressure ( ) . LiveInRegs ) ; BotRPTracker . addLiveRegs ( RPTracker . getPressure ( ) . LiveOutRegs ) ; for ( const auto & RegMaskPair : RPTracker . getPressure ( ) . LiveInRegs ) { if ( TargetRegisterInfo :: isVirtualRegister ( RegMaskPair . RegUnit ) ) LiveInRegs . insert ( RegMaskPair . RegUnit ) ; } LiveOutRegs . clear ( ) ; for ( const auto & RegMaskPair : RPTracker . getPressure ( ) . LiveOutRegs ) { unsigned Reg = RegMaskPair . RegUnit ; if ( TargetRegisterInfo :: isVirtualRegister ( Reg ) && isDefBetween ( Reg , LIS -> getInstructionIndex ( * BeginBlock ) . getRegSlot ( ) , LIS -> getInstructionIndex ( * EndBlock ) . getRegSlot ( ) , MRI , LIS ) ) { LiveOutRegs . insert ( Reg ) ; } } LiveInPressure = TopPressure . MaxSetPressure ; LiveOutPressure = BotPressure


<s> return RepeatReasonSet & ( <NUM_LIT>


<s> bool isScheduled ( ) { return Scheduled


<s> SUnit * Pred = PredDep . getSUnit ( ) ; if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * Pred -> getInstr ( ) ) ) { IsLowLatencyUser = true ; } if ( Pred -> NodeNum >= DAGSize ) continue ; unsigned PredPos = ScheduledSUnitsInv [ Pred -> NodeNum ] ; if ( PredPos >= MinPos ) MinPos = PredPos + <NUM_LIT> ; } if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * SU -> getInstr ( ) ) ) { unsigned BestPos = LastLowLatencyUser + <NUM_LIT> ; if ( ( int ) BestPos <= LastLowLatencyPos ) BestPos = LastLowLatencyPos + <NUM_LIT> ; if ( BestPos < MinPos ) BestPos = MinPos ; if ( BestPos < i ) { for ( unsigned u = i ; u > BestPos ; -- u ) { ++ ScheduledSUnitsInv [ ScheduledSUnits [ u - <NUM_LIT> ] ] ; ScheduledSUnits [ u ] = ScheduledSUnits [ u - <NUM_LIT> ] ; } ScheduledSUnits [ BestPos ] = SU -> NodeNum ; ScheduledSUnitsInv [ SU -> NodeNum ] = BestPos ; } LastLowLatencyPos = BestPos ; if ( IsLowLatencyUser ) LastLowLatencyUser = BestPos ; } else if ( IsLowLatencyUser ) { LastLowLatencyUser = i ; } else if ( SU -> getInstr ( ) -> getOpcode ( ) == <STR_LIT> ) { bool CopyForLowLat = false ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * Succ


<s> for ( unsigned u = i ; u > BestPos ; -- u ) { ++ ScheduledSUnitsInv [ ScheduledSUnits [ u - <NUM_LIT> ] ] ; ScheduledSUnits [ u ] = ScheduledSUnits [ u - <NUM_LIT> ] ; } ScheduledSUnits [ BestPos ] = SU -> NodeNum ; ScheduledSUnitsInv [ SU -> NodeNum ] = BestPos ; } LastLowLatencyPos = BestPos ; if ( IsLowLatencyUser ) LastLowLatencyUser = BestPos ; } else if ( IsLowLatencyUser ) { LastLowLatencyUser = i ; } else if ( SU -> getInstr ( ) -> getOpcode ( ) == <STR_LIT> ) { bool CopyForLowLat = false ; for ( SDep & SuccDep : SU -> Succs ) { SUnit * Succ = SuccDep . getSUnit ( ) ; if ( <ISA_LIT>TII -> isLowLatencyInstruction ( * Succ -> getInstr ( ) ) ) { CopyForLowLat = true ; } } if ( ! CopyForLowLat ) continue ; if ( MinPos < i ) { for ( unsigned u = i ; u > MinPos ;